update script and fixes

lib/BlockDb.js

@@ -326,6 +326,7 @@ BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) {
   if (o.multipleSpentAttempts) {
     async.eachLimit(o.multipleSpentAttempts, CONCURRENCY,
       function(oi, e_c) {
+        // Only one will be confirmed
         self.getBlockForTx(oi.spentTxId, function(err, hash, height) {
           if (err) return;
           if (height>=0) {
@@ -417,7 +418,17 @@ BlockDb.prototype.migrateV02cleanup = function(cb) {
         end: k + '~'
       })
       .pipe(d.createWriteStream({type:'del'}))
-      .on('close',cb);
+      .on('close', function(err){
+        if (err) return cb(err);
+        var k = 'txa-';
+        var d = self.txDb._db;
+        d.createReadStream({
+          start: k,
+          end: k + '~'
+        })
+        .pipe(d.createWriteStream({type:'del'}))
+        .on('close', cb);
+      });
     });
 };
 

lib/BlockExtractor.js

@@ -9,62 +9,53 @@ var  bitcore  = require('bitcore'),
      async    = require('async');
 
 function BlockExtractor(dataDir, network) {
-
-  var self = this;
   var path = dataDir + '/blocks/blk*.dat';
 
-  self.dataDir = dataDir;
+  this.dataDir = dataDir;
-  self.files   = glob.sync(path);
+  this.files   = glob.sync(path);
-  self.nfiles  = self.files.length;
+  this.nfiles  = this.files.length;
 
-  if (self.nfiles === 0)
+  if (this.nfiles === 0)
     throw new Error('Could not find block files at: ' + path);
 
-  self.currentFileIndex = 0;
+  this.currentFileIndex = 0;
-  self.isCurrentRead    = false;
+  this.isCurrentRead    = false;
-  self.currentBuffer    = null;
+  this.currentBuffer    = null;
-  self.currentParser    = null;
+  this.currentParser    = null;
-  self.network = network === 'testnet' ? networks.testnet: networks.livenet;
+  this.network = network === 'testnet' ? networks.testnet: networks.livenet;
-  self.magic   = self.network.magic.toString('hex');
+  this.magic   = this.network.magic.toString('hex');
 }
 
 BlockExtractor.prototype.currentFile = function() {
-  var self = this;
+  return this.files[this.currentFileIndex];
-
-  return self.files[self.currentFileIndex];
 };
 
 
 BlockExtractor.prototype.nextFile = function() {
-  var self = this;
+  if (this.currentFileIndex < 0) return false;
-
-  if (self.currentFileIndex < 0) return false;
 
   var ret  = true;
 
-  self.isCurrentRead = false;
+  this.isCurrentRead = false;
-  self.currentBuffer = null;
+  this.currentBuffer = null;
-  self.currentParser = null;
+  this.currentParser = null;
 
-  if (self.currentFileIndex < self.nfiles - 1) {
+  if (this.currentFileIndex < this.nfiles - 1) {
-    self.currentFileIndex++;
+    this.currentFileIndex++;
   }
   else {
-    self.currentFileIndex=-1;
+    this.currentFileIndex=-1;
     ret = false;
   }
   return ret;
 };
 
 BlockExtractor.prototype.readCurrentFileSync = function() {
-  var self = this;
+  if (this.currentFileIndex < 0 || this.isCurrentRead) return;
 
-  if (self.currentFileIndex < 0 || self.isCurrentRead) return;
+  this.isCurrentRead = true;
 
-
+  var fname = this.currentFile();
-  self.isCurrentRead = true;
-
-  var fname = self.currentFile();
   if (!fname) return;
 
 
@@ -81,80 +72,70 @@ BlockExtractor.prototype.readCurrentFileSync = function() {
 
   fs.readSync(fd, buffer, 0, size, 0);
 
-  self.currentBuffer = buffer;
+  this.currentBuffer = buffer;
-  self.currentParser = new Parser(buffer);
+  this.currentParser = new Parser(buffer);
 };
 
 
 
-BlockExtractor.prototype.getNextBlock = function(cb) {
-  var self = this;
 
+BlockExtractor.prototype._getMagic = function() {
+  if (!this.currentParser) 
+    return null;
+
+  var byte0 = this.currentParser ? this.currentParser.buffer(1).toString('hex') : null;
+
+
+
+  // Grab 3 bytes from block without removing them
+  var p = this.currentParser.pos;
+  var bytes123 = this.currentParser.subject.toString('hex',p,p+3);
+  var magic = byte0 + bytes123;
+
+  if (magic !=='00000000' && magic !== this.magic) {
+    if(this.errorCount++ > 4)
+      throw new Error('CRITICAL ERROR: Magic number mismatch: ' +
+                          magic + '!=' + this.magic);
+    magic=null;
+  }
+  
+  if (magic==='00000000') 
+    magic =null;
+
+  return magic;
+};
+
+BlockExtractor.prototype.getNextBlock = function(cb) {
   var b;
   var magic;
-  async.series([
+  var isFinished = 0;
-    function (a_cb) {
 
-      async.whilst(
+  while(!magic && !isFinished)  {
-        function() {
+    this.readCurrentFileSync();
-          return (!magic || magic === '00000000');
+    magic= this._getMagic();
-        },
-        function(w_cb) {
-          magic = null;
 
-          self.readCurrentFileSync();
+    if (!this.currentParser || this.currentParser.eof() ) {
-          if (self.currentFileIndex < 0) return cb();
 
-          var byte0 = self.currentParser ? self.currentParser.buffer(1).toString('hex') : null;
+      if (this.nextFile()) {
+        console.log('Moving forward to file:' + this.currentFile() );
+        magic = null;
+      } else {
+        console.log('Finished all files');
+        isFinished = 1;
+      }
+    }
+  }
+  if (isFinished)
+    return cb();
 
-          if (byte0) {
+  // Remove 3 bytes from magic and spacer
-            // Grab 3 bytes from block without removing them
+  this.currentParser.buffer(3+4);
-            var p = self.currentParser.pos;
-            var bytes123 = self.currentParser.subject.toString('hex',p,p+3);
-            magic = byte0 + bytes123;
 
-            if (magic !=='00000000' && magic !== self.magic) {
+  b = new Block();
-
+  b.parse(this.currentParser);
-              if (self.errorCount++ > 4) 
+  b.getHash();
-                return cb(new Error('CRITICAL ERROR: Magic number mismatch: ' +
+  this.errorCount=0;
-                    magic + '!=' + self.magic));
+  return cb(null,b);
-
-              magic=null;
-            }
-          } 
-
-          if (!self.currentParser || self.currentParser.eof() ) {
-            if (self.nextFile()) 
-              console.log('Moving forward to file:' + self.currentFile() );
-            else 
-              console.log('Finished all files');
-
-            magic = null;
-            return w_cb();
-          }
-          else {
-            return w_cb();
-          }
-        }, a_cb);
-    },
-    function (a_cb) {
-      if (!magic) return a_cb();
-      // Remove 3 bytes from magic and spacer
-      self.currentParser.buffer(3+4);
-      return a_cb();
-    },
-    function (a_cb) {
-      if (!magic) return a_cb();
-
-      b = new Block();
-      b.parse(self.currentParser);
-      b.getHash();
-      self.errorCount=0;
-      return a_cb();
-    },
-  ], function(err) {
-    return cb(err,b);
-  });
 };
 
 module.exports = require('soop')(BlockExtractor);

lib/HistoricSync.js

@@ -15,6 +15,10 @@ var bitcoreUtil    = bitcore.util;
 var logger = require('./logger').logger;
 var info   = logger.info;
 var error  = logger.error;
+var PERCENTAGE_TO_START_FROM_RPC = 1.1;
+
+// TODO  TODO TODO
+//var PERCENTAGE_TO_START_FROM_RPC = 0.98;
 
 //  var Deserialize = require('bitcore/Deserialize');
 var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@@ -244,7 +248,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) {
   var self = this;
 
   if ( opts.forceRPC || !config.bitcoind.dataDir ||
-    self.height > self.blockChainHeight * 0.9) return next();
+    self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next();
 
 
   try {
@@ -279,6 +283,9 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) {
           });
         });
       }, function(err){
+        console.log('\tFOUND Starting Block!');
+
+        // TODO SET HEIGHT
         return next(err);
       });
   });
@@ -394,6 +401,7 @@ HistoricSync.prototype.start = function(opts, next) {
       function (w_cb) {
         self.getFn(function(err,blockInfo) {
           if (err) return w_cb(self.setError(err));
+
           if (blockInfo && blockInfo.hash && (!opts.stopAt  || opts.stopAt !== blockInfo.hash)) {
             self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) {
               if (err) return w_cb(self.setError(err));

lib/TransactionDb.js

@@ -11,7 +11,6 @@ var SPENT_PREFIX = 'txs-'; //txs-<txid(out)>-<n(out)>-<txid(in)>-<n(in)> = ts
 // to sum up addr balance (only outs, spents are gotten later)
 var ADDR_PREFIX = 'txa2-'; //txa-<addr>-<tsr>-<txid>-<n> 
 // tsr = 1e13-js_timestamp
-
 // => + btc_sat [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs]
 // |balance:txApperances
 
@@ -201,6 +200,26 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) {
         i.valueSat = ret.valueSat;
         i.value = ret.valueSat / util.COIN;
         valueIn += i.valueSat;
+
+        if (ret.multipleSpentAttempt || !ret.spentTxId ||
+          (ret.spentTxId && ret.spentTxId !== info.txid)
+        ) {
+          if (ret.multipleSpentAttempts) {
+            ret.multipleSpentAttempts.forEach(function(mul) {
+              if (mul.spentTxId !== info.txid) {
+                i.doubleSpentTxID = ret.spentTxId;
+                i.doubleSpentIndex = ret.spentIndex;
+              }
+            });
+          } else if (!ret.spentTxId) {
+            i.dbError = 'Input spent not registered';
+          } else {
+            i.doubleSpentTxID = ret.spentTxId;
+            i.doubleSpentIndex = ret.spentIndex;
+          }
+        } else {
+          i.doubleSpentTxID = null;
+        }
         return c_in();
       });
     },
@@ -640,22 +659,26 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) {
 
 TransactionDb.prototype.checkVersion02 = function(cb) {
   var k = 'txa-';
+  var isV2=1;
   db.createReadStream({
     start: k,
-    end: k + '~'
+    end: k + '~',
+    limit: 1,
   })
     .on('data', function(data) {
-console.log('[TransactionDb.js.689]',data); //TODO
+      isV2=0;
-      return cb(0);
     })
     .on('end', function (){
-      return cb(1);
+      return cb(isV2);
     });
 };
 
 TransactionDb.prototype.migrateV02 = function(cb) {
   var k = 'txa-';
   var dbScript = [];
+  var c=0;
+  var c2=0;
+  var N=50000;
   db.createReadStream({
     start: k,
     end: k + '~'
@@ -668,12 +691,19 @@ TransactionDb.prototype.migrateV02 = function(cb) {
         key: ADDR_PREFIX + k[1] + '-' + (END_OF_WORLD_TS - parseInt(v[1])) + '-' + k[3] + '-' + [4],
         value: v[0],
       });
+      if (c++>N) {
+        console.log('\t%dM txs processed', ((c2+=N)/1e6).toFixed(3)); //TODO
+        db.batch(dbScript,function () {
+          c=0;
+          dbScript=[];
+        });
+      }
     })
     .on('error', function(err) {
       return cb(err);
     })
     .on('end', function (){
-      db.batch(dbScript,cb);
+      return cb();
     });
 };