refactor sync

lib/HistoricSync.js

@@ -14,7 +14,8 @@ function spec() {
   var Sync = require('./Sync').class();
   var sockets = require('../app/controllers/socket.js');
   var BlockExtractor = require('./BlockExtractor.js').class();
-	//var Deserialize = require('bitcore/Deserialize');
+  //  var bitcoreUtil = require('bitcore/util/util');
+	//  var Deserialize = require('bitcore/Deserialize');
 
 
   var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@@ -122,10 +123,10 @@ function spec() {
     if (self.opts.shouldBroadcastSync) {
       sockets.broadcastSyncInfo(self.info());
     }
-
+//
- // if (self.syncPercentage > 10) {
+//   if (self.syncPercentage > 10) {
- //   process.exit(-1);
+//     process.exit(-1);
- // }
+//   }
   };
 
   HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) {
@@ -228,7 +229,6 @@ function spec() {
     //get Info
     self.blockExtractor.getNextBlock(function(err, b) {
       if (err || ! b) return cb(err);
-
       blockInfo = b.getStandardizedObject(b.txs, self.network);
       // blockInfo.curWork = Deserialize.intFromCompact(b.bits);
       // We keep the RPC field names
@@ -323,62 +323,109 @@ function spec() {
     });
   };
 
-  HistoricSync.prototype.importHistory = function(scanOpts, next) {
+  HistoricSync.prototype.smartImport = function(scanOpts, next) {
     var self = this;
 
+    var genesis, count;
     var lastBlock;
     var tip;
 
     async.series([
-    function(cb) {
+      function(s_c) {
-      if (scanOpts.destroy) {
+        if (!scanOpts.destroy) return s_c();
+        
         p('Deleting DB...');
-        return self.sync.destroy(cb);
+        return self.sync.destroy(s_c);
-      }
+      },
-      return cb();
+      function(s_c) {
-    },
+        self.sync.bDb.has(self.genesis, function(err, b) {
-    function (cb) {
+          genesis = b;
-      return self.updateBlockCount(cb);
+          return s_c(err);
-    },
+        });
-    function(cb) {
+      },
-      if (!scanOpts.reverse) return cb();
+      function(s_c) {
-      self.rpc.getBlockHash(self.blockChainHeight, function(err, res) {
+        self.countNotOrphan(function(err, c) {
-        if (err) return cb(err);
+          count = c;
-        lastBlock = res.result;
+          return s_c(err);
-        return cb();
+        });
-      });
+      },
-    },
+      function(s_c) {
-    function(cb) {
+        if (!config.bitcoind.dataDir) return s_c();
-      if (!scanOpts.reverse) return cb();
+        if (scanOpts.startFile) {
-      self.sync.bDb.getTip(function(err, res) {
+          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-        if (err) return cb(err);
+          self.blockExtractor.currentFileIndex = scanOpts.startFile;
-        tip = res;
+          return s_c();
+        }
+        self.sync.bDb.getLastFileIndex(function(err, idx) {
+          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
+          if (idx) self.blockExtractor.currentFileIndex = idx;
+          return s_c(err);
+        });
+      },
+      function(s_c) {
+        self.updateBlockCount(s_c);
+      },
 
-        console.log('Old Tip:', tip);
+      // define sync strategy
-        return cb();
+      function(s_c) {
-      });
+        if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) {
-    },
 
-    function(cb) {
+          // Full sync.
-      if (scanOpts.reverse) {
+          if (!genesis)
+            p('Could not find Genesis block. Running FULL SYNC');
+          else
+            p('Less that 90% of current blockchain is stored. Running FULL SYNC',
+                parseInt(count/self.blockChainHeight*100));
+
+          if (config.bitcoind.dataDir) {
+            p('bitcoind dataDir configured...importing blocks from .dat files');
+            p('Starting from file: ' + self.blockExtractor.currentFileIndex);
+            scanOpts.fromFiles = true;
+          }
+          else {
+            scanOpts.reverse = true;
+          }
+        }
+        else {
+          p('Genesis block found. Syncing upto old TIP.');
+          p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks');
+          scanOpts.reverse = true;
+        }
+
+        if (!scanOpts.reverse) return s_c();
+
+        self.rpc.getBlockHash(self.blockChainHeight, function(err, res) {
+          if (err) return s_c(err);
+          lastBlock = res.result;
+          return s_c();
+        });
+      },
+      function(s_c) {
+        if (!scanOpts.reverse) return s_c();
+        self.sync.bDb.getTip(function(err, res) {
+          if (err) return s_c(err);
+          tip = res;
+
+          console.log('Old Tip:', tip);
+          return s_c();
+        });
+      },
+      function(s_c) {
+        if (!scanOpts.reverse) return s_c();
         self.countNotOrphan(function(err, count) {
-          if (err) return cb(err);
+          if (err) return s_c(err);
 
           self.syncedBlocks =  count || 0;
-          return cb();
+          return s_c();
         });
-      }
+      }],
-      else {
+      function(err) {
-          return cb();
+      // SETUP Sync params
-      }
+      var start, end;
-    },
-    ], function(err) {
       if (err) {
         self.setError(err);
         return next(err, 0);
       }
-      // SETUP Sync params
-      var start, end;
 
       if (!self.step) {
         var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
@@ -431,70 +478,6 @@ function spec() {
           return next(err);
         });
       }
-
-    });
-  };
-
-
-  HistoricSync.prototype.smartImport = function(scanOpts, next) {
-    var self = this;
-
-    var genesis, count;
-    async.series([
-      function(s_c) {
-        self.sync.bDb.has(self.genesis, function(err, b) {
-          genesis = b;
-          return s_c(err);
-        });
-      },
-      function(s_c) {
-        if (scanOpts.destroy) return s_c();
-
-        self.countNotOrphan(function(err, c) {
-          count = c;
-          return s_c(err);
-        });
-      },
-      function(s_c) {
-        if (!config.bitcoind.dataDir) return s_c();
-
-        self.sync.bDb.getLastFileIndex(function(err, idx) {
-          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-          if (idx) self.blockExtractor.currentFileIndex = idx;
-          return s_c(err);
-        });
-      },
-      function(s_c) {
-        self.updateBlockCount(s_c);
-      }],
-      function(err) {
-        if (err) return next(err);
-
-        if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) {
-
-          // Full sync.
-
-          if (!genesis)
-            p('Could not find Genesis block. Running FULL SYNC');
-          else
-            p('Less that 90% of current blockchain is stored. Running FULL SYNC',
-                parseInt(count/self.blockChainHeight*100));
-
-          if (config.bitcoind.dataDir) {
-            p('bitcoind dataDir configured...importing blocks from .dat files');
-            p('Starting from file: ' + self.blockExtractor.currentFileIndex);
-            scanOpts.fromFiles = true;
-          }
-          else {
-            scanOpts.reverse = true;
-          }
-        }
-        else {
-          p('Genesis block found. Syncing upto old TIP.');
-          p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks');
-          scanOpts.reverse = true;
-        }
-        return self.importHistory(scanOpts, next);
     });
   };
 

lib/Sync.js

@@ -143,7 +143,6 @@ function spec() {
       function(err) {
         if (!err) self._handleBroadcast(b.hash, null, updatedAddrs);
         if (err && err.toString().match(/WARN/) ) {
-          //console.log(err);
           err=null;
         }
         return cb(err);

util/sync.js

@@ -12,12 +12,8 @@ var async = require('async');
 
 program
   .version(SYNC_VERSION)
-  .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet')
   .option('-D --destroy', 'Remove current DB (and start from there)', 0)
-  .option('-R --reverse', 'Sync backwards', 0)
+  .option('-S --startfile', 'Number of file from bitcoind to start(default=0)')
-  .option('-U --uptoexisting', 'Sync only until old Tip block is found', 0)
-  .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0)
-  .option('-S --smart', 'genesis stored? uptoexisting = 1, fromFiles=1 [default]', true)
   .option('-v --verbose', 'Verbose 0/1', 0)
   .parse(process.argv);
 
@@ -33,19 +29,10 @@ async.series([
     historicSync.init(program, cb);
   },
   function(cb) {
-
+    historicSync.smartImport({
-    if (typeof program.smart === 'undefined' || parseInt(program.smart) ) {
+      destroy: program.destroy,
-      historicSync.smartImport({
+      startFile: program.startfile,
-        destroy: program.destroy,
+    },cb);
-      },cb);
-    }
-    else {
-      historicSync.importHistory({
-        destroy: program.destroy,
-        reverse: program.reverse,
-        fromFiles: program.fromfiles,
-      }, cb);
-    }
   },
   ],
   function(err) {