better smart syncs

app/models/Block.js

@@ -27,6 +27,7 @@ var BlockSchema = new Schema({
   },
   time: Number,
   nextBlockHash: String,
+  isOrphan: Boolean,
 });
 
 /**

app/models/Transaction.js

@@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
       }
       else {
         if ( !i.coinbase ) {
-            console.log ('TX: %s,%d could not parse INPUT', txid, i.n);
+            console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n);
         }
         return next_in();
       }
@@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time,  cb) {
           }, next_out);
         }
         else {
-          console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n);
+          console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n);
           return next_out();
         }
       },

lib/HistoricSync.js

@@ -13,14 +13,16 @@ function spec() {
   var Sync            = require('./Sync').class();
 
   function HistoricSync(opts) {
-    this.block_count= 0;
-    this.block_total= 0;
     this.network    = config.network === 'testnet' ? networks.testnet: networks.livenet;
 
     var genesisHashReversed = new Buffer(32);
     this.network.genesisBlock.hash.copy(genesisHashReversed);
     this.genesis = genesisHashReversed.reverse().toString('hex');
     this.sync       = new Sync(opts);
+
+    //available status: new / syncing / finished / aborted
+    this.status   = 'new';
+    this.syncInfo = {};
   }
 
   function p() {
@@ -32,8 +34,9 @@ function spec() {
     console.log.apply(this, args);
   }
 
-  var progress_bar = function(string, current, total) {
+  var printProgress = function(i) {
-    p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
+    var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
+    p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
   };
 
   HistoricSync.prototype.init = function(opts,cb) {
@@ -47,7 +50,6 @@ function spec() {
   };
 
   HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
-
     var self = this;
 
     // recursion end.
@@ -71,8 +73,11 @@ function spec() {
       },
       //show some (inacurate) status
       function(c) {
-        if (self.block_count % 1000 === 1) {
+        var step = parseInt(self.syncInfo.blocksToSync / 100);
-          progress_bar('sync status:', self.block_count, self.block_total);
+        if (step < 10) step = 10;
+
+        if (self.syncInfo.syncedBlocks % step === 1) {
+          printProgress(self.syncInfo);
         }
         return c();
       },
@@ -113,15 +118,30 @@ function spec() {
       ],
       function (err){
 
-        if (err)
+        if (err) {
-          p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count);
+          self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
+          self.status = 'aborted';
+          p(self.err);
+        }
 
-        if (opts.uptoexisting && existed) {
+        else {
-          p('DONE. Found existing block: ', blockHash);
+          self.err = null;
-          return cb(err);
+          self.status = 'syncing';
+        }
+
+        if (opts.uptoexisting && existed ) {
+          if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) {
+            self.status = 'finished';
+            p('DONE. Found existing block: ', blockHash);
+            return cb(err);
+          }
+          else {
+            p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks);
+          }
         }
 
         if (blockEnd && blockEnd === blockHash) {
+          self.status = 'finished';
           p('DONE. Found END block: ', blockHash);
           return cb(err);
         }
@@ -129,7 +149,7 @@ function spec() {
 
         // Continue
         if (blockInfo && blockInfo.result) {
-          self.block_count++;
+          self.syncInfo.syncedBlocks++;
           if (opts.prev && blockInfo.result.previousblockhash) {
             return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
           }
@@ -144,11 +164,10 @@ function spec() {
   HistoricSync.prototype.import_history = function(opts, next) {
     var self = this;
 
-    var retry_attemps = 100;
     var retry_secs    = 2;
 
-    var block_best;
+    var bestBlock;
-    var block_height;
+    var blockChainHeight;
 
     async.series([
       function(cb) {
@@ -175,41 +194,56 @@ function spec() {
           return cb();
         }
       },
-      function(cb) {
-        self.rpc.getInfo(function(err, res) {
-          if (err) return cb(err);
 
-          self.block_total = res.result.blocks;
-          return cb();
-        });
-      },
       // We are not using getBestBlockHash, because is not available in all clients
       function(cb) {
         if (!opts.reverse) return cb();
 
         self.rpc.getBlockCount(function(err, res) {
           if (err) return cb(err);
-          block_height = res.result;
+          blockChainHeight = res.result;
           return cb();
         });
       },
       function(cb) {
         if (!opts.reverse) return cb();
 
-        self.rpc.getBlockHash(block_height, function(err, res) {
+        self.rpc.getBlockHash(blockChainHeight, function(err, res) {
           if (err) return cb(err);
 
-          block_best = res.result;
+          bestBlock = res.result;
+
           return cb();
         });
       },
+      function(cb) {
+        // This is only to inform progress.
+        if (!opts.uptoexisting) {
+          self.rpc.getInfo(function(err, res) {
+            if (err) return cb(err);
+            self.syncInfo.blocksToSync = res.result.blocks;
+            return cb();
+          });
+        }
+        else {
+          // should be isOrphan = true or null to be more accurate.
+          Block.count({ isOrphan: null}, function(err, count) {
+            if (err) return cb(err);
+
+            self.syncInfo.blocksToSync = blockChainHeight - count;
+            if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
+            return cb();
+          });
+         }
+      },
       ],
       function(err) {
 
+
         var start, end;
         function sync() {
           if (opts.reverse) {
-            start     = block_best;
+            start     = bestBlock;
             end       = self.genesis;
             opts.prev      = true;
           }
@@ -219,25 +253,38 @@ function spec() {
             opts.next      = true;
           }
 
+          self.syncInfo = util._extend(self.syncInfo, {
+            start: start,
+            isStartGenesis: start === self.genesis,
+            end:   end,
+            isEndGenesis: end === self.genesis,
+            scanningForward: opts.next,
+            scanningBackward: opts.prev,
+            uptoexisting: opts.uptoexisting,
+            syncedBlocks: 0,
+          });
+
           p('Starting from: ', start);
           p('         to  : ', end);
           p('         opts: ', JSON.stringify(opts));
 
           self.getPrevNextBlock( start, end, opts , function(err) {
-            if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
+            if (err && err.message.match(/ECONNREFUSED/)){
               setTimeout(function() {
                 p('Retrying in %d secs', retry_secs);
                 sync();
               }, retry_secs * 1000);
             }
             else
-              return next(err, self.block_count);
+              return next(err);
           });
         }
+
         if (!err)
           sync();
-        else
+        else {
           return next(err, 0);
+        }
     });
   };
 
@@ -246,6 +293,7 @@ function spec() {
     var self = this;
 
     Block.findOne({hash:self.genesis}, function(err, b){
+
       if (err) return next(err);
 
 
@@ -253,7 +301,7 @@ function spec() {
         p('Could not find Genesis block. Running FULL SYNC');
       }
       else {
-        p('Genesis block found. Syncing upto know blocks.');
+        p('Genesis block found. Syncing upto known blocks.');
       }
 
       var opts = {

lib/Sync.js

@@ -15,6 +15,48 @@ function spec() {
     this.tx_count   = 0;
   }
 
+  Sync.prototype.init = function(opts, cb) {
+    var that = this;
+
+    that.opts = opts;
+
+    if (!(opts && opts.skip_db_connection)) {
+
+
+      if (!mongoose.connection.readyState == 1) {
+        mongoose.connect(config.db, function(err) {
+          if (err) {
+            console.log('CRITICAL ERROR: connecting to mongoDB:',err);
+            return (err);
+          }
+        });
+      }
+
+      that.db = mongoose.connection;
+
+      that.db.on('error', function(err) {
+          console.log('MongoDB ERROR:' + err);
+          return cb(err);
+      });
+
+      that.db.on('disconnect', function(err) {
+          console.log('MongoDB disconnect:' + err);
+          return cb(err);
+      });
+
+      return that.db.once('open', function(err) {
+        return cb(err);
+      });
+    }
+    else return cb();
+  };
+
+  Sync.prototype.close = function() {
+    if ( this.db && this.db.readyState ) {
+      this.db.close();
+    }
+  };
+
   Sync.prototype.storeBlock = function(block, cb) {
     var that = this;
 
@@ -55,40 +97,6 @@ function spec() {
       return cb(err);
     });
   };
-
-
-  Sync.prototype.init = function(opts, cb) {
-    var that = this;
-
-    that.opts = opts;
-
-    if (!(opts && opts.skip_db_connection)) {
-      if (!mongoose.connection) {
-        mongoose.connect(config.db, {server: {auto_reconnect: true}} );
-      }
-
-      this.db = mongoose.connection;
-
-      this.db.on('error', function(err) {
-          console.log('connection error:' + err);
-          mongoose.disconnect();
-      });
-
-      this.db.on('disconnect', function(err) {
-          console.log('disconnect:' + err);
-          mongoose.connect(config.db, {server: {auto_reconnect: true}} );
-      });
-
-      return that.db.once('open', cb);
-    }
-    else return cb();
-  };
-
-  Sync.prototype.close = function() {
-    if (!(this.opts && this.opts.skip_db_connection)) {
-      this.db.close();
-    }
-  };
   return Sync;
 }
 module.defineClass(spec);

server.js

@@ -50,8 +50,11 @@ if (!config.disableHistoricSync) {
     skip_db_connection: true,
     networkName: config.network
   }, function() {
-    hs.smart_import(function(){
+    hs.smart_import(function(err){
-      console.log('[historic_sync] finished!');
+      var txt= 'ended.';
+      if (err) txt = 'ABORTED with error: ' + err.message;
+
+      console.log('[historic_sync] ' + txt, hs.syncInfo);
     });
   });
 }

util/sync.js

@@ -29,7 +29,6 @@ if (program.remove) {
 
 }
 */
-
 async.series([
   function(cb) {
     historicSync.init(program, cb);
@@ -46,18 +45,14 @@ async.series([
       }, cb);
     }
   },
-  function(cb) {
-    historicSync.close();
-    return cb();
-  },
   ],
-  function(err, count) {
+  function(err) {
+    historicSync.close();
     if (err) {
       console.log('CRITICAL ERROR: ', err);
     }
     else {
-      console.log('Finished. [%d blocks synced]', count[1]);
+      console.log('Finished.\n Status:\n', historicSync.syncInfo);
     }
-    return;
 });