Fixed reorg.

lib/scaffold/start.js

@@ -102,7 +102,7 @@ function lookInBuiltInPath(req, service) {
     var serviceFile = path.resolve(__dirname, '../services/' + service.name);
     return req(serviceFile);
   } catch (e) {
-    if (e.code !== "MODULE_NOT_FOUND") {
+    if (e.code !== 'MODULE_NOT_FOUND') {
       log.error(e);
     }
     log.info('Checked the built-in path: lib/services, for service: ' + service.name);

lib/services/block/index.js

@@ -22,10 +22,9 @@ var BlockService = function(options) {
   this._header = this.node.services.header;
   this._timestamp = this.node.services.timestamp;
 
-  this._blockCount = 0;
   this.GENESIS_HASH = constants.BITCOIN_GENESIS_HASH[this.node.network];
   this._initialSync = true;
-  this._reorgBackToBlock = 1202419; // use this to rewind your indexes to a specific point by height or hash
+  this._reorgBackToBlock = null; // use this to rewind your indexes to a specific point by height or hash
   this._timeOfLastBlockReport = Date.now() - 30000;
   this._blocksInQueue = 0;
 };
@@ -161,7 +160,7 @@ BlockService.prototype._reorgBackTo = function(callback) {
       return callback(err || new Error('Header not found to reorg back to.'));
     }
     log.info('Block Service: we found the block to reorg back to, commencing reorg...');
-    self._handleReorg(header.hash, callback);
+    self._handleReorg(header, callback);
   });
 };
 
@@ -189,11 +188,11 @@ BlockService.prototype._checkTip = function(callback) {
       return callback();
     }
 
-    self._findCommonAncestor(function(err, commonAncestorHash) {
+    self._findCommonAncestor(function(err, commonAncestorHeader) {
       if(err) {
         return callback(err);
       }
-      self._handleReorg(commonAncestorHash, callback);
+      self._handleReorg(commonAncestorHeader, callback);
     });
 
   });
@@ -203,30 +202,41 @@ BlockService.prototype._findCommonAncestor = function(callback) {
 
   var self = this;
   var hash = self._tip.hash;
+  var header;
 
-  self._header.getAllHeaders(function(err, headers) {
+  async.until(function() {
 
-    if(err || !headers) {
+    return header;
-      return callback(err || new Error('headers required.'));
-    }
 
-    async.until(function() {
+  }, function(next) {
-      return headers.get(hash);
+
-    }, function(next) {
+    self._getBlock(hash, function(err, block) {
-      self._getBlock(hash, function(err, block) {
+
-        if(err || !block) {
+      if (err || !block) {
-          return next(err || new Error('block must be found in order to find common ancestor.'));
+        return callback(err || new Error('Block Service: went looking for the tip block, but found nothing.'));
+      }
+
+      hash = bcoin.util.revHex(block.prevBlock);
+
+      self._header.getBlockHeader(hash, function(err, _header) {
+
+        if (err) {
+          return next(err);
         }
-        hash = bcoin.util.revHex(block.prevBlock);
+
+        header = _header;
         next();
       });
-    }, function(err) {
-      if(err) {
-        return callback(err);
-      }
-      callback(null, hash);
     });
+  }, function(err) {
+
+    if (err) {
+      return callback(err);
+    }
+
+    callback(null, header);
   });
+
 };
 
 BlockService.prototype._resetTip = function(callback) {
@@ -285,8 +295,9 @@ BlockService.prototype._resetTip = function(callback) {
         return callback(err ||
           new Error('Block Service: none of the blocks from the headers match what is already indexed in the block service.'));
       }
-      self._setTip({ hash: block.rhash(), height: height + 1 });
+
-      callback();
+      self._setTip({ hash: block.rhash(), height: height + 1 }, callback);
+
     });
 
   });
@@ -343,10 +354,8 @@ BlockService.prototype.start = function(callback) {
       }
 
       self._blockProcessor = async.queue(self._onBlock.bind(self));
-      self._setTip(tip);
       self._bus = self.node.openBus({remoteAddress: 'localhost-block'});
-
+      self._setTip(tip, callback);
-      callback();
     });
 
   });
@@ -447,19 +456,10 @@ BlockService.prototype.onReorg = function(args, callback) {
 
   var block = args[1][0];
 
-  self._setTip({ hash: block.rhash(), height: self._tip.height - 1 });
-  var tipOps = utils.encodeTip(self._tip, self.name);
-
   var removalOps = [{
-    type: 'put',
-    key: tipOps.key,
-    value: tipOps.value
-  }];
-
-  removalOps.push({
     type: 'del',
     key: self._encoding.encodeBlockKey(block.rhash()),
-  });
+  }];
 
   setImmediate(function() {
     callback(null, removalOps);
@@ -540,7 +540,17 @@ BlockService.prototype._startBlockSubscription = function() {
 
 };
 
-BlockService.prototype._handleReorg = function(commonAncestorHash, callback) {
+BlockService.prototype._saveTip = function(tip, callback) {
+
+  var tipOps = utils.encodeTip({
+    hash: tip.hash,
+    height: tip.height
+  }, this.name);
+
+  this._db.put(tipOps.key, tipOps.value, callback);
+};
+
+BlockService.prototype._handleReorg = function(commonAncestorHeader, callback) {
 
   var self = this;
 
@@ -548,42 +558,71 @@ BlockService.prototype._handleReorg = function(commonAncestorHash, callback) {
   self._p2p.clearInventoryCache();
 
   log.warn('Block Service: chain reorganization detected, current height/hash: ' + self._tip.height + '/' +
-    self._tip.hash + ' common ancestor hash: ' + commonAncestorHash);
+    self._tip.hash + ' common ancestor hash: ' + commonAncestorHeader.hash + ' at height: ' + commonAncestorHeader.height);
 
+  var oldTip = { height: self._tip.height, hash: self._tip.hash };
+
+  async.series([
+    self._setTip.bind(self, { hash: commonAncestorHeader.hash, height: commonAncestorHeader.height }),
+    self._processReorg.bind(self, commonAncestorHeader, oldTip),
+  ], callback);
+
+};
+
+BlockService.prototype._processReorg = function(commonAncestorHeader, oldTip, callback) {
+
+  var self = this;
   var operations = [];
-  var tip = self._tip;
+  var tip = oldTip;
   var blockCount = 0;
+  var bar = new utils.IndeterminateProgressBar();
+
+  log.info('Block Service: Processing the reorganization.');
+
+  if (commonAncestorHeader.hash === tip.hash) {
+    return callback(null, []);
+  }
 
-  // we don't know how many blocks we need to remove until we've reached the common ancestor
   async.whilst(
+
     function() {
-      return tip.hash !== commonAncestorHash;
+
+      bar.tick();
+      return tip.hash !== commonAncestorHeader.hash;
+
     },
 
     function(next) {
+
       async.waterfall([
 
         self._getReorgBlock.bind(self, tip),
 
         function(block, next) {
+
           tip = {
             hash: bcoin.util.revHex(block.prevBlock),
             height: tip.height - 1
           };
+
           next(null, block);
+
         },
 
         function(block, next) {
-          self._onReorg(commonAncestorHash, block, next);
+          self._onReorg(commonAncestorHeader.hash, block, next);
         }
 
       ], function(err, ops) {
+
         if(err) {
           return next(err);
         }
+
         blockCount++;
         operations = operations.concat(ops);
         next();
+
       });
     },
 
@@ -684,27 +723,19 @@ BlockService.prototype._saveBlock = function(block, callback) {
     service.onBlock.call(service, block, next);
 
   }, function(err, ops) {
+
     if (err) {
       return callback(err);
     }
 
     self._db.batch(_.compact(_.flattenDeep(ops)), function(err) {
+
       if (err) {
         return callback(err);
       }
 
-      var tipOps = utils.encodeTip({
+      self._setTip({ hash: block.rhash(), height: self._tip.height + 1 }, callback);
-        hash: block.rhash(),
-        height: self._tip.height + 1
-      }, self.name);
 
-      self._db.put(tipOps.key, tipOps.value, function(err) {
-        if(err) {
-          return callback(err);
-        }
-        self._setTip({ hash: block.rhash(), height: self._tip.height + 1 });
-        callback();
-      });
     });
   });
 };
@@ -742,14 +773,20 @@ BlockService.prototype.onBlock = function(block, callback) {
   });
 };
 
-BlockService.prototype._setTip = function(tip) {
+BlockService.prototype._setTip = function(tip, callback) {
   log.debug('Block Service: Setting tip to height: ' + tip.height);
   log.debug('Block Service: Setting tip to hash: ' + tip.hash);
   this._tip = tip;
+  this._saveTip(tip, callback);
 };
 
 BlockService.prototype._onSynced = function() {
   var self = this;
+
+  if (this._reportInterval) {
+    clearInterval(this._reportInterval);
+  }
+
   self._logProgress();
   self._initialSync = false;
   self._startBlockSubscription();

lib/utils.js

@@ -103,21 +103,37 @@ utils.SimpleMap = function SimpleMap() {
   };
 };
 
+utils.IndeterminateProgressBar = function IndeterminateProgressBar() {
+
+  var states = ['|', '/', '-', '\\'];
+
+  this.state = 0;
+
+  this.tick = function() {
+    process.stdout.clearLine();
+    process.stdout.cursorTo(0);
+    process.stdout.write(states[this.state++ % states.length]);
+  };
+};
+
 utils.convertMillisecondsToHumanReadable = function(ms) {
   var ret = '';
 
+  var minutes;
+  var seconds;
+
   if (!ms && ms !== 0) {
     return 'invalid number of ms.';
   }
 
   if (ms >= 60000) {
-    var minutes = Math.floor(ms / 60000);
+    minutes = Math.floor(ms / 60000);
-    var ms = ms % 60000;
+    ms = ms % 60000;
   }
 
   if (ms >= 1000) {
-    var seconds = Math.floor(ms / 1000);
+    seconds = Math.floor(ms / 1000);
-    var ms = ms % 1000;
+    ms = ms % 1000;
   }
 
   if (minutes) {

package-lock.json

@@ -211,6 +211,19 @@
           "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.7.tgz",
           "integrity": "sha512-LxFiV5mefv0ley0SzqkOPR1bC4EbpPx8LkOz5vMe/Yi15t5hzwgO/G+tc7wOtL4PZTYjwHu8JnEiSLumuSjSfA=="
         },
+        "leveldown": {
+          "version": "1.7.2",
+          "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.7.2.tgz",
+          "integrity": "sha1-XjRnuyfuJGpKe429j7KxYgam64s=",
+          "optional": true,
+          "requires": {
+            "abstract-leveldown": "2.6.1",
+            "bindings": "1.2.1",
+            "fast-future": "1.0.2",
+            "nan": "2.6.2",
+            "prebuild-install": "2.2.0"
+          }
+        },
         "ms": {
           "version": "0.7.3",
           "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.3.tgz",
@@ -2616,15 +2629,35 @@
       }
     },
     "leveldown": {
-      "version": "1.7.2",
+      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.7.2.tgz",
+      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.8.0.tgz",
-      "integrity": "sha1-XjRnuyfuJGpKe429j7KxYgam64s=",
+      "integrity": "sha512-4r02OXouz24Sxs+i8ZNgDZxrRMRZq6BWS7pj6vjKEb8DFnEsveyaPIxF9Y3uHL5+jftWUDsmThT90epRrC6aGw==",
       "requires": {
-        "abstract-leveldown": "2.6.1",
+        "abstract-leveldown": "2.7.0",
-        "bindings": "1.2.1",
+        "bindings": "1.3.0",
         "fast-future": "1.0.2",
-        "nan": "2.6.2",
+        "nan": "2.7.0",
         "prebuild-install": "2.2.0"
+      },
+      "dependencies": {
+        "abstract-leveldown": {
+          "version": "2.7.0",
+          "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-2.7.0.tgz",
+          "integrity": "sha512-maam3ZrTeORbXKEJUeJZkYOsorEwr060WitXuQlUuIFlg0RofyyHts49wtaVmShJ6l0wEWB0ZtPhf6QYBA7D2w==",
+          "requires": {
+            "xtend": "4.0.1"
+          }
+        },
+        "bindings": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz",
+          "integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw=="
+        },
+        "nan": {
+          "version": "2.7.0",
+          "resolved": "https://registry.npmjs.org/nan/-/nan-2.7.0.tgz",
+          "integrity": "sha1-2Vv3IeyHfgjbJ27T/G63j5CDrUY="
+        }
       }
     },
     "levelup": {