Chnaged reorg to use cache hashes.

lib/services/block/index.js

@@ -11,6 +11,7 @@ var assert = require('assert');
 var constants = require('../../constants');
 var bcoin = require('bcoin');
 var _ = require('lodash');
+var LRU = require('lru-cache');
 
 var BlockService = function(options) {
 
@@ -25,8 +26,9 @@ var BlockService = function(options) {
   this.GENESIS_HASH = constants.BITCOIN_GENESIS_HASH[this.node.network];
   this._initialSync = false;
   this._serviceIniting = false;
-  this._reorgBackToBlock = null; // use this to rewind your indexes to a specific point by height or hash
   this._blocksInQueue = 0;
+  this._recentBlockHashesCount = options.recentBlockHashesCount || 50; // if you expect this chain to reorg deeper than 50, set this
+  this._recentBlockHashes = new LRU(this._recentBlockHashesCount);
 };
 
 inherits(BlockService, BaseService);
@@ -155,29 +157,12 @@ BlockService.prototype.getRawBlock = function(hash, callback) {
   });
 };
 
-BlockService.prototype._reorgBackTo = function(callback) {
-  var self = this;
-  self._header.getBlockHeader(self._reorgBackToBlock, function(err, header) {
-    if (err || !header) {
-      return callback(err || new Error('Header not found to reorg back to.'));
-    }
-    log.info('Block Service: we found the block to reorg back to, commencing reorg...');
-    self._handleReorg(header, callback);
-  });
-};
-
 BlockService.prototype._checkTip = function(callback) {
 
   var self = this;
 
   log.info('Block Service: checking the saved tip...');
 
-  if (self._reorgBackToBlock) {
-    self._reorgBackToBlock = false;
-    log.warn('Block Service: we were asked to reorg back to block: ' + self._reorgBackToBlock);
-    return self._reorgBackTo(callback);
-  }
-
   self._header.getBlockHeader(self._tip.height, function(err, header) {
 
     if (err) {
@@ -191,53 +176,64 @@ BlockService.prototype._checkTip = function(callback) {
       return callback();
     }
 
-    self._findCommonAncestor(function(err, commonAncestorHeader) {
+    self._findCommonAncestorAndBlockHashesToRemove(function(err, commonAncestorHeader, hashesToRemove) {
-      if(err) {
+
+      if (err) {
         return callback(err);
       }
-      self._handleReorg(commonAncestorHeader, callback);
+
+      self._handleReorg(commonAncestorHeader, hashesToRemove, callback);
     });
 
   });
 };
 
-BlockService.prototype._findCommonAncestor = function(callback) {
+BlockService.prototype._findCommonAncestorAndBlockHashesToRemove = function(callback) {
 
   var self = this;
-  var hash = self._tip.hash;
+
+  var hashes = [{
+    hash: self._tip.hash,
+    height: self._tip.height
+  }];
+
   var header;
+  var iterCount = 0;
 
   async.until(function() {
 
-    return header;
+    return header || iterCount++ >= self._recentBlockHashesCount;
 
   }, function(next) {
 
-    self._getBlock(hash, function(err, block) {
+    var hash = self._recentBlockHashes.get(hash);
 
-      if (err || !block) {
+    hashes.push({
-        return callback(err || new Error('Block Service: went looking for the tip block, but found nothing.'));
+      tip: hash,
+      height: hashes[hashes.length - 1].height - 1
+    });
+
+    self._header.getBlockHeader(hash, function(err, _header) {
+
+      if (err) {
+        return next(err);
       }
 
-      hash = bcoin.util.revHex(block.prevBlock);
+      header = _header;
-
+      next();
-      self._header.getBlockHeader(hash, function(err, _header) {
-
-        if (err) {
-          return next(err);
-        }
-
-        header = _header;
-        next();
-      });
     });
+
   }, function(err) {
 
     if (err) {
       return callback(err);
     }
 
-    callback(null, header);
+    // ensure the common ancestor hash is not in the blocks to remove hashes
+    hashes.pop();
+    assert(hashes.length >= 1, 'Block Service: we expected to remove at least one block, but we did not have at least one block.');
+    callback(null, header, hashes);
+
   });
 
 };
@@ -374,13 +370,51 @@ BlockService.prototype.start = function(callback) {
         self._reorging = false;
       });
 
-      self._setTip(tip, callback);
+      self._setTip(tip, function(err) {
+        if (err) {
+          return callback(err);
+        }
+        self._loadRecentBlockHashes(callback);
+      });
     });
 
   });
 
 };
 
+BlockService.prototype._loadRecentBlockHashes = function(callback) {
+
+  var self = this;
+  var hash = self._tip.hash;
+
+  async.times(Math.min(self._tip.height, self._recentBlockHashesCount), function(n, next) {
+
+    self.getBlock(hash, function(err, block) {
+
+      if (err) {
+        return callback(err);
+      }
+
+      var prevHash = bcoin.util.revHex(block.prevBlock);
+      self._recentBlockHashes.set(hash, prevHash);
+      hash = prevHash;
+      next();
+
+    });
+
+  }, function(err) {
+
+    if (err) {
+      return callback(err);
+    }
+
+    log.info('Block Service: loaded: ' + self._recentBlockHashesCount + ' hashes from the index.');
+    callback();
+
+  });
+
+};
+
 BlockService.prototype.stop = function(callback) {
   setImmediate(callback);
 };
@@ -620,7 +654,7 @@ BlockService.prototype._saveTip = function(tip, callback) {
   this._db.put(tipOps.key, tipOps.value, callback);
 };
 
-BlockService.prototype._handleReorg = function(commonAncestorHeader, callback) {
+BlockService.prototype._handleReorg = function(commonAncestorHeader, hashesToRemove, callback) {
 
   var self = this;
 
@@ -630,84 +664,57 @@ BlockService.prototype._handleReorg = function(commonAncestorHeader, callback) {
   log.warn('Block Service: chain reorganization detected, current height/hash: ' + self._tip.height + '/' +
     self._tip.hash + ' common ancestor hash: ' + commonAncestorHeader.hash + ' at height: ' + commonAncestorHeader.height);
 
-  var oldTip = { height: self._tip.height, hash: self._tip.hash };
-
   async.series([
     self._setTip.bind(self, { hash: commonAncestorHeader.hash, height: commonAncestorHeader.height }),
-    self._processReorg.bind(self, commonAncestorHeader, oldTip),
+    self._processReorg.bind(self, commonAncestorHeader, hashesToRemove),
   ], callback);
 
 };
 
-BlockService.prototype._processReorg = function(commonAncestorHeader, oldTip, callback) {
+BlockService.prototype._processReorg = function(commonAncestorHeader, hashesToRemove, callback) {
 
   var self = this;
   var operations = [];
-  var tip = oldTip;
   var blockCount = 0;
   var bar = new utils.IndeterminateProgressBar();
 
   log.info('Block Service: Processing the reorganization.');
 
-  if (commonAncestorHeader.hash === tip.hash) {
+  async.eachSeries(hashesToRemove, function(tip, next) {
-    return callback(null, []);
-  }
-
-  async.whilst(
-
-    function() {
 
       if (process.stdout.isTTY) {
         bar.tick();
       }
-      return tip.hash !== commonAncestorHeader.hash;
 
-    },
+      self._getReorgBlock(tip, function(err, block) {
 
-    function(next) {
+        if (err || !block)  {
-
+          return next(err || new Error('Block Service: block should be in the index.'));
-      async.waterfall([
-
-        self._getReorgBlock.bind(self, tip),
-
-        function(block, next) {
-
-          tip = {
-            hash: bcoin.util.revHex(block.prevBlock),
-            height: tip.height - 1
-          };
-
-          next(null, block);
-
-        },
-
-        function(block, next) {
-          self._onReorg(commonAncestorHeader.hash, block, next);
         }
 
-      ], function(err, ops) {
+        self._onReorg(commonAncestorHeader.hash, block, function(err, ops) {
 
-        if(err) {
+          if (err) {
-          return next(err);
+            return next(err);
-        }
+          }
 
-        blockCount++;
+          blockCount++;
-        operations = operations.concat(ops);
+          operations = operations.concat(ops);
-        next();
+          self._recentBlockHashes.del(tip.hash);
+          next();
+        });
 
       });
-    },
+  }, function(err) {
 
-    function(err) {
+    if (err) {
+      return callback(err);
+    }
 
-      if (err) {
+    log.info('Block Service: removed ' + blockCount + ' block(s) during the reorganization event.');
-        return callback(err);
+    self._db.batch(_.compact(_.flattenDeep(operations)), callback);
-      }
 
-      log.info('Block Service: removed ' + blockCount + ' block(s) during the reorganization event.');
+  });
-      self._db.batch(_.compact(_.flattenDeep(operations)), callback);
-
-    });
 };
 
 BlockService.prototype._getReorgBlock = function(tip, callback) {
@@ -811,6 +818,7 @@ BlockService.prototype._saveBlock = function(block, callback) {
         return callback(err);
       }
 
+      self._recentBlockHashes.set(block.rhash(), bcoin.util.revHex(block.prevBlock));
       self._setTip({ hash: block.rhash(), height: self._tip.height + 1 }, callback);
 
     });

lib/utils.js

@@ -7,6 +7,9 @@ var BN = require('bn.js');
 var utils = {};
 
 utils.isHeight = function(blockArg) {
+  if (!blockArg && blockArg !== 0) {
+    return false;
+  }
   return _.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg));
 };
 

test/services/block/index.unit.js

@@ -44,16 +44,16 @@ describe('Block Service', function() {
     });
   });
 
-  describe('#_findCommonAncestor', function() {
+  describe('#_findCommonAncestorAndBlockHashesToRemove', function() {
 
-    it('should find the common ancestor between the current chain and the new chain', function(done) {
+    it('should find the common ancestor and hashes between the current chain and the new chain', function(done) {
 
       sandbox.stub(blockService, '_getBlock').callsArgWith(1, null, block2);
       blockService._tip = { hash: 'aa' };
       var headers = new utils.SimpleMap();
 
       blockService._header = { getBlockHeader: sandbox.stub().callsArgWith(1, null, { hash: 'bb' }) };
-      blockService._findCommonAncestor(function(err, header) {
+      blockService._findCommonAncestorAndBlockHashesToRemove(function(err, header, hashes) {
 
         if(err) {
           return done(err);
@@ -158,6 +158,7 @@ describe('Block Service', function() {
       var getPrefix = sandbox.stub().callsArgWith(1, null, blockService._encoding);
       var getServiceTip = sandbox.stub().callsArgWith(1, null, { height: 1, hash: 'aa' });
       var performSanityCheck = sandbox.stub(blockService, '_performSanityCheck').callsArgWith(1, null, { hash: 'aa', height: 123 });
+      var loadRecentBlockHashes = sandbox.stub(blockService, '_loadRecentBlockHashes').callsArgWith(0, null, new utils.SimpleMap());
       var setTip = sandbox.stub(blockService, '_setTip').callsArgWith(1, null);
       blockService.node = { openBus: sandbox.stub() };
       blockService._db = { getPrefix: getPrefix, getServiceTip: getServiceTip };