Fixed reorg where we don't have all the previous blocks.

lib/services/block/index.js

@@ -717,7 +717,7 @@ BlockService.prototype._processBlock = function(block, callback) {
     return callback();
   }
 
-  log.info('Block Service: new block: ' + block.rhash());
+  log.debug('Block Service: new block: ' + block.rhash());
 
   // common case
   if (!self._detectReorg(block)) {

lib/services/header/index.js

@@ -124,7 +124,7 @@ HeaderService.prototype.getBestHeight = function() {
   return this._tip.height;
 };
 
-HeaderService.prototype._adjustTip = function() {
+HeaderService.prototype._adjustTipBackToCheckpoint = function() {
 
   if (this._checkpoint === -1 || this._tip.height < this._checkpoint) {
 
@@ -190,13 +190,13 @@ HeaderService.prototype.start = function(callback) {
 
       self._tip = tip;
 
-      self._adjustTip();
+      self._adjustTipBackToCheckpoint();
 
       if (self._tip.height === 0) {
         return self._setGenesisBlock(next);
       }
 
-      self._getLastHeader(next);
+      self._adjustHeadersForCheckPointTip(next);
 
     },
   ], function(err) {
@@ -272,7 +272,7 @@ HeaderService.prototype._processBlocks = function(block, callback) {
 
   var self = this;
 
-  if (self.node.stopping) {
+  if (self.node.stopping || self._reorging) {
     return callback();
   }
 
@@ -291,16 +291,6 @@ HeaderService.prototype._processBlocks = function(block, callback) {
 
 };
 
-HeaderService.prototype._findCommonAncestor = function(block, callback) {
-  var self = this;
-  self.getBlockHeader(bcoin.util.revHex(block.prevBlock), function(err, header) {
-    if (err) {
-      return callback(err);
-    }
-    callback(null, header);
-  });
-};
-
 HeaderService.prototype._persistHeader = function(block, callback) {
 
   var self = this;
@@ -309,30 +299,17 @@ HeaderService.prototype._persistHeader = function(block, callback) {
     return self._syncBlock(block, callback);
   }
 
-  // this will lead to rechecking the network for more headers after our last header
+  self._reorging =  true;
-  // and calling on=Headers on each service that implements it
-  self._initialSync = true;
 
-  self._findCommonAncestor(block, function(err, commonHeader) {
+  self._handleReorg(block, function(err) {
 
-    if (err || !commonHeader) {
+    if(err) {
-      return callback(err ||
+      return callback(err);
-        new Error('Header Service: a common header could not be found between the new block: ' +
-          block.rhash() + ' and the current tip: ' + self._tip.hash));
     }
 
-    self._handleReorg(block, commonHeader, function(err) {
+    self._startSync();
-
+    callback();
-      if(err) {
-        return callback(err);
-      }
-
-      self._syncBlock(block, callback);
-
-    });
-
   });
-
 };
 
 HeaderService.prototype._formatHeader = function(block) {
@@ -524,6 +501,7 @@ HeaderService.prototype._onHeadersSave = function(callback) {
   log.info('Header Service: sync complete.');
   self._initialSync = false;
 
+  // this is where the other services are called to let them know we have a good set of headers
   async.eachSeries(self.node.services, function(service, next) {
     if (service.onHeaders) {
       return service.onHeaders.call(service, next);
@@ -603,71 +581,42 @@ HeaderService.prototype._detectReorg = function(block) {
   return bcoin.util.revHex(block.prevBlock) !== this._lastHeader.hash;
 };
 
-HeaderService.prototype._handleReorg = function(block, commonHeader, callback) {
+HeaderService.prototype._handleReorg = function(block, callback) {
 
   var self = this;
 
   log.warn('Header Service: Reorganization detected, current tip hash: ' +
-    self._tip.hash + ', new block causing the reorg: ' + block.rhash() +
+    self._tip.hash + ', new block causing the reorg: ' + block.rhash());
-    ' common ancestor hash: ' + commonHeader.hash + ' and height: ' +
-    commonHeader.height);
 
-  var reorgHeader = self._formatHeader(block);
+  // at this point, we have a block that does not directly link to our
+  // last header. This is all we know for sure. We may not have this block's
+  // previous blocks either, which means we need to go out and re-retrieve
+  // a list of the latest headers and gather those blocks. If the peer hasn't
+  // completed its own reorganization, we may need to defer the rest of the system
+  // reorg until we get a list of headers that correctly links from this block
+  // all the way back to the genesis block.
 
-  self.getAllHeaders(function(err, headers) {
+  // first, we'll adjust the tip back to the last checkpoint just like we do when
+  // the service starts up.
+  self._adjustTipBackToCheckpoint();
 
-    if (err || !headers) {
+  // then, we'll get the last header from the database which will nuke out all the
-      return callback(err || new Error('Missing headers'));
+  // headers that are greater than new tip height.
+  self._adjustHeadersForCheckPointTip(function(err) {
+    if(err) {
+      return callback(err);
     }
 
-    var hash = block.rhash();
+    // we will set the reorg block here so that when startSync completes,
-    headers.set(hash, reorgHeader); // appends to the end
+    // it can check to ensure the final set of header contains the reorg block.
-
+    // in other words, our peer has completed its own reorg process and is delievering
-    // this will ensure our own headers collection is correct
+    // us a valid set of headers.
-    self._onReorg(reorgHeader, headers, commonHeader, callback);
+    self._reorgBlock = block;
+    callback();
   });
 
 };
 
-HeaderService.prototype._onReorg = function(reorgHeader, headers, commonHeader, callback) {
-  // remove all headers with a height greater than commonHeader
-
-  assert(this._tip.hash !== commonHeader.hash, 'Header Service: we were asked to reorg from and to the same hash.');
-
-  var ops = [];
-  var startingHeight = this._tip.height;
-  var hash = this._tip.hash;
-  var headerCount = 0;
-
-  while(hash !== commonHeader.hash) {
-
-    headerCount++;
-
-    var header = headers.getIndex(startingHeight--);
-
-    assert(header, 'Expected to have a header at this height, but did not. Reorg failed.');
-    hash = header.prevHash;
-
-    ops.push({
-      type: 'del',
-      key: this._encoding.encodeHeaderHashKey(header.hash)
-    });
-    ops.push({
-      type: 'del',
-      key: this._encoding.encodeHeaderHeightKey(header.height)
-    });
-
-  }
-
-  // setting our tip to the common ancestor
-  this._tip.hash = commonHeader.hash;
-  this._tip.height = commonHeader.height;
-  this._lastHeader = commonHeader;
-
-  log.info('Header Service: removed ' + headerCount + ' header(s) during the reorganization event.');
-  this._db.batch(ops, callback);
-};
-
 HeaderService.prototype._setListeners = function() {
   this._p2p.on('bestHeight', this._onBestHeight.bind(this));
 };
@@ -700,6 +649,7 @@ HeaderService.prototype._startSync = function() {
 
   }, function() {
 
+    self._reorging = false;
     var numNeeded = self._bestHeight - self._tip.height;
 
     // common case
@@ -841,15 +791,8 @@ HeaderService.prototype._handleLowTipHeight = function() {
           self._handleError(err);
         }
 
-        self._syncBlock(block, function(err) {
+        // run start sync again. This time we should be back on track.
-
+        self._startSync();
-          if (err) {
-            self._handleError(err);
-          }
-
-          // run start sync again. This time we should be back on track.
-          self._startSync();
-        });
 
       });
     });
@@ -963,7 +906,7 @@ HeaderService.prototype.getLastHeader = function() {
   return this._lastHeader;
 };
 
-HeaderService.prototype._getLastHeader = function(callback) {
+HeaderService.prototype._adjustHeadersForCheckPointTip = function(callback) {
 
   var self = this;
 
@@ -991,13 +934,20 @@ HeaderService.prototype._getLastHeader = function(callback) {
 
     // any records with a height greater than our current tip height can be scheduled for removal
     // because they will be replaced shortly
+    // and for every height record, we must also remove its hash record
     if (header.height > self._tip.height) {
       removalOps.push({
         type: 'del',
         key: data.key
       });
+      removalOps.push({
+        type: 'del',
+        key: self._encoding.encodeHeaderHashKey(header.hash)
+      });
       return;
-    } else if (header.height === self._tip.height) {
+    }
+
+    if (header.height === self._tip.height) {
       self._lastHeader = header;
     }
 

package-lock.json

@@ -2629,9 +2629,9 @@
       }
     },
     "leveldown": {
-      "version": "1.9.0",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.9.0.tgz",
+      "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-2.0.0.tgz",
-      "integrity": "sha512-3MwcrnCUIuFiKp/jSrG1UqDTV4k1yH8f5HH6T9dpqCKG+lRxcfo2KwAqbzTT+TTKfCbaATeHMy9mm1y6sI3ZvA==",
+      "integrity": "sha512-KZOtzMj/XW+0J+pwdLOmnu3qAgjAUL74OBHx3+s9aM+uWPvyj5XJoNUe4nPkTTi6/bA9OxzFVVY5dCN5YBbbqQ==",
       "requires": {
         "abstract-leveldown": "2.7.1",
         "bindings": "1.3.0",