Fixed initial sync issue with headers.

lib/services/header/index.js

@@ -26,7 +26,6 @@ var HeaderService = function(options) {
   this.subscriptions.block = [];
   this._checkpoint = options.checkpoint || 2000;
   this.GENESIS_HASH = constants.BITCOIN_GENESIS_HASH[this.node.network];
-  this._newBlocksHeight = 0;
 };
 
 inherits(HeaderService, BaseService);
@@ -93,14 +92,17 @@ HeaderService.prototype.start = function(callback) {
       self._db.getServiceTip(self.name, next);
     },
     function(tip, next) {
+
       self._tip = tip;
-      self._originalTip = {
+      log.debug('Header Service: original tip height is: ' + self._tip.height);
-        hash: self._tip.hash,
+      log.debug('Header Service: original tip hash is: ' + self._tip.hash);
-        height: self._tip.height
+
-      };
+      self._originalTip = Object.assign(self._tip, {});
 
       if (self._tip.height === 0) {
 
+        assert(self._tip.hash === self.GENESIS_HASH, 'Expected tip hash to be genesis hash, but it was not.');
+
         var genesisHeader = {
           hash: self.GENESIS_HASH,
           height: 0,
@@ -113,7 +115,7 @@ HeaderService.prototype.start = function(callback) {
           merkleRoot: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'
         };
 
-        self._headers.set(self.GENESIS_HASH, genesisHeader);
+        self._headers.set(self.GENESIS_HASH, genesisHeader, 0);
 
         self._db._store.put(self._encoding.encodeHeaderKey(0, self.GENESIS_HASH),
           self._encoding.encodeHeaderValue(genesisHeader), next);
@@ -187,7 +189,6 @@ HeaderService.prototype._onBlock = function(block) {
     header = block.toHeaders().toJSON();
     header.timestamp = header.ts;
     header.prevHash = header.prevBlock;
-    header.height = ++this._newBlocksHeight;
     this._onHeaders([header]);
   }
 
@@ -203,25 +204,25 @@ HeaderService.prototype._onHeaders = function(headers) {
   log.debug('Header Service: Received: ' + headers.length + ' header(s).');
 
   var dbOps = [];
+  var headerListLength = 0;
 
   for(var i = 0; i < headers.length; i++) {
 
     var header = headers[i];
+
+    // headers that come from a call to getheaders and not a new block comoing in
     if (header instanceof Header) {
       header = header.toObject();
-      header.height = ++self._tip.height;
+      self._tip.height++;
+      self._tip.hash = header.hash;
+      headerListLength = headers.length;
     }
 
-    header.chainwork = self._getChainwork(header).toString(16, 64);
+    var prevHeader = this._headers.get(header.prevHash);
-    self._lastChainwork = header.chainwork;
+    assert(prevHeader, 'We must have a previous header in order to calculate this header\'s data.');
 
-    self._tip.hash = header.hash;
+    header.height = prevHeader.height + 1;
-
+    header.chainwork = self._getChainwork(header, prevHeader).toString(16, 64);
-    dbOps.push({
-      type: 'put',
-      key: self._encoding.encodeHeaderKey(header.height, header.hash),
-      value: self._encoding.encodeHeaderValue(header)
-    });
 
     var newHdr = {
       hash: header.hash,
@@ -230,8 +231,15 @@ HeaderService.prototype._onHeaders = function(headers) {
       chainwork: header.chainwork
     };
 
+    // note: this could lead to nulls in positions we don't yet have headers for
+    // that should be ok because eventually all the missing headers will fill in
     self._headers.set(header.hash, newHdr, header.height);
 
+    dbOps.push({
+      type: 'put',
+      key: self._encoding.encodeHeaderKey(header.height, header.hash),
+      value: self._encoding.encodeHeaderValue(header)
+    });
   }
 
   var tipOps = utils.encodeTip(self._tip, self.name);
@@ -246,16 +254,27 @@ HeaderService.prototype._onHeaders = function(headers) {
 
     if(err) {
       log.error(err);
-      this.node.stop();
+      self.node.stop();
       return;
     }
 
-    if (self._tip.height < self._bestHeight) {
+    // once we've received less than 2000 headers, we know we are fully sync as far as headers
+    // if we reveive 2000 headers and there are no more after that, (the best height is a multiple of 2000),
+    // then we'll just wait for the next block to come in. This should be pretty rare occurrence
+
+    if (headerListLength === 2000) {
       self._sync();
       return;
     }
 
-    log.debug('Header Service: ' + self._headers.getIndex(self._bestHeight).hash + ' is the best block hash.');
+    // at this point, we should have no empty items in this._headers, everything should be filled in
+    assert(!self._headers.hasNullItems(), 'Header list is not complete yet peer has sent all available headers.');
+
+    var bestHeader = self._headers.getLastIndex();
+    self._tip.height = bestHeader.height;
+    self._tip.hash = bestHeader.hash;
+
+    log.debug('Header Service: ' + bestHeader.hash + ' is the best block hash.');
 
     // at this point, we can check our header list to see if our starting tip diverged from the tip
     // that we have now
@@ -265,23 +284,47 @@ HeaderService.prototype._onHeaders = function(headers) {
     }
 
     log.debug('Header Service: emitting headers to block service.');
+    // populate next hash fields on each header
+    self._populateNextHashes();
+
     self.emit('headers', self._headers);
 
   });
 
 };
 
+HeaderService.prototype._populateNextHashes = function() {
+
+  var count = 0;
+
+  while (count < this._headers.length) {
+    var hdr = this._headers.getIndex(count);
+    var nextHdr = this._headers.getIndex(++count);
+    if (nextHdr) {
+      hdr.nextHash = nextHdr.hash;
+    }
+  }
+
+};
+
 HeaderService.prototype._detectReorg = function() {
+
   // is our original tip's height and hash the same after we rewound by the checkpoint amount of blocks
   // and re-imported? If so, then we've reorg'ed since we've been shut down.
-
   var headerHash = this._headers.getIndex(this._originalTip.height).hash;
+
+  log.debug('Header Service: attempting to detect a reorg situation after fully syncing headers, original tip height is: ' +
+    this._originalTip.height + ' original tip hash: '  + this._originalTip.hash +  ' headers list size: ' + this._headers.size +
+      ' hash at original tip height in headers list: ' + headerHash);
+
   assert(headerHash, 'Expected a header to exist at height ' + this._originalTip.height);
 
   if (this._originalTip.hash !== headerHash) {
     return true;
   }
+
   return false;
+
 };
 
 HeaderService.prototype._handleReorg = function() {
@@ -299,7 +342,6 @@ HeaderService.prototype._onBestHeight = function(height) {
     height + ' tip height: ' + this._tip.height);
   log.debug('Header Service: Best Height is: ' + height);
   this._bestHeight = height;
-  this._newBlocksHeight = this._bestHeight;
   this._startSync();
 };
 
@@ -357,6 +399,7 @@ HeaderService.prototype._getPersistedHeaders = function(callback) {
 
   stream.on('data', function(data) {
     var header  = self._encoding.decodeHeaderValue(data.value);
+
     // any records with a height greater than our current tip height can be scheduled for removal
     // because they will be replaced shortly
     if (header.height > self._tip.height) {
@@ -367,7 +410,6 @@ HeaderService.prototype._getPersistedHeaders = function(callback) {
       return;
     }
 
-    // hold a bit less in memory
     var newHdr = {
       hash: header.hash,
       prevHash: header.prevHash,
@@ -395,9 +437,9 @@ HeaderService.prototype._getPersistedHeaders = function(callback) {
 
 };
 
-HeaderService.prototype._getChainwork = function(header) {
+HeaderService.prototype._getChainwork = function(header, prevHeader) {
 
-  var prevChainwork = new BN(new Buffer(this._lastChainwork || HeaderService.STARTING_CHAINWORK, 'hex'));
+  var prevChainwork = new BN(new Buffer(prevHeader.chainwork, 'hex'));
 
   return this._computeChainwork(header.bits, prevChainwork);
 };

lib/utils.js

@@ -72,6 +72,10 @@ utils.SimpleMap = function SimpleMap() {
   this.size = 0;
   this.length = 0;
 
+  this.hasNullItems = function() {
+    return array.length !== _.compact(array).length;
+  };
+
   this.get = function (key) {
     return array[object[key]];
   };