From abf52f8136689f38afbbadb8dc9802eb288a52e9 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 10 Aug 2017 11:23:28 -0400 Subject: [PATCH] Fixed initial sync issue with headers. --- lib/services/header/index.js | 92 ++++++++++++++++++++++++++---------- lib/utils.js | 4 ++ 2 files changed, 71 insertions(+), 25 deletions(-) diff --git a/lib/services/header/index.js b/lib/services/header/index.js index ab5e4dfd..90d006f6 100644 --- a/lib/services/header/index.js +++ b/lib/services/header/index.js @@ -26,7 +26,6 @@ var HeaderService = function(options) { this.subscriptions.block = []; this._checkpoint = options.checkpoint || 2000; this.GENESIS_HASH = constants.BITCOIN_GENESIS_HASH[this.node.network]; - this._newBlocksHeight = 0; }; inherits(HeaderService, BaseService); @@ -93,14 +92,17 @@ HeaderService.prototype.start = function(callback) { self._db.getServiceTip(self.name, next); }, function(tip, next) { + self._tip = tip; - self._originalTip = { - hash: self._tip.hash, - height: self._tip.height - }; + log.debug('Header Service: original tip height is: ' + self._tip.height); + log.debug('Header Service: original tip hash is: ' + self._tip.hash); + + self._originalTip = Object.assign(self._tip, {}); if (self._tip.height === 0) { + assert(self._tip.hash === self.GENESIS_HASH, 'Expected tip hash to be genesis hash, but it was not.'); + var genesisHeader = { hash: self.GENESIS_HASH, height: 0, @@ -113,7 +115,7 @@ HeaderService.prototype.start = function(callback) { merkleRoot: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b' }; - self._headers.set(self.GENESIS_HASH, genesisHeader); + self._headers.set(self.GENESIS_HASH, genesisHeader, 0); self._db._store.put(self._encoding.encodeHeaderKey(0, self.GENESIS_HASH), self._encoding.encodeHeaderValue(genesisHeader), next); @@ -187,7 +189,6 @@ HeaderService.prototype._onBlock = function(block) { header = block.toHeaders().toJSON(); header.timestamp = header.ts; header.prevHash = header.prevBlock; - header.height = ++this._newBlocksHeight; this._onHeaders([header]); } @@ -203,25 +204,25 @@ HeaderService.prototype._onHeaders = function(headers) { log.debug('Header Service: Received: ' + headers.length + ' header(s).'); var dbOps = []; + var headerListLength = 0; for(var i = 0; i < headers.length; i++) { var header = headers[i]; + + // headers that come from a call to getheaders and not a new block comoing in if (header instanceof Header) { header = header.toObject(); - header.height = ++self._tip.height; + self._tip.height++; + self._tip.hash = header.hash; + headerListLength = headers.length; } - header.chainwork = self._getChainwork(header).toString(16, 64); - self._lastChainwork = header.chainwork; + var prevHeader = this._headers.get(header.prevHash); + assert(prevHeader, 'We must have a previous header in order to calculate this header\'s data.'); - self._tip.hash = header.hash; - - dbOps.push({ - type: 'put', - key: self._encoding.encodeHeaderKey(header.height, header.hash), - value: self._encoding.encodeHeaderValue(header) - }); + header.height = prevHeader.height + 1; + header.chainwork = self._getChainwork(header, prevHeader).toString(16, 64); var newHdr = { hash: header.hash, @@ -230,8 +231,15 @@ HeaderService.prototype._onHeaders = function(headers) { chainwork: header.chainwork }; + // note: this could lead to nulls in positions we don't yet have headers for + // that should be ok because eventually all the missing headers will fill in self._headers.set(header.hash, newHdr, header.height); + dbOps.push({ + type: 'put', + key: self._encoding.encodeHeaderKey(header.height, header.hash), + value: self._encoding.encodeHeaderValue(header) + }); } var tipOps = utils.encodeTip(self._tip, self.name); @@ -246,16 +254,27 @@ HeaderService.prototype._onHeaders = function(headers) { if(err) { log.error(err); - this.node.stop(); + self.node.stop(); return; } - if (self._tip.height < self._bestHeight) { + // once we've received less than 2000 headers, we know we are fully sync as far as headers + // if we reveive 2000 headers and there are no more after that, (the best height is a multiple of 2000), + // then we'll just wait for the next block to come in. This should be pretty rare occurrence + + if (headerListLength === 2000) { self._sync(); return; } - log.debug('Header Service: ' + self._headers.getIndex(self._bestHeight).hash + ' is the best block hash.'); + // at this point, we should have no empty items in this._headers, everything should be filled in + assert(!self._headers.hasNullItems(), 'Header list is not complete yet peer has sent all available headers.'); + + var bestHeader = self._headers.getLastIndex(); + self._tip.height = bestHeader.height; + self._tip.hash = bestHeader.hash; + + log.debug('Header Service: ' + bestHeader.hash + ' is the best block hash.'); // at this point, we can check our header list to see if our starting tip diverged from the tip // that we have now @@ -265,23 +284,47 @@ HeaderService.prototype._onHeaders = function(headers) { } log.debug('Header Service: emitting headers to block service.'); + // populate next hash fields on each header + self._populateNextHashes(); + self.emit('headers', self._headers); }); }; +HeaderService.prototype._populateNextHashes = function() { + + var count = 0; + + while (count < this._headers.length) { + var hdr = this._headers.getIndex(count); + var nextHdr = this._headers.getIndex(++count); + if (nextHdr) { + hdr.nextHash = nextHdr.hash; + } + } + +}; + HeaderService.prototype._detectReorg = function() { + // is our original tip's height and hash the same after we rewound by the checkpoint amount of blocks // and re-imported? If so, then we've reorg'ed since we've been shut down. - var headerHash = this._headers.getIndex(this._originalTip.height).hash; + + log.debug('Header Service: attempting to detect a reorg situation after fully syncing headers, original tip height is: ' + + this._originalTip.height + ' original tip hash: ' + this._originalTip.hash + ' headers list size: ' + this._headers.size + + ' hash at original tip height in headers list: ' + headerHash); + assert(headerHash, 'Expected a header to exist at height ' + this._originalTip.height); if (this._originalTip.hash !== headerHash) { return true; } + return false; + }; HeaderService.prototype._handleReorg = function() { @@ -299,7 +342,6 @@ HeaderService.prototype._onBestHeight = function(height) { height + ' tip height: ' + this._tip.height); log.debug('Header Service: Best Height is: ' + height); this._bestHeight = height; - this._newBlocksHeight = this._bestHeight; this._startSync(); }; @@ -357,6 +399,7 @@ HeaderService.prototype._getPersistedHeaders = function(callback) { stream.on('data', function(data) { var header = self._encoding.decodeHeaderValue(data.value); + // any records with a height greater than our current tip height can be scheduled for removal // because they will be replaced shortly if (header.height > self._tip.height) { @@ -367,7 +410,6 @@ HeaderService.prototype._getPersistedHeaders = function(callback) { return; } - // hold a bit less in memory var newHdr = { hash: header.hash, prevHash: header.prevHash, @@ -395,9 +437,9 @@ HeaderService.prototype._getPersistedHeaders = function(callback) { }; -HeaderService.prototype._getChainwork = function(header) { +HeaderService.prototype._getChainwork = function(header, prevHeader) { - var prevChainwork = new BN(new Buffer(this._lastChainwork || HeaderService.STARTING_CHAINWORK, 'hex')); + var prevChainwork = new BN(new Buffer(prevHeader.chainwork, 'hex')); return this._computeChainwork(header.bits, prevChainwork); }; diff --git a/lib/utils.js b/lib/utils.js index c2fbe274..6a3ea229 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -72,6 +72,10 @@ utils.SimpleMap = function SimpleMap() { this.size = 0; this.length = 0; + this.hasNullItems = function() { + return array.length !== _.compact(array).length; + }; + this.get = function (key) { return array[object[key]]; };