From 54d08d57bf79dff91b073306fbc2ba48fe94d089 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 5 Jul 2017 08:56:45 -0400 Subject: [PATCH] wip --- lib/scaffold/start.js | 9 +- lib/services/block/encoding.js | 46 ++- lib/services/block/index.js | 491 +++++++++++++++------------- lib/services/db/index.js | 32 +- lib/services/p2p/index.js | 8 +- scratch.txt | 512 ------------------------------ test/services/block/encoding.js | 125 +++++--- test/services/block/index.unit.js | 491 +++++++++++++++++++++------- 8 files changed, 804 insertions(+), 910 deletions(-) delete mode 100644 scratch.txt diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 7f6e77bf..6f4b71ee 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -105,8 +105,7 @@ function lookInCwd(req, service) { try { return req(process.cwd + '/' + service); } catch(e) { - log.info('Checked the current working directory, but did not find the service. ' + - 'Error caught: ' + e.message); + log.info('Checked the current working directory for service: ' + service.name); } } @@ -115,8 +114,7 @@ function lookInBuiltInPath(req, service) { var serviceFile = path.resolve(__dirname, '../services/' + service.name); return req(serviceFile); } catch(e) { - log.info('Checked the built-in path: lib/services, but did not find the service. ' + - 'Error caught: ' + e.message); + log.info('Checked the built-in path: lib/services, for service: ' + service.name); } } @@ -129,8 +127,7 @@ function lookInModuleManifest(req, service) { return req(serviceModule); } } catch(e) { - log.info('Checked the module\'s package.json for \'bitcoreNode\' field, but found no service. ' + - 'Error caught: ' + e.message); + log.info('Checked the module\'s package.json for service: ' + service.name); } } diff --git a/lib/services/block/encoding.js b/lib/services/block/encoding.js index 70c7b9ec..ec471f3d 100644 --- a/lib/services/block/encoding.js +++ b/lib/services/block/encoding.js @@ -5,15 +5,19 @@ var Block = require('bitcore-lib').Block; function Encoding(servicePrefix) { this.servicePrefix = servicePrefix; + this.blockPrefix = new Buffer('00', 'hex'); + this.hashPrefix = new Buffer('01', 'hex'); + this.heightPrefix = new Buffer('02', 'hex'); } -Encoding.prototype.encodeBlockKey = function(header) { - var headerBuf = new Buffer(JSON.stringify(header), 'utf8'); - return Buffer.concat([ this.servicePrefix, headerBuf ]); + +// ---- hash --> block +Encoding.prototype.encodeBlockKey = function(hash) { + return Buffer.concat([ this.servicePrefix, this.blockPrefix, new Buffer(hash, 'hex') ]); }; Encoding.prototype.decodeBlockKey = function(buffer) { - return JSON.parse(buffer.slice(2).toString('utf8')); + return buffer.slice(3).toString('hex'); }; Encoding.prototype.encodeBlockValue = function(block) { @@ -24,4 +28,38 @@ Encoding.prototype.decodeBlockValue = function(buffer) { return Block.fromBuffer(buffer); }; + +// ---- hash --> header +Encoding.prototype.encodeHashKey = function(hash) { + var hashBuf = new Buffer(hash, 'hex'); + return Buffer.concat([ this.servicePrefix, this.hashPrefix, hashBuf ]); +}; + +Encoding.prototype.decodeHashKey = function(buffer) { + return buffer.slice(3).toString('hex'); +}; + +Encoding.prototype.encodeHeaderValue = function(header) { + return new Buffer(JSON.stringify(header), 'utf8'); +}; + +Encoding.prototype.decodeHeaderValue = function(buffer) { + return JSON.parse(buffer.toString('utf8')); +}; + + +// ---- height --> header +Encoding.prototype.encodeHeightKey = function(height) { + + var heightBuf = new Buffer(4); + heightBuf.writeUInt32BE(height); + return Buffer.concat([ this.servicePrefix, this.heightPrefix, heightBuf ]); + +}; + +Encoding.prototype.decodeHeightKey = function(buffer) { + return buffer.readUInt32BE(3); +}; + + module.exports = Encoding; diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 0b17954a..ac54401b 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -9,10 +9,13 @@ var LRU = require('lru-cache'); var utils = require('../../utils'); var _ = require('lodash'); var assert = require('assert'); +var BN = require('bn.js'); +var consensus = require('bcoin').consensus; +var constants = require('../../constants'); var BlockService = function(options) { BaseService.call(this, options); - this.tip = null; + this._tip = null; this._p2p = this.node.services.p2p; this._db = this.node.services.db; this._subscriptions = {}; @@ -24,14 +27,17 @@ var BlockService = function(options) { length: function(n) { return n.length * (1 * 1024 * 1024); // 50 MB of blocks } - }); // header -> block - this._chainTips = LRU(50); // chain tip hash -> [ tip-prev hash, tip-prevprev hash, tip-prevprevprev hash, ... ] + }); // hash -> block + this._chainTips = []; // list of all chain tips, including main chain and any chains that were orphaned after a reorg + this._maxLookback = 100; }; inherits(BlockService, BaseService); BlockService.dependencies = [ 'p2p', 'db' ]; +BlockService.MAX_CHAINWORK = new BN(1).ushln(256); + BlockService.prototype.start = function(callback) { var self = this; @@ -43,7 +49,7 @@ BlockService.prototype.start = function(callback) { } self.prefix = prefix; - self.encoding = new Encoding(self.prefix); + self._encoding = new Encoding(self.prefix); self._setListeners(); callback(); }); @@ -51,10 +57,7 @@ BlockService.prototype.start = function(callback) { }; BlockService.prototype.stop = function(callback) { - - if (callback) { - setImmediate(callback); - } + callback(); }; BlockService.prototype.getAPIMethods = function() { @@ -100,12 +103,66 @@ BlockService.prototype.unsubscribe = function(name, emitter) { }; +BlockService.prototype._startSyncHeaders = function() { + + var numHeadersNeeded = this._bestHeight - this._tip.height; + if (numHeadersNeeded <= 0) { + return; + } + + log.info('Gathering: ' + numHeadersNeeded + ' block headers from the peer-to-peer network.'); + + this._p2pHeaderCallsNeeded = numHeadersNeeded % 2000 + 1; // we may make one extra call, but this is ok + this._latestHeaderHash = this._tip.hash || constants.BITCOIN_GENESIS_HASH[this.node.getNetworkName]; + this._syncHeaders(); +}; + + +BlockService.prototype._syncHeaders = function() { + + if (--this._p2pHeaderCallsNeeded > 0) { + this._p2p.getHeaders({ startHash: this._latestHeaderHash }); + return; + } + log,info('Header sync complete.'); + this._startSyncBlocks(); + +}; + +BlockService.prototype._startSyncBlocks = function() { + + var numHeadersNeeded = this._bestHeight - this._tip.height; + if (numHeadersNeeded <= 0) { + return; + } + + log.info('Gathering: ' + numHeadersNeeded + ' blocks from the peer-to-peer network.'); + + this._p2pBlockCallsNeeded = numHeadersNeeded % 500 + 1; // we may make one extra call, but this is ok + this._latestBlockHash = this._tip.hash || constants.BITCOIN_GENESIS_HASH[this.node.getNetworkName]; + this._syncBlocks(); + +}; + +BlockService.prototype._syncBlocks = function() { + + if (--this._p2pBlockCallsNeeded > 0) { + this._p2p.getBlocks({ startHash: this._latestBlockHash }); + return; + } + + log,info('Header sync complete.'); + + this._startSyncBlocks(); + +}; + BlockService.prototype._reportBootStatus = function() { - var blockInfoString = utils.getBlockInfoString(this.tip.height, this.bestHeight); + var blockInfoString = utils.getBlockInfoString(this._tip.height, this._bestHeight); - log.info('Block Service tip is currently height: ' + this.tip.height + ' hash: ' + - this.tip.hash + ' P2P network best height: ' + this._bestHeight + '. Block Service is: ' + + log.info('Block Service tip is currently height: ' + this._tip.height + ' hash: ' + + this._tip.hash + ' P2P network best height: ' + this._bestHeight + '. Block Service is: ' + blockInfoString); }; @@ -133,112 +190,43 @@ BlockService.prototype._onHeaders = function(headers) { log.info('New headers received.'); this._cacheHeaders(headers); + this._latestBlockHash = headers[headers.length - 1].hash; }; BlockService.prototype._blockAlreadyProcessed = function(block) { - return this._blockHeaderQueue.get(block.hash); + return this._blockHeaderQueue.get(block.hash) ? true : false; }; -/* - The block service maintains a set of chain tips. This set includes all the block chains that have - been created, including orphaned chains. - - Because blocks can be delievered to us out of order, these out of order blocks enter this collection - as new chains. We won't yet have the block's parent block. - - This creates a unique problem. Until we get a complete chain (including the out of order blocks), we - won't know for sure if a reorg has taken place in the unknown ancestor of the out of order blocks. - So, we have to defer broadcaating blocks until we have a complete chain with unsent blocks. - -*/ -BlockService.prototype._mergeBlockIntoChainTips = function(block) { +BlockService.prototype._updateChainTips = function(block, state) { var prevHash = utils.reverseBufferToString(block.header.prevHash); - var chain = this._chainTips.get(prevHash); + // otherwise we are not an orphan, so we could be a reorg or normal + // if this is a normal state, then we are just adding ourselves as the tip on the main chain + if (state === 'normal') { - // No matter what, our own parent can no longer be the tip of any chain - this._chainTips.del(prevHash); + var index = this._chainTips.indexOf(prevHash); + assert(index > -1, 'Block state is normal, ' + + 'yet the previous block hash is missing from the chain tips collection.'); - // This is the normal case where blocks are received in order. - // We could still be missing blocks from our main chain if there is not a complete - // chain between our tip and this latest block. - if (chain) { - chain.unshift(prevHash); - chain.set(block.hash, chain); + this._chainTips.push(block.hash); + this._chainTips.splice(index, 1); return; + } - // This is where we have an out of order block arriving, - // but it may fill in gaps in a chain (making that chain the active one). - // We should check for chains that have our hash listed as the last entry. - // This means our children listed us as their parent before now, but now that - // we have arrived, we know our parent's hadh and this may be an tip in another chain. - // So, we put the chains together (think of playing solitaire). - // If unificatiion is done, then this chain becomes the active one. - var chainTips = this._attemptChainUnification(block); + // if this is a reorg state, then a new chain tip will be added to the list + if (state === 'reorg') { - // if we get more than one chainTip in chainTips, then we have the case where the main chain forked - // whilst we were building orphan chains and waiting for blocks to arrive to fill in the gaps. - // This situation should be a rare event, but can happen. This function won't determine which is the most - // valid chain, but leave it up to others. + this._chainTips.push(block.hash); - // This is the out of order condition. We can't know which chain we belong to. - // Our hash was not referenced in any chain, therefore our parent wasn't either. - // Only choice to make our hash the tip of its own chain into our parent arrives. - if (chainTips.length < 1) { - this._chainTips.set(block.hash, [prevHash]); } }; -BlockService.prototype._findChainTipsWithHash = function(hash, pos) { - // we could have more than one chain that contains this hash as the entry in position (pos) - // (although this would be extremely rare). We should return all the chain tips that apply. - var chainTips = []; - this._chainTips.forEach(function(v, k) { - if (pos === 'last') { - if (v[v.length - 1] === hash) { - return chainTips.push(k); - } - } - if (v.indexOf(hash) > -1) { - chainTips.push(k); - } - }); - return chainTips; -}; - -/* - The purpose of this function is to look for the opportunity for chains to unify because - new information has arrived (a new block). - - If we have the condition where this block's parent is the tip of a chain -and- our own - block hash is the last block (oldest) in a chain, then unification is possible. - It is possible to find more than one chain where our block hash is the last entry. - In this case, each of those chains forked in blocks that came after us. -*/ -BlockService.prototype._attemptChainUnification = function(block) { - - var prevHash = utils.reverseBufferToString(block.header.prevHash); - - var possibleNewChainTips = this._findChainTipWithHash(block.hash, 'last'); - var orphanChain = this._chainTips[prevHash]; - - if (orphanChain && possibleNewChainTips.length > 0) { - for(var i = 0; i < possibleNewChainTips.length; i++) { - var newChain = this._chainTips[possibleNewChainTips[i]]; - this.chainTips[possibleNewChainTips[i]] = newChain.concat(orphanChain); - } - } - - return possibleNewChainTips; - -}; - BlockService.prototype._onBlock = function(block) { // 1. have we already seen this block? @@ -252,151 +240,148 @@ BlockService.prototype._onBlock = function(block) { // 3. store the block for safe keeping this._cacheBlock(block); - // 4. merge this block into the set of chain tips - this._mergeBlockIntoChainTips(block); - - // 5. determine block state, reorg, orphaned, normal + // 4. determine block state, reorg, orphaned, normal var blockState = this._determineBlockState(block); + // 5. add block hash to chain tips + this._updateChainTips(block); + // 6. react to state of block switch (blockState) { case 'orphaned': + // nothing to do, but wait until ancestor blocks come in break; case 'reorg': this.emit('reorg', block); break; default: - // at this point, we know we have a complete chain containing our tip and this new block - var activeChainTip = this._selectActiveChain(); - this._sendAllUnsentBlocksFromAcitveChain(activeChainTip); + // send all unsent blocks now that we have a complete chain + this._sendDelta(); break; } - }; /* Since blocks can arrive out of order from our trusted peer, we can't rely on the latest block - being the tip of the main/active chain. We should, instead, take the chain with the most work completed (the heaviest). + being the tip of the main/active chain. We should, instead, take the chain with the most work completed. We need not concern ourselves whether or not the block is valid, we trust our peer to do this validation. */ BlockService.prototype._selectActiveChain = function() { var chainTip; - var mostChainWork = 0; + var mostChainWork = new BN(0); - this._chainTips.forEach(function(v, k) { - var work = this._computeChainWork(k); - if (work > mostChainWork) { + var self = this; + + for(var i = 0; i < this._chainTips.length; i++) { + + var work = self._getChainwork(this._chainTips[i]); + + if (work.gt(mostChainWork)) { mostChainWork = work; - chainTip = k; + chainTip = this._chainTips[i]; } - }); + } return chainTip; }; -/* -arith_uint256 GetBlockProof(const CBlockIndex& block) -{ - arith_uint256 bnTarget; - bool fNegative; - bool fOverflow; - bnTarget.SetCompact(block.nBits, &fNegative, &fOverflow); - if (fNegative || fOverflow || bnTarget == 0) - return 0; - // We need to compute 2**256 / (bnTarget+1), but we can't represent 2**256 - // as it's too large for a arith_uint256. However, as 2**256 is at least as large - // as bnTarget+1, it is equal to ((2**256 - bnTarget - 1) / (bnTarget+1)) + 1, - // or ~bnTarget / (nTarget+1) + 1. - return (~bnTarget / (bnTarget + 1)) + 1; -} +BlockService.prototype._getChainwork = function(tipHash) { + var header = this._blockHeaderQueue.get(tipHash); + assert(header, 'expected to find a header in block header queue, but did not find any.'); -*/ + var prevHeader = this._blockHeaderQueue.get(header.prevHash); + assert(header, 'expected to find a previous header in block header queue, but did not find any.'); -BlockService.prototype._computeChainWork = function(chainTip) { - // pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - //for super old forks that have cycled out of our cache, just return zero work + //we persist the chainWork to avoid recalculating it on boot + var chainwork = new BN(new Buffer(header.chainwork || '00', 'hex')); - - // given the block: - // we compute the target, which is a 256 bit numnber - // but we need to check to ensure the result isn't negative or overflows or is zero - // - var blockHeader = this._blockHeaderQueue.get(chainTip); - if (!blockHeader) { - return 0; + if (chainwork.gtn(0)) { + return chainwork; } - arith_uint256 bnTarget; - bool fNegative; - bool fOverflow; - bnTarget.SetCompact(block.nBits, &fNegative, &fOverflow); - if (fNegative || fOverflow || bnTarget == 0) - return 0; - // We need to compute 2**256 / (bnTarget+1), but we can't represent 2**256 - // as it's too large for a arith_uint256. However, as 2**256 is at least as large - // as bnTarget+1, it is equal to ((2**256 - bnTarget - 1) / (bnTarget+1)) + 1, - // or ~bnTarget / (nTarget+1) + 1. - return (~bnTarget / (bnTarget + 1)) + 1; - blockHeader.chainwork; + var prevChainwork = new BN(new Buffer(prevHeader.chainwork || '00', 'hex')); + + chainwork = this._computeChainwork(header.bits, prevChainwork); + + header.chainwork = chainwork.toBuffer().toString('hex'); + this._blockHeaderQueue.set(tipHash, header); + + return chainwork; }; -BlockService.prototype._setCompact = function() { - nt nSize = nCompact >> 24; - uint32_t nWord = nCompact & 0x007fffff; - if (nSize <= 3) { - nWord >>= 8 * (3 - nSize); - *this = nWord; - } else { - *this = nWord; - *this <<= 8 * (nSize - 3); +BlockService.prototype._computeChainwork = function(bits, prev) { + + var target = consensus.fromCompact(bits); + + if (target.isNeg() || target.cmpn(0) === 0) { + return new BN(0); } - if (pfNegative) - *pfNegative = nWord != 0 && (nCompact & 0x00800000) != 0; - if (pfOverflow) - *pfOverflow = nWord != 0 && ((nSize > 34) || - (nWord > 0xff && nSize > 33) || - (nWord > 0xffff && nSize > 32)); - return *this; + + var proof = BlockService.MAX_CHAINWORK.div(target.iaddn(1)); + + if (!prev) { + return proof; + } + + return proof.iadd(prev); + }; -BlockService.prototype._getAllUnsentBlocksFromActiveChain = function(block) { +BlockService.prototype._getDelta = function(tip) { - var blocksToSend = [block]; + var blocks = []; + var _tip = tip; - if (!this._chainTips.get(block.hash)) { + while (_tip !== this._tip.hash) { + var hdr = this._blockHeaderQueue.get(_tip); + var blk = this._blockQueue.get(_tip); + _tip = hdr.prevHash; + blocks.push(blk); + } - var keys = this._chainTips.keys(); + return blocks; - for(var i = 0; i < keys.length; i++) { +}; - var key = keys[i]; - var searchChain = this._chainTips.get(key); - var index = searchChain.indexOf(block.hash); +BlockService.prototype._checkChain = function(tip) { - if (index > -1) { - var additionalBlockHashes = [key].concat(searchChain.slice(0, index)); - var additionalBlocks = this._getBlocksFromHashes(additionalBlockHashes); - blocksToSend.concat(additionalBlocks); - blocksToSend.reverse(); - break; - } + var _tip = tip; + + for(var i = 0; i < this._maxLookback; i++) { + if (_tip === this._tip.hash) { + return true; } + var prevHeader = this._blockHeaderQueue.get(_tip); + if (!prevHeader) { + return false; + } + _tip = prevHeader.prevHash; } + + return false; + }; -BlockService.prototype._sendAllUnsentBlocksFromActiveChain = function(tip) { +BlockService.prototype._sendDelta = function() { - var blocks = this._getAllUnsentBlocksFromActiveChain(tip); + // when this function is called, we know, for sure, that we have a complete chain of unsent block(s). + // our task is to send all blocks between active chain's tip and our tip. + var activeChainTip = this._selectActiveChain(); - for(var j = 0; j < blocks.length; j++) { - this._broadcast(this._subscriptions.block, 'block/block', blocks[j]); + // it is essential that the activeChainTip be in the same chain as our current tip + assert(this._checkChain(activeChainTip), 'The chain with the greatest work does not include our current tip.'); + + var blocks = this._getDelta(activeChainTip); + + for(var i = 0; i < blocks.length; i++) { + this._broadcast(this._subscriptions.block, 'block/block', blocks[i]); } - this._setTip(blocks[j-1]); + this._setTip(blocks[i-1]); }; @@ -406,15 +391,8 @@ BlockService.prototype._getBlocksFromHashes = function(hashes) { var blocks = hashes.map(function(hash) { - var hdr = self._blockHeaderQueue.get(hash); + var block = self._blockQueue.get(hash); - if (!hdr) { - log.error('header for hash: ' + hash + ' could not found in our in-memory block header cache.'); - this.node.stop(); - return; - } - - var block = self._blockQueue.get(hdr); if (!block) { log.error('block: ' + hash + ' was not found in our in-memory block cache.'); this.node.stop(); @@ -433,12 +411,12 @@ BlockService.prototype._handleReorg = function(block) { this._reorging = true; log.warn('Chain reorganization detected! Our current block tip is: ' + - this.tip.hash + ' the current block: ' + block.hash + '.'); + this._tip.hash + ' the current block: ' + block.hash + '.'); var commonAncestor = this._findCommonAncestor(block); if (!commonAncestor) { - log.error('A common ancestor block between hash: ' + this.tip.hash + ' (our current tip) and: ' + + log.error('A common ancestor block between hash: ' + this._tip.hash + ' (our current tip) and: ' + block.hash + ' (the forked block) could not be found. Bitcore-node must exit.'); this.node.stop(); return; @@ -456,7 +434,7 @@ BlockService.prototype._findCommonAncestor = function(block) { assert(this._chainTips.length > 1, 'chain tips collection should have at least 2 chains in order to find a common ancestor.'); - var oldChain = this._chainTips.get(this.tip.hash); + var oldChain = this._chainTips.get(this._tip.hash); var newChain = this._chainTips.get(block.hash); if (!newChain) { @@ -473,9 +451,9 @@ BlockService.prototype._findCommonAncestor = function(block) { }; BlockService.prototype._setTip = function(block) { - this.tip.height = block.height; - this.tip.hash = block.hash; - this._db.setServiceTip('block', this.tip); + this._tip.height = block.height; + this._tip.hash = block.hash; + this._db.setServiceTip('block', this._tip); }; BlockService.prototype._determineBlockState = function(block) { @@ -523,21 +501,26 @@ BlockService.prototype._determineBlockState = function(block) { BlockService.prototype._isOrphanBlock = function(block) { - // all blocks that we've seen before should be in the blockHeaderQueue, if this block's prev - // hash isn't, then we definitely have an orphan block. + // if any of our ancestors are missing from the queue, then this is an orphan block var prevHash = utils.reverseBufferToString(block.header.prevHash); - return !this._blockHeaderQueue.get(prevHash); + var pastBlockCount = Math.min(this._maxLookback, this._blockHeaderQueue.length); + for(var i = 0; i < pastBlockCount; i++) { + var prevHeader = this._blockHeaderQueue.get(prevHash); + if (!prevHeader) { + return true; + } + prevHash = prevHeader.prevHash; + } + + return false; }; BlockService.prototype._isChainReorganizing = function(block) { - if (!this._isOrphanBlock()) { - var prevHash = utils.reverseBufferToString(block.header.prevHash); - return prevHash !== this.tip.hash; - } + var prevHash = utils.reverseBufferToString(block.header.prevHash); - return false; + return prevHash !== this._tip.hash; }; @@ -547,23 +530,40 @@ BlockService.prototype._broadcast = function(subscribers, name, entity) { } }; +BlockService.prototype._cacheHeaders = function(headers) { + + // 1. save to in-memory cache first + this._blockHeaderQueue.set(block.hash, block.header.toObject()); + + // 2. save in db + var operations = this._getHeaderOperations(block); + + this._db.batch(operations, function(err) { + + if(err) { + log.error('There was an error attempting to save header for block hash: ' + block.hash); + this._db.emit('error', err); + return; + } + + log.debug('Success saving block hash ' + block.hash); + }); +}; + BlockService.prototype._cacheBlock = function(block) { - log.debug('Setting block: ' + block.hash + ' in the block cache.'); + log.debug('Setting block: "' + block.hash + '" in the block cache.'); // 1. set the block queue, which holds full blocks in memory this._blockQueue.set(block.hash, block); - // 2. set the block header queue, which holds hash -> header -and- height -> headeer in memory - this._blockHeader.set(block.hash, block.header.toObject()); // we don't know about the height yet - - // 3. store the block in the database + // 2. store the block in the database var operations = this._getBlockOperations(block); this._db.batch(operations, function(err) { if(err) { - log.error('There was an error attempting to save block hash: ' + block.hash); + log.error('There was an error attempting to save block for hash: ' + block.hash); this._db.emit('error', err); return; } @@ -593,6 +593,7 @@ BlockService.prototype._setBlockHeaderQueue = function(header) { }; BlockService.prototype._setListeners = function() { + var self = this; self._p2p.once('bestHeight', function(height) { @@ -600,11 +601,26 @@ BlockService.prototype._setListeners = function() { self._bestHeight = height; // once we have best height, we know the p2p network is ready to go - self.once('tip-block', function(tip) { - self._tip = tip; + self._db.once('tip-block', function(tip) { + + if (tip) { + + self._tip = tip; + + } else { + + self._tip = { + height: 0, + hash: constants.BITCOIN_GENESIS_HASH[self.node.getNetworkName()] + }; + + } self._startSubscriptions(); + self._startSyncHeaders(); + }); + self._loadTip(); }); @@ -624,32 +640,61 @@ BlockService.prototype._onDbError = function(err) { this.node.stop(); }; -BlockService.prototype._getBlockOperations = function(obj) { +BlockService.prototype._getHeaderOperations = function(block) { var self = this; - if (_.isArray(obj)) { + // block or [block] + if (_.isArray(block)) { var ops = []; - _.forEach(obj, function(block) { - ops.push(self._getBlockOperations(block)); + _.forEach(block, function(b) { + ops.push(self._getBlockOperations(b)); }); return _.flatten(ops); } var operations = []; + // hash operations.push({ type: 'put', - key: self.encoding.encodeBlockHashKey(obj.hash), - value: self.encoding.encodeBlockHeightValue(obj.height) + key: self._encoding.encodeHashKey(block.hash), + value: self._encoding.encodeHeaderValue(block.header) }); + // height operations.push({ type: 'put', - key: self.encoding.encodeBlockHeightKey(obj.height), - value: self.encoding.encodeBlockHashValue(obj.hash) + key: self._encoding.encodeHeightKey(block.height), + value: self._encoding.encodeHeaderValue(block.header) }); + return operations; +}; + +BlockService.prototype._getBlockOperations = function(block) { + + var self = this; + + // block or [block] + if (_.isArray(block)) { + var ops = []; + _.forEach(block, function(b) { + ops.push(self._getBlockOperations(b)); + }); + return _.flatten(ops); + } + + var operations = []; + + // block + operations.push({ + type: 'put', + key: self._encoding.encodeBlockKey(block.hash), + value: self._encoding.encodeBlockValue(block) + }); + + return operations; }; diff --git a/lib/services/db/index.js b/lib/services/db/index.js index c5395c25..77efa725 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -212,29 +212,41 @@ DB.prototype.getPublishEvents = function() { return []; }; -DB.prototype.getServiceTip = function(service) { - var keyBuf = Buffer.concat([ this.dbPrefix, new Buffer('tip-', 'utf8'), new Buffer(service, 'utf8') ]); - this.get(keyBuf, function(err, tipBuf) { +DB.prototype.getServiceTip = function(serviceName) { + + var keyBuf = Buffer.concat([ this.dbPrefix, new Buffer('tip-' + serviceName, 'utf8') ]); + + var self = this; + self.get(keyBuf, function(err, tipBuf) { if (err) { this.emit('error', err); return; } - var heightBuf = new Buffer(4); - var tip = { - height: heightBuf.readUInt32BE(), - hash: tipBuf.slice(4) + + var tip = null; + if (tipBuf) { + var heightBuf = new Buffer(4); + tip = { + height: heightBuf.readUInt32BE(), + hash: tipBuf.slice(4) + }; } - this.emit('tip-' + service, tip); + + self.emit('tip-' + serviceName, tip); }); + }; -DB.prototype.setServiceTip = function(service, tip) { - var keyBuf = Buffer.concat([ this.dbPrefix, new Buffer('tip-', 'utf8'), new Buffer(service, 'utf8') ]); +DB.prototype.setServiceTip = function(serviceName, tip) { + + var keyBuf = Buffer.concat([ this.dbPrefix, new Buffer('tip-' + serviceName, 'utf8') ]); var heightBuf = new Buffer(4); var valBuf = Buffer.concat([ heightBuf.writeUInt32BE(tip.height), new Buffer(tip.hash, 'hex') ]); + this.put(keyBuf, valBuf, function(err) { this.emit('error', err); }); + }; DB.prototype.getPrefix = function(service, callback) { diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js index 4da99c6e..dfc0fa8f 100644 --- a/lib/services/p2p/index.js +++ b/lib/services/p2p/index.js @@ -21,7 +21,7 @@ var P2P = function(options) { this._startBcoinIfNecessary(); this._initP2P(); this._initPubSub(); - this._bcoin; + this._bcoin = null; }; @@ -37,7 +37,7 @@ P2P.prototype.getAPIMethods = function() { var methods = [ ['getHeaders', this, this.getHeaders, 1], ['getMempool', this, this.getMempool, 0], - ['getBlocks', this, this.getBlocks, 1] + ['getBlocks', this, this.getBlocks, 1], ['clearInventoryCache', this, this.clearInventoryCache, 0] ]; return methods; @@ -151,7 +151,7 @@ P2P.prototype._broadcast = function(subscribers, name, entity) { P2P.prototype._connect = function() { this._connectCalled = this._connectCalled > 0 ? 2 : 1; - if (this._connectCalled > 1) { + if (this._connectCalled > 1 || !this._bcoin) { log.info('Connecting to p2p network.'); this._pool.connect(); } @@ -179,7 +179,7 @@ P2P.prototype._getPeer = function() { P2P.prototype._hasPeers = function() { return this._options && this._options.peers && - this._options.peers.host; + this._options.peers.length > 0; }; P2P.prototype._initCache = function() { diff --git a/scratch.txt b/scratch.txt deleted file mode 100644 index 90c1e30d..00000000 --- a/scratch.txt +++ /dev/null @@ -1,512 +0,0 @@ -AddressService.prototype.concurrentBlockHandler = function(block, connectBlock, callback) { - var self = this; - - var txs = block.transactions; - var height = block.__height; - - var action = 'put'; - var reverseAction = 'del'; - if (!connectBlock) { - action = 'del'; - reverseAction = 'put'; - } - - var operations = []; - - for(var i = 0; i < txs.length; i++) { - - var tx = txs[i]; - var txid = tx.id; - var inputs = tx.inputs; - var outputs = tx.outputs; - - // Subscription messages - var txmessages = {}; - - var outputLength = outputs.length; - for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { - var output = outputs[outputIndex]; - - var script = output.script; - - if(!script) { - log.debug('Invalid script'); - continue; - } - - var address = self.getAddressString(script); - if(!address) { - continue; - } - - var key = self._encoding.encodeAddressIndexKey(address, height, txid); - operations.push({ - type: action, - key: key - }); - - // Collect data for subscribers - if (txmessages[address]) { - txmessages[address].outputIndexes.push(outputIndex); - } else { - txmessages[address] = { - tx: tx, - height: height, - outputIndexes: [outputIndex], - address: address, - timestamp: block.header.timestamp - }; - } - } - - if(tx.isCoinbase()) { - continue; - } - - //TODO deal with P2PK - for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { - var input = inputs[inputIndex]; - - if(!input.script) { - log.debug('Invalid script'); - continue; - } - - var inputAddress = self.getAddressString(input.script); - - if(!inputAddress) { - continue; - } - - var inputKey = self._encoding.encodeAddressIndexKey(inputAddress, height, txid); - - operations.push({ - type: action, - key: inputKey - }); - - } - } - setImmediate(function() { - callback(null, operations); - }); -}; - -AddressService.prototype.blockHandler = function(block, connectBlock, callback) { - var self = this; - - var txs = block.transactions; - - var action = 'put'; - var reverseAction = 'del'; - if (!connectBlock) { - action = 'del'; - reverseAction = 'put'; - } - - var operations = []; - - async.eachSeries(txs, function(tx, next) { - var txid = tx.id; - var inputs = tx.inputs; - var outputs = tx.outputs; - - var outputLength = outputs.length; - for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { - var output = outputs[outputIndex]; - - var script = output.script; - - if(!script) { - log.debug('Invalid script'); - continue; - } - - var address = self.getAddressString(script); - - if(!address) { - continue; - } - - var key = self._encoding.encodeUtxoIndexKey(address, txid, outputIndex); - var value = self._encoding.encodeUtxoIndexValue(block.__height, output.satoshis, output._scriptBuffer); - operations.push({ - type: action, - key: key, - value: value - }); - - } - - if(tx.isCoinbase()) { - return next(); - } - - //TODO deal with P2PK - async.each(inputs, function(input, next) { - if(!input.script) { - log.debug('Invalid script'); - return next(); - } - - var inputAddress = self.getAddressString(input.script); - - if(!inputAddress) { - return next(); - } - - var inputKey = self._encoding.encodeUtxoIndexKey(inputAddress, input.prevTxId, input.outputIndex); - //common case is connecting blocks and deleting outputs spent by these inputs - if (connectBlock) { - operations.push({ - type: 'del', - key: inputKey - }); - next(); - } else { // uncommon and slower, this happens during a reorg - self.node.services.transaction.getTransaction(input.prevTxId.toString('hex'), {}, function(err, tx) { - var utxo = tx.outputs[input.outputIndex]; - var inputValue = self._encoding.encodeUtxoIndexValue(tx.__height, utxo.satoshis, utxo._scriptBuffer); - operations.push({ - type: 'put', - key: inputKey, - value: inputValue - }); - next(); - }); - } - }, function(err) { - if(err) { - return next(err); - } - next(); - }); - }, function(err) { - //we are aync predicated on reorg sitch - if(err) { - return callback(err); - } - callback(null, operations); - }); -}; -TransactionService.prototype.blockHandler = function(block, connectBlock, callback) { - var self = this; - var action = 'put'; - var reverseAction = 'del'; - if (!connectBlock) { - action = 'del'; - reverseAction = 'put'; - } - - var operations = []; - - this.currentTransactions = {}; - - async.series([ - function(next) { - self.node.services.timestamp.getTimestamp(block.hash, function(err, timestamp) { - if(err) { - return next(err); - } - block.__timestamp = timestamp; - next(); - }); - }, function(next) { - async.eachSeries(block.transactions, function(tx, next) { - tx.__timestamp = block.__timestamp; - tx.__height = block.__height; - - self._getInputValues(tx, function(err, inputValues) { - if(err) { - return next(err); - } - tx.__inputValues = inputValues; - self.currentTransactions[tx.id] = tx; - - operations.push({ - type: action, - key: self.encoding.encodeTransactionKey(tx.id), - value: self.encoding.encodeTransactionValue(tx) - }); - next(); - }); - }, function(err) { - if(err) { - return next(err); - } - next(); - }); - }], function(err) { - if(err) { - return callback(err); - } - callback(null, operations); - }); - -}; -MempoolService.prototype.blockHandler = function(block, connectBlock, callback) { - var self = this; - - if (!self._handleBlocks) { - return setImmediate(callback); - } - - var txs = block.transactions; - - var action = 'del'; - if (!connectBlock) { - action = 'put'; - } - - for(var i = 0; i < txs.length; i++) { - - var tx = txs[i]; - self._updateMempool(tx, action); - } - setImmediate(callback); -}; -WalletService.prototype.blockHandler = function(block, connectBlock, callback) { - - var opts = { - block: block, - connectBlock: connectBlock, - serial: true - }; - this._blockHandler(opts, callback); - -}; - -WalletService.prototype.concurrentBlockHandler = function(block, connectBlock, callback) { - - var opts = { - block: block, - connectBlock: connectBlock - }; - this._blockHandler(opts, callback); - -}; - -WalletService.prototype._blockHandler = function(opts, callback) { - - var self = this; - - if (!self._checkAddresses()) { - return setImmediate(function() { - callback(null, []); - }); - } - - async.mapSeries(opts.block.transactions, function(tx, next) { - - self._processTransaction(opts, tx, next); - - }, function(err, operations) { - - if(err) { - return callback(err); - } - - var ret = _.compact(_.flattenDeep(operations)); - callback(null, ret); - - }); - -}; - -WalletService.prototype._processTransaction = function(opts, tx, callback) { - var self = this; - - tx.outputs.forEach(function(output, index) { - output.index = index; - }); - - var ioData = tx.inputs.concat(tx.outputs); - - async.mapSeries(ioData, function(io, next) { - if (opts.serial) { - self._processSerialIO(opts, tx, io, next); - } else { - self._processConcurrentIO(opts, tx, io, next); - } - }, function(err, operations) { - if(err) { - return callback(err); - } - callback(null, operations); - }); - -}; - -WalletService.prototype._processConcurrentIO = function(opts, tx, io, callback) { - - var self = this; - var walletIds = self._getWalletIdsFromScript(io); - - if (!walletIds) { - return callback(); - } - var actions = self._getActions(opts.connectBlock); - - var operations = walletIds.map(function(walletId) { - return { - type: actions[0], - key: self._encoding.encodeWalletTransactionKey(walletId, opts.block.__height, tx.id) - }; - }); - - setImmediate(function() { - callback(null, operations); - }); - -}; - -WalletService.prototype._processSerialIO = function(opts, tx, io, callback) { - var fn = this._processSerialOutput; - if (io instanceof Input) { - fn = this._processSerialInput; - } - fn.call(this, opts, tx, io, callback); -}; - -WalletService.prototype._getWalletIdsFromScript = function(io) { - - if(!io.script) { - log.debug('Invalid script'); - return; - } - - return this._addressMap[this.getAddressString(io)]; - -}; - -WalletService.prototype._getActions = function(connect) { - var action = 'put'; - var reverseAction = 'del'; - if (!connect) { - action = 'del'; - reverseAction = 'put'; - } - return [action, reverseAction]; -}; - -WalletService.prototype._processSerialOutput = function(opts, tx, output, callback) { - - var self = this; - var walletIds = self._getWalletIdsFromScript(output); - - if (!walletIds) { - return callback(); - } - - var actions = self._getActions(opts.connectBlock); - - async.mapSeries(walletIds, function(walletId, next) { - - self.balances[walletId] = self.balances[walletId] || 0; - self.balances[walletId] += opts.connectBlock ? output.satoshis : (-1 * output.satoshis); - - var operations = [ - { - type: actions[0], - key: self._encoding.encodeWalletUtxoKey(walletId, tx.id, output.index), - value: self._encoding.encodeWalletUtxoValue(opts.block.__height, output.satoshis, output._scriptBuffer) - }, - { - type: actions[0], - key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, output.satoshis, tx.id, output.index), - value: self._encoding.encodeWalletUtxoSatoshisValue(opts.block.__height, output._scriptBuffer) - }, - { - type: 'put', - key: self._encoding.encodeWalletBalanceKey(walletId), - value: self._encoding.encodeWalletBalanceValue(self.balances[walletId]) - } - ]; - - next(null, operations); - - }, function(err, operations) { - - if(err) { - return callback(err); - } - - callback(null, operations); - - }); - -}; - -WalletService.prototype._processSerialInput = function(opts, tx, input, callback) { - - var self = this; - - var walletIds = input.script && input.script.isPublicKeyIn() ? - ['p2pk'] : - self._getWalletIdsFromScript(input); - - if (!walletIds) { - return callback(); - } - - var actions = self._getActions(opts.connectBlock); - - async.mapSeries(walletIds, function(walletId, next) { - - self.node.services.transaction.getTransaction(input.prevTxId.toString('hex'), {}, function(err, tx) { - - if(err) { - return next(err); - } - - var utxo = tx.outputs[input.outputIndex]; - - if (walletId === 'p2pk') { - - var pubKey = utxo.script.getPublicKey().toString('hex'); - walletId = self._addressMap[pubKey]; - - if (!walletId) { - return next(null, []); - } - - } - - self.balances[walletId] = self.balances[walletId] || 0; - self.balances[walletId] += opts.connectBlock ? (-1 * utxo.satoshis) : utxo.satoshis; - - var operations = [ - { - type: actions[1], - key: self._encoding.encodeWalletUtxoKey(walletId, input.prevTxId, input.outputIndex), - value: self._encoding.encodeWalletUtxoValue(tx.__height, utxo.satoshis, utxo._scriptBuffer) - }, - { - type: actions[1], - key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, utxo.satoshis, tx.id, input.outputIndex), - value: self._encoding.encodeWalletUtxoSatoshisValue(tx.__height, utxo._scriptBuffer) - }, - { - type: 'put', - key: self._encoding.encodeWalletBalanceKey(walletId), - value: self._encoding.encodeWalletBalanceValue(self.balances[walletId]) - } - ]; - - next(null, operations); - - }); - }, function(err, operations) { - - if(err) { - return callback(err); - } - - callback(null, operations); - - }); - -}; - -WalletService.prototype._loadAllAddresses = function(callback) { - diff --git a/test/services/block/encoding.js b/test/services/block/encoding.js index 36a47356..108b2c80 100644 --- a/test/services/block/encoding.js +++ b/test/services/block/encoding.js @@ -1,63 +1,100 @@ 'use strict'; var should = require('chai').should(); +var Block = require('bitcore-lib').Block; var Encoding = require('../../../lib/services/block/encoding'); -describe('Wallet-Api service encoding', function() { +describe('Block service encoding', function() { var servicePrefix = new Buffer('0000', 'hex'); + + var blockPrefix = new Buffer('00', 'hex'); + var hashPrefix = new Buffer('01', 'hex'); + var heightPrefix = new Buffer('02', 'hex'); + var encoding = new Encoding(servicePrefix); - var block = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add'; + var hash = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add'; var height = 1; + var header = { hash: hash, height: height }; + var block = new Block(new Buffer('0100000095194b8567fe2e8bbda931afd01a7acd399b9325cb54683e64129bcd00000000660802c98f18fd34fd16d61c63cf447568370124ac5f3be626c2e1c3c9f0052d19a76949ffff001d33f3c25d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d014dffffffff0100f2052a01000000434104e70a02f5af48a1989bf630d92523c9d14c45c75f7d1b998e962bff6ff9995fc5bdb44f1793b37495d80324acba7c8f537caaf8432b8d47987313060cc82d8a93ac00000000', 'hex')); + + describe('Block', function() { + + it('should encode block key' , function() { + encoding.encodeBlockKey(hash).should.deep.equal(Buffer.concat([ + servicePrefix, + blockPrefix, + new Buffer(hash, 'hex') + ])); + }); + + it('should decode block key' , function() { + var buf = Buffer.concat([ + servicePrefix, + blockPrefix, + new Buffer(hash, 'hex') + ]); + + var actual = encoding.decodeBlockKey(buf); + actual.should.deep.equal(hash); + }); + + it('should encode block value', function() { + encoding.encodeBlockValue(block).should.deep.equal( + block.toBuffer()); + }); + + it('shound decode block value', function() { + var ret = encoding.decodeBlockValue(block.toBuffer()); + ret.should.deep.equal(ret); + }); - it('should encode block hash key' , function() { - encoding.encodeBlockHashKey(block).should.deep.equal(Buffer.concat([ - servicePrefix, - new Buffer('00', 'hex'), - new Buffer(block, 'hex') - ])); }); - it('should decode block hash key' , function() { - encoding.decodeBlockHashKey(Buffer.concat([ - servicePrefix, - new Buffer('00', 'hex'), - new Buffer(block, 'hex') - ])).should.deep.equal(block); + describe('Hash', function() { + it('should encode hash key', function() { + encoding.encodeHashKey(hash).should.deep.equal(Buffer.concat([ + servicePrefix, + hashPrefix, + new Buffer(hash, 'hex') + ])); + }); + + it('should decode hash key', function() { + encoding.decodeHashKey(Buffer.concat([ + servicePrefix, + hashPrefix, + new Buffer(hash, 'hex') + ])).should.deep.equal(hash); + }); + + it('should encode header value', function() { + encoding.encodeHeaderValue(header).should.deep.equal(new Buffer(JSON.stringify(header), 'utf8')); + }); + + it('should decode hash value', function() { + encoding.decodeHeaderValue(new Buffer(JSON.stringify(header), 'utf8')).should.deep.equal(header); + }); }); - it('should encode block hash value', function() { - encoding.encodeBlockHashValue(block).should.deep.equal( - new Buffer(block, 'hex')); + describe('Height', function() { + it('should encode height key', function() { + encoding.encodeHeightKey(height).should.deep.equal(Buffer.concat([ + servicePrefix, + heightPrefix, + new Buffer('00000001', 'hex') + ])); + }); + + it('should decode height key', function() { + encoding.decodeHeightKey(Buffer.concat([ + servicePrefix, + heightPrefix, + new Buffer('00000001', 'hex') + ])).should.deep.equal(height); + }); }); - it('shound decode block hash value', function() { - encoding.decodeBlockHashValue(new Buffer(block, 'hex')).should.deep.equal(block); - }); - - it('should encode block height key', function() { - encoding.encodeBlockHeightKey(height).should.deep.equal(Buffer.concat([ - servicePrefix, - new Buffer('01', 'hex'), - new Buffer('00000001', 'hex') - ])); - }); - - it('should decode block height key', function() { - encoding.decodeBlockHeightKey(Buffer.concat([ - servicePrefix, - new Buffer('01', 'hex'), - new Buffer('00000001', 'hex') - ])).should.deep.equal(height); - }); - - it('should encode block height value', function() { - encoding.encodeBlockHeightValue(height).should.deep.equal(new Buffer('00000001', 'hex')); - }); - - it('should decode block height value', function() { - encoding.decodeBlockHeightValue(new Buffer('00000001', 'hex')).should.deep.equal(height); - }); }); diff --git a/test/services/block/index.unit.js b/test/services/block/index.unit.js index dbdf8ca2..e82a5b54 100644 --- a/test/services/block/index.unit.js +++ b/test/services/block/index.unit.js @@ -2,6 +2,13 @@ var expect = require('chai').expect; var BlockService = require('../../../lib/services/block'); +var BN = require('bn.js'); +var assert = require('chai').assert; +var crypto = require('crypto'); +var sinon = require('sinon'); +var Block = require('bitcore-lib').Block; +var Encoding = require('../../../lib/services/block/encoding'); +var EventEmitter = require('events').EventEmitter; var LRU = require('lru-cache'); describe('Block Service', function() { @@ -10,12 +17,24 @@ describe('Block Service', function() { beforeEach(function() { blockService = new BlockService({ node: { services: []}}); - blockService._chainTips = LRU(50); + blockService._chainTips = ['00']; + blockService._encoding = new Encoding(new Buffer('0000', 'hex')); }); + describe('#_blockAlreadyProcessed', function() { - describe('Chain Tips', function() { - it('should merge blocks into chain tips using active chain blocks only' , function() { + it('should detect that a block has already been delivered to us', function() { + blockService._blockHeaderQueue.set('aa', {}); + expect(blockService._blockAlreadyProcessed({ hash: 'aa' })).to.be.true; + expect(blockService._blockAlreadyProcessed('bb')).to.be.false; + blockService._blockHeaderQueue.reset(); + }); + + }); + + describe('#_updateChainTips', function() { + + it('should set chain tips under normal block arrival conditions, in order arrival' , function() { var blocks = ['aa','bb','cc','dd','ee']; @@ -27,146 +46,404 @@ describe('Block Service', function() { } var block = { header: { prevHash: buf }, hash: n }; - blockService._mergeBlockIntoChainTips(block); + blockService._updateChainTips(block, 'normal'); }); expect(blockService._chainTips.length).to.equal(1); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'cc', 'bb', 'aa', '00']); + expect(blockService._chainTips).to.deep.equal(['ee']); }); - it('should merge blocks into chain tips using out of order blocks' , function() { + it('should not set chain tips if not in normal or reorg state' , function() { - var blocks = ['ee','aa','bb','dd','cc']; - var prevBlocks = ['dd','00','aa','cc','bb']; + var block = { header: { prevHash: new Buffer('aa', 'hex') }}; + blockService._updateChainTips(block, 'orphan'); + expect(blockService._chainTips).to.deep.equal(['00']); - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - - - expect(blockService._chainTips.length).to.equal(1); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'cc', 'bb', 'aa', '00']); }); - it('should merge blocks where there is a fork (a parent block has more than one child)' , function() { + it('should set chain tips when there is a reorg taking place' , function() { - var blocks = ['aa','bb','cc','dd','ee']; - var prevBlocks = ['00','aa','aa','cc','dd']; + var block = { hash: 'ee', header: { prevHash: 'dd' } }; + blockService._updateChainTips(block, 'reorg'); + expect(blockService._chainTips).to.deep.equal(['00', 'ee']); - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); + }); + }); - expect(blockService._chainTips.length).to.equal(2); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'cc', 'aa', '00']); - expect(blockService._chainTips.get('bb')).to.deep.equal(['aa', '00']); + describe('#_isOrphanBlock', function() { + + beforeEach(function() { + var prevHash = '00000000'; + for(var i = 0; i < 110; i++) { + var newHash = crypto.randomBytes(4); + blockService._blockHeaderQueue.set(newHash, { prevHash: new Buffer(prevHash, 'hex') }); + prevHash = newHash; + } }); - it('should merge blocks where there is a fork (a parent block has more than one child) and blocks are received out of order' , function() { - - var blocks = ['cc','aa','bb','ee','dd']; - var prevBlocks = ['aa','00','aa','dd','cc']; - - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - - - expect(blockService._chainTips.length).to.equal(2); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'cc', 'aa', '00']); - expect(blockService._chainTips.get('bb')).to.deep.equal(['aa', '00']); + it('should detect an orphaned block', function() { + var block = { hash: 'ee', header: { prevHash: new Buffer('aa', 'hex') }}; + expect(blockService._isOrphanBlock(block)).to.be.true; }); - it('should merge blocks where there is a three-way fork (a parent block has more than one child) and blocks are received out of order' , function() { - - var blocks = ['cc','aa','bb','ee','dd']; - var prevBlocks = ['aa','00','aa','dd','aa']; - - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - - - expect(blockService._chainTips.length).to.equal(3); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'aa', '00']); - expect(blockService._chainTips.get('bb')).to.deep.equal(['aa', '00']); - expect(blockService._chainTips.get('cc')).to.deep.equal(['aa', '00']); + it('should not detect an orphaned block', function() { + var block = { hash: 'new', header: { prevHash: '00' }}; + expect(blockService._isOrphanBlock(block)).to.be.true; }); }); - it('shoudd merge blocks where there is three-way fork and blocks are received in order.', function() { + describe('#_getChainwork', function() { - var blocks = ['aa','bb','cc','dd','ee']; - var prevBlocks = ['00','aa','aa','aa','dd']; - - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - - - expect(blockService._chainTips.length).to.equal(3); - expect(blockService._chainTips.get('ee')).to.deep.equal(['dd', 'aa', '00']); - expect(blockService._chainTips.get('bb')).to.deep.equal(['aa', '00']); - expect(blockService._chainTips.get('cc')).to.deep.equal(['aa', '00']); - }); - - describe('Reorgs', function() { - - it('should find a common ancestor in the normal case', function() { - - var blocks = ['aa', 'bb', 'cc', 'dd']; - var prevBlocks = ['00', 'aa', 'bb', 'bb']; - - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - - blockService.tip = { hash: 'cc', height: 3 }; - var commonAncestor = blockService._findCommonAncestor({ hash: 'dd' }); - expect(commonAncestor).to.equal('bb'); + it('should get chainwork, chainwork already on header', function() { + var expected = new BN(new Buffer('000000000000000000000000000000000000000000677c7b8122f9902c79f4e0', 'hex')); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', chainwork: '000000000000000000000000000000000000000000677c7b8122f9902c79f4e0'}); + blockService._blockHeaderQueue.set('aa', { prevHash: '00', chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); + var actual = blockService._getChainwork('bb'); + assert(actual.eq(expected), 'not equal: actual: ' + actual + ' expected: ' + expected); }); - it('should find a common ancestor in the case where more than one block is built on an alternative chain before reorg is discovered', function() { + it('should get chainwork, chainwork not already on header', function() { + var expected = new BN(new Buffer('000000000000000000000000000000000000000000677c7b8122f9902c79f4e0', 'hex')); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', bits: 0x18018d30 }); + blockService._blockHeaderQueue.set('aa', { prevHash: '00', chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); + var actual = blockService._getChainwork('bb'); + assert(actual.eq(expected), 'not equal: actual: ' + actual + ' expected: ' + expected); - // even though 'ee' is the tip on the alt chain, 'dd' was the last block to come in that was not an orphan block. So 'dd' was the first to allow - // reorg validation - var blocks = ['aa', 'bb', 'cc', 'ee', 'dd']; - var prevBlocks = ['00', 'aa', 'bb', 'dd', 'bb']; + }); + }); - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); - blockService.tip = { hash: 'cc', height: 3 }; - var commonAncestor = blockService._findCommonAncestor({ hash: 'dd' }); - expect(commonAncestor).to.equal('bb'); + describe('#_computeChainwork', function() { + + it('should calculate chain work correctly', function() { + var expected = new BN(new Buffer('000000000000000000000000000000000000000000677c7b8122f9902c79f4e0', 'hex')); + var prev = new BN(new Buffer('000000000000000000000000000000000000000000677bd68118a98f8779ea90', 'hex')); + + var actual = blockService._computeChainwork(0x18018d30, prev); + assert(actual.eq(expected), 'not equal: actual: ' + actual + ' expected: ' + expected); + }); + + }); + + describe('#_selectActiveChain', function() { + + it('should select active chain based on most chain work', function() { + blockService._blockHeaderQueue.set('cc', { prevHash: '00', bits: 0x18018d30 }); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', bits: 0x18018d30 }); + blockService._blockHeaderQueue.set('aa', { chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); + blockService._blockHeaderQueue.set('00', { chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea8f'}); + blockService._chainTips.push('bb'); + blockService._chainTips.push('cc'); + + var expected = 'bb'; + var actual = blockService._selectActiveChain(); + expect(actual).to.equal(expected); + }); + + }); + + describe('#_cacheBlock', function() { + + it('should set the block in the block queue and db', function() { + var sandbox = sinon.sandbox.create(); + var spy1 = sandbox.spy(); + var stub1 = sandbox.stub(); + blockService._blockQueue = { set: stub1 }; + var block = {}; + sandbox.stub(blockService, '_getBlockOperations'); + blockService._db = { batch: spy1 }; + blockService._cacheBlock(block); + expect(spy1.calledOnce).to.be.true; + expect(stub1.calledOnce).to.be.true; + sandbox.restore(); + }); + + }); + + describe('#_isChainReorganizing', function() { + + it('should decide that chain is reorging', function() { + blockService._tip = { hash: 'aa' }; + var block = { header: { prevHash: new Buffer('00', 'hex') }}; + expect(blockService._isChainReorganizing(block)).to.be.true; + }); + + it('should decide that chain is not reorging', function() { + blockService._tip = { hash: 'aa' }; + var block = { header: { prevHash: new Buffer('aa', 'hex') }}; + expect(blockService._isChainReorganizing(block)).to.be.false; + }); + + }); + + describe('#_getBlockOperations', function() { + + it('should get block operations when given one block', function() { + var block = new Block(new Buffer('0100000095194b8567fe2e8bbda931afd01a7acd399b9325cb54683e64129bcd00000000660802c98f18fd34fd16d61c63cf447568370124ac5f3be626c2e1c3c9f0052d19a76949ffff001d33f3c25d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d014dffffffff0100f2052a01000000434104e70a02f5af48a1989bf630d92523c9d14c45c75f7d1b998e962bff6ff9995fc5bdb44f1793b37495d80324acba7c8f537caaf8432b8d47987313060cc82d8a93ac00000000', 'hex')); + var ops = blockService._getBlockOperations(block); + + expect(ops[0]).to.deep.equal({ type: 'put', key: blockService._encoding.encodeBlockKey(block.hash), value: blockService._encoding.encodeBlockValue(block) }); + + }); + + it('should get block operations when given more than one block', function() { + var block = new Block(new Buffer('0100000095194b8567fe2e8bbda931afd01a7acd399b9325cb54683e64129bcd00000000660802c98f18fd34fd16d61c63cf447568370124ac5f3be626c2e1c3c9f0052d19a76949ffff001d33f3c25d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d014dffffffff0100f2052a01000000434104e70a02f5af48a1989bf630d92523c9d14c45c75f7d1b998e962bff6ff9995fc5bdb44f1793b37495d80324acba7c8f537caaf8432b8d47987313060cc82d8a93ac00000000', 'hex')); + var ops = blockService._getBlockOperations([block, block]); + + expect(ops[0]).to.deep.equal({ type: 'put', key: blockService._encoding.encodeBlockKey(block.hash), value: blockService._encoding.encodeBlockValue(block) }); + expect(ops[1]).to.deep.equal({ type: 'put', key: blockService._encoding.encodeBlockKey(block.hash), value: blockService._encoding.encodeBlockValue(block) }); }); }); - describe('Send all unsent blocks from main chain', function() { + describe('#_onBlock', function() { - it('should send all unsent blocks from the active/main chain', function() { - var blocks = ['ee','aa','bb','dd','cc']; - var prevBlocks = ['dd','00','aa','cc','bb']; + it('should perform all the steps for onBlock handler (normal)', function() { + + var sandbox = sinon.sandbox.create(); + var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); + var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); + var blockState = sandbox.stub(blockService, '_determineBlockState').returns('normal'); + var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + var sendAllUnsent = sandbox.stub(blockService, '_sendDelta'); + + blockService._onBlock({ hash: 'aa' }); + expect(alreadyProcessed.callCount).to.equal(1); + expect(cacheBlock.callCount).to.equal(1); + expect(blockState.callCount).to.equal(1); + expect(updateChainTips.callCount).to.equal(1); + expect(sendAllUnsent.callCount).to.equal(1); + + sandbox.restore(); - blocks.forEach(function(n, index) { - var block = { header: { prevHash: new Buffer(prevBlocks[index], 'hex') }, hash: n }; - blockService._mergeBlockIntoChainTips(block); - }); }); + it('should perform all the steps for onBlock handler (reorg)', function() { + + var sandbox = sinon.sandbox.create(); + var block = { hash: 'aa' }; + var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); + var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); + var blockState = sandbox.stub(blockService, '_determineBlockState').returns('reorg'); + var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + + var reorgListener = blockService.on('reorg', function(block) { + expect(block).to.equal(block); + }); + + blockService._onBlock(block); + expect(alreadyProcessed.callCount).to.equal(1); + expect(cacheBlock.callCount).to.equal(1); + expect(blockState.callCount).to.equal(1); + expect(updateChainTips.callCount).to.equal(1); + + sandbox.restore(); + + }); + + it('should perform all the steps for onBlock handler (orphaned)', function() { + + var sandbox = sinon.sandbox.create(); + var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); + var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); + var blockState = sandbox.stub(blockService, '_determineBlockState').returns('orphaned'); + var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + + blockService._onBlock({ hash: 'aa' }); + expect(alreadyProcessed.callCount).to.equal(1); + expect(cacheBlock.callCount).to.equal(1); + expect(blockState.callCount).to.equal(1); + expect(updateChainTips.callCount).to.equal(1); + + sandbox.restore(); + + }); + }); + + describe('#_determineBlockState', function() { + + it('should determine the block in a normal state', function() { + var sandbox = sinon.sandbox.create(); + var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(false); + var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(false); + expect(blockService._determineBlockState({})).to.equal('normal'); + sandbox.restore(); + }); + + it('should determine the block in a orphan state', function() { + var sandbox = sinon.sandbox.create(); + var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(false); + var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(true); + expect(blockService._determineBlockState({})).to.equal('orphaned'); + sandbox.restore(); + }); + + it('should determine the block in a reorg state', function() { + var sandbox = sinon.sandbox.create(); + var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(true); + var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(false); + expect(blockService._determineBlockState({})).to.equal('reorg'); + sandbox.restore(); + }); + + }); + + describe('#start', function() { + + var sandbox; + + beforeEach(function() { + sandbox = sinon.sandbox.create(); + blockService._db = { + getPrefix: sandbox.stub().callsArgWith(1, null, new Buffer('0000', 'hex')) }; + var setListeners = sandbox.stub(blockService, '_setListeners'); + }); + + after(function() { + sandbox.restore(); + }); + + it('should get the prefix', function(done) { + blockService.start(function() { + expect(blockService._encoding).to.be.an.instanceof(Encoding); + done(); + }); + }); + }); + + describe('#stop', function() { + + it('should call stop', function(done) { + blockService.stop(done); + }); + + }); + + describe('#_loadTip', function() { + + var tip = { hash: 'aa', height: 1 }; + var sandbox; + var testEmitter = new EventEmitter(); + + beforeEach(function() { + sandbox = sinon.sandbox.create(); + + blockService._db = { + + getServiceTip: function(name) { + testEmitter.emit('tip-' + name, tip); + } + + }; + }); + + after(function() { + sandbox.restore(); + }); + + it('should load the tip from the db service', function() { + testEmitter.on('tip-block', function(_tip) { + expect(_tip).to.deep.equal(tip); + }); + blockService._loadTip(); + }); + }); + + describe('#_sendDelta', function() { + + var sandbox; + beforeEach(function() { + sandbox = sinon.sandbox.create(); + }); + + after(function() { + sandbox.restore(); + }); + + it('should send all unsent blocks for the active chain', function() { + var activeChain = sandbox.stub(blockService, '_selectActiveChain').returns('aa'); + var checkChain = sandbox.stub(blockService, '_checkChain').returns(true); + var getDelta = sandbox.stub(blockService, '_getDelta').returns(['aa', '00']); + var broadcast = sandbox.stub(blockService, '_broadcast'); + var setTip = sandbox.stub(blockService, '_setTip'); + + blockService._sendDelta(); + expect(activeChain.calledOnce).to.be.true; + expect(checkChain.calledOnce).to.be.true; + expect(getDelta.calledOnce).to.be.true; + expect(broadcast.calledTwice).to.be.true; + expect(setTip.calledOnce).to.be.true; + }); + }); + + describe('#_getDelta', function() { + + var sandbox; + beforeEach(function() { + sandbox = sinon.sandbox.create(); + }); + + after(function() { + sandbox.restore(); + }); + + it('should get all unsent blocks for the active chain', function() { + + var expected = [ 'block bb', 'block cc' ]; + blockService._tip = { hash: 'aa' }; + blockService._blockHeaderQueue = LRU(5); + blockService._blockQueue = LRU(5); + blockService._blockHeaderQueue.set('cc', { prevHash: 'bb' }); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); + blockService._blockQueue.set('bb', 'block cc'); + blockService._blockQueue.set('aa', 'block 00'); + blockService._blockQueue.set('cc', 'block bb'); + var actual = blockService._getDelta('cc'); + expect(actual).to.deep.equal(expected); + }); + }); + + describe('#_checkChain', function() { + + var sandbox; + beforeEach(function() { + sandbox = sinon.sandbox.create(); + }); + + after(function() { + sandbox.restore(); + }); + + it('should check that blocks between the active chain tip and the block service tip are in the same chain and the chain is complete.', function() { + + blockService._tip = { hash: 'aa' }; + blockService._blockHeaderQueue = LRU(5); + blockService._blockQueue = LRU(5); + blockService._blockHeaderQueue.set('cc', { prevHash: 'bb' }); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); + blockService._blockQueue.set('bb', 'block cc'); + blockService._blockQueue.set('cc', 'block bb'); + var result = blockService._checkChain('cc'); + expect(result).to.be.true; + blockService._blockQueue.reset(); + blockService._blockHeaderQueue.reset(); + }); + + it('should check that blocks between the active chain tip and the block service tip are in a different chain.', function() { + + blockService._tip = { hash: 'aa' }; + blockService._blockHeaderQueue = LRU(5); + blockService._blockQueue = LRU(5); + blockService._blockHeaderQueue.set('cc', { prevHash: 'xx' }); + blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); + blockService._blockQueue.set('bb', 'block cc'); + blockService._blockQueue.set('cc', 'block bb'); + var result = blockService._checkChain('cc'); + expect(result).to.be.false; + }); }); }); -