From 67e9d63f4b3af5230a77d127d0e8bb3ac986b510 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Sun, 16 Jul 2017 17:29:58 -0400 Subject: [PATCH] wip --- lib/services/address/index.js | 37 +- lib/services/block/encoding.js | 12 +- lib/services/block/index.js | 145 +++-- lib/services/db/index.js | 101 ++-- lib/services/timestamp/encoding.js | 15 +- lib/services/timestamp/index.js | 203 ++++--- lib/services/transaction/encoding.js | 6 +- lib/services/transaction/index.js | 286 +++++----- lib/utils.js | 13 + test/services/address/encoding.unit.js | 5 +- .../block/{encoding.js => encoding.unit.js} | 55 +- test/services/block/index.unit.js | 514 ++++++++---------- .../{timestamp.unit.js => encoding.unit.js} | 10 +- test/services/transaction/encoding.unit.js | 2 +- 14 files changed, 673 insertions(+), 731 deletions(-) rename test/services/block/{encoding.js => encoding.unit.js} (56%) rename test/services/timestamp/{timestamp.unit.js => encoding.unit.js} (81%) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index ee3f74f2..fe29aa5e 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -12,28 +12,6 @@ var _ = bitcore.deps._; var Encoding = require('./encoding'); var utils = require('../../utils'); var Transform = require('stream').Transform; -/* - -1. getAddressSummary -2. getAddressUnspentOutputs -3. bitcoind.height -4. getBlockHeader -5. getDetailedTransaction -6. getTransaction -7. sendTransaction -8. getInfo -9. bitcoind.tiphash -10. getBestBlockHash -11. isSynced -12. getAddressHistory -13. getBlock -14. getRawBlock -15. getBlockHashesByTimestamp -16. estimateFee -17. getBlockOverview -18. syncPercentage - -*/ var AddressService = function(options) { BaseService.call(this, options); @@ -405,7 +383,6 @@ AddressService.prototype._setListeners = function() { var self = this; - self._db.on('error', self._onDbError.bind(self)); self.on('reorg', self._handleReorg.bind(self)); }; @@ -431,21 +408,13 @@ AddressService.prototype._onBlock = function(block) { var operations = []; block.transactions.forEach(function(tx) { - operations.concat(self._processTransaction(tx, { block: block, connect: connect })); + operations.concat(self._processTransaction(tx, { block: block })); }); if (operations && operations.length > 0) { - self._db.batch(operations, function(err) { + self._db.batch(operations); - if(err) { - log.error('Address Service: Error saving block with hash: ' + block.hash); - this._db.emit('error', err); - return; - } - - log.debug('Address Service: Success saving block hash ' + block.hash); - }); } }; @@ -537,6 +506,4 @@ AddressService.prototype._processTransaction = function(opts, tx) { }; - - module.exports = AddressService; diff --git a/lib/services/block/encoding.js b/lib/services/block/encoding.js index cbc017e9..3987dda8 100644 --- a/lib/services/block/encoding.js +++ b/lib/services/block/encoding.js @@ -4,15 +4,15 @@ var Block = require('bitcore-lib').Block; // stores -- block header as key, block itself as value (optionally) function Encoding(servicePrefix) { - this.servicePrefix = servicePrefix; - this.blockPrefix = new Buffer('00', 'hex'); - this.metaPrefix = new Buffer('01', 'hex'); + this._servicePrefix = servicePrefix; + this._blockPrefix = new Buffer('00', 'hex'); + this._metaPrefix = new Buffer('01', 'hex'); } // ---- hash --> rawblock Encoding.prototype.encodeBlockKey = function(hash) { - return Buffer.concat([ this.servicePrefix, this.blockPrefix, new Buffer(hash, 'hex') ]); + return Buffer.concat([ this._servicePrefix, this._blockPrefix, new Buffer(hash, 'hex') ]); }; Encoding.prototype.decodeBlockKey = function(buffer) { @@ -31,7 +31,7 @@ Encoding.prototype.decodeBlockValue = function(buffer) { Encoding.prototype.encodeMetaKey = function(height) { var heightBuf = new Buffer(4); heightBuf.writeUInt32BE(height); - return Buffer.concat([ this.blockPrefix, this.metaPrefix, heightBuf ]); + return Buffer.concat([ this._servicePrefix, this._metaPrefix, heightBuf ]); }; Encoding.prototype.decodeMetaKey = function(buffer) { @@ -40,7 +40,7 @@ Encoding.prototype.decodeMetaKey = function(buffer) { Encoding.prototype.encodeMetaValue = function(value) { // { chainwork: hex-string, hash: hex-string } - return Buffer([ new Buffer(value.hash, 'hex'), new Buffer(value.chainwork, 'hex') ]); + return Buffer.concat([ new Buffer(value.hash, 'hex'), new Buffer(value.chainwork, 'hex') ]); }; Encoding.prototype.decodeMetaValue = function(buffer) { diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 95cc07a1..ba116bf1 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -14,28 +14,6 @@ var BN = require('bn.js'); var consensus = require('bcoin').consensus; var constants = require('../../constants'); -/* - -1. getAddressSummary -2. getAddressUnspentOutputs -3. bitcoind.height -4. getBlockHeader -5. getDetailedTransaction -6. getTransaction -7. sendTransaction -8. getInfo -9. bitcoind.tiphash -10. getBestBlockHash -11. isSynced -12. getAddressHistory -13. getBlock -14. getRawBlock -15. getBlockHashesByTimestamp -16. estimateFee -17. getBlockOverview -18. syncPercentage - -*/ var BlockService = function(options) { BaseService.call(this, options); @@ -61,7 +39,6 @@ var BlockService = function(options) { // keep track of out-of-order blocks, this is a list of chains (which are lists themselves) // e.g. [ [ block5, block4 ], [ block8, block7 ] ]; - // TODO: persist this to disk, we can't hold too many blocks in memory this._incompleteChains = []; // list of all chain tips, including main chain and any chains that were orphaned after a reorg this._chainTips = []; @@ -93,15 +70,15 @@ BlockService.prototype.getBestBlockHash = function(callback) { return callback(null, this._meta[this._meta.length - 1].hash); }; -BlockService.prototype.getBlock = function(hash, callback) { +BlockService.prototype.getBlock = function(arg, callback) { - blockArg = this._getHash(blockArg); + var hash = this._getHash(arg); - if (!blockArg) { + if (!hash) { return callback(); } - this._getBlock(blockArg, callback); + this._getBlock(hash, callback); }; @@ -342,7 +319,7 @@ BlockService.prototype._findCommonAncestor = function(block) { _newTip = newBlk.prevHash; if (_newTip === _oldTip) { - return _newTip; + return this._blockQueue.get(_newTip); } } }; @@ -426,6 +403,28 @@ BlockService.prototype._getIncompleteChainIndexes = function(block) { return ret; }; +BlockService.prototype._getOldBlocks = function(currentHash, commonAncestorHash) { + // the old blocks should be in the meta colection + + if (currentHash === commonAncestorHash || !commonAncestor || !currentHash) { + return; + } + + var oldBlocks; + for(var i = this._meta.length - 1; i > 0; --i) { + var item = this._meta[i]; + if (item.hash === currentHash) { + oldBlocks = [this._blockQueue.get(currentHash)]; + continue; + } + if (item.hash === commonAncestorHash) { + return oldBlocks; + } + oldBlocks.push(this._blockQueue.get(item.hash)); + } + +}; + BlockService.prototype._handleReorg = function(block) { this._reorging = true; @@ -434,25 +433,67 @@ BlockService.prototype._handleReorg = function(block) { this._tip.hash + ' the current block: ' + block.hash + '.'); var commonAncestor = this._findCommonAncestor(block); + var oldBlocks = this._getOldBlocks(this._tip.hash, block.hash, commonAncestor.hash); - if (!commonAncestor) { + if (!commonAncestor || !oldBlocks || oldBlock.length < 1) { log.error('A common ancestor block between hash: ' + this._tip.hash + ' (our current tip) and: ' + block.hash + ' (the forked block) could not be found. Bitcore-node must exit.'); - log.permitWrites = false; this.node.stop(); return; } - log.warn('A common ancestor block was found to at hash: ' + commonAncestor + '.'); + log.warn('A common ancestor block was found to at hash: ' + commonAncestor.hash + '.'); - this._broadcast(this.subscriptions.reorg, 'block/reorg', [commonAncestor, [block]]); + this._broadcast(this.subscriptions.reorg, 'block/reorg', [oldBlocks, [block], commonAncestor]); - this._onReorg(commonAncestor, [block]); + this._onReorg(oldBlocks, [block], commonAncestor); this._reorging = false; }; -BlockService.prototype._onReorg = function(commonAncestor, newBlockList) { +BlockService.prototype._onReorg = function(oldBlockList, newBlockList, commonAncestor) { + + // set the tip to the common ancestor in case something goes wrong with the reorg + var tipOps = utils.encodeTip({ height: commonAncestor.header.height }); + + var removalOps = [{ + type: 'put', + key: tipOps.key, + value: tipOps.value + }]; + + + // remove all the old blocks that we reorg from + oldBlockList.forEach(function(block) { + removalOps.concat([ + { + type: 'del', + key: this.encoding.encodeBlockKey(block.header.timestamp), + }, + { + type: 'del', + key: this.encoding.encodeMetaKey(block.header.height), + } + ]); + }); + + this._db.batch(removalOps); + + // remove the blocks from the in-memory meta list + var newMetaHeight = commonAncestor.header.height; + for(var i = this._meta.length - 1; i > 0; --i) { + if (i > newMetaHeight) { + this._meta.splice(i, 1); + } + } + + //call onBlock for each of the new blocks + newBlockList.forEach(this._onBlock.bind(this)); + // if the common ancestor block height is greater than our own, then nothing to do for the reorg + if (this._tip.height <= commonAncestor.header.height) { + return; + } + }; BlockService.prototype._isChainReorganizing = function(block) { @@ -489,8 +530,6 @@ BlockService.prototype._loadMeta = function(callback) { var stream = this._db.createReadStream(criteria); - stream.on('error', self._onDbError.bind(self)); - stream.on('end', function() { if (self._meta.length < 1) { self._meta.push({ @@ -500,6 +539,7 @@ BlockService.prototype._loadMeta = function(callback) { } callback(); }); + stream.on('data', function(data) { self._meta.push(self._encoding.decodeMetaValue(data.value)); }); @@ -550,13 +590,6 @@ BlockService.prototype._onBlock = function(block) { } }; -BlockService.prototype._onDbError = function(err) { - - log.error('Block Service: Error: ' + err + ' not recovering.'); - this.node.stop(); - -}; - BlockService.prototype._saveMetaData = function(block) { var item = { chainwork: this._getChainwork(block.hash).toString(16, 64), @@ -565,9 +598,28 @@ BlockService.prototype._saveMetaData = function(block) { this._meta.push(item); - // save meta position in db - this._db.put(this._encoding.encodeMetaKey(this._meta.length - 1), - this._encoding.encodeMetaValue(item)); + var operations = []; + + // tip + this._tip.hash = block.hash; + this._tip.height = block.height; + + var tipInfo = utils.encodeTip(this._tip, this.name); + + operations.push({ + type: 'put', + key: tipInfo.key, + value: tipInfo.value + }); + + //meta positions in db + operations.push({ + type: 'put', + key: this._encoding.encodeMetaKey(this._meta.length - 1), + value: this._encoding.encodeMetaValue(item) + }); + + this._db.batch(operations); }; @@ -622,7 +674,6 @@ BlockService.prototype._setListeners = function() { var self = this; self._p2p.once('bestHeight', self._onBestHeight.bind(self)); - self._db.on('error', self._onDbError.bind(self)); }; diff --git a/lib/services/db/index.js b/lib/services/db/index.js index ac458ad6..fc454e9a 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -11,6 +11,7 @@ var Networks = bitcore.Networks; var $ = bitcore.util.preconditions; var Service = require('../../service'); var constants = require('../../constants'); +var log = require('../../index').log; function DB(options) { @@ -45,6 +46,12 @@ util.inherits(DB, Service); DB.dependencies = []; +DB.prototype._onError = function(err) { + log.error('Db Service: error: ' + err); + this.node.stop(); + process.exit(-1); +}; + DB.prototype._setDataPath = function() { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); if (this.node.network === Networks.livenet) { @@ -143,69 +150,46 @@ DB.prototype.get = function(key, options, callback) { }; -DB.prototype.put = function(key, value, options, callback) { +DB.prototype.put = function(key, value, options) { var self = this; - var cb = callback; - var opts = options; - if (typeof callback !== 'function') { - cb = options; - opts = {}; + + if (self._stopping) { + return; } - if (!self._stopping) { - self._operationsQueue++; - self._store.put(key, value, opts, function(err) { + self._operationsQueue++; + self._store.put(key, value, options, function(err) { - self._operationsQueue--; + self._operationsQueue--; - if (err) { - self.emit('error', err); - return; - } - - if (typeof cb === 'function') { - cb(err); - } - - }); - - } else { - if (typeof cb === 'function') { - cb(); + if (err) { + self.emit('error', err); + return; } - } + + }); + }; -DB.prototype.batch = function(ops, options, callback) { +DB.prototype.batch = function(ops, options) { var self = this; - var cb = callback; - var opts = options; - if (typeof callback !== 'function') { - cb = options; - opts = {}; + + if (self._stopping) { + return; } - if (!self._stopping) { - self._operationsQueue += ops.length; - self._store.batch(ops, opts, function(err) { + self._operationsQueue += ops.length; + self._store.batch(ops, options, function(err) { - self._operationsQueue -= ops.length; + self._operationsQueue -= ops.length; - if (err) { - self.emit('error', err); - return; - } + if (err) { + self.emit('error', err); + return; + } - if (typeof cb === 'function') { - cb(err); - } + }); - }); - } else { - - setImmediate(cb); - - } }; DB.prototype.createReadStream = function(op) { @@ -213,6 +197,9 @@ DB.prototype.createReadStream = function(op) { if (!this._stopping) { stream = this._store.createReadStream(op); } + if (stream) { + stream.on('error', this.onError.bind(this)); + } return stream; }; @@ -221,6 +208,9 @@ DB.prototype.createKeyStream = function(op) { if (!this._stopping) { stream = this._store.createKeyStream(op); } + if (stream) { + stream.on('error', this.onError.bind(this)); + } return stream; }; @@ -284,21 +274,6 @@ DB.prototype.getServiceTip = function(serviceName, callback) { }; -DB.prototype.setServiceTip = function(serviceName, tip) { - - var self = this; - var keyBuf = Buffer.concat([ self.dbPrefix, new Buffer('tip-' + serviceName, 'utf8') ]); - var heightBuf = new Buffer(4); - heightBuf.writeUInt32BE(tip.height); - - var valBuf = Buffer.concat([ heightBuf, new Buffer(tip.hash, 'hex') ]); - self.put(keyBuf, valBuf, function(err) { - if (err) { - self.emit('error', err); - } - }); - -}; DB.prototype.getPrefix = function(service, callback) { var self = this; diff --git a/lib/services/timestamp/encoding.js b/lib/services/timestamp/encoding.js index 77f4759e..6b0edfb6 100644 --- a/lib/services/timestamp/encoding.js +++ b/lib/services/timestamp/encoding.js @@ -1,15 +1,18 @@ 'use strict'; function Encoding(servicePrefix) { - this.servicePrefix = servicePrefix; + this._servicePrefix = servicePrefix; + this._blockPrefix = new Buffer('00', 'hex'); + this._timestampPrefix = new Buffer('01', 'hex'); } +// ---- block hash -> timestamp Encoding.prototype.encodeBlockTimestampKey = function(hash) { - return Buffer.concat([this.servicePrefix, new Buffer(hash, 'hex')]); + return Buffer.concat([this._servicePrefix, this._blockPrefix, new Buffer(hash, 'hex')]); }; Encoding.prototype.decodeBlockTimestampKey = function(buffer) { - return buffer.slice(2).toString('hex'); + return buffer.slice(3).toString('hex'); }; Encoding.prototype.encodeBlockTimestampValue = function(timestamp) { @@ -22,14 +25,16 @@ Encoding.prototype.decodeBlockTimestampValue = function(buffer) { return buffer.readDoubleBE(); }; + +// ---- timestamp -> block hash Encoding.prototype.encodeTimestampBlockKey = function(timestamp) { var timestampBuffer = new Buffer(new Array(8)); timestampBuffer.writeDoubleBE(timestamp); - return Buffer.concat([this.servicePrefix, timestampBuffer]); + return Buffer.concat([this._servicePrefix, this._timestampPrefix, timestampBuffer]); }; Encoding.prototype.decodeTimestampBlockKey = function(buffer) { - return buffer.readDoubleBE(2); + return buffer.readDoubleBE(3); }; Encoding.prototype.encodeTimestampBlockValue = function(hash) { diff --git a/lib/services/timestamp/index.js b/lib/services/timestamp/index.js index 77f2c234..d6430f84 100644 --- a/lib/services/timestamp/index.js +++ b/lib/services/timestamp/index.js @@ -1,37 +1,22 @@ 'use strict'; +var BaseService = require('../../service'); var Encoding = require('./encoding'); +var assert = require('assert'); +var _ = require('lodash'); +var index = require('../../index'); +var log = index.log; +var LRU = require('lru-cache'); var inherits = require('util').inherits; -var LRU = require('lru-cache'); var utils = require('../../../lib/utils'); -/* - -1. getAddressSummary -2. getAddressUnspentOutputs -3. bitcoind.height -4. getBlockHeader -5. getDetailedTransaction -6. getTransaction -7. sendTransaction -8. getInfo -9. bitcoind.tiphash -10. getBestBlockHash -11. isSynced -12. getAddressHistory -13. getBlock -14. getRawBlock -15. getBlockHashesByTimestamp -16. estimateFee -17. getBlockOverview -18. syncPercentage - -*/ function TimestampService(options) { BaseService.call(this, options); this._db = this.node.services.db; this._tip = null; + this._lastBlockTimestamp = 0; + this._cache = new LRU(10); } inherits(TimestampService, BaseService); @@ -49,7 +34,51 @@ TimestampService.prototype.syncPercentage = function(callback) { return callback(null, ((this._tip.height / this._block.getBestBlockHeight()) * 100).toFixed(2) + '%'); }; -TimestampService.prototype.getBlockHashesByTimestamp = function(callback) { +TimestampService.prototype.getBlockHashesByTimestamp = function(low, high, callback) { + + assert(_.isNumber(low) && _.isNumber(high) && low < high, + 'start time and end time must be integers representing the number of seconds since epoch.'); + + var self = this; + var result; + var lastEntry; + + var start = self._encoding.encodeTimestampBlockKey(low); + + var criteria = { + gte: start, + lt: utils.getTerminalKey(start) + }; + + var tsStream = self._db.createReadStream(criteria); + + tsStream.on('data', function(data) { + var value = self._encoding.decodeTimestampBlockValue(data.value); + if (!result) { + result.push(value); + } + lastEntry = value; + }); + + var streamErr; + tsStream.on('error', function(err) { + streamErr = err; + }); + + tsStream.on('end', function() { + + if(streamErr) { + return callback(streamErr); + } + + if (!result) { + return callback(); + } + + return callback(null, result.push(lastEntry)); + + }); + }; TimestampService.prototype.start = function(callback) { @@ -95,7 +124,7 @@ TimestampService.prototype._startSubscriptions = function() { this._bus.subscribe('block/block'); }; -BlockService.prototype._sync = function() { +TimestampService.prototype._sync = function() { if (--this._p2pBlockCallsNeeded > 0) { @@ -107,6 +136,7 @@ BlockService.prototype._sync = function() { } }; + TimestampService.prototype._setListeners = function() { var self = this; @@ -129,71 +159,98 @@ TimestampService.prototype.stop = function(callback) { TimestampService.prototype._onBlock = function(block) { - - var prevHash = utils.reverseBufferToString(block.header.prevHash); - var operations = []; + var ts = block.header.timestamp; + + if (ts <= this._lastBlockTimestamp) { + ts = this._lastBlockTimestamp + 1; + } + + this._lastBlockTimestamp = ts; + + this._tip.hash = block.hash; + this._tip.height++; + this._cache.set(block.hash, ts); + + var tipInfo = utils.encodeTip(this._tip, this.name); + operations = operations.concat([ { - type: action, - key: self.encoding.encodeTimestampBlockKey(item.value), - value: self.encoding.encodeTimestampBlockValue(item.key) + type: 'put', + key: this.encoding.encodeTimestampBlockKey(ts), + value: this.encoding.encodeTimestampBlockValue(block.hash) }, { - type: action, - key: self.encoding.encodeBlockTimestampKey(item.key), - value: self.encoding.encodeBlockTimestampValue(item.value) + type: 'put', + key: this.encoding.encodeBlockTimestampKey(block.hash), + value: this.encoding.encodeBlockTimestampValue(ts) + }, + { + type: 'put', + key: tipInfo.key, + value: tipInfo.value } + ]); - callback(null, operations); + this._db.batch(operations); }; +TimestampService.prototype._onReorg = function(oldBlockList, newBlockList, commonAncestor) { + + // if the common ancestor block height is greater than our own, then nothing to do for the reorg + if (this._tip.height <= commonAncestor.header.height) { + return; + } + + // set the tip to the common ancestor in case something goes wrong with the reorg + var tipOps = utils.encodeTip({ height: commonAncestor.header.height }); + + var removalOps = [{ + type: 'put', + key: tipOps.key, + value: tipOps.value + }]; + + + // remove all the old blocks that we reorg from + oldBlockList.forEach(function(block) { + removalOps.concat([ + { + type: 'del', + key: this.encoding.encodeTimestampBlockKey(block.header.timestamp), + }, + { + type: 'del', + key: this.encoding.encodeBlockTimestampKey(block.hash), + } + ]); + }); + + this._db.batch(removalOps); + + // set the last time stamp to the common ancestor + this._lastBlockTimestamp = commonAncestor.header.timestamp; + + //call onBlock for each of the new blocks + newBlockList.forEach(this._onBlock.bind(this)); + +}; + + +TimestampService.prototype.getTimestampSync = function(hash) { + return this._cache.get(hash); +}; + TimestampService.prototype.getTimestamp = function(hash, callback) { - this._getValue(hash, callback); + this._db.get(this._encoding.encodeBlockTimestampKey(hash), callback); }; TimestampService.prototype.getHash = function(timestamp, callback) { - this._getValue(timestamp, callback); + this._db.get(this._encoding.encodeTimestampBlockKey(timestamp), callback); }; -TimestampService.prototype._getValue = function(key, callback) { - - var self = this; - var keyBuf, fn; - - if (key.length === 64){ - keyBuf = self.encoding.encodeBlockTimestampKey(key); - fn = self.encoding.decodeBlockTimestampValue; - } else { - keyBuf = self.encoding.encodeTimestampBlockKey(key); - fn = self.encoding.decodeTimestampBlockValue; - } - - self.db.get(keyBuf, function(err, value) { - - if (err) { - return callback(err); - } - - callback(null, fn(value)); - - }); -}; - -TimestampService.prototype._onReorg = function(commonAncestor, newBlockList) { -}; - -TimestampService.prototype.getBlockHashesByTimestamp = function(high, low, options, callback) { - - var self = this; - if (_.isFunction(options)) { - callback = options; - options = {}; - } - -}; module.exports = TimestampService; diff --git a/lib/services/transaction/encoding.js b/lib/services/transaction/encoding.js index 6831fe79..c500a31d 100644 --- a/lib/services/transaction/encoding.js +++ b/lib/services/transaction/encoding.js @@ -1,9 +1,8 @@ 'use strict'; -var bitcore = require('bitcore-lib'); +var tx = require('bcoin').tx; function Encoding(servicePrefix) { - //if you add any more encoding keys, be sure to add a subkey this.servicePrefix = servicePrefix; } @@ -44,7 +43,8 @@ Encoding.prototype.decodeTransactionValue = function(buffer) { for(var i = 0; i < inputValuesLength; i++) { inputValues.push(buffer.readDoubleBE(i * 8 + 14)); } - var transaction = new bitcore.Transaction(buffer.slice(inputValues.length * 8 + 14)); + var transaction = tx.fromRaw(buffer.slice(inputValues.length * 8 + 14)); + transaction.__height = height; transaction.__inputValues = inputValues; transaction.__timestamp = timestamp; diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js index 395ef6a0..5c0c000a 100644 --- a/lib/services/transaction/index.js +++ b/lib/services/transaction/index.js @@ -1,11 +1,12 @@ 'use strict'; -var assert = require('assert'); var async = require('async'); var BaseService = require('../../service'); var inherits = require('util').inherits; var Encoding = require('./encoding'); -var levelup = require('levelup'); +var utils = require('../../lib/utils'); +var _ = require('lodash'); +var LRU = require('lru-cache'); function TransactionService(options) { BaseService.call(this, options); @@ -13,6 +14,8 @@ function TransactionService(options) { this._mempool = this.node.services._mempool; this._block = this.node.services.block; this._p2p = this.node.services.p2p; + this._timestamp = this.node.services.timestamp; + this._inputValuesCache = LRU(1E6); // this should speed up syncing } inherits(TransactionService, BaseService); @@ -25,28 +28,7 @@ TransactionService.dependencies = [ 'mempool' ]; -/* - -1. getAddressSummary -2. getAddressUnspentOutputs -3. bitcoind.height -4. getBlockHeader -5. getDetailedTransaction -6. getTransaction -7. sendTransaction -8. getInfo -9. bitcoind.tiphash -10. getBestBlockHash -11. isSynced -12. getAddressHistory -13. getBlock -14. getRawBlock -15. getBlockHashesByTimestamp -16. estimateFee -17. getBlockOverview -18. syncPercentage - -*/ +// ---- start public function protorypes TransactionService.prototype.getAPIMethods = function() { return [ ['getRawTransaction', this, this.getRawTransaction, 1], @@ -57,6 +39,9 @@ TransactionService.prototype.getAPIMethods = function() { ]; }; +TransactionService.prototype.getDetailedTransaction = function(txid, options, callback) { + this.getTransaction(txid, options, callback); +}; TransactionService.prototype.getTransaction = function(txid, options, callback) { @@ -80,7 +65,8 @@ TransactionService.prototype.getTransaction = function(txid, options, callback) } if (memTx) { - return callback(null, { tx: memTx, confirmations: 0}); + memTx.confirmations = 0; + return callback(null, memTx); } return callback(); @@ -89,7 +75,8 @@ TransactionService.prototype.getTransaction = function(txid, options, callback) } else { if (tx) { - return callback(null, { tx: tx, confirmations: this._p2p.getBestHeight - tx.__height }); + tx.confirmations = this._p2p.getBestHeight - tx.__height; + return callback(null, tx); } return callback(); @@ -97,84 +84,12 @@ TransactionService.prototype.getTransaction = function(txid, options, callback) }); - -}; - -TransactionService.prototype._onBlock = function(block, connectBlock, callback) { - var self = this; - var action = 'put'; - var reverseAction = 'del'; - if (!connectBlock) { - action = 'del'; - reverseAction = 'put'; - } - - var operations = []; - - this.currentTransactions = {}; - - async.series([ - function(next) { - self.node.services.timestamp.getTimestamp(block.hash, function(err, timestamp) { - if(err) { - return next(err); - } - block.__timestamp = timestamp; - next(); - }); - }, function(next) { - async.eachSeries(block.transactions, function(tx, next) { - tx.__timestamp = block.__timestamp; - tx.__height = block.__height; - - self._getInputValues(tx, function(err, inputValues) { - if(err) { - return next(err); - } - tx.__inputValues = inputValues; - self.currentTransactions[tx.id] = tx; - - operations.push({ - type: action, - key: self.encoding.encodeTransactionKey(tx.id), - value: self.encoding.encodeTransactionValue(tx) - }); - next(); - }); - }, function(err) { - if(err) { - return next(err); - } - next(); - }); - }], function(err) { - if(err) { - return callback(err); - } - callback(null, operations); - }); - }; TransactionService.prototype.sendTransaction = function(tx, callback) { this._p2p.sendTransaction(tx, callback); }; -TransactionService.prototype._setListeners = function() { - - var self = this; - - self._db.on('error', self._onDbError.bind(self)); - self.on('reorg', self._onReorg.bind(self)); - -}; - -TransactionService.prototype._onDbError = function(error) { -}; - -TransactionService.prototype._onReorg = function(commonAncestor, newBlockList) { -}; - TransactionService.prototype.start = function(callback) { var self = this; @@ -202,7 +117,121 @@ TransactionService.prototype.start = function(callback) { }); }; -TransactionService.prototype._onTransaction = function(transaction) { +TransactionService.prototype.stop = function(callback) { + setImmediate(callback); +}; + +TransactionService.prototype.syncPercentage = function(callback) { + +}; + +// --- start private prototype functions +TransactionService.prototype._cacheOutputValues = function(tx) { + + var values = tx.outputs.map(function(output) { + return output.satoshis; + }); + + this._inputValuesCache.set(tx.id, values); + +}; + +TransactionService.prototype._getBlockTimestamp = function(hash) { + return this._timestamp.getTimestamp(hash); +}; + +TransactionService.prototype._getInputValues = function(tx, opts) { + + return tx.inputs.forEach(function(input) { + var value = this._inputValuesCache.get(input.prevTxId); + if (value) { + return value[input.outputIndex]; + } + return null; + }); +}; + +TransactionService.prototype._onBlock = function(block) { + + var self = this; + + var operations = block.transactions.map(function(tx) { + return self._processTransaction(tx, { block: block }); + }); + + if (operations && operations.length > 0) { + + self._db.batch(operations); + + } + +}; + +TransactionService.prototype._onReorg = function(oldBlockList, newBlockList, commonAncestor) { + + // if the common ancestor block height is greater than our own, then nothing to do for the reorg + if (this._tip.height <= commonAncestor.header.height) { + return; + } + + // set the tip to the common ancestor in case something goes wrong with the reorg + var tipOps = utils.encodeTip({ hash: commonAncestor.hash, height: commonAncestor.header.height }); + + var removalOps = [{ + type: 'put', + key: tipOps.key, + value: tipOps.value + }]; + + // remove all the old blocks that we reorg from + oldBlockList.forEach(function(block) { + removalOps.concat([ + { + type: 'del', + key: this.encoding.encodeAddressIndexKey(), + }, + { + type: 'del', + key: this.encoding.encodeBlockTimestampKey(block.hash), + } + ]); + }); + + this._db.batch(removalOps); + + //call onBlock for each of the new blocks + newBlockList.forEach(this._onBlock.bind(this)); +}; + +TransactionService.prototype._processTransaction = function(tx, opts) { + + // squirrel away he current outputs + this._cacheOutputValues(tx); + + // this index is very simple txid -> tx, but we also need to find each + // input's prev output value, the adjusted timestamp for the block and + // the tx's block height + + // input values + tx.__inputValues = this._getInputValues(tx); //if there are any nulls here, this is a cache miss + //timestamp + tx.__timestamp = this._getBlockTimestamp(opts.block); + //height + tx.__height = opts.block.height; + + return { + key: this._encoding.encodeTransactionKey(tx.id), + value: this._encoding.encodeTransactionValue(tx) + }; + +}; + +TransactionService.prototype._setListeners = function() { + + var self = this; + + self.on('reorg', self._onReorg.bind(self)); + }; TransactionService.prototype._startSubscriptions = function() { @@ -220,63 +249,4 @@ TransactionService.prototype._startSubscriptions = function() { this._bus.subscribe('block/block'); }; -TransactionService.prototype.stop = function(callback) { - setImmediate(callback); -}; - -TransactionService.prototype.syncPercentage = function(callback) { - -}; - -TransactionService.prototype._getMissingInputValues = function(tx, callback) { - - var self = this; - - if (tx.isCoinbase()) { - return callback(null, []); - } - - async.eachOf(tx.inputs, function(input, index, next) { - if (tx.__inputValues[index]) { - return next(); - } - self.getTransaction(input.prevTxId.toString('hex'), {}, function(err, prevTx) { - if(err) { - return next(err); - } - if (!prevTx) { - return next(new Error('previous Tx missing.')); - } - if (!prevTx.outputs[input.outputIndex]) { - return next(new Error('Input did not have utxo.')); - } - var satoshis = prevTx.outputs[input.outputIndex].satoshis; - tx.__inputValues[index] = satoshis; - next(); - }); - }, callback); - -}; - -TransactionService.prototype._getInputValues = function(tx, callback) { - var self = this; - - if (tx.isCoinbase()) { - return callback(null, []); - } - - async.mapLimit(tx.inputs, this.concurrency, function(input, next) { - self.getTransaction(input.prevTxId.toString('hex'), {}, function(err, prevTx) { - if(err) { - return next(err); - } - if (!prevTx.outputs[input.outputIndex]) { - return next(new Error('Input did not have utxo: ' + prevTx.id + ' for tx: ' + tx.id)); - } - var satoshis = prevTx.outputs[input.outputIndex].satoshis; - next(null, satoshis); - }); - }, callback); -}; - module.exports = TransactionService; diff --git a/lib/utils.js b/lib/utils.js index 09a6f4f0..6460d821 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -217,4 +217,17 @@ utils.removeItemsByIndexList = function(indexList, list) { return ret; }; +utils.encodeTip = function(tip, name) { + var key = Buffer.concat([ new Buffer('00', 'hex'), + new Buffer('tip-' + name, 'utf8') ]); + + var heightBuf = new Buffer(4); + heightBuf.writeUInt32BE(tip.height); + + var value = Buffer.concat([ heightBuf, new Buffer(tip.hash, 'hex') ]); + return { key: key, value: value }; + +}; + + module.exports = utils; diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js index 42a8dc2a..5556dabe 100644 --- a/test/services/address/encoding.unit.js +++ b/test/services/address/encoding.unit.js @@ -21,7 +21,10 @@ describe('Address service encoding', function() { addressSizeBuf, new Buffer(address), new Buffer('00000001', 'hex'), - new Buffer(txid, 'hex')]); + new Buffer(txid, 'hex'), + new Buffer('00000000', 'hex'), + new Buffer('00', 'hex') + ]); var outputIndex = 5; var utxoKeyBuf = Buffer.concat([ servicePrefix, diff --git a/test/services/block/encoding.js b/test/services/block/encoding.unit.js similarity index 56% rename from test/services/block/encoding.js rename to test/services/block/encoding.unit.js index 108b2c80..40491698 100644 --- a/test/services/block/encoding.js +++ b/test/services/block/encoding.unit.js @@ -10,13 +10,11 @@ describe('Block service encoding', function() { var servicePrefix = new Buffer('0000', 'hex'); var blockPrefix = new Buffer('00', 'hex'); - var hashPrefix = new Buffer('01', 'hex'); - var heightPrefix = new Buffer('02', 'hex'); + var metaPrefix = new Buffer('01', 'hex'); var encoding = new Encoding(servicePrefix); var hash = '91b58f19b6eecba94ed0f6e463e8e334ec0bcda7880e2985c82a8f32e4d03add'; var height = 1; - var header = { hash: hash, height: height }; var block = new Block(new Buffer('0100000095194b8567fe2e8bbda931afd01a7acd399b9325cb54683e64129bcd00000000660802c98f18fd34fd16d61c63cf447568370124ac5f3be626c2e1c3c9f0052d19a76949ffff001d33f3c25d0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d014dffffffff0100f2052a01000000434104e70a02f5af48a1989bf630d92523c9d14c45c75f7d1b998e962bff6ff9995fc5bdb44f1793b37495d80324acba7c8f537caaf8432b8d47987313060cc82d8a93ac00000000', 'hex')); describe('Block', function() { @@ -52,49 +50,28 @@ describe('Block service encoding', function() { }); - describe('Hash', function() { - it('should encode hash key', function() { - encoding.encodeHashKey(hash).should.deep.equal(Buffer.concat([ - servicePrefix, - hashPrefix, - new Buffer(hash, 'hex') - ])); + describe('Meta', function() { + + var heightBuf = new Buffer(4); + heightBuf.writeUInt32BE(height); + + it('should encode meta key', function() { + encoding.encodeMetaKey(height).should.deep.equal(Buffer.concat([ servicePrefix, metaPrefix, heightBuf ])); }); - it('should decode hash key', function() { - encoding.decodeHashKey(Buffer.concat([ - servicePrefix, - hashPrefix, - new Buffer(hash, 'hex') - ])).should.deep.equal(hash); + it('should decode meta key', function() { + encoding.decodeMetaKey(Buffer.concat([ servicePrefix, metaPrefix, heightBuf ])).should.equal(height); }); - it('should encode header value', function() { - encoding.encodeHeaderValue(header).should.deep.equal(new Buffer(JSON.stringify(header), 'utf8')); + it('should encode meta value', function() { + encoding.encodeMetaValue({ chainwork: '00000001', hash: hash }).should.deep.equal( + Buffer.concat([ new Buffer(hash, 'hex'), new Buffer('00000001', 'hex') ])); }); - it('should decode hash value', function() { - encoding.decodeHeaderValue(new Buffer(JSON.stringify(header), 'utf8')).should.deep.equal(header); + it('should decode meta value', function() { + encoding.decodeMetaValue(Buffer.concat([ new Buffer(hash, 'hex'), new Buffer('00000001', 'hex') ])).should.deep.equal( + { chainwork: '00000001', hash: hash }); }); }); - - describe('Height', function() { - it('should encode height key', function() { - encoding.encodeHeightKey(height).should.deep.equal(Buffer.concat([ - servicePrefix, - heightPrefix, - new Buffer('00000001', 'hex') - ])); - }); - - it('should decode height key', function() { - encoding.decodeHeightKey(Buffer.concat([ - servicePrefix, - heightPrefix, - new Buffer('00000001', 'hex') - ])).should.deep.equal(height); - }); - }); - }); diff --git a/test/services/block/index.unit.js b/test/services/block/index.unit.js index 5f5fed7a..2b63d334 100644 --- a/test/services/block/index.unit.js +++ b/test/services/block/index.unit.js @@ -8,7 +8,6 @@ var crypto = require('crypto'); var sinon = require('sinon'); var Block = require('bitcore-lib').Block; var Encoding = require('../../../lib/services/block/encoding'); -var EventEmitter = require('events').EventEmitter; var LRU = require('lru-cache'); var constants = require('../../../lib/constants'); @@ -17,40 +16,24 @@ describe('Block Service', function() { var blockService; beforeEach(function() { - blockService = new BlockService({ node: { services: []}}); - blockService._chainTips = ['00']; - blockService._encoding = new Encoding(new Buffer('0000', 'hex')); - }); - - describe.only('#_applyHeaderHeights', function() { - it('should apply heights to the list of headers', function() { - var genesis = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206'; - blockService._blockHeaderQueue = LRU(100); - blockService._blockHeaderQueue.set(genesis, { prevHash: '00' }); - blockService._latestHeaderHashReceived = '100'; - blockService.node = { getNetworkName: function() { return 'regtest'; } }; - for(var i = 0; i < 100; i++) { - var prevHash = i.toString(); - if (i === 0) { - prevHash = genesis; - } - blockService._blockHeaderQueue.set((i+1).toString(), { prevHash: prevHash }); - } - - blockService._applyHeaderHeights(); - for(var j = 0; j < blockService._applyHeaderHeights.length; j++) { - expect(blockService._applyHeaderHeights[j]).to.equal(j.toString()); + blockService = new BlockService({ + node: { + getNetworkName: function() { return 'regtest'; }, + services: [] } }); + blockService._chainTips = ['00']; + blockService._encoding = new Encoding(new Buffer('0000', 'hex')); }); describe('#_blockAlreadyProcessed', function() { it('should detect that a block has already been delivered to us', function() { - blockService._blockHeaderQueue.set('aa', {}); + blockService._blockQueue = LRU(1); + blockService._blockQueue.set('aa', '00'); expect(blockService._blockAlreadyProcessed({ hash: 'aa' })).to.be.true; expect(blockService._blockAlreadyProcessed('bb')).to.be.false; - blockService._blockHeaderQueue.reset(); + blockService._blockQueue.reset(); }); }); @@ -73,57 +56,6 @@ describe('Block Service', function() { }); - describe('#_cacheHeader', function() { - it('should cache header', function() { - var toObj = sinon.stub().returns('header'); - var header = { hash: 'aa', toObject: toObj }; - blockService._blockHeaderQueue = LRU(1); - blockService._cacheHeader(header); - expect(toObj.calledOnce); - expect(blockService._blockHeaderQueue.get('aa')).to.equal('header'); - }); - }); - - describe('#_checkChain', function() { - - var sandbox; - beforeEach(function() { - sandbox = sinon.sandbox.create(); - }); - - after(function() { - sandbox.restore(); - }); - - it('should check that blocks between the active chain tip and the block service tip are in the same chain and the chain is complete.', function() { - - blockService._tip = { hash: 'aa' }; - blockService._blockHeaderQueue = LRU(5); - blockService._blockQueue = LRU(5); - blockService._blockHeaderQueue.set('cc', { prevHash: 'bb' }); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); - blockService._blockQueue.set('bb', 'block cc'); - blockService._blockQueue.set('cc', 'block bb'); - var result = blockService._checkChain('cc'); - expect(result).to.be.true; - blockService._blockQueue.reset(); - blockService._blockHeaderQueue.reset(); - }); - - it('should check that blocks between the active chain tip and the block service tip are in a different chain.', function() { - - blockService._tip = { hash: 'aa' }; - blockService._blockHeaderQueue = LRU(5); - blockService._blockQueue = LRU(5); - blockService._blockHeaderQueue.set('cc', { prevHash: 'xx' }); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); - blockService._blockQueue.set('bb', 'block cc'); - blockService._blockQueue.set('cc', 'block bb'); - var result = blockService._checkChain('cc'); - expect(result).to.be.false; - }); - }); - describe('#_computeChainwork', function() { it('should calculate chain work correctly', function() { @@ -141,23 +73,23 @@ describe('Block Service', function() { it('should determine the block in a normal state', function() { var sandbox = sinon.sandbox.create(); var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(false); - var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(false); + var stub2 = sandbox.stub(blockService, '_isOutOfOrder').returns(false); expect(blockService._determineBlockState({})).to.equal('normal'); sandbox.restore(); }); - it('should determine the block in a orphan state', function() { + it('should determine the block in a outoforder state', function() { var sandbox = sinon.sandbox.create(); var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(false); - var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(true); - expect(blockService._determineBlockState({})).to.equal('orphaned'); + var stub2 = sandbox.stub(blockService, '_isOutOfOrder').returns(true); + expect(blockService._determineBlockState({})).to.equal('outoforder'); sandbox.restore(); }); it('should determine the block in a reorg state', function() { var sandbox = sinon.sandbox.create(); var stub1 = sandbox.stub(blockService, '_isChainReorganizing').returns(true); - var stub2 = sandbox.stub(blockService, '_isOrphanBlock').returns(false); + var stub2 = sandbox.stub(blockService, '_isOutOfOrder').returns(false); expect(blockService._determineBlockState({})).to.equal('reorg'); sandbox.restore(); }); @@ -177,10 +109,10 @@ describe('Block Service', function() { it('should find the common ancestor between the current chain and the new chain', function() { var block = { hash: 'cc' }; blockService._tip = { hash: 'bb' } - blockService._blockHeaderQueue = new LRU(5); - blockService._blockHeaderQueue.set('aa', { prevHash: '00' }); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); - blockService._blockHeaderQueue.set('cc', { prevHash: 'aa' }); + blockService._blockQueue = new LRU(5); + blockService._blockQueue.set('aa', { prevHash: '00' }); + blockService._blockQueue.set('bb', { prevHash: 'aa' }); + blockService._blockQueue.set('cc', { prevHash: 'aa' }); blockService._chainTips = [ 'cc', 'bb' ]; var commonAncestor = blockService._findCommonAncestor(block); expect(commonAncestor).to.equal('aa'); @@ -188,6 +120,26 @@ describe('Block Service', function() { }); + describe('#getBestBlockHash', function() { + it('should get best block hash', function() { + }); + }); + + describe('#getBlock', function() { + it('should get block', function() { + }); + }); + + describe('#getBlockHashesByTimestamp', function() { + it('should get block hashes by timestamp', function() { + }); + }); + + describe('#getBlockHeader', function() { + it('should get block header', function() { + }); + }); + describe('#_getBlockOperations', function() { it('should get block operations when given one block', function() { @@ -209,25 +161,22 @@ describe('Block Service', function() { }); + describe('#getBlockOverview', function() { + it('should get block overview', function() { + }); + }; + describe('#_getChainwork', function() { - it('should get chainwork, chainwork already on header', function() { + it('should get chainwork', function() { var expected = new BN(new Buffer('000000000000000000000000000000000000000000677c7b8122f9902c79f4e0', 'hex')); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', chainwork: '000000000000000000000000000000000000000000677c7b8122f9902c79f4e0'}); - blockService._blockHeaderQueue.set('aa', { prevHash: '00', chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); + blockService._meta = [ { chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90', hash: 'aa' } ]; + blockService._blockQueue = LRU(1); + blockService._blockQueue.set('bb', { header: { bits: 0x18018d30 }}); var actual = blockService._getChainwork('bb'); assert(actual.eq(expected), 'not equal: actual: ' + actual + ' expected: ' + expected); - }); - it('should get chainwork, chainwork not already on header', function() { - var expected = new BN(new Buffer('000000000000000000000000000000000000000000677c7b8122f9902c79f4e0', 'hex')); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', bits: 0x18018d30 }); - blockService._blockHeaderQueue.set('aa', { prevHash: '00', chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); - var actual = blockService._getChainwork('bb'); - assert(actual.eq(expected), 'not equal: actual: ' + actual + ' expected: ' + expected); - - }); }); describe('#_getDelta', function() { @@ -243,20 +192,23 @@ describe('Block Service', function() { it('should get all unsent blocks for the active chain', function() { - var expected = [ 'block bb', 'block cc' ]; + var block1 = { header: { prevHash: new Buffer('00', 'hex') }}; + var block2 = { header: { prevHash: new Buffer('aa', 'hex') }}; + var block3 = { header: { prevHash: new Buffer('bb', 'hex') }}; + var expected = [ block2, block3 ]; blockService._tip = { hash: 'aa' }; - blockService._blockHeaderQueue = LRU(5); - blockService._blockQueue = LRU(5); - blockService._blockHeaderQueue.set('cc', { prevHash: 'bb' }); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa' }); - blockService._blockQueue.set('bb', 'block cc'); - blockService._blockQueue.set('aa', 'block 00'); - blockService._blockQueue.set('cc', 'block bb'); + blockService._blockQueue = LRU(3); + blockService._blockQueue.set('aa', block1); + blockService._blockQueue.set('bb', block2); + blockService._blockQueue.set('cc', block3); var actual = blockService._getDelta('cc'); expect(actual).to.deep.equal(expected); }); }); + describe('#getRawBlock', function() { + }); + describe('#_isChainReorganizing', function() { it('should decide that chain is reorging', function() { @@ -273,58 +225,30 @@ describe('Block Service', function() { }); - describe('#_isOrphanBlock', function() { + describe('#_isOutOfOrder', function() { beforeEach(function() { var prevHash = '00000000'; for(var i = 0; i < 110; i++) { var newHash = crypto.randomBytes(4); - blockService._blockHeaderQueue.set(newHash, { prevHash: new Buffer(prevHash, 'hex') }); + var mock = { toBuffer: function() { return new Buffer(new Array(10), 'hex') } }; + blockService._blockQueue.set(newHash, mock); prevHash = newHash; } }); it('should detect an orphaned block', function() { var block = { hash: 'ee', header: { prevHash: new Buffer('aa', 'hex') }}; - expect(blockService._isOrphanBlock(block)).to.be.true; + expect(blockService._isOutOfOrder(block)).to.be.true; }); it('should not detect an orphaned block', function() { var block = { hash: 'new', header: { prevHash: '00' }}; - expect(blockService._isOrphanBlock(block)).to.be.true; + expect(blockService._isOutOfOrder(block)).to.be.true; }); }); - describe('#_loadTip', function() { - - var tip = { hash: 'aa', height: 1 }; - var sandbox; - var testEmitter = new EventEmitter(); - - beforeEach(function() { - sandbox = sinon.sandbox.create(); - - blockService._db = { - - getServiceTip: function(name) { - testEmitter.emit('tip-' + name, tip); - } - - }; - }); - - after(function() { - sandbox.restore(); - }); - - it('should load the tip from the db service', function() { - testEmitter.on('tip-block', function(_tip) { - expect(_tip).to.deep.equal(tip); - }); - blockService._loadTip(); - }); - }); describe('#_onBestHeight', function() { var sandbox; @@ -337,13 +261,10 @@ describe('Block Service', function() { }); it('should set best height', function() { - var tips = sandbox.stub(); - var loadTip = sandbox.stub(blockService, '_loadTip'); - blockService._db = { once: tips }; + var startSync = sandbox.stub(blockService, '_startSync'); blockService._onBestHeight(123); expect(blockService._bestHeight).to.equal(123); - expect(tips.calledTwice).to.be.true; - expect(loadTip.calledOnce).to.be.true; + expect(startSync.calledOnce).to.be.true; }); }); @@ -356,8 +277,9 @@ describe('Block Service', function() { var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); var blockState = sandbox.stub(blockService, '_determineBlockState').returns('normal'); - var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + var updateChainTips = sandbox.stub(blockService, '_updateChainInfo'); var sendAllUnsent = sandbox.stub(blockService, '_sendDelta'); + var saveMetaData = sandbox.stub(blockService, '_saveMetaData'); blockService._onBlock({ hash: 'aa' }); expect(alreadyProcessed.callCount).to.equal(1); @@ -365,6 +287,7 @@ describe('Block Service', function() { expect(blockState.callCount).to.equal(1); expect(updateChainTips.callCount).to.equal(1); expect(sendAllUnsent.callCount).to.equal(1); + expect(saveMetaData.callCount).to.equal(1); sandbox.restore(); @@ -377,7 +300,7 @@ describe('Block Service', function() { var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); var blockState = sandbox.stub(blockService, '_determineBlockState').returns('reorg'); - var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + var updateChainTips = sandbox.stub(blockService, '_updateChainInfo'); var reorgListener = blockService.on('reorg', function(block) { expect(block).to.equal(block); @@ -398,8 +321,8 @@ describe('Block Service', function() { var sandbox = sinon.sandbox.create(); var alreadyProcessed = sandbox.stub(blockService, '_blockAlreadyProcessed').returns(false); var cacheBlock = sandbox.stub(blockService, '_cacheBlock'); - var blockState = sandbox.stub(blockService, '_determineBlockState').returns('orphaned'); - var updateChainTips = sandbox.stub(blockService, '_updateChainTips'); + var blockState = sandbox.stub(blockService, '_determineBlockState').returns('outoforder'); + var updateChainTips = sandbox.stub(blockService, '_updateChainInfo'); blockService._onBlock({ hash: 'aa' }); expect(alreadyProcessed.callCount).to.equal(1); @@ -420,103 +343,25 @@ describe('Block Service', function() { }); }); - describe('#_onHeaders', function() { - var sandbox; - beforeEach(function() { - sandbox = sinon.sandbox.create(); - }); - - after(function() { - sandbox.restore(); - }); - - it('should handle header delivery', function() { - var headers = [ { hash: '00' }, { hash: 'aa' } ]; - var cache = sandbox.stub(blockService, '_cacheHeaders'); - blockService._onHeaders(headers); - expect(cache.calledOnce).to.be.true; - expect(blockService._latestHeaderHash).to.equal('aa'); - }); - - }); - - describe('#_onTipHeader', function() { - var sandbox; - beforeEach(function() { - sandbox = sinon.sandbox.create(); - }); - - after(function() { - sandbox.restore(); - }); - - it('should set initial tip after receiving from db', function() { - var startSubs = sandbox.stub(blockService, '_startSubscriptions'); - var startSync = sandbox.stub(blockService, '_startSync'); - var tip = { height: 100, hash: 'aa' }; - blockService._onTipHeader(tip); - expect(blockService._headerTip).to.deep.equal(tip); - expect(startSubs.calledOnce).to.be.true; - expect(startSync.calledOnce).to.be.true; - }); - - it('should set initial tip after not receiving from db', function() { - var startSubs = sandbox.stub(blockService, '_startSubscriptions'); - var startSync = sandbox.stub(blockService, '_startSync'); - var tip = null; - blockService.node = { getNetworkName: function() { return 'regtest'; } }; - blockService._onTipHeader(tip); - expect(blockService._headerTip).to.deep.equal({ height: 0, hash: constants.BITCOIN_GENESIS_HASH['regtest']}); - expect(startSubs.calledOnce).to.be.true; - expect(startSync.calledOnce).to.be.true; - }); - - }); - - describe('#_onTipBlock', function() { - - var sandbox; - beforeEach(function() { - sandbox = sinon.sandbox.create(); - }); - - after(function() { - sandbox.restore(); - }); - - it('should set initial tip after receiving from db', function() { - var tip = { height: 100, hash: 'aa' }; - blockService._onTipBlock(tip); - expect(blockService._tip).to.deep.equal(tip); - }); - - it('should set initial tip after not receiving from db', function() { - var tip = null; - blockService.node = { getNetworkName: function() { return 'regtest'; } }; - blockService._onTipBlock(tip); - expect(blockService._tip).to.deep.equal({ height: 0, hash: constants.BITCOIN_GENESIS_HASH['regtest']}); - }); - - }); - - describe('#_reportBootStatus', function() { - it('should report info on boot', function() { - blockService._tip = { height: 100, hash: 'aa' }; - blockService._reportBootStatus(); - }); - }); describe('#_selectActiveChain', function() { - it('should select active chain based on most chain work', function() { - blockService._blockHeaderQueue.set('cc', { prevHash: '00', bits: 0x18018d30 }); - blockService._blockHeaderQueue.set('bb', { prevHash: 'aa', bits: 0x18018d30 }); - blockService._blockHeaderQueue.set('aa', { chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea90'}); - blockService._blockHeaderQueue.set('00', { chainwork: '000000000000000000000000000000000000000000677bd68118a98f8779ea8f'}); - blockService._chainTips.push('bb'); - blockService._chainTips.push('cc'); + var sandbox; + beforeEach(function() { + sandbox = sinon.sandbox.create(); + }); - var expected = 'bb'; + after(function() { + sandbox.restore(); + }); + + it('should select active chain based on most chain work', function() { + blockService._chainTips = [ 'aa', 'cc' ]; + + var getCW = sandbox.stub(blockService, '_getChainwork'); + getCW.onCall(0).returns(new BN(1)); + getCW.onCall(1).returns(new BN(2)); + var expected = 'cc'; var actual = blockService._selectActiveChain(); expect(actual).to.equal(expected); }); @@ -535,15 +380,14 @@ describe('Block Service', function() { }); it('should send all unsent blocks for the active chain', function() { + blockService._meta = [ { hash: 'aa' } ]; var activeChain = sandbox.stub(blockService, '_selectActiveChain').returns('aa'); - var checkChain = sandbox.stub(blockService, '_checkChain').returns(true); var getDelta = sandbox.stub(blockService, '_getDelta').returns(['aa', '00']); var broadcast = sandbox.stub(blockService, '_broadcast'); var setTip = sandbox.stub(blockService, '_setTip'); blockService._sendDelta(); expect(activeChain.calledOnce).to.be.true; - expect(checkChain.calledOnce).to.be.true; expect(getDelta.calledOnce).to.be.true; expect(broadcast.calledTwice).to.be.true; expect(setTip.calledOnce).to.be.true; @@ -600,8 +444,8 @@ describe('Block Service', function() { blockService._startSubscriptions(); expect(blockService._subscribed).to.be.true; expect(openBus.calledOnce).to.be.true; - expect(on.calledTwice).to.be.true; - expect(subscribe.calledTwice).to.be.true; + expect(on.calledOnce).to.be.true; + expect(subscribe.calledOnce).to.be.true; }); }); @@ -618,22 +462,10 @@ describe('Block Service', function() { sandbox.restore(); }); - it('should start the sync of headers if type set', function() { - blockService._tip = { height: 100 }; - blockService._bestHeight = 200; - var stub = sandbox.stub(blockService, '_sync'); - blockService._startSync('header'); - expect(stub.calledOnce).to.be.true; - expect(stub.calledWith('header')).to.be.true; - expect(blockService._latestHeaderHash).to.equal(constants.BITCOIN_GENESIS_HASH[blockService.node.getNetworkName()]); - expect(blockService._p2pHeaderCallsNeeded).to.equal(1); - }); - it('should start the sync of blocks if type set', function() { var stub = sandbox.stub(blockService, '_sync'); - blockService._startSync('block'); + blockService._startSync(); expect(stub.calledOnce).to.be.true; - expect(stub.calledWith('block')).to.be.true; expect(blockService._latestBlockHash).to.equal(constants.BITCOIN_GENESIS_HASH[blockService.node.getNetworkName()]); expect(blockService._p2pBlockCallsNeeded).to.equal(1); }); @@ -644,32 +476,32 @@ describe('Block Service', function() { blockService._p2pBlockCallsNeeded = 2; var getBlocks = sinon.stub(); blockService._p2p = { getBlocks: getBlocks }; - blockService._sync('block'); + blockService._sync(); expect(blockService._p2pBlockCallsNeeded).to.equal(1); expect(getBlocks.calledOnce).to.be.true; }); }); - describe('#_sync', function() { - it('should sync headers', function() { - blockService._p2pHeaderCallsNeeded = 2; - var getHeaders = sinon.stub(); - blockService._p2p = { getHeaders: getHeaders }; - blockService._sync('header'); - expect(blockService._p2pHeaderCallsNeeded).to.equal(1); - expect(getHeaders.calledOnce).to.be.true; - }); - }); - describe('#start', function() { var sandbox; + var getServiceTip; + var getServicePrefix; + var loadMeta; + var startSubs; beforeEach(function() { sandbox = sinon.sandbox.create(); + getServiceTip = sandbox.stub().callsArgWith(1, null, { height: 100, hash: 'aa' }); + getServicePrefix = sandbox.stub().callsArgWith(1, null, new Buffer('0000', 'hex')); + loadMeta = sandbox.stub(blockService, '_loadMeta').callsArgWith(0, null); + startSubs = sandbox.stub(blockService, '_startSubscriptions'); + blockService._db = { - getPrefix: sandbox.stub().callsArgWith(1, null, new Buffer('0000', 'hex')) }; + getPrefix: getServicePrefix, + getServiceTip: getServiceTip + }; var setListeners = sandbox.stub(blockService, '_setListeners'); }); @@ -680,6 +512,10 @@ describe('Block Service', function() { it('should get the prefix', function(done) { blockService.start(function() { expect(blockService._encoding).to.be.an.instanceof(Encoding); + expect(getServiceTip.calledOnce).to.be.true; + expect(getServicePrefix.calledOnce).to.be.true; + expect(loadMeta.calledOnce).to.be.true; + expect(startSubs.calledOnce).to.be.true; done(); }); }); @@ -693,42 +529,128 @@ describe('Block Service', function() { }); - describe('#_updateChainTips', function() { + describe('#_updateChainInfo', function() { - it('should set chain tips under normal block arrival conditions, in order arrival' , function() { + var sandbox; - var blocks = ['aa','bb','cc','dd','ee']; - - blocks.forEach(function(n, index) { - - var buf = new Buffer('00', 'hex'); - if (index) { - buf = new Buffer(blocks[index-1], 'hex'); - } - - var block = { header: { prevHash: buf }, hash: n }; - blockService._updateChainTips(block, 'normal'); - }); - - expect(blockService._chainTips.length).to.equal(1); - expect(blockService._chainTips).to.deep.equal(['ee']); + beforeEach(function() { + sandbox = sinon.sandbox.create(); }); - it('should not set chain tips if not in normal or reorg state' , function() { + after(function() { + sandbox.restore(); + }); - var block = { header: { prevHash: new Buffer('aa', 'hex') }}; - blockService._updateChainTips(block, 'orphan'); - expect(blockService._chainTips).to.deep.equal(['00']); + it('should update chain tips and incomplete block list for the out of order state', function() { + var stub = sandbox.stub(blockService, '_updateOutOfOrderStateChainInfo'); + blockService._updateChainInfo({ header: { prevHash: new Buffer('aa', 'hex')}}, 'outoforder'); + expect(stub.calledOnce).to.be.true; + }); + + it('should update chain tips and incomplete block list for normal state', function() { + var stub = sandbox.stub(blockService, '_updateNormalStateChainInfo'); + blockService._updateChainInfo({ header: { prevHash: new Buffer('aa', 'hex')}}, 'normal'); + expect(stub.calledOnce).to.be.true; + }); + + it('should update chain tips and incomplete block list for reorg state', function() { + var stub = sandbox.stub(blockService, '_updateReorgStateChainInfo'); + blockService._updateChainInfo({ header: { prevHash: new Buffer('aa', 'hex')}}, 'reorg'); + expect(stub.calledOnce).to.be.true; + }); + }); + + + describe('#_updateOutOfOrderStateChainInfo', function() { + + var block1 = { hash: 'aa', header: { prevHash: new Buffer('99', 'hex') } }; + var block2 = { hash: 'bb', header: { prevHash: new Buffer('aa', 'hex') } }; + var block3 = { hash: 'cc', header: { prevHash: new Buffer('bb', 'hex') } }; + var block4 = { hash: 'dd', header: { prevHash: new Buffer('cc', 'hex') } }; + var block5 = { hash: 'ee', header: { prevHash: new Buffer('dd', 'hex') } }; + var block6_1 = { hash: '11', header: { prevHash: new Buffer('ee', 'hex') } }; + var block6 = { hash: 'ff', header: { prevHash: new Buffer('ee', 'hex') } }; + + it('should join chains if needed', function() { + + blockService._incompleteChains = [ [block6, block5], [block3, block2] ]; + blockService._updateOutOfOrderStateChainInfo(block4); + expect(blockService._incompleteChains).to.deep.equal([ [ block6, block5, block4, block3, block2] ]); }); - it('should set chain tips when there is a reorg taking place' , function() { + it('should join chains if needed different order', function() { - var block = { hash: 'ee', header: { prevHash: 'dd' } }; - blockService._updateChainTips(block, 'reorg'); - expect(blockService._chainTips).to.deep.equal(['00', 'ee']); + blockService._incompleteChains = [ [block3, block2], [block6, block5] ]; + blockService._updateOutOfOrderStateChainInfo(block4); + expect(blockService._incompleteChains).to.deep.equal([ [ block6, block5, block4, block3, block2] ]); }); + + it('should not join chains', function() { + blockService._incompleteChains = [ [block2, block1], [block6, block5] ]; + blockService._updateOutOfOrderStateChainInfo(block4); + expect(blockService._incompleteChains).to.deep.equal([ [ block2, block1 ], [ block6, block5, block4 ] ]); + }); + + it('should not join chains, only add a new chain', function() { + blockService._incompleteChains = [ [block2, block1] ]; + blockService._updateOutOfOrderStateChainInfo(block6); + expect(blockService._incompleteChains).to.deep.equal([ [ block2, block1 ], [ block6 ] ]); + }); + + it('should join multiple different chains', function() { + blockService._incompleteChains = [ [block2, block1], [block6], [block6_1], [block4] ]; + blockService._updateOutOfOrderStateChainInfo(block5); + expect(blockService._incompleteChains).to.deep.equal([ + [ block2, block1 ], + [ block6, block5, block4 ], + [ block6_1, block5, block4 ] + ]); + }); + }); + + describe('#_updateReorgStateChainInfo', function() { + it('should update chain tips for reorg situation', function() { + blockService._chainTips = []; + blockService._updateReorgStateChainInfo({ hash: 'aa' }); + expect(blockService._chainTips).to.deep.equal([ 'aa' ]); + }); + }); + + describe('#_updateNormalStateChainInfo', function() { + + it('should update chain tips when there is no incomplete chains', function() { + var block2 = { hash: 'bb', header: { prevHash: new Buffer('aa', 'hex') } }; + blockService._incompleteChains = []; + blockService._chainTips = [ 'aa' ]; + blockService._updateNormalStateChainInfo(block2, 'aa'); + expect(blockService._chainTips).to.deep.equal([ 'bb' ]); + expect(blockService._incompleteChains).to.deep.equal([]); + }); + + it('should update chain tips when there is an incomplete chain', function() { + var block2 = { hash: 'bb', header: { prevHash: new Buffer('aa', 'hex') } }; + var block3 = { hash: 'cc', header: { prevHash: new Buffer('bb', 'hex') } }; + var block4 = { hash: 'dd', header: { prevHash: new Buffer('cc', 'hex') } }; + blockService._incompleteChains = [ [ block4, block3 ] ]; + blockService._chainTips = [ 'aa' ]; + blockService._updateNormalStateChainInfo(block2, 'aa'); + expect(blockService._chainTips).to.deep.equal([ 'dd' ]); + expect(blockService._incompleteChains).to.deep.equal([]); + }); + + it('should update chain tip when there are mulitipla chain tips', function() { + var block4 = { hash: 'dd', header: { prevHash: new Buffer('cc', 'hex') } }; + var block5 = { hash: 'ee', header: { prevHash: new Buffer('dd', 'hex') } }; + var block6_1 = { hash: '11', header: { prevHash: new Buffer('ee', 'hex') } }; + var block6 = { hash: 'ff', header: { prevHash: new Buffer('ee', 'hex') } }; + blockService._incompleteChains = [ [ block6, block5 ], [ block6_1, block5 ] ]; + blockService._chainTips = [ 'cc' ]; + blockService._updateNormalStateChainInfo(block4, 'cc'); + expect(blockService._chainTips).to.deep.equal([ '11', 'ff' ]); + expect(blockService._incompleteChains).to.deep.equal([]); + }); }); }); diff --git a/test/services/timestamp/timestamp.unit.js b/test/services/timestamp/encoding.unit.js similarity index 81% rename from test/services/timestamp/timestamp.unit.js rename to test/services/timestamp/encoding.unit.js index e7256dc9..bfe4a02d 100644 --- a/test/services/timestamp/timestamp.unit.js +++ b/test/services/timestamp/encoding.unit.js @@ -6,6 +6,8 @@ var Encoding = require('../../../lib/services/timestamp/encoding'); describe('Timestamp service encoding', function() { var servicePrefix = new Buffer('0000', 'hex'); + var blockPrefix = new Buffer('00', 'hex'); + var timestampPrefix = new Buffer('01', 'hex'); var encoding = new Encoding(servicePrefix); var blockhash = '00000000000000000115b92b1ff4377441049bff75c6c48b626eb99e8b744297'; var timestamp = 5; @@ -13,11 +15,11 @@ describe('Timestamp service encoding', function() { timestampBuf.writeDoubleBE(timestamp); it('should encode block timestamp key' , function() { - encoding.encodeBlockTimestampKey(blockhash).should.deep.equal(Buffer.concat([servicePrefix, new Buffer(blockhash, 'hex')])); + encoding.encodeBlockTimestampKey(blockhash).should.deep.equal(Buffer.concat([servicePrefix, blockPrefix, new Buffer(blockhash, 'hex')])); }); it('should decode block timestamp key', function() { - var blockTimestampKey = encoding.decodeBlockTimestampKey(Buffer.concat([servicePrefix, new Buffer(blockhash, 'hex')])); + var blockTimestampKey = encoding.decodeBlockTimestampKey(Buffer.concat([servicePrefix, blockPrefix, new Buffer(blockhash, 'hex')])); blockTimestampKey.should.equal(blockhash); }); @@ -30,11 +32,11 @@ describe('Timestamp service encoding', function() { }); it('should encode timestamp block key', function() { - encoding.encodeTimestampBlockKey(timestamp).should.deep.equal(Buffer.concat([servicePrefix, timestampBuf])); + encoding.encodeTimestampBlockKey(timestamp).should.deep.equal(Buffer.concat([servicePrefix, timestampPrefix, timestampBuf])); }); it('should decode timestamp block key', function() { - encoding.decodeTimestampBlockKey(Buffer.concat([servicePrefix, timestampBuf])).should.equal(timestamp); + encoding.decodeTimestampBlockKey(Buffer.concat([servicePrefix, timestampPrefix, timestampBuf])).should.equal(timestamp); }); it('should encode timestamp block value', function() { diff --git a/test/services/transaction/encoding.unit.js b/test/services/transaction/encoding.unit.js index 7a450ec2..1c1e5dc2 100644 --- a/test/services/transaction/encoding.unit.js +++ b/test/services/transaction/encoding.unit.js @@ -37,6 +37,6 @@ describe('Transaction service encoding', function() { tx.__height.should.equal(2); tx.__timestamp.should.equal(1); tx.__inputValues.should.deep.equal([2,3]); - tx.toBuffer().toString('hex').should.equal(txHex); + tx.toRaw().toString('hex').should.equal(txHex); }); });