From 04c5f94997e1ba3c15917e1a9cb007dd488e9d7d Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 5 Mar 2016 02:18:39 -0800 Subject: [PATCH] new chaindb. allow for real fork resolution. --- lib/bcoin/chain.js | 1651 +++++++++++++++++++++---------------- lib/bcoin/chainblock.js | 212 +++-- lib/bcoin/chaindb.js | 973 ++++++++-------------- lib/bcoin/compactblock.js | 7 +- lib/bcoin/lru.js | 17 + lib/bcoin/node.js | 3 + lib/bcoin/pool.js | 167 ++-- 7 files changed, 1535 insertions(+), 1495 deletions(-) diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index ee194597..816b3e78 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -33,7 +33,6 @@ function Chain(node, options) { bcoin.debug = this.options.debug; this.node = node; - this.request = new utils.RequestCache(); this.loading = false; this.mempool = node.mempool; this.blockdb = node.blockdb; @@ -49,6 +48,8 @@ function Chain(node, options) { this.invalid = {}; this.bestHeight = -1; this.lastUpdate = utils.now(); + this.tip = null; + this.height = -1; this.orphan = { map: {}, @@ -146,7 +147,7 @@ Chain.prototype._init = function _init() { utils.debug('Reason: %s', err.message); } - self.db.load(start || 0, function(err) { + self.db.load(function(err) { if (err) throw err; @@ -154,26 +155,23 @@ Chain.prototype._init = function _init() { if (err) throw err; - self.loading = false; - self.emit('load'); + self.db.getTip(function(err, tip) { + if (err) + throw err; + + assert(tip); + + self.tip = tip; + self.height = tip.height; + self.loading = false; + self.emit('load'); + }); }); }); }); }); }; -Chain.prototype.__defineGetter__('tip', function() { - return this.db.tip; -}); - -Chain.prototype.__defineGetter__('height', function() { - return this.db.height; -}); - -// Maybe do this: -// Chain.prototype._lock = function _lock(func, args, callback, force) { -// And return wrapped callback with an unlock call in it - Chain.prototype._lock = function _lock(func, args, force) { var self = this; var block, called; @@ -305,7 +303,7 @@ Chain.prototype._preload = function _preload(callback) { stream.on('error', function(err) { var start = Math.max(0, height - 2); - self.resetHeightAsync(start, function(e) { + self.resetHeight(start, function(e) { if (e) throw e; return callback(err, start + 1); @@ -344,7 +342,7 @@ Chain.prototype._preload = function _preload(callback) { if (lastEntry && entry.prevBlock !== lastEntry.hash) { start = Math.max(0, height - 2); stream.destroy(); - return self.resetHeightAsync(start, function(e) { + return self.resetHeight(start, function(e) { if (e) throw e; return callback(new Error('Corrupt headers.'), start + 1); @@ -356,7 +354,7 @@ Chain.prototype._preload = function _preload(callback) { if (!block.verifyHeaders()) { start = Math.max(0, height - 2); stream.destroy(); - return self.resetHeightAsync(start, function(e) { + return self.resetHeight(start, function(e) { if (e) throw e; return callback(new Error('Bad headers.'), start + 1); @@ -380,7 +378,7 @@ Chain.prototype._preload = function _preload(callback) { self.db._cache(entry); self.db._populate(entry); } else { - self.db.saveAsync(entry); + self.db.save(entry); } height++; @@ -415,183 +413,411 @@ Chain.prototype._removeBlock = function _removeBlock(tip, callback) { Chain.prototype._verifyContext = function _verifyContext(block, prev, callback) { var self = this; - var flags; - flags = this._verify(block, prev); - - if (flags === false) - return callback(null, false); - - this._checkDuplicates(block, prev, function(err, result) { + this._verify(block, prev, function(err, flags) { if (err) return callback(err); - if (!result) + if (flags === false) return callback(null, false); - self._checkInputs(block, prev, flags, function(err, result) { + self._checkDuplicates(block, prev, function(err, result) { if (err) return callback(err); if (!result) return callback(null, false); - return callback(null, true); + self._checkInputs(block, prev, flags, function(err, result) { + if (err) + return callback(err); + + if (!result) + return callback(null, false); + + return callback(null, true); + }); }); }); }; -Chain.prototype._verify = function _verify(block, prev) { +Chain.prototype._verify = function _verify(block, prev, callback) { + var self = this; var flags = constants.flags.MANDATORY_VERIFY_FLAGS; - var height, ts, i, tx, cb, coinbaseHeight, medianTime; - var locktimeMedian, segwit; + var height, ts, i, tx, cb, coinbaseHeight; + var locktimeMedian, segwit, check; + + function done(err, result) { + prev.free(); + callback(err, result); + } if (!block.verify()) - return false; + return done(null, false); // Skip the genesis block if (block.isGenesis()) - return flags; + return done(null, flags); // Ensure it's not an orphan if (!prev) { utils.debug('Block has no previous entry: %s', block.rhash); - return false; + return done(null, false); + } + + prev.alloc(function(err) { + if (err) + return callback(err); + + height = prev.height + 1; + medianTime = prev.getMedianTime(); + + // Ensure the timestamp is correct + if (block.ts <= medianTime) { + utils.debug('Block time is lower than median: %s', block.rhash); + return done(null, false); + } + + if (block.bits !== self.getTarget(prev, block)) { + utils.debug('Block is using wrong target: %s', block.rhash); + return done(null, false); + } + + // For some reason bitcoind has p2sh in the + // mandatory flags by default, when in reality + // it wasn't activated until march 30th 2012. + // The first p2sh output and redeem script + // appeared on march 7th 2012, only it did + // not have a signature. See: + // https://blockchain.info/tx/6a26d2ecb67f27d1fa5524763b49029d7106e91e3cc05743073461a719776192 + // https://blockchain.info/tx/9c08a4d78931342b37fd5f72900fb9983087e6f46c4a097d8a1f52c74e28eaf6 + if (block.ts < constants.block.bip16time) + flags &= ~constants.flags.VERIFY_P2SH; + + // Only allow version 2 blocks (coinbase height) + // once the majority of blocks are using it. + if (block.version < 2 && prev.isOutdated(2)) { + utils.debug('Block is outdated (v2): %s', block.rhash); + return done(null, false); + } + + // Only allow version 3 blocks (sig validation) + // once the majority of blocks are using it. + if (block.version < 3 && prev.isOutdated(3)) { + utils.debug('Block is outdated (v3): %s', block.rhash); + return done(null, false); + } + + // Only allow version 4 blocks (checklocktimeverify) + // once the majority of blocks are using it. + if (block.version < 4 && prev.isOutdated(4)) { + utils.debug('Block is outdated (v4): %s', block.rhash); + return done(null, false); + } + + // Only allow version 5 blocks (segwit) + // once the majority of blocks are using it. + if (network.segwitHeight !== -1 && height >= network.segwitHeight) { + if (block.version < 5 && prev.isOutdated(5)) { + utils.debug('Block is outdated (v5): %s', block.rhash); + return done(null, false); + } + } + + // Only allow version 8 blocks (locktime median past) + // once the majority of blocks are using it. + // if (block.version < 8 && prev.isOutdated(8)) { + // utils.debug('Block is outdated (v8): %s', block.rhash); + // return false); + // } + + // Make sure the height contained in the coinbase is correct. + if (network.block.bip34height !== -1 && height >= network.block.bip34height) { + if (block.version >= 2 && prev.isUpgraded(2)) + coinbaseHeight = true; + } + + // Signature validation is now enforced (bip66) + if (block.version >= 3 && prev.isUpgraded(3)) + flags |= constants.flags.VERIFY_DERSIG; + + // CHECKLOCKTIMEVERIFY is now usable (bip65) + if (block.version >= 4 && prev.isUpgraded(4)) + flags |= constants.flags.VERIFY_CHECKLOCKTIMEVERIFY; + + // Segregrated witness is now usable (the-bip-that-really-needs-to-be-rewritten) + if (network.segwitHeight !== -1 && height >= network.segwitHeight) { + if (block.version >= 5 && prev.isUpgraded(5) ) { + flags |= constants.flags.VERIFY_WITNESS; + segwit = true; + } + } + + // Can't verify any further when merkleblock or headers. + if (block.type !== 'block') + return done(null, flags); + + // Make sure the height contained in the coinbase is correct. + if (coinbaseHeight) { + if (block.getCoinbaseHeight() !== height) { + utils.debug('Block has bad coinbase height: %s', block.rhash); + return done(null, false); + } + } + + if (block.version >= 5 && segwit) { + if (block.commitmentHash !== block.getCommitmentHash()) { + utils.debug('Block failed witnessroot test: %s', block.rhash); + return done(null, false); + } + } else { + if (block.hasWitness()) { + utils.debug('Unexpected witness data found: %s', block.rhash); + return done(null, false); + } + } + + // Get timestamp for tx.isFinal(). + ts = locktimeMedian ? medianTime : block.ts; + + // Check all transactions + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + + // Transactions must be finalized with + // regards to nSequence and nLockTime. + if (!tx.isFinal(height, ts)) { + utils.debug('TX is not final: %s (%s)', block.rhash, i); + return done(null, false); + } + } + + return done(null, flags); + }); +}; + +Chain.prototype._verify2 = function _verify(block, prev, callback) { + var self = this; + var flags = constants.flags.MANDATORY_VERIFY_FLAGS; + var height, ts, i, tx, cb, coinbaseHeight; + var locktimeMedian, segwit, check; + + if (!block.verify()) + return callback(null, false); + + // Skip the genesis block + if (block.isGenesis()) + return callback(null, flags); + + // Ensure it's not an orphan + if (!prev) { + utils.debug('Block has no previous entry: %s', block.rhash); + return callback(null, false); } height = prev.height + 1; - medianTime = prev.getMedianTime(); + prev.getMedianTime(function(err, medianTime) { + if (err) + return callback(err); - // Ensure the timestamp is correct - if (block.ts <= medianTime) { - utils.debug('Block time is lower than median: %s', block.rhash); - return false; - } - - // Ensure the miner's target is equal to what we expect - if (block.bits !== this.getTarget(prev, block)) { - utils.debug('Block is using wrong target: %s', block.rhash); - return false; - } - - // For some reason bitcoind has p2sh in the - // mandatory flags by default, when in reality - // it wasn't activated until march 30th 2012. - // The first p2sh output and redeem script - // appeared on march 7th 2012, only it did - // not have a signature. See: - // https://blockchain.info/tx/6a26d2ecb67f27d1fa5524763b49029d7106e91e3cc05743073461a719776192 - // https://blockchain.info/tx/9c08a4d78931342b37fd5f72900fb9983087e6f46c4a097d8a1f52c74e28eaf6 - if (block.ts < constants.block.bip16time) - flags &= ~constants.flags.VERIFY_P2SH; - - // Only allow version 2 blocks (coinbase height) - // once the majority of blocks are using it. - if (block.version < 2 && prev.isOutdated(2)) { - utils.debug('Block is outdated (v2): %s', block.rhash); - return false; - } - - // Only allow version 3 blocks (sig validation) - // once the majority of blocks are using it. - if (block.version < 3 && prev.isOutdated(3)) { - utils.debug('Block is outdated (v3): %s', block.rhash); - return false; - } - - // Only allow version 4 blocks (checklocktimeverify) - // once the majority of blocks are using it. - if (block.version < 4 && prev.isOutdated(4)) { - utils.debug('Block is outdated (v4): %s', block.rhash); - return false; - } - - // Only allow version 5 blocks (segwit) - // once the majority of blocks are using it. - if (height >= network.segwitHeight) { - if (block.version < 5 && prev.isOutdated(5)) { - utils.debug('Block is outdated (v5): %s', block.rhash); - return false; + // Ensure the timestamp is correct + if (block.ts <= medianTime) { + utils.debug('Block time is lower than median: %s', block.rhash); + return callback(null, false); } - } - // Only allow version 8 blocks (locktime median past) - // once the majority of blocks are using it. - // if (block.version < 8 && prev.isOutdated(8)) { - // utils.debug('Block is outdated (v8): %s', block.rhash); - // return false; - // } + // Ensure the miner's target is equal to what we expect + self.getTarget(prev, block, function(err, target) { + if (err) + return callback(err); - // Make sure the height contained in the coinbase is correct. - if (network.block.bip34height !== -1 && height >= network.block.bip34height) { - if (block.version >= 2 && prev.isUpgraded(2)) - coinbaseHeight = true; - } + if (block.bits !== target) { + utils.debug('Block is using wrong target: %s', block.rhash); + return callback(null, false); + } - // Signature validation is now enforced (bip66) - if (block.version >= 3 && prev.isUpgraded(3)) - flags |= constants.flags.VERIFY_DERSIG; + // For some reason bitcoind has p2sh in the + // mandatory flags by default, when in reality + // it wasn't activated until march 30th 2012. + // The first p2sh output and redeem script + // appeared on march 7th 2012, only it did + // not have a signature. See: + // https://blockchain.info/tx/6a26d2ecb67f27d1fa5524763b49029d7106e91e3cc05743073461a719776192 + // https://blockchain.info/tx/9c08a4d78931342b37fd5f72900fb9983087e6f46c4a097d8a1f52c74e28eaf6 + if (block.ts < constants.block.bip16time) + flags &= ~constants.flags.VERIFY_P2SH; - // CHECKLOCKTIMEVERIFY is now usable (bip65) - if (block.version >= 4 && prev.isUpgraded(4)) - flags |= constants.flags.VERIFY_CHECKLOCKTIMEVERIFY; + check = []; - // Segregrated witness is now usable (the-bip-that-really-needs-to-be-rewritten) - if (network.segwitHeight !== -1 && height >= network.segwitHeight) { - if (block.version >= 5 && prev.isUpgraded(5) ) { - flags |= constants.flags.VERIFY_WITNESS; - segwit = true; - } - } + // Only allow version 2 blocks (coinbase height) + // once the majority of blocks are using it. + if (block.version < 2) { + check.push([2, function(result) { + if (result) + utils.debug('Block is outdated (v2): %s', block.rhash); + }]); + } - // Use nLockTime median past (bip113) - // https://github.com/btcdrak/bips/blob/d4c9a236ecb947866c61aefb868b284498489c2b/bip-0113.mediawiki - // Support version bits: - // https://gist.github.com/sipa/bf69659f43e763540550 - // http://lists.linuxfoundation.org/pipermail/bitcoin-dev/2015-August/010396.html - // if (block.version >= 8 && prev.isUpgraded(8)) - // locktimeMedian = true; + // Only allow version 3 blocks (sig validation) + // once the majority of blocks are using it. + if (block.version < 3) { + check.push([3, function(result) { + if (result) + utils.debug('Block is outdated (v3): %s', block.rhash); + }]); + } - // Can't verify any further when merkleblock or headers. - if (block.type !== 'block') - return flags; + // Only allow version 4 blocks (checklocktimeverify) + // once the majority of blocks are using it. + if (block.version < 4) { + check.push([4, function(result) { + if (result) + utils.debug('Block is outdated (v4): %s', block.rhash); + }]); + } - // Make sure the height contained in the coinbase is correct. - if (coinbaseHeight) { - if (block.getCoinbaseHeight() !== height) { - utils.debug('Block has bad coinbase height: %s', block.rhash); - return false; - } - } + // Only allow version 5 blocks (segwit) + // once the majority of blocks are using it. + if (network.segwitHeight !== -1 && height >= network.segwitHeight) { + if (block.version < 5) { + check.push([5, function(result) { + if (result) + utils.debug('Block is outdated (v5): %s', block.rhash); + }]); + } + } - if (block.version >= 5 && segwit) { - if (block.commitmentHash !== block.getCommitmentHash()) { - utils.debug('Block failed witnessroot test: %s', block.rhash); - return false; - } - } else { - if (block.hasWitness()) { - utils.debug('Unexpected witness data found: %s', block.rhash); - return false; - } - } + // Only allow version 8 blocks (locktime median past) + // once the majority of blocks are using it. + // if (block.version < 8 && prev.isOutdated(8)) { + // utils.debug('Block is outdated (v8): %s', block.rhash); + // return false; + // } - // Get timestamp for tx.isFinal(). - ts = locktimeMedian ? medianTime : block.ts; + check = []; + utils.forEachSerial(check, function(item, next) { + var version = item[0]; + var cb = item[1]; + prev.isOutdated(version, function(err, result) { + if (err) + return next(err); - // Check all transactions - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; + cb(result); - // Transactions must be finalized with - // regards to nSequence and nLockTime. - if (!tx.isFinal(height, ts)) { - utils.debug('TX is not final: %s (%s)', block.rhash, i); - return false; - } - } + if (!result) + return callback(null, false); - return flags; + next(); + }); + }, function(err) { + if (err) + return callback(err); + + check = []; + + // Make sure the height contained in the coinbase is correct. + if (network.block.bip34height !== -1 && height >= network.block.bip34height) { + check.push([2, function(result) { + if (result) + coinbaseHeight = true; + }]); + } + + // Signature validation is now enforced (bip66) + if (block.version >= 3) { + check.push([3, function(result) { + if (result) + flags |= constants.flags.VERIFY_DERSIG; + }]); + } + + // CHECKLOCKTIMEVERIFY is now usable (bip65) + if (block.version >= 4) { + check.push([4, function(result) { + if (result) + flags |= constants.flags.VERIFY_CHECKLOCKTIMEVERIFY; + }]); + } + + // Segregrated witness is now usable (the-bip-that-really-needs-to-be-rewritten) + if (network.segwitHeight !== -1 && height >= network.segwitHeight) { + if (block.version >= 5) { + check.push([5, function(result) { + if (result) { + flags |= constants.flags.VERIFY_WITNESS; + segwit = true; + } + }]); + } + } + + segwit = true; + check = []; + // Use nLockTime median past (bip113) + // https://github.com/btcdrak/bips/blob/d4c9a236ecb947866c61aefb868b284498489c2b/bip-0113.mediawiki + // Support version bits: + // https://gist.github.com/sipa/bf69659f43e763540550 + // http://lists.linuxfoundation.org/pipermail/bitcoin-dev/2015-August/010396.html + // if (block.version >= 8 && prev.isUpgraded(8)) + // locktimeMedian = true; + + utils.forEachSerial(check, function(item, next) { + var version = item[0]; + var cb = item[1]; + prev.isUpgraded(version, function(err, result) { + if (err) + return next(err); + cb(result); + next(); + }); + }, function(err) { + if (err) + return callback(err); + + // Can't verify any further when merkleblock or headers. + if (block.type !== 'block') + return callback(null, flags); + + // Make sure the height contained in the coinbase is correct. + if (coinbaseHeight) { + if (block.getCoinbaseHeight() !== height) { + utils.debug('Block has bad coinbase height: %s', block.rhash); + return callback(null, false); + } + } + + if (block.version >= 5 && segwit) { + if (block.commitmentHash !== block.getCommitmentHash()) { + utils.debug('Block failed witnessroot test: %s', block.rhash); + return callback(null, false); + } + } else { + if (block.hasWitness()) { + utils.debug('Unexpected witness data found: %s', block.rhash); + return callback(null, false); + } + } + + // Get timestamp for tx.isFinal(). + ts = locktimeMedian ? medianTime : block.ts; + + // Check all transactions + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + + // Transactions must be finalized with + // regards to nSequence and nLockTime. + if (!tx.isFinal(height, ts)) { + utils.debug('TX is not final: %s (%s)', block.rhash, i); + return callback(null, false); + } + } + + return callback(null, flags); + }); + }); + }); + }); }; Chain.prototype._checkDuplicates = function _checkDuplicates(block, prev, callback) { @@ -766,60 +992,146 @@ Chain.prototype._fillBlock = function _fillBlock(block, callback) { }); }; -Chain.prototype._addEntry = function _addEntry(entry, block, callback) { +Chain.prototype.getHeight = function getHeight(hash) { + if (Buffer.isBuffer(hash)) + hash = utils.toHex(hash); + else if (hash.hash) + hash = hash.hash('hex'); + + if (this.db.hasCache(hash)) + return this.db.getCache(hash).height; + + return -1; +}; + +Chain.prototype._findFork = function _findFork(fork, longer, callback) { + (function find() { + if (fork.hash === longer.hash) + return callback(null, fork); + + (function next() { + if (longer.height <= fork.height) + return done(); + + self.db.get(longer.prevBlock, function(err, entry) { + if (err) + return callback(err); + + if (!entry) + return callback(new Error('No previous entry for new tip.')); + + longer = entry; + + next(); + }); + })(); + + function done() { + if (fork.hash === longer.hash) + return callback(null, fork); + + self.db.get(fork.prevBlock, function(err, entry) { + if (err) + return callback(err); + + if (!entry) + return callback(new Error('No previous entry for old tip.')); + + fork = entry; + + find(); + }); + } + })(); +}; + +Chain.prototype._reorganize = function _reorganize(entry, block, callback) { var self = this; - var existing, now; - callback = utils.asyncify(callback); - - // Already added - if (this.db.has(entry.height)) { - assert(this.db.getHeight(entry.hash) === entry.height); - return callback(null, false); - } - - // Duplicate height (do a sync call here since this is cached) - existing = this.db.getSync(entry.height); - if (existing && existing.hash === entry.hash) - return callback(null, false); - - now = utils.now(); - this.lastUpdate = now; - - this._saveBlock(block, function(err) { + function done(err) { if (err) return callback(err); - self.db.saveAsync(entry, function(err) { - if (err) - return callback(err); + self.emit('fork', block, { + height: entry.height, + expected: self.tip.hash, + received: entry.hash, + checkpoint: false + }); - return callback(null, true); + return callback(); + } + + return this._findFork(this.tip, entry, function(err, fork) { + if (err) + return done(err); + + assert(fork); + + self.db.resetHeight(fork.height, function(err) { + if (err) + return done(err); + + if (!self.blockdb) + return done(); + + self.blockdb.resetHeight(fork.height, function(err) { + if (err) + return done(err); + + return done(); + }, function(block) { + self.emit('remove block', block); + }); + }, function(entry) { + self.emit('remove entry', entry); }); }); }; -Chain.prototype.resetHeight = function resetHeight(height, force) { +Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { var self = this; - if (height === this.db.getSize() - 1) - return; + callback = utils.asyncify(callback); - this.db.resetHeightSync(height, function(entry) { - self.emit('remove entry', entry); - }); + this.lastUpdate = utils.now(); - // Reset the orphan map completely. There may - // have been some orphans on a forked chain we - // no longer need. - this.purgeOrphans(); - this.purgePending(); + if (!this.tip) { + if (entry.hash !== network.genesis.hash) + return callback(new Error('Bad genesis block.')); + + done(); + } else if (entry.prevBlock === this.tip.hash) { + done(); + } else { + self._reorganize(entry, block, done); + } + + function done(err) { + if (err) + return callback(err); + + self._saveBlock(block, function(err) { + if (err) + return callback(err); + + self.db.save(entry, function(err) { + if (err) + return callback(err); + + self.tip = entry; + self.height = entry.height; + + return callback(); + }); + }); + } }; -Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback, force) { +Chain.prototype.resetHeight = function resetHeight(height, callback, force) { var self = this; - var unlock = this._lock(resetHeightAsync, [height, callback], force); + var unlock = this._lock(resetHeight, [height, callback], force); if (!unlock) return; @@ -829,10 +1141,7 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback, f callback(err, result); } - if (height === this.db.getSize() - 1) - return utils.nextTick(done); - - this.db.resetHeightAsync(height, function(err) { + this.db.resetHeight(height, function(err) { if (err) return done(err); @@ -863,47 +1172,50 @@ Chain.prototype.revertHeight = function revertHeight(height, callback, force) { callback(err, result); } - chainHeight = this.db.getSize() - 1; - - if (chainHeight < 0) - return done(new Error('Bad chain height.')); - - if (chainHeight < height) - return done(new Error('Cannot reset height.')); - - if (chainHeight === height) - return done(); - - this.resetHeightAsync(height, function(err) { + this.db.getChainHeight(function(err, chainHeight) { if (err) return done(err); - if (!self.blockdb) + if (chainHeight == null || chainHeight < 0) + return done(new Error('Bad chain height.')); + + if (chainHeight < height) + return done(new Error('Cannot reset height.')); + + if (chainHeight === height) return done(); - self.blockdb.getHeight(function(err, blockHeight) { + this.resetHeight(height, function(err) { if (err) return done(err); - if (blockHeight < 0) - return done(new Error('Bad block height.')); - - if (blockHeight < height) - return done(new Error('Cannot reset height.')); - - if (blockHeight === height) + if (!self.blockdb) return done(); - self.blockdb.resetHeight(height, function(err) { + self.blockdb.getHeight(function(err, blockHeight) { if (err) return done(err); - return done(); - }, function(block) { - self.emit('remove block', block); + if (blockHeight < 0) + return done(new Error('Bad block height.')); + + if (blockHeight < height) + return done(new Error('Cannot reset height.')); + + if (blockHeight === height) + return done(); + + self.blockdb.resetHeight(height, function(err) { + if (err) + return done(err); + + return done(); + }, function(block) { + self.emit('remove block', block); + }); }); - }); - }, true); + }, true); + }); }; Chain.prototype._revertLast = function _revertLast(existing, callback, force) { @@ -918,7 +1230,7 @@ Chain.prototype._revertLast = function _revertLast(existing, callback, force) { callback(err, result); } - this.resetHeightAsync(existing.height - 1, function(err) { + this.resetHeight(existing.height - 1, function(err) { if (err) return done(err); @@ -949,42 +1261,45 @@ Chain.prototype.syncHeight = function syncHeight(callback, force) { callback(err, result); } - chainHeight = this.db.getSize() - 1; - - if (chainHeight < 0) - return done(new Error('Bad chain height.')); - - if (!this.blockdb) - return done(); - - this.blockdb.getHeight(function(err, blockHeight) { + this.db.getChainHeight(function(err, chainHeight) { if (err) return done(err); - if (blockHeight < 0) - return done(new Error('Bad block height.')); + if (chainHeight == null || chainHeight < 0) + return done(new Error('Bad chain height.')); - if (blockHeight === chainHeight) + if (!this.blockdb) return done(); - utils.debug('ChainDB and BlockDB are out of sync.'); + this.blockdb.getHeight(function(err, blockHeight) { + if (err) + return done(err); - if (blockHeight < chainHeight) { - utils.debug('ChainDB is higher than BlockDB. Syncing...'); - return self.resetHeightAsync(blockHeight, done, true); - } - - if (blockHeight > chainHeight) { - utils.debug('BlockDB is higher than ChainDB. Syncing...'); - self.blockdb.resetHeight(chainHeight, function(err) { - if (err) - return done(err); + if (blockHeight < 0) + return done(new Error('Bad block height.')); + if (blockHeight === chainHeight) return done(); - }, function(block) { - self.emit('remove block', block); - }); - } + + utils.debug('ChainDB and BlockDB are out of sync.'); + + if (blockHeight < chainHeight) { + utils.debug('ChainDB is higher than BlockDB. Syncing...'); + return self.resetHeight(blockHeight, done, true); + } + + if (blockHeight > chainHeight) { + utils.debug('BlockDB is higher than ChainDB. Syncing...'); + self.blockdb.resetHeight(chainHeight, function(err) { + if (err) + return done(err); + + return done(); + }, function(block) { + self.emit('remove block', block); + }); + } + }); }); }; @@ -995,14 +1310,14 @@ Chain.prototype.resetTime = function resetTime(ts) { return this.resetHeight(entry.height); }; -Chain.prototype.resetTimeAsync = function resetTimeAsync(ts, callback, force) { +Chain.prototype.resetTime = function resetTime(ts, callback, force) { var self = this; - var unlock = this._lock(resetTimeAsync, [ts, callback], force); + var unlock = this._lock(resetTime, [ts, callback], force); if (!unlock) return; - this.byTimeAsync(ts, function(err, entry) { + this.byTime(ts, function(err, entry) { if (err) { unlock(); if (callback) @@ -1017,7 +1332,7 @@ Chain.prototype.resetTimeAsync = function resetTimeAsync(ts, callback, force) { return; } - self.resetHeightAsync(entry.height, function(err) { + self.resetHeight(entry.height, function(err) { unlock(); if (callback) callback(err); @@ -1045,308 +1360,242 @@ Chain.prototype.add = function add(initial, peer, callback, force) { (function next(block) { var hash = block.hash('hex'); var prevHash = block.prevBlock; - var prevHeight, height, checkpoint, prev, orphan; - - // Find the previous block height/index. - prevHeight = self.db.getHeight(prevHash); - height = prevHeight === -1 ? -1 : prevHeight + 1; + var height, checkpoint, prev, orphan; // We already have this block. - if (self.db.has(hash) || self.hasPending(hash)) { - self.emit('exists', block, { - height: height, - hash: hash - }, peer); - return done(); - } - - // Do not revalidate known invalid blocks. - if (self.invalid[hash] || self.invalid[prevHash]) { - self.emit('invalid', block, { - height: height, - hash: hash, - seen: !!self.invalid[hash], - chain: !!self.invalid[prevHash] - }, peer); - self.invalid[hash] = true; - return done(); - } - - // Validate the block we want to add. - // This is only necessary for new - // blocks coming in, not the resolving - // orphans. - if (block === initial && !block.verify()) { - self.invalid[hash] = true; - self.emit('invalid', block, { - height: height, - hash: hash, - seen: false, - chain: false - }, peer); - return done(); - } - - // Special case for genesis block. - if (block.isGenesis()) - return done(); - - // If the block is already known to be - // an orphan, ignore it. - orphan = self.orphan.map[prevHash]; - if (orphan) { - // If the orphan chain forked, simply - // reset the orphans and find a new peer. - if (orphan.hash('hex') !== hash) { - self.purgeOrphans(); - self.purgePending(); - - self.emit('fork', block, { - height: -1, - expected: orphan.hash('hex'), - received: hash, - checkpoint: false - }, peer); - - return done(); - } - - self.emit('orphan', block, { - height: -1, - hash: hash, - seen: true - }, peer); - - return done(); - } - - // Update the best height based on the coinbase. - // We do this even for orphans (peers will send - // us their highest block during the initial - // getblocks sync, making it an orphan). - if (block.getCoinbaseHeight() > self.bestHeight) - self.bestHeight = block.getCoinbaseHeight(); - - // If previous block wasn't ever seen, - // add it current to orphans and break. - if (prevHeight === -1) { - self.orphan.count++; - self.orphan.size += block.getSize(); - self.orphan.map[prevHash] = block; - self.orphan.bmap[hash] = block; - self.emit('orphan', block, { - height: -1, - hash: hash, - seen: false - }, peer); - return done(); - } - - // Verify the checkpoint. - checkpoint = network.checkpoints[height]; - if (checkpoint) { - self.emit('checkpoint', block, { - height: height, - hash: hash, - checkpoint: checkpoint - }, peer); - - // Block did not match the checkpoint. The - // chain could be reset to the last sane - // checkpoint, but it really isn't necessary, - // so we don't do it. The misbehaving peer has - // been killed and hopefully we find a peer - // who isn't trying to fool us. - if (hash !== checkpoint) { - self.purgeOrphans(); - self.purgePending(); - - self.emit('fork', block, { - height: height, - expected: checkpoint, - received: hash, - checkpoint: true - }, peer); - - return done(); - } - } - - // Lookup previous entry. - // We can do this synchronously: - // This will be cached in 99.9% of cases. - if (!self.db.isCached(prevHeight)) - utils.debug('Warning: height %d is not cached.', prevHeight); - - try { - prev = self.db.getSync(prevHeight); - } catch (e) { - return done(e); - } - - assert(prev); - - // Explanation: we try to keep as much data - // off the javascript heap as possible. Blocks - // in the future may be 8mb or 20mb, who knows. - // In fullnode-mode we store the blocks in - // "compact" form (the headers plus the raw - // Buffer object) until they're ready to be - // fully validated here. They are deserialized, - // validated, and emitted. Hopefully the deserialized - // blocks get cleaned up by the GC quickly. - if (block.type === 'compactblock') { - try { - block = block.toBlock(); - } catch (e) { - // Ugly hack to handle - // the error properly. - peer.parser.emit('error', e); - return done(e); - } - } - - // Do "contextual" verification on our block - // now that we're certain its previous - // block is in the chain. - self._verifyContext(block, prev, function(err, verified) { - var entry, existing; - + self.db.has(hash, function(err, existing) { if (err) return done(err); - if (!verified) { - self.invalid[hash] = true; - self.emit('invalid', block, { - height: height, - hash: hash, - seen: false, - chain: false + if (existing || self.hasPending(hash)) { + self.emit('exists', block, { + height: -1, + hash: hash }, peer); return done(); } - // Create a new chain entry. - entry = new bcoin.chainblock(self, { - hash: hash, - version: block.version, - prevBlock: block.prevBlock, - merkleRoot: block.merkleRoot, - ts: block.ts, - bits: block.bits, - nonce: block.nonce, - height: height - }); - - // Real fork resolution would just be this. - // if (entry.chainwork.cmp(self.tip.chainwork) > 0) - // return self.setBestChain(entry); - // return done(); - - // See if the height already exists (fork). - // Do this synchronously: This will - // be cached in 99.9% of cases. - if (self.db.has(entry.height)) { - if (!self.db.isCached(entry.height)) - utils.debug('Warning: height %d is not cached.', entry.height); - - try { - existing = self.db.getSync(entry.height); - } catch (e) { - return done(e); - } - - // Shouldn't be the same by this point. - assert(existing.hash !== entry.hash); - - // A valid block with an already existing - // height came in, that spells fork. We - // don't store by hash so we can't compare - // chainworks. We reset the chain, find a - // new peer, and wait to see who wins. - assert(self.db.getHeight(entry.hash) === -1); - - // The tip has more chainwork, it is a - // higher height than the entry. This is - // not an alternate tip. Ignore it. - if (existing.chainwork.cmp(entry.chainwork) > 0) - return done(); - - // The block has equal chainwork (an - // alternate tip). Reset the chain, find - // a new peer, and wait to see who wins. - // return self.revertHeight(existing.height - 1, function(err) { - return self._revertLast(existing, function(err, existingBlock) { - if (err) - return done(err); - - self.emit('fork', block, { - height: existing.height, - expected: existing.hash, - received: entry.hash, - checkpoint: false - }, peer); - - return done(); - }, true); - } - - // Add entry if we do not have it. - assert(self.db.getHeight(entry.hash) === -1); - - // Update the block height - block.height = entry.height; - block.txs.forEach(function(tx) { - tx.height = entry.height; - }); - - // Attempt to add block to the chain index. - self._addEntry(entry, block, function(err, success) { + // Find the previous block height/index. + self.db.get(prevHash, function(err, prev) { if (err) return done(err); - // Result should never be `unchanged` since - // we already verified there were no - // duplicate heights, etc. - assert(success === true); + height = !prev ? -1 : prev.height + 1; - // Keep track of the number of blocks we - // added and the number of orphans resolved. - total++; + // Do not revalidate known invalid blocks. + if (self.invalid[hash] || self.invalid[prevHash]) { + self.emit('invalid', block, { + height: height, + hash: hash, + seen: !!self.invalid[hash], + chain: !!self.invalid[prevHash] + }, peer); + self.invalid[hash] = true; + return done(); + } - // Emit our block (and potentially resolved - // orphan) so the programmer can save it. - self.emit('block', block, entry, peer); - if (block.hash('hex') !== initial.hash('hex')) - self.emit('resolved', block, entry, peer); + // Validate the block we want to add. + // This is only necessary for new + // blocks coming in, not the resolving + // orphans. + if (block === initial && !block.verify()) { + self.invalid[hash] = true; + self.emit('invalid', block, { + height: height, + hash: hash, + seen: false, + chain: false + }, peer); + return done(); + } - self.emit('add block', block); + // Special case for genesis block. + if (block.isGenesis()) + return done(); - // Fulfill request - self.request.fulfill(hash, block); + // If the block is already known to be + // an orphan, ignore it. + orphan = self.orphan.map[prevHash]; + if (orphan) { + // If the orphan chain forked, simply + // reset the orphans. + if (orphan.hash('hex') !== hash) { + self.purgeOrphans(); + self.purgePending(); - handleOrphans(); + self.emit('fork', block, { + height: -1, + expected: orphan.hash('hex'), + received: hash, + checkpoint: false + }, peer); + + return done(); + } + + self.emit('orphan', block, { + height: -1, + hash: hash, + seen: true + }, peer); + + return done(); + } + + // Update the best height based on the coinbase. + // We do this even for orphans (peers will send + // us their highest block during the initial + // getblocks sync, making it an orphan). + if (block.getCoinbaseHeight() > self.bestHeight) + self.bestHeight = block.getCoinbaseHeight(); + + // If previous block wasn't ever seen, + // add it current to orphans and break. + if (!prev) { + self.orphan.count++; + self.orphan.size += block.getSize(); + self.orphan.map[prevHash] = block; + self.orphan.bmap[hash] = block; + self.emit('orphan', block, { + height: -1, + hash: hash, + seen: false + }, peer); + return done(); + } + + // Verify the checkpoint. + checkpoint = network.checkpoints[height]; + if (checkpoint) { + self.emit('checkpoint', block, { + height: height, + hash: hash, + checkpoint: checkpoint + }, peer); + + // Block did not match the checkpoint. The + // chain could be reset to the last sane + // checkpoint, but it really isn't necessary, + // so we don't do it. The misbehaving peer has + // been killed and hopefully we find a peer + // who isn't trying to fool us. + if (hash !== checkpoint) { + self.purgeOrphans(); + self.purgePending(); + + self.emit('fork', block, { + height: height, + expected: checkpoint, + received: hash, + checkpoint: true + }, peer); + + return done(); + } + } + + assert(prev); + + // Explanation: we try to keep as much data + // off the javascript heap as possible. Blocks + // in the future may be 8mb or 20mb, who knows. + // In fullnode-mode we store the blocks in + // "compact" form (the headers plus the raw + // Buffer object) until they're ready to be + // fully validated here. They are deserialized, + // validated, and emitted. Hopefully the deserialized + // blocks get cleaned up by the GC quickly. + if (block.type === 'compactblock') { + try { + block = block.toBlock(); + } catch (e) { + // Ugly hack to handle + // the error properly. + peer.parser.emit('error', e); + return done(e); + } + } + + // Do "contextual" verification on our block + // now that we're certain its previous + // block is in the chain. + self._verifyContext(block, prev, function(err, verified) { + var entry; + + if (err) + return done(err); + + if (!verified) { + self.invalid[hash] = true; + self.emit('invalid', block, { + height: height, + hash: hash, + seen: false, + chain: false + }, peer); + return done(); + } + + // Create a new chain entry. + entry = new bcoin.chainblock(self, { + hash: hash, + version: block.version, + prevBlock: block.prevBlock, + merkleRoot: block.merkleRoot, + ts: block.ts, + bits: block.bits, + nonce: block.nonce, + height: height + }, prev); + + // Set main chain only if chainwork is higher. + if (entry.chainwork.cmp(self.tip.chainwork) <= 0) + return done(); + + // Add entry if we do not have it. + + // Update the block height + block.height = entry.height; + block.txs.forEach(function(tx) { + tx.height = entry.height; + }); + + // Attempt to add block to the chain index. + self._setBestChain(entry, block, function(err) { + if (err) + return done(err); + + // Keep track of the number of blocks we + // added and the number of orphans resolved. + total++; + + // Emit our block (and potentially resolved + // orphan) so the programmer can save it. + self.emit('block', block, entry, peer); + if (block.hash('hex') !== initial.hash('hex')) + self.emit('resolved', block, entry, peer); + + self.emit('add block', block); + + // No orphan chain. + if (!self.orphan.map[hash]) + return done(); + + // An orphan chain was found, start resolving. + block = self.orphan.map[hash]; + delete self.orphan.bmap[block.hash('hex')]; + delete self.orphan.map[hash]; + self.orphan.count--; + self.orphan.size -= block.getSize(); + + next(block); + }); + }); }); }); - - function handleOrphans() { - if (!self.orphan.map[hash]) - return done(); - - // An orphan chain was found, start resolving. - block = self.orphan.map[hash]; - delete self.orphan.bmap[block.hash('hex')]; - delete self.orphan.map[hash]; - self.orphan.count--; - self.orphan.size -= block.getSize(); - - next(block); - } })(initial); function done(err) { - var item; - // Failsafe for large orphan chains. Do not // allow more than 20mb stored in memory. if (self.orphan.size > self.orphanLimit) @@ -1440,55 +1689,23 @@ Chain.prototype.purgePending = function purgePending() { }); }; -Chain.prototype.has = function has(hash) { - if (this.hasBlock(hash)) - return true; - +Chain.prototype.has = function has(hash, callback) { if (this.hasOrphan(hash)) - return true; + return callback(null, true); if (this.hasPending(hash)) - return true; + return callback(null, true); - return false; + return this.hasBlock(hash, callback); }; -Chain.prototype.byTime = function byTime(ts) { - var start = 0; - var end = this.height + 1; - var pos, delta, entry; - - if (ts >= this.tip.ts) - return this.tip; - - // Do a binary search for a block - // mined within an hour of the - // timestamp. - while (start < end) { - pos = (start + end) >> 1; - entry = this.db.getSync(pos); - delta = Math.abs(ts - entry.ts); - - if (delta <= 60 * 60) - return entry; - - if (ts < entry.ts) { - end = pos; - } else { - start = pos + 1; - } - } - - return this.db.getSync(start); -}; - -Chain.prototype.byTimeAsync = function byTimeAsync(ts, callback, force) { +Chain.prototype.byTime = function byTime(ts, callback, force) { var self = this; var start = 0; var end = this.height + 1; var pos, delta; - var unlock = this._lock(byTimeAsync, [ts, callback], force); + var unlock = this._lock(byTime, [ts, callback], force); if (!unlock) return; @@ -1505,7 +1722,7 @@ Chain.prototype.byTimeAsync = function byTimeAsync(ts, callback, force) { return callback(null, result); } - self.db.getAsync(start, function(err, entry) { + self.db.get(start, function(err, entry) { unlock(); callback(err, entry); }); @@ -1523,7 +1740,7 @@ Chain.prototype.byTimeAsync = function byTimeAsync(ts, callback, force) { pos = (start + end) >> 1; - self.db.getAsync(pos, function(err, entry) { + self.db.get(pos, function(err, entry) { if (err) return done(err); @@ -1543,16 +1760,16 @@ Chain.prototype.byTimeAsync = function byTimeAsync(ts, callback, force) { })(); }; -Chain.prototype.hasBlock = function hasBlock(hash) { +Chain.prototype.hasBlock = function hasBlock(hash, callback) { if (typeof hash === 'number') - return this.db.has(hash); + return this.db.has(hash, callback); if (Buffer.isBuffer(hash)) hash = utils.toHex(hash); else if (hash.hash) hash = hash.hash('hex'); - return this.db.has(hash); + return this.db.has(hash, callback); }; Chain.prototype.hasOrphan = function hasOrphan(hash) { @@ -1568,28 +1785,16 @@ Chain.prototype.hasPending = function hasPending(hash) { return !!this.pendingBlocks[hash]; }; -Chain.prototype.getEntry = function getEntry(hash) { +Chain.prototype.getEntry = function getEntry(hash, callback) { if (typeof hash === 'number') - return this.db.getSync(hash); + return this.db.get(hash, callback); if (Buffer.isBuffer(hash)) hash = utils.toHex(hash); else if (hash.hash) hash = hash.hash('hex'); - return this.db.getSync(hash); -}; - -Chain.prototype.getEntryAsync = function getEntryAsync(hash, callback) { - if (typeof hash === 'number') - return this.db.getAsync(hash, callback); - - if (Buffer.isBuffer(hash)) - hash = utils.toHex(hash); - else if (hash.hash) - hash = hash.hash('hex'); - - return this.db.getAsync(hash, callback); + return this.db.get(hash, callback); }; Chain.prototype.getOrphan = function getOrphan(hash) { @@ -1631,26 +1836,10 @@ Chain.prototype.getProgress = function getProgress() { return Math.min(1, this.tip.ts / (utils.now() - 40 * 60)); }; -Chain.prototype.getHashRange = function getHashRange(start, end) { - var hashes = []; - var i; - - start = this.byTime(start); - end = this.byTime(end); - - if (!start || !end) - return hashes; - - for (i = start.height; i < end.height + 1; i++) - hashes.push(this.db.getSync(i).hash); - - return hashes; -}; - -Chain.prototype.getHashRangeAsync = function getHashRangeAsync(start, end, callback, force) { +Chain.prototype.getHashRange = function getHashRange(start, end, callback, force) { var self = this; - var unlock = this._lock(getHashRangeAsync, [start, end, callback], force); + var unlock = this._lock(getHashRange, [start, end, callback], force); if (!unlock) return; @@ -1659,11 +1848,11 @@ Chain.prototype.getHashRangeAsync = function getHashRangeAsync(start, end, callb callback(err, result); } - this.byTimeAsync(start, function(err, start) { + this.byTime(start, function(err, start) { if (err) return done(err); - self.byTimeAsync(end, function(err, end) { + self.byTime(end, function(err, end) { var hashes, i; if (err) @@ -1675,7 +1864,7 @@ Chain.prototype.getHashRangeAsync = function getHashRangeAsync(start, end, callb return done(null, hashes); utils.forRange(start.height, end.height + 1, function(i, next) { - self.db.getAsync(i, function(err, entry) { + self.db.get(i, function(err, entry) { if (err) return next(err); @@ -1695,62 +1884,14 @@ Chain.prototype.getHashRangeAsync = function getHashRangeAsync(start, end, callb }, true); }; -Chain.prototype.getLocator = function getLocator(start) { - var hashes = []; - var top = this.height; - var step = 1; - var i, existing; - - if (start) { - if (Buffer.isBuffer(start)) - start = utils.toHex(start); - else if (start.hash) - start = start.hash('hex'); - } - - if (typeof start === 'string') { - top = this.db.getHeight(start); - if (top === -1) { - // We could simply `return [start]` here, - // but there is no standardized "spacing" - // for locator hashes. Pretend this hash - // is our tip. This is useful for getheaders - // when not using headers-first. - hashes.push(start); - top = this.height; - } - } else if (typeof start === 'number') { - top = start; - } - - assert(this.db.has(top)); - - i = top; - for (;;) { - existing = this.db.getSync(i); - assert(existing); - hashes.push(existing.hash); - i = i - step; - if (i <= 0) { - if (i + step !== 0) - hashes.push(network.genesis.hash); - break; - } - if (hashes.length >= 10) - step *= 2; - } - - return hashes; -}; - -Chain.prototype.getLocatorAsync = function getLocatorAsync(start, callback, force) { +Chain.prototype.getLocator = function getLocator(start, callback, force) { var self = this; var hashes = []; var top = this.height; var step = 1; var i; - var unlock = this._lock(getLocatorAsync, [start, callback], force); + var unlock = this._lock(getLocator, [start, callback], force); if (!unlock) return; @@ -1761,57 +1902,69 @@ Chain.prototype.getLocatorAsync = function getLocatorAsync(start, callback, forc start = start.hash('hex'); } - if (typeof start === 'string') { - top = this.db.getHeight(start); - if (top === -1) { - // We could simply `return [start]` here, - // but there is no standardized "spacing" - // for locator hashes. Pretend this hash - // is our tip. This is useful for getheaders - // when not using headers-first. - hashes.push(start); - top = this.height; + function getTop(callback) { + if (typeof start === 'string') { + self.db.getHeight(start, function(err, top) { + if (err) + return callback(err); + + if (top === -1) { + // We could simply `return [start]` here, + // but there is no standardized "spacing" + // for locator hashes. Pretend this hash + // is our tip. This is useful for getheaders + // when not using headers-first. + hashes.push(start); + top = self.height; + } + + return callback(null, top); + }); + } else if (typeof start === 'number') { + top = start; } - } else if (typeof start === 'number') { - top = start; + return callback(null, top); } - assert(this.db.has(top)); - - callback = utils.asyncify(callback); - - i = top; - for (;;) { - hashes.push(i); - i = i - step; - if (i <= 0) { - if (i + step !== 0) - hashes.push(network.genesis.hash); - break; - } - if (hashes.length >= 10) - step *= 2; - } - - utils.forEach(hashes, function(height, next, i) { - if (typeof height === 'string') - return next(); - - self.db.getAsync(height, function(err, existing) { - if (err) - return next(err); - - assert(existing); - - hashes[i] = existing.hash; - - next(); - }); - }, function(err) { - unlock(); - if (err) + getTop(function(err, top) { + if (err) { + unlock(); return callback(err); - return callback(null, hashes); + } + + i = top; + for (;;) { + hashes.push(i); + i = i - step; + if (i <= 0) { + if (i + step !== 0) + hashes.push(network.genesis.hash); + break; + } + if (hashes.length >= 10) + step *= 2; + } + + utils.forEach(hashes, function(height, next, i) { + if (typeof height === 'string') + return next(); + + self.db.get(height, function(err, existing) { + if (err) + return next(err); + + assert(existing); + + hashes[i] = existing.hash; + + next(); + }); + }, function(err) { + unlock(); + if (err) + return callback(err); + return callback(null, hashes); + }); }); }; @@ -1840,23 +1993,69 @@ Chain.prototype.getOrphanRoot = function getOrphanRoot(hash) { }; }; -Chain.prototype.getHeight = function getHeight(hash) { - return this.db.getHeight(hash); -}; - -Chain.prototype.getSize = function getSize() { - return this.db.getSize(); -}; - -Chain.prototype.getCurrentTarget = function getCurrentTarget() { +Chain.prototype.getCurrentTarget = function getCurrentTarget(callback) { if (!this.tip) - return utils.toCompact(network.powLimit); - return this.getTarget(this.tip); + return callback(null, utils.toCompact(network.powLimit)); + return this.getTarget(this.tip, null, callback); +}; + +Chain.prototype.getTargetAsync = function getTarget(last, block, callback) { + var self = this; + var powLimit = utils.toCompact(network.powLimit); + var ts, first; + var i = 0; + + callback = utils.asyncify(callback); + + // Genesis + if (!last) + return callback(null, powLimit); + + // Do not retarget + if ((last.height + 1) % network.powDiffInterval !== 0) { + if (network.powAllowMinDifficultyBlocks) { + // Special behavior for testnet: + ts = block ? (block.ts || block) : utils.now(); + if (ts > last.ts + network.powTargetSpacing * 2) + return callback(null, powLimit); + + (function next(err, last) { + if (err) + return callback(err); + + assert(last); + + if (last.height > 0 + && last.height % network.powDiffInterval !== 0 + && last.bits === powLimit) { + return self.db.get(last.prevBlock, next); + } + + return callback(null, last.bits); + })(null, last); + + return; + } + return callback(null, last.bits); + } + + (function next(err, first) { + if (err) + return callback(err); + + i++; + assert(first); + + if (i >= network.powDiffInterval) + return callback(null, self.retarget(last, first)); + + self.db.get(first.prevBlock, next); + })(null, last); }; Chain.prototype.getTarget = function getTarget(last, block) { var powLimit = utils.toCompact(network.powLimit); - var ts, first, i; + var ts, first, i, prev; // Genesis if (!last) @@ -1870,10 +2069,12 @@ Chain.prototype.getTarget = function getTarget(last, block) { if (ts > last.ts + network.powTargetSpacing * 2) return powLimit; - while (last.prev + i = 1; + prev = last.previous; + while (prev[i] && last.height % network.powDiffInterval !== 0 && last.bits === powLimit) { - last = last.prev; + last = prev[i++]; } return last.bits; @@ -1882,17 +2083,13 @@ Chain.prototype.getTarget = function getTarget(last, block) { } // Back 2 weeks - // NOTE: This is cached. - first = this.db.getSync(last.height - (network.powDiffInterval - 1)); + first = last.previous[network.powDiffInterval - 1]; assert(first); return this.retarget(last, first); }; -// Legacy -Chain.prototype.target = Chain.prototype.getTarget; - Chain.prototype.retarget = function retarget(last, first) { var powTargetTimespan = new bn(network.powTargetTimespan); var actualTimespan, target; diff --git a/lib/bcoin/chainblock.js b/lib/bcoin/chainblock.js index 4ba6a379..7896c6c6 100644 --- a/lib/bcoin/chainblock.js +++ b/lib/bcoin/chainblock.js @@ -18,7 +18,7 @@ var fs = bcoin.fs; * ChainBlock */ -function ChainBlock(chain, data) { +function ChainBlock(chain, data, prev) { if (!(this instanceof ChainBlock)) return new ChainBlock(chain, data); @@ -31,18 +31,14 @@ function ChainBlock(chain, data) { this.bits = data.bits; this.nonce = data.nonce; this.height = data.height; - this.chainwork = data.chainwork || this.getChainwork(); + this.chainwork = data.chainwork || this.getChainwork(prev); + + assert(this.chainwork); + + this.previous = []; } -ChainBlock.BLOCK_SIZE = 112; - -ChainBlock.prototype.__defineGetter__('prev', function() { - return this.chain.db.getSync(this.height - 1); -}); - -ChainBlock.prototype.__defineGetter__('next', function() { - return this.chain.db.getSync(this.height + 1); -}); +ChainBlock.BLOCK_SIZE = 116; ChainBlock.prototype.getProof = function getProof() { var target = utils.fromCompact(this.bits); @@ -51,16 +47,12 @@ ChainBlock.prototype.getProof = function getProof() { return new bn(1).ushln(256).div(target.addn(1)); }; -ChainBlock.prototype.getChainwork = function getChainwork() { - var prev; - +ChainBlock.prototype.getChainwork = function getChainwork(prev) { // Workaround for genesis block // being added _in_ chaindb. if (!this.chain.db) return this.getProof(); - prev = this.prev; - return (prev ? prev.chainwork : new bn(0)).add(this.getProof()); }; @@ -68,43 +60,6 @@ ChainBlock.prototype.isGenesis = function isGenesis() { return this.hash === network.genesis.hash; }; -ChainBlock.prototype.getMedianTime = function getMedianTime() { - var entry = this; - var median = []; - var timeSpan = constants.block.medianTimespan; - var i; - - for (i = 0; i < timeSpan && entry; i++, entry = entry.prev) - median.push(entry.ts); - - median = median.sort(); - - return median[median.length / 2 | 0]; -}; - -ChainBlock.prototype.isOutdated = function isOutdated(version) { - return this.isSuperMajority(version, network.block.majorityRejectOutdated); -}; - -ChainBlock.prototype.isUpgraded = function isUpgraded(version) { - return this.isSuperMajority(version, network.block.majorityEnforceUpgrade); -}; - -ChainBlock.prototype.isSuperMajority = function isSuperMajority(version, required) { - var entry = this; - var found = 0; - var majorityWindow = network.block.majorityWindow; - var i; - - for (i = 0; i < majorityWindow && found < required && entry; i++) { - if (entry.version >= version) - found++; - entry = entry.prev; - } - - return found >= required; -}; - ChainBlock.prototype.toJSON = function toJSON() { return { hash: this.hash, @@ -131,14 +86,14 @@ ChainBlock.prototype.toRaw = function toRaw() { utils.writeU32(res, this.ts, 68); utils.writeU32(res, this.bits, 72); utils.writeU32(res, this.nonce, 76); - utils.copy(new Buffer(this.chainwork.toArray('be', 32)), res, 80); + utils.writeU32(res, this.height, 80); + utils.copy(new Buffer(this.chainwork.toArray('be', 32)), res, 84); return res; }; -ChainBlock.fromRaw = function fromRaw(chain, height, p) { +ChainBlock.fromRaw = function fromRaw(chain, p) { return new ChainBlock(chain, { - height: height, hash: utils.toHex(utils.dsha256(p.slice(0, 80))), version: utils.read32(p, 0), prevBlock: utils.toHex(p.slice(4, 36)), @@ -146,10 +101,153 @@ ChainBlock.fromRaw = function fromRaw(chain, height, p) { ts: utils.readU32(p, 68), bits: utils.readU32(p, 72), nonce: utils.readU32(p, 76), - chainwork: new bn(p.slice(80, 112), 'be') + height: utils.readU32(p, 80), + chainwork: new bn(p.slice(84, 116), 'be') }); }; +ChainBlock.prototype.getMedianTimeAsync = function getMedianTime(callback) { + var self = this; + var median = []; + var timeSpan = constants.block.medianTimespan; + var i = 0; + + (function next(err, entry) { + if (err) + return callback(err); + + if (!entry || i >= timeSpan) { + median = median.sort(); + return callback(null, median[median.length / 2 | 0]); + } + + median[i] = entry.ts; + i++; + + self.chain.db.get(entry.prevBlock, next); + })(null, this); +}; + +ChainBlock.prototype.isOutdatedAsync = function isOutdated(version, callback) { + return this.isSuperMajority(version, network.block.majorityRejectOutdated, callback); +}; + +ChainBlock.prototype.isUpgradedAsync = function isUpgraded(version, callback) { + return this.isSuperMajority(version, network.block.majorityEnforceUpgrade, callback); +}; + +ChainBlock.prototype.isSuperMajorityAsync = function isSuperMajority(version, required, callback) { + var self = this; + var found = 0; + var majorityWindow = network.block.majorityWindow; + var i = 0; + + (function next(err, entry) { + if (err) + return callback(err); + + if (!entry || i >= majorityWindow || found >= required) + return callback(null, found >= required); + + if (entry.version >= version) + found++; + + i++; + + self.chain.db.get(entry.prevBlock, next); + })(null, this); +}; + +ChainBlock.prototype.alloc = function alloc(callback) { + var majorityWindow = network.block.majorityWindow; + var medianTimespan = constants.block.medianTimespan; + var powDiffInterval = network.powDiffInterval; + var allowMinDiff = network.powAllowMinDifficultyBlocks; + var max = Math.max(majorityWindow, medianTimespan); + if ((this.height + 1) % powDiffInterval === 0 || allowMinDiff) + max = Math.max(max, powDiffInterval); + return this._alloc(max, callback); +}; + +ChainBlock.prototype._alloc = function _alloc(max, callback) { + var self = this; + var entry = this; + + assert(this.previous.length === 0); + + // Try to do this iteratively and synchronously + // so we don't have to wait on nextTicks. + for (;;) { + this.previous.push(entry); + + if (this.previous.length >= max) + return callback(); + + if (!this.chain.db.cacheHash.has(entry.prevBlock)) + break; + + entry = this.chain.db.cacheHash.get(entry.prevBlock); + } + + (function next(err, entry) { + if (err) { + self.free(); + return callback(err); + } + + if (!entry) + return callback(); + + self.previous.push(entry); + + if (self.previous.length >= max) + return callback(); + + self.chain.db.get(entry.prevBlock, next); + })(null, entry); +}; + +ChainBlock.prototype.free = function free() { + this.previous.length = 0; +}; + +ChainBlock.prototype.getMedianTime = function getMedianTime() { + var entry = this; + var median = []; + var timeSpan = constants.block.medianTimespan; + var i; + + for (i = 0; i < timeSpan && entry; i++, entry = this.previous[i]) + median.push(entry.ts); + + median = median.sort(); + + return median[median.length / 2 | 0]; +}; + +ChainBlock.prototype.isOutdated = function isOutdated(version) { + return this.isSuperMajority(version, network.block.majorityRejectOutdated); +}; + +ChainBlock.prototype.isUpgraded = function isUpgraded(version) { + return this.isSuperMajority(version, network.block.majorityEnforceUpgrade); +}; + +ChainBlock.prototype.isSuperMajority = function isSuperMajority(version, required) { + var entry = this; + var found = 0; + var majorityWindow = network.block.majorityWindow; + var i; + + for (i = 0; i < majorityWindow && found < required && entry; i++) { + if (entry.version >= version) + found++; + entry = this.previous[i + 1]; + } + + return found >= required; +}; + /** * Expose */ diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 13857f5c..fd14d44c 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -37,25 +37,27 @@ function ChainDB(node, chain, options) { if (!this.file) { this.file = bcoin.prefix - + '/chain-' + + '/chainindex-' + (options.spv ? 'spv-' : '') + network.type + '.db'; } - this.heightLookup = {}; this.queue = {}; this.queueSize = 0; - this.cache = {}; - this.bufferPool = { used: {} }; - this.highest = -1; - this.tip = null; - this.height = -1; this.size = 0; this.fd = null; this.loading = false; this.loaded = false; + // Keep track of block hashes in a + // bloom filter to avoid DB lookups. + // 1% false positive rate for 800k blocks + // http://hur.st/bloomfilter?n=800000&p=0.01 (m=936kb, k=7) + // 10% false positive rate for 800k blocks + // http://hur.st/bloomfilter?n=800000&p=0.10 (m=468kb, k=3) + this.bloom = new bcoin.bloom(937 * 1024, 7, 0xdeadbeef); + // Need to cache up to the retarget interval // if we're going to be checking the damn // target all the time. @@ -64,12 +66,42 @@ function ChainDB(node, chain, options) { else this._cacheWindow = network.block.majorityWindow + 1; + this.cacheHash = new DumbCache(this._cacheWindow * 200); // (not hashcash) + this.cacheHeight = new DumbCache(this._cacheWindow * 200); + this._init(); } utils.inherits(ChainDB, EventEmitter); ChainDB.prototype._init = function _init() { + var levelup = require('levelup'); + + bcoin.ensurePrefix(); + + if (+process.env.BCOIN_FRESH === 1 && bcoin.cp) + bcoin.cp.execFileSync('rm', ['-rf', this.file], { stdio: 'ignore' }); + + this.db = new levelup(this.file, { + keyEncoding: 'ascii', + valueEncoding: 'binary', + createIfMissing: true, + errorIfExists: false, + compression: false, + cacheSize: 16 * 1024 * 1024, + writeBufferSize: 8 * 1024 * 1024, + // blockSize: 4 * 1024, + maxOpenFiles: 8192, + // blockRestartInterval: 16, + db: bcoin.isBrowser + ? require('level-js') + : require('level' + 'down') + }); +}; + +ChainDB.prototype.load = function load(callback) { + var self = this; + var genesis = bcoin.chainblock.fromJSON(this.chain, { hash: network.genesis.hash, version: network.genesis.version, @@ -81,715 +113,380 @@ ChainDB.prototype._init = function _init() { height: 0 }); - if (!bcoin.fs) { - utils.debug('`fs` module not available. Falling back to ramdisk.'); - this.ramdisk = new bcoin.ramdisk(40 * 1024 * 1024); - this.saveSync(genesis); - return; - } - - bcoin.ensurePrefix(); - - if (+process.env.BCOIN_FRESH === 1) { - try { - fs.unlinkSync(this.file); - } catch (e) { - ; - } - } - - if (!this.exists()) - fs.writeFileSync(this.file, new Buffer([])); - - this.size = this.getFileSize(); - - if (this.size % BLOCK_SIZE !== 0) { - utils.debug('Blockchain is at an odd length. Truncating.'); - fs.truncateSync(this.file, this.size - (this.size % BLOCK_SIZE)); - this.size = this.getFileSize(); - assert(this.size % BLOCK_SIZE === 0); - } - - this.fd = fs.openSync(this.file, 'r+'); - - if (this.size === 0) { - this.saveSync(genesis); - } else { - this.getSync(0); - assert(this.tip.hash === genesis.hash); - } -}; - -ChainDB.prototype.load = function load(start, callback) { - var self = this; - var count = this.getSize(); - var i = start || 0; - var lastEntry; - this.loading = true; - utils.debug('Starting chain load at height: %s', i); + utils.debug('Starting chain load.'); function finish(err) { + if (err) + return callback(err); + self.loading = false; self.loaded = true; self.emit('load'); - if (err) - return callback(err); + utils.debug('Chain successfully loaded.'); callback(); } - function done(height) { - if (height != null) { - utils.debug( - 'Blockchain is corrupt after height %d. Resetting.', - height); - return self.resetHeightAsync(height, finish); - } - utils.debug('Chain successfully loaded.'); - finish(); - } - - (function next() { - if (i >= count) - return done(); - - self.getAsync(i, function(err, entry) { - if (err) - return callback(err); - - // Do some paranoid checks. - if (lastEntry && entry.prevBlock !== lastEntry.hash) - return done(Math.max(0, i - 2)); - - if (i % 10000 === 0) - utils.debug('Loaded %d blocks.', i); - - lastEntry = entry; - i += 1; - next(); - }); - })(); -}; - -ChainDB.prototype.closeSync = function closeSync() { - if (!bcoin.fs) { - this.ramdisk = null; - return; - } - - fs.close(this.fd); - this.fd = null; -}; - -ChainDB.prototype.closeAsync = function closeAsync(callback) { - var self = this; - - callback = utils.asyncify(callback); - - if (!bcoin.fs) { - this.ramdisk = null; - return callback(); - } - - fs.close(this.fd, function(err) { - if (err) - return callback(err); - - self.fd = null; - return callback(); + this.has(genesis.hash, function(err, exists) { + assert(!err); + if (!exists) + self.save(genesis, finish); + else + finish(); }); }; -ChainDB.prototype._malloc = function _malloc(size) { - if (!this.options.usePool) - return new Buffer(size); - - if (!this.bufferPool[size]) - this.bufferPool[size] = new Buffer(size); - - if (this.bufferPool.used[size] === this.bufferPool[size]) - return new Buffer(size); - - this.bufferPool.used[size] = this.bufferPool[size]; - - return this.bufferPool[size]; +ChainDB.prototype.close = function close(callback) { + callback = utils.ensure(callback); + this.db.close(callback); }; -ChainDB.prototype._free = function _free(buf) { - if (!this.options.usePool) - return; - - if (this.bufferPool.used[buf.length] === buf) { - assert(this.bufferPool[buf.length] === buf); - delete this.bufferPool.used[buf.length]; - } +ChainDB.prototype.addCache = function addCache(entry) { + this.cacheHash.set(entry.hash, entry); + this.cacheHeight.set(entry.height, entry); }; -ChainDB.prototype.exists = function exists() { - if (!bcoin.fs) - return true; - - try { - fs.statSync(this.file); - return true; - } catch (e) { +ChainDB.prototype.hasCache = function hasCache(hash) { + if (hash == null || hash < 0) return false; - } + + if (typeof hash === 'number') + return this.cacheHeight.has(hash); + + return this.cacheHash.has(hash); }; -ChainDB.prototype.getFileSize = function getFileSize() { - if (!bcoin.fs) - return this.ramdisk.size; +ChainDB.prototype.getCache = function getCache(hash) { + if (hash == null || hash < 0) + return; - try { - return fs.statSync(this.file).size; - } catch (e) { - return 0; - } + if (typeof hash === 'number') + return this.cacheHeight.get(hash); + + return this.cacheHash.get(hash); }; -ChainDB.prototype.getSize = function getSize() { - var len = this.size / BLOCK_SIZE; - assert(len % 1 === 0); - return len; +ChainDB.prototype.getHeight = function getHeight(hash, callback) { + if (typeof hash === 'number') + return callback(null, hash); + + if (this.cacheHash.has(hash)) + return callback(null, this.cacheHash.get(hash).height); + + if (!this.bloom.test(hash, 'hex')) + return callback(null, -1); + + this.db.get('c/h/' + hash, function(err, height) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (height == null) + return callback(null, -1); + + return callback(null, utils.readU32(height, 0)); + }); }; -ChainDB.prototype._cache = function _cache(entry) { - if (entry.height === this.highest + 1) { - this.highest = entry.height; - delete this.cache[entry.height - this._cacheWindow]; - this.cache[entry.height] = entry; - assert(Object.keys(this.cache).length <= this._cacheWindow); - } -}; - -ChainDB.prototype.isCached = function isCached(height) { - if (this.queue[height] != null) - return true; - - if (this.cache[height] != null) - return true; - - return false; -}; - -ChainDB.prototype.getHeight = function getHeight(hash) { - var height = this.heightLookup[hash]; - - if (height == null) - return -1; - - return height; -}; - -ChainDB.prototype._populate = function _populate(entry) { - this.heightLookup[entry.hash] = entry.height; - - if (!this.tip || entry.height > this.tip.height) { - this.tip = entry; - this.height = this.tip.height; - this.emit('tip', this.tip); - } -}; - -ChainDB.prototype.getSync = function getSync(height, force) { - var data, entry; - +ChainDB.prototype.getHash = function getHash(height, callback) { if (typeof height === 'string') - height = this.heightLookup[height]; + return callback(null, height); - if (height < 0 || height == null) - return; + if (this.cacheHeight.has(height)) + return callback(null, this.cacheHeight.get(height).hash); - if (!force) { - if ((height + 1) * BLOCK_SIZE > this.size) - return; - } + this.db.get('c/b/' + height, function(err, hash) { + if (err && err.type !== 'NotFoundError') + return callback(err); - if (this.cache[height]) - return this.cache[height]; + if (hash == null) + return callback(); - if (this.queue[height]) - return this.queue[height]; - - data = this._readSync(BLOCK_SIZE, height * BLOCK_SIZE); - - if (!data) - return; - - entry = bcoin.chainblock.fromRaw(this.chain, height, data); - - // Populate the entry. - this._populate(entry); - - // Cache the past 1001 blocks in memory - // (necessary for isSuperMajority) - this._cache(entry); - - return entry; + return callback(null, utils.toHex(hash)); + }); }; -ChainDB.prototype.getAsync = function getAsync(height, callback, force) { +ChainDB.prototype.getChainHeight = function getChainHeight(callback) { + return this.getTip(function(err, entry) { + if (err) + return callback(err); + + if (!entry) + return callback(null, -1); + + return callback(null, entry.height); + }); +}; + +ChainDB.prototype.getBoth = function getBoth(block, callback) { + var hash, height; + + if (block == null || block < 0) + return callback(null, null, -1); + + if (typeof block === 'string') + hash = block; + else + height = block; + + if (!hash) { + return this.getHash(height, function(err, hash) { + if (err) + return callback(err); + + if (hash == null) + height = -1; + + return callback(null, hash, height); + }); + } + + return this.getHeight(hash, function(err, height) { + if (err) + return callback(err); + + if (height === -1) + hash = null; + + return callback(null, hash, height); + }); +}; + +ChainDB.prototype.getEntry = function getEntry(hash, callback) { + var self = this; + var entry; + + if (hash == null || hash < 0) + return callback(); + + return this.getBoth(hash, function(err, hash, height) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!hash) + return callback(); + + if (self.cacheHash.has(hash)) + return callback(null, self.cacheHash.get(hash)); + + return self.db.get('c/c/' + hash, function(err, data) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!data) + return callback(); + + entry = bcoin.chainblock.fromRaw(self.chain, data); + + return callback(null, entry); + }); + }); +}; + +ChainDB.prototype.get = function get(height, callback, force) { var self = this; callback = utils.asyncify(callback); - if (typeof height === 'string') - height = this.heightLookup[height]; - - if (height < 0 || height == null) - return callback(); - - if (!force) { - if ((height + 1) * BLOCK_SIZE > this.size) - return callback(); - } - - if (this.cache[height]) - return callback(null, this.cache[height]); - - if (this.queue[height]) - return callback(null, this.queue[height]); - - return this._readAsync(BLOCK_SIZE, height * BLOCK_SIZE, function(err, data) { - var entry; - + return this.getEntry(height, function(err, entry) { if (err) return callback(err); - if (!data) + if (!entry) return callback(); - entry = bcoin.chainblock.fromRaw(self.chain, height, data); - - // Populate the entry. - self._populate(entry); - // Cache the past 1001 blocks in memory // (necessary for isSuperMajority) - self._cache(entry); + self.addCache(entry); return callback(null, entry); }); }; -ChainDB.prototype.saveSync = function saveSync(entry) { - var raw, offset; - - assert(entry.height >= 0); - - if (entry.height * BLOCK_SIZE !== this.size) { - utils.debug('Warning attempt to write to height: %d/%d', - entry.height, this.getSize() - 1); - return false; - } - - // Cache the past 1001 blocks in memory - // (necessary for isSuperMajority) - this._cache(entry); - - // Populate the entry. - this._populate(entry); - - raw = entry.toRaw(); - offset = entry.height * BLOCK_SIZE; - - return this._writeSync(raw, offset); -}; - -ChainDB.prototype.saveAsync = function saveAsync(entry, callback) { +ChainDB.prototype.save = function save(entry, callback) { var self = this; - var raw, offset; + var batch, height; callback = utils.asyncify(callback); assert(entry.height >= 0); - if (entry.height * BLOCK_SIZE !== this.size) { - utils.debug('Warning attempt to write to height: %d/%d', - entry.height, this.getSize() - 1); - return callback(); - } - // Cache the past 1001 blocks in memory // (necessary for isSuperMajority) - this._cache(entry); + this.addCache(entry); - // Populate the entry. - this._populate(entry); + this.bloom.add(entry.hash, 'hex'); - // Something is already writing. - assert(!this.queue[entry.height]); + batch = this.db.batch(); + height = new Buffer(4); + utils.writeU32(height, entry.height, 0); - // Speed up writes by doing them asynchronously - // and keeping the data to be written in memory. - this.queue[entry.height] = entry; - this.queueSize++; + batch.put('c/b/' + entry.height, new Buffer(entry.hash, 'hex')); + batch.put('c/h/' + entry.hash, height); + batch.put('c/c/' + entry.hash, entry.toRaw()); + batch.put('c/n/' + entry.prevBlock, new Buffer(entry.hash, 'hex')); + batch.put('c/t', new Buffer(entry.hash, 'hex')); - // Write asynchronously to the db. - raw = entry.toRaw(); - offset = entry.height * BLOCK_SIZE; - - return this._writeAsync(raw, offset, function(err, success) { + return batch.write(function(err) { if (err) return callback(err); - assert(self.queue[entry.height]); - - delete self.queue[entry.height]; - self.queueSize--; - - if (self.queueSize === 0) - self.emit('flush'); - - return callback(null, success); - }); -}; - -ChainDB.prototype.resetHeightSync = function resetHeightSync(height, emit) { - var self = this; - var osize = this.size; - var ohighest = this.highest; - var otip = this.tip; - var size, count, existing; - - if (typeof height === 'string') - height = this.heightLookup[height]; - - assert(height >= 0); - assert(this.tip); - - size = (height + 1) * BLOCK_SIZE; - count = this.getSize(); - - if (height > count - 1) - throw new Error('Height too high.'); - - if (height === count - 1) - return; - - for (i = height + 1; i < count; i++) { - existing = this.getSync(i); - - assert(existing); - - // Emit the blocks we remove. - if (emit) - emit(existing); - - // Warn of potential race condition - // (handled with _onFlush). - if (this.queue[i]) - utils.debug('Warning: write job in progress.'); - - delete this.cache[i]; - delete this.heightLookup[existing.hash]; - } - - // Prevent any more writes - // by setting this early. - this.size = size; - this.highest = height; - this.tip = this.getSync(height); - assert(this.tip); - this.height = this.tip.height; - this.emit('tip', this.tip); - - // This will be synchronous 99% of the time. - this._onFlush(function() { - try { - if (!bcoin.fs) - self.ramdisk.truncate(size); - else - fs.ftruncateSync(self.fd, size); - } catch (e) { - self.size = osize; - self.highest = ohighest; - self.tip = otip; - self.height = self.tip.height; - self.emit('tip', self.tip); - throw e; - } - }); -}; - -ChainDB.prototype.resetHeightAsync = function resetHeightAsync(height, callback, emit) { - var self = this; - var osize = this.size; - var ohighest = this.highest; - var otip = this.tip; - var size, count; - - callback = utils.asyncify(callback); - - if (typeof height === 'string') - height = this.heightLookup[height]; - - assert(height >= 0); - assert(this.tip); - - size = (height + 1) * BLOCK_SIZE; - count = this.getSize(); - - if (height > count - 1) - return callback(new Error('Height too high')); - - if (height === count - 1) - return callback(); - - // Prevent any more writes - // by setting this early. - this.size = size; - this.highest = height; - - this.getAsync(height, function(err, tip) { - if (err) - return done(err); - - self.tip = tip; - assert(self.tip); - self.height = self.tip.height; - self.emit('tip', self.tip); - - function finish(err) { - if (err) { - self.size = osize; - self.highest = ohighest; - self.tip = otip; - self.height = self.tip.height; - self.emit('tip', self.tip); - return callback(err); - } - - callback(); - } - - utils.forRange(height + 1, count, function(i, next) { - self.getAsync(i, function(err, existing) { - if (err) - return next(err); - - assert(existing); - - // Emit the blocks we remove. - if (emit) - emit(existing); - - // Warn of potential race condition - // (handled with _onFlush). - if (self.queue[i]) - utils.debug('Warning: write job in progress.'); - - delete self.cache[i]; - delete self.heightLookup[existing.hash]; - - return next(); - }, true); - }, function(err) { - if (err) - return finish(err); - - self._onFlush(function() { - if (!bcoin.fs) { - self.ramdisk.truncate(size); - return finish(); - } - - fs.ftruncate(self.fd, size, function(err) { - if (err) - return finish(err); - - return finish(); - }); - }); - }); - }); -}; - -ChainDB.prototype._onFlush = function _onFlush(callback) { - if (this.queueSize === 0) - return callback(); - this.once('flush', callback); -}; - -ChainDB.prototype.has = function has(height) { - if (typeof height === 'string') - height = this.heightLookup[height]; - - if (height < 0 || height == null) - return false; - - if ((height + 1) * BLOCK_SIZE <= this.size) - return true; - - return false; -}; - -ChainDB.prototype._ioError = function _ioError(name, size, offset) { - return new Error(name - + '() failed at offset ' - + offset - + ' with ' - + size - + ' bytes left.'); -}; - -ChainDB.prototype._readSync = function _readSync(size, offset) { - var index = 0; - var data, bytes; - - if (offset < 0 || offset == null) - return; - - if (!bcoin.fs) - return this.ramdisk.read(size, offset); - - data = this._malloc(size); - - try { - while (bytes = fs.readSync(this.fd, data, index, size, offset)) { - index += bytes; - size -= bytes; - offset += bytes; - if (index === data.length) { - this._free(data); - return data; - } - } - } catch (e) { - this._free(data); - throw e; - } - - this._free(data); - - throw this._ioError('_readSync', size, offset); -}; - -ChainDB.prototype._readAsync = function _readAsync(size, offset, callback) { - var self = this; - var index = 0; - var data, bytes; - - callback = utils.asyncify(callback); - - if (offset < 0 || offset == null) - return callback(); - - if (!bcoin.fs) - return callback(null, this.ramdisk.read(size, offset)); - - data = this._malloc(size); - - (function next() { - fs.read(self.fd, data, index, size, offset, function(err, bytes) { - if (err) { - self._free(data); - return callback(err); - } - - if (!bytes) - return callback(self._ioError('_readAsync', size, offset)); - - index += bytes; - size -= bytes; - offset += bytes; - - if (index === data.length) { - self._free(data); - return callback(null, data); - } - - next(); - }); - })(); -}; - -ChainDB.prototype._writeSync = function _writeSync(data, offset) { - var size = data.length; - var added = Math.max(0, (offset + data.length) - this.size); - var index = 0; - var bytes; - - if (offset < 0 || offset == null) - return false; - - if (!bcoin.fs) { - this.size += added; - this.ramdisk.write(data, offset); - return; - } - - try { - while (bytes = fs.writeSync(this.fd, data, index, size, offset)) { - index += bytes; - size -= bytes; - offset += bytes; - if (index === data.length) { - this.size += added; - return true; - } - } - } catch (e) { - throw e; - } - - fs.fsyncSync(this.fd); - - throw this._ioError('_writeSync', size, offset); -}; - -ChainDB.prototype._writeAsync = function _writeAsync(data, offset, callback) { - var self = this; - var added = Math.max(0, (offset + data.length) - this.size); - var size = data.length; - var index = 0; - - callback = utils.asyncify(callback); - - if (offset < 0 || offset == null) - return callback(null, false); - - if (!bcoin.fs) { - this.size += added; - this.ramdisk.write(data, offset); return callback(null, true); - } + }); +}; - this.size += added; +ChainDB.prototype.getTip = function getTip(callback) { + var self = this; + return this.db.get('c/t', function(err, hash) { + if (err && err.type !== 'NotFoundError') + return callback(err); - (function next() { - fs.write(self.fd, data, index, size, offset, function(err, bytes) { - if (err) { - self.size -= (added - index); + if (!hash) + return callback(); + + return self.get(utils.toHex(hash), callback); + }); +}; + +ChainDB.prototype.remove = function remove(block, callback, emit) { + var blocks = []; + var entry; + + this.getBoth(block, function(err, hash, height) { + if (err) + return callback(err); + + if (hash == null) + return callback(); + + (function next(err, nextHash) { + if (err && err.type !== 'NotFoundError') return callback(err); - } - if (!bytes) - return callback(self._ioError('_writeAsync', size, offset)); + if (!nextHash) + return done(); - index += bytes; - size -= bytes; - offset += bytes; + entry = { + hash: utils.toHex(nextHash), + height: height++ + }; - if (index === data.length) { - // Don't fsync when we're - // potentially preloading headers. - if (!self.chain.loaded) - return callback(null, true); - return fs.fsync(self.fd, function(err) { + self.cacheHash.remove(entry.hash); + self.cacheHeight.remove(entry.height); + + if (emit) + emit(entry); + + blocks.push(entry); + + self.db.get('c/n/' + entry.hash, next); + })(null, hash); + }); + + function done() { + var batch = self.db.batch(); + utils.forEach(blocks, function(entry, next, i) { + batch.del('c/b/' + entry.height); + batch.del('c/h/' + entry.hash); + batch.del('c/c/' + entry.hash); + + if (i === 0) { + return self.get(entry.hash, function(err, entry) { if (err) return callback(err); - return callback(null, true); + + assert(entry); + + return self.get(entry.prevBlock, function(err, entry) { + if (err) + return callback(err); + + assert(entry); + + batch.put('c/t', new Buffer(entry.hash, 'hex')); + batch.del('c/n/' + entry.hash); + + next(); + }); }); } - next(); + batch.del('c/n/' + blocks[i - 1].hash); + + return utils.nextTick(next); + }, function(err) { + if (err) + return callback(err); + + batch.write(callback); }); - })(); + } +}; + +ChainDB.prototype.resetHeight = function resetHeight(height, callback, emit) { + var self = this; + + callback = utils.asyncify(callback); + + return this.removeEntry(height + 1, callback, emit); +}; + +ChainDB.prototype.has = function has(height, callback) { + if (height == null || height < 0) + return callback(null, false); + + return this.getBoth(height, function(err, hash, height) { + if (err) + return callback(err); + return callback(null, hash != null); + }); +}; + +function DumbCache(size) { + this.data = {}; + this.count = 0; + this.size = size; +} + +DumbCache.prototype.set = function set(key, value) { + key = key + ''; + + assert(value !== undefined); + + if (this.count > this.size) + this.reset(); + + if (this.data[key] === undefined) + this.count++; + + this.data[key] = value; +}; + +DumbCache.prototype.remove = function remove(key) { + key = key + ''; + + if (this.data[key] === undefined) + return; + + this.count--; + delete this.data[key]; +}; + +DumbCache.prototype.get = function get(key) { + key = key + ''; + return this.data[key]; +}; + +DumbCache.prototype.has = function has(key) { + key = key + ''; + return this.data[key] !== undefined; +}; + +DumbCache.prototype.reset = function reset() { + this.data = {}; + this.count = 0; }; /** diff --git a/lib/bcoin/compactblock.js b/lib/bcoin/compactblock.js index d2098d07..dbc5d11c 100644 --- a/lib/bcoin/compactblock.js +++ b/lib/bcoin/compactblock.js @@ -37,8 +37,11 @@ CompactBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() { return this.coinbaseHeight; }; -CompactBlock.prototype.toBlock = function toBlock(peer) { - return new bcoin.block(bcoin.protocol.parser.parseBlock(this._raw)); +CompactBlock.prototype.toBlock = function toBlock() { + var block = new bcoin.block(bcoin.protocol.parser.parseBlock(this._raw)); + if (this.valid != null) + block.valid = this.valid; + return block; }; /** diff --git a/lib/bcoin/lru.js b/lib/bcoin/lru.js index 512ef722..0b3a94ac 100644 --- a/lib/bcoin/lru.js +++ b/lib/bcoin/lru.js @@ -81,6 +81,23 @@ LRU.prototype._compact = function _compact() { item.prev = null; }; +LRU.prototype.reset = function reset() { + var item, next; + + for (item = this.head; item; item = next) { + delete this.data[item.key]; + next = item.next; + item.prev = null; + item.next = null; + } + + assert(!item); + + this.size = 0; + this.head = null; + this.tail = null; +}; + LRU.prototype.set = function set(key, value) { var item; diff --git a/lib/bcoin/node.js b/lib/bcoin/node.js index 177d1375..355dd245 100644 --- a/lib/bcoin/node.js +++ b/lib/bcoin/node.js @@ -108,6 +108,7 @@ Fullnode.prototype._init = function _init() { self.emit('wallet tx', tx, ids); }); + if (0) this.on('tx', function(tx) { self.walletdb.addTX(tx, function(err) { if (err) @@ -116,6 +117,7 @@ Fullnode.prototype._init = function _init() { }); // Emit events for valid blocks and TXs. + if (0) this.chain.on('block', function(block) { self.emit('block', block); block.txs.forEach(function(tx) { @@ -128,6 +130,7 @@ Fullnode.prototype._init = function _init() { }); // Update the mempool. + if (0) this.chain.on('add block', function(block) { self.mempool.addBlock(block); }); diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 9f216e33..d93988f4 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -243,7 +243,7 @@ Pool.prototype._init = function _init() { Pool.prototype.getBlocks = function getBlocks(peer, top, stop) { var self = this; this.chain.onFlush(function() { - self.chain.getLocatorAsync(top, function(err, locator) { + self.chain.getLocator(top, function(err, locator) { if (err) throw err; @@ -256,7 +256,7 @@ Pool.prototype.resolveOrphan = function resolveOrphan(peer, top, orphan) { var self = this; assert(orphan); this.chain.onFlush(function() { - self.chain.getLocatorAsync(top, function(err, locator) { + self.chain.getLocator(top, function(err, locator) { if (err) throw err; @@ -280,10 +280,10 @@ Pool.prototype.resolveOrphan = function resolveOrphan(peer, top, orphan) { return; } - if (self.chain.has(orphan.soil)) { - utils.debug('Already have orphan "soil". Race condition?'); - return; - } + // if (self.chain.has(orphan.soil)) { + // utils.debug('Already have orphan "soil". Race condition?'); + // return; + // } peer.getBlocks(locator, orphan.root); }); @@ -293,7 +293,7 @@ Pool.prototype.resolveOrphan = function resolveOrphan(peer, top, orphan) { Pool.prototype.getHeaders = function getHeaders(peer, top, stop) { var self = this; this.chain.onFlush(function() { - self.chain.getLocatorAsync(top, function(err, locator) { + self.chain.getLocator(top, function(err, locator) { if (err) throw err; @@ -564,8 +564,11 @@ Pool.prototype._handleHeaders = function _handleHeaders(headers, peer) { if (!header.verify()) break; - if (!self.chain.has(hash)) - self.getData(peer, self.block.type, hash); + self.chain.has(hash, function(err, has) { + assert(!err); + if (!has) + self.getData(peer, self.block.type, hash); + }); last = hash; } @@ -608,39 +611,51 @@ Pool.prototype._handleBlocks = function _handleBlocks(hashes, peer) { this.emit('blocks', hashes); + var req = []; this.chain.onFlush(function() { - for (i = 0; i < hashes.length; i++) { - hash = hashes[i]; - + utils.forEachSerial(hashes, function(hash, next, i) { // Resolve orphan chain. if (self.chain.hasOrphan(hash)) { utils.debug('Peer sent a hash that is already a known orphan.'); self.resolveOrphan(peer, null, hash); - continue; + return utils.nextTick(next); } // Request a block if we don't have it. - if (!self.chain.has(hash)) { - self.getData(peer, self.block.type, hash); - continue; - } + self.chain.has(hash, function(err, has) { + assert(!err); - // Normally we request the hashContinue. - // In the odd case where we already have - // it, we can do one of two things: either - // force re-downloading of the block to - // continue the sync, or do a getblocks - // from the last hash. - if (i === hashes.length - 1) { - // Request more hashes: - // self.getBlocks(peer, hash, null); + if (!has) { + req.push([peer, self.block.type, hash]); + return next(); + } - // Re-download the block (traditional method): - self.getData(peer, self.block.type, hash, { force: true }); + // Normally we request the hashContinue. + // In the odd case where we already have + // it, we can do one of two things: either + // force re-downloading of the block to + // continue the sync, or do a getblocks + // from the last hash. + if (i === hashes.length - 1) { + // Request more hashes: + // self.getBlocks(peer, hash, null); - continue; - } - } + // Re-download the block (traditional method): + req.push([peer, self.block.type, hash, { force: true }]); + + return next(); + } + + return next(); + }); + }, function(err) { + if (err) + return self.emit('error', err); + + req.forEach(function(item) { + self.getData(item[0], item[1], item[2], item[3]); + }); + }); }); // Reset interval to avoid calling getblocks unnecessarily @@ -659,12 +674,15 @@ Pool.prototype._handleInv = function _handleInv(hashes, peer) { for (i = 0; i < hashes.length; i++) { hash = utils.toHex(hashes[i]); - if (!this.chain.has(hash)) { + this.chain.has(hash, function(err, has) { + assert(!err); + if (has) + return; if (this.options.headers) this.getHeaders(this.peers.load, null, hash); else this.getData(peer, this.block.type, hash); - } + }); } }; @@ -720,7 +738,7 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) { self.chain.orphan.count, self.request.activeBlocks, peer.queue.block.length, - self.chain.getCurrentTarget(), + 0, self.peers.all.length, self.chain.pending.length, self.chain.bestHeight, @@ -1327,7 +1345,7 @@ Pool.prototype.searchWallet = function(wallet, callback) { if (!height || height === -1) height = self.chain.height - (7 * 24 * 6); - self.chain.resetHeightAsync(height, function(err) { + self.chain.resetHeight(height, function(err) { if (err) { utils.debug('Failed to reset height: %s', err.stack + ''); return callback(err); @@ -1349,7 +1367,7 @@ Pool.prototype.searchWallet = function(wallet, callback) { if (!ts) ts = utils.now() - 7 * 24 * 3600; - self.chain.resetTimeAsync(ts, function(err) { + self.chain.resetTime(ts, function(err) { if (err) { utils.debug('Failed to reset time: %s', err.stack + ''); return callback(err); @@ -1416,7 +1434,7 @@ Pool.prototype.search = function search(id, range, callback) { this.on('block', onBlock); if (range.start < this.chain.tip.ts) { - this.chain.resetTimeAsync(range.start, function(err) { + this.chain.resetTime(range.start, function(err) { if (err) return done(err); @@ -1446,50 +1464,57 @@ Pool.prototype.getData = function getData(peer, type, hash, options, callback) { if (Buffer.isBuffer(hash)) hash = utils.toHex(hash); - if (this.request.map[hash]) { - if (callback) - this.request.map[hash].callback.push(callback); - return; + function has(cb) { + if (!options.force && type !== self.tx.type) + return self.chain.has(hash, cb); + return cb(null, false); } - if (!options.force && type !== self.tx.type) { - if (this.chain.has(hash)) + has(function(err, res) { + assert(!err); + if (res) return; - } - if (options.noQueue) - return; + if (self.request.map[hash]) { + if (callback) + self.request.map[hash].callback.push(callback); + return; + } - item = new LoadRequest(this, peer, type, hash, callback); + item = new LoadRequest(self, peer, type, hash, callback); - if (type === self.tx.type) { - if (peer.queue.tx.length === 0) { - utils.nextTick(function() { - utils.debug( - 'Requesting %d/%d txs from %s with getdata', - peer.queue.tx.length, - self.request.activeTX, - peer.host); + if (options.noQueue) + return; - peer.getData(peer.queue.tx); - peer.queue.tx.length = 0; + if (type === self.tx.type) { + if (peer.queue.tx.length === 0) { + utils.nextTick(function() { + utils.debug( + 'Requesting %d/%d txs from %s with getdata', + peer.queue.tx.length, + self.request.activeTX, + peer.host); + + peer.getData(peer.queue.tx); + peer.queue.tx.length = 0; + }); + } + + peer.queue.tx.push(item.start()); + + return; + } + + if (peer.queue.block.length === 0) { + self.chain.onFlush(function() { + utils.nextTick(function() { + self.scheduleRequests(peer); + }); }); } - peer.queue.tx.push(item.start()); - - return; - } - - if (peer.queue.block.length === 0) { - this.chain.onFlush(function() { - utils.nextTick(function() { - self.scheduleRequests(peer); - }); - }); - } - - peer.queue.block.push(item); + peer.queue.block.push(item); + }); }; Pool.prototype.scheduleRequests = function scheduleRequests(peer) {