From 6b51badfa9c21de9b85d292ae061049ccaf2ef31 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 5 Mar 2016 06:19:05 -0800 Subject: [PATCH] better reorgs. --- lib/bcoin/blockdb.js | 400 +++++++++++++++++++++++++++++++++++++++---- lib/bcoin/chain.js | 265 ++++++++++++---------------- lib/bcoin/chaindb.js | 156 +++++++++-------- lib/bcoin/pool.js | 2 +- 4 files changed, 560 insertions(+), 263 deletions(-) diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index ffa0ce1c..75282ce1 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -397,6 +397,287 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { }); }; +BlockDB.prototype.saveBlock = function saveBlock(block, callback) { + var self = this; + + this.data.saveAsync(block._raw, function(err, data) { + var batch, blockOffset; + + if (err) + return callback(err); + + batch = self.index.batch(); + + block._fileOffset = data.offset; + assert(block._size === data.size); + + blockOffset = self.createOffset(block._size, block._fileOffset, block.height); + + batch.put('b/b/' + block.hash('hex'), blockOffset); + + self.connectBlock(block, callback, batch, blockOffset); + }); +}; + +BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { + var self = this; + + this._getCoinBlock(hash, function(err, block) { + var batch; + + if (err) + return callback(err); + + if (!block) + return callback(); + + batch = self.index.batch(); + + batch.del('b/b/' + block.hash('hex')); + + function cb(err) { + if (err) + return callback(err); + + return callback(null, block); + + // XXX This seems to be truncating too much right now + self.data.truncateAsync(block._fileOffset, function(err) { + if (err) + return callback(err); + return callback(null, block); + }); + } + + self.disconnectBlock(hash, cb, batch); + }); +}; + +BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch, blockOffset) { + var self = this; + + this._getCoinBlock(block, function(err, block) { + if (err) + return callback(err); + + if (!block) { + assert(!batch); + return callback(); + } + + if (!batch) + batch = self.index.batch(); + + if (!blockOffset) + blockOffset = self.createOffset(block._size, block._fileOffset, block.height); + + batch.put('b/h/' + block.height, blockOffset); + + block.txs.forEach(function(tx, i) { + var hash = tx.hash('hex'); + var uniq = {}; + var txOffset; + + txOffset = self.createOffset( + tx._size, + block._fileOffset + tx._offset, + block.height + ); + + batch.put('t/t/' + hash, txOffset); + + tx.inputs.forEach(function(input) { + var type = input.getType(); + var address = input.getAddress(); + var uaddr; + + if (input.isCoinbase()) + return; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + if (uaddr) + batch.put('t/a/' + uaddr + '/' + hash, txOffset); + + if (address) { + batch.del( + 'u/a/' + address + + '/' + input.prevout.hash + + '/' + input.prevout.index); + } + + batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index); + + if (self.options.cache) + self.cache.unspent.remove(input.prevout.hash + '/' + input.prevout.index); + }); + + tx.outputs.forEach(function(output, i) { + var type = output.getType(); + var address = output.getAddress(); + var uaddr, coinOffset; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + coinOffset = self.createOffset( + output._size, + block._fileOffset + tx._offset + output._offset, + block.height + ); + + if (uaddr) + batch.put('t/a/' + uaddr + '/' + hash, txOffset); + + if (address) + batch.put('u/a/' + address + '/' + hash + '/' + i, coinOffset); + + batch.put('u/t/' + hash + '/' + i, coinOffset); + }); + }); + + batch.write(function(err) { + if (err) + return callback(err); + self.emit('save block', block); + return callback(null, block); + }); + }); +}; + +BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, batch) { + var self = this; + + this._getCoinBlock(hash, function(err, block) { + var batch; + + if (err) + return callback(err); + + if (!block) { + assert(!batch); + return callback(); + } + + if (!batch) + batch = self.index.batch(); + + if (typeof hash === 'string') + assert(block.hash('hex') === hash); + + batch.del('b/h/' + block.height); + + block.txs.forEach(function(tx, i) { + var hash = tx.hash('hex'); + var uniq = {}; + + if (self.options.cache) + self.cache.tx.remove(hash); + + batch.del('t/t/' + hash); + + tx.inputs.forEach(function(input) { + var type = input.getType(); + var address = input.getAddress(); + var uaddr, coinOffset; + + if (input.isCoinbase()) + return; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + assert(input.output._fileOffset >= 0); + + coinOffset = self.createOffset( + input.output._size, + input.output._fileOffset, + input.output.height + ); + + if (uaddr) + batch.del('t/a/' + uaddr + '/' + hash); + + if (address) { + batch.put('u/a/' + address + + '/' + input.prevout.hash + + '/' + input.prevout.index, + coinOffset); + } + + batch.put('u/t/' + + input.prevout.hash + + '/' + input.prevout.index, + coinOffset); + }); + + tx.outputs.forEach(function(output, i) { + var type = output.getType(); + var address = output.getAddress(); + var uaddr; + + if (type === 'pubkey' || type === 'multisig') + address = null; + + uaddr = address; + + if (uaddr) { + if (!uniq[uaddr]) + uniq[uaddr] = true; + else + uaddr = null; + } + + if (uaddr) + batch.del('t/a/' + uaddr + '/' + hash); + + if (address) + batch.del('u/a/' + address + '/' + hash + '/' + i); + + batch.del('u/t/' + hash + '/' + i); + + if (self.options.cache) + self.cache.unspent.remove(hash + '/' + i); + }); + }); + + batch.write(function(err) { + if (err) + return callback(err); + self.emit('remove block', block); + return callback(null, block); + }); + }); +}; + BlockDB.prototype.fillCoins = function fillCoins(txs, callback) { var self = this; var pending = txs.length; @@ -788,6 +1069,93 @@ BlockDB.prototype.getTX = function getTX(hash, callback) { }); }; +BlockDB.prototype.getFullTX = function getFullTX(hash, callback) { + var self = this; + + return this.getTX(hash, function(err, tx) { + if (err) + return callback(err); + + if (!tx) + return callback(); + + return self.fillTX(tx, function(err) { + if (err) + return callback(err); + + return callback(null, tx); + }); + }); +}; + +BlockDB.prototype.getFullBlock = function getFullBlock(hash, callback) { + var self = this; + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillTXs(block.txs, function(err) { + if (err) + return callback(err); + + return callback(null, block); + }); + }); +}; + +BlockDB.prototype._getCoinBlock = function _getCoinBlock(hash, callback) { + var self = this; + + if (hash instanceof bcoin.block) + return callback(null, hash); + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillBlock(block, callback); + }); +}; + +BlockDB.prototype.fillBlock = function fillBlock(block, callback) { + var self = this; + + return this.fillCoins(block.txs, function(err) { + var coins, i, tx, hash, j, input, id; + + if (err) + return callback(err); + + coins = {}; + + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + hash = tx.hash('hex'); + + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + id = input.prevout.hash + '/' + input.prevout.index; + if (!input.output && coins[id]) { + input.output = coins[id]; + delete coins[id]; + } + } + + for (j = 0; j < tx.outputs.length; j++) + coins[hash + '/' + j] = bcoin.coin(tx, j); + } + + return callback(null, block); + }); +}; + BlockDB.prototype.getBlock = function getBlock(hash, callback) { var self = this; var id = 'b/b/' + hash; @@ -995,7 +1363,7 @@ BlockDB.prototype.getHeight = function getHeight(callback) { })(); }; -BlockDB.prototype.resetHeight = function resetHeight(height, callback, emit) { +BlockDB.prototype.reset = function reset(height, callback, emit) { var self = this; this.getHeight(function(err, currentHeight) { if (err) @@ -1035,36 +1403,6 @@ BlockDB.prototype.getTip = function getTip(callback) { return callback(null, tip); }; -BlockDB.prototype.resetHash = function resetHash(hash, callback, emit) { - var self = this; - this.getTip(function(err, tip) { - if (err) - return callback(err); - - if (!tip) - return callback(new Error('Cannot reset to hash ' + hash)); - - (function next(tip) { - if (tip === hash) - return callback(); - - self.removeBlock(tip, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(new Error('Cannot reset all blocks.')); - - // Emit the blocks we removed. - if (emit) - emit(block); - - next(block.prevBlock); - }); - })(hash); - }); -}; - BlockDB.prototype._getEntry = function _getEntry(height, callback) { if (!this.node) return callback(); diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index f327d9cc..1e23e6f6 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -151,7 +151,7 @@ Chain.prototype._init = function _init() { if (err) throw err; - self.syncHeight(function(err) { + self._syncHeight(function(err) { if (err) throw err; @@ -303,7 +303,7 @@ Chain.prototype._preload = function _preload(callback) { stream.on('error', function(err) { var start = Math.max(0, height - 2); - self.resetHeight(start, function(e) { + self.reset(start, function(e) { if (e) throw e; return callback(err, start + 1); @@ -345,7 +345,7 @@ Chain.prototype._preload = function _preload(callback) { if (lastEntry && entry.prevBlock !== lastEntry.hash) { start = Math.max(0, height - 2); stream.destroy(); - return self.resetHeight(start, function(err) { + return self.reset(start, function(err) { if (err) throw err; return callback(new Error('Corrupt headers.'), start + 1); @@ -357,7 +357,7 @@ Chain.prototype._preload = function _preload(callback) { if (!block.verifyHeaders()) { start = Math.max(0, height - 2); stream.destroy(); - return self.resetHeight(start, function(err) { + return self.reset(start, function(err) { if (err) throw err; return callback(new Error('Bad headers.'), start + 1); @@ -651,7 +651,7 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, flags, callbac scriptCheck = false; } - this._fillBlock(block, function(err) { + this.blockdb.fillBlock(block, function(err) { var i, j, input, hash; var sigops = 0; @@ -740,38 +740,6 @@ Chain.prototype._checkReward = function _checkReward(block) { return true; }; -Chain.prototype._fillBlock = function _fillBlock(block, callback) { - var self = this; - - return this.blockdb.fillCoins(block.txs, function(err) { - var coins, i, tx, hash, j, input, id; - - if (err) - return callback(err); - - coins = {}; - - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; - hash = tx.hash('hex'); - - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - id = input.prevout.hash + '/' + input.prevout.index; - if (!input.output && coins[id]) { - input.output = coins[id]; - delete coins[id]; - } - } - - for (j = 0; j < tx.outputs.length; j++) - coins[hash + '/' + j] = bcoin.coin(tx, j); - } - - return callback(); - }); -}; - Chain.prototype.getHeight = function getHeight(hash) { if (Buffer.isBuffer(hash)) hash = utils.toHex(hash); @@ -828,12 +796,6 @@ Chain.prototype._findFork = function _findFork(fork, longer, callback) { Chain.prototype._reorganize = function _reorganize(entry, callback) { var self = this; - // Find the fork and connect/disconnect blocks. - // NOTE: Bitcoind disconnects and reconnects the - // forked block for some reason. We don't do this - // since it was already emitted for the wallet - // and mempool to handle. Technically bitcoind - // shouldn't have done it either. return this._findFork(this.tip, entry, function(err, fork) { if (err) return callback(err); @@ -842,34 +804,7 @@ Chain.prototype._reorganize = function _reorganize(entry, callback) { // Disconnect blocks/txs. function disconnect(callback) { - self.db.resetHash(fork.hash, function(err) { - if (err) - return callback(err); - - if (!self.blockdb) - return callback(); - - self.blockdb.resetHash(fork.hash, function(err) { - if (err) - return callback(err); - - return callback(); - }, function(block) { - self.emit('remove block', block); - }); - }, function(entry) { - self.emit('remove entry', entry); - }); - } - - // Connect blocks/txs. - function connect(callback) { - var entries = []; - (function collect(entry) { - if (entry.hash === fork.hash) - return finish(); - self.db.get(entry.prevBlock, function(err, entry) { if (err) return callback(err); @@ -877,6 +812,67 @@ Chain.prototype._reorganize = function _reorganize(entry, callback) { assert(entry); entries.push(entry); + + if (entry.hash === fork.hash) + return finish(); + + collect(entry); + }); + })(self.tip); + + function finish() { + assert(entries.length > 0); + + utils.forEachSerial(entries, function(entry, next) { + self.db.disconnect(entry, function(err) { + if (err) + return next(err); + + self.emit('remove entry', entry); + + next(); + }); + }, function(err) { + if (err) + return callback(err); + + if (!self.blockdb) + return callback(); + + utils.forEachSerial(entries, function(entry, next) { + self.blockdb.disconnectBlock(entry.hash, function(err, block) { + if (err) + return next(err); + + self.emit('remove block', entry); + + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(); + }); + }); + } + } + + // Connect blocks/txs. + function connect(callback) { + var entries = []; + + (function collect(entry) { + self.db.get(entry.prevBlock, function(err, entry) { + if (err) + return callback(err); + + assert(entry); + + entries.push(entry); + + if (entry.hash === fork.hash) + return finish(); + collect(entry); }); })(entry); @@ -885,29 +881,39 @@ Chain.prototype._reorganize = function _reorganize(entry, callback) { entries = entries.slice().reverse(); assert(entries.length > 0); - entries.forEach(function(entry) { - self.emit('add entry', entry); - }); - - if (!self.blockdb) - return callback(); - - utils.forEachSerial(entries, function(err, entry) { - return self.blockdb.getBlock(entry.hash, function(err, block) { + utils.forEachSerial(entries, function(entry, next) { + self.db.connect(entry, function(err) { if (err) - return callback(err); + return next(err); - assert(block); + self.emit('add entry', entry); - self.emit('add block', block); - - next(); + return next(); }); }, function(err) { if (err) return callback(err); - return callback(); + if (!self.blockdb) + return callback(); + + utils.forEachSerial(entries, function(err, entry) { + self.blockdb.connectBlock(entry.hash, function(err, block) { + if (err) + return callback(err); + + assert(block); + + self.emit('add block', block); + + next(); + }); + }, function(err) { + if (err) + return callback(err); + + return callback(); + }); }); } } @@ -972,101 +978,52 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { } }; -Chain.prototype.resetHeight = function resetHeight(height, callback, force) { +Chain.prototype.reset = function reset(height, callback, force) { var self = this; + var chainHeight; - var unlock = this._lock(resetHeight, [height, callback], force); + var unlock = this._lock(reset, [height, callback], force); if (!unlock) return; + callback = utils.ensure(callback); + function done(err, result) { - unlock(); - if (callback) - callback(err, result); - } - - this.db.resetHeight(height, function(err) { - if (err) - return done(err); - // Reset the orphan map completely. There may // have been some orphans on a forked chain we // no longer need. self.purgeOrphans(); self.purgePending(); - return done(); - }, function(entry) { - self.emit('remove entry', entry); - }); -}; - -Chain.prototype.revertHeight = function revertHeight(height, callback, force) { - var self = this; - var chainHeight; - - var unlock = this._lock(revertHeight, [height, callback], force); - if (!unlock) - return; - - callback = utils.asyncify(callback); - - function done(err, result) { unlock(); callback(err, result); } - this.db.getChainHeight(function(err, chainHeight) { + this.db.reset(height, function(err) { if (err) return done(err); - if (chainHeight == null || chainHeight < 0) - return done(new Error('Bad chain height.')); - - if (chainHeight < height) - return done(new Error('Cannot reset height.')); - - if (chainHeight === height) + if (!self.blockdb) return done(); - this.resetHeight(height, function(err) { + self.blockdb.reset(height, function(err) { if (err) return done(err); - if (!self.blockdb) - return done(); - - self.blockdb.getHeight(function(err, blockHeight) { - if (err) - return done(err); - - if (blockHeight < 0) - return done(new Error('Bad block height.')); - - if (blockHeight < height) - return done(new Error('Cannot reset height.')); - - if (blockHeight === height) - return done(); - - self.blockdb.resetHeight(height, function(err) { - if (err) - return done(err); - - return done(); - }, function(block) { - self.emit('remove block', block); - }); - }); - }, true); + return done(); + }, function(block) { + self.emit('remove block', block); + }); + }, function(entry) { + self.emit('remove entry', block); }); }; -Chain.prototype.syncHeight = function syncHeight(callback, force) { +Chain.prototype._syncHeight = function _syncHeight(callback, force) { var self = this; var chainHeight; - var unlock = this._lock(syncHeight, [callback], force); + var unlock = this._lock(_syncHeight, [callback], force); if (!unlock) return; @@ -1101,18 +1058,16 @@ Chain.prototype.syncHeight = function syncHeight(callback, force) { if (blockHeight < chainHeight) { utils.debug('ChainDB is higher than BlockDB. Syncing...'); - return self.resetHeight(blockHeight, done, true); + return self.db.reset(blockHeight, done); } if (blockHeight > chainHeight) { utils.debug('BlockDB is higher than ChainDB. Syncing...'); - self.blockdb.resetHeight(chainHeight, function(err) { + self.blockdb.reset(chainHeight, function(err) { if (err) return done(err); return done(); - }, function(block) { - self.emit('remove block', block); }); } }); @@ -1141,7 +1096,7 @@ Chain.prototype.resetTime = function resetTime(ts, callback, force) { return; } - self.resetHeight(entry.height, function(err) { + self.reset(entry.height, function(err) { unlock(); if (callback) callback(err); diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index db1e5b0b..8919218d 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -360,87 +360,66 @@ ChainDB.prototype.getTip = function getTip(callback) { }); }; -ChainDB.prototype.remove = function remove(block, callback, emit) { +ChainDB.prototype.connect = function connect(block, callback, emit) { var self = this; - var blocks = []; - var entry, hash, height; + var batch; - this.get(block, function(err, data) { + this._get(block, function(err, entry) { if (err) return callback(err); - if (!data) + if (!entry) return callback(); - hash = data.hash; - height = data.height; + batch = self.db.batch(); - (function next(err, nextHash) { - if (err && err.type !== 'NotFoundError') - return callback(err); + batch.put('c/b/' + entry.height, new Buffer(entry.hash, 'hex')); + batch.put('c/t', new Buffer(entry.hash, 'hex')); - if (!nextHash) - return done(); + self.cacheHeight.set(entry.height, entry); - entry = { - hash: utils.toHex(nextHash), - height: height++ - }; - - self.cacheHash.remove(entry.hash); - self.cacheHeight.remove(entry.height); - - if (emit) - emit(entry); - - blocks.push(entry); - - self.db.get('c/n/' + entry.hash, next); - })(null, hash); - }); - - function done() { - var batch = self.db.batch(); - utils.forEach(blocks, function(entry, next, i) { - batch.del('c/b/' + entry.height); - batch.del('c/h/' + entry.hash); - batch.del('c/c/' + entry.hash); - - if (i === 0) { - return self.get(entry.hash, function(err, entry) { - if (err) - return callback(err); - - assert(entry); - - return self.get(entry.prevBlock, function(err, entry) { - if (err) - return callback(err); - - assert(entry); - - batch.put('c/t', new Buffer(entry.hash, 'hex')); - batch.del('c/n/' + entry.hash); - - next(); - }); - }); - } - - batch.del('c/n/' + blocks[i - 1].hash); - - return utils.nextTick(next); - }, function(err) { + batch.write(function(err) { if (err) return callback(err); - - batch.write(callback); + return callback(null, entry); }); - } + }); +}; + +ChainDB.prototype.disconnect = function disconnect(block, callback) { + var self = this; + var batch; + + this._get(block, function(err, entry) { + if (err) + return callback(err); + + if (!entry) + return callback(); + + batch = self.db.batch(); + + batch.del('c/b/' + entry.height); + batch.put('c/t', new Buffer(entry.prevBlock, 'hex')); + + self.cacheHeight.remove(entry.height); + + batch.write(function(err) { + if (err) + return callback(err); + return callback(null, entry); + }); + }); +}; + +ChainDB.prototype._get = function _get(block, callback) { + if (block instanceof bcoin.chainblock) + return callback(null, block); + return this.get(block, callback); }; ChainDB.prototype.getNextHash = function getNextHash(hash, callback) { - return this.get('c/n/' + hash, function(err, nextHash) { + return this.db.get('c/n/' + hash, function(err, nextHash) { if (err && err.type !== 'NotFoundError') return callback(err); @@ -451,24 +430,49 @@ ChainDB.prototype.getNextHash = function getNextHash(hash, callback) { }); }; -ChainDB.prototype.resetHeight = function resetHeight(height, callback, emit) { +ChainDB.prototype.reset = function reset(block, callback, emit) { var self = this; + var batch; - callback = utils.asyncify(callback); - - return this.removeEntry(height + 1, callback, emit); -}; - -ChainDB.prototype.resetHash = function resetHash(hash, callback, emit) { - var self = this; - return this.getNextHash(hash, function(err, hash) { + this.get(block, function(err, entry) { if (err) return callback(err); - if (!hash) + if (!entry) return callback(); - return self.remove(hash, callback, emit); + self.getTip(function(err, tip) { + if (err) + return callback(err); + + if (!tip) + return callback(); + + batch = self.db.batch(); + + (function next(err, tip) { + if (err) + return done(err); + + if (!tip) + return done(); + + if (tip.hash === entry.hash) { + batch.put('c/t', new Buffer(tip.hash, 'hex')); + return batch.write(callback); + } + + batch.del('c/b/' + tip.height); + batch.del('c/h/' + tip.hash); + batch.del('c/c/' + tip.hash); + batch.del('c/n/' + tip.prevBlock); + + if (emit) + emit(tip); + + self.get(tip.prevBlock, next); + })(null, tip); + }); }); }; diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 30c6ee6d..b63a99e2 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -1342,7 +1342,7 @@ Pool.prototype.searchWallet = function(wallet, callback) { if (!height || height === -1) height = self.chain.height - (7 * 24 * 6); - self.chain.resetHeight(height, function(err) { + self.chain.reset(height, function(err) { if (err) { utils.debug('Failed to reset height: %s', err.stack + ''); return callback(err);