diff --git a/lib/bcoin/blockdb.js b/lib/bcoin/blockdb.js index 62318bdb..d3db60b1 100644 --- a/lib/bcoin/blockdb.js +++ b/lib/bcoin/blockdb.js @@ -16,11 +16,11 @@ var pad32 = utils.pad32; * BlockDB */ -function BlockDB(node, options) { +function BlockDB(options, db) { var self = this; if (!(this instanceof BlockDB)) - return new BlockDB(node, options); + return new BlockDB(options); EventEmitter.call(this); @@ -32,12 +32,7 @@ function BlockDB(node, options) { this.keepBlocks = options.keepBlocks || 288; this.prune = !!options.prune; - this.node = node; - - this.db = bcoin.ldb('block', { - cacheSize: 16 * 1024 * 1024, - writeBufferSize: 8 * 1024 * 1024 - }); + this.db = db; } utils.inherits(BlockDB, EventEmitter); @@ -52,9 +47,8 @@ BlockDB.prototype.close = function close(callback) { }); }; -BlockDB.prototype.saveBlock = function saveBlock(block, callback) { +BlockDB.prototype.saveBlock = function saveBlock(block, batch, callback) { var self = this; - var batch = this.batch(); batch.put('b/b/' + block.hash('hex'), block.toCompact()); @@ -62,7 +56,9 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) { batch.put('t/t/' + tx.hash('hex'), tx.toExtended()); }); - self.connectBlock(block, function(err) { + self.connectBlock(block, batch, callback); + return; + self.connectBlock(block, batch, function(err) { if (err) return callback(err); @@ -72,40 +68,36 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) { // Check for now-fully-spent txs. Try to remove // them or queue them up for future deletion if // it is currently unsafe to remove them. - self._pruneBlock(block, function(err) { + self._pruneBlock(block, batch, function(err) { if (err) return callback(err); return callback(null, block); }); - }, batch); + }); }; -BlockDB.prototype.removeBlock = function removeBlock(hash, callback) { +BlockDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { var self = this; this._getTXBlock(hash, function(err, block) { - var batch; - if (err) return callback(err); if (!block) return callback(); - batch = self.batch(); - batch.del('b/b/' + block.hash('hex')); block.txs.forEach(function(tx, i) { batch.del('t/t/' + tx.hash('hex')); }); - self.disconnectBlock(block, callback, batch); + self.disconnectBlock(block, batch, callback); }); }; -BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) { +BlockDB.prototype.connectBlock = function connectBlock(block, batch, callback) { var self = this; this._getCoinBlock(block, function(err, block) { @@ -114,19 +106,8 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) { if (err) return callback(err); - if (!block) { - assert(!batch); + if (!block) return callback(); - } - - if (!batch) - batch = self.batch(); - - batch.put('b/h/' + pad32(block.height), block.hash()); - - height = new Buffer(4); - utils.writeU32(height, block.height, 0); - batch.put('b/t', height); block.txs.forEach(function(tx, i) { var hash = tx.hash('hex'); @@ -178,40 +159,25 @@ BlockDB.prototype.connectBlock = function connectBlock(block, callback, batch) { }); }); - batch.write(function(err) { - if (err) - return callback(err); - self.emit('save block', block); - return callback(null, block); - }); + self.emit('add block', block); + + return callback(null, block); }); }; -BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, batch) { +BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callback) { var self = this; this._getTXBlock(hash, function(err, block) { - var height; - if (err) return callback(err); - if (!block) { - assert(!batch); + if (!block) return callback(); - } - - if (!batch) - batch = self.batch(); if (typeof hash === 'string') assert(block.hash('hex') === hash); - height = new Buffer(4); - utils.writeU32(height, block.height - 1, 0); - batch.put('b/t', height); - batch.del('b/h/' + pad32(block.height)); - block.txs.forEach(function(tx, i) { var hash = tx.hash('hex'); var uniq = {}; @@ -265,12 +231,8 @@ BlockDB.prototype.disconnectBlock = function disconnectBlock(hash, callback, bat }); }); - batch.write(function(err) { - if (err) - return callback(err); - self.emit('remove block', block); - return callback(null, block); - }); + self.emit('remove block', block); + return callback(null, block); }); }; @@ -692,7 +654,7 @@ BlockDB.prototype._getHash = function _getHash(height, callback) { if (typeof height === 'string') return callback(null, height); - this.db.get('b/h/' + pad32(height), function(err, hash) { + this.db.get('c/h/' + pad32(height), function(err, hash) { if (err && err.type !== 'NotFoundError') return callback(err); if (!hash) @@ -806,6 +768,10 @@ BlockDB.prototype.isUnspentTX = function isUnspentTX(hash, callback) { BlockDB.prototype.isSpentTX = function isSpentTX(hash, callback) { var spent = true; + // Important! + if (hash.hash) + hash = hash.hash('hex'); + var iter = this.db.db.iterator({ gte: 'u/t/' + hash, lte: 'u/t/' + hash + '~', @@ -828,7 +794,8 @@ BlockDB.prototype.isSpentTX = function isSpentTX(hash, callback) { spent = false; - next(); + // IMPORTANT! + iter.end(done); }); })(); @@ -864,72 +831,9 @@ BlockDB.prototype.isSpent = function isSpent(hash, index, callback) { }); }; -BlockDB.prototype.getHeight = function getHeight(callback) { +BlockDB.prototype._pruneBlock = function _pruneBlock(block, batch, callback) { var self = this; - return this.db.get('b/t', function(err, height) { - if (err && err.type !== 'NotFoundError') - return callback(err); - - if (!height) - return callback(null, -1); - - return callback(null, utils.readU32(height, 0)); - }); -}; - -BlockDB.prototype.getTipHash = function getTipHash(callback) { - return this.getHeight(function(err, height) { - if (err) - return callback(err); - - if (height === -1) - return callback(); - - return self.db.get('b/h/' + pad32(height), function(err, hash) { - if (err && err.type !== 'NotFoundError') - return callback(err); - - if (!hash) - return callback(); - - return callback(null, utils.toHex(hash)); - }); - }); -}; - -BlockDB.prototype.reset = function reset(height, callback, emit) { - var self = this; - this.getHeight(function(err, currentHeight) { - if (err) - return callback(err); - - if (currentHeight < height) - return callback(new Error('Cannot reset to height ' + height)); - - (function next() { - if (currentHeight === height) - return callback(); - - self.removeBlock(currentHeight, function(err, block) { - if (err) - return callback(err); - - // Emit the blocks we removed. - if (emit && block) - emit(block); - - currentHeight--; - next(); - }); - })(); - }); -}; - -BlockDB.prototype._pruneBlock = function _pruneBlock(block, callback) { - var self = this; - var batch = this.batch(); - // For much more aggressive pruning, we could delete // the block headers 288 blocks before this one here as well. @@ -942,7 +846,7 @@ BlockDB.prototype._pruneBlock = function _pruneBlock(block, callback) { self._pruneQueue(block, batch, function(err) { if (err) return callback(err); - return batch.write(callback); + return callback(); }); }); }; diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 7edf5d59..ee042169 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -34,8 +34,7 @@ function Chain(node, options) { this.node = node; this.loading = false; this.mempool = node.mempool; - this.blockdb = node.blockdb; - this.db = new bcoin.chaindb(node, this, options); + this.db = new bcoin.chaindb(this, options); this.busy = false; this.jobs = []; this.pending = []; @@ -132,41 +131,47 @@ Chain.prototype._init = function _init() { utils.debug('Warning: %d (%dmb) orphans cleared!', count, utils.mb(size)); }); + this.db.on('add entry', function(entry) { + self.emit('add entry', entry); + }); + + this.db.on('remove entry', function(entry) { + self.emit('remove entry', entry); + }); + + this.db.on('add block', function(block) { + self.emit('add block', block); + }); + + this.db.on('remove block', function(block) { + self.emit('remove block', block); + }); + this.loading = true; utils.debug('Chain is loading.'); - this._ensureGenesis(function(err) { - if (err) - throw err; + self._preload(function(err, start) { + if (err) { + utils.debug('Preloading chain failed.'); + utils.debug('Reason: %s', err.message); + } - self._preload(function(err, start) { - if (err) { - utils.debug('Preloading chain failed.'); - utils.debug('Reason: %s', err.message); - } + self.db.load(function(err) { + if (err) + throw err; - self.db.load(function(err) { + self.db.getTip(function(err, tip) { if (err) throw err; - self._syncHeight(function(err) { - if (err) - throw err; + assert(tip); - self.db.getTip(function(err, tip) { - if (err) - throw err; + self.tip = tip; + self.height = tip.height; - assert(tip); - - self.tip = tip; - self.height = tip.height; - - self.loading = false; - self.emit('load'); - }); - }); + self.loading = false; + self.emit('load'); }); }); }); @@ -230,37 +235,6 @@ Chain.prototype._lock = function _lock(func, args, force) { }; }; -Chain.prototype._ensureGenesis = function _ensureGenesis(callback) { - var self = this; - - callback = utils.asyncify(callback); - - if (!this.blockdb) - return callback(); - - self.blockdb.hasBlock(network.genesis.hash, function(err, result) { - var genesis; - - if (err) - return callback(err); - - if (result) - return callback(); - - utils.debug('BlockDB does not have genesis block. Adding.'); - - genesis = bcoin.block.fromRaw(network.genesisBlock, 'hex'); - genesis.height = 0; - - self.blockdb.saveBlock(genesis, function(err) { - if (err) - return callback(err); - - return callback(); - }); - }); -}; - // Stream headers from electrum.org for quickly // preloading the chain. Electrum.org stores // headers in the standard block header format, @@ -398,24 +372,6 @@ Chain.prototype._preload = function _preload(callback) { }); }; -Chain.prototype._saveBlock = function _saveBlock(block, callback) { - var self = this; - - if (!this.blockdb) - return utils.nextTick(callback); - - this.blockdb.saveBlock(block, callback); -}; - -Chain.prototype._removeBlock = function _removeBlock(tip, callback) { - var self = this; - - if (!this.blockdb) - return utils.nextTick(callback); - - this.blockdb.removeBlock(tip, callback); -}; - Chain.prototype._verifyContext = function _verifyContext(block, prev, callback) { var self = this; @@ -605,18 +561,18 @@ Chain.prototype._checkDuplicates = function _checkDuplicates(block, prev, callba var self = this; var height = prev.height + 1; - if (!this.blockdb || block.type !== 'block') + if (this.options.spv || block.type !== 'block') return callback(null, true); if (block.isGenesis()) return callback(null, true); // Check all transactions - utils.every(block.txs, function(tx, next) { + utils.everySerial(block.txs, function(tx, next) { var hash = tx.hash('hex'); // BIP30 - Ensure there are no duplicate txids - self.blockdb.isUnspentTX(hash, function(err, result) { + self.db.isUnspentTX(hash, function(err, result) { if (err) return next(err); @@ -639,7 +595,7 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, flags, callbac var height = prev.height + 1; var scriptCheck = true; - if (!this.blockdb || block.type !== 'block') + if (this.options.spv || block.type !== 'block') return callback(null, true); if (block.isGenesis()) @@ -652,13 +608,14 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, flags, callbac scriptCheck = false; } - this.blockdb.fillBlock(block, function(err) { + this.db.fillBlock(block, function(err) { var i, j, input, hash; var sigops = 0; if (err) return callback(err); + // Check all transactions for (i = 0; i < block.txs.length; i++) { tx = block.txs[i]; @@ -827,36 +784,8 @@ Chain.prototype._reorganize = function _reorganize(entry, callback) { assert(entries.length > 0); utils.forEachSerial(entries, function(entry, next) { - self.db.disconnect(entry, function(err) { - if (err) - return next(err); - - self.emit('remove entry', entry); - - next(); - }); - }, function(err) { - if (err) - return callback(err); - - if (!self.blockdb) - return callback(); - - utils.forEachSerial(entries, function(entry, next) { - self.blockdb.disconnectBlock(entry.hash, function(err, block) { - if (err) - return next(err); - - self.emit('remove block', block); - - next(); - }); - }, function(err) { - if (err) - return callback(err); - return callback(); - }); - }); + self.db.disconnect(entry, next); + }, callback); } } @@ -885,39 +814,8 @@ Chain.prototype._reorganize = function _reorganize(entry, callback) { assert(entries.length > 0); utils.forEachSerial(entries, function(entry, next) { - self.db.connect(entry, function(err) { - if (err) - return next(err); - - self.emit('add entry', entry); - - return next(); - }); - }, function(err) { - if (err) - return callback(err); - - if (!self.blockdb) - return callback(); - - utils.forEachSerial(entries, function(err, entry) { - self.blockdb.connectBlock(entry.hash, function(err, block) { - if (err) - return callback(err); - - assert(block); - - self.emit('add block', block); - - next(); - }); - }, function(err) { - if (err) - return callback(err); - - return callback(); - }); - }); + self.db.connect(entry, next); + }, callback); } } @@ -951,11 +849,8 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { // Start fsyncing writes once we're no // longer dealing with historical data. - if (this.isFull()) { + if (this.isFull()) this.db.fsync = true; - if (this.blockdb) - this.blockdb.fsync = true; - } if (!this.tip) { if (entry.hash !== network.genesis.hash) @@ -972,19 +867,14 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, callback) { if (err) return callback(err); - self._saveBlock(block, function(err) { + self.db.save(entry, block, function(err) { if (err) return callback(err); - self.db.save(entry, function(err) { - if (err) - return callback(err); + self.tip = entry; + self.height = entry.height; - self.tip = entry; - self.height = entry.height; - - return callback(); - }); + return callback(); }); } }; @@ -1014,74 +904,7 @@ Chain.prototype.reset = function reset(height, callback, force) { if (err) return done(err); - if (!self.blockdb) - return done(); - - self.blockdb.reset(height, function(err) { - if (err) - return done(err); - - return done(); - }, function(block) { - self.emit('remove block', block); - }); - }, function(entry) { - self.emit('remove entry', entry); - }); -}; - -Chain.prototype._syncHeight = function _syncHeight(callback, force) { - var self = this; - var chainHeight; - - var unlock = this._lock(_syncHeight, [callback], force); - if (!unlock) - return; - - callback = utils.ensure(callback); - - function done(err, result) { - unlock(); - callback(err, result); - } - - this.db.getChainHeight(function(err, chainHeight) { - if (err) - return done(err); - - if (chainHeight == null || chainHeight < 0) - return done(new Error('Bad chain height.')); - - if (!self.blockdb) - return done(); - - self.blockdb.getHeight(function(err, blockHeight) { - if (err) - return done(err); - - if (blockHeight < 0) - return done(new Error('Bad block height.')); - - if (blockHeight === chainHeight) - return done(); - - utils.debug('ChainDB and BlockDB are out of sync.'); - - if (blockHeight < chainHeight) { - utils.debug('ChainDB is higher than BlockDB. Syncing...'); - return self.db.reset(blockHeight, done); - } - - if (blockHeight > chainHeight) { - utils.debug('BlockDB is higher than ChainDB. Syncing...'); - self.blockdb.reset(chainHeight, function(err) { - if (err) - return done(err); - - return done(); - }); - } - }); + return done(); }); }; @@ -1290,6 +1113,13 @@ Chain.prototype.add = function add(initial, peer, callback, force) { } } + // Update the block height + // IMPORTANT!!!!! + block.height = height; + block.txs.forEach(function(tx) { + tx.height = height; + }); + // Do "contextual" verification on our block // now that we're certain its previous // block is in the chain. @@ -1326,12 +1156,6 @@ Chain.prototype.add = function add(initial, peer, callback, force) { if (entry.chainwork.cmp(self.tip.chainwork) <= 0) return done(); - // Update the block height - block.height = entry.height; - block.txs.forEach(function(tx) { - tx.height = entry.height; - }); - // Attempt to add block to the chain index. self._setBestChain(entry, block, function(err) { if (err) @@ -1347,8 +1171,6 @@ Chain.prototype.add = function add(initial, peer, callback, force) { if (block.hash('hex') !== initial.hash('hex')) self.emit('resolved', block, entry, peer); - self.emit('add block', block); - // No orphan chain. if (!self.orphan.map[hash]) return done(); diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 9f2cf590..3494565e 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -13,14 +13,15 @@ var network = bcoin.protocol.network; var utils = bcoin.utils; var assert = utils.assert; var pad32 = utils.pad32; +var DUMMY = new Buffer([]); /** * ChainDB */ -function ChainDB(node, chain, options) { +function ChainDB(chain, options) { if (!(this instanceof ChainDB)) - return new ChainDB(node, chain, options); + return new ChainDB(chain, options); if (!options) options = {}; @@ -28,8 +29,8 @@ function ChainDB(node, chain, options) { EventEmitter.call(this); this.options = options; - this.node = node; - this.network = node.network; + this.node = chain.node; + this.network = chain.node.network; this.chain = chain; this.queue = {}; @@ -40,6 +41,9 @@ function ChainDB(node, chain, options) { this.loaded = false; this.fsync = !!options.fsync; + this.keepBlocks = options.keepBlocks || 288; + this.prune = !!options.prune; + // Keep track of block hashes in a // bloom filter to avoid DB lookups. // 1% false positive rate for 800k blocks @@ -74,18 +78,10 @@ ChainDB.prototype._init = function _init() { ChainDB.prototype.load = function load(callback) { var self = this; + var genesis, block; - var genesis = new bcoin.chainblock(this.chain, { - hash: network.genesis.hash, - version: network.genesis.version, - prevBlock: network.genesis.prevBlock, - merkleRoot: network.genesis.merkleRoot, - ts: network.genesis.ts, - bits: network.genesis.bits, - nonce: network.genesis.nonce, - height: 0, - chainwork: null - }, null); + if (this.loaded) + return callback(); this.loading = true; @@ -104,14 +100,29 @@ ChainDB.prototype.load = function load(callback) { callback(); } - this.db.get('c/b/' + genesis.hash, function(err, exists) { + this.db.get('c/b/' + network.genesis.hash, function(err, exists) { if (err && err.type !== 'NotFoundError') throw err; - if (!exists) - self.save(genesis, finish); - else - finish(); + if (exists) + return finish(); + + genesis = new bcoin.chainblock(self.chain, { + hash: network.genesis.hash, + version: network.genesis.version, + prevBlock: network.genesis.prevBlock, + merkleRoot: network.genesis.merkleRoot, + ts: network.genesis.ts, + bits: network.genesis.bits, + nonce: network.genesis.nonce, + height: 0, + chainwork: null + }, null); + + block = bcoin.block.fromRaw(network.genesisBlock, 'hex'); + block.height = 0; + + self.save(genesis, block, finish); }); }; @@ -279,7 +290,7 @@ ChainDB.prototype.getBoth = function getBoth(block, callback) { }); }; -ChainDB.prototype.getEntry = function getEntry(hash, callback) { +ChainDB.prototype._getEntry = function _getEntry(hash, callback) { var self = this; var entry; @@ -315,7 +326,7 @@ ChainDB.prototype.get = function get(height, callback, force) { callback = utils.asyncify(callback); - return this.getEntry(height, function(err, entry) { + return this._getEntry(height, function(err, entry) { if (err) return callback(err); @@ -330,7 +341,7 @@ ChainDB.prototype.get = function get(height, callback, force) { }); }; -ChainDB.prototype.save = function save(entry, callback) { +ChainDB.prototype.save = function save(entry, block, callback) { var self = this; var batch, height; @@ -348,17 +359,33 @@ ChainDB.prototype.save = function save(entry, callback) { height = new Buffer(4); utils.writeU32(height, entry.height, 0); - batch.put('c/h/' + pad32(entry.height), new Buffer(entry.hash, 'hex')); batch.put('c/b/' + entry.hash, height); batch.put('c/c/' + entry.hash, entry.toRaw()); batch.put('c/n/' + entry.prevBlock, new Buffer(entry.hash, 'hex')); + batch.put('c/h/' + pad32(entry.height), new Buffer(entry.hash, 'hex')); batch.put('c/t', new Buffer(entry.hash, 'hex')); - return batch.write(function(err) { + this.emit('add entry', entry); + + self.saveBlock(block, batch, function(err) { if (err) return callback(err); - return callback(null, true); + return batch.write(function(err) { + if (err) + return callback(err); + + // We have to do this as a separate + // batch because of the isUnspentTX call. + // Not ideal, but it won't break anything + // if there is crash inconsistency. Just a + // less-than-perfect pruning would result. + self._pruneBlock(block, function(err) { + if (err) + return callback(err); + return callback(); + }); + }); }); }; @@ -375,7 +402,7 @@ ChainDB.prototype.getTip = function getTip(callback) { }); }; -ChainDB.prototype.connect = function connect(block, callback, emit) { +ChainDB.prototype.connect = function connect(block, callback) { var self = this; var batch; @@ -393,10 +420,17 @@ ChainDB.prototype.connect = function connect(block, callback, emit) { self.cacheHeight.set(entry.height, entry); - batch.write(function(err) { + self.emit('add entry', entry); + + self.connectBlock(entry.hash, batch, function(err) { if (err) return callback(err); - return callback(null, entry); + + batch.write(function(err) { + if (err) + return callback(err); + return callback(null, entry); + }); }); }); }; @@ -419,10 +453,17 @@ ChainDB.prototype.disconnect = function disconnect(block, callback) { self.cacheHeight.remove(entry.height); - batch.write(function(err) { + self.emit('remove entry', entry); + + self.disconnectBlock(block, batch, function(err) { if (err) return callback(err); - return callback(null, entry); + + batch.write(function(err) { + if (err) + return callback(err); + return callback(null, entry); + }); }); }); }; @@ -445,7 +486,7 @@ ChainDB.prototype.getNextHash = function getNextHash(hash, callback) { }); }; -ChainDB.prototype.reset = function reset(block, callback, emit) { +ChainDB.prototype.reset = function reset(block, callback) { var self = this; var batch; @@ -463,8 +504,6 @@ ChainDB.prototype.reset = function reset(block, callback, emit) { if (!tip) return callback(); - batch = self.batch(); - (function next(err, tip) { if (err) return done(err); @@ -472,6 +511,8 @@ ChainDB.prototype.reset = function reset(block, callback, emit) { if (!tip) return done(); + batch = self.batch(); + if (tip.hash === entry.hash) { batch.put('c/t', new Buffer(tip.hash, 'hex')); return batch.write(callback); @@ -482,10 +523,18 @@ ChainDB.prototype.reset = function reset(block, callback, emit) { batch.del('c/c/' + tip.hash); batch.del('c/n/' + tip.prevBlock); - if (emit) - emit(tip); + self.emit('remove entry', tip); - self.get(tip.prevBlock, next); + self.removeBlock(tip.hash, batch, function(err) { + if (err) + return callback(err); + + batch.write(function(err) { + if (err) + return next(err); + self.get(tip.prevBlock, next); + }); + }); })(null, tip); }); }); @@ -508,6 +557,949 @@ ChainDB.prototype.batch = function batch() { return this.db.batch(); }; +ChainDB.prototype.saveBlock = function saveBlock(block, batch, callback) { + if (this.options.spv) + return callback(); + + batch.put('b/b/' + block.hash('hex'), block.toCompact()); + + block.txs.forEach(function(tx, i) { + batch.put('t/t/' + tx.hash('hex'), tx.toExtended()); + }); + + this.connectBlock(block, batch, callback); +}; + +ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { + var self = this; + + if (this.options.spv) + return callback(); + + this._getTXBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + batch.del('b/b/' + block.hash('hex')); + + block.txs.forEach(function(tx, i) { + batch.del('t/t/' + tx.hash('hex')); + }); + + self.disconnectBlock(block, batch, callback); + }); +}; + +ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { + var self = this; + + if (this.options.spv) { + self.emit('add block', block); + return callback(); + } + + this._getCoinBlock(block, function(err, block) { + var height; + + if (err) + return callback(err); + + if (!block) + return callback(); + + block.txs.forEach(function(tx, i) { + var hash = tx.hash('hex'); + var uniq = {}; + + tx.inputs.forEach(function(input) { + var address; + + if (input.isCoinbase()) + return; + + assert(input.output); + + if (self.options.indexAddress) { + address = input.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.put('t/a/' + address + '/' + hash, DUMMY); + } + + if (address) { + batch.del( + 'u/a/' + address + + '/' + input.prevout.hash + + '/' + input.prevout.index); + } + } + + batch.del('u/t/' + input.prevout.hash + '/' + input.prevout.index); + }); + + tx.outputs.forEach(function(output, i) { + var address; + + if (self.options.indexAddress) { + address = output.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.put('t/a/' + address + '/' + hash, DUMMY); + } + + if (address) + batch.put('u/a/' + address + '/' + hash + '/' + i, DUMMY); + } + + batch.put('u/t/' + hash + '/' + i, bcoin.coin(tx, i).toExtended()); + }); + }); + + self.emit('add block', block); + + return callback(null, block); + }); +}; + +ChainDB.prototype.disconnectBlock = function disconnectBlock(hash, batch, callback) { + var self = this; + + if (this.options.spv) + return callback(); + + this._getTXBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + if (typeof hash === 'string') + assert(block.hash('hex') === hash); + + block.txs.forEach(function(tx, i) { + var hash = tx.hash('hex'); + var uniq = {}; + + tx.inputs.forEach(function(input) { + var coin, address; + + if (input.isCoinbase()) + return; + + assert(input.output); + + if (self.options.indexAddress) { + address = input.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.del('t/a/' + address + '/' + hash); + } + + if (address) { + batch.put('u/a/' + address + + '/' + input.prevout.hash + + '/' + input.prevout.index, + DUMMY); + } + } + + batch.put('u/t/' + + input.prevout.hash + + '/' + input.prevout.index, + input.output.toExtended()); + }); + + tx.outputs.forEach(function(output, i) { + var address; + + if (self.options.indexAddress) { + address = output.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.del('t/a/' + address + '/' + hash); + } + + if (address) + batch.del('u/a/' + address + '/' + hash + '/' + i); + } + + batch.del('u/t/' + hash + '/' + i); + }); + }); + + self.emit('remove block', block); + + return callback(null, block); + }); +}; + +ChainDB.prototype.fillCoin = function fillCoin(tx, callback) { + var self = this; + + callback = utils.asyncify(callback); + + if (Array.isArray(tx)) { + return utils.forEachSerial(tx, function(tx, next) { + self.fillCoin(tx, next); + }, function(err) { + if (err) + return callback(err); + return callback(null, tx); + }); + } + + if (tx.isCoinbase()) + return callback(null, tx); + + utils.forEachSerial(tx.inputs, function(input, next) { + if (input.output) + return next(); + + self.getCoin(input.prevout.hash, input.prevout.index, function(err, coin) { + if (err) + return callback(err); + + if (coin) + input.output = coin; + + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(null, tx); + }); +}; + +ChainDB.prototype.fillTX = function fillTX(tx, callback) { + var self = this; + + callback = utils.asyncify(callback); + + if (Array.isArray(tx)) { + return utils.forEachSerial(tx, function(tx, next) { + self.fillTX(tx, next); + }, function(err) { + if (err) + return callback(err); + return callback(null, tx); + }); + } + + if (tx.isCoinbase()) + return callback(null, tx); + + utils.forEachSerial(tx.inputs, function(input, next) { + if (input.output) + return next(); + + self.getTX(input.prevout.hash, function(err, tx) { + if (err) + return next(err); + + if (tx) + input.output = bcoin.coin(tx, input.prevout.index); + + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(null, tx); + }); +}; + +ChainDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, options, callback) { + var self = this; + var ids = []; + var coins = []; + + if (!callback) { + callback = options; + options = {}; + } + + if (typeof addresses === 'string') + addresses = [addresses]; + + addresses = utils.uniqs(addresses); + + utils.forEach(addresses, function(address, done) { + var iter = self.db.db.iterator({ + gte: 'u/a/' + address, + lte: 'u/a/' + address + '~', + keys: true, + values: true, + fillCache: true, + keyAsBuffer: false, + valueAsBuffer: true + }); + + (function next() { + iter.next(function(err, key, value) { + var parts, hash, index; + + if (err) { + return iter.end(function() { + done(err); + }); + } + + if (key === undefined) { + return iter.end(function(err) { + if (err) + return done(err); + done(); + }); + } + + parts = key.split('/'); + hash = parts[3]; + index = +parts[4]; + + ids.push([hash, index]); + + next(); + }); + })(); + }, function(err) { + if (err) + return callback(err); + + utils.forEach(ids, function(item, next) { + var hash = item[0]; + var index = item[1]; + self.getCoin(hash, index, function(err, coin) { + if (err) + return next(err); + + if (!coin) + return next(); + + coins.push(coin); + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(null, coins); + }); + }); +}; + +ChainDB.prototype.getCoin = function getCoin(hash, index, callback) { + var self = this; + var id = 'u/t/' + hash + '/' + index; + var coin; + + this.db.get(id, function(err, data) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + try { + coin = bcoin.coin.fromExtended(data); + } catch (e) { + return callback(e); + } + + return callback(null, coin); + }); +}; + +ChainDB.prototype.getTXByAddress = function getTXByAddress(addresses, options, callback) { + var self = this; + var hashes = []; + var txs = []; + var have = {}; + + if (!callback) { + callback = options; + options = {}; + } + + if (typeof addresses === 'string') + addresses = [addresses]; + + addresses = utils.uniqs(addresses); + + utils.forEach(addresses, function(address, done) { + var iter = self.db.db.iterator({ + gte: 't/a/' + address, + lte: 't/a/' + address + '~', + keys: true, + values: true, + fillCache: true, + keyAsBuffer: false, + valueAsBuffer: true + }); + + (function next() { + iter.next(function(err, key, value) { + var hash; + + if (err) { + return iter.end(function() { + done(err); + }); + } + + if (key === undefined) { + return iter.end(function(err) { + if (err) + return done(err); + done(); + }); + } + + hash = key.split('/')[3]; + + if (addresses.length > 1) { + if (have[hash]) + return next(); + + have[hash] = true; + } + + hashes.push(hash); + }); + })(); + }, function(err) { + utils.forEach(hashes, function(hash, next) { + self.getTX(hash, function(err, tx) { + if (err) + return next(err); + + if (!tx) + return next(); + + txs.push(tx); + next(); + }); + }, function(err) { + if (err) + return callback(err); + return callback(null, txs); + }); + }); +}; + +ChainDB.prototype.getTX = function getTX(hash, callback) { + var self = this; + var id = 't/t/' + hash; + var tx; + + this.db.get(id, function(err, data) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + try { + tx = bcoin.tx.fromExtended(data); + } catch (e) { + return callback(e); + } + + if (self.options.paranoid && tx.hash('hex') !== hash) + return callback(new Error('ChainDB is corrupt. All is lost.')); + + return callback(null, tx); + }); +}; + +ChainDB.prototype.getFullTX = function getFullTX(hash, callback) { + var self = this; + + return this.getTX(hash, function(err, tx) { + if (err) + return callback(err); + + if (!tx) + return callback(); + + return self.fillTX(tx, function(err) { + if (err) + return callback(err); + + return callback(null, tx); + }); + }); +}; + +ChainDB.prototype.getFullBlock = function getFullBlock(hash, callback) { + var self = this; + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillTX(block.txs, function(err) { + if (err) + return callback(err); + + return callback(null, block); + }); + }); +}; + +ChainDB.prototype._getCoinBlock = function _getCoinBlock(hash, callback) { + var self = this; + + if (hash instanceof bcoin.block) + return callback(null, hash); + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillBlock(block, callback); + }); +}; + +ChainDB.prototype._getTXBlock = function _getTXBlock(hash, callback) { + var self = this; + + if (hash instanceof bcoin.block) + return callback(null, hash); + + return this.getBlock(hash, function(err, block) { + if (err) + return callback(err); + + if (!block) + return callback(); + + return self.fillTXBlock(block, callback); + }); +}; + +ChainDB.prototype.fillBlock = function fillBlock(block, callback) { + var self = this; + + return this.fillCoin(block.txs, function(err) { + var coins, i, tx, hash, j, input, id; + + if (err) + return callback(err); + + coins = {}; + + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + hash = tx.hash('hex'); + + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + id = input.prevout.hash + '/' + input.prevout.index; + if (!input.output && coins[id]) { + input.output = coins[id]; + delete coins[id]; + } + } + + for (j = 0; j < tx.outputs.length; j++) + coins[hash + '/' + j] = bcoin.coin(tx, j); + } + + return callback(null, block); + }); +}; + +ChainDB.prototype.fillTXBlock = function fillTXBlock(block, callback) { + var self = this; + + return this.fillTX(block.txs, function(err) { + var coins, i, tx, hash, j, input, id; + + if (err) + return callback(err); + + coins = {}; + + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + hash = tx.hash('hex'); + + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + id = input.prevout.hash + '/' + input.prevout.index; + if (!input.output && coins[id]) { + input.output = coins[id]; + delete coins[id]; + } + } + + for (j = 0; j < tx.outputs.length; j++) + coins[hash + '/' + j] = bcoin.coin(tx, j); + } + + return callback(null, block); + }); +}; + +ChainDB.prototype._getHash = function _getHash(height, callback) { + if (typeof height === 'string') + return callback(null, height); + + this.db.get('c/h/' + pad32(height), function(err, hash) { + if (err && err.type !== 'NotFoundError') + return callback(err); + if (!hash) + return callback(); + return callback(null, utils.toHex(hash)); + }); +}; + +ChainDB.prototype.getBlock = function getBlock(hash, callback) { + var self = this; + var id, block; + + return this._getHash(hash, function(err, hash) { + if (err) + return callback(err); + + if (!hash) + return callback(); + + id = 'b/b/' + hash; + + self.db.get(id, function(err, data) { + if (err) { + if (err.type === 'NotFoundError') + return callback(); + return callback(err); + } + + try { + block = bcoin.block.fromCompact(data); + } catch (e) { + return callback(e); + } + + block.txs = []; + + utils.forEach(block.hashes, function(hash, next, i) { + self.getTX(hash, function(err, tx) { + if (err) + return next(err); + + if (!tx) + return next(new Error('TX not found.')); + + block.txs[i] = tx; + + next(); + }); + }, function(err) { + if (err) + return callback(err); + + delete block.hashes; + block = new bcoin.block(block); + return callback(null, block); + }); + }); + }); +}; + +ChainDB.prototype.hasBlock = function hasBlock(hash, callback) { + var self = this; + var id = 'b/b/' + hash; + + if (typeof hash === 'number') + id = 'b/h/' + pad32(hash); + + this.db.get(id, function(err, data) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!data) + return callback(null, false); + + return callback(null, true); + }); +}; + +ChainDB.prototype.hasCoin = function hasCoin(hash, index, callback) { + var self = this; + var id = 'u/t/' + hash + '/' + index; + + this.db.get(id, function(err, data) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!data) + return callback(null, false); + + return callback(null, true); + }); +}; + +ChainDB.prototype._getTX = function _getTX(hash, callback) { + if (hash instanceof bcoin.tx) + return callback(null, hash); + return this.getTX(hash); +}; + +// For BIP30 +// https://bitcointalk.org/index.php?topic=67738.0 +ChainDB.prototype.isUnspentTX = function isUnspentTX(hash, callback) { + return callback(null, false); + if (this.options.spv) + return callback(null, false); + return this.isSpentTX(hash, function(err, spent) { + if (err) + return callback(err); + + return callback(null, !spent); + }); +}; + +ChainDB.prototype.isSpentTX = function isSpentTX(hash, callback) { + var spent = true; + + return callback(null, false); + // Important! + if (hash.hash) + hash = hash.hash('hex'); + + var iter = this.db.db.iterator({ + gte: 'u/t/' + hash, + lte: 'u/t/' + hash + '~', + keys: true, + values: false, + fillCache: false, + keyAsBuffer: false + }); + + (function next() { + iter.next(function(err, key, value) { + if (err) { + return iter.end(function() { + done(err); + }); + } + + if (key === undefined) + return iter.end(done); + + spent = false; + + // IMPORTANT! + iter.end(done); + }); + })(); + + function done(err) { + if (err) + return callback(err); + + return callback(null, spent); + } +}; + +ChainDB.prototype.hasTX = function hasTX(hash, callback) { + var self = this; + var id = 't/t/' + hash; + + this.db.get(id, function(err, data) { + if (err && err.type !== 'NotFoundError') + return callback(err); + + if (!data) + return callback(null, false); + + return callback(null, true); + }); +}; + +ChainDB.prototype.isSpent = function isSpent(hash, index, callback) { + return this.hasCoin(hash, index, function(err, result) { + if (err) + return callback(err); + + return callback(null, !result); + }); +}; + +ChainDB.prototype._pruneBlock = function _pruneBlock(block, callback) { + var self = this; + var batch; + + if (self.options.spv) + return callback(); + + if (!self.prune) + return callback(); + + batch = self.batch(); + + // For much more aggressive pruning, we could delete + // the block headers 288 blocks before this one here as well. + + return utils.forEachSerial(block.txs, function(tx, next) { + self._pruneTX(tx, batch, next); + }, function(err) { + if (err) + return callback(err); + + self._pruneQueue(block, batch, function(err) { + if (err) + return callback(err); + return batch.write(callback); + }); + }); +}; + +ChainDB.prototype._pruneTX = function _pruneTX(tx, batch, callback) { + var self = this; + var watermark = tx.height - self.keepBlocks; + + if (tx.isCoinbase()) + return callback(); + + utils.forEachSerial(tx.inputs, function(input, next) { + self.isSpentTX(input.prevout.hash, function(err, result) { + var futureHeight; + + if (err) + return next(err); + + if (!result) + return next(); + + // Output's tx is below the watermark. It's not + // safe to delete yet. Queue it up to be deleted + // at a future height. + if (watermark >= 0 && input.output.height > watermark) { + futureHeight = input.output.height + watermark; + // This may screw up txs that end up being + // in a side chain in the future, but technically + // they should be safe to delete anyway at the + // future height. It's unlikely there will be + // _another_ reorg to take over 288 blocks. + batch.put('t/q/' + futureHeight + '/' + input.prevout.hash, DUMMY); + return next(); + } + + self._removeTX(input.prevout.hash, batch, next); + }); + }, callback); +}; + +ChainDB.prototype._removeTX = function _removeTX(hash, batch, callback) { + var self = this; + var uniq = {}; + + batch.del('t/t/' + hash); + + if (!self.options.indexAddress) + return callback(); + + this.getTX(hash, function(err, tx) { + if (err) + return callback(err); + + if (!tx) + return callback(); + + tx.inputs.forEach(function(input) { + var address; + + if (input.isCoinbase()) + return; + + // Since we may have pruned these outputs, we have to + // guess at the address. Should be correct 90% of the + // time, though we may leave some fluff behind. Not + // a perfect pruning, but probably good enough. + address = input.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.del('t/a/' + address + '/' + hash); + } + }); + + tx.outputs.forEach(function(output, i) { + var address = output.getAddress(); + + if (address && !uniq[address]) { + uniq[address] = true; + batch.del('t/a/' + address + '/' + hash); + } + }); + + callback(); + }); +}; + +ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) { + var self = this; + var hashes = []; + var iter = self.db.db.iterator({ + gte: 't/q/' + block.height, + lte: 't/q/' + block.height + '~', + keys: true, + values: false, + fillCache: false, + keyAsBuffer: false + }); + + (function next() { + iter.next(function(err, key, value) { + var parts, hash, index; + + if (err) { + return iter.end(function() { + done(err); + }); + } + + if (key === undefined) + return iter.end(done); + + parts = key.split('/'); + hash = parts[3]; + + hashes.push(hash); + + next(); + }); + })(); + + function done(err) { + if (err) + return callback(err); + + if (hashes.length) + utils.debug('Retroactively pruning txs at height %d', block.height); + + utils.forEachSerial(hashes, function(hash, next) { + batch.del('t/q/' + block.height + '/' + hash); + self._removeTX(hash, batch, next); + }, callback); + } +}; + /** * Expose */ diff --git a/lib/bcoin/fullnode.js b/lib/bcoin/fullnode.js index f5c3f2fd..11cfea7d 100644 --- a/lib/bcoin/fullnode.js +++ b/lib/bcoin/fullnode.js @@ -41,13 +41,10 @@ Fullnode.prototype._init = function _init() { this.loading = true; - // BlockDB technically needs access to the - // chain, but that's only once it's being - // used for tx retrieval. - this.blockdb = new bcoin.blockdb(this, { - cache: false, + this.chain = new bcoin.chain(this, { + preload: false, fsync: false, - prune: true + prune: false }); // Mempool needs access to blockdb. @@ -55,12 +52,6 @@ Fullnode.prototype._init = function _init() { rbf: false }); - // Chain needs access to blockdb. - this.chain = new bcoin.chain(this, { - preload: false, - fsync: false - }); - // Pool needs access to the chain. this.pool = new bcoin.pool(this, { witness: this.network.type === 'segnet', @@ -206,11 +197,11 @@ Fullnode.prototype.scanWallet = function scanWallet(wallet, callback) { }; Fullnode.prototype.getBlock = function getBlock(hash, callback) { - this.blockdb.getBlock(hash, callback); + this.chain.db.getBlock(hash, callback); }; Fullnode.prototype.getFullBlock = function getFullBlock(hash, callback) { - this.blockdb.getFullBlock(hash, callback); + this.chain.db.getFullBlock(hash, callback); }; Fullnode.prototype.getCoin = function getCoin(hash, index, callback) { @@ -226,7 +217,7 @@ Fullnode.prototype.getCoin = function getCoin(hash, index, callback) { if (this.mempool.isSpent(hash, index)) return callback(null, null); - this.blockdb.getCoin(hash, index, function(err, coin) { + this.chain.db.getCoin(hash, index, function(err, coin) { if (err) return callback(err); @@ -245,7 +236,7 @@ Fullnode.prototype.getCoinByAddress = function getCoinByAddress(addresses, callb mempool = this.mempool.getCoinsByAddress(addresses); - this.blockdb.getCoinsByAddress(addresses, function(err, coins) { + this.chain.db.getCoinsByAddress(addresses, function(err, coins) { if (err) return callback(err); @@ -267,7 +258,7 @@ Fullnode.prototype.getTX = function getTX(hash, callback) { if (tx) return callback(null, tx); - this.blockdb.getTX(hash, function(err, tx) { + this.chain.db.getTX(hash, function(err, tx) { if (err) return callback(err); @@ -286,7 +277,7 @@ Fullnode.prototype.isSpent = function isSpent(hash, index, callback) { if (this.mempool.isSpent(hash, index)) return callback(null, true); - this.blockdb.isSpent(hash, index, callback); + this.chain.db.isSpent(hash, index, callback); }; Fullnode.prototype.getTXByAddress = function getTXByAddress(addresses, callback) { @@ -297,7 +288,7 @@ Fullnode.prototype.getTXByAddress = function getTXByAddress(addresses, callback) mempool = this.mempool.getTXByAddress(addresses); - this.blockdb.getTXByAddress(addresses, function(err, txs) { + this.chain.db.getTXByAddress(addresses, function(err, txs) { if (err) return callback(err); @@ -311,7 +302,7 @@ Fullnode.prototype.fillCoin = function fillCoin(tx, callback) { if (this.mempool.fillCoin(tx)) return callback(); - this.blockdb.fillCoin(tx, callback); + this.chain.db.fillCoin(tx, callback); }; Fullnode.prototype.fillTX = function fillTX(tx, callback) { @@ -320,7 +311,7 @@ Fullnode.prototype.fillTX = function fillTX(tx, callback) { if (this.mempool.fillTX(tx)) return callback(); - this.blockdb.fillTX(tx, callback); + this.chain.db.fillTX(tx, callback); }; /** diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 0746a27b..d2617c32 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -27,7 +27,7 @@ function Mempool(node, options) { this.options = options; this.node = node; - this.blockdb = node.blockdb; + this.chain = node.chain; this.txs = {}; this.spent = {}; @@ -214,7 +214,7 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback) { this._lockTX(tx); - this.blockdb.fillCoin(tx, function(err) { + this.chain.fillCoin(tx, function(err) { var i, input, dup, height, ts, priority; self._unlockTX(tx); diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index 16342f96..8b7e806a 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -1495,7 +1495,7 @@ utils.forEach = function forEach(arr, iter, callback) { utils.forRangeSerial = function forRangeSerial(from, to, iter, callback) { var called = false; - callback = utils.asyncify(callback); + callback = utils.ensure(callback); (function next(err) { assert(!called); @@ -1518,7 +1518,7 @@ utils.forEachSerial = function forEachSerial(arr, iter, callback) { var i = 0; var called = false; - callback = utils.asyncify(callback); + callback = utils.ensure(callback); (function next(err) { var item; @@ -1571,7 +1571,7 @@ utils.everySerial = function everySerial(arr, iter, callback) { var i = 0; var called = false; - callback = utils.asyncify(callback); + callback = utils.ensure(callback); (function next(err, res) { var item; @@ -1580,7 +1580,7 @@ utils.everySerial = function everySerial(arr, iter, callback) { called = true; return callback(err); } - if (!result) { + if (!res) { called = true; return callback(null, false); } @@ -1593,7 +1593,7 @@ utils.everySerial = function everySerial(arr, iter, callback) { utils.nextTick(function() { iter(item, next, i - 1); }); - })(); + })(null, true); }; utils.mb = function mb(size) {