diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index f67f864f..173d8a79 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -310,7 +310,7 @@ Chain.prototype._preload = function _preload(callback) { if (!unlock) return; - self.db.save(entry, null, true, function(err) { + self.db.save(entry, null, null, true, function(err) { if (err) { stream.destroy(); locker.destroy(); @@ -451,11 +451,11 @@ Chain.prototype._verifyContext = function _verifyContext(block, prev, callback) if (err) return callback(err); - self._checkInputs(block, prev, state, function(err) { + self._checkInputs(block, prev, state, function(err, view) { if (err) return callback(err); - return callback(); + return callback(null, view); }); }); }); @@ -838,7 +838,7 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, state, callbac historical = true; } - this.db.fillBlock(block, function(err) { + this.db.getCoinView(block, function(err, view) { var ret = {}; var sigops = 0; @@ -849,6 +849,7 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, state, callbac utils.forEachSerial(block.txs, function(tx, next) { // Ensure tx is not double spending an output. if (!tx.isCoinbase()) { + view.fill(tx); if (!tx.hasCoins()) { assert(!historical, 'BUG: Spent inputs in historical data!'); return next(new VerifyError(block, @@ -890,6 +891,8 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, state, callbac } } + view.add(tx.toCoins()); + return next(); }); }, function(err) { @@ -922,7 +925,7 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, state, callbac 100)); } - return callback(); + return callback(null, view); }); }); }); @@ -1141,7 +1144,7 @@ Chain.prototype.connect = function connect(entry, callback) { assert(prev); - self._verifyContext(block, prev, function(err) { + self._verifyContext(block, prev, function(err, view) { if (err) { if (err.type === 'VerifyError') { self.invalid[entry.hash] = true; @@ -1155,7 +1158,7 @@ Chain.prototype.connect = function connect(entry, callback) { return callback(err); } - self.db.connect(entry, block, function(err) { + self.db.connect(entry, block, view, function(err) { if (err) return callback(err); @@ -1196,7 +1199,7 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, prev, callb // Do "contextual" verification on our block // now that we're certain its previous // block is in the chain. - self._verifyContext(block, prev, function(err) { + self._verifyContext(block, prev, function(err, view) { if (err) { // Couldn't verify block. // Revert the height. @@ -1216,7 +1219,7 @@ Chain.prototype._setBestChain = function _setBestChain(entry, block, prev, callb } // Save block and connect inputs. - self.db.save(entry, block, true, function(err) { + self.db.save(entry, block, view, true, function(err) { if (err) return callback(err); @@ -1564,7 +1567,7 @@ Chain.prototype.add = function add(block, callback, force) { // our tip's. Add the block but do _not_ // connect the inputs. if (entry.chainwork.cmp(self.tip.chainwork) <= 0) { - return self.db.save(entry, block, false, function(err) { + return self.db.save(entry, block, null, false, function(err) { if (err) return done(err); diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 145a0b33..82bb7625 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -93,8 +93,8 @@ function ChainDB(chain, options) { // Average number of outputs per tx: 2.2 // Average size of outputs per tx: 74b // Average number of txs: 2300 - // Key size: 68b (* 2) - this.coinWindow = ((165 * 1024 + 5000 * 9) + (5000 * 68 * 2)) * 5; + // Key size: 66b (* 2) + this.coinWindow = ((165 * 1024 + 2300 * 4) + (2300 * 66 * 2)) * 5; this.coinCache = new NullCache(this.coinWindow); this.cacheHash = new bcoin.lru(this.cacheWindow, 1); @@ -160,7 +160,7 @@ ChainDB.prototype._init = function _init() { block = bcoin.block.fromRaw(self.network.genesisBlock, 'hex'); block.setHeight(0); - self.save(genesis, block, true, finish); + self.save(genesis, block, null, true, finish); }); }); }; @@ -423,7 +423,7 @@ ChainDB.prototype.get = function get(hash, callback) { * @param {Function} callback */ -ChainDB.prototype.save = function save(entry, block, connect, callback) { +ChainDB.prototype.save = function save(entry, block, view, connect, callback) { var batch, hash, height; callback = utils.ensure(callback); @@ -443,7 +443,7 @@ ChainDB.prototype.save = function save(entry, block, connect, callback) { this.cacheHash.set(entry.hash, entry); if (!connect) { - return this.saveBlock(block, batch, false, function(err) { + return this.saveBlock(block, view, batch, false, function(err) { if (err) return callback(err); return batch.write(callback); @@ -458,7 +458,7 @@ ChainDB.prototype.save = function save(entry, block, connect, callback) { this.emit('add entry', entry); - this.saveBlock(block, batch, true, function(err) { + this.saveBlock(block, view, batch, true, function(err) { if (err) return callback(err); return batch.write(callback); @@ -492,7 +492,7 @@ ChainDB.prototype.getTip = function getTip(callback) { * @param {Function} callback - Returns [Error, {@link ChainEntry}]. */ -ChainDB.prototype.connect = function connect(entry, block, callback) { +ChainDB.prototype.connect = function connect(entry, block, view, callback) { var batch = this.db.batch(); var hash = new Buffer(entry.hash, 'hex'); @@ -505,7 +505,7 @@ ChainDB.prototype.connect = function connect(entry, block, callback) { this.emit('add entry', entry); - this.connectBlock(block, batch, function(err) { + this.connectBlock(block, view, batch, function(err) { if (err) return callback(err); @@ -527,24 +527,24 @@ ChainDB.prototype.disconnect = function disconnect(block, callback) { var self = this; var batch; - this._ensureEntry(block, function(err, entry) { + batch = this.db.batch(); + + batch.del('n/' + entry.prevBlock); + batch.del('H/' + pad32(entry.height)); + batch.put('R', new Buffer(entry.prevBlock, 'hex')); + + this.cacheHeight.remove(entry.height); + + this.emit('remove entry', entry); + + this.getBlock(entry.hash, function(err, block) { if (err) return callback(err); - if (!entry) - return callback(new Error('Entry not found.')); + if (!block) + return callback(new Error('Block not found.')); - batch = self.db.batch(); - - batch.del('n/' + entry.prevBlock); - batch.del('H/' + pad32(entry.height)); - batch.put('R', new Buffer(entry.prevBlock, 'hex')); - - self.cacheHeight.remove(entry.height); - - self.emit('remove entry', entry); - - self.disconnectBlock(entry.hash, batch, function(err) { + self.disconnectBlock(block, batch, function(err) { if (err) return callback(err); @@ -557,12 +557,6 @@ ChainDB.prototype.disconnect = function disconnect(block, callback) { }); }; -ChainDB.prototype._ensureEntry = function _ensureEntry(block, callback) { - if (block instanceof bcoin.chainentry) - return callback(null, block); - return this.get(block, callback); -}; - /** * Get the _next_ block hash (does not work by height). * @param {Hash} hash @@ -701,7 +695,7 @@ ChainDB.prototype.has = function has(height, callback) { * @param {Function} callback - Returns [Error, {@link Block}]. */ -ChainDB.prototype.saveBlock = function saveBlock(block, batch, connect, callback) { +ChainDB.prototype.saveBlock = function saveBlock(block, view, batch, connect, callback) { if (this.options.spv) return utils.nextTick(callback); @@ -710,7 +704,7 @@ ChainDB.prototype.saveBlock = function saveBlock(block, batch, connect, callback if (!connect) return utils.nextTick(callback); - this.connectBlock(block, batch, callback); + this.connectBlock(block, view, batch, callback); }; /** @@ -724,10 +718,7 @@ ChainDB.prototype.saveBlock = function saveBlock(block, batch, connect, callback ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { var self = this; - if (this.options.spv) - return utils.nextTick(callback); - - this._ensureHistory(hash, function(err, block) { + this.getBlock(hash, function(err, block) { if (err) return callback(err); @@ -736,6 +727,9 @@ ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { batch.del('b/' + block.hash('hex')); + if (self.options.spv) + return callback(null, block); + self.disconnectBlock(block, batch, callback); }); }; @@ -747,13 +741,13 @@ ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) { * @param {Function} callback - Returns [Error, {@link Block}]. */ -ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { +ChainDB.prototype.connectBlock = function connectBlock(block, view, batch, callback) { var self = this; var undo = new BufferWriter(); - var i, j, tx, input, output, key, addresses, address, hash, coin; + var i, j, tx, input, output, key, addresses, address, hash, coins, raw; if (this.options.spv) { - self.emit('add block', block); + this.emit('add block', block); return utils.nextTick(callback); } @@ -761,81 +755,76 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { if (this.chain.isGenesis(block)) return utils.nextTick(callback); - this._ensureBlock(block, function(err, block) { - if (err) - return callback(err); + for (i = 0; i < block.txs.length; i++) { + tx = block.txs[i]; + hash = tx.hash('hex'); - if (!block) - return callback(); - - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; - hash = tx.hash('hex'); - - if (self.options.indexTX) { - batch.put('t/' + hash, tx.toExtended()); - if (self.options.indexAddress) { - addresses = tx.getHashes(); - for (j = 0; j < addresses.length; j++) { - address = addresses[j]; - batch.put('T/' + address + '/' + hash, DUMMY); - } + if (this.options.indexTX) { + batch.put('t/' + hash, tx.toExtended()); + if (this.options.indexAddress) { + addresses = tx.getHashes(); + for (j = 0; j < addresses.length; j++) { + address = addresses[j]; + batch.put('T/' + address + '/' + hash, DUMMY); } } - - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - key = input.prevout.hash + '/' + input.prevout.index; - - if (tx.isCoinbase()) - break; - - assert(input.coin); - - if (self.options.indexAddress) { - address = input.getHash(); - if (address) - batch.del('C/' + address + '/' + key); - } - - batch.del('c/' + key); - - Framer.coin(input.coin, false, undo); - - self.coinCache.remove(key); - } - - for (j = 0; j < tx.outputs.length; j++) { - output = tx.outputs[j]; - key = hash + '/' + j; - - if (output.script.isUnspendable()) - continue; - - coin = bcoin.coin.fromTX(tx, j).toRaw(); - - if (self.options.indexAddress) { - address = output.getHash(); - if (address) - batch.put('C/' + address + '/' + key, DUMMY); - } - - batch.put('c/' + key, coin); - - self.coinCache.set(key, coin); - } } - if (undo.written > 0) - batch.put('u/' + block.hash('hex'), undo.render()); + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + key = input.prevout.hash + '/' + input.prevout.index; - self.emit('add block', block); + if (tx.isCoinbase()) + break; - self._pruneBlock(block, batch, function(err) { - if (err) - return callback(err); - return callback(null, block); - }); + assert(input.coin); + + if (this.options.indexAddress) { + address = input.getHash(); + if (address) + batch.del('C/' + address + '/' + key); + } + + Framer.coin(input.coin, false, undo); + } + + for (j = 0; j < tx.outputs.length; j++) { + output = tx.outputs[j]; + + if (output.script.isUnspendable()) + continue; + + if (this.options.indexAddress) { + address = output.getHash(); + if (address) + batch.put('C/' + address + '/' + key, DUMMY); + } + } + } + + view = view.toArray(); + + for (i = 0; i < view.length; i++) { + coins = view[i]; + if (coins.count() === 0) { + batch.del('c/' + coins.hash); + this.coinCache.remove(coins.hash); + } else { + raw = coins.toRaw(); + batch.put('c/' + coins.hash, raw); + this.coinCache.set(coins.hash, raw); + } + } + + if (undo.written > 0) + batch.put('u/' + block.hash('hex'), undo.render()); + + this.emit('add block', block); + + this._pruneBlock(block, batch, function(err) { + if (err) + return callback(err); + return callback(null, block); }); }; @@ -848,18 +837,15 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) { ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callback) { var self = this; - var i, j, tx, input, output, key, addresses, address, hash, coin; + var i, j, tx, input, output, key, addresses, address, hash, view, coins, raw; if (this.options.spv) return utils.nextTick(callback); - this._ensureHistory(block, function(err, block) { + this.getUndoView(block, function(err, view) { if (err) return callback(err); - if (!block) - return callback(new Error('Block not found.')); - for (i = block.txs.length - 1; i >= 0; i--) { tx = block.txs[i]; hash = tx.hash('hex'); @@ -889,14 +875,10 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb if (address) batch.put('C/' + address + '/' + key, DUMMY); } - - coin = input.coin.toRaw(); - - batch.put('c/' + key, coin); - - self.coinCache.set(key, coin); } + view.add(tx.toCoins()); + for (j = 0; j < tx.outputs.length; j++) { output = tx.outputs[j]; key = hash + '/' + j; @@ -910,9 +892,21 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb batch.del('C/' + address + '/' + key); } - batch.del('c/' + key); + view.spend(hash, j); + } + } - self.coinCache.remove(key); + view = view.toArray(); + + for (i = 0; i < view.length; i++) { + coins = view[i]; + if (coins.count() === 0) { + batch.del('c/' + coins.hash); + self.coinCache.remove(coins.hash); + } else { + raw = coins.toRaw(); + batch.put('c/' + coins.hash, raw); + self.coinCache.set(coins.hash, raw); } } @@ -933,16 +927,6 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb ChainDB.prototype.fillCoins = function fillCoins(tx, callback) { var self = this; - if (Array.isArray(tx)) { - return utils.forEachSerial(tx, function(tx, next) { - self.fillCoins(tx, next); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); - } - if (tx.isCoinbase()) return utils.asyncify(callback)(null, tx); @@ -975,16 +959,6 @@ ChainDB.prototype.fillCoins = function fillCoins(tx, callback) { ChainDB.prototype.fillHistory = function fillHistory(tx, callback) { var self = this; - if (Array.isArray(tx)) { - return utils.forEachSerial(tx, function(tx, next) { - self.fillHistory(tx, next); - }, function(err) { - if (err) - return callback(err); - return callback(null, tx); - }); - } - if (tx.isCoinbase()) return utils.asyncify(callback)(null, tx); @@ -1017,28 +991,54 @@ ChainDB.prototype.fillHistory = function fillHistory(tx, callback) { ChainDB.prototype.getCoin = function getCoin(hash, index, callback) { var self = this; - var key = hash + '/' + index; - var coin; + var coins = this.coinCache.get(hash); - coin = this.coinCache.get(key); - if (coin) { + if (coins) { callback = utils.asyncify(callback); + + if (raw) + return callback(null, coins); + try { - coin = bcoin.coin.fromRaw(coin); - coin.hash = hash; - coin.index = index; + coins = bcoin.coins.parseCoin(coins, hash, index); } catch (e) { return callback(e); } - return callback(null, coin); + + return callback(null, coins); } - this.db.fetch('c/' + key, function(data) { - var coin = bcoin.coin.fromRaw(data); - coin.hash = hash; - coin.index = index; - self.coinCache.set(key, data); - return coin; + this.db.fetch('c/' + hash, function(data) { + self.coinCache.set(hash, data); + return bcoin.coins.parseCoin(data, hash, index); + }, callback); +}; + +/** + * Get coins (unspents only). + * @param {Hash} hash + * @param {Function} callback - Returns [Error, {@link Coins}]. + */ + +ChainDB.prototype.getCoins = function getCoins(hash, callback) { + var self = this; + var coins = this.coinCache.get(hash); + + if (coins) { + callback = utils.asyncify(callback); + + try { + coins = bcoin.coins.fromRaw(coins, hash); + } catch (e) { + return callback(e); + } + + return callback(null, coins); + } + + this.db.fetch('c/' + hash, function(data) { + self.coinCache.set(hash, data); + return bcoin.coins.fromRaw(data, hash); }, callback); }; @@ -1096,27 +1096,32 @@ ChainDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, call utils.forEachSerial(addresses, function(address, next) { address = bcoin.address.getHash(address); + if (!address) return next(); - self.db.lookup({ + + self.db.iterate({ gte: 'C/' + address, lte: 'C/' + address + '~', transform: function(key) { key = key.split('/'); - return 'c/' + key[2] + '/' + key[3]; - }, - parse: function(data, key) { - var coin = bcoin.coin.fromRaw(data); - var hash = key.split('/'); - coin.hash = hash[1]; - coin.index = +hash[2]; - return coin; + return [key[2], +key[3]]; } - }, function(err, coin) { + }, function(err, keys) { if (err) return next(err); - coins = coins.concat(coin); - next(); + + utils.forEachSerial(keys, function(key, next) { + self.getCoin(key[0], key[1], function(err, coin) { + if (err) + return callback(err); + + if (coin) + coins.push(coin); + + return next(); + }); + }, next); }); }, function(err) { if (err) @@ -1217,76 +1222,40 @@ ChainDB.prototype.getFullBlock = function getFullBlock(hash, callback) { if (!block) return callback(); - return self.fillHistoryBlock(block, callback); + return self.getUndoView(block, function(err, view) { + if (err) + return callback(err); + + return callback(null, block); + }); }); }; -ChainDB.prototype._ensureBlock = function _ensureBlock(hash, callback) { - var self = this; - - if (hash instanceof bcoin.block) - return utils.asyncify(callback)(null, hash); - - return this.getBlock(hash, function(err, block) { - if (err) - return callback(err); - - if (!block) - return callback(); - - return self.fillBlock(block, callback); - }); -}; - -ChainDB.prototype._ensureHistory = function _ensureHistory(hash, callback) { - if (hash instanceof bcoin.block) - return utils.asyncify(callback)(null, hash); - - return this.getFullBlock(hash, callback); -}; - /** * Fill a block with coins (unspent only). * @param {Block} block * @param {Function} callback - Returns [Error, {@link Block}]. */ -ChainDB.prototype.fillBlock = function fillBlock(block, callback) { - var coins, spent, i, tx, hash, j, input, key; +ChainDB.prototype.getCoinView = function getCoinView(block, callback) { + var self = this; + var view = new bcoin.coinview(); - return this.fillCoins(block.txs, function(err) { + utils.forEachSerial(block.getPrevout(), function(prevout, next) { + self.getCoins(prevout, function(err, coins) { + if (err) + return next(err); + + if (coins) + view.add(coins); + + next(); + }); + }, function(err) { if (err) return callback(err); - coins = {}; - spent = {}; - - for (i = 0; i < block.txs.length; i++) { - tx = block.txs[i]; - hash = tx.hash('hex'); - - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - key = input.prevout.hash + '/' + input.prevout.index; - - if (spent[key]) { - delete input.coin; - continue; - } - - spent[key] = true; - - if (!input.coin && coins[key]) { - input.coin = coins[key]; - delete coins[key]; - } - } - - for (j = 0; j < tx.outputs.length; j++) - coins[hash + '/' + j] = bcoin.coin.fromTX(tx, j); - } - - return callback(null, block); + return callback(null, view); }); }; @@ -1317,31 +1286,39 @@ ChainDB.prototype.getUndoCoins = function getUndoCoins(hash, callback) { * @param {Function} callback - Returns [Error, {@link Block}]. */ -ChainDB.prototype.fillHistoryBlock = function fillHistoryBlock(block, callback) { - var i, j, k, tx, input; +ChainDB.prototype.getUndoView = function getUndoView(block, callback) { + var self = this; + var i, j, k, tx, input, coin; - return this.getUndoCoins(block.hash('hex'), function(err, coins) { + return this.getCoinView(block, function(err, view) { if (err) return callback(err); - if (!coins) - return callback(null, block); + return self.getUndoCoins(block.hash('hex'), function(err, coins) { + if (err) + return callback(err); - for (i = 0, k = 0; i < block.txs.length; i++) { - tx = block.txs[i]; + if (!coins) + return callback(null, view); - if (tx.isCoinbase()) - continue; + for (i = 0, k = 0; i < block.txs.length; i++) { + tx = block.txs[i]; - for (j = 0; j < tx.inputs.length; j++) { - input = tx.inputs[j]; - input.coin = coins[k++]; - input.coin.hash = input.prevout.hash; - input.coin.index = input.prevout.index; + if (tx.isCoinbase()) + continue; + + for (j = 0; j < tx.inputs.length; j++) { + input = tx.inputs[j]; + coin = coins[k++]; + coin.hash = input.prevout.hash; + coin.index = input.prevout.index; + input.coin = coin; + view.addCoin(coin); + } } - } - return callback(null, block); + return callback(null, view); + }); }); }; @@ -1395,32 +1372,13 @@ ChainDB.prototype.isUnspentTX = function isUnspentTX(hash, callback) { */ ChainDB.prototype.isSpentTX = function isSpentTX(hash, callback) { - var iter; - if (hash.hash) hash = hash.hash('hex'); - iter = this.db.iterator({ - gte: 'c/' + hash, - lte: 'c/' + hash + '~', - keys: true, - values: false, - fillCache: false, - keyAsBuffer: false - }); - - iter.next(function(err, key, value) { - if (err) { - return iter.end(function() { - callback(err); - }); - } - - iter.end(function(err) { - if (err) - return callback(err); - return callback(null, key === undefined); - }); + this.getCoins(hash, function(err, coins) { + if (err) + return callback(err); + return callback(null, !coins); }); };