diff --git a/lib/bcoin/http/rpc.js b/lib/bcoin/http/rpc.js index e190651c..b137c431 100644 --- a/lib/bcoin/http/rpc.js +++ b/lib/bcoin/http/rpc.js @@ -1051,9 +1051,9 @@ RPC.prototype.getdifficulty = function getdifficulty(args, callback) { RPC.prototype.getmempoolinfo = function getmempoolinfo(args, callback) { callback(null, { - size: this.mempool.total, - bytes: this.mempool.size, - usage: this.mempool.size, + size: this.mempool.totalTX, + bytes: this.mempool.getSize(), + usage: this.mempool.getSize(), maxmempool: constants.mempool.MAX_MEMPOOL_SIZE, mempoolminfee: +utils.btc(this.mempool.minFeeRate) }); diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index cc6826cf..1dedf519 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -16,12 +16,10 @@ var bcoin = require('./env'); var AsyncObject = require('./async'); var constants = bcoin.protocol.constants; var utils = require('./utils'); -var RBT = require('./rbt'); var assert = utils.assert; var BufferWriter = require('./writer'); var BufferReader = require('./reader'); var VerifyError = bcoin.errors.VerifyError; -var ptrSize; /** * Represents a mempool. @@ -80,14 +78,14 @@ function Mempool(options) { this.locker = new bcoin.locker(this, this.addTX); this.size = 0; + this.totalOrphans = 0; + this.totalSpent = 0; + this.totalTX = 0; + this.waiting = {}; this.orphans = {}; - this.totalOrphans = 0; - this.spent = 0; - this.total = 0; this.tx = {}; this.spents = {}; - this.time = new RBT(timeCmp); this.coinIndex = new AddressIndex(this); this.txIndex = new AddressIndex(this); @@ -102,7 +100,6 @@ function Mempool(options) { : this.network.requireStandard; this.rejectAbsurdFees = this.options.rejectAbsurdFees !== false; this.prematureWitness = !!this.options.prematureWitness; - this.accurateMemory = !!this.options.accurateMemory; this.maxSize = options.maxSize || constants.mempool.MAX_MEMPOOL_SIZE; this.blockSinceBump = false; @@ -131,7 +128,7 @@ Mempool.prototype._open = function open(callback) { */ Mempool.prototype._close = function destroy(callback) { - callback(); + return callback(); }; /** @@ -251,20 +248,28 @@ Mempool.prototype.removeBlock = function removeBlock(block, callback, force) { Mempool.prototype.limitMempoolSize = function limitMempoolSize(entryHash, callback) { var self = this; var trimmed = false; - var end, entries, hashes, entry; + var end, hashes, entry; if (this.getSize() <= this.maxSize) return callback(null, trimmed); + hashes = Object.keys(this.tx); end = utils.now() - constants.mempool.MEMPOOL_EXPIRY; - entries = this.getRange(0, end); - utils.forEachSerial(entries, function(entry, next) { + utils.forEachSerial(hashes, function(hash, next) { + var entry = self.getEntry(hash); + + if (!entry) + return next(); + if (self.getSize() <= self.maxSize) return callback(null, trimmed); - if (!trimmed) - trimmed = entry.tx.hash('hex') === entryHash; + if (entry.ts >= end) + return next(); + + if (!trimmed && hash === entryHash) + trimmed = true; self.removeUnchecked(entry, true, next, true); }, function(err) { @@ -274,8 +279,6 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(entryHash, callba if (self.getSize() <= self.maxSize) return callback(null, trimmed); - hashes = self.getSnapshot(); - utils.forEachSerial(hashes, function(hash, next) { if (self.getSize() <= self.maxSize) return callback(null, trimmed); @@ -285,8 +288,8 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(entryHash, callba if (!entry) return next(); - if (!trimmed) - trimmed = hash === entryHash; + if (!trimmed && hash === entryHash) + trimmed = true; self.removeUnchecked(entry, true, next, true); }, function(err) { @@ -375,8 +378,7 @@ Mempool.prototype.getCoin = function getCoin(hash, index) { */ Mempool.prototype.isSpent = function isSpent(hash, index) { - var key = hash + index; - return this.spents[key]; + return this.spents[hash + index]; }; /** @@ -503,31 +505,6 @@ Mempool.prototype.hasTX = function hasTX(hash) { return this.tx[hash] != null; }; -/** - * Find transactions within a range. - * @param {Object} range - * @param {Function} callback - Returns [Error, {@link TX}[]]. - */ - -Mempool.prototype.getRange = function getRange(start, end) { - var items = this.time.range(start, end); - var entries = []; - var i, item, hash, entry; - - for (i = 0; i < items.length; i++) { - item = items[i]; - hash = item.value.toString('hex'); - entry = this.getEntry(hash); - if (!entry) { - this.time.remove(item.key); - continue; - } - entries.push(entry); - } - - return entries; -}; - /** * Test the mempool to see if it contains a transaction or an orphan. * @param {Hash} hash @@ -716,9 +693,9 @@ Mempool.prototype.addUnchecked = function addUnchecked(entry, callback, force) { this._addUnchecked(entry); - this.spent += entry.tx.inputs.length; + this.totalSpent += entry.tx.inputs.length; this.size += this.memUsage(entry.tx); - this.total++; + this.totalTX++; this.emit('tx', entry.tx); this.emit('add tx', entry.tx); @@ -773,41 +750,36 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(entry, limit, callb callback = utils.wrap(callback, unlock); - hash = entry.tx.hash('hex'); + this.removeOrphan(entry.tx); - this.fillAllHistory(entry.tx, function(err) { + this._removeUnchecked(entry, limit, function(err) { if (err) return callback(err); - self.removeOrphan(entry.tx); + self.totalSpent -= entry.tx.inputs.length; + self.size -= self.memUsage(entry.tx); + self.totalTX--; - self._removeUnchecked(entry, limit, function(err) { - if (err) - return callback(err); + if (self.fees) { + hash = entry.tx.hash('hex'); + self.fees.removeTX(hash); + } - self.spent -= entry.tx.inputs.length; - self.size -= self.memUsage(entry.tx); - self.total--; - - if (self.fees) - self.fees.removeTX(hash); - - if (limit) { - self.logger.spam('Removed tx %s from mempool.', entry.tx.rhash); - rate = bcoin.tx.getRate(entry.sizes, entry.fees); - rate += self.minReasonableFee; - if (rate > self.minFeeRate) { - self.minFeeRate = rate; - self.blockSinceBump = false; - } - } else { - self.logger.spam('Removed block tx %s from mempool.', entry.tx.rhash); + if (limit) { + self.logger.spam('Removed tx %s from mempool.', entry.tx.rhash); + rate = bcoin.tx.getRate(entry.sizes, entry.fees); + rate += self.minReasonableFee; + if (rate > self.minFeeRate) { + self.minFeeRate = rate; + self.blockSinceBump = false; } + } else { + self.logger.spam('Removed block tx %s from mempool.', entry.tx.rhash); + } - self.emit('remove tx', entry.tx); + self.emit('remove tx', entry.tx); - return callback(); - }); + return callback(); }); }; @@ -1012,6 +984,52 @@ Mempool.prototype.countAncestors = function countAncestors(tx) { return max; }; +/** + * Count the highest number of + * descendants a transaction may have. + * @param {TX} tx + * @param {Function} callback - Returns [Error, Number]. + */ + +Mempool.prototype.countDescendants = function countDescendants(tx) { + var max = 0; + var hash = tx.hash('hex'); + var i, count, next; + + for (i = 0; i < tx.outputs.length; i++) { + next = this.isSpent(hash, i); + if (!next) + continue; + count = 1; + count += this.countDescendants(next); + if (count > max) + max = count; + } + + return max; +}; + +/** + * Find a unconfirmed transactions that + * this transaction depends on. + * @param {TX} tx + * @param {Function} callback - Returns [Error, Number]. + */ + +Mempool.prototype.getDepends = function getDepends(tx) { + var prevout = tx.getPrevout(); + var depends = []; + var i, hash; + + for (i = 0; i < prevout.length; i++) { + hash = prevout[i].hash; + if (this.hasTX(hash)) + depends.push(hash); + } + + return depends; +}; + /** * Store an orphaned transaction. * @param {TX} tx @@ -1250,7 +1268,6 @@ Mempool.prototype.fillAllHistory = function fillAllHistory(tx, callback) { Mempool.prototype.fillAllCoins = function fillAllCoins(tx, callback) { var self = this; - var doubleSpend = false; this.fillCoins(tx); @@ -1261,10 +1278,8 @@ Mempool.prototype.fillAllCoins = function fillAllCoins(tx, callback) { var hash = input.prevout.hash; var index = input.prevout.index; - if (self.isSpent(hash, index)) { - doubleSpend = true; + if (self.isSpent(hash, index)) return next(); - } self.chain.db.getCoin(hash, index, function(err, coin) { if (err) @@ -1281,7 +1296,7 @@ Mempool.prototype.fillAllCoins = function fillAllCoins(tx, callback) { if (err) return callback(err); - return callback(null, tx, doubleSpend); + return callback(null, tx); }); }; @@ -1337,7 +1352,6 @@ Mempool.prototype.isDoubleSpend = function isDoubleSpend(tx) { */ Mempool.prototype.getConfidence = function getConfidence(hash, callback) { - var self = this; var tx; callback = utils.asyncify(callback); @@ -1346,17 +1360,17 @@ Mempool.prototype.getConfidence = function getConfidence(hash, callback) { tx = hash; hash = hash.hash('hex'); } else { - tx = self.getTX(hash); + tx = this.getTX(hash); } - if (self.hasTX(hash)) + if (this.hasTX(hash)) return callback(null, constants.confidence.PENDING); - if (tx && self.isDoubleSpend(tx)) + if (tx && this.isDoubleSpend(tx)) return callback(null, constants.confidence.INCONFLICT); if (tx && tx.block) { - return self.chain.db.isMainChain(tx.block, function(err, result) { + return this.chain.db.isMainChain(tx.block, function(err, result) { if (err) return callback(err); @@ -1367,7 +1381,7 @@ Mempool.prototype.getConfidence = function getConfidence(hash, callback) { }); } - self.chain.db.hasCoins(hash, function(err, existing) { + this.chain.db.hasCoins(hash, function(err, existing) { if (err) return callback(err); @@ -1388,10 +1402,9 @@ Mempool.prototype.getConfidence = function getConfidence(hash, callback) { Mempool.prototype._addUnchecked = function _addUnchecked(entry) { var tx = entry.tx; var hash = tx.hash('hex'); - var i, input, output, key, coin, spender; + var i, input, output, key, coin; this.tx[hash] = entry; - this.time.insert(entry.ts, hash); if (this.options.indexAddress) this.indexTX.addTX(tx); @@ -1413,7 +1426,6 @@ Mempool.prototype._addUnchecked = function _addUnchecked(entry) { for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; - key = hash + i; if (output.script.isUnspendable()) continue; @@ -1459,7 +1471,6 @@ Mempool.prototype._removeUnchecked = function _removeUnchecked(entry, limit, cal return callback(err); delete self.tx[hash]; - self.time.remove(entry.ts); if (self.options.indexAddress) self.txIndex.addTX(tx); @@ -1473,26 +1484,14 @@ Mempool.prototype._removeUnchecked = function _removeUnchecked(entry, limit, cal delete self.spents[key]; - // We only disconnect inputs if this - // is a limited transaction. For block - // transactions, the coins are still - // spent. They were spent on the - // blockchain. - if (!limit) - continue; - - assert(input.coin); - - if (input.coin.height !== -1) - continue; - - if (self.options.indexAddress) + if (self.options.indexAddress) { + assert(input.coin); self.coinIndex.removeCoin(input.coin); + } } for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; - key = hash + i; if (output.script.isUnspendable()) continue; @@ -1551,60 +1550,37 @@ Mempool.prototype._removeSpenders = function _removeSpenders(entry, limit, callb */ Mempool.prototype.memUsage = function memUsage(tx) { - if (this.accurateMemory) - return this.memUsageAccurate(tx); - return this.memUsageBitcoind(tx); -}; - -/** - * Calculate the memory usage of a transaction - * accurately (the amount bcoin is actually using). - * @param {TX} tx - * @returns {Number} Usage in bytes. - */ - -Mempool.prototype.memUsageAccurate = function memUsageAccurate(tx) { - return 0 - + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended - + (2 + 64) // t - + (2 + 10 + 1 + 64) // m - + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s - + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c -}; - -/** - * Calculate the memory usage of a transaction based on - * bitcoind's memory estimation algorithm. This will - * _not_ be accurate to bcoin's actual memory usage, - * but it helps accurately replicate the bitcoind - * mempool. - * @see DynamicMemoryUsage() - * @param {TX} tx - * @returns {Number} Usage in bytes. - */ - -Mempool.prototype.memUsageBitcoind = function memUsageBitcoind(tx) { var mem = 0; - var i, j, input; + var i, j, input, output; - mem += mallocUsage(tx.inputs.length); - mem += mallocUsage(tx.outputs.length); - - for (i = 0; i < tx.inputs.length; i++) - mem += mallocUsage(tx.inputs[i].script.getSize()); - - for (i = 0; i < tx.outputs.length; i++) - mem += mallocUsage(tx.outputs[i].script.getSize()); - - mem += mallocUsage(tx.inputs.length); + mem += 8; for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - mem += mallocUsage(input.witness.items.length); + if (input.coin) { + mem += 93; + mem += input.coin.script.getSize() * 2; + } + mem += 76; + mem += input.script.getSize() * 2; for (j = 0; j < input.witness.items.length; j++) - mem += mallocUsage(input.witness.items[j].length); + mem += input.witness.items[j].length; } + for (i = 0; i < tx.outputs.length; i++) { + output = tx.outputs[i]; + mem += 8; + mem += output.script.getSize() * 2; + } + + mem += 8; + + // Map entry + mem += 64; + + // Spent entry + mem += tx.outputs.length * 72; + return mem; }; @@ -1615,14 +1591,7 @@ Mempool.prototype.memUsageBitcoind = function memUsageBitcoind(tx) { */ Mempool.prototype.getSize = function getSize() { - if (this.accurateMemory) - return this.size; - - return mallocUsage(162 + 15 * ptrSize) * this.total // entries - + mallocUsage(this.spent) // mapNextTx - + mallocUsage(this.total) // mapDeltas - + mallocUsage(this.total) // mapLinks - + this.size; + return this.size; }; /** @@ -1862,86 +1831,28 @@ MempoolEntry.prototype.isFree = function isFree(height) { return priority > constants.tx.FREE_THRESHOLD; }; -/* - * Helpers - */ - -/** - * "Guessed" pointer size based on ISA. This - * assumes 64 bit for arm since the arm - * version number is not exposed by node.js. - * @memberof Mempool - * @const {Number} - */ - -ptrSize = (process.platform == null - || process.platform === 'x64' - || process.platform === 'ia64' - || process.platform === 'arm') ? 8 : 4; - -/** - * Calculate malloc usage based on pointer size. - * If you're scratching your head as to why this - * function is here, it is only here to accurately - * replicate bitcoind's memory usage algorithm. - * (I know javascript doesn't have malloc or - * pointers). - * @memberof Mempool - * @param {Number} alloc - Size of Buffer object. - * @returns {Number} Allocated size. - */ - -function mallocUsage(alloc) { - if (alloc === 0) - return 0; - if (ptrSize === 8) - return ((alloc + 31) >>> 4) << 4; - return ((alloc + 15) >>> 3) << 3; -} - /** * Address Index */ function AddressIndex(mempool) { this.mempool = mempool; - this.tree = new RBT(); + this.map = {}; } -AddressIndex.prototype.search = function(hash) { - return this.tree.range(hash + '/', hash + '/~'); -}; - -AddressIndex.prototype.set = function set(hash, postfix, value) { - return this.tree.insert(hash + '/' + postfix, value); -}; - -AddressIndex.prototype.remove = function remove(hash, postfix) { - return this.tree.remove(hash + '/' + postfix); -}; - AddressIndex.prototype.searchCoin = function searchCoin(address) { - var items = this.search(address); + var items = this.map[address]; var out = []; var i, item, outpoint, coin; + if (!items) + return out; + for (i = 0; i < items.length; i++) { item = items[i]; - - try { - outpoint = bcoin.outpoint.fromRaw(item.value); - } catch (e) { - this.tree.remove(item.key); - continue; - } - + outpoint = bcoin.outpoint.fromRaw(item); coin = this.mempool.getCoin(outpoint.hash, outpoint.index); - - if (!coin) { - this.tree.remove(item.key); - continue; - } - + assert(coin); out.push(coin); } @@ -1949,20 +1860,17 @@ AddressIndex.prototype.searchCoin = function searchCoin(address) { }; AddressIndex.prototype.searchTX = function searchTX(address) { - var items = this.search(address); + var items = this.map[address]; var out = []; - var i, item, hash, tx; + var i, hash, tx; + + if (!items) + return out; for (i = 0; i < items.length; i++) { - item = items[i]; - hash = item.value.toString('hex'); - tx = this.mempool.getEntry(hash); - - if (!tx) { - this.tree.remove(item.key); - continue; - } - + hash = items[i].toString('hex'); + tx = this.mempool.getTX(hash); + assert(tx); out.push(tx); } @@ -1971,47 +1879,75 @@ AddressIndex.prototype.searchTX = function searchTX(address) { AddressIndex.prototype.addTX = function addTX(tx) { var hashes = tx.getHashes('hex'); - var i, hash; + var i, hash, items; for (i = 0; i < hashes.length; i++) { hash = hashes[i]; - this.set(hash, tx.hash('hex'), tx.hash()); + items = this.map[hash]; + + if (!items) { + items = []; + this.map[hash] = items; + } + + utils.binaryInsert(items, tx.hash(), utils.cmp); } }; AddressIndex.prototype.removeTX = function removeTX(tx) { var hashes = tx.getHashes('hex'); - var i, hash; + var i, hash, items; for (i = 0; i < hashes.length; i++) { hash = hashes[i]; - this.remove(hash, tx.hash('hex')); + items = this.map[hash]; + + if (!items) + continue; + + utils.binaryRemove(items, tx.hash(), utils.cmp); + + if (items.length === 0) + delete this.map[hash]; } }; AddressIndex.prototype.addCoin = function addCoin(coin) { var hash = coin.getHash('hex'); - var outpoint; + var outpoint, items; if (!hash) return; - outpoint = bcoin.outpoint(coin.hash, coin.index); - this.set(hash, coin.hash + coin.index, outpoint.toRaw()); + items = this.map[hash]; + + if (!items) { + items = []; + this.map[hash] = items; + } + + outpoint = bcoin.outpoint(coin.hash, coin.index).toRaw(); + utils.binaryInsert(items, outpoint, utils.cmp); }; AddressIndex.prototype.removeCoin = function removeCoin(coin) { var hash = coin.getHash('hex'); + var outpoint, items; if (!hash) return; - this.remove(hash, coin.hash + coin.index); -}; + items = this.map[hash]; -function timeCmp(a, b) { - return a - b; -} + if (!items) + return; + + outpoint = bcoin.outpoint(coin.hash, coin.index).toRaw(); + utils.binaryRemove(items, outpoint, utils.cmp); + + if (items.length === 0) + delete this.map[hash]; +}; /* * Expose diff --git a/lib/bcoin/miner.js b/lib/bcoin/miner.js index d877147c..d84f744c 100644 --- a/lib/bcoin/miner.js +++ b/lib/bcoin/miner.js @@ -231,7 +231,7 @@ Miner.prototype.stop = function stop() { Miner.prototype.createBlock = function createBlock(version, callback) { var self = this; var ts = Math.max(bcoin.now(), this.chain.tip.ts + 1); - var attempt; + var i, attempt, txs, tx; if (typeof version === 'function') { callback = version; @@ -280,31 +280,14 @@ Miner.prototype.createBlock = function createBlock(version, callback) { if (!self.mempool) return callback(null, attempt); - self.mempool.getSnapshot(function(err, hashes) { - if (err) - return callback(err); + txs = self.mempool.getHistory(); - utils.forEachSerial(hashes, function(hash, next) { - self.mempool.getTX(hash, function(err, tx) { - if (err) - return next(err); + for (i = 0; i < txs.length; i++) { + tx = txs[i]; + attempt.addTX(tx); + } - self.mempool.fillAllCoins(tx, function(err) { - if (err) - return next(err); - - attempt.addTX(tx); - - next(); - }); - }); - }, function(err) { - if (err) - return callback(err); - - return callback(null, attempt); - }); - }); + return callback(null, attempt); }); }); }; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 17be2078..d83c0cb1 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -1117,36 +1117,16 @@ Peer.prototype._handleGetUTXOs = function _handleGetUTXOs(payload) { if (this.chain.db.options.spv) return done(); - function checkMempool(hash, index, callback) { - if (!self.mempool) - return callback(); - - if (!payload.mempool) - return callback(); - - callback(null, self.mempool.getCoin(hash, index)); - } - - function isSpent(hash, index, callback) { - if (!self.mempool) - return callback(null, false); - - if (!payload.mempool) - return callback(null, false); - - callback(null, self.mempool.isSpent(hash, index)); - } - if (payload.prevout.length > 15) return done(); utils.forEachSerial(payload.prevout, function(prevout, next) { var hash = prevout.hash; var index = prevout.index; + var coin; - checkMempool(hash, index, function(err, coin) { - if (err) - return next(err); + if (self.mempool && payload.mempool) { + coin = self.mempool.getCoin(hash, index); if (coin) { hits.push(1); @@ -1154,30 +1134,25 @@ Peer.prototype._handleGetUTXOs = function _handleGetUTXOs(payload) { return next(); } - isSpent(hash, index, function(err, result) { - if (err) - return next(err); + if (self.mempool.isSpent(hash, index)) { + hits.push(0); + return next(); + } + } - if (result) { - hits.push(0); - return next(); - } + self.chain.db.getCoin(hash, index, function(err, coin) { + if (err) + return next(err); - self.chain.db.getCoin(hash, index, function(err, coin) { - if (err) - return next(err); + if (!coin) { + hits.push(0); + return next(); + } - if (!coin) { - hits.push(0); - return next(); - } + hits.push(1); + coins.push(coin); - hits.push(1); - coins.push(coin); - - next(); - }); - }); + next(); }); }, function(err) { if (err)