From a940ea51582f97c688c2b331aaf120317dacf378 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 11 May 2016 19:09:48 -0700 Subject: [PATCH] mempool size. --- lib/bcoin/mempool.js | 231 +++++++++++++++++++++++++++---------------- 1 file changed, 148 insertions(+), 83 deletions(-) diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 84e37bea..c425ec0c 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -77,6 +77,8 @@ function Mempool(options) { this.tx = null; this.size = 0; this.orphans = 0; + this.spent = 0; + this.total = 0; this.freeCount = 0; this.lastTime = 0; @@ -145,12 +147,11 @@ Mempool.prototype._init = function _init() { unlock(); return self.emit('error', err); } - self.dynamicMemoryUsage(function(err, size) { - if (err) - self.emit('error', err); - else - self.size = size; - + self.initialMemoryUsage(function(err) { + if (err) { + unlock(); + return self.emit('error', err); + } self.chain.open(function(err) { if (err) { unlock(); @@ -165,8 +166,28 @@ Mempool.prototype._init = function _init() { }); }; -Mempool.prototype.dynamicMemoryUsage = function dynamicMemoryUsage(callback) { - return this.db.approximateSize('m', 'm~', callback); +/** + * Tally up total memory usage from database. + * @param {Function} callback - Returns [Error, Number]. + */ + +Mempool.prototype.initialMemoryUsage = function initialMemoryUsage(callback) { + var self = this; + var i, tx; + + this.getHistory(function(err, txs) { + if (err) + return callback(err); + + for (i = 0; i < txs.length; i++) { + tx = txs[i]; + self.size += self.memUsage(tx); + self.spent += tx.inputs.length; + self.total++; + } + + return callback(); + }); }; /** @@ -341,7 +362,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { var self = this; var rate; - if (this.size <= this.maxSize) + if (this.getSize() <= this.maxSize) return callback(null, true); this.tx.getRange({ @@ -352,6 +373,9 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { return callback(err); utils.forEachSerial(function(tx, next) { + if (self.getSize() <= self.maxSize) + return next(); + self.removeUnchecked(tx, function(err) { if (err) return next(err); @@ -370,12 +394,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { if (err) return callback(err); - self.purgeOrphans(function(err) { - if (err) - return callback(err); - - return callback(self.size <= self.maxSize); - }); + return callback(null, self.getSize() <= self.maxSize); }); }); }; @@ -385,61 +404,49 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { * @param {Function} callback */ -Mempool.prototype.purgeOrphans = function purgeOrphans(callback) { +Mempool.prototype.pruneOrphans = function pruneOrphans(callback) { var self = this; - var batch = this.db.batch(); - var tx; + var hashes = []; + var iter; callback = utils.ensure(callback); - utils.forEachSerial(['O', 'o'], function(type, callback) { - var iter = self.db.iterator({ - gte: type, - lte: type + '~', - keys: true, - values: true, - fillCache: false, - keyAsBuffer: false - }); + iter = this.db.iterator({ + gte: 'O', + lte: 'O~', + keys: true, + values: false, + fillCache: false, + keyAsBuffer: false + }); - (function next() { - iter.next(function(err, key, value) { - if (err) { - return iter.end(function() { - callback(err); - }); - } - - if (type === 'O') { - try { - tx = bcoin.tx.fromExtended(value, true); - } catch (e) { - return callback(e); - } - self.size -= memOrphan(tx); - } - - if (key === undefined) - return iter.end(callback); - - batch.del(key); - - next(); - }); - })(); - }, function(err) { + function done(err) { if (err) return callback(err); - batch.write(function(err) { - if (err) - return callback(err); + utils.forEachSerial(hashes, function(hash, next) { + if (self.orphans <= constants.mempool.MAX_ORPHAN_TX) + return next(); + self.removeOrphan(hash, next); + }, callback); + } - self.orphans = 0; + (function next() { + iter.next(function(err, key, value) { + if (err) { + return iter.end(function() { + callback(err); + }); + } - return callback(); + if (key === undefined) + return iter.end(done); + + hashes.push(key.split('/')[1]); + + next(); }); - }); + })(); }; /** @@ -586,7 +593,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { if (tx.ts !== 0) { return callback(new VerifyError(tx, 'alreadyknown', - 'txn-already-in-mempool', + 'txn-already-known', 0)); } @@ -658,7 +665,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { return callback(err); if (!tx.hasCoins()) { - if (self.size > self.maxSize) { + if (self.getSize() > self.maxSize) { return callback(new VerifyError(tx, 'insufficientfee', 'mempool full', @@ -708,7 +715,9 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) { if (err) return callback(err); - self.size += mem(tx); + self.spent += tx.inputs.length; + self.size += self.memUsage(tx); + self.total++; self.emit('tx', tx); self.emit('add tx', tx); @@ -767,7 +776,9 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback, limit if (err) return callback(err); - self.size -= mem(tx); + self.spent -= tx.inputs.length; + self.size -= self.memUsage(tx); + self.total--; self.emit('remove tx', tx); @@ -1009,9 +1020,15 @@ Mempool.prototype.countAncestors = function countAncestors(tx, callback) { Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) { var self = this; var prevout = {}; - var batch = this.db.batch(); - var hash = tx.hash('hex'); - var i, input, p; + var i, hash, batch, input, p; + + if (tx.getSize() > 5000) { + bcoin.debug('Ignoring large orphan: %s', tx.rhash); + return callback(); + } + + hash = tx.hash('hex'); + batch = this.db.batch(); for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; @@ -1044,12 +1061,11 @@ Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) { return callback(err); self.orphans++; - self.size += memOrphan(tx); batch.put('O/' + hash, tx.toExtended(true)); if (self.orphans > constants.mempool.MAX_ORPHAN_TX) { - return self.purgeOrphans(function(err) { + return self.pruneOrphans(function(err) { if (err) return callback(err); batch.write(callback); @@ -1267,7 +1283,7 @@ Mempool.prototype.removeOrphan = function removeOrphan(tx, callback) { if (err) return callback(err); - self.size -= memOrphan(tx); + self.orphans--; batch.write(callback); }); @@ -1664,6 +1680,57 @@ Mempool.prototype._removeUnchecked = function removeUnchecked(hash, callback, fo }); }; +Mempool.prototype.memUsage = function memUsage(tx) { + if (this.accurateMemory) + return this.memUsageAccurate(tx); + return this.memUsageBitcoind(tx); +}; + +Mempool.prototype.memUsageAccurate = function memUsage(tx) { + return 0 + + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended + + (2 + 64) // t + + (2 + 10 + 1 + 64) // m + + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s + + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c +}; + +Mempool.prototype.memUsageBitcoind = function memUsage(tx) { + var mem = 0; + var i, input; + + mem += mallocUsage(tx.inputs.length) + mem += mallocUsage(tx.outputs.length); + + for (i = 0; i < tx.inputs.length; i++) + mem += mallocUsage(tx.inputs[i].script.getSize()) + 0; + + for (i = 0; i < tx.outputs.length; i++) + mem += mallocUsage(tx.outputs[i].script.getSize()); + + mem += mallocUsage(tx.inputs.length); + + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + mem += mallocUsage(input.witness.length); + for (j = 0; j < input.witness.length; j++) + mem += mallocUsage(input.witness[j].length); + } + + return mem; +}; + +Mempool.prototype.getSize = function getSize() { + if (this.accurateMemory) + return this.size; + + return mallocUsage(162 + 15 * ptrSize) * this.total // entries + + mallocUsage(this.spent) // mapNextTx + + mallocUsage(this.total) // mapDeltas + + mallocUsage(this.total) // mapLinks + + this.size; +}; + function MempoolTX(options) { this.tx = options.tx; this.height = options.height; @@ -1737,21 +1804,19 @@ MempoolTX.prototype.isFree = function isFree(height) { * Helpers */ -function mem(tx) { - return 0 - + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended - + (2 + 64) // t - + (2 + 10 + 1 + 64) // m - + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s - + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c -} +// Assume 64 bit for arm since version +// number is not exposed by node.js. +var ptrSize = (process.platform == null + || process.platform === 'x64' + || process.platform === 'ia64' + || process.platform === 'arm') ? 8 : 4; -function memOrphan(tx) { - return 0 - + (2 + 64 + 32) // o - + (2 + 64) // O - + (tx.getSize() + 4 + 32 + 4 + 4 + 4 - + (tx.inputs.length >>> 1) * 80); // extended +function mallocUsage(alloc) { + if (alloc == 0) + return 0; + if (ptrSize === 8) + return ((alloc + 31) >> 4) << 4; + return ((alloc + 15) >> 3) << 3; } return Mempool;