diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 743d3cc1..84e37bea 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -38,7 +38,7 @@ var DUMMY = new Buffer([0]); * @param {Number?} options.limitFreeRelay * @param {Boolean?} options.relayPriority * @param {Boolean?} options.requireStandard - * @param {Boolean?} options.rejectInsaneFees + * @param {Boolean?} options.rejectAbsurdFees * @param {Boolean?} options.relay * @property {Boolean} loaded * @property {Object} db @@ -47,7 +47,6 @@ var DUMMY = new Buffer([0]); * @property {Locker} locker * @property {Number} freeCount * @property {Number} lastTime - * @property {String} backend * @emits Mempool#open * @emits Mempool#error * @emits Mempool#tx @@ -88,10 +87,15 @@ function Mempool(options) { this.requireStandard = this.options.requireStandard != null ? this.options.requireStandard : network.requireStandard; - this.rejectInsaneFees = this.options.rejectInsaneFees !== false; + this.rejectAbsurdFees = this.options.rejectAbsurdFees !== false; + this.prematureWitness = !!this.options.prematureWitness; - // Use an in-memory binary search tree by default - this.backend = this.options.memory === false ? 'leveldb' : 'memory'; + this.maxSize = options.maxSize || constants.mempool.MAX_MEMPOOL_SIZE; + this.minFeeRate = 0; + this.blockSinceBump = false; + this.lastFeeUpdate = utils.now(); + this.minReasonableFee = constants.tx.MIN_FEE; + this.minRelayFee = constants.tx.MIN_FEE; this._init(); } @@ -116,7 +120,7 @@ Mempool.prototype._init = function _init() { var options = { name: this.options.name || 'mempool', location: this.options.location, - db: this.options.db || this.backend + db: this.options.db || 'memory' }; assert(unlock); @@ -269,7 +273,15 @@ Mempool.prototype.addBlock = function addBlock(block, callback, force) { return next(); }); }); - }, callback); + }, function(err) { + if (err) + return callback(err); + + self.blockSinceBump = true; + self.lastFeeUpdate = utils.now(); + + return callback(); + }); }); }; @@ -327,8 +339,9 @@ Mempool.prototype.removeBlock = function removeBlock(block, callback, force) { Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { var self = this; + var rate; - if (this.size <= constants.mempool.MAX_MEMPOOL_SIZE) + if (this.size <= this.maxSize) return callback(null, true); this.tx.getRange({ @@ -339,7 +352,20 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { return callback(err); utils.forEachSerial(function(tx, next) { - self.removeUnchecked(tx, next); + self.removeUnchecked(tx, function(err) { + if (err) + return next(err); + + rate = tx.getFee().muln(1000).divn(tx.getVirtualSize()).toNumber(); + rate += self.minReasonableFee; + + if (rate > self.minFeeRate) { + self.minFeeRate = rate; + self.blockSinceBump = false; + } + + next(); + }); }, function(err) { if (err) return callback(err); @@ -348,7 +374,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { if (err) return callback(err); - return callback(self.size <= constants.mempool.MAX_MEMPOOL_SIZE); + return callback(self.size <= self.maxSize); }); }); }); @@ -362,6 +388,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) { Mempool.prototype.purgeOrphans = function purgeOrphans(callback) { var self = this; var batch = this.db.batch(); + var tx; callback = utils.ensure(callback); @@ -370,7 +397,7 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) { gte: type, lte: type + '~', keys: true, - values: false, + values: true, fillCache: false, keyAsBuffer: false }); @@ -383,6 +410,15 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) { }); } + if (type === 'O') { + try { + tx = bcoin.tx.fromExtended(value, true); + } catch (e) { + return callback(e); + } + self.size -= memOrphan(tx); + } + if (key === undefined) return iter.end(callback); @@ -399,15 +435,9 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) { if (err) return callback(err); - self.dynamicMemoryUsage(function(err, size) { - if (err) - return callback(err); + self.orphans = 0; - self.size = size; - self.orphans = 0; - - return callback(); - }); + return callback(); }); }); }; @@ -539,6 +569,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { var self = this; var flags = constants.flags.STANDARD_VERIFY_FLAGS; var lockFlags = constants.flags.STANDARD_LOCKTIME_FLAGS; + var hash = tx.hash('hex'); var ret = {}; var now; @@ -559,11 +590,6 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { 0)); } - if (!this.chain.segwitActive) { - if (tx.hasWitness()) - return callback(new VerifyError(tx, 'nonstandard', 'no-witness-yet', 0)); - } - if (!tx.isSane(ret)) return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score)); @@ -582,6 +608,11 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { } } + if (!this.chain.segwitActive && !this.prematureWitness) { + if (tx.hasWitness()) + return callback(new VerifyError(tx, 'nonstandard', 'no-witness-yet', 0)); + } + this.chain.checkFinal(this.chain.tip, tx, lockFlags, function(err, isFinal) { if (err) return callback(err); @@ -589,7 +620,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { if (!isFinal) return callback(new VerifyError(tx, 'nonstandard', 'non-final', 0)); - self.seenTX(tx, function(err, exists) { + self.has(hash, function(err, exists) { if (err) return callback(err); @@ -600,47 +631,59 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) { 0)); } - self.isDoubleSpend(tx, function(err, doubleSpend) { + self.chain.db.isUnspentTX(hash, function(err, exists) { if (err) return callback(err); - if (doubleSpend) { + if (exists) { return callback(new VerifyError(tx, - 'duplicate', - 'bad-txns-inputs-spent', + 'alreadyknown', + 'txn-already-known', 0)); } - self.fillAllCoins(tx, function(err) { + self.isDoubleSpend(tx, function(err, doubleSpend) { if (err) return callback(err); - if (!tx.hasCoins()) { - if (self.totalSize > constants.mempool.MAX_MEMPOOL_SIZE) { - return callback(new VerifyError(tx, - 'insufficientfee', - 'mempool full', - 0)); - } - return self.storeOrphan(tx, callback); + if (doubleSpend) { + return callback(new VerifyError(tx, + 'duplicate', + 'bad-txns-inputs-spent', + 0)); } - self.verify(tx, function(err) { + self.fillAllCoins(tx, function(err) { if (err) return callback(err); - self.limitMempoolSize(function(err, result) { - if (err) - return callback(err); - - if (!result) { + if (!tx.hasCoins()) { + if (self.size > self.maxSize) { return callback(new VerifyError(tx, 'insufficientfee', 'mempool full', 0)); } + return self.storeOrphan(tx, callback); + } - self.addUnchecked(tx, callback); + self.verify(tx, function(err) { + if (err) + return callback(err); + + self.limitMempoolSize(function(err, result) { + if (err) + return callback(err); + + if (!result) { + return callback(new VerifyError(tx, + 'insufficientfee', + 'mempool full', + 0)); + } + + self.addUnchecked(tx, callback); + }); }); }); }); @@ -665,7 +708,7 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) { if (err) return callback(err); - self.size += tx.getSize(); + self.size += mem(tx); self.emit('tx', tx); self.emit('add tx', tx); @@ -708,8 +751,10 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) { * @param {Function} callback */ -Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback) { +Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback, limit) { var self = this; + var rate; + this.fillAllHistory(tx, function(err, tx) { if (err) return callback(err); @@ -721,14 +766,57 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback) { self._removeUnchecked(tx, function(err) { if (err) return callback(err); - self.size -= tx.getSize(); + + self.size -= mem(tx); + self.emit('remove tx', tx); + return callback(); }); }); }); }; +/** + * Calculate and update the minimum rolling fee rate. + * @returns {Number} Rate. + */ + +Mempool.prototype.getMinRate = function getMinRate() { + var now, halflife, exp; + + if (!this.blockSinceBump || this.minFeeRate == 0) + return this.minFeeRate; + + now = utils.now(); + + if (now > this.lastFeeUpdate + 10) { + halflife = constants.mempool.FEE_HALFLIFE; + + if (this.size < this.maxSize / 4) { + halflife /= 4; + halflife |= 0; + } else if (this.size < this.maxSize / 2) { + halflife /= 2; + halflife |= 0; + } + + this.minFeeRate /= Math.pow(2.0, (now - this.lastFeeUpdate) / halflife | 0); + this.minFeeRate |= 0; + this.lastFeeUpdate = now; + + if (this.minFeeRate < this.minReasonableFee / 2) { + this.minFeeRate = 0; + return 0; + } + } + + if (this.minFeeRate > this.minReasonableFee) + return this.minFeeRate; + + return this.minReasonableFee; +}; + /** * Verify a transaction with mempool standards. * @param {TX} tx @@ -742,7 +830,7 @@ Mempool.prototype.verify = function verify(tx, callback) { var flags = constants.flags.STANDARD_VERIFY_FLAGS; var mandatory = constants.flags.MANDATORY_VERIFY_FLAGS; var ret = {}; - var fee, now, free, minFee; + var fee, modFee, now, size, rejectFee, minRelayFee; if (this.chain.segwitActive) mandatory |= constants.flags.VERIFY_WITNESS; @@ -772,37 +860,47 @@ Mempool.prototype.verify = function verify(tx, callback) { 0)); } - if (!tx.checkInputs(height, ret)) - return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score)); - + // In reality we would apply deltas + // to the modified fee (only for mining). fee = tx.getFee(); - minFee = tx.getMinFee(); - if (fee.cmp(minFee) < 0) { - if (self.relayPriority) { - free = tx.isFree(height); - if (!free) { - return callback(new VerifyError(tx, - 'insufficientfee', - 'insufficient priority', - 0)); - } - } else { + modFee = fee; + size = tx.getVirtualSize(); + rejectFee = tx.getMinFee(size, self.getMinRate()); + minRelayFee = tx.getMinFee(size, self.minRelayFee); + + if (rejectFee.cmpn(0) > 0 && modFee.cmp(rejectFee) < 0) { + return callback(new VerifyError(tx, + 'insufficientfee', + 'mempool min fee not met', + 0)); + } + + if (self.relayPriority && modFee.cmp(minRelayFee) < 0) { + if (!tx.isFree(height, size)) { return callback(new VerifyError(tx, 'insufficientfee', - 'insufficient fee', + 'insufficient priority', 0)); } } - if (self.limitFree && free) { + // Continuously rate-limit free (really, very-low-fee) + // transactions. This mitigates 'penny-flooding'. i.e. + // sending thousands of free transactions just to be + // annoying or make others' transactions take longer + // to confirm. + if (self.limitFree && modFee.cmp(minRelayFee) < 0) { now = utils.now(); if (!self.lastTime) self.lastTime = now; + // Use an exponentially decaying ~10-minute window: self.freeCount *= Math.pow(1 - 1 / 600, now - self.lastTime); self.lastTime = now; + // The limitFreeRelay unit is thousand-bytes-per-minute + // At default rate it would take over a month to fill 1GB if (self.freeCount > self.limitFreeRelay * 10 * 1000) { return callback(new VerifyError(tx, 'insufficientfee', @@ -810,12 +908,15 @@ Mempool.prototype.verify = function verify(tx, callback) { 0)); } - self.freeCount += tx.getSize(); + self.freeCount += size; } - if (self.rejectInsaneFees && fee.cmp(minFee.muln(10000)) > 0) + if (self.rejectAbsurdFees && fee.cmp(minRelayFee.muln(10000)) > 0) return callback(new VerifyError(tx, 'highfee', 'absurdly-high-fee', 0)); + if (!tx.checkInputs(height, ret)) + return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score)); + self.countAncestors(tx, function(err, count) { if (err) return callback(err); @@ -943,6 +1044,7 @@ Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) { return callback(err); self.orphans++; + self.size += memOrphan(tx); batch.put('O/' + hash, tx.toExtended(true)); @@ -1164,6 +1266,9 @@ Mempool.prototype.removeOrphan = function removeOrphan(tx, callback) { }, function(err) { if (err) return callback(err); + + self.size -= memOrphan(tx); + batch.write(callback); }); }); @@ -1559,5 +1664,95 @@ Mempool.prototype._removeUnchecked = function removeUnchecked(hash, callback, fo }); }; +function MempoolTX(options) { + this.tx = options.tx; + this.height = options.height; + this.priority = options.priority; + this.chainValue = options.chainValue; + + this.ts = options.ts; + this.count = options.count; + this.size = options.size; + this.fees = options.fees; +} + +MempoolTX.fromTX = function fromTX(tx, height) { + var data = tx.getPriority(height); + + return new MempoolTX({ + tx: tx, + height: height, + priority: data.priority, + chainValue: data.value, + ts: utils.now(), + count: 1, + size: tx.getVirtualSize(), + fees: tx.getFee() + }); +}; + +MempoolTX.prototype.toRaw = function toRaw() { + var p = new BufferWriter(); + bcoin.protocol.framer.tx(this.tx, p); + p.writeU32(this.height); + p.writeVarint(this.priority); + p.writeVarint(this.chainValue); + p.writeU32(this.ts); + p.writeU32(this.count); + p.writeU32(this.size); + p.writeVarint(this.fees); + return p.render(); +}; + +MempoolTX.fromRaw = function fromRaw(data, saveCoins) { + var p = new BufferReader(data); + return new MempoolTX({ + tx: bcoin.protocol.parser.parseTX(p), + height: p.readU32(), + priority: p.readVarint(true), + chainValue: p.readVarint(true), + ts: p.readU32(), + count: p.readU32(), + size: p.readU32(), + fees: p.readVarint(true) + }); +}; + +MempoolTX.prototype.getPriority = function getPriority(height) { + var heightDelta = Math.max(0, height - this.height); + var modSize = this.tx.getModifiedSize(); + var deltaPriority = new bn(heightDelta).mul(this.chainValue).divn(modSize); + var result = this.priority.add(deltaPriority); + if (result.cmpn(0) < 0) + result = new bn(0); + return result; +}; + +MempoolTX.prototype.isFree = function isFree(height) { + var priority = this.getPriority(height); + return priority.cmp(constants.tx.FREE_THRESHOLD) > 0; +}; + +/* + * Helpers + */ + +function mem(tx) { + return 0 + + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended + + (2 + 64) // t + + (2 + 10 + 1 + 64) // m + + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s + + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c +} + +function memOrphan(tx) { + return 0 + + (2 + 64 + 32) // o + + (2 + 64) // O + + (tx.getSize() + 4 + 32 + 4 + 4 + 4 + + (tx.inputs.length >>> 1) * 80); // extended +} + return Mempool; }; diff --git a/lib/bcoin/mtx.js b/lib/bcoin/mtx.js index b99ac201..0d1d4d65 100644 --- a/lib/bcoin/mtx.js +++ b/lib/bcoin/mtx.js @@ -1148,6 +1148,10 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) { size = tx.maxSize(options, true); if (tryFree) { + // Note that this will only work + // if the mempool's rolling reject + // fee is zero (i.e. the mempool is + // not full). if (tx.isFree(network.height + 1, size)) { fee = new bn(0); break; @@ -1156,9 +1160,9 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) { } if (options.accurate) - fee = tx.getMinFee(size); + fee = tx.getMinFee(size, options.rate); else - fee = tx.getMaxFee(size); + fee = tx.getMaxFee(size, options.rate); // Failed to get enough funds, add more coins. if (!isFull()) @@ -1182,7 +1186,7 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) { if (typeof options.subtractFee === 'number') { i = options.subtractFee; - if (i > tx.outputs.length - 1) + if (!tx.outputs[i]) throw new Error('Subtraction index does not exist.'); if (tx.outputs[i].value.cmp(minValue) < 0) diff --git a/lib/bcoin/protocol/constants.js b/lib/bcoin/protocol/constants.js index 0421ac3d..3f446db2 100644 --- a/lib/bcoin/protocol/constants.js +++ b/lib/bcoin/protocol/constants.js @@ -453,7 +453,7 @@ exports.mempool = { * Maximum mempool size in bytes. */ - MAX_MEMPOOL_SIZE: 300 << 20, + MAX_MEMPOOL_SIZE: 300 * 1000000, /** * The time at which transactions @@ -466,7 +466,13 @@ exports.mempool = { * Maximum number of orphan transactions. */ - MAX_ORPHAN_TX: 100 + MAX_ORPHAN_TX: 100, + + /** + * Decay of minimum fee rate. + */ + + FEE_HALFLIFE: 60 * 60 * 12 }; /** diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index a8a5d135..c8ae440d 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -1357,7 +1357,7 @@ TX.prototype.getModifiedSize = function getModifiedSize(size) { */ TX.prototype.getPriority = function getPriority(height, size) { - var sum, i, input, age; + var sum, i, input, age, value; if (this.isCoinbase()) return new bn(0); @@ -1369,9 +1369,10 @@ TX.prototype.getPriority = function getPriority(height, size) { } if (size == null) - size = this.getModifiedSize(); + size = this.maxSize(); sum = new bn(0); + value = new bn(0); for (i = 0; i < this.inputs.length; i++) { input = this.inputs[i]; @@ -1384,11 +1385,15 @@ TX.prototype.getPriority = function getPriority(height, size) { if (input.coin.height <= height) { age = height - input.coin.height; + value.iadd(input.coin.value); sum.iadd(input.coin.value.muln(age)); } } - return sum.divn(size); + return { + value: value, + priority: sum.divn(size) + }; }; /** @@ -1412,7 +1417,7 @@ TX.prototype.isFree = function isFree(height, size) { height = network.height + 1; } - priority = this.getPriority(height, size); + priority = this.getPriority(height, size).priority; return priority.cmp(constants.tx.FREE_THRESHOLD) > 0; }; @@ -1422,19 +1427,23 @@ TX.prototype.isFree = function isFree(height, size) { * to be relayable (not the constant min relay fee). * @param {Number?} size - If not present, max size * estimation will be calculated and used. + * @param {Number?} rate - Rate of satoshi per kB. * @returns {BN} fee */ -TX.prototype.getMinFee = function getMinFee(size) { +TX.prototype.getMinFee = function getMinFee(size, rate) { var fee; if (size == null) size = this.maxSize(); - fee = new bn(constants.tx.MIN_FEE).muln(size).divn(1000); + if (rate == null) + rate = constants.tx.MIN_FEE; - if (fee.cmpn(0) === 0 && constants.tx.MIN_FEE > 0) - fee = new bn(constants.tx.MIN_FEE); + fee = new bn(rate).muln(size).divn(1000); + + if (fee.cmpn(0) === 0 && rate > 0) + fee = new bn(rate); return fee; }; @@ -1445,19 +1454,23 @@ TX.prototype.getMinFee = function getMinFee(size) { * when taking into account size. * @param {Number?} size - If not present, max size * estimation will be calculated and used. + * @param {Number?} rate - Rate of satoshi per kB. * @returns {BN} fee */ -TX.prototype.getMaxFee = function getMaxFee(size) { +TX.prototype.getMaxFee = function getMaxFee(size, rate) { var fee; if (size == null) size = this.maxSize(); - fee = new bn(constants.tx.MIN_FEE).muln(Math.ceil(size / 1000)); + if (rate == null) + rate = constants.tx.MIN_FEE; - if (fee.cmpn(0) === 0 && constants.tx.MIN_FEE > 0) - fee = new bn(constants.tx.MIN_FEE); + fee = new bn(rate).muln(Math.ceil(size / 1000)); + + if (fee.cmpn(0) === 0 && rate > 0) + fee = new bn(rate); return fee; }; @@ -1617,13 +1630,13 @@ TX.prototype.inspect = function inspect() { hash: utils.revHex(this.hash('hex')), witnessHash: utils.revHex(this.witnessHash('hex')), size: this.getSize(), - virtualSize: this.getVirtualSize(), + virtualSize: this.maxSize(), height: this.height, value: utils.btc(this.getOutputValue()), fee: utils.btc(this.getFee()), minFee: utils.btc(this.getMinFee()), confirmations: this.getConfirmations(), - priority: this.getPriority().toString(10), + priority: this.getPriority().priority.toString(10), date: utils.date(this.ts || this.ps), block: this.block ? utils.revHex(this.block) : null, ts: this.ts,