From 59f06bb06cd7a6e35874de07f0d1661d4fdcb164 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 11 Jun 2016 15:45:26 -0700 Subject: [PATCH] start estimating fees. --- lib/bcoin/env.js | 2 +- lib/bcoin/fees.js | 91 +++++++++++++++++++++----------------------- lib/bcoin/mempool.js | 48 ++++++++++++++++++++--- lib/bcoin/network.js | 4 ++ lib/bcoin/peer.js | 2 +- lib/bcoin/tx.js | 2 +- 6 files changed, 94 insertions(+), 55 deletions(-) diff --git a/lib/bcoin/env.js b/lib/bcoin/env.js index 17eed165..d127ce90 100644 --- a/lib/bcoin/env.js +++ b/lib/bcoin/env.js @@ -138,8 +138,8 @@ function Environment(options) { this.lowlevelup = require('./lowlevelup'); this.uri = require('./uri'); - this.network = require('./network'); this.protocol = require('./protocol'); + this.network = require('./network'); this.errors = require('./errors'); this.ldb = require('./ldb'); this.profiler = require('./profiler'); diff --git a/lib/bcoin/fees.js b/lib/bcoin/fees.js index 86e7f506..ed1acfc4 100644 --- a/lib/bcoin/fees.js +++ b/lib/bcoin/fees.js @@ -8,6 +8,9 @@ var bcoin = require('./env'); var constants = bcoin.protocol.constants; +var global = bcoin.utils.global; +var Float64Array = global.Float64Array || Array; +var Int32Array = global.Int32Array || Array; /* * Constants @@ -27,7 +30,7 @@ var MAX_PRIORITY = 1e16; var INF_PRIORITY = 1e9 * constants.MAX_MONEY; var FEE_SPACING = 1.1; var PRI_SPACING = 2; -var FREE_THRESHOLD = constants.FREE_THRESHOLD; +var FREE_THRESHOLD = constants.tx.FREE_THRESHOLD; /** * Confirmation stats. @@ -45,7 +48,11 @@ function ConfirmStats(buckets, maxConfirms, decay, type) { if (!(this instanceof ConfirmStats)) return new ConfirmStats(buckets, maxConfirms, decay, type); - this.buckets = new Array(buckets.length); + this.maxConfirms = maxConfirms; + this.decay = decay; + this.type = type; + + this.buckets = new Float64Array(buckets.length); this.bucketMap = {}; for (i = 0; i < buckets.length; i++) { @@ -53,36 +60,23 @@ function ConfirmStats(buckets, maxConfirms, decay, type) { this.bucketMap[buckets[i]] = i; } - this.maxConfirms = maxConfirms; - this.decay = decay; - this.type = type; - this.confAvg = new Array(maxConfirms); this.curBlockConf = new Array(maxConfirms); this.unconfTX = new Array(maxConfirms); for (i = 0; i < maxConfirms; i++) { - this.confAvg[i] = new Array(buckets.length); - this.curBlockConf[i] = new Array(buckets.length); - this.unconfTX[i] = new Array(buckets.length); + this.confAvg[i] = new Float64Array(buckets.length); + this.curBlockConf[i] = new Int32Array(buckets.length); + this.unconfTX[i] = new Int32Array(buckets.length); } - this.oldUnconfTX = new Array(buckets.length); - this.curBlockTX = new Array(buckets.length); - this.txAvg = new Array(buckets.length); - this.curBlockVal = new Array(buckets.length); - this.avg = new Array(buckets.length); + this.oldUnconfTX = new Int32Array(buckets.length); + this.curBlockTX = new Int32Array(buckets.length); + this.txAvg = new Float64Array(buckets.length); + this.curBlockVal = new Float64Array(buckets.length); + this.avg = new Float64Array(buckets.length); } -/** - * Get max confirmations. - * @returns {Number} - */ - -ConfirmStats.prototype.getMaxConfirms = function getMaxConfirms() { - return this.confAvg.length; -}; - /** * Clear data for the current block. * @param {Number} height @@ -114,6 +108,7 @@ ConfirmStats.prototype.record = function record(blocks, val) { return; bucketIndex = this.bucketMap[val]; + for (i = blocks; i <= this.curBlockConf.length; i++) this.curBlockConf[i - 1][bucketIndex]++; @@ -168,8 +163,8 @@ ConfirmStats.prototype.estimateMedian = function estimateMedian(target, needed, conf += this.confAvg[target - 1][i]; total += this.txAvg[i]; - for (j = target; j < this.getMaxConfirms(); j++) - extra += this.unconfTX[(height - j) % bins][i]; + for (j = target; j < this.maxConfirms; j++) + extra += this.unconfTX[Math.max(height - j, 0) % bins][i]; extra += this.oldUnconfTX[i]; @@ -198,8 +193,8 @@ ConfirmStats.prototype.estimateMedian = function estimateMedian(target, needed, minBucket = bestNear < bestFar ? bestNear : bestFar; maxBucket = bestNear > bestFar ? bestNear : bestFar; - for (j = minBucket; j <= maxBucket; j++) - sum += this.txAvg[j]; + for (i = minBucket; i <= maxBucket; i++) + sum += this.txAvg[i]; if (found && sum !== 0) { sum = sum / 2; @@ -213,7 +208,7 @@ ConfirmStats.prototype.estimateMedian = function estimateMedian(target, needed, } } - bcoin.debug('estimatefee: %d:' + bcoin.debug('estimatefee: target=%d.' + ' For conf success %s %d need %s %s: %d from buckets %d - %d.' + ' Cur Bucket stats %d% %d/%d (%d mempool).', target, @@ -224,7 +219,7 @@ ConfirmStats.prototype.estimateMedian = function estimateMedian(target, needed, median, this.buckets[minBucket], this.buckets[maxBucket], - 100 * conf / (total + extra), + 100 * conf / Math.max(1, total + extra), conf, total, extra); @@ -295,10 +290,10 @@ ConfirmStats.prototype.removeTX = function removeTX(entryHeight, bestHeight, buc */ function PolicyEstimator(minRelay, network) { - var feelist, prilist, boundary; + var fee, priority, boundary; if (!(this instanceof PolicyEstimator)) - return new PolicyEstimator(minRelay); + return new PolicyEstimator(minRelay, network); this.network = bcoin.network.get(network); @@ -306,36 +301,36 @@ function PolicyEstimator(minRelay, network) { ? MIN_FEERATE : minRelay; - feelist = []; + fee = []; for (boundary = this.minTrackedFee; boundary <= MAX_FEERATE; boundary *= FEE_SPACING) { - feelist.push(boundary); + fee.push(boundary); } - feelist.push(INF_FEERATE); + fee.push(INF_FEERATE); this.feeStats = new ConfirmStats( - feelist, MAX_BLOCK_CONFIRMS, + fee, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, 'FeeRate'); this.minTrackedPri = FREE_THRESHOLD < MIN_PRIORITY ? MIN_PRIORITY : FREE_THRESHOLD; - prilist = []; + priority = []; for (boundary = this.minTrackedPri; boundary <= MAX_PRIORITY; boundary *= PRI_SPACING) { - prilist.push(boundary); + priority.push(boundary); } - prilist.push(INF_PRIORITY); + priority.push(INF_PRIORITY); this.priStats = new ConfirmStats( - prilist, MAX_BLOCK_CONFIRMS, + priority, MAX_BLOCK_CONFIRMS, DEFAULT_DECAY, 'Priority'); this.feeUnlikely = 0; @@ -422,8 +417,8 @@ PolicyEstimator.prototype.processTX = function processTX(entry, current) { if (entry.dependencies) return; - fee = entry.tx.getFee(); - rate = entry.tx.getRate(); + fee = entry.getFee(); + rate = entry.getRate(); priority = entry.getPriority(height); bcoin.debug('estimatefee: Processing mempool tx %s.', hash); @@ -438,9 +433,11 @@ PolicyEstimator.prototype.processTX = function processTX(entry, current) { blockHeight: height, bucketIndex: this.feeStats.addTX(height, rate) }; + bcoin.debug('estimatefee: Rate: %d.', this.estimateFee(1, true)); } else { bcoin.debug('estimatefee: Not adding ts %s.', hash); } + }; /** @@ -462,8 +459,8 @@ PolicyEstimator.prototype.processBlockTX = function processBlockTX(height, entry return; } - fee = entry.tx.getFee(); - rate = entry.tx.getRate(); + fee = entry.getFee(); + rate = entry.getRate(); priority = entry.getPriority(height); if (fee === 0 || this.isPriPoint(rate, priority)) @@ -544,7 +541,7 @@ PolicyEstimator.prototype.processBlock = function processBlock(height, entries, PolicyEstimator.prototype.estimateFee = function estimateFee(target, smart) { var rate, minPoolFee; - if (target <= 0 || target > this.feeStats.getMaxConfirms()) + if (target <= 0 || target > this.feeStats.maxConfirms) return 0; if (!smart) { @@ -559,7 +556,7 @@ PolicyEstimator.prototype.estimateFee = function estimateFee(target, smart) { } rate = -1; - while (rate < 0 && target <= this.feeStats.getMaxConfirms()) { + while (rate < 0 && target <= this.feeStats.maxConfirms) { rate = this.feeStats.estimateMedian( target++, SUFFICIENT_FEETXS, MIN_SUCCESS_PCT, true, this.bestHeight); @@ -587,7 +584,7 @@ PolicyEstimator.prototype.estimateFee = function estimateFee(target, smart) { PolicyEstimator.prototype.estimatePriority = function estimatePriority(target, smart) { var minPoolFee, priority; - if (target <= 0 || target > this.priStats.getMaxConfirms()) + if (target <= 0 || target > this.priStats.maxConfirms) return -1; if (!smart) { @@ -602,7 +599,7 @@ PolicyEstimator.prototype.estimatePriority = function estimatePriority(target, s return INF_PRIORITY; priority = -1; - while (priority < 0 && target <= this.priStats.getMaxConfirms()) { + while (priority < 0 && target <= this.priStats.maxConfirms) { priority = this.priStats.estimateMedian( target++, SUFFICIENT_PRITXS, MIN_SUCCESS_PCT, true, this.bestHeight); diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js index 9b22f311..c016d862 100644 --- a/lib/bcoin/mempool.js +++ b/lib/bcoin/mempool.js @@ -228,11 +228,13 @@ Mempool.prototype.destroy = function destroy(callback) { Mempool.prototype.addBlock = function addBlock(block, callback, force) { var self = this; var unlock = this._lock(addBlock, [block, callback], force); + var entries; if (!unlock) return; callback = utils.wrap(callback, unlock); + entries = []; utils.forEachSerial(block.txs.slice().reverse(), function(tx, next) { var hash = tx.hash('hex'); @@ -255,6 +257,8 @@ Mempool.prototype.addBlock = function addBlock(block, callback, force) { self.emit('confirmed', tx, block); + entries.push(entry); + return next(); }, true); }); @@ -265,6 +269,8 @@ Mempool.prototype.addBlock = function addBlock(block, callback, force) { self.blockSinceBump = true; self.lastFeeUpdate = utils.now(); + self.network.fees.processBlock(block.height, entries, self.chain.isFull()); + return callback(); }); }; @@ -792,6 +798,8 @@ Mempool.prototype.addUnchecked = function addUnchecked(entry, callback, force) { self.emit('tx', entry.tx); self.emit('add tx', entry.tx); + self.network.fees.processTX(entry, self.chain.isFull()); + bcoin.debug('Added tx %s to the mempool.', entry.tx.rhash); resolved = self.resolveOrphans(entry.tx); @@ -855,6 +863,8 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(entry, limit, callb self.size -= self.memUsage(entry.tx); self.total--; + self.network.fees.removeTX(entry.tx.hash('hex')); + if (limit) { rate = bcoin.tx.getRate(entry.size, entry.fees); rate += self.minReasonableFee; @@ -1788,12 +1798,14 @@ function MempoolEntry(options) { this.tx = options.tx; this.height = options.height; + this.size = options.size; this.priority = options.priority; + this.fee = options.fee; this.ts = options.ts; this.chainValue = options.chainValue; this.count = options.count; - this.size = options.size; + this.sizes = options.sizes; this.fees = options.fees; this.dependencies = options.dependencies; } @@ -1808,6 +1820,8 @@ function MempoolEntry(options) { MempoolEntry.fromTX = function fromTX(tx, height) { var data = tx.getPriority(height); var dependencies = false; + var size = tx.getVirtualSize(); + var fee = tx.getFee(); var i; for (i = 0; i < tx.inputs.length; i++) { @@ -1820,12 +1834,14 @@ MempoolEntry.fromTX = function fromTX(tx, height) { return new MempoolEntry({ tx: tx, height: height, + size: size, priority: data.priority, + fee: fee, chainValue: data.value, ts: utils.now(), count: 1, - size: tx.getVirtualSize(), - fees: tx.getFee(), + sizes: size, + fees: fee, dependencies: dependencies }); }; @@ -1844,11 +1860,13 @@ MempoolEntry.prototype.toRaw = function toRaw(writer) { bcoin.protocol.framer.renderTX(this.tx, true, p); p.writeU32(this.height); + p.writeU32(this.size); p.writeDouble(this.priority); + p.writeVarint(this.fee); p.writeVarint(this.chainValue); p.writeU32(this.ts); p.writeU32(this.count); - p.writeU32(this.size); + p.writeU32(this.sizes); p.writeVarint(this.fees); p.writeU8(this.dependencies ? 1 : 0); @@ -1869,11 +1887,13 @@ MempoolEntry.fromRaw = function fromRaw(data) { return new MempoolEntry({ tx: bcoin.tx.fromRaw(p), height: p.readU32(), + size: p.readU32(), priority: p.readDouble(), + fee: p.readVarint(), chainValue: p.readVarint(), ts: p.readU32(), count: p.readU32(), - size: p.readU32(), + sizes: p.readU32(), fees: p.readVarint(), dependencies: p.readU8() === 1 }); @@ -1897,6 +1917,24 @@ MempoolEntry.prototype.getPriority = function getPriority(height) { return result; }; +/** + * Get fee. + * @returns {Amount} + */ + +MempoolEntry.prototype.getFee = function getFee() { + return this.fee; +}; + +/** + * Calculate fee rate. + * @returns {Rate} + */ + +MempoolEntry.prototype.getRate = function getRate() { + return bcoin.tx.getRate(this.size, this.fee); +}; + /** * Test whether the entry is free with * the current priority (calculated by diff --git a/lib/bcoin/network.js b/lib/bcoin/network.js index b13f40d1..d60ea64d 100644 --- a/lib/bcoin/network.js +++ b/lib/bcoin/network.js @@ -7,7 +7,9 @@ var utils = require('./utils'); var assert = utils.assert; +var constants = require('./protocol/constants'); var network = require('./protocol/network'); +var PolicyEstimator = require('./fees'); /** * Represents a network. @@ -41,6 +43,8 @@ function Network(options) { this[key] = value; } + this.fees = new PolicyEstimator(constants.tx.MIN_RELAY, this); + if (!Network[this.type]) Network[this.type] = this; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 2f6fbfa1..56a8dd4b 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -1304,7 +1304,7 @@ Peer.prototype._handleGetData = function _handleGetData(items) { // the `mempool` handler, but it would be // too slow. if (self.feeRate !== -1) { - if (bcoin.tx.getRate(entry.size, entry.fees) < self.feeRate) + if (entry.getRate() < self.feeRate) return next(); } diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js index 8496b510..62209d26 100644 --- a/lib/bcoin/tx.js +++ b/lib/bcoin/tx.js @@ -1744,7 +1744,7 @@ TX.getMinFee = function getMinFee(size, rate) { * @returns {Rate} */ -TX.getRate = function(size, fee) { +TX.getRate = function getRate(size, fee) { return Math.floor(fee * 1000 / size); };