mempool. fees.

lib/bcoin/mempool.js

@@ -38,7 +38,7 @@ var DUMMY = new Buffer([0]);
  * @param {Number?} options.limitFreeRelay
  * @param {Boolean?} options.relayPriority
  * @param {Boolean?} options.requireStandard
- * @param {Boolean?} options.rejectInsaneFees
+ * @param {Boolean?} options.rejectAbsurdFees
  * @param {Boolean?} options.relay
  * @property {Boolean} loaded
  * @property {Object} db
@@ -47,7 +47,6 @@ var DUMMY = new Buffer([0]);
  * @property {Locker} locker
  * @property {Number} freeCount
  * @property {Number} lastTime
- * @property {String} backend
  * @emits Mempool#open
  * @emits Mempool#error
  * @emits Mempool#tx
@@ -88,10 +87,15 @@ function Mempool(options) {
   this.requireStandard = this.options.requireStandard != null
     ? this.options.requireStandard
     : network.requireStandard;
-  this.rejectInsaneFees = this.options.rejectInsaneFees !== false;
+  this.rejectAbsurdFees = this.options.rejectAbsurdFees !== false;
+  this.prematureWitness = !!this.options.prematureWitness;
 
-  // Use an in-memory binary search tree by default
-  this.backend = this.options.memory === false ? 'leveldb' : 'memory';
+  this.maxSize = options.maxSize || constants.mempool.MAX_MEMPOOL_SIZE;
+  this.minFeeRate = 0;
+  this.blockSinceBump = false;
+  this.lastFeeUpdate = utils.now();
+  this.minReasonableFee = constants.tx.MIN_FEE;
+  this.minRelayFee = constants.tx.MIN_FEE;
 
   this._init();
 }
@@ -116,7 +120,7 @@ Mempool.prototype._init = function _init() {
   var options = {
     name: this.options.name || 'mempool',
     location: this.options.location,
-    db: this.options.db || this.backend
+    db: this.options.db || 'memory'
   };
 
   assert(unlock);
@@ -269,7 +273,15 @@ Mempool.prototype.addBlock = function addBlock(block, callback, force) {
           return next();
         });
       });
-    }, callback);
+    }, function(err) {
+      if (err)
+        return callback(err);
+
+      self.blockSinceBump = true;
+      self.lastFeeUpdate = utils.now();
+
+      return callback();
+    });
   });
 };
 
@@ -327,8 +339,9 @@ Mempool.prototype.removeBlock = function removeBlock(block, callback, force) {
 
 Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
   var self = this;
+  var rate;
 
-  if (this.size <= constants.mempool.MAX_MEMPOOL_SIZE)
+  if (this.size <= this.maxSize)
     return callback(null, true);
 
   this.tx.getRange({
@@ -339,7 +352,20 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
       return callback(err);
 
     utils.forEachSerial(function(tx, next) {
-      self.removeUnchecked(tx, next);
+      self.removeUnchecked(tx, function(err) {
+        if (err)
+          return next(err);
+
+        rate = tx.getFee().muln(1000).divn(tx.getVirtualSize()).toNumber();
+        rate += self.minReasonableFee;
+
+        if (rate > self.minFeeRate) {
+          self.minFeeRate = rate;
+          self.blockSinceBump = false;
+        }
+
+        next();
+      });
     }, function(err) {
       if (err)
         return callback(err);
@@ -348,7 +374,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
         if (err)
           return callback(err);
 
-        return callback(self.size <= constants.mempool.MAX_MEMPOOL_SIZE);
+        return callback(self.size <= self.maxSize);
       });
     });
   });
@@ -362,6 +388,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
 Mempool.prototype.purgeOrphans = function purgeOrphans(callback) {
   var self = this;
   var batch = this.db.batch();
+  var tx;
 
   callback = utils.ensure(callback);
 
@@ -370,7 +397,7 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) {
       gte: type,
       lte: type + '~',
       keys: true,
-      values: false,
+      values: true,
       fillCache: false,
       keyAsBuffer: false
     });
@@ -383,6 +410,15 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) {
           });
         }
 
+        if (type === 'O') {
+          try {
+            tx = bcoin.tx.fromExtended(value, true);
+          } catch (e) {
+            return callback(e);
+          }
+          self.size -= memOrphan(tx);
+        }
+
         if (key === undefined)
           return iter.end(callback);
 
@@ -399,15 +435,9 @@ Mempool.prototype.purgeOrphans = function purgeOrphans(callback) {
       if (err)
         return callback(err);
 
-      self.dynamicMemoryUsage(function(err, size) {
-        if (err)
-          return callback(err);
+      self.orphans = 0;
 
-        self.size = size;
-        self.orphans = 0;
-
-        return callback();
-      });
+      return callback();
     });
   });
 };
@@ -539,6 +569,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
   var self = this;
   var flags = constants.flags.STANDARD_VERIFY_FLAGS;
   var lockFlags = constants.flags.STANDARD_LOCKTIME_FLAGS;
+  var hash = tx.hash('hex');
   var ret = {};
   var now;
 
@@ -559,11 +590,6 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
       0));
   }
 
-  if (!this.chain.segwitActive) {
-    if (tx.hasWitness())
-      return callback(new VerifyError(tx, 'nonstandard', 'no-witness-yet', 0));
-  }
-
   if (!tx.isSane(ret))
     return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score));
 
@@ -582,6 +608,11 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
     }
   }
 
+  if (!this.chain.segwitActive && !this.prematureWitness) {
+    if (tx.hasWitness())
+      return callback(new VerifyError(tx, 'nonstandard', 'no-witness-yet', 0));
+  }
+
   this.chain.checkFinal(this.chain.tip, tx, lockFlags, function(err, isFinal) {
     if (err)
       return callback(err);
@@ -589,7 +620,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
     if (!isFinal)
       return callback(new VerifyError(tx, 'nonstandard', 'non-final', 0));
 
-    self.seenTX(tx, function(err, exists) {
+    self.has(hash, function(err, exists) {
       if (err)
         return callback(err);
 
@@ -600,47 +631,59 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
           0));
       }
 
-      self.isDoubleSpend(tx, function(err, doubleSpend) {
+      self.chain.db.isUnspentTX(hash, function(err, exists) {
         if (err)
           return callback(err);
 
-        if (doubleSpend) {
+        if (exists) {
           return callback(new VerifyError(tx,
-            'duplicate',
-            'bad-txns-inputs-spent',
+            'alreadyknown',
+            'txn-already-known',
             0));
         }
 
-        self.fillAllCoins(tx, function(err) {
+        self.isDoubleSpend(tx, function(err, doubleSpend) {
           if (err)
             return callback(err);
 
-          if (!tx.hasCoins()) {
-            if (self.totalSize > constants.mempool.MAX_MEMPOOL_SIZE) {
-              return callback(new VerifyError(tx,
-                'insufficientfee',
-                'mempool full',
-                0));
-            }
-            return self.storeOrphan(tx, callback);
+          if (doubleSpend) {
+            return callback(new VerifyError(tx,
+              'duplicate',
+              'bad-txns-inputs-spent',
+              0));
           }
 
-          self.verify(tx, function(err) {
+          self.fillAllCoins(tx, function(err) {
             if (err)
               return callback(err);
 
-            self.limitMempoolSize(function(err, result) {
-              if (err)
-                return callback(err);
-
-              if (!result) {
+            if (!tx.hasCoins()) {
+              if (self.size > self.maxSize) {
                 return callback(new VerifyError(tx,
                   'insufficientfee',
                   'mempool full',
                   0));
               }
+              return self.storeOrphan(tx, callback);
+            }
 
-              self.addUnchecked(tx, callback);
+            self.verify(tx, function(err) {
+              if (err)
+                return callback(err);
+
+              self.limitMempoolSize(function(err, result) {
+                if (err)
+                  return callback(err);
+
+                if (!result) {
+                  return callback(new VerifyError(tx,
+                    'insufficientfee',
+                    'mempool full',
+                    0));
+                }
+
+                self.addUnchecked(tx, callback);
+              });
             });
           });
         });
@@ -665,7 +708,7 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) {
     if (err)
       return callback(err);
 
-    self.size += tx.getSize();
+    self.size += mem(tx);
     self.emit('tx', tx);
     self.emit('add tx', tx);
 
@@ -708,8 +751,10 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) {
  * @param {Function} callback
  */
 
-Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback) {
+Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback, limit) {
   var self = this;
+  var rate;
+
   this.fillAllHistory(tx, function(err, tx) {
     if (err)
       return callback(err);
@@ -721,14 +766,57 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback) {
       self._removeUnchecked(tx, function(err) {
         if (err)
           return callback(err);
-        self.size -= tx.getSize();
+
+        self.size -= mem(tx);
+
         self.emit('remove tx', tx);
+
         return callback();
       });
     });
   });
 };
 
+/**
+ * Calculate and update the minimum rolling fee rate.
+ * @returns {Number} Rate.
+ */
+
+Mempool.prototype.getMinRate = function getMinRate() {
+  var now, halflife, exp;
+
+  if (!this.blockSinceBump || this.minFeeRate == 0)
+    return this.minFeeRate;
+
+  now = utils.now();
+
+  if (now > this.lastFeeUpdate + 10) {
+    halflife = constants.mempool.FEE_HALFLIFE;
+
+    if (this.size < this.maxSize / 4) {
+      halflife /= 4;
+      halflife |= 0;
+    } else if (this.size < this.maxSize / 2) {
+      halflife /= 2;
+      halflife |= 0;
+    }
+
+    this.minFeeRate /= Math.pow(2.0, (now - this.lastFeeUpdate) / halflife | 0);
+    this.minFeeRate |= 0;
+    this.lastFeeUpdate = now;
+
+    if (this.minFeeRate < this.minReasonableFee / 2) {
+      this.minFeeRate = 0;
+      return 0;
+    }
+  }
+
+  if (this.minFeeRate > this.minReasonableFee)
+    return this.minFeeRate;
+
+  return this.minReasonableFee;
+};
+
 /**
  * Verify a transaction with mempool standards.
  * @param {TX} tx
@@ -742,7 +830,7 @@ Mempool.prototype.verify = function verify(tx, callback) {
   var flags = constants.flags.STANDARD_VERIFY_FLAGS;
   var mandatory = constants.flags.MANDATORY_VERIFY_FLAGS;
   var ret = {};
-  var fee, now, free, minFee;
+  var fee, modFee, now, size, rejectFee, minRelayFee;
 
   if (this.chain.segwitActive)
     mandatory |= constants.flags.VERIFY_WITNESS;
@@ -772,37 +860,47 @@ Mempool.prototype.verify = function verify(tx, callback) {
         0));
     }
 
-    if (!tx.checkInputs(height, ret))
-      return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score));
-
+    // In reality we would apply deltas
+    // to the modified fee (only for mining).
     fee = tx.getFee();
-    minFee = tx.getMinFee();
-    if (fee.cmp(minFee) < 0) {
-      if (self.relayPriority) {
-        free = tx.isFree(height);
-        if (!free) {
-          return callback(new VerifyError(tx,
-            'insufficientfee',
-            'insufficient priority',
-            0));
-        }
-      } else {
+    modFee = fee;
+    size = tx.getVirtualSize();
+    rejectFee = tx.getMinFee(size, self.getMinRate());
+    minRelayFee = tx.getMinFee(size, self.minRelayFee);
+
+    if (rejectFee.cmpn(0) > 0 && modFee.cmp(rejectFee) < 0) {
+      return callback(new VerifyError(tx,
+        'insufficientfee',
+        'mempool min fee not met',
+        0));
+    }
+
+    if (self.relayPriority && modFee.cmp(minRelayFee) < 0) {
+      if (!tx.isFree(height, size)) {
         return callback(new VerifyError(tx,
           'insufficientfee',
-          'insufficient fee',
+          'insufficient priority',
           0));
       }
     }
 
-    if (self.limitFree && free) {
+    // Continuously rate-limit free (really, very-low-fee)
+    // transactions. This mitigates 'penny-flooding'. i.e.
+    // sending thousands of free transactions just to be
+    // annoying or make others' transactions take longer
+    // to confirm.
+    if (self.limitFree && modFee.cmp(minRelayFee) < 0) {
       now = utils.now();
 
       if (!self.lastTime)
         self.lastTime = now;
 
+      // Use an exponentially decaying ~10-minute window:
       self.freeCount *= Math.pow(1 - 1 / 600, now - self.lastTime);
       self.lastTime = now;
 
+      // The limitFreeRelay unit is thousand-bytes-per-minute
+      // At default rate it would take over a month to fill 1GB
       if (self.freeCount > self.limitFreeRelay * 10 * 1000) {
         return callback(new VerifyError(tx,
           'insufficientfee',
@@ -810,12 +908,15 @@ Mempool.prototype.verify = function verify(tx, callback) {
           0));
       }
 
-      self.freeCount += tx.getSize();
+      self.freeCount += size;
     }
 
-    if (self.rejectInsaneFees && fee.cmp(minFee.muln(10000)) > 0)
+    if (self.rejectAbsurdFees && fee.cmp(minRelayFee.muln(10000)) > 0)
       return callback(new VerifyError(tx, 'highfee', 'absurdly-high-fee', 0));
 
+    if (!tx.checkInputs(height, ret))
+      return callback(new VerifyError(tx, 'invalid', ret.reason, ret.score));
+
     self.countAncestors(tx, function(err, count) {
       if (err)
         return callback(err);
@@ -943,6 +1044,7 @@ Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) {
       return callback(err);
 
     self.orphans++;
+    self.size += memOrphan(tx);
 
     batch.put('O/' + hash, tx.toExtended(true));
 
@@ -1164,6 +1266,9 @@ Mempool.prototype.removeOrphan = function removeOrphan(tx, callback) {
     }, function(err) {
       if (err)
         return callback(err);
+
+      self.size -= memOrphan(tx);
+
       batch.write(callback);
     });
   });
@@ -1559,5 +1664,95 @@ Mempool.prototype._removeUnchecked = function removeUnchecked(hash, callback, fo
   });
 };
 
+function MempoolTX(options) {
+  this.tx = options.tx;
+  this.height = options.height;
+  this.priority = options.priority;
+  this.chainValue = options.chainValue;
+
+  this.ts = options.ts;
+  this.count = options.count;
+  this.size = options.size;
+  this.fees = options.fees;
+}
+
+MempoolTX.fromTX = function fromTX(tx, height) {
+  var data = tx.getPriority(height);
+
+  return new MempoolTX({
+    tx: tx,
+    height: height,
+    priority: data.priority,
+    chainValue: data.value,
+    ts: utils.now(),
+    count: 1,
+    size: tx.getVirtualSize(),
+    fees: tx.getFee()
+  });
+};
+
+MempoolTX.prototype.toRaw = function toRaw() {
+  var p = new BufferWriter();
+  bcoin.protocol.framer.tx(this.tx, p);
+  p.writeU32(this.height);
+  p.writeVarint(this.priority);
+  p.writeVarint(this.chainValue);
+  p.writeU32(this.ts);
+  p.writeU32(this.count);
+  p.writeU32(this.size);
+  p.writeVarint(this.fees);
+  return p.render();
+};
+
+MempoolTX.fromRaw = function fromRaw(data, saveCoins) {
+  var p = new BufferReader(data);
+  return new MempoolTX({
+    tx: bcoin.protocol.parser.parseTX(p),
+    height: p.readU32(),
+    priority: p.readVarint(true),
+    chainValue: p.readVarint(true),
+    ts: p.readU32(),
+    count: p.readU32(),
+    size: p.readU32(),
+    fees: p.readVarint(true)
+  });
+};
+
+MempoolTX.prototype.getPriority = function getPriority(height) {
+  var heightDelta = Math.max(0, height - this.height);
+  var modSize = this.tx.getModifiedSize();
+  var deltaPriority = new bn(heightDelta).mul(this.chainValue).divn(modSize);
+  var result = this.priority.add(deltaPriority);
+  if (result.cmpn(0) < 0)
+    result = new bn(0);
+  return result;
+};
+
+MempoolTX.prototype.isFree = function isFree(height) {
+  var priority = this.getPriority(height);
+  return priority.cmp(constants.tx.FREE_THRESHOLD) > 0;
+};
+
+/*
+ * Helpers
+ */
+
+function mem(tx) {
+  return 0
+    + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended
+    + (2 + 64) // t
+    + (2 + 10 + 1 + 64) // m
+    + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s
+    + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c
+}
+
+function memOrphan(tx) {
+  return 0
+    + (2 + 64 + 32) // o
+    + (2 + 64) // O
+    + (tx.getSize() + 4 + 32 + 4 + 4 + 4
+    + (tx.inputs.length >>> 1) * 80); // extended
+}
+
 return Mempool;
 };

lib/bcoin/mtx.js

@@ -1148,6 +1148,10 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) {
       size = tx.maxSize(options, true);
 
       if (tryFree) {
+        // Note that this will only work
+        // if the mempool's rolling reject
+        // fee is zero (i.e. the mempool is
+        // not full).
         if (tx.isFree(network.height + 1, size)) {
           fee = new bn(0);
           break;
@@ -1156,9 +1160,9 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) {
       }
 
       if (options.accurate)
-        fee = tx.getMinFee(size);
+        fee = tx.getMinFee(size, options.rate);
       else
-        fee = tx.getMaxFee(size);
+        fee = tx.getMaxFee(size, options.rate);
 
       // Failed to get enough funds, add more coins.
       if (!isFull())
@@ -1182,7 +1186,7 @@ MTX.prototype.selectCoins = function selectCoins(coins, options) {
     if (typeof options.subtractFee === 'number') {
       i = options.subtractFee;
 
-      if (i > tx.outputs.length - 1)
+      if (!tx.outputs[i])
         throw new Error('Subtraction index does not exist.');
 
       if (tx.outputs[i].value.cmp(minValue) < 0)

lib/bcoin/protocol/constants.js

@@ -453,7 +453,7 @@ exports.mempool = {
    * Maximum mempool size in bytes.
    */
 
-  MAX_MEMPOOL_SIZE: 300 << 20,
+  MAX_MEMPOOL_SIZE: 300 * 1000000,
 
   /**
    * The time at which transactions
@@ -466,7 +466,13 @@ exports.mempool = {
    * Maximum number of orphan transactions.
    */
 
-  MAX_ORPHAN_TX: 100
+  MAX_ORPHAN_TX: 100,
+
+  /**
+   * Decay of minimum fee rate.
+   */
+
+  FEE_HALFLIFE: 60 * 60 * 12
 };
 
 /**

lib/bcoin/tx.js

@@ -1357,7 +1357,7 @@ TX.prototype.getModifiedSize = function getModifiedSize(size) {
  */
 
 TX.prototype.getPriority = function getPriority(height, size) {
-  var sum, i, input, age;
+  var sum, i, input, age, value;
 
   if (this.isCoinbase())
     return new bn(0);
@@ -1369,9 +1369,10 @@ TX.prototype.getPriority = function getPriority(height, size) {
   }
 
   if (size == null)
-    size = this.getModifiedSize();
+    size = this.maxSize();
 
   sum = new bn(0);
+  value = new bn(0);
 
   for (i = 0; i < this.inputs.length; i++) {
     input = this.inputs[i];
@@ -1384,11 +1385,15 @@ TX.prototype.getPriority = function getPriority(height, size) {
 
     if (input.coin.height <= height) {
       age = height - input.coin.height;
+      value.iadd(input.coin.value);
       sum.iadd(input.coin.value.muln(age));
     }
   }
 
-  return sum.divn(size);
+  return {
+    value: value,
+    priority: sum.divn(size)
+  };
 };
 
 /**
@@ -1412,7 +1417,7 @@ TX.prototype.isFree = function isFree(height, size) {
       height = network.height + 1;
   }
 
-  priority = this.getPriority(height, size);
+  priority = this.getPriority(height, size).priority;
 
   return priority.cmp(constants.tx.FREE_THRESHOLD) > 0;
 };
@@ -1422,19 +1427,23 @@ TX.prototype.isFree = function isFree(height, size) {
  * to be relayable (not the constant min relay fee).
  * @param {Number?} size - If not present, max size
  * estimation will be calculated and used.
+ * @param {Number?} rate - Rate of satoshi per kB.
  * @returns {BN} fee
  */
 
-TX.prototype.getMinFee = function getMinFee(size) {
+TX.prototype.getMinFee = function getMinFee(size, rate) {
   var fee;
 
   if (size == null)
     size = this.maxSize();
 
-  fee = new bn(constants.tx.MIN_FEE).muln(size).divn(1000);
+  if (rate == null)
+    rate = constants.tx.MIN_FEE;
 
-  if (fee.cmpn(0) === 0 && constants.tx.MIN_FEE > 0)
-    fee = new bn(constants.tx.MIN_FEE);
+  fee = new bn(rate).muln(size).divn(1000);
+
+  if (fee.cmpn(0) === 0 && rate > 0)
+    fee = new bn(rate);
 
   return fee;
 };
@@ -1445,19 +1454,23 @@ TX.prototype.getMinFee = function getMinFee(size) {
  * when taking into account size.
  * @param {Number?} size - If not present, max size
  * estimation will be calculated and used.
+ * @param {Number?} rate - Rate of satoshi per kB.
  * @returns {BN} fee
  */
 
-TX.prototype.getMaxFee = function getMaxFee(size) {
+TX.prototype.getMaxFee = function getMaxFee(size, rate) {
   var fee;
 
   if (size == null)
     size = this.maxSize();
 
-  fee = new bn(constants.tx.MIN_FEE).muln(Math.ceil(size / 1000));
+  if (rate == null)
+    rate = constants.tx.MIN_FEE;
 
-  if (fee.cmpn(0) === 0 && constants.tx.MIN_FEE > 0)
-    fee = new bn(constants.tx.MIN_FEE);
+  fee = new bn(rate).muln(Math.ceil(size / 1000));
+
+  if (fee.cmpn(0) === 0 && rate > 0)
+    fee = new bn(rate);
 
   return fee;
 };
@@ -1617,13 +1630,13 @@ TX.prototype.inspect = function inspect() {
     hash: utils.revHex(this.hash('hex')),
     witnessHash: utils.revHex(this.witnessHash('hex')),
     size: this.getSize(),
-    virtualSize: this.getVirtualSize(),
+    virtualSize: this.maxSize(),
     height: this.height,
     value: utils.btc(this.getOutputValue()),
     fee: utils.btc(this.getFee()),
     minFee: utils.btc(this.getMinFee()),
     confirmations: this.getConfirmations(),
-    priority: this.getPriority().toString(10),
+    priority: this.getPriority().priority.toString(10),
     date: utils.date(this.ts || this.ps),
     block: this.block ? utils.revHex(this.block) : null,
     ts: this.ts,