mempool size.

lib/bcoin/mempool.js

@@ -77,6 +77,8 @@ function Mempool(options) {
   this.tx = null;
   this.size = 0;
   this.orphans = 0;
+  this.spent = 0;
+  this.total = 0;
 
   this.freeCount = 0;
   this.lastTime = 0;
@@ -145,12 +147,11 @@ Mempool.prototype._init = function _init() {
         unlock();
         return self.emit('error', err);
       }
-      self.dynamicMemoryUsage(function(err, size) {
+      self.initialMemoryUsage(function(err) {
-        if (err)
+        if (err) {
-          self.emit('error', err);
+          unlock();
-        else
+          return self.emit('error', err);
-          self.size = size;
+        }
-
         self.chain.open(function(err) {
           if (err) {
             unlock();
@@ -165,8 +166,28 @@ Mempool.prototype._init = function _init() {
   });
 };
 
-Mempool.prototype.dynamicMemoryUsage = function dynamicMemoryUsage(callback) {
+/**
-  return this.db.approximateSize('m', 'm~', callback);
+ * Tally up total memory usage from database.
+ * @param {Function} callback - Returns [Error, Number].
+ */
+
+Mempool.prototype.initialMemoryUsage = function initialMemoryUsage(callback) {
+  var self = this;
+  var i, tx;
+
+  this.getHistory(function(err, txs) {
+    if (err)
+      return callback(err);
+
+    for (i = 0; i < txs.length; i++) {
+      tx = txs[i];
+      self.size += self.memUsage(tx);
+      self.spent += tx.inputs.length;
+      self.total++;
+    }
+
+    return callback();
+  });
 };
 
 /**
@@ -341,7 +362,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
   var self = this;
   var rate;
 
-  if (this.size <= this.maxSize)
+  if (this.getSize() <= this.maxSize)
     return callback(null, true);
 
   this.tx.getRange({
@@ -352,6 +373,9 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
       return callback(err);
 
     utils.forEachSerial(function(tx, next) {
+      if (self.getSize() <= self.maxSize)
+        return next();
+
       self.removeUnchecked(tx, function(err) {
         if (err)
           return next(err);
@@ -370,12 +394,7 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
       if (err)
         return callback(err);
 
-      self.purgeOrphans(function(err) {
+      return callback(null, self.getSize() <= self.maxSize);
-        if (err)
-          return callback(err);
-
-        return callback(self.size <= self.maxSize);
-      });
     });
   });
 };
@@ -385,61 +404,49 @@ Mempool.prototype.limitMempoolSize = function limitMempoolSize(callback) {
  * @param {Function} callback
  */
 
-Mempool.prototype.purgeOrphans = function purgeOrphans(callback) {
+Mempool.prototype.pruneOrphans = function pruneOrphans(callback) {
   var self = this;
-  var batch = this.db.batch();
+  var hashes = [];
-  var tx;
+  var iter;
 
   callback = utils.ensure(callback);
 
-  utils.forEachSerial(['O', 'o'], function(type, callback) {
+  iter = this.db.iterator({
-    var iter = self.db.iterator({
+    gte: 'O',
-      gte: type,
+    lte: 'O~',
-      lte: type + '~',
+    keys: true,
-      keys: true,
+    values: false,
-      values: true,
+    fillCache: false,
-      fillCache: false,
+    keyAsBuffer: false
-      keyAsBuffer: false
+  });
-    });
 
-    (function next() {
+  function done(err) {
-      iter.next(function(err, key, value) {
-        if (err) {
-          return iter.end(function() {
-            callback(err);
-          });
-        }
-
-        if (type === 'O') {
-          try {
-            tx = bcoin.tx.fromExtended(value, true);
-          } catch (e) {
-            return callback(e);
-          }
-          self.size -= memOrphan(tx);
-        }
-
-        if (key === undefined)
-          return iter.end(callback);
-
-        batch.del(key);
-
-        next();
-      });
-    })();
-  }, function(err) {
     if (err)
       return callback(err);
 
-    batch.write(function(err) {
+    utils.forEachSerial(hashes, function(hash, next) {
-      if (err)
+      if (self.orphans <= constants.mempool.MAX_ORPHAN_TX)
-        return callback(err);
+        return next();
+      self.removeOrphan(hash, next);
+    }, callback);
+  }
 
-      self.orphans = 0;
+  (function next() {
+    iter.next(function(err, key, value) {
+      if (err) {
+        return iter.end(function() {
+          callback(err);
+        });
+      }
 
-      return callback();
+      if (key === undefined)
+        return iter.end(done);
+
+      hashes.push(key.split('/')[1]);
+
+      next();
     });
-  });
+  })();
 };
 
 /**
@@ -586,7 +593,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
   if (tx.ts !== 0) {
     return callback(new VerifyError(tx,
       'alreadyknown',
-      'txn-already-in-mempool',
+      'txn-already-known',
       0));
   }
 
@@ -658,7 +665,7 @@ Mempool.prototype.addTX = function addTX(tx, callback, force) {
               return callback(err);
 
             if (!tx.hasCoins()) {
-              if (self.size > self.maxSize) {
+              if (self.getSize() > self.maxSize) {
                 return callback(new VerifyError(tx,
                   'insufficientfee',
                   'mempool full',
@@ -708,7 +715,9 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, callback) {
     if (err)
       return callback(err);
 
-    self.size += mem(tx);
+    self.spent += tx.inputs.length;
+    self.size += self.memUsage(tx);
+    self.total++;
     self.emit('tx', tx);
     self.emit('add tx', tx);
 
@@ -767,7 +776,9 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(tx, callback, limit
         if (err)
           return callback(err);
 
-        self.size -= mem(tx);
+        self.spent -= tx.inputs.length;
+        self.size -= self.memUsage(tx);
+        self.total--;
 
         self.emit('remove tx', tx);
 
@@ -1009,9 +1020,15 @@ Mempool.prototype.countAncestors = function countAncestors(tx, callback) {
 Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) {
   var self = this;
   var prevout = {};
-  var batch = this.db.batch();
+  var i, hash, batch, input, p;
-  var hash = tx.hash('hex');
+
-  var i, input, p;
+  if (tx.getSize() > 5000) {
+    bcoin.debug('Ignoring large orphan: %s', tx.rhash);
+    return callback();
+  }
+
+  hash = tx.hash('hex');
+  batch = this.db.batch();
 
   for (i = 0; i < tx.inputs.length; i++) {
     input = tx.inputs[i];
@@ -1044,12 +1061,11 @@ Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) {
       return callback(err);
 
     self.orphans++;
-    self.size += memOrphan(tx);
 
     batch.put('O/' + hash, tx.toExtended(true));
 
     if (self.orphans > constants.mempool.MAX_ORPHAN_TX) {
-      return self.purgeOrphans(function(err) {
+      return self.pruneOrphans(function(err) {
         if (err)
           return callback(err);
         batch.write(callback);
@@ -1267,7 +1283,7 @@ Mempool.prototype.removeOrphan = function removeOrphan(tx, callback) {
       if (err)
         return callback(err);
 
-      self.size -= memOrphan(tx);
+      self.orphans--;
 
       batch.write(callback);
     });
@@ -1664,6 +1680,57 @@ Mempool.prototype._removeUnchecked = function removeUnchecked(hash, callback, fo
   });
 };
 
+Mempool.prototype.memUsage = function memUsage(tx) {
+  if (this.accurateMemory)
+    return this.memUsageAccurate(tx);
+  return this.memUsageBitcoind(tx);
+};
+
+Mempool.prototype.memUsageAccurate = function memUsage(tx) {
+  return 0
+    + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended
+    + (2 + 64) // t
+    + (2 + 10 + 1 + 64) // m
+    + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s
+    + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c
+};
+
+Mempool.prototype.memUsageBitcoind = function memUsage(tx) {
+  var mem = 0;
+  var i, input;
+
+  mem += mallocUsage(tx.inputs.length)
+  mem += mallocUsage(tx.outputs.length);
+
+  for (i = 0; i < tx.inputs.length; i++)
+    mem += mallocUsage(tx.inputs[i].script.getSize()) + 0;
+
+  for (i = 0; i < tx.outputs.length; i++)
+    mem += mallocUsage(tx.outputs[i].script.getSize());
+
+  mem += mallocUsage(tx.inputs.length);
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    mem += mallocUsage(input.witness.length);
+    for (j = 0; j < input.witness.length; j++)
+      mem += mallocUsage(input.witness[j].length);
+  }
+
+  return mem;
+};
+
+Mempool.prototype.getSize = function getSize() {
+  if (this.accurateMemory)
+    return this.size;
+
+  return mallocUsage(162 + 15 * ptrSize) * this.total // entries
+    + mallocUsage(this.spent) // mapNextTx
+    + mallocUsage(this.total) // mapDeltas
+    + mallocUsage(this.total) // mapLinks
+    + this.size;
+};
+
 function MempoolTX(options) {
   this.tx = options.tx;
   this.height = options.height;
@@ -1737,21 +1804,19 @@ MempoolTX.prototype.isFree = function isFree(height) {
  * Helpers
  */
 
-function mem(tx) {
+// Assume 64 bit for arm since version
-  return 0
+// number is not exposed by node.js.
-    + (tx.getSize() + 4 + 32 + 4 + 4 + 4) // extended
+var ptrSize = (process.platform == null
-    + (2 + 64) // t
+  || process.platform === 'x64'
-    + (2 + 10 + 1 + 64) // m
+  || process.platform === 'ia64'
-    + (tx.inputs.length * (2 + 64 + 1 + 2 + 32)) // s
+  || process.platform === 'arm') ? 8 : 4;
-    + (tx.outputs.length * (2 + 64 + 1 + 2 + 80)); // c
-}
 
-function memOrphan(tx) {
+function mallocUsage(alloc) {
-  return 0
+  if (alloc == 0)
-    + (2 + 64 + 32) // o
+    return 0;
-    + (2 + 64) // O
+  if (ptrSize === 8)
-    + (tx.getSize() + 4 + 32 + 4 + 4 + 4
+    return ((alloc + 31) >> 4) << 4;
-    + (tx.inputs.length >>> 1) * 80); // extended
+  return ((alloc + 15) >> 3) << 3;
 }
 
 return Mempool;