mempool: refactor coin and orphan handling.

lib/mempool/mempool.js

@@ -9,8 +9,6 @@
 var AsyncObject = require('../utils/async');
 var constants = require('../protocol/constants');
 var util = require('../utils/util');
-var BufferReader = require('../utils/reader');
-var BufferWriter = require('../utils/writer');
 var co = require('../utils/co');
 var assert = require('assert');
 var crypto = require('../crypto/crypto');
@@ -504,67 +502,6 @@ Mempool.prototype.getTXByAddress = function getTXByAddress(addresses) {
   return txs;
 };
 
-/**
- * Fill a transaction with all available transaction outputs
- * in the mempool. This differs from {@link Mempool#fillCoins}
- * in that it will fill with all historical coins and not
- * just unspent coins.
- * @param {TX} tx
- */
-
-Mempool.prototype.fillHistory = function fillHistory(tx) {
-  var i, input, prevout, prev;
-
-  if (tx.isCoinbase())
-    return;
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-    prevout = input.prevout;
-
-    if (input.coin)
-      continue;
-
-    prev = this.getTX(prevout.hash);
-
-    if (!prev)
-      continue;
-
-    if (prevout.index >= prev.outputs.length)
-      continue;
-
-    input.coin = Coin.fromTX(prev, prevout.index);
-  }
-};
-
-/**
- * Fill a transaction with all available (unspent) coins
- * in the mempool.
- * @param {TX} tx
- */
-
-Mempool.prototype.fillCoins = function fillCoins(tx) {
-  var i, input, prevout, coin;
-
-  if (tx.isCoinbase())
-    return;
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-    prevout = input.prevout;
-
-    if (input.coin)
-      continue;
-
-    coin = this.getCoin(prevout.hash, prevout.index);
-
-    if (!coin)
-      continue;
-
-    input.coin = coin;
-  }
-};
-
 /**
  * Test the mempool to see if it contains a transaction.
  * @param {Hash} hash
@@ -751,9 +688,7 @@ Mempool.prototype._addTX = co(function* _addTX(tx) {
       0);
   }
 
-  yield this.fillAllCoins(tx);
+  if (!(yield this.fillAllCoins(tx)))
-
-  if (!tx.hasCoins())
     return this.storeOrphan(tx);
 
   entry = MempoolEntry.fromTX(tx, this.chain.height);
@@ -796,7 +731,7 @@ Mempool.prototype.addUnchecked = co(function* addUnchecked(entry) {
  */
 
 Mempool.prototype._addUnchecked = co(function* addUnchecked(entry) {
-  var i, resolved, tx, orphan;
+  var i, resolved, tx;
 
   this.trackEntry(entry);
 
@@ -812,10 +747,9 @@ Mempool.prototype._addUnchecked = co(function* addUnchecked(entry) {
 
   for (i = 0; i < resolved.length; i++) {
     tx = resolved[i];
-    orphan = MempoolEntry.fromTX(tx, this.chain.height);
 
     try {
-      yield this.verify(orphan);
+      yield this._addTX(tx);
     } catch (err) {
       if (err.type === 'VerifyError') {
         this.logger.debug('Could not resolve orphan %s: %s.',
@@ -831,13 +765,6 @@ Mempool.prototype._addUnchecked = co(function* addUnchecked(entry) {
       continue;
     }
 
-    try {
-      yield this._addUnchecked(orphan);
-    } catch (err) {
-      this.emit('error', err);
-      continue;
-    }
-
     this.logger.spam('Resolved orphan %s in mempool.', tx.txid());
   }
 });
@@ -851,9 +778,9 @@ Mempool.prototype._addUnchecked = co(function* addUnchecked(entry) {
 
 Mempool.prototype.removeUnchecked = function removeUnchecked(entry, limit) {
   var tx = entry.tx;
-  var hash;
+  var hash = tx.hash('hex');
 
-  this.removeOrphan(tx);
+  this.removeOrphan(hash);
 
   // We do not remove spenders if this is
   // being removed for a block. The spenders
@@ -869,7 +796,6 @@ Mempool.prototype.removeUnchecked = function removeUnchecked(entry, limit) {
   this.untrackEntry(entry);
 
   if (this.fees) {
-    hash = tx.hash('hex');
     this.fees.removeTX(hash);
   }
 
@@ -1268,63 +1194,6 @@ Mempool.prototype.getDepends = function getDepends(tx) {
   return depends;
 };
 
-/**
- * Store an orphaned transaction.
- * @param {TX} tx
- */
-
-Mempool.prototype.storeOrphan = function storeOrphan(tx) {
-  var missing = {};
-  var i, hash, input, prev;
-
-  if (tx.getWeight() > constants.tx.MAX_WEIGHT) {
-    this.logger.debug('Ignoring large orphan: %s', tx.txid());
-    if (!tx.hasWitness())
-      this.rejects.add(tx.hash());
-    return;
-  }
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-
-    if (input.coin)
-      continue;
-
-    if (this.hasReject(input.prevout.hash)) {
-      this.logger.debug('Not storing orphan %s (rejected parents).', tx.txid());
-      this.rejects.add(tx.hash());
-      return;
-    }
-
-    missing[input.prevout.hash] = true;
-  }
-
-  hash = tx.hash('hex');
-  missing = Object.keys(missing);
-
-  assert(missing.length > 0);
-
-  for (i = 0; i < missing.length; i++) {
-    prev = missing[i];
-
-    if (!this.waiting[prev])
-      this.waiting[prev] = [];
-
-    this.waiting[prev].push(hash);
-  }
-
-  this.orphans[hash] = toOrphanRaw(tx);
-  this.totalOrphans++;
-
-  this.logger.debug('Added orphan %s to mempool.', tx.txid());
-
-  this.emit('add orphan', tx);
-
-  this.limitOrphans();
-
-  return missing;
-};
-
 /**
  * Return the full balance of all unspents in the mempool
  * (not very useful in practice, only used for testing).
@@ -1385,23 +1254,7 @@ Mempool.prototype.getHistory = function getHistory() {
  */
 
 Mempool.prototype.getOrphan = function getOrphan(hash) {
-  var data = this.orphans[hash];
+  return this.orphans[hash];
-  var orphan;
-
-  if (!data)
-    return;
-
-  try {
-    orphan = fromOrphanRaw(data);
-  } catch (e) {
-    delete this.orphans[hash];
-    this.logger.warning('%s %s',
-      'Warning: possible memory corruption.',
-      'Orphan failed deserialization.');
-    return;
-  }
-
-  return orphan;
 };
 
 /**
@@ -1413,6 +1266,63 @@ Mempool.prototype.hasOrphan = function hasOrphan(hash) {
   return this.orphans[hash] != null;
 };
 
+/**
+ * Store an orphaned transaction.
+ * @param {TX} tx
+ */
+
+Mempool.prototype.storeOrphan = function storeOrphan(tx) {
+  var missing = {};
+  var i, hash, input, prev;
+
+  if (tx.getWeight() > constants.tx.MAX_WEIGHT) {
+    this.logger.debug('Ignoring large orphan: %s', tx.txid());
+    if (!tx.hasWitness())
+      this.rejects.add(tx.hash());
+    return;
+  }
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+
+    if (input.coin)
+      continue;
+
+    if (this.hasReject(input.prevout.hash)) {
+      this.logger.debug('Not storing orphan %s (rejected parents).', tx.txid());
+      this.rejects.add(tx.hash());
+      return;
+    }
+
+    missing[input.prevout.hash] = true;
+  }
+
+  hash = tx.hash('hex');
+  missing = Object.keys(missing);
+
+  assert(missing.length > 0);
+
+  for (i = 0; i < missing.length; i++) {
+    prev = missing[i];
+
+    if (!this.waiting[prev])
+      this.waiting[prev] = [];
+
+    this.waiting[prev].push(hash);
+  }
+
+  this.orphans[hash] = new Orphan(tx, missing.length);
+  this.totalOrphans++;
+
+  this.logger.debug('Added orphan %s to mempool.', tx.txid());
+
+  this.emit('add orphan', tx);
+
+  this.limitOrphans();
+
+  return missing;
+};
+
 /**
  * Potentially resolve any transactions
  * that redeem the passed-in transaction.
@@ -1438,16 +1348,17 @@ Mempool.prototype.resolveOrphans = function resolveOrphans(tx) {
     if (!orphan)
       continue;
 
-    orphan.fillCoins(tx);
+    if (--orphan.missing === 0) {
-
-    if (orphan.hasCoins()) {
       this.totalOrphans--;
       delete this.orphans[orphanHash];
-      resolved.push(orphan);
+      try {
-      continue;
+        resolved.push(orphan.toTX());
+      } catch (e) {
+        this.logger.warning('%s %s',
+          'Warning: possible memory corruption.',
+          'Orphan failed deserialization.');
+      }
     }
-
-    this.orphans[orphanHash] = toOrphanRaw(orphan);
   }
 
   delete this.waiting[hash];
@@ -1457,19 +1368,25 @@ Mempool.prototype.resolveOrphans = function resolveOrphans(tx) {
 
 /**
  * Remove a transaction from the mempool.
- * @param {TX|Hash} tx
+ * @param {Hash} tx
  */
 
-Mempool.prototype.removeOrphan = function removeOrphan(tx) {
+Mempool.prototype.removeOrphan = function removeOrphan(hash) {
-  var i, j, hashes, prevout, prev, hash;
+  var orphan = this.getOrphan(hash);
+  var i, j, tx, hashes, prevout, prev;
 
-  if (typeof tx === 'string')
+  if (!orphan)
-    tx = this.getOrphan(tx);
-
-  if (!tx)
     return;
 
-  hash = tx.hash('hex');
+  try {
+    tx = orphan.toTX();
+  } catch (e) {
+    delete this.orphans[hash];
+    this.logger.warning('%s %s',
+      'Warning: possible memory corruption.',
+      'Orphan failed deserialization.');
+  }
+
   prevout = tx.getPrevout();
 
   for (i = 0; i < prevout.length; i++) {
@@ -1498,6 +1415,105 @@ Mempool.prototype.removeOrphan = function removeOrphan(tx) {
   this.totalOrphans--;
 };
 
+/**
+ * Test all of a transactions outpoints to see if they are doublespends.
+ * Note that this will only test against the mempool spents, not the
+ * blockchain's. The blockchain spents are not checked against because
+ * the blockchain does not maintain a spent list. The transaction will
+ * be seen as an orphan rather than a double spend.
+ * @param {TX} tx
+ * @returns {Promise} - Returns Boolean.
+ */
+
+Mempool.prototype.isDoubleSpend = function isDoubleSpend(tx) {
+  var spent = {};
+  var i, input, prevout, key;
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    prevout = input.prevout;
+    key = prevout.hash + prevout.index;
+
+    if (this.isSpent(prevout.hash, prevout.index))
+      return true;
+
+    if (spent[key])
+      return true;
+
+    spent[key] = true;
+  }
+
+  return false;
+};
+
+/**
+ * Fill a transaction with all available transaction outputs
+ * in the mempool. This differs from {@link Mempool#fillCoins}
+ * in that it will fill with all historical coins and not
+ * just unspent coins.
+ * @param {TX} tx
+ */
+
+Mempool.prototype.fillHistory = function fillHistory(tx) {
+  var found = true;
+  var i, input, prevout, prev;
+
+  if (tx.isCoinbase())
+    return false;
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    prevout = input.prevout;
+    prev = this.getTX(prevout.hash);
+
+    if (!prev) {
+      input.coin = null;
+      found = false;
+      continue;
+    }
+
+    if (prevout.index >= prev.outputs.length) {
+      input.coin = null;
+      found = false;
+      continue;
+    }
+
+    input.coin = Coin.fromTX(prev, prevout.index);
+  }
+
+  return found;
+};
+
+/**
+ * Fill a transaction with all available (unspent) coins
+ * in the mempool.
+ * @param {TX} tx
+ */
+
+Mempool.prototype.fillCoins = function fillCoins(tx) {
+  var found = true;
+  var i, input, prevout, coin;
+
+  if (tx.isCoinbase())
+    return false;
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    prevout = input.prevout;
+    coin = this.getCoin(prevout.hash, prevout.index);
+
+    if (!coin) {
+      input.coin = null;
+      found = false;
+      continue;
+    }
+
+    input.coin = coin;
+  }
+
+  return found;
+};
+
 /**
  * Fill transaction with all unspent _and spent_
  * coins. Similar to {@link Mempool#fillHistory}
@@ -1526,28 +1542,39 @@ Mempool.prototype.fillAllHistory = function fillAllHistory(tx) {
  */
 
 Mempool.prototype.fillAllCoins = co(function* fillAllCoins(tx) {
+  var found = true;
   var i, input, hash, index, coin;
 
-  this.fillCoins(tx);
-
-  if (tx.hasCoins())
-    return tx;
-
   for (i = 0; i < tx.inputs.length; i++) {
     input = tx.inputs[i];
     hash = input.prevout.hash;
     index = input.prevout.index;
 
-    if (this.isSpent(hash, index))
+    coin = this.getCoin(hash, index);
+
+    if (coin) {
+      input.coin = coin;
       continue;
+    }
+
+    if (this.isSpent(hash, index)) {
+      input.coin = null;
+      found = false;
+      continue;
+    }
 
     coin = yield this.chain.db.getCoin(hash, index);
 
-    if (coin)
+    if (!coin) {
-      input.coin = coin;
+      input.coin = null;
+      found = false;
+      continue;
+    }
+
+    input.coin = coin;
   }
 
-  return tx;
+  return found;
 });
 
 /**
@@ -1571,29 +1598,6 @@ Mempool.prototype.verifyLocks = function verifyLocks(tx, flags) {
   return this.chain.verifyLocks(this.chain.tip, tx, flags);
 };
 
-/**
- * Test all of a transactions outpoints to see if they are doublespends.
- * Note that this will only test against the mempool spents, not the
- * blockchain's. The blockchain spents are not checked against because
- * the blockchain does not maintain a spent list. The transaction will
- * be seen as an orphan rather than a double spend.
- * @param {TX} tx
- * @returns {Promise} - Returns Boolean.
- */
-
-Mempool.prototype.isDoubleSpend = function isDoubleSpend(tx) {
-  var i, input, prevout;
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-    prevout = input.prevout;
-    if (this.isSpent(prevout.hash, prevout.index))
-      return true;
-  }
-
-  return false;
-};
-
 /**
  * Calculate bitcoinj-style confidence.
  * @see http://bit.ly/1OVQwlO
@@ -1988,49 +1992,15 @@ AddressIndex.prototype.removeCoin = function removeCoin(coin) {
  * Helpers
  */
 
-function toOrphanRaw(tx) {
+function Orphan(tx, missing) {
-  var bw = new BufferWriter();
+  this.raw = tx.toRaw();
-  var i, input;
+  this.missing = missing;
-
-  tx.toWriter(bw);
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-
-    if (!input.coin) {
-      bw.writeU8(0);
-      continue;
-    }
-
-    bw.writeU8(1);
-    input.coin.toWriter(bw);
-  }
-
-  return bw.render();
-};
-
-function fromOrphanRaw(data) {
-  var br = new BufferReader(data);
-  var i, tx, input, coin;
-
-  tx = TX.fromReader(br);
-
-  for (i = 0; i < tx.inputs.length; i++) {
-    input = tx.inputs[i];
-
-    if (br.readU8() === 0)
-      continue;
-
-    coin = Coin.fromReader(br);
-    coin.hash = input.prevout.hash;
-    coin.index = input.prevout.index;
-
-    input.coin = coin;
-  }
-
-  return tx;
 }
 
+Orphan.prototype.toTX = function toTX() {
+  return TX.fromRaw(this.raw);
+};
+
 /*
  * Expose
  */

lib/primitives/tx.js

@@ -1121,6 +1121,19 @@ TX.prototype.fillCoins = function fillCoins(coins) {
   return result;
 };
 
+/**
+ * Empty coins.
+ */
+
+TX.prototype.emptyCoins = function emptyCoins() {
+  var i, input;
+
+  for (i = 0; i < this.inputs.length; i++) {
+    input = this.inputs[i];
+    input.coin = null;
+  }
+};
+
 /**
  * Check finality of transaction by examining nLockTime and nSequences.
  * @example