txdb: orphan resolution.

bench/walletdb.js

@@ -34,7 +34,9 @@ var walletdb = new bcoin.walletdb({
   name: 'wallet-test',
   // location: __dirname + '/../walletdb-bench',
   // db: 'leveldb'
-  db: 'memory'
+  db: 'memory',
+  resolution: false,
+  verify: false
 });
 
 var runBench = co(function* runBench() {

lib/node/fullnode.js

@@ -148,6 +148,7 @@ function Fullnode(options) {
     witness: this.options.witness,
     useCheckpoints: this.options.useCheckpoints,
     maxFiles: this.options.maxFiles,
+    resolution: true,
     verify: false
   });
 

lib/node/spvnode.js

@@ -86,6 +86,7 @@ function SPVNode(options) {
     location: this.location('walletdb'),
     witness: this.options.witness,
     maxFiles: this.options.maxFiles,
+    resolution: true,
     verify: true
   });
 

lib/wallet/browser.js

@@ -120,12 +120,6 @@ layout.txdb = {
   ss: function ss(key) {
     return this.hii(key);
   },
-  o: function o(hash, index) {
-    return this.hi('o', hash, index);
-  },
-  oo: function oo(key) {
-    return this.hii(key);
-  },
   p: function p(hash) {
     return this.ha('p', hash);
   },

lib/wallet/pathinfo.js

@@ -108,7 +108,9 @@ PathInfo.fromTX = function fromTX(wallet, tx, paths) {
  * @returns {Boolean}
  */
 
-PathInfo.prototype.hasPath = function hasPath(hash) {
+PathInfo.prototype.hasPath = function hasPath(output) {
+  var hash = output.getHash('hex');
+
   if (!hash)
     return false;
 
@@ -121,7 +123,9 @@ PathInfo.prototype.hasPath = function hasPath(hash) {
  * @returns {Path}
  */
 
-PathInfo.prototype.getPath = function getPath(hash) {
+PathInfo.prototype.getPath = function getPath(output) {
+  var hash = output.getHash('hex');
+
   if (!hash)
     return;
 

lib/wallet/txdb.js

@@ -136,12 +136,6 @@ var layout = {
   ss: function ss(key) {
     return layout.hii(key);
   },
-  o: function o(hash, index) {
-    return layout.hi(0x6f, hash, index);
-  },
-  oo: function oo(key) {
-    return layout.hii(key);
-  },
   p: function p(hash) {
     return layout.ha(0x70, hash);
   },
@@ -219,6 +213,10 @@ function TXDB(wallet) {
   this.balance = null;
   this.pending = null;
   this.events = [];
+
+  this.orphans = {};
+  this.count = {};
+  this.totalOrphans = 0;
 }
 
 /**
@@ -438,57 +436,183 @@ TXDB.prototype.getPathInfo = function getPathInfo(tx) {
 };
 
 /**
- * Add an orphan (tx hash + input index)
+ * Determine which transactions to add.
- * to orphan list. Stored by its required coin ID.
+ * Attempt to resolve orphans (for SPV).
- * @private
+ * @param {TX} tx
- * @param {Outpoint} prevout - Required coin hash & index.
+ * @returns {Promise}
- * @param {Buffer} input - Spender input hash and index.
- * @returns {Promise} - Returns Buffer.
  */
 
-TXDB.prototype.addOrphan = co(function* addOrphan(prevout, input) {
+TXDB.prototype.resolve = co(function* add(tx) {
-  var key = layout.o(prevout.hash, prevout.index);
+  var hash, result;
-  var data = yield this.get(key);
-  var p = new BufferWriter();
 
-  if (data)
+  if (!this.options.resolution)
-    p.writeBytes(data);
+    return [tx];
 
-  p.writeBytes(input);
+  hash = tx.hash('hex');
 
-  this.put(key, p.render());
+  if (yield this.hasTX(hash))
+    return [tx];
+
+  result = yield this.verifyInputs(tx);
+
+  if (!result)
+    return [];
+
+  return yield this.resolveOutputs(tx);
 });
 
 /**
- * Retrieve orphan list by coin ID.
+ * Verify inputs and potentially add orphans.
- * @private
+ * Used in SPV mode.
- * @param {Hash} hash
+ * @param {TX} tx
- * @param {Number} index
+ * @returns {Promise}
- * @returns {Promise} - Returns {@link Orphan}.
  */
 
-TXDB.prototype.getOrphans = co(function* getOrphans(hash, index) {
+TXDB.prototype.verifyInputs = co(function* verifyInputs(tx) {
-  var key = layout.o(hash, index);
+  var hash = tx.hash('hex');
-  var data = yield this.get(key);
+  var hasOrphans = false;
-  var items = [];
+  var orphans = [];
-  var i, inputs, input, tx, p;
+  var i, input, prevout, address;
+  var path, key, coin, spent;
 
-  if (!data)
+  if (tx.isCoinbase())
-    return;
+    return true;
 
-  p = new BufferReader(data);
+  if (this.count[hash])
-  inputs = [];
+    return false;
 
-  while (p.left())
+  for (i = 0; i < tx.inputs.length; i++) {
-    inputs.push(Outpoint.fromRaw(p));
+    input = tx.inputs[i];
+    prevout = input.prevout;
+    coin = yield this.getCoin(prevout.hash, prevout.index);
 
-  for (i = 0; i < inputs.length; i++) {
+    if (coin) {
-    input = inputs[i];
+      input.coin = coin;
-    tx = yield this.getTX(input.hash);
+
-    items.push(new Orphan(input, tx));
+      if (this.options.verify) {
+        if (!(yield tx.verifyInputAsync(i)))
+          return false;
+      }
+
+      continue;
+    }
+
+    spent = yield this.isSpent(prevout.hash, prevout.index);
+
+    if (!spent) {
+      address = input.getHash('hex');
+      path = yield this.wallet.hasPath(address);
+
+      if (!path)
+        continue;
+
+      orphans[i] = true;
+      continue;
+    }
+
+    coin = yield this.getSpentCoin(spent, prevout);
+    assert(coin);
+
+    input.coin = coin;
+
+    if (this.options.verify) {
+      if (!(yield tx.verifyInputAsync(i)))
+        return false;
+    }
   }
 
-  return items;
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    prevout = input.prevout;
+
+    if (!orphans[i])
+      continue;
+
+    key = prevout.hash + prevout.index;
+
+    if (this.totalOrphans > 20) {
+      this.logger.warning('Potential orphan flood!');
+      this.logger.warning(
+        'More than 20 orphans for %s. Purging.',
+        this.wallet.id);
+      this.totalOrphans = 0;
+      this.orphans = {};
+      this.count = {};
+    }
+
+    if (!this.orphans[key])
+      this.orphans[key] = [];
+
+    if (!this.count[hash])
+      this.count[hash] = 0;
+
+    this.orphans[key].push(new Orphan(tx, i));
+    this.count[hash]++;
+    this.totalOrphans++;
+
+    hasOrphans = true;
+  }
+
+  if (hasOrphans)
+    return false;
+
+  return true;
+});
+
+/**
+ * Resolve orphans for outputs.
+ * Used in SPV mode.
+ * @param {TX} tx
+ * @returns {Promise}
+ */
+
+TXDB.prototype.resolveOutputs = co(function* resolveOutputs(tx, resolved) {
+  var hash = tx.hash('hex');
+  var i, input, output, key, orphans, orphan, coin, valid;
+
+  if (!resolved)
+    resolved = [];
+
+  resolved.push(tx);
+
+  for (i = 0; i < tx.outputs.length; i++) {
+    output = tx.outputs[i];
+    key = hash + i;
+    orphans = this.orphans[key];
+
+    if (!orphans)
+      continue;
+
+    delete this.orphans[key];
+
+    coin = Coin.fromTX(tx, i);
+
+    while (orphans.length) {
+      orphan = orphans.pop();
+      valid = true;
+
+      input = orphan.tx.inputs[orphan.index];
+      input.coin = coin;
+
+      assert(input.prevout.hash === hash);
+      assert(input.prevout.index === i);
+
+      if (this.options.verify)
+        valid = yield orphan.tx.verifyInputAsync(orphan.index);
+
+      if (valid) {
+        if (--this.count[orphan.hash] === 0) {
+          delete this.count[orphan.hash];
+          yield this.resolveOutputs(orphan.tx, resolved);
+        }
+        break;
+      }
+
+      delete this.count[orphan.hash];
+    }
+  }
+
+  return resolved;
 });
 
 /**
@@ -500,63 +624,48 @@ TXDB.prototype.getOrphans = co(function* getOrphans(hash, index) {
  * @returns {Promise}
  */
 
-TXDB.prototype.verify = co(function* verify(tx, info) {
+TXDB.prototype.getInputs = co(function* getInputs(tx, info) {
   var spends = [];
-  var orphans = [];
+  var coins = [];
   var removed = {};
-  var i, input, prevout, address, coin, spent, conflict;
+  var i, input, prevout, coin, spent, conflict;
 
   if (tx.isCoinbase())
-    return orphans;
+    return coins;
 
   for (i = 0; i < tx.inputs.length; i++) {
     input = tx.inputs[i];
     prevout = input.prevout;
-    address = input.getHash('hex');
-
-    // Only bother if this input is ours.
-    if (!info.hasPath(address))
-      continue;
 
+    // Try to get the coin we're redeeming.
     coin = yield this.getCoin(prevout.hash, prevout.index);
 
     if (coin) {
-      // Add TX to inputs and spend money
       input.coin = coin;
-
+      coins.push(coin);
-      // Skip invalid transactions
-      if (this.options.verify) {
-        if (!(yield tx.verifyInputAsync(i)))
-          return;
-      }
-
       continue;
     }
 
+    // Is it already spent?
     spent = yield this.isSpent(prevout.hash, prevout.index);
 
-    // Orphan until we see a parent transaction.
+    // If we have no coin or spend
+    // record, this is not our input.
+    // This is assuming everything came
+    // in order!
     if (!spent) {
-      orphans[i] = true;
+      coins.push(null);
       continue;
     }
 
-    // We must be double-spending.
+    // Yikes, we're double spending. We
+    // still need the coin for after we
+    // resolve the conflict.
     coin = yield this.getSpentCoin(spent, prevout);
-
+    assert(coin);
-    // Double-spent orphan.
-    if (!coin) {
-      orphans[i] = true;
-      continue;
-    }
 
     input.coin = coin;
-
+    coins.push(coin);
-    // Skip invalid transactions
-    if (this.options.verify) {
-      if (!(yield tx.verifyInputAsync(i)))
-        return;
-    }
 
     spends[i] = spent;
   }
@@ -598,66 +707,7 @@ TXDB.prototype.verify = co(function* verify(tx, info) {
     this.emit('conflict', conflict.tx, conflict.info);
   }
 
-  return orphans;
+  return coins;
-});
-
-/**
- * Attempt to resolve orphans for an output.
- * @private
- * @param {TX} tx
- * @param {Number} index
- * @returns {Promise}
- */
-
-TXDB.prototype.resolveOrphans = co(function* resolveOrphans(tx, index) {
-  var hash = tx.hash('hex');
-  var i, orphans, coin, input, spender, orphan;
-
-  orphans = yield this.getOrphans(hash, index);
-
-  if (!orphans)
-    return false;
-
-  this.del(layout.o(hash, index));
-
-  coin = Coin.fromTX(tx, index);
-
-  // Add input to resolved orphan.
-  for (i = 0; i < orphans.length; i++) {
-    orphan = orphans[i];
-    spender = orphan.input;
-    tx = orphan.tx;
-
-    // Probably removed by some other means.
-    if (!tx)
-      continue;
-
-    input = tx.inputs[spender.index];
-    input.coin = coin;
-
-    assert(input.prevout.hash === hash);
-    assert(input.prevout.index === index);
-
-    // Verify that input script is correct, if not - add
-    // output to unspent and remove orphan from storage
-    if (!this.options.verify || (yield tx.verifyInputAsync(spender.index))) {
-      // Add the undo coin record which we never had.
-      this.put(layout.d(spender.hash, spender.index), coin.toRaw());
-      // Add the spender record back in case any evil
-      // transactions were removed with lazyRemove.
-      this.put(layout.s(hash, index), spender.toRaw());
-      return true;
-    }
-
-    yield this.lazyRemove(tx);
-  }
-
-  // We had orphans, but they were invalid. The
-  // balance will be (incorrectly) added outside.
-  // Subtract to compensate.
-  this.pending.sub(coin);
-
-  return false;
 });
 
 /**
@@ -696,7 +746,7 @@ TXDB.prototype.add = co(function* add(tx) {
 TXDB.prototype._add = co(function* add(tx, info) {
   var hash, path, account;
   var i, result, input, output, coin;
-  var prevout, key, address, spender, orphans;
+  var prevout, key, spender, coins;
 
   assert(!tx.mutable, 'Cannot add mutable TX to wallet.');
 
@@ -709,9 +759,9 @@ TXDB.prototype._add = co(function* add(tx, info) {
 
   // Verify and get coins.
   // This potentially removes double-spenders.
-  orphans = yield this.verify(tx, info);
+  coins = yield this.getInputs(tx, info);
 
-  if (!orphans)
+  if (!coins)
     return false;
 
   hash = tx.hash('hex');
@@ -743,56 +793,41 @@ TXDB.prototype._add = co(function* add(tx, info) {
     for (i = 0; i < tx.inputs.length; i++) {
       input = tx.inputs[i];
       prevout = input.prevout;
-
+      coin = coins[i];
-      address = input.getHash('hex');
-      path = info.getPath(address);
 
       // Only bother if this input is ours.
-      if (!path)
+      if (!coin)
         continue;
 
+      path = info.getPath(coin);
+      assert(path);
+
       key = prevout.hash + prevout.index;
 
       // s[outpoint-key] -> [spender-hash]|[spender-input-index]
       spender = Outpoint.fromTX(tx, i).toRaw();
       this.put(layout.s(prevout.hash, prevout.index), spender);
 
-      // Add orphan if no parent transaction known.
-      // Do not disconnect any coins.
-      if (orphans[i]) {
-        yield this.addOrphan(prevout, spender);
-        continue;
-      }
-
       this.del(layout.c(prevout.hash, prevout.index));
       this.del(layout.C(path.account, prevout.hash, prevout.index));
-      this.put(layout.d(hash, i), input.coin.toRaw());
+      this.put(layout.d(hash, i), coin.toRaw());
-      this.pending.sub(input.coin);
+      this.pending.sub(coin);
 
       this.coinCache.remove(key);
     }
   }
 
-  // Add unspent outputs or resolve orphans
+  // Add unspent outputs or resolve orphans.
   for (i = 0; i < tx.outputs.length; i++) {
     output = tx.outputs[i];
-    address = output.getHash('hex');
+    path = info.getPath(output);
     key = hash + i;
 
-    path = info.getPath(address);
+    // Do not add unspents for
-
+    // outputs that aren't ours.
-    // Do not add unspents for outputs that aren't ours.
     if (!path)
       continue;
 
-    orphans = yield this.resolveOrphans(tx, i);
-
-    // If this transaction resolves an orphan,
-    // it should not connect coins as they are
-    // already spent by the orphan it resolved.
-    if (orphans)
-      continue;
-
     coin = Coin.fromTX(tx, i);
 
     this.pending.add(coin);
@@ -840,10 +875,10 @@ TXDB.prototype.removeConflict = co(function* removeConflict(hash, ref, removed)
   if (!tx)
     throw new Error('Could not find spender.');
 
-  if (tx.ts !== 0) {
+  if (tx.height !== -1) {
     // If spender is confirmed and replacement
     // is not confirmed, do nothing.
-    if (ref.ts === 0)
+    if (ref.height === -1)
       return;
 
     // If both are confirmed but replacement
@@ -853,13 +888,12 @@ TXDB.prototype.removeConflict = co(function* removeConflict(hash, ref, removed)
   } else {
     // If spender is unconfirmed and replacement
     // is confirmed, do nothing.
-    if (ref.ts !== 0)
+    if (ref.height === -1) {
-      return;
+      // If both are unconfirmed but replacement
-
+      // is older than spender, do nothing.
-    // If both are unconfirmed but replacement
+      if (ref.ps < tx.ps)
-    // is older than spender, do nothing.
+        return;
-    if (ref.ps < tx.ps)
+    }
-      return;
   }
 
   info = yield this.removeRecursive(tx, removed);
@@ -1002,11 +1036,10 @@ TXDB.prototype.confirm = co(function* confirm(tx, info) {
 
   for (i = 0; i < tx.outputs.length; i++) {
     output = tx.outputs[i];
-    address = output.getHash('hex');
     key = hash + i;
 
     // Only update coins if this output is ours.
-    if (!info.hasPath(address))
+    if (!info.hasPath(output))
       continue;
 
     coin = yield this.getCoin(hash, i);
@@ -1108,7 +1141,7 @@ TXDB.prototype.lazyRemove = co(function* lazyRemove(tx) {
 TXDB.prototype.__remove = co(function* remove(tx, info) {
   var hash = tx.hash('hex');
   var i, path, account, key, prevout;
-  var address, input, output, coin;
+  var input, output, coin;
 
   this.del(layout.t(hash));
 
@@ -1139,12 +1172,11 @@ TXDB.prototype.__remove = co(function* remove(tx, info) {
       input = tx.inputs[i];
       key = input.prevout.hash + input.prevout.index;
       prevout = input.prevout;
-      address = input.getHash('hex');
 
       if (!input.coin)
         continue;
 
-      path = info.getPath(address);
+      path = info.getPath(input.coin);
 
       if (!path)
         continue;
@@ -1157,7 +1189,6 @@ TXDB.prototype.__remove = co(function* remove(tx, info) {
       this.put(layout.C(path.account, prevout.hash, prevout.index), DUMMY);
       this.del(layout.d(hash, i));
       this.del(layout.s(prevout.hash, prevout.index));
-      this.del(layout.o(prevout.hash, prevout.index));
 
       this.coinCache.set(key, coin);
     }
@@ -1166,9 +1197,7 @@ TXDB.prototype.__remove = co(function* remove(tx, info) {
   for (i = 0; i < tx.outputs.length; i++) {
     output = tx.outputs[i];
     key = hash + i;
-    address = output.getHash('hex');
+    path = info.getPath(output);
-
-    path = info.getPath(address);
 
     if (!path)
       continue;
@@ -2406,9 +2435,10 @@ function Conflict(tx, info) {
   this.info = info;
 }
 
-function Orphan(input, tx) {
+function Orphan(tx, i) {
-  this.input = input;
   this.tx = tx;
+  this.hash = tx.hash('hex');
+  this.index = i;
 }
 
 /*

lib/wallet/wallet.js

@@ -1092,6 +1092,24 @@ Wallet.prototype.getPath = co(function* getPath(address) {
   return path;
 });
 
+/**
+ * Test whether the wallet contains a path.
+ * @param {Address|Hash} address
+ * @returns {Promise} - Returns {Boolean}.
+ */
+
+Wallet.prototype.hasPath = co(function* hasPath(address) {
+  var hash = Address.getHash(address, 'hex');
+
+  if (!hash)
+    return false;
+
+  if (this.pathCache.has(hash))
+    return true;
+
+  return yield this.db.hasPath(this.wid, hash);
+});
+
 /**
  * Get all wallet paths.
  * @param {(String|Number)?} acct
@@ -1802,11 +1820,28 @@ Wallet.prototype.add = co(function* add(tx) {
 
 /**
  * Add a transaction to the wallet without a lock.
+ * Potentially resolves orphans.
+ * @private
  * @param {TX} tx
  * @returns {Promise}
  */
 
 Wallet.prototype._add = co(function* add(tx) {
+  var resolved = yield this.txdb.resolve(tx);
+  var i;
+
+  for (i = 0; i < resolved.length; i++)
+    yield this._insert(resolved[i]);
+});
+
+/**
+ * Insert a transaction into the wallet (no lock).
+ * @private
+ * @param {TX} tx
+ * @returns {Promise}
+ */
+
+Wallet.prototype._insert = co(function* insert(tx) {
   var info = yield this.getPathInfo(tx);
   var result, derived;
 

lib/wallet/walletdb.js

@@ -876,6 +876,18 @@ WalletDB.prototype.getPath = co(function* getPath(wid, hash) {
   return path;
 });
 
+/**
+ * Test whether a wallet contains a path.
+ * @param {WalletID} wid
+ * @param {Hash} hash
+ * @returns {Promise}
+ */
+
+WalletDB.prototype.hasPath = co(function* hasPath(wid, hash) {
+  var data = yield this.db.get(layout.P(wid, hash));
+  return data != null;
+});
+
 /**
  * Get all address hashes.
  * @returns {Promise}

test/wallet-test.js

@@ -42,6 +42,7 @@ describe('Wallet', function() {
   walletdb = new bcoin.walletdb({
     name: 'wallet-test',
     db: 'memory',
+    resolution: true,
     verify: true
   });
 
@@ -244,17 +245,20 @@ describe('Wallet', function() {
     yield walletdb.addTX(t4);
 
     balance = yield w.getBalance();
-    assert.equal(balance.total, 22500);
+    //assert.equal(balance.total, 22500);
+    assert.equal(balance.total, 0);
 
     yield walletdb.addTX(t1);
 
     balance = yield w.getBalance();
-    assert.equal(balance.total, 73000);
+    //assert.equal(balance.total, 73000);
+    assert.equal(balance.total, 51000);
 
     yield walletdb.addTX(t2);
 
     balance = yield w.getBalance();
-    assert.equal(balance.total, 47000);
+    //assert.equal(balance.total, 47000);
+    assert.equal(balance.total, 49000);
 
     yield walletdb.addTX(t3);