From c07848fadd2d4dc580bf9cb4e0df3e3df4e6aaa3 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 15 Oct 2016 21:02:40 -0700 Subject: [PATCH] txdb: orphan resolution. --- bench/walletdb.js | 4 +- lib/node/fullnode.js | 1 + lib/node/spvnode.js | 1 + lib/wallet/browser.js | 6 - lib/wallet/pathinfo.js | 8 +- lib/wallet/txdb.js | 394 ++++++++++++++++++++++------------------- lib/wallet/wallet.js | 35 ++++ lib/wallet/walletdb.js | 12 ++ test/wallet-test.js | 10 +- 9 files changed, 277 insertions(+), 194 deletions(-) diff --git a/bench/walletdb.js b/bench/walletdb.js index a067fe85..f631421e 100644 --- a/bench/walletdb.js +++ b/bench/walletdb.js @@ -34,7 +34,9 @@ var walletdb = new bcoin.walletdb({ name: 'wallet-test', // location: __dirname + '/../walletdb-bench', // db: 'leveldb' - db: 'memory' + db: 'memory', + resolution: false, + verify: false }); var runBench = co(function* runBench() { diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 15e6fb9c..4516cedb 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -148,6 +148,7 @@ function Fullnode(options) { witness: this.options.witness, useCheckpoints: this.options.useCheckpoints, maxFiles: this.options.maxFiles, + resolution: true, verify: false }); diff --git a/lib/node/spvnode.js b/lib/node/spvnode.js index 6218dcc8..cf5c2e15 100644 --- a/lib/node/spvnode.js +++ b/lib/node/spvnode.js @@ -86,6 +86,7 @@ function SPVNode(options) { location: this.location('walletdb'), witness: this.options.witness, maxFiles: this.options.maxFiles, + resolution: true, verify: true }); diff --git a/lib/wallet/browser.js b/lib/wallet/browser.js index fc679130..22f2fa45 100644 --- a/lib/wallet/browser.js +++ b/lib/wallet/browser.js @@ -120,12 +120,6 @@ layout.txdb = { ss: function ss(key) { return this.hii(key); }, - o: function o(hash, index) { - return this.hi('o', hash, index); - }, - oo: function oo(key) { - return this.hii(key); - }, p: function p(hash) { return this.ha('p', hash); }, diff --git a/lib/wallet/pathinfo.js b/lib/wallet/pathinfo.js index 444c5dde..d3e462a3 100644 --- a/lib/wallet/pathinfo.js +++ b/lib/wallet/pathinfo.js @@ -108,7 +108,9 @@ PathInfo.fromTX = function fromTX(wallet, tx, paths) { * @returns {Boolean} */ -PathInfo.prototype.hasPath = function hasPath(hash) { +PathInfo.prototype.hasPath = function hasPath(output) { + var hash = output.getHash('hex'); + if (!hash) return false; @@ -121,7 +123,9 @@ PathInfo.prototype.hasPath = function hasPath(hash) { * @returns {Path} */ -PathInfo.prototype.getPath = function getPath(hash) { +PathInfo.prototype.getPath = function getPath(output) { + var hash = output.getHash('hex'); + if (!hash) return; diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index 18d5fe9e..d29d9557 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -136,12 +136,6 @@ var layout = { ss: function ss(key) { return layout.hii(key); }, - o: function o(hash, index) { - return layout.hi(0x6f, hash, index); - }, - oo: function oo(key) { - return layout.hii(key); - }, p: function p(hash) { return layout.ha(0x70, hash); }, @@ -219,6 +213,10 @@ function TXDB(wallet) { this.balance = null; this.pending = null; this.events = []; + + this.orphans = {}; + this.count = {}; + this.totalOrphans = 0; } /** @@ -438,57 +436,183 @@ TXDB.prototype.getPathInfo = function getPathInfo(tx) { }; /** - * Add an orphan (tx hash + input index) - * to orphan list. Stored by its required coin ID. - * @private - * @param {Outpoint} prevout - Required coin hash & index. - * @param {Buffer} input - Spender input hash and index. - * @returns {Promise} - Returns Buffer. + * Determine which transactions to add. + * Attempt to resolve orphans (for SPV). + * @param {TX} tx + * @returns {Promise} */ -TXDB.prototype.addOrphan = co(function* addOrphan(prevout, input) { - var key = layout.o(prevout.hash, prevout.index); - var data = yield this.get(key); - var p = new BufferWriter(); +TXDB.prototype.resolve = co(function* add(tx) { + var hash, result; - if (data) - p.writeBytes(data); + if (!this.options.resolution) + return [tx]; - p.writeBytes(input); + hash = tx.hash('hex'); - this.put(key, p.render()); + if (yield this.hasTX(hash)) + return [tx]; + + result = yield this.verifyInputs(tx); + + if (!result) + return []; + + return yield this.resolveOutputs(tx); }); /** - * Retrieve orphan list by coin ID. - * @private - * @param {Hash} hash - * @param {Number} index - * @returns {Promise} - Returns {@link Orphan}. + * Verify inputs and potentially add orphans. + * Used in SPV mode. + * @param {TX} tx + * @returns {Promise} */ -TXDB.prototype.getOrphans = co(function* getOrphans(hash, index) { - var key = layout.o(hash, index); - var data = yield this.get(key); - var items = []; - var i, inputs, input, tx, p; +TXDB.prototype.verifyInputs = co(function* verifyInputs(tx) { + var hash = tx.hash('hex'); + var hasOrphans = false; + var orphans = []; + var i, input, prevout, address; + var path, key, coin, spent; - if (!data) - return; + if (tx.isCoinbase()) + return true; - p = new BufferReader(data); - inputs = []; + if (this.count[hash]) + return false; - while (p.left()) - inputs.push(Outpoint.fromRaw(p)); + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + prevout = input.prevout; + coin = yield this.getCoin(prevout.hash, prevout.index); - for (i = 0; i < inputs.length; i++) { - input = inputs[i]; - tx = yield this.getTX(input.hash); - items.push(new Orphan(input, tx)); + if (coin) { + input.coin = coin; + + if (this.options.verify) { + if (!(yield tx.verifyInputAsync(i))) + return false; + } + + continue; + } + + spent = yield this.isSpent(prevout.hash, prevout.index); + + if (!spent) { + address = input.getHash('hex'); + path = yield this.wallet.hasPath(address); + + if (!path) + continue; + + orphans[i] = true; + continue; + } + + coin = yield this.getSpentCoin(spent, prevout); + assert(coin); + + input.coin = coin; + + if (this.options.verify) { + if (!(yield tx.verifyInputAsync(i))) + return false; + } } - return items; + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + prevout = input.prevout; + + if (!orphans[i]) + continue; + + key = prevout.hash + prevout.index; + + if (this.totalOrphans > 20) { + this.logger.warning('Potential orphan flood!'); + this.logger.warning( + 'More than 20 orphans for %s. Purging.', + this.wallet.id); + this.totalOrphans = 0; + this.orphans = {}; + this.count = {}; + } + + if (!this.orphans[key]) + this.orphans[key] = []; + + if (!this.count[hash]) + this.count[hash] = 0; + + this.orphans[key].push(new Orphan(tx, i)); + this.count[hash]++; + this.totalOrphans++; + + hasOrphans = true; + } + + if (hasOrphans) + return false; + + return true; +}); + +/** + * Resolve orphans for outputs. + * Used in SPV mode. + * @param {TX} tx + * @returns {Promise} + */ + +TXDB.prototype.resolveOutputs = co(function* resolveOutputs(tx, resolved) { + var hash = tx.hash('hex'); + var i, input, output, key, orphans, orphan, coin, valid; + + if (!resolved) + resolved = []; + + resolved.push(tx); + + for (i = 0; i < tx.outputs.length; i++) { + output = tx.outputs[i]; + key = hash + i; + orphans = this.orphans[key]; + + if (!orphans) + continue; + + delete this.orphans[key]; + + coin = Coin.fromTX(tx, i); + + while (orphans.length) { + orphan = orphans.pop(); + valid = true; + + input = orphan.tx.inputs[orphan.index]; + input.coin = coin; + + assert(input.prevout.hash === hash); + assert(input.prevout.index === i); + + if (this.options.verify) + valid = yield orphan.tx.verifyInputAsync(orphan.index); + + if (valid) { + if (--this.count[orphan.hash] === 0) { + delete this.count[orphan.hash]; + yield this.resolveOutputs(orphan.tx, resolved); + } + break; + } + + delete this.count[orphan.hash]; + } + } + + return resolved; }); /** @@ -500,63 +624,48 @@ TXDB.prototype.getOrphans = co(function* getOrphans(hash, index) { * @returns {Promise} */ -TXDB.prototype.verify = co(function* verify(tx, info) { +TXDB.prototype.getInputs = co(function* getInputs(tx, info) { var spends = []; - var orphans = []; + var coins = []; var removed = {}; - var i, input, prevout, address, coin, spent, conflict; + var i, input, prevout, coin, spent, conflict; if (tx.isCoinbase()) - return orphans; + return coins; for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; prevout = input.prevout; - address = input.getHash('hex'); - - // Only bother if this input is ours. - if (!info.hasPath(address)) - continue; + // Try to get the coin we're redeeming. coin = yield this.getCoin(prevout.hash, prevout.index); if (coin) { - // Add TX to inputs and spend money input.coin = coin; - - // Skip invalid transactions - if (this.options.verify) { - if (!(yield tx.verifyInputAsync(i))) - return; - } - + coins.push(coin); continue; } + // Is it already spent? spent = yield this.isSpent(prevout.hash, prevout.index); - // Orphan until we see a parent transaction. + // If we have no coin or spend + // record, this is not our input. + // This is assuming everything came + // in order! if (!spent) { - orphans[i] = true; + coins.push(null); continue; } - // We must be double-spending. + // Yikes, we're double spending. We + // still need the coin for after we + // resolve the conflict. coin = yield this.getSpentCoin(spent, prevout); - - // Double-spent orphan. - if (!coin) { - orphans[i] = true; - continue; - } + assert(coin); input.coin = coin; - - // Skip invalid transactions - if (this.options.verify) { - if (!(yield tx.verifyInputAsync(i))) - return; - } + coins.push(coin); spends[i] = spent; } @@ -598,66 +707,7 @@ TXDB.prototype.verify = co(function* verify(tx, info) { this.emit('conflict', conflict.tx, conflict.info); } - return orphans; -}); - -/** - * Attempt to resolve orphans for an output. - * @private - * @param {TX} tx - * @param {Number} index - * @returns {Promise} - */ - -TXDB.prototype.resolveOrphans = co(function* resolveOrphans(tx, index) { - var hash = tx.hash('hex'); - var i, orphans, coin, input, spender, orphan; - - orphans = yield this.getOrphans(hash, index); - - if (!orphans) - return false; - - this.del(layout.o(hash, index)); - - coin = Coin.fromTX(tx, index); - - // Add input to resolved orphan. - for (i = 0; i < orphans.length; i++) { - orphan = orphans[i]; - spender = orphan.input; - tx = orphan.tx; - - // Probably removed by some other means. - if (!tx) - continue; - - input = tx.inputs[spender.index]; - input.coin = coin; - - assert(input.prevout.hash === hash); - assert(input.prevout.index === index); - - // Verify that input script is correct, if not - add - // output to unspent and remove orphan from storage - if (!this.options.verify || (yield tx.verifyInputAsync(spender.index))) { - // Add the undo coin record which we never had. - this.put(layout.d(spender.hash, spender.index), coin.toRaw()); - // Add the spender record back in case any evil - // transactions were removed with lazyRemove. - this.put(layout.s(hash, index), spender.toRaw()); - return true; - } - - yield this.lazyRemove(tx); - } - - // We had orphans, but they were invalid. The - // balance will be (incorrectly) added outside. - // Subtract to compensate. - this.pending.sub(coin); - - return false; + return coins; }); /** @@ -696,7 +746,7 @@ TXDB.prototype.add = co(function* add(tx) { TXDB.prototype._add = co(function* add(tx, info) { var hash, path, account; var i, result, input, output, coin; - var prevout, key, address, spender, orphans; + var prevout, key, spender, coins; assert(!tx.mutable, 'Cannot add mutable TX to wallet.'); @@ -709,9 +759,9 @@ TXDB.prototype._add = co(function* add(tx, info) { // Verify and get coins. // This potentially removes double-spenders. - orphans = yield this.verify(tx, info); + coins = yield this.getInputs(tx, info); - if (!orphans) + if (!coins) return false; hash = tx.hash('hex'); @@ -743,56 +793,41 @@ TXDB.prototype._add = co(function* add(tx, info) { for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; prevout = input.prevout; - - address = input.getHash('hex'); - path = info.getPath(address); + coin = coins[i]; // Only bother if this input is ours. - if (!path) + if (!coin) continue; + path = info.getPath(coin); + assert(path); + key = prevout.hash + prevout.index; // s[outpoint-key] -> [spender-hash]|[spender-input-index] spender = Outpoint.fromTX(tx, i).toRaw(); this.put(layout.s(prevout.hash, prevout.index), spender); - // Add orphan if no parent transaction known. - // Do not disconnect any coins. - if (orphans[i]) { - yield this.addOrphan(prevout, spender); - continue; - } - this.del(layout.c(prevout.hash, prevout.index)); this.del(layout.C(path.account, prevout.hash, prevout.index)); - this.put(layout.d(hash, i), input.coin.toRaw()); - this.pending.sub(input.coin); + this.put(layout.d(hash, i), coin.toRaw()); + this.pending.sub(coin); this.coinCache.remove(key); } } - // Add unspent outputs or resolve orphans + // Add unspent outputs or resolve orphans. for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; - address = output.getHash('hex'); + path = info.getPath(output); key = hash + i; - path = info.getPath(address); - - // Do not add unspents for outputs that aren't ours. + // Do not add unspents for + // outputs that aren't ours. if (!path) continue; - orphans = yield this.resolveOrphans(tx, i); - - // If this transaction resolves an orphan, - // it should not connect coins as they are - // already spent by the orphan it resolved. - if (orphans) - continue; - coin = Coin.fromTX(tx, i); this.pending.add(coin); @@ -840,10 +875,10 @@ TXDB.prototype.removeConflict = co(function* removeConflict(hash, ref, removed) if (!tx) throw new Error('Could not find spender.'); - if (tx.ts !== 0) { + if (tx.height !== -1) { // If spender is confirmed and replacement // is not confirmed, do nothing. - if (ref.ts === 0) + if (ref.height === -1) return; // If both are confirmed but replacement @@ -853,13 +888,12 @@ TXDB.prototype.removeConflict = co(function* removeConflict(hash, ref, removed) } else { // If spender is unconfirmed and replacement // is confirmed, do nothing. - if (ref.ts !== 0) - return; - - // If both are unconfirmed but replacement - // is older than spender, do nothing. - if (ref.ps < tx.ps) - return; + if (ref.height === -1) { + // If both are unconfirmed but replacement + // is older than spender, do nothing. + if (ref.ps < tx.ps) + return; + } } info = yield this.removeRecursive(tx, removed); @@ -1002,11 +1036,10 @@ TXDB.prototype.confirm = co(function* confirm(tx, info) { for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; - address = output.getHash('hex'); key = hash + i; // Only update coins if this output is ours. - if (!info.hasPath(address)) + if (!info.hasPath(output)) continue; coin = yield this.getCoin(hash, i); @@ -1108,7 +1141,7 @@ TXDB.prototype.lazyRemove = co(function* lazyRemove(tx) { TXDB.prototype.__remove = co(function* remove(tx, info) { var hash = tx.hash('hex'); var i, path, account, key, prevout; - var address, input, output, coin; + var input, output, coin; this.del(layout.t(hash)); @@ -1139,12 +1172,11 @@ TXDB.prototype.__remove = co(function* remove(tx, info) { input = tx.inputs[i]; key = input.prevout.hash + input.prevout.index; prevout = input.prevout; - address = input.getHash('hex'); if (!input.coin) continue; - path = info.getPath(address); + path = info.getPath(input.coin); if (!path) continue; @@ -1157,7 +1189,6 @@ TXDB.prototype.__remove = co(function* remove(tx, info) { this.put(layout.C(path.account, prevout.hash, prevout.index), DUMMY); this.del(layout.d(hash, i)); this.del(layout.s(prevout.hash, prevout.index)); - this.del(layout.o(prevout.hash, prevout.index)); this.coinCache.set(key, coin); } @@ -1166,9 +1197,7 @@ TXDB.prototype.__remove = co(function* remove(tx, info) { for (i = 0; i < tx.outputs.length; i++) { output = tx.outputs[i]; key = hash + i; - address = output.getHash('hex'); - - path = info.getPath(address); + path = info.getPath(output); if (!path) continue; @@ -2406,9 +2435,10 @@ function Conflict(tx, info) { this.info = info; } -function Orphan(input, tx) { - this.input = input; +function Orphan(tx, i) { this.tx = tx; + this.hash = tx.hash('hex'); + this.index = i; } /* diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index cb1d60a8..5da88dcd 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -1092,6 +1092,24 @@ Wallet.prototype.getPath = co(function* getPath(address) { return path; }); +/** + * Test whether the wallet contains a path. + * @param {Address|Hash} address + * @returns {Promise} - Returns {Boolean}. + */ + +Wallet.prototype.hasPath = co(function* hasPath(address) { + var hash = Address.getHash(address, 'hex'); + + if (!hash) + return false; + + if (this.pathCache.has(hash)) + return true; + + return yield this.db.hasPath(this.wid, hash); +}); + /** * Get all wallet paths. * @param {(String|Number)?} acct @@ -1802,11 +1820,28 @@ Wallet.prototype.add = co(function* add(tx) { /** * Add a transaction to the wallet without a lock. + * Potentially resolves orphans. + * @private * @param {TX} tx * @returns {Promise} */ Wallet.prototype._add = co(function* add(tx) { + var resolved = yield this.txdb.resolve(tx); + var i; + + for (i = 0; i < resolved.length; i++) + yield this._insert(resolved[i]); +}); + +/** + * Insert a transaction into the wallet (no lock). + * @private + * @param {TX} tx + * @returns {Promise} + */ + +Wallet.prototype._insert = co(function* insert(tx) { var info = yield this.getPathInfo(tx); var result, derived; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index c39737b8..7430e1dd 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -876,6 +876,18 @@ WalletDB.prototype.getPath = co(function* getPath(wid, hash) { return path; }); +/** + * Test whether a wallet contains a path. + * @param {WalletID} wid + * @param {Hash} hash + * @returns {Promise} + */ + +WalletDB.prototype.hasPath = co(function* hasPath(wid, hash) { + var data = yield this.db.get(layout.P(wid, hash)); + return data != null; +}); + /** * Get all address hashes. * @returns {Promise} diff --git a/test/wallet-test.js b/test/wallet-test.js index a410c1db..d3755cdf 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -42,6 +42,7 @@ describe('Wallet', function() { walletdb = new bcoin.walletdb({ name: 'wallet-test', db: 'memory', + resolution: true, verify: true }); @@ -244,17 +245,20 @@ describe('Wallet', function() { yield walletdb.addTX(t4); balance = yield w.getBalance(); - assert.equal(balance.total, 22500); + //assert.equal(balance.total, 22500); + assert.equal(balance.total, 0); yield walletdb.addTX(t1); balance = yield w.getBalance(); - assert.equal(balance.total, 73000); + //assert.equal(balance.total, 73000); + assert.equal(balance.total, 51000); yield walletdb.addTX(t2); balance = yield w.getBalance(); - assert.equal(balance.total, 47000); + //assert.equal(balance.total, 47000); + assert.equal(balance.total, 49000); yield walletdb.addTX(t3);