From 9e4db47792c68ed7ada8f08730ee1496f8bbda9a Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 6 Dec 2016 21:09:40 -0800 Subject: [PATCH] wallet: improve size estimation. --- lib/primitives/mtx.js | 419 ++++++++++-------------------------------- lib/primitives/tx.js | 23 +-- lib/wallet/wallet.js | 104 ++++++++++- test/wallet-test.js | 2 +- 4 files changed, 201 insertions(+), 347 deletions(-) diff --git a/lib/primitives/mtx.js b/lib/primitives/mtx.js index d9ccb114..e9236abe 100644 --- a/lib/primitives/mtx.js +++ b/lib/primitives/mtx.js @@ -9,6 +9,7 @@ var assert = require('assert'); var util = require('../utils/util'); +var co = require('../utils/co'); var btcutils = require('../btc/utils'); var constants = require('../protocol/constants'); var Script = require('../script/script'); @@ -618,76 +619,6 @@ MTX.prototype.signVector = function signVector(prev, vector, sig, ring) { return false; }; -/** - * Combine and sort multisig signatures for script. - * Mimics bitcoind's behavior. - * @param {Number} index - * @param {Script} prev - * @param {Witness|Script} vector - * @param {Number} version - * @param {Buffer} data - * @return {Boolean} - */ - -MTX.prototype.combine = function combine(index, prev, vector, version, data) { - var m = prev.getSmall(0); - var sigs = []; - var map = {}; - var result = false; - var i, j, sig, type, msg, key, pub, res; - - if (data) - sigs.push(data); - - for (i = 1; i < vector.length; i++) { - sig = vector.get(i); - if (Script.isSignature(sig)) - sigs.push(sig); - } - - for (i = 0; i < sigs.length; i++) { - sig = sigs[i]; - type = sig[sig.length - 1]; - - msg = this.signatureHash(index, prev, type, version); - - for (j = 1; j < prev.length - 2; j++) { - key = prev.get(j); - pub = key.toString('hex'); - - if (map[pub]) - continue; - - res = Script.checksig(msg, sig, key); - - if (res) { - map[pub] = sig; - if (util.equal(sig, data)) - result = true; - break; - } - } - } - - vector.clear(); - vector.push(opcodes.OP_0); - - for (i = 1; i < prev.length - 2; i++) { - key = prev.get(i); - pub = key.toString('hex'); - sig = map[pub]; - if (sig) - vector.push(sig); - } - - while (vector.length - 1 < m) - vector.push(opcodes.OP_0); - - vector.compile(); - - return result; -}; - /** * Create a signature suitable for inserting into scriptSigs/witnesses. * @param {Number} index - Index of input being signed. @@ -912,82 +843,34 @@ MTX.prototype.signAsync = function signAsync(ring, type) { return workerPool.sign(this, ring, type); }; -/** - * Test whether the transaction at least - * has all script templates built. - * @returns {Boolean} - */ - -MTX.prototype.isScripted = function isScripted() { - var i; - - if (this.outputs.length === 0) - return false; - - if (this.inputs.length === 0) - return false; - - for (i = 0; i < this.inputs.length; i++) { - if (!this.isInputScripted(i)) - return false; - } - - return true; -}; - -/** - * Test whether the input at least - * has all script templates built. - * @returns {Boolean} - */ - -MTX.prototype.isInputScripted = function isInputScripted(index) { - var input = this.inputs[index]; - - assert(input, 'Input does not exist.'); - - if (input.script.raw.length === 0 - && input.witness.items.length === 0) { - return false; - } - - return true; -}; - /** * Estimate maximum possible size. - * @param {Object?} options - Wallet or options object. - * @param {Number} options.m - Multisig `m` value. - * @param {Number} options.n - Multisig `n` value. + * @param {Function?} estimate - Input script size estimator. * @returns {Number} */ -MTX.prototype.maxSize = function maxSize(options) { - var scale = constants.WITNESS_SCALE_FACTOR; - var i, j, input, total, size, prev, m, n, sz; - var witness, hadWitness, redeem; - - if (!options && this.isScripted()) - return this.getVirtualSize(); - - if (!options) - options = {}; +MTX.prototype.estimateSize = co(function* estimateSize(estimate) { + var total = 0; + var i, input, output, size, prev; // Calculate the size, minus the input scripts. - total = this.getBaseSize(); + total += 4; + total += encoding.sizeVarint(this.inputs.length); + total += this.inputs.length * 40; - for (i = 0; i < this.inputs.length; i++) { - input = this.inputs[i]; - size = input.script.getSize(); - total -= encoding.sizeVarint(size) + size; + total += encoding.sizeVarint(this.outputs.length); + + for (i = 0; i < this.outputs.length; i++) { + output = this.outputs[i]; + total += output.getSize(); } + total += 4; + // Add size for signatures and public keys for (i = 0; i < this.inputs.length; i++) { input = this.inputs[i]; size = 0; - witness = false; - redeem = null; // We're out of luck here. // Just assume it's a p2pkh. @@ -996,150 +879,92 @@ MTX.prototype.maxSize = function maxSize(options) { continue; } - // Get the previous output's script + // Previous output script. prev = input.coin.script; - // If we have access to the redeem script, - // we can use it to calculate size much easier. - if (prev.isScripthash()) { - // Need to add the redeem script size - // here since it will be ignored by - // the isMultisig clause. - // OP_PUSHDATA2 [redeem] - redeem = this._guessRedeem(prev.get(1), options); - if (redeem) { - prev = redeem; - sz = prev.getSize(); - size += Script.sizePush(sz); - size += sz; - } - } - - if (prev.isProgram()) { - witness = true; - - // Now calculating vsize. - if (redeem) { - // The regular redeem script - // is now worth 4 points. - size += encoding.sizeVarint(size); - size *= 4; - } else { - // Add one varint byte back - // for the 0-byte input script. - size += 1 * 4; - } - - // Add 2 bytes for flag and marker. - if (!hadWitness) - size += 2; - - hadWitness = true; - - if (prev.isWitnessScripthash()) { - redeem = this._guessRedeem(prev.get(1), options); - if (redeem) { - prev = redeem; - sz = prev.getSize(); - size += encoding.sizeVarint(sz); - size += sz; - } - } else if (prev.isWitnessPubkeyhash()) { - prev = Script.fromPubkeyhash(prev.get(1)); - } - } - + // P2PK if (prev.isPubkey()) { - // P2PK + // varint script size + size += 1; // OP_PUSHDATA0 [signature] size += 1 + 73; - } else if (prev.isPubkeyhash()) { - // P2PKH + total += size; + continue; + } + + // P2PKH + if (prev.isPubkeyhash()) { + // varint script size + size += 1; // OP_PUSHDATA0 [signature] size += 1 + 73; // OP_PUSHDATA0 [key] size += 1 + 33; - } else if (prev.isMultisig()) { + total += size; + continue; + } + + if (prev.isMultisig()) { // Bare Multisig - // Get the previous m value: - m = prev.getSmall(0); // OP_0 size += 1; // OP_PUSHDATA0 [signature] ... - size += (1 + 73) * m; - } else if (prev.isScripthash() || prev.isWitnessScripthash()) { - // P2SH Multisig - // This technically won't work well for other - // kinds of P2SH. It will also over-estimate - // the fee by a lot (at least 10000 satoshis - // since we don't have access to the m and n - // values), which will be recalculated later. - // If fee turns out to be smaller later, we - // simply add more of the fee to the change - // output. - // m value - m = options.m || 2; - // n value - n = options.n || 3; - // OP_0 + size += (1 + 73) * prev.getSmall(0); + // varint len + size += encoding.sizeVarint(size); + total += size; + continue; + } + + // P2WPKH + if (prev.isWitnessPubkeyhash()) { + // varint-items-len size += 1; - // OP_PUSHDATA0 [signature] ... - size += (1 + 73) * m; - // OP_PUSHDATA2 [redeem] - size += 3; - // m value - size += 1; - // OP_PUSHDATA0 [key] ... - size += (1 + 33) * n; - // n value - size += 1; - // OP_CHECKMULTISIG - size += 1; - } else { - // OP_PUSHDATA0 [signature] - for (j = 0; j < prev.length; j++) { - if (Script.isKey(prev.get(j))) - size += 1 + 73; + // varint-len [signature] + size += 1 + 73; + // varint-len [key] + size += 1 + 33; + // vsize + size = (size + scale - 1) / scale | 0; + total += size; + continue; + } + + if (estimate) { + size = yield estimate(prev); + if (size !== -1) { + total += size; + continue; } } - if (witness) { - // Calculate vsize if - // we're a witness program. - size = (size + scale - 1) / scale | 0; - } else { - // Byte for varint - // size of input script. - size += encoding.sizeVarint(size); + // P2SH + if (prev.isScripthash()) { + // varint size + total += 2; + // 2-of-3 multisig input + total += 257; + continue; } - total += size; + // P2WSH + if (prev.isWitnessScripthash()) { + // varint-len + size += 1; + // 2-of-3 multisig input + size += 257; + // vsize + size = (size + scale - 1) / scale | 0; + total += size; + continue; + } + + // Unknown. + total += 110; } return total; -}; - -/** - * "Guess" a redeem script based on some options. - * @private - * @param {Object} options - * @param {Buffer} hash - * @returns {Script|null} - */ - -MTX.prototype._guessRedeem = function guessRedeem(options, hash) { - switch (hash.length) { - case 20: - if (options.witness) { - if (options.n > 1) - return Script.fromProgram(0, constants.ZERO_HASH); - return Script.fromProgram(0, constants.ZERO_HASH160); - } - return options.script; - case 32: - return options.script; - } -}; +}); /** * Select necessary coins based on total output value. @@ -1151,8 +976,6 @@ MTX.prototype._guessRedeem = function guessRedeem(options, hash) { * @param {Boolean} options.round - Whether to round to the nearest * kilobyte for fee calculation. * See {@link TX#getMinFee} vs. {@link TX#getRoundFee}. - * @param {Boolean} options.free - Do not apply a fee if the - * transaction priority is high enough to be considered free. * @param {Amount?} options.hardFee - Use a hard fee rather * than calculating one. * @param {Rate?} options.rate - Rate used for fee calculation. @@ -1236,26 +1059,15 @@ MTX.prototype.subtractFee = function subtractFee(fee, index) { * @returns {CoinSelector} */ -MTX.prototype.fund = function fund(coins, options) { - var i, select, change, changeAddress; +MTX.prototype.fund = co(function* fund(coins, options) { + var i, select, change; + assert(options, 'Options are required.'); + assert(options.changeAddress, 'Change address is required.'); assert(this.inputs.length === 0, 'TX is already filled.'); // Select necessary coins. - select = this.selectCoins(coins, options); - - // We need a change address. - changeAddress = select.changeAddress; - - // If change address is not available, - // send back to one of the coins' addresses. - for (i = 0; i < select.chosen.length && !changeAddress; i++) - changeAddress = select.chosen[i].getAddress(); - - // Will only happen in rare cases where - // we're redeeming all non-standard coins. - if (!changeAddress) - throw new Error('No change address available.'); + select = yield this.selectCoins(coins, options); // Add coins to transaction. for (i = 0; i < select.chosen.length; i++) @@ -1267,7 +1079,7 @@ MTX.prototype.fund = function fund(coins, options) { // Add a change output. this.addOutput({ - address: changeAddress, + address: select.changeAddress, value: select.change }); @@ -1284,7 +1096,7 @@ MTX.prototype.fund = function fund(coins, options) { } return select; -}; +}); /** * Sort inputs and outputs according to BIP69. @@ -1464,7 +1276,6 @@ function CoinSelector(tx, options) { this.selection = 'age'; this.shouldSubtract = false; this.subtractFee = null; - this.free = false; this.height = -1; this.confirmations = -1; this.hardFee = -1; @@ -1474,10 +1285,7 @@ function CoinSelector(tx, options) { this.changeAddress = null; // Needed for size estimation. - this.m = null; - this.n = null; - this.witness = false; - this.script = null; + this.estimate = null; if (options) this.fromOptions(options); @@ -1502,11 +1310,6 @@ CoinSelector.prototype.fromOptions = function fromOptions(options) { this.shouldSubtract = options.subtractFee !== false; } - if (options.free != null) { - assert(typeof options.free === 'boolean'); - this.free = options.free; - } - if (options.height != null) { assert(util.isNumber(options.height)); assert(options.height >= -1); @@ -1552,26 +1355,9 @@ CoinSelector.prototype.fromOptions = function fromOptions(options) { } } - if (options.m != null) { - assert(util.isNumber(options.m)); - assert(options.m >= 1); - this.m = options.m; - } - - if (options.n != null) { - assert(util.isNumber(options.n)); - assert(options.n >= 1); - this.n = options.n; - } - - if (options.witness != null) { - assert(typeof options.witness === 'boolean'); - this.witness = options.witness; - } - - if (options.script) { - assert(options.script instanceof Script); - this.script = options.script; + if (options.estimate) { + assert(typeof options.estimate === 'function'); + this.estimate = options.estimate; } return this; @@ -1717,13 +1503,13 @@ CoinSelector.prototype.fund = function fund() { * @returns {CoinSelector} */ -CoinSelector.prototype.select = function select(coins) { +CoinSelector.prototype.select = co(function* select(coins) { this.init(coins); if (this.hardFee !== -1) this.selectHard(this.hardFee); else - this.selectEstimate(constants.tx.MIN_FEE); + yield this.selectEstimate(constants.tx.MIN_FEE); if (!this.isFull()) { // Still failing to get enough funds. @@ -1737,14 +1523,14 @@ CoinSelector.prototype.select = function select(coins) { this.change = this.tx.getInputValue() - this.total(); return this; -}; +}); /** * Initialize selection based on size estimate. * @param {Amount} fee */ -CoinSelector.prototype.selectEstimate = function selectEstimate(fee) { +CoinSelector.prototype.selectEstimate = co(function* selectEstimate(fee) { var size; // Initial fee. @@ -1764,23 +1550,10 @@ CoinSelector.prototype.selectEstimate = function selectEstimate(fee) { value: 0 }); - if (this.free && this.height !== -1) { - size = this.tx.maxSize(this); - - // Note that this will only work - // if the mempool's rolling reject - // fee is zero (i.e. the mempool is - // not full). - if (this.tx.isFree(this.height + 1, size)) { - this.fee = 0; - return; - } - } - // Keep recalculating fee and funding // until we reach some sort of equilibrium. do { - size = this.tx.maxSize(this); + size = yield this.tx.estimateSize(this.estimate); this.fee = this.getFee(size); @@ -1795,7 +1568,7 @@ CoinSelector.prototype.selectEstimate = function selectEstimate(fee) { if (!this.isFull()) this.fund(); } while (!this.isFull() && this.index < this.coins.length); -}; +}); /** * Initiate selection based on a hard fee. diff --git a/lib/primitives/tx.js b/lib/primitives/tx.js index 71f871c0..ced145af 100644 --- a/lib/primitives/tx.js +++ b/lib/primitives/tx.js @@ -1719,17 +1719,6 @@ TX.prototype.checkInputs = function checkInputs(spendHeight, ret) { return true; }; -/** - * Estimate the max possible size of transaction once the - * inputs are scripted. If the transaction is non-mutable, - * this will just return the virtual size. - * @returns {Number} size - */ - -TX.prototype.maxSize = function maxSize() { - return this.getVirtualSize(); -}; - /** * Calculate the modified size of the transaction. This * is used in the mempool for calculating priority. @@ -1742,7 +1731,7 @@ TX.prototype.getModifiedSize = function getModifiedSize(size) { var i, input, offset; if (size == null) - size = this.maxSize(); + size = this.getVirtualSize(); for (i = 0; i < this.inputs.length; i++) { input = this.inputs[i]; @@ -1773,7 +1762,7 @@ TX.prototype.getPriority = function getPriority(height, size) { return sum; if (size == null) - size = this.maxSize(); + size = this.getVirtualSize(); for (i = 0; i < this.inputs.length; i++) { input = this.inputs[i]; @@ -1853,7 +1842,7 @@ TX.prototype.isFree = function isFree(height, size) { TX.prototype.getMinFee = function getMinFee(size, rate) { if (size == null) - size = this.maxSize(); + size = this.getVirtualSize(); return btcutils.getMinFee(size, rate); }; @@ -1870,7 +1859,7 @@ TX.prototype.getMinFee = function getMinFee(size, rate) { TX.prototype.getRoundFee = function getRoundFee(size, rate) { if (size == null) - size = this.maxSize(); + size = this.getVirtualSize(); return btcutils.getRoundFee(size, rate); }; @@ -1884,7 +1873,7 @@ TX.prototype.getRoundFee = function getRoundFee(size, rate) { TX.prototype.getRate = function getRate(size) { if (size == null) - size = this.maxSize(); + size = this.getVirtualSize(); return btcutils.getRate(size, this.getFee()); }; @@ -2065,7 +2054,7 @@ TX.prototype.inspect = function inspect() { hash: this.rhash(), witnessHash: this.rwhash(), size: this.getSize(), - virtualSize: this.maxSize(), + virtualSize: this.getVirtualSize(), height: this.height, value: Amount.btc(this.getOutputValue()), fee: Amount.btc(this.getFee()), diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 5d4052e3..36be4120 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -12,6 +12,7 @@ var EventEmitter = require('events').EventEmitter; var constants = require('../protocol/constants'); var Network = require('../protocol/network'); var util = require('../utils/util'); +var encoding = require('../utils/encoding'); var Locker = require('../utils/locker'); var co = require('../utils/co'); var crypto = require('../crypto/crypto'); @@ -23,6 +24,7 @@ var TXDB = require('./txdb'); var Path = require('./path'); var common = require('./common'); var Address = require('../primitives/address'); +var Script = require('../script/script'); var MTX = require('../primitives/mtx'); var WalletKey = require('./walletkey'); var HD = require('../hd/hd'); @@ -1430,24 +1432,114 @@ Wallet.prototype._fund = co(function* fund(tx, options) { // Don't use any locked coins. coins = this.txdb.filterLocked(coins); - tx.fund(coins, { + yield tx.fund(coins, { selection: options.selection, round: options.round, confirmations: options.confirmations, - free: options.free, hardFee: options.hardFee, subtractFee: options.subtractFee, changeAddress: account.change.getAddress(), height: this.db.state.height, rate: rate, maxFee: options.maxFee, - m: account.m, - n: account.n, - witness: account.witness, - script: account.receive.script + estimate: this.estimate.bind(this) }); }); +/** + * Get account by address. + * @param {Address} address + * @returns {Account} + */ + +Wallet.prototype.getAccountByAddress = co(function* getAccountByAddress(address) { + var hash = Address.getHash(address, 'hex'); + var path, account; + + if (!hash) + return; + + path = yield this.getPath(hash); + + if (!path) + return; + + return yield this.getAccount(path.account); +}); + +/** + * Input size estimator for max possible tx size. + * @param {Script} prev + * @returns {Number} + */ + +Wallet.prototype.estimate = co(function* estimate(prev) { + var scale = constants.WITNESS_SCALE_FACTOR; + var address = prev.getAddress(); + var account = yield this.getAccountByAddress(address); + var size = 0; + + if (!account) + return -1; + + if (prev.isScripthash()) { + // Nested bullshit. + if (account.witness) { + switch (account.type) { + case Account.types.PUBKEYHASH: + size += 23; // redeem script + size *= 4; // vsize + break; + case Account.types.MULTISIG: + size += 35; // redeem script + size *= 4; // vsize + break; + } + } + } + + switch (account.type) { + case Account.types.PUBKEYHASH: + // P2PKH + // OP_PUSHDATA0 [signature] + size += 1 + 73; + // OP_PUSHDATA0 [key] + size += 1 + 33; + break; + case Account.types.MULTISIG: + // P2SH Multisig + // OP_0 + size += 1; + // OP_PUSHDATA0 [signature] ... + size += (1 + 73) * account.m; + // OP_PUSHDATA2 [redeem] + size += 3; + // m value + size += 1; + // OP_PUSHDATA0 [key] ... + size += (1 + 33) * account.n; + // n value + size += 1; + // OP_CHECKMULTISIG + size += 1; + break; + } + + if (account.witness) { + // Varint witness items length. + size += 1; + // Calculate vsize if + // we're a witness program. + size = (size + scale - 1) / scale | 0; + } else { + // Byte for varint + // size of input script. + size += encoding.sizeVarint(size); + } + + return size; +}); + /** * Build a transaction, fill it with outputs and inputs, * sort the members according to BIP69, set locktime, diff --git a/test/wallet-test.js b/test/wallet-test.js index 9064b6c4..610dbe7b 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -195,7 +195,7 @@ describe('Wallet', function() { .addInput(src, 0) .addOutput(w.getAddress(), 5460); - maxSize = tx.maxSize(); + maxSize = yield tx.estimateSize(); yield w.sign(tx);