wallet: improve size estimation.

lib/primitives/mtx.js

@@ -9,6 +9,7 @@
 
 var assert = require('assert');
 var util = require('../utils/util');
+var co = require('../utils/co');
 var btcutils = require('../btc/utils');
 var constants = require('../protocol/constants');
 var Script = require('../script/script');
@@ -618,76 +619,6 @@ MTX.prototype.signVector = function signVector(prev, vector, sig, ring) {
   return false;
 };
 
-/**
- * Combine and sort multisig signatures for script.
- * Mimics bitcoind's behavior.
- * @param {Number} index
- * @param {Script} prev
- * @param {Witness|Script} vector
- * @param {Number} version
- * @param {Buffer} data
- * @return {Boolean}
- */
-
-MTX.prototype.combine = function combine(index, prev, vector, version, data) {
-  var m = prev.getSmall(0);
-  var sigs = [];
-  var map = {};
-  var result = false;
-  var i, j, sig, type, msg, key, pub, res;
-
-  if (data)
-    sigs.push(data);
-
-  for (i = 1; i < vector.length; i++) {
-    sig = vector.get(i);
-    if (Script.isSignature(sig))
-      sigs.push(sig);
-  }
-
-  for (i = 0; i < sigs.length; i++) {
-    sig = sigs[i];
-    type = sig[sig.length - 1];
-
-    msg = this.signatureHash(index, prev, type, version);
-
-    for (j = 1; j < prev.length - 2; j++) {
-      key = prev.get(j);
-      pub = key.toString('hex');
-
-      if (map[pub])
-        continue;
-
-      res = Script.checksig(msg, sig, key);
-
-      if (res) {
-        map[pub] = sig;
-        if (util.equal(sig, data))
-          result = true;
-        break;
-      }
-    }
-  }
-
-  vector.clear();
-  vector.push(opcodes.OP_0);
-
-  for (i = 1; i < prev.length - 2; i++) {
-    key = prev.get(i);
-    pub = key.toString('hex');
-    sig = map[pub];
-    if (sig)
-      vector.push(sig);
-  }
-
-  while (vector.length - 1 < m)
-    vector.push(opcodes.OP_0);
-
-  vector.compile();
-
-  return result;
-};
-
 /**
  * Create a signature suitable for inserting into scriptSigs/witnesses.
  * @param {Number} index - Index of input being signed.
@@ -912,82 +843,34 @@ MTX.prototype.signAsync = function signAsync(ring, type) {
   return workerPool.sign(this, ring, type);
 };
 
-/**
- * Test whether the transaction at least
- * has all script templates built.
- * @returns {Boolean}
- */
-
-MTX.prototype.isScripted = function isScripted() {
-  var i;
-
-  if (this.outputs.length === 0)
-    return false;
-
-  if (this.inputs.length === 0)
-    return false;
-
-  for (i = 0; i < this.inputs.length; i++) {
-    if (!this.isInputScripted(i))
-      return false;
-  }
-
-  return true;
-};
-
-/**
- * Test whether the input at least
- * has all script templates built.
- * @returns {Boolean}
- */
-
-MTX.prototype.isInputScripted = function isInputScripted(index) {
-  var input = this.inputs[index];
-
-  assert(input, 'Input does not exist.');
-
-  if (input.script.raw.length === 0
-      && input.witness.items.length === 0) {
-    return false;
-  }
-
-  return true;
-};
-
 /**
  * Estimate maximum possible size.
- * @param {Object?} options - Wallet or options object.
+ * @param {Function?} estimate - Input script size estimator.
- * @param {Number} options.m - Multisig `m` value.
- * @param {Number} options.n - Multisig `n` value.
  * @returns {Number}
  */
 
-MTX.prototype.maxSize = function maxSize(options) {
+MTX.prototype.estimateSize = co(function* estimateSize(estimate) {
-  var scale = constants.WITNESS_SCALE_FACTOR;
+  var total = 0;
-  var i, j, input, total, size, prev, m, n, sz;
+  var i, input, output, size, prev;
-  var witness, hadWitness, redeem;
-
-  if (!options && this.isScripted())
-    return this.getVirtualSize();
-
-  if (!options)
-    options = {};
 
   // Calculate the size, minus the input scripts.
-  total = this.getBaseSize();
+  total += 4;
+  total += encoding.sizeVarint(this.inputs.length);
+  total += this.inputs.length * 40;
 
-  for (i = 0; i < this.inputs.length; i++) {
+  total += encoding.sizeVarint(this.outputs.length);
-    input = this.inputs[i];
+
-    size = input.script.getSize();
+  for (i = 0; i < this.outputs.length; i++) {
-    total -= encoding.sizeVarint(size) + size;
+    output = this.outputs[i];
+    total += output.getSize();
   }
 
+  total += 4;
+
   // Add size for signatures and public keys
   for (i = 0; i < this.inputs.length; i++) {
     input = this.inputs[i];
     size = 0;
-    witness = false;
-    redeem = null;
 
     // We're out of luck here.
     // Just assume it's a p2pkh.
@@ -996,150 +879,92 @@ MTX.prototype.maxSize = function maxSize(options) {
       continue;
     }
 
-    // Get the previous output's script
+    // Previous output script.
     prev = input.coin.script;
 
-    // If we have access to the redeem script,
+    // P2PK
-    // we can use it to calculate size much easier.
-    if (prev.isScripthash()) {
-      // Need to add the redeem script size
-      // here since it will be ignored by
-      // the isMultisig clause.
-      // OP_PUSHDATA2 [redeem]
-      redeem = this._guessRedeem(prev.get(1), options);
-      if (redeem) {
-        prev = redeem;
-        sz = prev.getSize();
-        size += Script.sizePush(sz);
-        size += sz;
-      }
-    }
-
-    if (prev.isProgram()) {
-      witness = true;
-
-      // Now calculating vsize.
-      if (redeem) {
-        // The regular redeem script
-        // is now worth 4 points.
-        size += encoding.sizeVarint(size);
-        size *= 4;
-      } else {
-        // Add one varint byte back
-        // for the 0-byte input script.
-        size += 1 * 4;
-      }
-
-      // Add 2 bytes for flag and marker.
-      if (!hadWitness)
-        size += 2;
-
-      hadWitness = true;
-
-      if (prev.isWitnessScripthash()) {
-        redeem = this._guessRedeem(prev.get(1), options);
-        if (redeem) {
-          prev = redeem;
-          sz = prev.getSize();
-          size += encoding.sizeVarint(sz);
-          size += sz;
-        }
-      } else if (prev.isWitnessPubkeyhash()) {
-        prev = Script.fromPubkeyhash(prev.get(1));
-      }
-    }
-
     if (prev.isPubkey()) {
-      // P2PK
+      // varint script size
+      size += 1;
       // OP_PUSHDATA0 [signature]
       size += 1 + 73;
-    } else if (prev.isPubkeyhash()) {
+      total += size;
-      // P2PKH
+      continue;
+    }
+
+    // P2PKH
+    if (prev.isPubkeyhash()) {
+      // varint script size
+      size += 1;
       // OP_PUSHDATA0 [signature]
       size += 1 + 73;
       // OP_PUSHDATA0 [key]
       size += 1 + 33;
-    } else if (prev.isMultisig()) {
+      total += size;
+      continue;
+    }
+
+    if (prev.isMultisig()) {
       // Bare Multisig
-      // Get the previous m value:
-      m = prev.getSmall(0);
       // OP_0
       size += 1;
       // OP_PUSHDATA0 [signature] ...
-      size += (1 + 73) * m;
+      size += (1 + 73) * prev.getSmall(0);
-    } else if (prev.isScripthash() || prev.isWitnessScripthash()) {
+      // varint len
-      // P2SH Multisig
+      size += encoding.sizeVarint(size);
-      // This technically won't work well for other
+      total += size;
-      // kinds of P2SH. It will also over-estimate
+      continue;
-      // the fee by a lot (at least 10000 satoshis
+    }
-      // since we don't have access to the m and n
+
-      // values), which will be recalculated later.
+    // P2WPKH
-      // If fee turns out to be smaller later, we
+    if (prev.isWitnessPubkeyhash()) {
-      // simply add more of the fee to the change
+      // varint-items-len
-      // output.
-      // m value
-      m = options.m || 2;
-      // n value
-      n = options.n || 3;
-      // OP_0
       size += 1;
-      // OP_PUSHDATA0 [signature] ...
+      // varint-len [signature]
-      size += (1 + 73) * m;
+      size += 1 + 73;
-      // OP_PUSHDATA2 [redeem]
+      // varint-len [key]
-      size += 3;
+      size += 1 + 33;
-      // m value
+      // vsize
-      size += 1;
+      size = (size + scale - 1) / scale | 0;
-      // OP_PUSHDATA0 [key] ...
+      total += size;
-      size += (1 + 33) * n;
+      continue;
-      // n value
+    }
-      size += 1;
+
-      // OP_CHECKMULTISIG
+    if (estimate) {
-      size += 1;
+      size = yield estimate(prev);
-    } else {
+      if (size !== -1) {
-      // OP_PUSHDATA0 [signature]
+        total += size;
-      for (j = 0; j < prev.length; j++) {
+        continue;
-        if (Script.isKey(prev.get(j)))
-          size += 1 + 73;
       }
     }
 
-    if (witness) {
+    // P2SH
-      // Calculate vsize if
+    if (prev.isScripthash()) {
-      // we're a witness program.
+      // varint size
-      size = (size + scale - 1) / scale | 0;
+      total += 2;
-    } else {
+      // 2-of-3 multisig input
-      // Byte for varint
+      total += 257;
-      // size of input script.
+      continue;
-      size += encoding.sizeVarint(size);
     }
 
-    total += size;
+    // P2WSH
+    if (prev.isWitnessScripthash()) {
+      // varint-len
+      size += 1;
+      // 2-of-3 multisig input
+      size += 257;
+      // vsize
+      size = (size + scale - 1) / scale | 0;
+      total += size;
+      continue;
+    }
+
+    // Unknown.
+    total += 110;
   }
 
   return total;
-};
+});
-
-/**
- * "Guess" a redeem script based on some options.
- * @private
- * @param {Object} options
- * @param {Buffer} hash
- * @returns {Script|null}
- */
-
-MTX.prototype._guessRedeem = function guessRedeem(options, hash) {
-  switch (hash.length) {
-    case 20:
-      if (options.witness) {
-        if (options.n > 1)
-          return Script.fromProgram(0, constants.ZERO_HASH);
-        return Script.fromProgram(0, constants.ZERO_HASH160);
-      }
-      return options.script;
-    case 32:
-      return options.script;
-  }
-};
 
 /**
  * Select necessary coins based on total output value.
@@ -1151,8 +976,6 @@ MTX.prototype._guessRedeem = function guessRedeem(options, hash) {
  * @param {Boolean} options.round - Whether to round to the nearest
  * kilobyte for fee calculation.
  * See {@link TX#getMinFee} vs. {@link TX#getRoundFee}.
- * @param {Boolean} options.free - Do not apply a fee if the
- * transaction priority is high enough to be considered free.
  * @param {Amount?} options.hardFee - Use a hard fee rather
  * than calculating one.
  * @param {Rate?} options.rate - Rate used for fee calculation.
@@ -1236,26 +1059,15 @@ MTX.prototype.subtractFee = function subtractFee(fee, index) {
  * @returns {CoinSelector}
  */
 
-MTX.prototype.fund = function fund(coins, options) {
+MTX.prototype.fund = co(function* fund(coins, options) {
-  var i, select, change, changeAddress;
+  var i, select, change;
 
+  assert(options, 'Options are required.');
+  assert(options.changeAddress, 'Change address is required.');
   assert(this.inputs.length === 0, 'TX is already filled.');
 
   // Select necessary coins.
-  select = this.selectCoins(coins, options);
+  select = yield this.selectCoins(coins, options);
-
-  // We need a change address.
-  changeAddress = select.changeAddress;
-
-  // If change address is not available,
-  // send back to one of the coins' addresses.
-  for (i = 0; i < select.chosen.length && !changeAddress; i++)
-    changeAddress = select.chosen[i].getAddress();
-
-  // Will only happen in rare cases where
-  // we're redeeming all non-standard coins.
-  if (!changeAddress)
-    throw new Error('No change address available.');
 
   // Add coins to transaction.
   for (i = 0; i < select.chosen.length; i++)
@@ -1267,7 +1079,7 @@ MTX.prototype.fund = function fund(coins, options) {
 
   // Add a change output.
   this.addOutput({
-    address: changeAddress,
+    address: select.changeAddress,
     value: select.change
   });
 
@@ -1284,7 +1096,7 @@ MTX.prototype.fund = function fund(coins, options) {
   }
 
   return select;
-};
+});
 
 /**
  * Sort inputs and outputs according to BIP69.
@@ -1464,7 +1276,6 @@ function CoinSelector(tx, options) {
   this.selection = 'age';
   this.shouldSubtract = false;
   this.subtractFee = null;
-  this.free = false;
   this.height = -1;
   this.confirmations = -1;
   this.hardFee = -1;
@@ -1474,10 +1285,7 @@ function CoinSelector(tx, options) {
   this.changeAddress = null;
 
   // Needed for size estimation.
-  this.m = null;
+  this.estimate = null;
-  this.n = null;
-  this.witness = false;
-  this.script = null;
 
   if (options)
     this.fromOptions(options);
@@ -1502,11 +1310,6 @@ CoinSelector.prototype.fromOptions = function fromOptions(options) {
     this.shouldSubtract = options.subtractFee !== false;
   }
 
-  if (options.free != null) {
-    assert(typeof options.free === 'boolean');
-    this.free = options.free;
-  }
-
   if (options.height != null) {
     assert(util.isNumber(options.height));
     assert(options.height >= -1);
@@ -1552,26 +1355,9 @@ CoinSelector.prototype.fromOptions = function fromOptions(options) {
     }
   }
 
-  if (options.m != null) {
+  if (options.estimate) {
-    assert(util.isNumber(options.m));
+    assert(typeof options.estimate === 'function');
-    assert(options.m >= 1);
+    this.estimate = options.estimate;
-    this.m = options.m;
-  }
-
-  if (options.n != null) {
-    assert(util.isNumber(options.n));
-    assert(options.n >= 1);
-    this.n = options.n;
-  }
-
-  if (options.witness != null) {
-    assert(typeof options.witness === 'boolean');
-    this.witness = options.witness;
-  }
-
-  if (options.script) {
-    assert(options.script instanceof Script);
-    this.script = options.script;
   }
 
   return this;
@@ -1717,13 +1503,13 @@ CoinSelector.prototype.fund = function fund() {
  * @returns {CoinSelector}
  */
 
-CoinSelector.prototype.select = function select(coins) {
+CoinSelector.prototype.select = co(function* select(coins) {
   this.init(coins);
 
   if (this.hardFee !== -1)
     this.selectHard(this.hardFee);
   else
-    this.selectEstimate(constants.tx.MIN_FEE);
+    yield this.selectEstimate(constants.tx.MIN_FEE);
 
   if (!this.isFull()) {
     // Still failing to get enough funds.
@@ -1737,14 +1523,14 @@ CoinSelector.prototype.select = function select(coins) {
   this.change = this.tx.getInputValue() - this.total();
 
   return this;
-};
+});
 
 /**
  * Initialize selection based on size estimate.
  * @param {Amount} fee
  */
 
-CoinSelector.prototype.selectEstimate = function selectEstimate(fee) {
+CoinSelector.prototype.selectEstimate = co(function* selectEstimate(fee) {
   var size;
 
   // Initial fee.
@@ -1764,23 +1550,10 @@ CoinSelector.prototype.selectEstimate = function selectEstimate(fee) {
     value: 0
   });
 
-  if (this.free && this.height !== -1) {
-    size = this.tx.maxSize(this);
-
-    // Note that this will only work
-    // if the mempool's rolling reject
-    // fee is zero (i.e. the mempool is
-    // not full).
-    if (this.tx.isFree(this.height + 1, size)) {
-      this.fee = 0;
-      return;
-    }
-  }
-
   // Keep recalculating fee and funding
   // until we reach some sort of equilibrium.
   do {
-    size = this.tx.maxSize(this);
+    size = yield this.tx.estimateSize(this.estimate);
 
     this.fee = this.getFee(size);
 
@@ -1795,7 +1568,7 @@ CoinSelector.prototype.selectEstimate = function selectEstimate(fee) {
     if (!this.isFull())
       this.fund();
   } while (!this.isFull() && this.index < this.coins.length);
-};
+});
 
 /**
  * Initiate selection based on a hard fee.

lib/primitives/tx.js

@@ -1719,17 +1719,6 @@ TX.prototype.checkInputs = function checkInputs(spendHeight, ret) {
   return true;
 };
 
-/**
- * Estimate the max possible size of transaction once the
- * inputs are scripted. If the transaction is non-mutable,
- * this will just return the virtual size.
- * @returns {Number} size
- */
-
-TX.prototype.maxSize = function maxSize() {
-  return this.getVirtualSize();
-};
-
 /**
  * Calculate the modified size of the transaction. This
  * is used in the mempool for calculating priority.
@@ -1742,7 +1731,7 @@ TX.prototype.getModifiedSize = function getModifiedSize(size) {
   var i, input, offset;
 
   if (size == null)
-    size = this.maxSize();
+    size = this.getVirtualSize();
 
   for (i = 0; i < this.inputs.length; i++) {
     input = this.inputs[i];
@@ -1773,7 +1762,7 @@ TX.prototype.getPriority = function getPriority(height, size) {
     return sum;
 
   if (size == null)
-    size = this.maxSize();
+    size = this.getVirtualSize();
 
   for (i = 0; i < this.inputs.length; i++) {
     input = this.inputs[i];
@@ -1853,7 +1842,7 @@ TX.prototype.isFree = function isFree(height, size) {
 
 TX.prototype.getMinFee = function getMinFee(size, rate) {
   if (size == null)
-    size = this.maxSize();
+    size = this.getVirtualSize();
 
   return btcutils.getMinFee(size, rate);
 };
@@ -1870,7 +1859,7 @@ TX.prototype.getMinFee = function getMinFee(size, rate) {
 
 TX.prototype.getRoundFee = function getRoundFee(size, rate) {
   if (size == null)
-    size = this.maxSize();
+    size = this.getVirtualSize();
 
   return btcutils.getRoundFee(size, rate);
 };
@@ -1884,7 +1873,7 @@ TX.prototype.getRoundFee = function getRoundFee(size, rate) {
 
 TX.prototype.getRate = function getRate(size) {
   if (size == null)
-    size = this.maxSize();
+    size = this.getVirtualSize();
 
   return btcutils.getRate(size, this.getFee());
 };
@@ -2065,7 +2054,7 @@ TX.prototype.inspect = function inspect() {
     hash: this.rhash(),
     witnessHash: this.rwhash(),
     size: this.getSize(),
-    virtualSize: this.maxSize(),
+    virtualSize: this.getVirtualSize(),
     height: this.height,
     value: Amount.btc(this.getOutputValue()),
     fee: Amount.btc(this.getFee()),

lib/wallet/wallet.js

@@ -12,6 +12,7 @@ var EventEmitter = require('events').EventEmitter;
 var constants = require('../protocol/constants');
 var Network = require('../protocol/network');
 var util = require('../utils/util');
+var encoding = require('../utils/encoding');
 var Locker = require('../utils/locker');
 var co = require('../utils/co');
 var crypto = require('../crypto/crypto');
@@ -23,6 +24,7 @@ var TXDB = require('./txdb');
 var Path = require('./path');
 var common = require('./common');
 var Address = require('../primitives/address');
+var Script = require('../script/script');
 var MTX = require('../primitives/mtx');
 var WalletKey = require('./walletkey');
 var HD = require('../hd/hd');
@@ -1430,24 +1432,114 @@ Wallet.prototype._fund = co(function* fund(tx, options) {
   // Don't use any locked coins.
   coins = this.txdb.filterLocked(coins);
 
-  tx.fund(coins, {
+  yield tx.fund(coins, {
     selection: options.selection,
     round: options.round,
     confirmations: options.confirmations,
-    free: options.free,
     hardFee: options.hardFee,
     subtractFee: options.subtractFee,
     changeAddress: account.change.getAddress(),
     height: this.db.state.height,
     rate: rate,
     maxFee: options.maxFee,
-    m: account.m,
+    estimate: this.estimate.bind(this)
-    n: account.n,
-    witness: account.witness,
-    script: account.receive.script
   });
 });
 
+/**
+ * Get account by address.
+ * @param {Address} address
+ * @returns {Account}
+ */
+
+Wallet.prototype.getAccountByAddress = co(function* getAccountByAddress(address) {
+  var hash = Address.getHash(address, 'hex');
+  var path, account;
+
+  if (!hash)
+    return;
+
+  path = yield this.getPath(hash);
+
+  if (!path)
+    return;
+
+  return yield this.getAccount(path.account);
+});
+
+/**
+ * Input size estimator for max possible tx size.
+ * @param {Script} prev
+ * @returns {Number}
+ */
+
+Wallet.prototype.estimate = co(function* estimate(prev) {
+  var scale = constants.WITNESS_SCALE_FACTOR;
+  var address = prev.getAddress();
+  var account = yield this.getAccountByAddress(address);
+  var size = 0;
+
+  if (!account)
+    return -1;
+
+  if (prev.isScripthash()) {
+    // Nested bullshit.
+    if (account.witness) {
+      switch (account.type) {
+        case Account.types.PUBKEYHASH:
+          size += 23; // redeem script
+          size *= 4; // vsize
+          break;
+        case Account.types.MULTISIG:
+          size += 35; // redeem script
+          size *= 4; // vsize
+          break;
+      }
+    }
+  }
+
+  switch (account.type) {
+    case Account.types.PUBKEYHASH:
+      // P2PKH
+      // OP_PUSHDATA0 [signature]
+      size += 1 + 73;
+      // OP_PUSHDATA0 [key]
+      size += 1 + 33;
+      break;
+    case Account.types.MULTISIG:
+      // P2SH Multisig
+      // OP_0
+      size += 1;
+      // OP_PUSHDATA0 [signature] ...
+      size += (1 + 73) * account.m;
+      // OP_PUSHDATA2 [redeem]
+      size += 3;
+      // m value
+      size += 1;
+      // OP_PUSHDATA0 [key] ...
+      size += (1 + 33) * account.n;
+      // n value
+      size += 1;
+      // OP_CHECKMULTISIG
+      size += 1;
+      break;
+  }
+
+  if (account.witness) {
+    // Varint witness items length.
+    size += 1;
+    // Calculate vsize if
+    // we're a witness program.
+    size = (size + scale - 1) / scale | 0;
+  } else {
+    // Byte for varint
+    // size of input script.
+    size += encoding.sizeVarint(size);
+  }
+
+  return size;
+});
+
 /**
  * Build a transaction, fill it with outputs and inputs,
  * sort the members according to BIP69, set locktime,

test/wallet-test.js

@@ -195,7 +195,7 @@ describe('Wallet', function() {
       .addInput(src, 0)
       .addOutput(w.getAddress(), 5460);
 
-    maxSize = tx.maxSize();
+    maxSize = yield tx.estimateSize();
 
     yield w.sign(tx);