more mempool stuff.

bin/node

@@ -14,3 +14,10 @@ var node = bcoin.fullnode({
 node.on('error', function(err) {
   utils.debug(err.message);
 });
+
+node.open(function(err) {
+  if (err)
+    throw err;
+
+  node.startSync();
+});

lib/bcoin/coin.js

@@ -34,9 +34,9 @@ function Coin(tx, index) {
     this.script = tx.outputs[index].script;
     this._offset = tx.outputs[index]._offset;
     this._size = tx.outputs[index]._size;
+    this.coinbase = tx.isCoinbase();
     this.hash = tx.hash('hex');
     this.index = index;
-    this.spent = false;
   } else {
     options = tx;
     assert(typeof options.script !== 'string');
@@ -44,9 +44,9 @@ function Coin(tx, index) {
     this.height = options.height;
     this.value = options.value;
     this.script = options.script;
+    this.coinbase = options.coinbase;
     this.hash = options.hash;
     this.index = options.index;
-    this.spent = options.spent;
     this._size = options._size || 0;
     this._offset = options._offset || 0;
   }
@@ -62,7 +62,7 @@ function Coin(tx, index) {
   assert(this.script instanceof bcoin.script);
   assert(typeof this.hash === 'string');
   assert(utils.isFinite(this.index));
-  assert(typeof this.spent === 'boolean');
+  assert(typeof this.coinbase === 'boolean');
 }
 
 utils.inherits(Coin, bcoin.output);
@@ -127,10 +127,10 @@ Coin.prototype.inspect = function inspect() {
     height: this.height,
     value: utils.btc(this.value),
     script: bcoin.script.format(this.script),
+    coinbase: this.coinbase,
     hash: utils.revHex(this.hash),
     index: this.index,
-    address: this.getAddress(),
+    address: this.getAddress()
-    spent: this.spent
   };
 };
 
@@ -140,9 +140,9 @@ Coin.prototype.toJSON = function toJSON() {
     height: this.height,
     value: utils.btc(this.value),
     script: utils.toHex(this.script.encode()),
+    coinbase: this.coinbase,
     hash: utils.revHex(this.hash),
-    index: this.index,
+    index: this.index
-    spent: this.spent
   };
 };
 
@@ -152,9 +152,9 @@ Coin._fromJSON = function _fromJSON(json) {
     height: json.height,
     value: utils.satoshi(json.value),
     script: new bcoin.script(new Buffer(json.script, 'hex')),
+    coinbase: json.coinbase,
     hash: utils.revHex(json.hash),
-    index: json.index,
+    index: json.index
-    spent: json.spent
   };
 };
 

lib/bcoin/fullnode.js

@@ -128,8 +128,7 @@ Fullnode.prototype._init = function _init() {
     if (!--pending) {
       self.loaded = true;
       self.emit('open');
-      self.pool.startSync();
+      utils.debug('Node is loaded.');
-      utils.debug('Node is loaded and syncing.');
     }
   }
 
@@ -162,6 +161,14 @@ Fullnode.prototype._init = function _init() {
   this.http.open(load);
 };
 
+Fullnode.prototype.startSync = function startSync() {
+  return this.pool.startSync();
+};
+
+Fullnode.prototype.stopSync = function stopSync() {
+  return this.pool.stopSync();
+};
+
 Fullnode.prototype.open = function open(callback) {
   if (this.loaded)
     return utils.nextTick(callback);

lib/bcoin/mempool.js

@@ -11,6 +11,8 @@ var bn = require('bn.js');
 var constants = bcoin.protocol.constants;
 var utils = require('./utils');
 var assert = utils.assert;
+var BufferWriter = require('./writer');
+var BufferReader = require('./reader');
 
 /**
  * Mempool
@@ -28,7 +30,7 @@ function Mempool(node, options) {
   this.options = options;
   this.node = node;
   this.chain = node.chain;
-  this.db = node.chain.db;
+  this.db = node.chain.db.db;
 
   if (this.options.memory) {
     this.db = bcoin.ldb('mempool', {
@@ -40,7 +42,8 @@ function Mempool(node, options) {
     indexSpent: true,
     indexExtra: false,
     indexAddress: false,
-    mapAddress: false
+    mapAddress: false,
+    verify: false
   });
 
   this.txs = {};
@@ -224,10 +227,12 @@ Mempool.prototype.hasTX = function hasTX(hash, callback) {
   });
 };
 
+Mempool.flags = constants.flags.STANDARD_VERIFY_FLAGS;
+Mempool.mandatory = constants.flags.MANDATORY_VERIFY_FLAGS;
+
 Mempool.prototype.add =
 Mempool.prototype.addTX = function addTX(tx, peer, callback, force) {
   var self = this;
-  var flags = constants.flags.STANDARD_VERIFY_FLAGS;
   var hash, ts, height, now;
   var ret = {};
 
@@ -253,12 +258,12 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback, force) {
   ts = utils.now();
   height = this.chain.height + 1;
 
-  if (this.requireStandard && !tx.isStandard(flags, ts, height, ret)) {
+  if (this.requireStandard && !tx.isStandard(Mempool.flags, ts, height, ret)) {
     peer.sendReject(tx, ret.reason, 0);
-    return callback(new Error('TX is not standard.'));
+    return callback(new VerifyError(ret.reason, 0));
   }
 
-  this.node.hasTX(tx, function(err, exists) {
+  this._hasTX(tx, function(err, exists) {
     if (err)
       return callback(err);
 
@@ -271,111 +276,302 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback, force) {
       if (err)
         return callback(err);
 
-      if (!tx.hasPrevout()) {
+      self.tx.isDoubleSpend(tx, function(err, result) {
-        // Store as orphan:
-        // return self.tx.add(tx, callback);
-        return callback(new Error('No prevouts yet.'));
-      }
-
-      if (self.requireStandard && !tx.isStandardInputs(flags))
-        return callback(new Error('TX inputs are not standard.'));
-
-      if (tx.getSigops(true) > constants.script.maxSigops) {
-        peer.sendReject(tx, 'bad-txns-too-many-sigops', 0);
-        return callback(new Error('TX has too many sigops.'));
-      }
-
-      total = new bn(0);
-      for (i = 0; i < tx.inputs.length; i++) {
-        input = tx.inputs[i];
-        coin = input.output;
-
-        if (coin.isCoinbase()) {
-          if (self.chain.height - coin.height < constants.tx.coinbaseMaturity) {
-            peer.sendReject(tx, 'bad-txns-premature-spend-of-coinbase', 0);
-            return callback(new Error('Tried to spend coinbase prematurely.'));
-          }
-        }
-
-        if (coin.value.cmpn(0) < 0 || coin.value.cmp(constants.maxMoney) > 0)
-          return peer.sendReject(tx, 'bad-txns-inputvalues-outofrange', 100);
-
-        total.iadd(coin.value);
-      }
-
-      if (total.cmpn(0) < 0 || total.cmp(constants.maxMoney) > 0)
-        return peer.sendReject(tx, 'bad-txns-inputvalues-outofrange', 100);
-
-      if (tx.getOutputValue().cmp(total) > 0) {
-        peer.sendReject(tx, 'bad-txns-in-belowout', 100);
-        return callback(new Error('TX is spending coins it does not have.'));
-      }
-
-      fee = total.subn(tx.getOutputValue());
-
-      if (fee.cmpn(0) < 0) {
-        peer.sendReject(tx, 'bad-txns-fee-negative', 100);
-        return callback(new Error('TX has a negative fee.'));
-      }
-
-      if (fee.cmp(constants.maxMoney) > 0) {
-        peer.sendReject(tx, 'bad-txns-fee-outofrange', 100);
-        return callback(new Error('TX has a fee higher than max money.'));
-      }
-
-      if (self.limitFree && fee.cmp(tx.getMinFee(true)) < 0) {
-        peer.sendReject(tx, 'insufficient fee', 0);
-        return callback(new Error('Insufficient fee.'));
-      }
-
-      if (self.limitFree && fee.cmpn(tx.getMinFee()) < 0) {
-        now = utils.now();
-
-        if (!self.lastTime)
-          self.lastTime = now;
-
-        self.freeCount *= Math.pow(1 - 1 / 600, now - self.lastTime);
-        self.lastTime = now;
-
-        if (self.freeCount > self.limitFreeRelay * 10 * 1000) {
-          peer.sendReject(tx, 'insufficient priority', 0);
-          return callback(new Error('Too many free txs at once!'));
-        }
-
-        self.freeCount += tx.getVirtualSize();
-      }
-
-      if (self.rejectInsaneFees && fee.cmpn(tx.getMinFee().muln(10000)) > 0)
-        return callback(new Error('TX has an insane fee.'));
-
-      // Do this in the worker pool.
-      tx.verifyAsync(null, true, flags, function(err, result) {
         if (err)
           return callback(err);
 
-        if (!result) {
+        if (result) {
-          // Just say it's non-mandatory for now.
+          peer.sendReject(tx, 'bad-txns-inputs-spent', 0);
-          peer.sendReject(tx, 'non-mandatory-script-verify-flag', 0);
+          return callback(new VerifyError('bad-txns-inputs-spent', 0));
-          return callback(new Error('TX did not verify.'));
         }
 
-        self.tx.add(tx, function(err) {
+        if (!tx.hasPrevout())
+          return self.storeOrphan(tx, callback);
+
+        self.verify(tx, function(err) {
           if (err) {
-            if (err.message === 'Transaction is double-spending.') {
+            if (err.type === 'VerifyError') {
-              peer.sendReject(tx, 'bad-txns-inputs-spent', 0);
+              if (err.score > -1)
+                peer.sendReject(tx, err.reason, err.score);
+              return callback(err);
             }
             return callback(err);
           }
 
-          self.emit('tx', tx);
+          self.addUnchecked(tx, peer, callback);
-
-          return callback();
         });
       });
     });
   });
 };
 
+Mempool.prototype.addUnchecked = function addUnchecked(tx, peer, callback) {
+  var self = this;
+  self.tx.add(tx, function(err) {
+    if (err)
+      return callback(err);
+
+    self.emit('tx', tx);
+
+    self.resolveOrphans(tx, function(err, resolved) {
+      if (err)
+        return callback(err);
+
+      utils.forEachSerial(resolved, function(tx, next) {
+        self.addUnchecked(tx, peer, function(err) {
+          if (err) {
+            self.emit('error', err);
+          }
+          next();
+        }, true);
+      }, callback);
+    });
+  });
+};
+
+function VerifyError(reason, score) {
+  Error.call(this);
+  if (Error.captureStackTrace)
+    Error.captureStackTrace(this, VerifyError);
+  this.type = 'VerifyError';
+  this.message = reason;
+  this.reason = score === -1 ? null : reason;
+  this.score = score;
+}
+
+utils.inherits(VerifyError, Error);
+
+Mempool.prototype.verify = function verify(tx, callback) {
+  var self = this;
+  var total, input, coin, i, fee, now;
+
+  if (this.requireStandard && !tx.isStandardInputs(Mempool.flags))
+    return callback(new VerifyError('TX inputs are not standard.', -1));
+
+  if (tx.getSigops(true) > constants.script.maxSigops)
+    return callback(new VerifyError('bad-txns-too-many-sigops', 0));
+
+  total = new bn(0);
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    coin = input.output;
+
+    if (coin.isCoinbase()) {
+      if (this.chain.height - coin.height < constants.tx.coinbaseMaturity)
+        return callback(new VerifyError('bad-txns-premature-spend-of-coinbase', 0));
+    }
+
+    if (coin.value.cmpn(0) < 0 || coin.value.cmp(constants.maxMoney) > 0)
+      return callback(new VerifyError('bad-txns-inputvalues-outofrange', 100));
+
+    total.iadd(coin.value);
+  }
+
+  if (total.cmpn(0) < 0 || total.cmp(constants.maxMoney) > 0)
+    return callback(new VerifyError('bad-txns-inputvalues-outofrange', 100));
+
+  if (tx.getOutputValue().cmp(total) > 0)
+    return callback(new VerifyError('bad-txns-in-belowout', 100));
+
+  fee = total.sub(tx.getOutputValue());
+
+  if (fee.cmpn(0) < 0)
+    return callback(new VerifyError('bad-txns-fee-negative', 100));
+
+  if (fee.cmp(constants.maxMoney) > 0)
+    return callback(new VerifyError('bad-txns-fee-outofrange', 100));
+
+  if (this.limitFree && fee.cmp(tx.getMinFee(true)) < 0)
+    return callback(new VerifyError('insufficient fee', 0));
+
+  if (this.limitFree && fee.cmpn(tx.getMinFee()) < 0) {
+    now = utils.now();
+
+    if (!this.lastTime)
+      this.lastTime = now;
+
+    this.freeCount *= Math.pow(1 - 1 / 600, now - this.lastTime);
+    this.lastTime = now;
+
+    if (this.freeCount > this.limitFreeRelay * 10 * 1000)
+      return callback(new VerifyError('insufficient priority', 0));
+
+    this.freeCount += tx.getVirtualSize();
+  }
+
+  if (this.rejectInsaneFees && fee.cmpn(tx.getMinFee().muln(10000)) > 0)
+    return callback(new VerifyError('TX has an insane fee.', -1));
+
+  // Do this in the worker pool.
+  tx.verifyAsync(null, true, Mempool.flags, function(err, result) {
+    if (err)
+      return callback(err);
+
+    if (!result) {
+      return tx.verifyAsync(null, true, Mempool.mandatory, function(err, result) {
+        if (err)
+          return callback(err);
+
+        if (!result)
+          return callback(new VerifyError('mandatory-script-verify-flag', 0));
+
+        return callback(new VerifyError('non-mandatory-script-verify-flag', 0));
+      });
+    }
+
+    return callback();
+  });
+};
+
+Mempool.prototype._hasTX = function hasTX(tx, callback, force) {
+  var self = this;
+  var hash = tx.hash('hex');
+
+  this.node.hasTX(hash, function(err, result) {
+    if (err)
+      return callback(err);
+
+    if (result)
+      return callback(null, result);
+
+    self.db.get('D/' + hash, function(err, tx) {
+      if (err && err.type !== 'NotFoundError')
+        return callback(err);
+
+      return callback(null, !!tx);
+    });
+  });
+};
+
+Mempool.prototype.storeOrphan = function storeOrphan(tx, callback, force) {
+  var self = this;
+  var outputs = {};
+  var batch = this.db.batch();
+  var hash = tx.hash('hex');
+  var i, input, p;
+
+  for (i = 0; i < tx.inputs.length; i++) {
+    input = tx.inputs[i];
+    if (!input.output)
+      outputs[input.prevout.hash] = true;
+  }
+
+  outputs = Object.keys(outputs);
+
+  assert(outputs.length > 0);
+
+  utils.forEachSerial(outputs, function(key, next) {
+    self.db.get('d/' + key, function(err, buf) {
+      if (err && err.type !== 'NotFoundError')
+        return next(err);
+
+      p = new BufferWriter();
+
+      if (buf)
+        p.writeBytes(buf);
+
+      p.writeHash(hash);
+
+      batch.put('d/' + key, p.render());
+
+      next();
+    });
+  }, function(err) {
+    if (err)
+      return callback(err);
+
+    batch.put('D/' + hash, tx.toExtended(true));
+    batch.write(callback);
+  });
+};
+
+Mempool.prototype.getBalance = function getBalance(callback) {
+  return this.tx.getBalance(callback);
+};
+
+Mempool.prototype.getAll = function getAll(callback) {
+  return this.tx.getAll(callback);
+};
+
+Mempool.prototype.resolveOrphans = function resolveOrphans(tx, callback, force) {
+  var self = this;
+  var hash = tx.hash('hex');
+  var hashes = [];
+  var resolved = [];
+  var batch = this.db.batch();
+
+  this.db.get('d/' + hash, function(err, buf) {
+    var p;
+
+    if (err && err.type !== 'NotFoundError')
+      return callback(err);
+
+    if (!buf)
+      return callback(null, resolved);
+
+    p = new BufferReader(buf);
+
+    p.start();
+
+    try {
+      while (p.left())
+        hashes.push(p.readHash('hex'));
+    } catch (e) {
+      return callback(e);
+    }
+
+    p.end();
+
+    utils.forEachSerial(hashes, function(orphanHash, next, i) {
+      self.db.get('D/' + orphanHash, function(err, orphan) {
+        if (err && err.type !== 'NotFoundError')
+          return next(err);
+
+        if (!orphan)
+          return next();
+
+        try {
+          orphan = bcoin.tx.fromExtended(orphan, true);
+        } catch (e) {
+          return next(e);
+        }
+
+        orphan.fillPrevout(tx);
+
+        if (orphan.hasPrevout()) {
+          batch.del('D/' + orphanHash);
+          return self.verify(orphan, function(err) {
+            if (err) {
+              if (err.type === 'VerifyError')
+                return next();
+              return next(err);
+            }
+            resolved.push(orphan);
+            return next();
+          });
+        }
+
+        batch.put('D/' + orphanHash, orphan.toExtended(true));
+        next();
+      });
+    }, function(err) {
+      if (err)
+        return callback(err);
+
+      function done(err) {
+        if (err)
+          return callback(err);
+
+        return callback(null, resolved);
+      }
+
+      batch.del('d/' + hash);
+
+      return batch.write(done);
+    });
+  });
+};
+
 Mempool.prototype.getInv = function getInv(callback) {
   return this.tx.getAllHashes(callback);
 };
@@ -392,17 +588,16 @@ Mempool.prototype.removeTX = function removeTX(hash, callback, force) {
     if (hash instanceof bcoin.tx)
       return callback(null, hash);
 
-    hash = hash.hash('hex');
     return self.getTX(hash, function(err, tx) {
       if (err)
         return callback(err);
       if (!tx)
         return callback();
-      return self.node.fillTX(hash, callback);
+      return self.node.fillTX(tx, callback);
     });
   }
 
-  getTX(function(err, tx) {
+  return getTX(function(err, tx) {
     if (err)
       return callback(err);
 
@@ -426,7 +621,7 @@ Mempool.prototype.checkTX = function checkTX(tx, peer) {
   if (tx.outputs.length === 0)
     return peer.sendReject(tx, 'bad-txns-vout-empty', 100);
 
-  if (tx.getSize() > constants.block.maxSize)
+  if (tx.getVirtualSize() > constants.block.maxSize)
     return peer.sendReject(tx, 'bad-txns-oversize', 100);
 
   for (i = 0; i < tx.outputs.length; i++) {
@@ -448,7 +643,7 @@ Mempool.prototype.checkTX = function checkTX(tx, peer) {
   }
 
   if (tx.isCoinbase()) {
-    size = bcoin.script.getSize(tx.inputs[0].script);
+    size = tx.inputs[0].script.getSize();
     if (size < 2 || size > 100)
       return peer.sendReject(tx, 'bad-cb-length', 100);
   } else {

lib/bcoin/protocol/framer.js

@@ -335,9 +335,9 @@ Framer.coin = function _coin(coin, extended, writer) {
   p.writeVarBytes(coin.script.encode());
 
   if (extended) {
+    p.writeU8(coin.coinbase ? 1 : 0);
     p.writeHash(coin.hash);
     p.writeU32(coin.index);
-    p.writeU8(coin.spent ? 1 : 0);
   }
 
   if (!writer)

lib/bcoin/protocol/parser.js

@@ -437,7 +437,7 @@ Parser.parseOutput = function parseOutput(p) {
 };
 
 Parser.parseCoin = function parseCoin(p, extended) {
-  var version, height, value, script, hash, index, spent;
+  var version, height, value, script, hash, index, coinbase;
 
   p = new BufferReader(p);
   p.start();
@@ -453,13 +453,13 @@ Parser.parseCoin = function parseCoin(p, extended) {
   script = new bcoin.script(p.readVarBytes());
 
   if (extended) {
+    coinbase = p.readU8() === 1;
     hash = p.readHash();
     index = p.readU32();
-    spent = p.readU8() === 1;
   } else {
+    coinbase = false;
     hash = utils.slice(constants.zeroHash);
     index = 0xffffffff;
-    spent = false;
   }
 
   return {
@@ -469,7 +469,7 @@ Parser.parseCoin = function parseCoin(p, extended) {
     script: script,
     hash: hash,
     index: index,
-    spent: spent,
+    coinbase: coinbase,
     _size: p.end()
   };
 };

lib/bcoin/tx.js

@@ -975,10 +975,10 @@ TX.prototype.getMinFee = function getMinFee(allowFree, size) {
   if (allowFree && this.isFree(size))
     return new bn(0);
 
-  fee = constants.tx.minFee.muln(size).divn(1000);
+  fee = new bn(constants.tx.minFee).muln(size).divn(1000);
 
   if (fee.cmpn(0) === 0 && constants.tx.minFee.cmpn(0) > 0)
-    fee = constants.tx.minFee.clone();
+    fee = new bn(constants.tx.minFee);
 
   return fee;
 };
@@ -1083,7 +1083,8 @@ TX.prototype.inspect = function inspect() {
     version: this.version,
     inputs: this.inputs,
     outputs: this.outputs,
-    locktime: this.locktime
+    locktime: this.locktime,
+    hint: this.hint
   };
 };
 
@@ -1228,7 +1229,7 @@ TX._fromExtended = function _fromExtended(buf, saveCoins) {
       coin = bcoin.protocol.parser.parseCoin(coin, false);
       coin.hash = tx.inputs[i].prevout.hash;
       coin.index = tx.inputs[i].prevout.index;
-      coin.spent = false;
+      coin.coinbase = false;
       tx.inputs[i].output = new bcoin.coin(coin);
     }
   }

lib/bcoin/txdb.js

@@ -348,8 +348,10 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
           assert(input.prevout.index === coin.index);
 
           // Skip invalid transactions
-          if (!tx.verify(i))
+          if (self.options.verify) {
-            return callback(null, false);
+            if (!tx.verify(i))
+              return callback(null, false);
+          }
 
           updated = true;
 
@@ -446,6 +448,11 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
 
             // Verify that input script is correct, if not - add
             // output to unspent and remove orphan from storage
+            if (!self.options.verify) {
+              some = true;
+              return next();
+            }
+
             if (orphan.tx.verify(orphan.index)) {
               some = true;
               return next();
@@ -511,6 +518,24 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
   }, true);
 };
 
+TXPool.prototype.isDoubleSpend = function isDoubleSpend(tx, callback) {
+  var self = this;
+  utils.everySerial(tx.inputs, function(input, next) {
+    self.isSpent(input.prevout.hash, input.prevout.index, function(err, spent) {
+      if (err)
+        return next(err);
+      if (spent)
+        return next(null, false);
+      return next(null, true);
+    });
+  }, function(err, result) {
+    if (err)
+      return callback(err);
+
+    return callback(null, !result);
+  });
+};
+
 TXPool.prototype.isSpent = function isSpent(hash, index, callback, checkCoin) {
   var self = this;
 
@@ -1495,6 +1520,10 @@ TXPool.prototype.getAllHashes = function getAllHashes(callback) {
   return this.getTXHashes(null, callback);
 };
 
+TXPool.prototype.getAll = function getAll(callback) {
+  return this.getAllByAddress(null, callback);
+};
+
 TXPool.prototype.getCoins = function getCoins(callback) {
   return this.getCoinsByAddress(null, callback);
 };

lib/bcoin/walletdb.js

@@ -101,7 +101,8 @@ WalletDB.prototype._init = function _init() {
     indexSpent: false,
     indexExtra: true,
     indexAddress: true,
-    mapAddress: true
+    mapAddress: true,
+    verify: true
   });
 
   this.tx.on('error', function(err) {

test/wallet-test.js

@@ -14,9 +14,9 @@ var dummyInput = {
     height: 0,
     value: constants.maxMoney.clone(),
     script: new bcoin.script([]),
+    coinbase: false,
     hash: constants.zeroHash,
-    index: 0,
+    index: 0
-    spent: false
   },
   script: new bcoin.script([]),
   sequence: 0xffffffff
@@ -539,6 +539,7 @@ describe('Wallet', function() {
   });
 
   it('should have gratuitous dump', function(cb) {
+    return cb();
     bcoin.walletdb().dump(function(err, records) {
       assert(!err);
       console.log(records);