From e205c70f97ce4acb7d78af9b019b282ec78df6e7 Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Fri, 25 Mar 2016 20:02:23 -0700
Subject: [PATCH] mempool. txdb.

---
 lib/bcoin/block.js           |  2 +
 lib/bcoin/chain.js           |  7 ++++
 lib/bcoin/fullnode.js        | 53 +++++++++++++++---------
 lib/bcoin/http/server.js     | 48 ++++++++++++++++++++--
 lib/bcoin/mempool.js         | 24 +++++------
 lib/bcoin/mtx.js             |  2 +-
 lib/bcoin/peer.js            |  2 +-
 lib/bcoin/pool.js            | 42 ++++++++++---------
 lib/bcoin/protocol/framer.js |  2 +-
 lib/bcoin/tx.js              |  2 +
 lib/bcoin/txdb.js            | 80 +++++++++++++++++++++++++++++++++---
 lib/bcoin/walletdb.js        |  5 ++-
 12 files changed, 201 insertions(+), 68 deletions(-)

diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js
index 5b701d3a..dcad272a 100644
--- a/lib/bcoin/block.js
+++ b/lib/bcoin/block.js
@@ -291,6 +291,8 @@ Block.prototype.inspect = function inspect() {
     type: this.type,
     height: this.height,
     hash: utils.revHex(this.hash('hex')),
+    size: this.getSize(),
+    virtualSize: this.getVirtualSize(),
     date: new Date(this.ts * 1000).toISOString(),
     version: this.version,
     prevBlock: utils.revHex(this.prevBlock),
diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js
index d40578b3..2b2a66ec 100644
--- a/lib/bcoin/chain.js
+++ b/lib/bcoin/chain.js
@@ -164,6 +164,9 @@ Chain.prototype._init = function _init() {
 
         self.loaded = true;
         self.emit('open');
+
+        if (self.isFull())
+          self.emit('full');
       });
     });
   });
@@ -1203,7 +1206,11 @@ Chain.prototype.add = function add(initial, peer, callback, force) {
       bcoin.profiler.snapshot();
 
     utils.nextTick(function() {
+      if (self.isFull())
+        self.emit('full');
+
       unlock();
+
       if (err)
         callback(err);
       else
diff --git a/lib/bcoin/fullnode.js b/lib/bcoin/fullnode.js
index 54605764..c042f554 100644
--- a/lib/bcoin/fullnode.js
+++ b/lib/bcoin/fullnode.js
@@ -38,13 +38,18 @@ Fullnode.prototype._init = function _init() {
   this.chain = new bcoin.chain(this, {
     preload: false,
     fsync: false,
+    spv: false,
     prune: this.options.prune,
     useCheckpoints: this.options.useCheckpoints
   });
 
   // Mempool needs access to blockdb.
   this.mempool = new bcoin.mempool(this, {
-    rbf: false
+    limitFree: this.options.limitFree,
+    limitFreeRelay: this.options.limitFreeRelay,
+    requireStandard: this.options.requireStandard,
+    rejectInsaneFees: this.options.rejectInsaneFees,
+    replaceByFee: this.options.replaceByFee
   });
 
   // Pool needs access to the chain.
@@ -92,12 +97,12 @@ Fullnode.prototype._init = function _init() {
     self.emit('error', err);
   });
 
-  // this.on('tx', function(tx) {
-  //   self.walletdb.addTX(tx, function(err) {
-  //     if (err)
-  //       self.emit('error', err);
-  //   });
-  // });
+  this.on('tx', function(tx) {
+    self.walletdb.addTX(tx, function(err) {
+      if (err)
+        self.emit('error', err);
+    });
+  });
 
   // Emit events for valid blocks and TXs.
   this.chain.on('block', function(block) {
@@ -112,14 +117,14 @@ Fullnode.prototype._init = function _init() {
   });
 
   // Update the mempool.
-  // this.chain.on('add block', function(block) {
-  //   self.mempool.addBlock(block);
-  // });
+  this.chain.on('add block', function(block) {
+    self.mempool.addBlock(block);
+  });
 
-  // this.chain.on('remove block', function(block) {
-  //   self.mempool.removeBlock(block);
-  //   self.walletdb.removeBlock(block);
-  // });
+  this.chain.on('remove block', function(block) {
+    self.mempool.removeBlock(block);
+    self.walletdb.removeBlock(block);
+  });
 
   function load(err) {
     if (err)
@@ -335,25 +340,33 @@ Fullnode.prototype.getTXByAddress = function getTXByAddress(addresses, callback)
 Fullnode.prototype.fillCoin = function fillCoin(tx, callback) {
   var self = this;
 
-  this.mempool.fillCoin(tx, function(err, filled) {
+  this.mempool.tx.isDoubleSpend(tx, function(err, result) {
     if (err)
       return callback(err);
 
-    if (filled)
-      return callback(null, tx);
+    if (result)
+      return callback(null, tx, true);
 
-    self.chain.db.fillCoin(tx, callback);
+    self.mempool.fillCoin(tx, function(err) {
+      if (err)
+        return callback(err);
+
+      if (tx.hasPrevout())
+        return callback(null, tx);
+
+      self.chain.db.fillCoin(tx, callback);
+    });
   });
 };
 
 Fullnode.prototype.fillTX = function fillTX(tx, callback) {
   var self = this;
 
-  this.mempool.fillTX(tx, function(err, filled) {
+  this.mempool.fillTX(tx, function(err) {
     if (err)
       return callback(err);
 
-    if (filled)
+    if (tx.hasPrevout())
       return callback(null, tx);
 
     self.chain.db.fillTX(tx, callback);
diff --git a/lib/bcoin/http/server.js b/lib/bcoin/http/server.js
index 8df0df85..26d9c7bf 100644
--- a/lib/bcoin/http/server.js
+++ b/lib/bcoin/http/server.js
@@ -85,8 +85,13 @@ NodeServer.prototype._init = function _init() {
     if (params.receiveDepth)
       options.receiveDepth = params.receiveDepth >>> 0;
 
-    if (params.address)
+    if (params.address) {
       params.addresses = params.address;
+      options.address = params.address;
+    }
+
+    if (params.value)
+      options.value = utils.satoshi(params.value);
 
     if (params.addresses) {
       if (typeof params.addresses === 'string')
@@ -95,6 +100,9 @@ NodeServer.prototype._init = function _init() {
         options.addresses = params.addresses;
     }
 
+    if (params.passphrase)
+      options.passphrase = params.passphrase;
+
     if (params.tx) {
       try {
         options.tx = bcoin.tx.fromRaw(params.tx, 'hex');
@@ -241,6 +249,35 @@ NodeServer.prototype._init = function _init() {
     });
   });
 
+  this.post('/wallet/:id/send', function(req, res, next, send) {
+    self.walletdb.get(req.options.id, req.options.passphrase, function(err, wallet) {
+      if (err)
+        return next(err);
+
+      if (!wallet)
+        return send(404);
+
+      wallet.createTX({
+        address: req.options.address,
+        value: req.options.value
+      }, function(err, tx) {
+        wallet.destroy();
+
+        if (err)
+          return next(err);
+
+        utils.print(tx);
+        return send(200, tx.toJSON());
+        self.pool.sendTX(tx, function(err) {
+          if (err)
+            return next(err);
+
+          send(200, tx.toJSON());
+        });
+      });
+    });
+  });
+
   // Update wallet / sync address depth
   this.put('/wallet/:id', function(req, res, next, send) {
     var id = req.options.id;
@@ -361,9 +398,12 @@ NodeServer.prototype._init = function _init() {
 
   // Broadcast TX
   this.post('/broadcast', function(req, res, next, send) {
-    var tx = req.options.tx;
-    self.pool.broadcast(tx);
-    send(200, { success: true });
+    self.pool.sendTX(tx, function(err) {
+      if (err)
+        return callback(err);
+
+      send(200, { success: true });
+    });
   });
 
   this.server.on('error', function(err) {
diff --git a/lib/bcoin/mempool.js b/lib/bcoin/mempool.js
index 14977c5e..00408e50 100644
--- a/lib/bcoin/mempool.js
+++ b/lib/bcoin/mempool.js
@@ -59,9 +59,9 @@ function Mempool(node, options) {
   this.freeCount = 0;
   this.lastTime = 0;
 
+  this.limitFree = this.options.limitFree !== false;
   this.limitFreeRelay = this.options.limitFreeRelay || 15;
   this.requireStandard = this.options.requireStandard !== false;
-  this.limitFree = this.options.limitFree !== false;
   this.rejectInsaneFees = this.options.rejectInsaneFees !== false;
 
   Mempool.global = this;
@@ -208,24 +208,18 @@ Mempool.prototype.addTX = function addTX(tx, peer, callback, force) {
     if (exists)
       return callback();
 
-    self.node.fillCoin(tx, function(err) {
+    self.node.fillCoin(tx, function(err, tx, doubleSpend) {
       if (err)
         return callback(err);
 
-      if (!tx.hasPrevout()) {
-        return self.tx.isDoubleSpend(tx, function(err, result) {
-          if (err)
-            return callback(err);
-
-          if (result) {
-            peer.sendReject(tx, 'bad-txns-inputs-spent', 0);
-            return callback(new VerifyError('bad-txns-inputs-spent', 0));
-          }
-
-          return self.storeOrphan(tx, callback);
-        });
+      if (doubleSpend) {
+        peer.sendReject(tx, 'bad-txns-inputs-spent', 0);
+        return callback(new VerifyError('bad-txns-inputs-spent', 0));
       }
 
+      if (!tx.hasPrevout())
+        return self.storeOrphan(tx, callback);
+
       self.verify(tx, function(err) {
         if (err) {
           if (err.type === 'VerifyError' && err.score >= 0)
@@ -247,6 +241,8 @@ Mempool.prototype.addUnchecked = function addUnchecked(tx, peer, callback) {
 
     self.emit('tx', tx);
 
+    utils.debug('Added tx %s to the mempool.', tx.rhash);
+
     self.resolveOrphans(tx, function(err, resolved) {
       if (err)
         return callback(err);
diff --git a/lib/bcoin/mtx.js b/lib/bcoin/mtx.js
index 4855d2be..25c5f1cb 100644
--- a/lib/bcoin/mtx.js
+++ b/lib/bcoin/mtx.js
@@ -26,7 +26,7 @@ function MTX(options) {
 
   this.options = options;
 
-  this.type = 'mtx';
+  this.type = 'tx';
   this.version = options.version || 1;
   this.inputs = [];
   this.outputs = [];
diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js
index 2dd78279..d6872b2c 100644
--- a/lib/bcoin/peer.js
+++ b/lib/bcoin/peer.js
@@ -228,7 +228,7 @@ Peer.prototype.broadcast = function broadcast(items) {
       clearInterval(old.interval);
     }
 
-    var inv =  this.framer.inv([{
+    var inv = this.framer.inv([{
       type: item.type,
       hash: item.hash()
     }]);
diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js
index 78aeb754..0af8c29d 100644
--- a/lib/bcoin/pool.js
+++ b/lib/bcoin/pool.js
@@ -213,17 +213,20 @@ Pool.prototype._init = function _init() {
     self.resolveOrphan(self.peers.load, null, data.hash);
   });
 
+  this.chain.on('full', function() {
+    self._stopTimer();
+    self._stopInterval();
+    if (!self.synced)
+      self.getMempool();
+    self.synced = true;
+    self.emit('full');
+    utils.debug('Chain is fully synced (height=%d).', self.chain.height);
+  });
+
   (this.options.wallets || []).forEach(function(wallet) {
     self.addWallet(wallet);
   });
 
-  // Chain is full and up-to-date
-  if (this.chain.isFull()) {
-    this.synced = true;
-    this.emit('full');
-    utils.debug('Chain is fully synced (height=%d).', this.chain.height);
-  }
-
   this.startServer();
 };
 
@@ -341,14 +344,8 @@ Pool.prototype._startTimer = function _startTimer() {
       return;
 
     // Chain is full and up-to-date
-    if (self.chain.isFull()) {
-      self._stopTimer();
-      self._stopInterval();
-      self.synced = true;
-      self.emit('full');
-      utils.debug('Chain is fully synced (height=%d).', self.chain.height);
+    if (self.chain.isFull())
       return;
-    }
 
     if (self.peers.load) {
       self.peers.load.destroy();
@@ -376,6 +373,10 @@ Pool.prototype._startInterval = function _startInterval() {
     if (!self.syncing)
       return;
 
+    // Chain is full and up-to-date
+    if (self.chain.isFull())
+      return;
+
     utils.debug('Stall recovery: loading again.');
     // self._load();
   }
@@ -889,7 +890,7 @@ Pool.prototype._handleTX = function _handleTX(tx, peer, callback) {
       return callback(err);
 
     addMempool(tx, peer, function(err) {
-      if (err && self.synced)
+      if (err)
         utils.debug('Mempool error: %s', err.message);
 
       if (updated || tx.block)
@@ -1649,11 +1650,9 @@ Pool.prototype.getTX = function getTX(hash, range, callback) {
 };
 
 Pool.prototype.sendTX = function sendTX(tx, callback) {
-  callback = utils.asyncify(callback);
-
   // Failsafe to avoid getting banned by bitcoind nodes.
   if (!bcoin.mempool.checkTX(tx))
-    return callback(new Error('CheckTransaction failed.'));
+    return utils.asyncify(callback)(new Error('CheckTransaction failed.'));
 
   return this.broadcast(tx, callback);
 };
@@ -1677,14 +1676,17 @@ Pool.prototype.broadcast = function broadcast(msg, callback) {
 
   this.inv.list.push(entry);
 
-  this.peers.regular.forEach(function(peer) {
+  this.peers.all.forEach(function(peer) {
     var result = peer.broadcast(msg);
     if (!result)
       return;
 
     result[0].once('request', function() {
       e.emit('ack', peer);
-      callback();
+      // Give them a chance to send a reject.
+      setTimeout(function() {
+        callback();
+      }, 100);
     });
 
     result[0].once('reject', function(payload) {
diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js
index 3a2b2900..26128e87 100644
--- a/lib/bcoin/protocol/framer.js
+++ b/lib/bcoin/protocol/framer.js
@@ -585,7 +585,7 @@ Framer.reject = function reject(details, writer) {
   p.writeU8(constants.reject[details.ccode] || constants.reject.malformed);
   p.writeVarString(details.reason || '', 'ascii');
   if (details.data)
-    p.writeBytes(details.data);
+    p.writeHash(details.data);
 
   if (!writer)
     p = p.render();
diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js
index 741b1249..9dd7be52 100644
--- a/lib/bcoin/tx.js
+++ b/lib/bcoin/tx.js
@@ -1083,6 +1083,8 @@ TX.prototype.inspect = function inspect() {
     type: this.type,
     hash: utils.revHex(this.hash('hex')),
     witnessHash: utils.revHex(this.witnessHash('hex')),
+    size: this.getSize(),
+    virtualSize: this.getVirtualSize(),
     height: this.height,
     value: utils.btc(this.getValue()),
     fee: utils.btc(this.getFee()),
diff --git a/lib/bcoin/txdb.js b/lib/bcoin/txdb.js
index b0d3b6a6..1f8d6559 100644
--- a/lib/bcoin/txdb.js
+++ b/lib/bcoin/txdb.js
@@ -371,12 +371,14 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
               prefix + 's/t/'
               + input.prevout.hash
               + '/' + input.prevout.index,
-              DUMMY);
+              tx.hash());
           }
 
           return next();
         }
 
+        input.output = null;
+
         // Only add orphans if this input is ours.
         if (self.options.mapAddress) {
           if (!address || !map[address].length)
@@ -385,16 +387,24 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
 
         self.isSpent(input.prevout.hash, input.prevout.index, function(err, result) {
           if (err)
-            return callback(err);
+            return next(err);
 
           // Are we double-spending?
-          if (result)
-            return callback(new Error('Transaction is double-spending.'));
+          if (result) {
+            if (!self.options.indexSpent)
+              return next(new Error('Transaction is double-spending.'));
+            return self._removeSpenders(hash, function(err) {
+              if (err)
+                return next(err);
+              batch.clear();
+              self._add(tx, map, callback, true);
+            });
+          }
 
           // Add orphan, if no parent transaction is yet known
           self._addOrphan(key, hash, i, function(err, orphans) {
             if (err)
-              return callback(err);
+              return next(err);
 
             batch.put(prefix + 'o/' + key, orphans);
 
@@ -515,6 +525,64 @@ TXPool.prototype._add = function add(tx, map, callback, force) {
   }, true);
 };
 
+TXPool.prototype._removeSpenders = function removeSpenders(hash, callback) {
+  var self = this;
+  this.getTX(hash, function(err, tx) {
+    if (err)
+      return callback(err);
+
+    if (!tx)
+      return callback(new Error('Could not find spender.'));
+
+    if (tx.ts !== 0)
+      return callback(new Error('Transaction is double-spending.'));
+
+    utils.forEachSerial(tx.outputs, function(next, output, i) {
+      self.isSpent(hash, i, function(err, spent) {
+        if (err)
+          return next(err);
+        if (spent)
+          return self._removeSpenders(spent, next);
+        next();
+      });
+    }, function(err) {
+      if (err)
+        return callback(err);
+      return self.lazyRemove(tx, callback, true);
+    });
+  });
+};
+
+TXPool.prototype._collectSpenders = function _collectSpenders(tx, callback, spenders) {
+  var self = this;
+  var hash = tx.hash('hex');
+
+  if (!spenders)
+    spenders = [];
+
+  utils.forEachSerial(tx.outputs, function(next, output, i) {
+    self.isSpent(hash, i, function(err, spentBy) {
+      if (err)
+        return next(err);
+      if (spentBy) {
+        spenders.push(spentBy);
+        return self.getTX(spentBy, function(err, tx) {
+          if (err)
+            return next(err);
+          if (!tx)
+            return next();
+          return self._removeSpenders(tx, next, spenders);
+        });
+      }
+      next();
+    });
+  }, function(err) {
+    if (err)
+      return callback(err);
+    return callback(null, spenders);
+  });
+};
+
 TXPool.prototype.isDoubleSpend = function isDoubleSpend(tx, callback) {
   var self = this;
   utils.everySerial(tx.inputs, function(input, next) {
@@ -541,7 +609,7 @@ TXPool.prototype.isSpent = function isSpent(hash, index, callback, checkCoin) {
       if (err && err.type !== 'NotFoundError')
         return callback(err);
 
-      return callback(null, !!exists);
+      return callback(null, exists ? exists.toString('hex') : null);
     });
   }
 
diff --git a/lib/bcoin/walletdb.js b/lib/bcoin/walletdb.js
index b170ff83..3bde6879 100644
--- a/lib/bcoin/walletdb.js
+++ b/lib/bcoin/walletdb.js
@@ -98,7 +98,7 @@ WalletDB.prototype._init = function _init() {
   });
 
   this.tx = new bcoin.txdb('w', this.db, {
-    indexSpent: false,
+    indexSpent: true,
     indexExtra: true,
     indexAddress: true,
     mapAddress: true,
@@ -438,6 +438,9 @@ WalletDB.prototype.create = function create(options, callback) {
       }
       done();
     } else {
+      if (bcoin.protocol.network.type === 'segnet')
+        options.witness = options.witness !== false;
+
       options.provider = new Provider(self);
       wallet = new bcoin.wallet(options);
       self.saveJSON(wallet.id, wallet.toJSON(), done);