From 845a987e00217e054aa1ffe50b2c47215163d3e8 Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Fri, 29 Apr 2016 01:41:51 -0700
Subject: [PATCH] undo coins.

---
 lib/bcoin/block.js   |  65 ---------
 lib/bcoin/chain.js   |   2 +-
 lib/bcoin/chaindb.js | 336 ++++++++++++++++---------------------------
 lib/bcoin/tx.js      |   2 +-
 4 files changed, 129 insertions(+), 276 deletions(-)

diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js
index 68e639d1..ba0add07 100644
--- a/lib/bcoin/block.js
+++ b/lib/bcoin/block.js
@@ -635,71 +635,6 @@ Block.fromRaw = function fromRaw(data, enc, type) {
   return new Block(Block.parseRaw(data, enc));
 };
 
-/**
- * Serialize a block to BCoin "compact format".
- * This is the serialization format BCoin uses internally
- * to store blocks in the database. It includes the height,
- * but does not include transaction data, only transaction
- * hashes.
- * @returns {Buffer}
- */
-
-Block.prototype.toCompact = function toCompact() {
-  var p = new BufferWriter();
-  var height = this.height;
-
-  if (height === -1)
-    height = 0x7fffffff;
-
-  p.writeBytes(this.abbr());
-  p.writeU32(height);
-  p.writeVarint(this.txs.length);
-
-  this.txs.forEach(function(tx) {
-    p.writeHash(tx.hash());
-  });
-
-  return p.render();
-};
-
-/**
- * Parse a transaction in "compact" serialization format.
- * @param {Buffer} buf
- * @returns {NakedBlock} - A "naked" block object with a `hashes` vector.
- */
-
-Block.parseCompact = function parseCompact(buf) {
-  var p = new BufferReader(buf);
-  var hashes = [];
-  var version = p.readU32(); // Technically signed
-  var prevBlock = p.readHash('hex');
-  var merkleRoot = p.readHash('hex');
-  var ts = p.readU32();
-  var bits = p.readU32();
-  var nonce = p.readU32();
-  var height = p.readU32();
-  var txCount = p.readVarint();
-  var i;
-
-  for (i = 0; i < txCount; i++)
-    hashes.push(p.readHash('hex'));
-
-  if (height === 0x7fffffff)
-    height = -1;
-
-  return {
-    version: version,
-    prevBlock: prevBlock,
-    merkleRoot: merkleRoot,
-    ts: ts,
-    bits: bits,
-    nonce: nonce,
-    height: height,
-    totalTX: txCount,
-    hashes: hashes
-  };
-};
-
 /**
  * Convert the Block to a MerkleBlock.
  * @param {Bloom} filter - Bloom filter for transactions
diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js
index 3dd56179..675800e9 100644
--- a/lib/bcoin/chain.js
+++ b/lib/bcoin/chain.js
@@ -1938,7 +1938,7 @@ Chain.prototype.getLocator = function getLocator(start, callback, force) {
         step *= 2;
     }
 
-    utils.forEach(hashes, function(height, next, i) {
+    utils.forEachSerial(hashes, function(height, next, i) {
       if (typeof height === 'string')
         return next();
 
diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js
index 2cfeba79..1536ab15 100644
--- a/lib/bcoin/chaindb.js
+++ b/lib/bcoin/chaindb.js
@@ -13,6 +13,10 @@ var utils = require('./utils');
 var assert = utils.assert;
 var pad32 = utils.pad32;
 var DUMMY = new Buffer([0]);
+var BufferWriter = require('./writer');
+var BufferReader = require('./reader');
+var Framer = bcoin.protocol.framer;
+var Parser = bcoin.protocol.parser;
 
 /**
  * The database backend for the {@link Chain} object.
@@ -716,17 +720,12 @@ ChainDB.prototype.has = function has(height, callback) {
  */
 
 ChainDB.prototype.saveBlock = function saveBlock(block, batch, connect, callback) {
-  var i, tx;
+  var i, j, tx, hash, addresses, address;
 
   if (this.options.spv)
     return utils.nextTick(callback);
 
-  batch.put('b/b/' + block.hash('hex'), block.toCompact());
-
-  for (i = 0; i < block.txs.length; i++) {
-    tx = block.txs[i];
-    batch.put('t/t/' + tx.hash('hex'), tx.toExtended());
-  }
+  batch.put('b/b/' + block.hash('hex'), block.render());
 
   if (!connect)
     return utils.nextTick(callback);
@@ -758,11 +757,6 @@ ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) {
 
     batch.del('b/b/' + block.hash('hex'));
 
-    for (i = 0; i < block.txs.length; i++) {
-      tx = block.txs[i];
-      batch.del('t/t/' + tx.hash('hex'));
-    }
-
     self.disconnectBlock(block, batch, callback);
   });
 };
@@ -776,7 +770,8 @@ ChainDB.prototype.removeBlock = function removeBlock(hash, batch, callback) {
 
 ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
   var self = this;
-  var i, j, tx, input, output, key, address, hash, uniq, coin;
+  var undo = new BufferWriter();
+  var i, j, tx, input, output, key, addresses, address, hash, coin;
 
   if (this.options.spv) {
     self.emit('add block', block);
@@ -797,7 +792,17 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
     for (i = 0; i < block.txs.length; i++) {
       tx = block.txs[i];
       hash = tx.hash('hex');
-      uniq = {};
+
+      if (self.options.indexTX) {
+        batch.put('t/t/' + hash, tx.toExtended());
+        if (self.options.indexAddress) {
+          addresses = tx.getAddresses();
+          for (j = 0; j < addresses.length; j++) {
+            address = addresses[j];
+            batch.put('t/a/' + address + '/' + hash, DUMMY);
+          }
+        }
+      }
 
       for (j = 0; j < tx.inputs.length; j++) {
         input = tx.inputs[j];
@@ -810,17 +815,14 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
 
         if (self.options.indexAddress) {
           address = input.getAddress();
-
-          if (address && !uniq[address] && !self.prune) {
-            uniq[address] = true;
-            batch.put('t/a/' + address + '/' + hash, DUMMY);
-          }
-
           if (address)
             batch.del('u/a/' + address + '/' + key);
         }
 
         batch.del('u/t/' + key);
+
+        Framer.coin(input.coin, false, undo);
+
         self.coinCache.remove(key);
       }
 
@@ -831,21 +833,19 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
 
         if (self.options.indexAddress) {
           address = output.getAddress();
-
-          if (address && !uniq[address] && !self.prune) {
-            uniq[address] = true;
-            batch.put('t/a/' + address + '/' + hash, DUMMY);
-          }
-
           if (address)
             batch.put('u/a/' + address + '/' + key, DUMMY);
         }
 
         batch.put('u/t/' + key, coin.toRaw());
+
         self.coinCache.set(key, coin);
       }
     }
 
+    if (undo.written > 0)
+      batch.put('b/u/' + block.hash('hex'), undo.render());
+
     self.emit('add block', block);
 
     self._pruneBlock(block, batch, function(err) {
@@ -865,7 +865,7 @@ ChainDB.prototype.connectBlock = function connectBlock(block, batch, callback) {
 
 ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callback) {
   var self = this;
-  var i, j, tx, input, output, key, address, hash, uniq;
+  var i, j, tx, input, output, key, addresses, address, hash;
 
   if (this.options.spv)
     return utils.nextTick(callback);
@@ -880,7 +880,17 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb
     for (i = block.txs.length - 1; i >= 0; i--) {
       tx = block.txs[i];
       hash = tx.hash('hex');
-      uniq = {};
+
+      if (self.options.indexTX) {
+        batch.del('t/t/' + hash);
+        if (self.options.indexAddress) {
+          addresses = tx.getAddresses();
+          for (j = 0; j < addresses.length; j++) {
+            address = addresses[j];
+            batch.del('t/a/' + address + '/' + hash);
+          }
+        }
+      }
 
       for (j = 0; j < tx.inputs.length; j++) {
         input = tx.inputs[j];
@@ -893,17 +903,12 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb
 
         if (self.options.indexAddress) {
           address = input.getAddress();
-
-          if (address && !uniq[address] && !self.prune) {
-            uniq[address] = true;
-            batch.del('t/a/' + address + '/' + hash);
-          }
-
           if (address)
             batch.put('u/a/' + address + '/' + key, DUMMY);
         }
 
         batch.put('u/t/' + key, input.coin.toRaw());
+
         self.coinCache.set(key, input.coin);
       }
 
@@ -913,21 +918,18 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb
 
         if (self.options.indexAddress) {
           address = output.getAddress();
-
-          if (address && !uniq[address] && !self.prune) {
-            uniq[address] = true;
-            batch.del('t/a/' + address + '/' + hash);
-          }
-
           if (address)
             batch.del('u/a/' + address + '/' + key);
         }
 
         batch.del('u/t/' + key);
+
         self.coinCache.remove(key);
       }
     }
 
+    batch.del('b/u/' + block.hash('hex'));
+
     self.emit('remove block', block);
 
     return callback(null, block);
@@ -998,27 +1000,6 @@ ChainDB.prototype.fillHistory = function fillHistory(tx, callback) {
   if (tx.isCoinbase())
     return utils.asyncify(callback)(null, tx);
 
-  if (this.prune) {
-    return utils.forEachSerial(tx.inputs, function(input, next) {
-      if (input.coin)
-        return next();
-
-      self._getPruneCoin(input.prevout.hash, input.prevout.index, function(err, coin) {
-        if (err)
-          return callback(err);
-
-        if (coin)
-          input.coin = coin;
-
-        next();
-      });
-    }, function(err) {
-      if (err)
-        return callback(err);
-      return callback(null, tx);
-    });
-  }
-
   utils.forEachSerial(tx.inputs, function(input, next) {
     if (input.coin)
       return next();
@@ -1055,7 +1036,7 @@ ChainDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, call
 
   addresses = utils.uniqs(addresses);
 
-  utils.forEach(addresses, function(address, done) {
+  utils.forEachSerial(addresses, function(address, done) {
     var iter = self.db.iterator({
       gte: 'u/a/' + address + '/',
       lte: 'u/a/' + address + '/~',
@@ -1092,7 +1073,7 @@ ChainDB.prototype.getCoinsByAddress = function getCoinsByAddress(addresses, call
     if (err)
       return callback(err);
 
-    utils.forEach(ids, function(item, next) {
+    utils.forEachSerial(ids, function(item, next) {
       var hash = item[0];
       var index = item[1];
       self.getCoin(hash, index, function(err, coin) {
@@ -1167,7 +1148,7 @@ ChainDB.prototype.getTXByAddress = function getTXByAddress(addresses, callback)
 
   addresses = utils.uniqs(addresses);
 
-  utils.forEach(addresses, function(address, done) {
+  utils.forEachSerial(addresses, function(address, done) {
     var iter = self.db.iterator({
       gte: 't/a/' + address + '/',
       lte: 't/a/' + address + '/~',
@@ -1209,7 +1190,7 @@ ChainDB.prototype.getTXByAddress = function getTXByAddress(addresses, callback)
     if (err)
       return callback(err);
 
-    utils.forEach(hashes, function(hash, next) {
+    utils.forEachSerial(hashes, function(hash, next) {
       self.getTX(hash, function(err, tx) {
         if (err)
           return next(err);
@@ -1240,11 +1221,11 @@ ChainDB.prototype.getTX = function getTX(hash, callback) {
   var tx;
 
   this.db.get(key, function(err, data) {
-    if (err) {
-      if (err.type === 'NotFoundError')
-        return callback();
+    if (err && err.type !== 'NotFoundError')
       return callback(err);
-    }
+
+    if (!data)
+      return callback();
 
     try {
       tx = bcoin.tx.fromExtended(data);
@@ -1353,7 +1334,7 @@ ChainDB.prototype._ensureHistory = function _ensureHistory(hash, callback) {
     if (!block)
       return callback();
 
-    return self.fillHistoryBlock(block, callback);
+    return self.fillUndoBlock(block, callback);
   });
 };
 
@@ -1429,6 +1410,70 @@ ChainDB.prototype.fillHistoryBlock = function fillHistoryBlock(block, callback)
   });
 };
 
+/**
+ * Get coins necessary to be resurrected during a reorg.
+ * @param {Hash} hash
+ * @param {Function} callback - Returns [Error, Object].
+ */
+
+ChainDB.prototype.getUndoCoins = function getUndoCoins(hash, callback) {
+  var coins, p, coin, i, tx;
+
+  return this.db.get('b/u/' + hash, function(err, data) {
+    if (err && err.type !== 'NotFoundError')
+      return callback(err);
+
+    if (!data)
+      return callback();
+
+    coins = [];
+    p = new BufferReader(data);
+
+    p.start();
+
+    while (p.left()) {
+      coin = new bcoin.coin(Parser.parseCoin(p, false));
+      coins.push(coin);
+    }
+
+    p.end();
+
+    return callback(null, coins);
+  });
+};
+
+/**
+ * Fill a block with coins necessary to be resurrected during a reorg.
+ * @param {Block} block
+ * @param {Function} callback - Returns [Error, {@link Block}].
+ */
+
+ChainDB.prototype.fillUndoBlock = function fillUndoBlock(block, callback) {
+  var i, j, k, tx, input;
+
+  return this.getUndoCoins(block.hash('hex'), function(err, coins) {
+    if (err)
+      return callback(err);
+
+    if (!coins)
+      return callback(null, block);
+
+    for (i = 0, k = 0; i < block.txs.length; i++) {
+      tx = block.txs[i];
+
+      if (tx.isCoinbase())
+        continue;
+
+      for (j = 0; j < tx.inputs.length; j++) {
+        input = tx.inputs[j];
+        input.coin = coins[k++];
+      }
+    }
+
+    return callback(null, block);
+  });
+};
+
 /**
  * Retrieve a block from the database (not filled with coins).
  * @param {Hash} hash
@@ -1439,7 +1484,7 @@ ChainDB.prototype.getBlock = function getBlock(hash, callback) {
   var self = this;
   var key, block;
 
-  return this.getHash(hash, function(err, hash) {
+  return this.getBoth(hash, function(err, hash, height) {
     if (err)
       return callback(err);
 
@@ -1456,47 +1501,17 @@ ChainDB.prototype.getBlock = function getBlock(hash, callback) {
         return callback();
 
       try {
-        block = bcoin.block.parseCompact(data);
+        block = bcoin.block.fromRaw(data);
+        block.setHeight(height);
       } catch (e) {
         return callback(e);
       }
 
-      block.txs = [];
-
-      utils.forEach(block.hashes, function(hash, next, i) {
-        self.getTX(hash, function(err, tx) {
-          if (err)
-            return next(err);
-
-          if (!tx)
-            return next(new Error('TX not found.'));
-
-          block.txs[i] = tx;
-
-          next();
-        });
-      }, function(err) {
-        if (err)
-          return callback(err);
-
-        delete block.hashes;
-        block = new bcoin.block(block);
-
-        if (self.options.paranoid)
-          assert(block.hash('hex') === hash, 'Database is corrupt.');
-
-        return callback(null, block);
-      });
+      return callback(null, block);
     });
   });
 };
 
-ChainDB.prototype._getTX = function _getTX(hash, callback) {
-  if (hash instanceof bcoin.tx)
-    return callback(null, hash);
-  return this.getTX(hash);
-};
-
 /**
  * Check whether a transaction is unspent (i.e. not yet _fully_ spent).
  * @see https://bitcointalk.org/index.php?topic=67738.0
@@ -1524,10 +1539,12 @@ ChainDB.prototype.isUnspentTX = function isUnspentTX(hash, callback) {
  */
 
 ChainDB.prototype.isSpentTX = function isSpentTX(hash, callback) {
+  var iter;
+
   if (hash.hash)
     hash = hash.hash('hex');
 
-  var iter = this.db.iterator({
+  iter = this.db.iterator({
     gte: 'u/t/' + hash,
     lte: 'u/t/' + hash + '~',
     keys: true,
@@ -1567,30 +1584,7 @@ ChainDB.prototype._pruneBlock = function _pruneBlock(block, batch, callback) {
 
   batch.put('b/q/' + futureHeight, block.hash());
 
-  for (i = 0; i < block.txs.length; i++) {
-    tx = block.txs[i];
-
-    for (j = 0; j < tx.inputs.length; j++) {
-      input = tx.inputs[j];
-      key = input.prevout.hash + '/' + input.prevout.index;
-
-      if (tx.isCoinbase())
-        break;
-
-      assert(input.coin);
-
-      batch.put('u/x/' + key, input.coin.toRaw());
-      batch.put('u/q/' + futureHeight + '/' + key, DUMMY);
-    }
-  }
-
-  this._pruneQueue(block, batch, callback);
-};
-
-ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) {
-  var self = this;
-  var key = 'b/q/' + pad32(block.height);
-  var i;
+  key = 'b/q/' + pad32(block.height);
 
   this.db.get(key, function(err, hash) {
     if (err && err.type !== 'NotFoundError')
@@ -1601,87 +1595,11 @@ ChainDB.prototype._pruneQueue = function _pruneQueue(block, batch, callback) {
 
     hash = hash.toString('hex');
 
-    self.db.get('b/b/' + hash, function(err, compact) {
-      if (err && err.type !== 'NotFoundError')
-        return callback(err);
+    batch.del(key);
+    batch.del('b/b/' + hash);
+    batch.del('b/u/' + hash);
 
-      batch.del(key);
-
-      if (!compact)
-        return callback();
-
-      try {
-        compact = bcoin.block.parseCompact(compact);
-      } catch (e) {
-        return callback(e);
-      }
-
-      batch.del('b/b/' + hash);
-
-      for (i = 0; i < compact.hashes.length; i++)
-        batch.del('t/t/' + compact.hashes[i]);
-
-      self._pruneCoinQueue(block, batch, callback);
-    });
-  });
-};
-
-ChainDB.prototype._pruneCoinQueue = function _pruneQueue(block, batch, callback) {
-  var iter = this.db.iterator({
-    gte: 'u/q/' + pad32(block.height),
-    lte: 'u/q/' + pad32(block.height) + '~',
-    keys: true,
-    values: false,
-    fillCache: false,
-    keyAsBuffer: false
-  });
-
-  (function next() {
-    iter.next(function(err, key, value) {
-      var parts, hash, index;
-
-      if (err) {
-        return iter.end(function() {
-          callback(err);
-        });
-      }
-
-      if (key === undefined)
-        return iter.end(callback);
-
-      parts = key.split('/');
-      hash = parts[3];
-      index = +parts[4];
-
-      batch.del(key);
-      batch.del('u/x/' + hash + '/' + index);
-
-      next();
-    });
-  })();
-};
-
-ChainDB.prototype._getPruneCoin = function _getPruneCoin(hash, index, callback) {
-  var self = this;
-  var key = 'u/x/' + hash + '/' + index;
-  var coin;
-
-  this.db.get(key, function(err, data) {
-    if (err && err.type !== 'NotFoundError')
-      return callback(err);
-
-    if (!data)
-      return self.getCoin(hash, index, callback);
-
-    try {
-      coin = bcoin.coin.fromRaw(data);
-      coin.hash = hash;
-      coin.index = index;
-    } catch (e) {
-      return callback(e);
-    }
-
-    return callback(null, coin);
+    return callback();
   });
 };
 
diff --git a/lib/bcoin/tx.js b/lib/bcoin/tx.js
index acbb0401..a16b3d31 100644
--- a/lib/bcoin/tx.js
+++ b/lib/bcoin/tx.js
@@ -838,7 +838,7 @@ TX.prototype.fillCoins = function fillCoins(coins) {
   var total = 0;
   var inputs, txs, key, i, input;
 
-  if (!Array.isArray(coins))
+  if ((coins instanceof bcoin.coin) || (coins instanceof bcoin.tx))
     coins = [coins];
 
   if (Array.isArray(coins)) {