From 5053fa2eeb1620dd4eb55153c4df3b0ce793605f Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Thu, 18 Feb 2016 21:01:34 -0800
Subject: [PATCH] move height lookups to chaindb.

---
 lib/bcoin/chain.js   | 178 ++++------------------------
 lib/bcoin/chaindb.js | 270 ++++++++++++++++++++++++++++---------------
 2 files changed, 199 insertions(+), 249 deletions(-)

diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js
index a531ef23..b030ae12 100644
--- a/lib/bcoin/chain.js
+++ b/lib/bcoin/chain.js
@@ -35,7 +35,6 @@ function Chain(options) {
     bcoin.debug = this.options.debug;
 
   this.db = new bcoin.chaindb(this);
-  this.heightLookup = {};
   this.request = new utils.RequestCache();
   this.loading = false;
   this.tip = null;
@@ -145,6 +144,11 @@ Chain.prototype._init = function _init() {
       self.mempool.removeBlock(block);
   });
 
+  this.db.on('tip', function(tip) {
+    self.tip = tip;
+    self.height = tip.height;
+  });
+
   this.loading = true;
 
   utils.debug('Chain is loading.');
@@ -154,56 +158,23 @@ Chain.prototype._init = function _init() {
       throw err;
 
     self._preload(function(err, start) {
-      var count = self.db.count();
-      var i = start || 1;
-      var lastEntry;
-
       if (err) {
         utils.debug('Preloading chain failed.');
         utils.debug('Reason: %s', err.message);
       }
 
-      utils.debug('Starting chain load at height: %s', i);
-
-      function done(height) {
-        if (height != null) {
-          utils.debug(
-            'Blockchain is corrupt after height %d. Resetting.',
-            height);
-          self.resetHeight(height);
-        } else {
-          utils.debug('Chain successfully loaded.');
-        }
+      self.db.load(start || 0, function(err) {
+        if (err)
+          throw err;
 
         self.syncHeight(function(err) {
           if (err)
             throw err;
+
           self.loading = false;
           self.emit('load');
         });
-      }
-
-      (function next() {
-        if (i >= count)
-          return done();
-
-        self.db.getAsync(i, function(err, entry) {
-          if (err)
-            throw err;
-
-          // Do some paranoid checks.
-          if (lastEntry && entry.prevBlock !== lastEntry.hash)
-            return done(Math.max(0, i - 2));
-
-          if (i % 10000 === 0)
-            utils.debug('Loaded %d blocks.', i);
-
-          lastEntry = entry;
-          self._saveEntry(entry);
-          i += 1;
-          next();
-        });
-      })();
+      });
     });
   });
 };
@@ -213,7 +184,7 @@ Chain.prototype._ensureGenesis = function _ensureGenesis(callback) {
 
   callback = utils.asyncify(callback);
 
-  this._saveEntry(bcoin.chainblock.fromJSON(this, {
+  this.db.save(bcoin.chainblock.fromJSON(this, {
     hash: network.genesis.hash,
     version: network.genesis.version,
     prevBlock: network.genesis.prevBlock,
@@ -222,7 +193,7 @@ Chain.prototype._ensureGenesis = function _ensureGenesis(callback) {
     bits: network.genesis.bits,
     nonce: network.genesis.nonce,
     height: 0
-  }), true);
+  }));
 
   if (!this.blockdb)
     return callback();
@@ -348,11 +319,7 @@ Chain.prototype._preload = function _preload(callback) {
         return;
       }
 
-      // Don't write blocks we already have
-      // (bad for calculating chainwork).
-      // self._saveEntry(entry, height > chainHeight);
-
-      self._saveEntry(entry, true);
+      self.db.save(entry);
 
       height++;
 
@@ -686,8 +653,8 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
   callback = utils.asyncify(callback);
 
   // Already added
-  if (this.heightLookup[entry.hash] != null) {
-    assert(this.heightLookup[entry.hash] === entry.height);
+  if (this.db.has(entry.height)) {
+    assert(this.db.getHeight(entry.hash) === entry.height);
     return callback(null, false);
   }
 
@@ -700,7 +667,7 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
     if (err)
       return callback(err);
 
-    self._saveEntry(entry, true, function(err) {
+    self.db.save(entry, function(err) {
       if (err)
         return callback(err);
 
@@ -709,36 +676,6 @@ Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
   });
 };
 
-Chain.prototype._saveEntry = function _saveEntry(entry, save, callback) {
-  this.heightLookup[entry.hash] = entry.height;
-
-  if (!this.tip || entry.height > this.tip.height) {
-    this.tip = entry;
-    this.height = this.tip.height;
-    this.emit('tip', this.tip);
-  }
-
-  if (save)
-    this.db.save(entry, callback);
-};
-
-Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
-  var heights = Object.keys(network.checkpoints).sort();
-  var index = heights.indexOf(height) - 1;
-  var checkpoint = network.checkpoint[index];
-
-  assert(index >= 0);
-  assert(checkpoint);
-
-  // This is the safest way to do it, the other
-  // possibility is to simply reset ignore the
-  // bad checkpoint block. The likelihood of
-  // someone carrying on an entire fork between
-  // to checkpoints is absurd, so this is
-  // probably _a lot_ of work for nothing.
-  this.resetHeight(checkpoint.height);
-};
-
 Chain.prototype.resetHeight = function resetHeight(height) {
   var self = this;
   var count = this.db.count();
@@ -750,15 +687,7 @@ Chain.prototype.resetHeight = function resetHeight(height) {
   if (height === count - 1)
     return;
 
-  for (i = height + 1; i < count; i++) {
-    existing = this.db.get(i);
-    assert(existing);
-    // this.db.remove(i);
-    this.db.drop(i);
-    delete this.heightLookup[existing.hash];
-  }
-
-  this.db.truncate(height);
+  this.db.resetHeight(height);
 
   // Reset the orphan map completely. There may
   // have been some orphans on a forked chain we
@@ -768,11 +697,6 @@ Chain.prototype.resetHeight = function resetHeight(height) {
   this.orphan.bmap = {};
   this.orphan.count = 0;
   this.orphan.size = 0;
-
-  this.tip = this.db.get(height);
-  assert(this.tip);
-  this.height = this.tip.height;
-  this.emit('tip', this.tip);
 };
 
 Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
@@ -789,26 +713,7 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
   lock = this.locked;
   this.locked = true;
 
-  i = height + 1;
-
-  function next() {
-    if (i === count)
-      return self.db.truncateAsync(height, done);
-
-    self.db.getAsync(i, function(err, existing) {
-      if (err)
-        return done(err);
-
-      assert(existing);
-
-      delete self.heightLookup[existing.hash];
-      self.db.drop(i);
-      i++;
-      next();
-    });
-  }
-
-  function done(err) {
+  this.db.resetHeightAsync(height, function(err) {
     self.locked = lock;
 
     if (err)
@@ -823,13 +728,8 @@ Chain.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
     self.orphan.count = 0;
     self.orphan.size = 0;
 
-    self.tip = self.db.get(height);
-    assert(self.tip);
-    self.height = self.tip.height;
-    self.emit('tip', self.tip);
-
     return callback();
-  }
+  });
 };
 
 Chain.prototype.revertHeight = function revertHeight(height, callback) {
@@ -1008,7 +908,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
     }
 
     // Find the previous block height/index.
-    prevHeight = self.heightLookup[prevHash];
+    prevHeight = self.db.getHeight(prevHash);
 
     // Validate the block we want to add.
     // This is only necessary for new
@@ -1140,7 +1040,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
       // don't store by hash so we can't compare
       // chainworks. We reset the chain, find a
       // new peer, and wait to see who wins.
-      assert(self.heightLookup[entry.hash] == null);
+      assert(self.db.getHeight(entry.hash) == null);
 
       // The tip has more chainwork, it is a
       // higher height than the entry. This is
@@ -1168,7 +1068,7 @@ Chain.prototype.add = function add(initial, peer, callback) {
     }
 
     // Add entry if we do not have it.
-    assert(self.heightLookup[entry.hash] == null);
+    assert(self.db.getHeight(entry.hash) == null);
 
     // Lookup previous entry.
     prev = self.db.get(prevHeight);
@@ -1314,7 +1214,7 @@ Chain.prototype.byHash = function byHash(hash) {
   else if (hash.hash)
     hash = hash.hash('hex');
 
-  return this.byHeight(this.heightLookup[hash]);
+  return this.byHeight(this.db.getHeight(hash));
 };
 
 Chain.prototype.byTime = function byTime(ts) {
@@ -1425,7 +1325,7 @@ Chain.prototype.getLocator = function getLocator(start) {
   }
 
   if (typeof start === 'string') {
-    top = this.heightLookup[start];
+    top = this.db.getHeight(start);
     if (top == null) {
       // We could simply `return [start]` here,
       // but there is no standardized "spacing"
@@ -1474,7 +1374,7 @@ Chain.prototype.getLocatorAsync = function getLocatorAsync(start, callback) {
   }
 
   if (typeof start === 'string') {
-    top = this.heightLookup[start];
+    top = this.db.getHeight(start);
     if (top == null) {
       // We could simply `return [start]` here,
       // but there is no standardized "spacing"
@@ -1668,34 +1568,6 @@ Chain.prototype.retarget = function retarget(last, first) {
   return utils.toCompact(target);
 };
 
-Chain.prototype.toJSON = function toJSON() {
-  var entries = [];
-  var count = this.db.count();
-  var i;
-
-  for (i = 0; i < count; i++)
-    entries.push(this.db.get(i));
-
-  return {
-    v: 2,
-    type: 'chain',
-    network: network.type,
-    entries: entries.map(function(entry) {
-      return entry.toJSON();
-    })
-  };
-};
-
-Chain.prototype.fromJSON = function fromJSON(json) {
-  assert.equal(json.v, 2);
-  assert.equal(json.type, 'chain');
-  assert.equal(json.network, network.type);
-
-  json.entries.forEach(function(entry) {
-    this._saveEntry(bcoin.chainblock.fromJSON(this, entry));
-  }, this);
-};
-
 /**
  * Expose
  */
diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js
index de227a15..3585ccfd 100644
--- a/lib/bcoin/chaindb.js
+++ b/lib/bcoin/chaindb.js
@@ -28,6 +28,8 @@ function ChainDB(chain, options) {
   if (!options)
     options = {};
 
+  EventEmitter.call(this);
+
   this.options = options;
   this.chain = chain;
   this.file = options.file;
@@ -35,12 +37,13 @@ function ChainDB(chain, options) {
   if (!this.file)
     this.file = process.env.HOME + '/bcoin-chain-' + network.type + '.db';
 
-  this._queue = [];
+  this.heightLookup = {};
+  this._queue = {};
   this._cache = {};
   this._bufferPool = { used: {} };
-  this._nullBlock = new Buffer(BLOCK_SIZE);
-  this._nullBlock.fill(0);
-  this.tip = -1;
+  this.highest = -1;
+  this.tip = null;
+  this.height = -1;
   this.size = 0;
   this.fd = null;
 
@@ -55,6 +58,8 @@ function ChainDB(chain, options) {
   this._init();
 }
 
+inherits(ChainDB, EventEmitter);
+
 ChainDB.prototype._init = function _init() {
   if (!bcoin.fs) {
     utils.debug('`fs` module not available. Falling back to ramdisk.');
@@ -85,6 +90,48 @@ ChainDB.prototype._init = function _init() {
   this.fd = fs.openSync(this.file, 'r+');
 };
 
+ChainDB.prototype.load = function load(start, callback) {
+  var self = this;
+  var count = this.count();
+  var i = start || 0;
+  var lastEntry;
+
+  utils.debug('Starting chain load at height: %s', i);
+
+  function done(height) {
+    if (height != null) {
+      utils.debug(
+        'Blockchain is corrupt after height %d. Resetting.',
+        height);
+      self.resetHeight(height);
+    } else {
+      utils.debug('Chain successfully loaded.');
+    }
+    callback();
+  }
+
+  (function next() {
+    if (i >= count)
+      return done();
+
+    self.getAsync(i, function(err, entry) {
+      if (err)
+        return callback(err);
+
+      // Do some paranoid checks.
+      if (lastEntry && entry.prevBlock !== lastEntry.hash)
+        return done(Math.max(0, i - 2));
+
+      if (i % 10000 === 0)
+        utils.debug('Loaded %d blocks.', i);
+
+      lastEntry = entry;
+      i += 1;
+      next();
+    });
+  })();
+};
+
 ChainDB.prototype.closeSync = function closeSync() {
   if (!bcoin.fs) {
     this.ramdisk = null;
@@ -167,14 +214,28 @@ ChainDB.prototype.count = function count() {
 };
 
 ChainDB.prototype.cache = function cache(entry) {
-  if (entry.height > this.tip) {
-    this.tip = entry.height;
+  if (entry.height > this.highest) {
+    this.highest = entry.height;
     delete this._cache[entry.height - this._cacheWindow];
     this._cache[entry.height] = entry;
     assert(Object.keys(this._cache).length <= this._cacheWindow);
   }
 };
 
+ChainDB.prototype.getHeight = function getHeight(hash) {
+  return this.heightLookup[hash];
+};
+
+ChainDB.prototype._populate = function _populate(entry) {
+  this.heightLookup[entry.hash] = entry.height;
+
+  if (!this.tip || entry.height > this.tip.height) {
+    this.tip = entry;
+    this.height = this.tip.height;
+    this.emit('tip', this.tip);
+  }
+};
+
 ChainDB.prototype.get = function get(height) {
   return this.getSync(height);
 };
@@ -182,6 +243,9 @@ ChainDB.prototype.get = function get(height) {
 ChainDB.prototype.getSync = function getSync(height) {
   var data, entry;
 
+  if (typeof height === 'string')
+    height = this.heightLookup[height];
+
   if (this._cache[height])
     return this._cache[height];
 
@@ -199,12 +263,10 @@ ChainDB.prototype.getSync = function getSync(height) {
   if (!data)
     return;
 
-  // Ignore if it is a null block.
-  if (utils.read32(data, 0) === 0)
-    return;
-
   entry = bcoin.chainblock.fromRaw(this.chain, height, data);
 
+  this._populate(entry);
+
   // Cache the past 1001 blocks in memory
   // (necessary for isSuperMajority)
   this.cache(entry);
@@ -217,6 +279,9 @@ ChainDB.prototype.getAsync = function getAsync(height, callback) {
 
   callback = utils.asyncify(callback);
 
+  if (typeof height === 'string')
+    height = this.heightLookup[height];
+
   if (this._cache[height])
     return callback(null, this._cache[height]);
 
@@ -241,12 +306,10 @@ ChainDB.prototype.getAsync = function getAsync(height, callback) {
     if (!data)
       return callback();
 
-    // Ignore if it is a null block.
-    if (utils.read32(data, 0) === 0)
-      return callback();
-
     entry = bcoin.chainblock.fromRaw(self.chain, height, data);
 
+    self._populate(entry);
+
     // Cache the past 1001 blocks in memory
     // (necessary for isSuperMajority)
     self.cache(entry);
@@ -260,13 +323,14 @@ ChainDB.prototype.save = function save(entry, callback) {
 };
 
 ChainDB.prototype.saveSync = function saveSync(entry) {
-  var self = this;
   var raw, offset;
 
   // Cache the past 1001 blocks in memory
   // (necessary for isSuperMajority)
   this.cache(entry);
 
+  this._populate(entry);
+
   raw = entry.toRaw();
   offset = entry.height * BLOCK_SIZE;
 
@@ -283,6 +347,8 @@ ChainDB.prototype.saveAsync = function saveAsync(entry, callback) {
   // (necessary for isSuperMajority)
   this.cache(entry);
 
+  this._populate(entry);
+
   // Something is already writing. Cancel it
   // and synchronously write the data after
   // it cancels.
@@ -336,119 +402,131 @@ ChainDB.prototype.drop = function drop(height) {
   delete this._cache[height];
 };
 
-ChainDB.prototype.remove = function remove(height) {
+ChainDB.prototype.resetHeight = function resetHeight(height) {
+  if (typeof height === 'string')
+    height = this.heightLookup[height];
+
   assert(height >= 0);
 
-  // Drop the queue and cache
-  this.drop(height);
-
-  // Write a null block
-  this._writeSync(this._nullBlock, height * BLOCK_SIZE);
-
-  // If we deleted several blocks at the end, go back
-  // to the last non-null block and truncate the file
-  // beyond that point.
-  if ((height + 1) * BLOCK_SIZE === this.size) {
-    while (this.isNull(height))
-      height--;
-
-    assert(height >= 0);
-
-    this.truncate(height);
-  }
-
-  return true;
-};
-
-ChainDB.prototype.truncate = function truncate(height) {
   var size = (height + 1) * BLOCK_SIZE;
+  var count = this.count();
+  if (height === count - 1)
+    return;
+  assert(height <= count - 1);
+  assert(this.tip);
+
+  for (i = height + 1; i < count; i++) {
+    existing = this.get(i);
+    assert(existing);
+    this.drop(i);
+    delete this.heightLookup[existing.hash];
+  }
 
   if (!bcoin.fs) {
     this.ramdisk.truncate(size);
+
     this.size = size;
-    this.tip = height;
+    this.highest = height;
+    this.tip = this.get(height);
+    assert(this.tip);
+    this.height = this.tip.height;
+    this.emit('tip', this.tip);
     return;
   }
 
   fs.ftruncateSync(this.fd, size);
+
   this.size = size;
-  this.tip = height;
+  this.highest = height;
+  this.tip = this.get(height);
+  assert(this.tip);
+  this.height = this.tip.height;
+  this.emit('tip', this.tip);
 };
 
-ChainDB.prototype.truncateAsync = function truncateAsync(height) {
+ChainDB.prototype.resetHeightAsync = function resetHeightAsync(height, callback) {
   var self = this;
-  var size = (height + 1) * BLOCK_SIZE;
+  var called;
+
+  if (typeof height === 'string')
+    height = this.heightLookup[height];
+
+  assert(height >= 0);
 
   callback = utils.asyncify(callback);
 
-  if (!bcoin.fs) {
-    this.ramdisk.truncate(size);
-    this.size = size;
-    this.tip = height;
+  var size = (height + 1) * BLOCK_SIZE;
+  var count = this.count() - 1;
+  if (height === count - 1)
     return callback();
+  assert(height <= count - 1);
+  assert(this.tip);
+
+  var pending = count - (height + 1);
+
+  for (i = height + 1; i < count; i++)
+    dropEntry(i);
+
+  function dropEntry(i) {
+    self.getAsync(i, function(err, existing) {
+      if (err)
+        return done(err);
+
+      assert(existing);
+      self.drop(i);
+      delete self.heightLookup[existing.hash];
+      if (!--pending)
+        done();
+    });
   }
 
-  fs.ftruncate(this.fd, size, function(err) {
+  function done(err) {
+    if (called)
+      return;
+
+    called = true;
+
     if (err)
       return callback(err);
 
-    self.size = size;
-    self.tip = height;
+    if (!bcoin.fs) {
+      self.ramdisk.truncate(size);
+      self.size = size;
+      self.highest = height;
+      self.tip = self.get(height);
+      assert(self.tip);
+      self.height = self.tip.height;
+      self.emit('tip', self.tip);
+      return callback();
+    }
 
-    return callback();
-  });
-};
+    fs.ftruncate(self.fd, size, function(err) {
+      if (err)
+        return callback(err);
 
-ChainDB.prototype.isNull = function isNull(height) {
-  var data = this._readSync(4, height * BLOCK_SIZE);
-  if (!data)
-    return false;
-  return utils.read32(data, 0) === 0;
+      self.size = size;
+      self.highest = height;
+      self.tip = self.get(height);
+      assert(self.tip);
+      self.height = self.tip.height;
+      self.emit('tip', self.tip);
+
+      return callback();
+    });
+  }
 };
 
 ChainDB.prototype.has = function has(height) {
-  var data;
+  if (typeof height === 'string')
+    height = this.heightLookup[height];
 
-  if (this._queue[height] || this._cache[height])
+  if (height < 0 || height == null)
+    return false;
+
+  if ((height + 1) * BLOCK_SIZE <= this.size)
     return true;
 
-  if (height < 0 || height == null)
-    return false;
-
-  if ((height + 1) * BLOCK_SIZE > this.size)
-    return false;
-
-  data = this._readSync(4, height * BLOCK_SIZE);
-
-  if (!data)
-    return false;
-
-  return utils.read32(data, 0) !== 0;
-};
-
-ChainDB.prototype.hasAsync = function hasAsync(height, callback) {
-  var data;
-
-  callback = utils.asyncify(callback);
-
-  if (this._queue[height] || this._cache[height])
-    return callback(null, true);
-
-  if (height < 0 || height == null)
-    return callback(null, false);
-
-  if ((height + 1) * BLOCK_SIZE > this.size)
-    return callback(null, false);
-
-  this._readAsync(4, height * BLOCK_SIZE, function(err, data) {
-    if (err)
-      return callback(err);
-
-    if (!data)
-      return callback(null, false);
-
-    return callback(null, utils.read32(data, 0) !== 0);
-  });
+  return false;
 };
 
 ChainDB.prototype._readSync = function _readSync(size, offset) {