blockchain db.

lib/bcoin/block.js

@@ -101,7 +101,7 @@ Block.prototype.abbr = function abbr() {
     return this._raw.slice();
 
   var res = new Array(80);
-  utils.writeU32(res, this.version, 0);
+  utils.write32(res, this.version, 0);
   utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4);
   utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36);
   utils.writeU32(res, this.ts, 68);

lib/bcoin/chain.js

@@ -9,6 +9,7 @@ var EventEmitter = require('events').EventEmitter;
 
 var bcoin = require('../bcoin');
 var bn = require('bn.js');
+var fs = require('fs');
 var constants = bcoin.protocol.constants;
 var network = bcoin.protocol.network;
 var utils = bcoin.utils;
@@ -28,6 +29,7 @@ function Chain(options) {
   this.prefix = 'bt/chain/';
   this.storage = this.options.storage;
   this.strict = this.options.strict || false;
+  this.db = new ChainDB(this);
 
   if (this.options.debug)
     bcoin.debug = this.options.debug;
@@ -42,12 +44,8 @@ function Chain(options) {
   };
 
   this.index = {
-    entries: [],
-    // Get hash by height
-    hashes: [],
-    // Get height by hash
     heights: {},
-    count: 0,
+    count: this.db.count(),
     lastTs: 0
   };
 
@@ -61,9 +59,11 @@ function Chain(options) {
       {
         hash: network.genesis.hash,
         version: network.genesis.version,
-        // prevBlock: network.genesis.prevBlock,
+        prevBlock: network.genesis.prevBlock,
+        merkleRoot: network.genesis.merkleRoot,
         ts: network.genesis.ts,
         bits: network.genesis.bits,
+        nonce: network.genesis.nonce,
         height: 0
       }
     ]
@@ -76,17 +76,12 @@ function Chain(options) {
     else if (network.type === 'testnet')
       this.fromJSON(require('./protocol/preload-test-full'));
     this.resetHeight(+process.env.BCOIN_START_HEIGHT);
-  } else {
-    if (+process.env.BCOIN_NO_COMPACT !== 1) {
-      if (!this.options.fullNode)
-        this.fromJSON(network.preload);
-    }
   }
 
-  this.tip = this.index.entries[this.index.entries.length - 1];
+  this.tip = this.db.get(this.index.count - 1);
 
   // Last TS after preload, needed for fill percent
-  this.index.lastTs = this.index.entries[this.index.entries.length - 1].ts;
+  this.index.lastTs = this.tip.ts;
 
   Chain.global = this;
 
@@ -159,6 +154,9 @@ Chain.prototype._init = function _init() {
   });
 };
 
+Chain.prototype.getEntry = function getEntry(height) {
+};
+
 Chain.prototype._addIndex = function _addIndex(entry, save) {
   var self = this;
 
@@ -169,7 +167,8 @@ Chain.prototype._addIndex = function _addIndex(entry, save) {
   }
 
   // Duplicate height
-  if (this.index.hashes[entry.height] === entry.hash)
+  var existing = this.db.get(entry.height);
+  if (existing && existing.hash === entry.hash)
     return Chain.codes.unchanged;
 
   // Fork at checkpoint
@@ -186,16 +185,14 @@ Chain.prototype._addIndex = function _addIndex(entry, save) {
     }
   }
 
-  this.index.entries[entry.height] = entry;
+  this.db.save(entry);
-  this.index.hashes[entry.height] = entry.hash;
   this.index.heights[entry.hash] = entry.height;
   this.index.count++;
 
-  this.tip = this.index.entries[this.index.entries.length - 1];
+  if (!this.tip || entry.height > this.tip.height)
-  this.emit('tip', this.tip);
+    this.tip = entry;
 
-  if (save)
+  this.emit('tip', this.tip);
-    this._save(entry.hash, entry);
 
   return Chain.codes.okay;
 };
@@ -219,13 +216,8 @@ Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
 
 Chain.prototype.resetHeight = function resetHeight(height) {
   var self = this;
-  var ahead = this.index.entries.slice(height + 1);
 
-  assert(height < this.index.entries.length);
+  assert(height < this.index.count);
-
-  // Nothing to do.
-  if (height === this.index.entries.length - 1)
-    return;
 
   // Reset the orphan map completely. There may
   // have been some orphans on a forked chain we
@@ -235,27 +227,14 @@ Chain.prototype.resetHeight = function resetHeight(height) {
   this.orphan.count = 0;
   this.orphan.size = 0;
 
-  // Rebuild the index from our new (lower) height.
+  for (var i = height + 1; height < this.index.count; i++) {
-  this.index.entries.length = height + 1;
+    var existing = this.db.get(i);
+    this.db.del(i);
+    delete this.index.heights[existing.hash];
+  }
 
-  this.index.heights = this.index.entries.reduce(function(out, entry) {
+  this.tip = this.db.get(height);
-    if (!self.options.fullNode) {
+  this.index.count = height + 1;
-      if (!entry)
-        return out;
-    }
-    out[entry.hash] = entry.height;
-    return out;
-  }, {});
-
-  this.index.hashes.length = height + 1;
-
-  if (!this.options.fullNode)
-    this.index.count -= this._count(ahead);
-  else
-    this.index.count = height + 1;
-
-  // Set and emit our new (old) tip.
-  this.tip = this.index.entries[this.index.entries.length - 1];
   this.emit('tip', this.tip);
 
   // The lastTs is supposed to be the last ts
@@ -264,17 +243,8 @@ Chain.prototype.resetHeight = function resetHeight(height) {
   // be higher. Reset it if necessary.
   this.index.lastTs = Math.min(
     this.index.lastTs,
-    this.index.entries[this.index.entries.length - 1].ts
+    this.tip.ts
   );
-
-  // Delete all the blocks now above us.
-  ahead.forEach(function(entry) {
-    if (!self.options.fullNode) {
-      if (!entry)
-        return;
-    }
-    self._delete(entry.hash);
-  });
 };
 
 Chain.prototype.resetTime = function resetTime(ts) {
@@ -349,15 +319,18 @@ Chain.prototype.add = function add(block, peer) {
     entry = new ChainBlock(this, {
       hash: hash,
       version: block.version,
-      // prevBlock: prevHash,
+      prevBlock: prevHash,
+      merkleRoot: block.merkleRoot,
       ts: block.ts,
       bits: block.bits,
+      nonce: block.nonce,
       height: prevHeight + 1
     });
 
     // Add entry if we do not have it (or if
     // there is another entry at its height)
-    if (this.index.hashes[entry.height] !== hash) {
+    var existing = this.db.get(entry.height);
+    if (!existing || existing.hash !== hash) {
       assert(this.index.heights[entry.hash] == null);
 
       // A valid block with an already existing
@@ -365,17 +338,18 @@ Chain.prototype.add = function add(block, peer) {
       // don't store by hash so we can't compare
       // chainworks. We reset the chain, find a
       // new peer, and wait to see who wins.
-      if (this.index.hashes[entry.height]) {
+      if (existing) {
         // The tip has more chainwork, it is a
         // higher height than the entry. This is
         // not an alternate tip. Ignore it.
+        if (0)
         if (this.tip.chainwork.cmp(entry.chainwork) > 0) {
           code = Chain.codes.unchanged;
           break;
         }
         // Get _our_ tip as opposed to
         // the attempted alternate tip.
-        tip = this.index.entries[entry.height];
+        tip = existing;
         // The block has equal chainwork (an
         // alternate tip). Reset the chain, find
         // a new peer, and wait to see who wins.
@@ -466,13 +440,6 @@ Chain.prototype.add = function add(block, peer) {
     this.orphan.size = 0;
   }
 
-  // Potentially compact the chain here. A
-  // full chain is not necessary for spv.
-  // if (!this.options.fullNode) {
-  //   if (this.size() > 100000)
-  //     this.compact();
-  // }
-
   if (code !== Chain.codes.okay) {
     if (!(this.options.multiplePeers && code === Chain.codes.newOrphan))
       utils.debug('Chain Error: %s', Chain.msg(code));
@@ -492,7 +459,9 @@ Chain.prototype.has = function has(hash) {
 };
 
 Chain.prototype.byHeight = function byHeight(height) {
-  return this.index.entries[height] || null;
+  if (height == null)
+    return;
+  return this.db.get(height);
 };
 
 Chain.prototype.byHash = function byHash(hash) {
@@ -505,13 +474,10 @@ Chain.prototype.byHash = function byHash(hash) {
 };
 
 Chain.prototype.byTime = function byTime(ts) {
-  for (var i = this.index.entries.length - 1; i >= 0; i--) {
+  for (var i = this.index.count - 1; i >= 0; i--) {
-    if (!this.options.fullNode) {
+    var existing = this.db.get(i);
-      if (!this.index.entries[i])
+    if (ts >= existing.ts)
-        continue;
+      return existing;
-    }
-    if (ts >= this.index.entries[i].ts)
-      return this.index.entries[i];
   }
   return null;
 };
@@ -540,11 +506,11 @@ Chain.prototype.getOrphan = function getOrphan(hash) {
 };
 
 Chain.prototype.getTip = function getTip() {
-  return this.index.entries[this.index.entries.length - 1];
+  return this.tip;
 };
 
 Chain.prototype.isFull = function isFull() {
-  var last = this.index.entries[this.index.entries.length - 1].ts;
+  var last = this.tip.ts;
   var delta = utils.now() - last;
   return delta < 40 * 60;
 };
@@ -564,18 +530,15 @@ Chain.prototype.hashRange = function hashRange(start, end) {
   if (!start || !end)
     return [];
 
-  hashes = this.index.hashes.slice(start.height, end.height + 1);
+  for (var i = start.height; i < end.height + 1; i++)
-
+    hashes.push(this.db.get(i).hash);
-  if (!this.options.fullNode)
-    hashes = this._filter(hashes);
 
   return hashes;
 };
 
 Chain.prototype.locatorHashes = function locatorHashes(start) {
-  var chain = this.index.hashes;
   var hashes = [];
-  var top = chain.length - 1;
+  var top = this.height();
   var step = 1;
   var i;
 
@@ -595,22 +558,23 @@ Chain.prototype.locatorHashes = function locatorHashes(start) {
       // is our tip. This is useful for getheaders
       // when not using headers-first.
       hashes.push(start);
-      top = chain.length - 1;
+      top = this.index.count - 1;
     }
   } else if (typeof start === 'number') {
     top = start;
   }
 
-  assert(chain[top]);
+  // assert(chain[top]);
 
   i = top;
   for (;;) {
-    if (chain[i])
+    var existing = this.db.get(i);
-      hashes.push(chain[i]);
+    if (existing)
+      hashes.push(existing.hash);
     i = i - step;
     if (i <= 0) {
       if (i + step !== 0)
-        hashes.push(chain[0]);
+        hashes.push(this.db.get(0).hash);
       break;
     }
     if (hashes.length >= 10)
@@ -694,12 +658,7 @@ Chain.prototype.target = function target(last, block) {
   }
 
   // Back 2 weeks
-  // first = last;
+  first = this.db.get(last.height - (network.powDiffInterval - 1));
-  // for (i = 0; first && i < network.powDiffInterval - 1; i++)
-  //   first = first.prev;
-
-  // Back 2 weeks
-  first = this.index.entries[last.height - (network.powDiffInterval - 1)];
 
   assert(first);
 
@@ -731,117 +690,9 @@ Chain.prototype.retarget = function retarget(last, first) {
   return utils.toCompact(target);
 };
 
-Chain.prototype.compact = function compact(keep) {
-  var entries;
-
-  if (+process.env.BCOIN_NO_COMPACT === 1)
-    return;
-
-  entries = this._compact(keep);
-
-  this.index.entries = {};
-  this.index.hashes = [];
-  this.index.heights = {};
-  this.index.count = 0;
-
-  json.entries.forEach(function(entry) {
-    this._addIndex(ChainBlock.fromJSON(this, entry));
-  }, this);
-};
-
-Chain.prototype._compact = function _compact(keep) {
-  var entries;
-  var last, first, delta1, delta2, delta3, lastTs, lastHeight;
-  var i, ts, delta, hdelta;
-
-  if (+process.env.BCOIN_NO_COMPACT === 1)
-    return this.index.entries;
-
-  entries = this._filter(this.index.entries);
-
-  if (!keep)
-    keep = network.powDiffInterval + 10;
-
-  // Keep only last ~2016 consequent blocks, dilate others at:
-  // 7 day range for blocks before 2013
-  // 12 hour for blocks before 2014
-  // 6 hour for blocks in 2014 and after it
-  // (or at maximum 250 block range)
-  last = entries.slice(-keep);
-
-  first = [];
-
-  delta1 = 7 * 24 * 3600;
-  delta2 = 12 * 3600;
-  delta3 = 6 * 3600;
-
-  lastTs = 0;
-  lastHeight = -1000;
-
-  for (i = 0; i < entries.length - keep; i++) {
-    ts = entries[i].ts;
-
-    delta = ts < 1356984000
-      ? delta1
-      : ts < 1388520000 ? delta2 : delta3;
-
-    hdelta = entries[i].height - lastHeight;
-
-    if (ts - lastTs < delta && hdelta < 250)
-      continue;
-
-    lastTs = ts;
-    lastHeight = entries[i].height;
-    first.push(this.index.entries[i]);
-  }
-
-  return first.concat(last);
-};
-
-Chain.prototype._save = function(hash, obj) {
-  var self = this;
-
-  if (!this.storage)
-    return;
-
-  this.storage.put(this.prefix + hash, obj.toJSON(), function(err) {
-    if (err)
-      self.emit('error', err);
-  });
-};
-
-Chain.prototype._delete = function(hash) {
-  var self = this;
-
-  if (!this.storage)
-    return;
-
-  this.storage.del(this.prefix + hash, function(err) {
-    if (err)
-      self.emit('error', err);
-  });
-};
-
-Chain.prototype._count = function(obj) {
-  for (var i = 0, c = 0; i < obj.length; i++)
-    if (obj[i])
-      c++;
-  return c;
-};
-
-Chain.prototype._filter = function(obj) {
-  for (var i = 0, a = []; i < obj.length; i++)
-    if (obj[i])
-      a.push(obj[i]);
-  return a;
-};
-
 Chain.prototype.toJSON = function toJSON() {
   var entries = this.index.entries;
 
-  if (!this.options.fullNode)
-    entries = this._compact();
-
   return {
     v: 2,
     type: 'chain',
@@ -858,10 +709,200 @@ Chain.prototype.fromJSON = function fromJSON(json) {
   assert.equal(json.network, network.type);
 
   json.entries.forEach(function(entry) {
-    this._addIndex(ChainBlock.fromJSON(this, entry));
+    this._addIndex(ChainBlock.fromJSON(this, entry), true);
   }, this);
+};
 
-  assert(this.index.entries.length > 0);
+/**
+ * ChainDB
+ */
+
+var BLOCK_SIZE = 80;
+// var BLOCK_SIZE = 144;
+
+function ChainDB(chain) {
+  this.file = process.env.HOME + '/bcoin-' + network.type + '.blockchain';
+  try {
+    fs.unlinkSync(this.file);
+  } catch (e) {
+  }
+  var exists = false;
+  try {
+    fs.accessSync(file);
+    exists = true;
+  } catch (e) {
+    exists = false;
+  }
+  if (!exists)
+    fs.writeFileSync(this.file, new Buffer(0));
+  this.fd = fs.openSync(this.file, 'r+');
+  this.chain = chain;
+  this._buffer = [];
+  this._cache = {};
+  this._bufferPool = {};
+  this._nullBlock = new Buffer(BLOCK_SIZE);
+  this._nullBlock.fill(0);
+}
+
+ChainDB.prototype.getBuffer = function(size) {
+  if (!this._bufferPool[size])
+    this._bufferPool[size] = new Buffer(size);
+  return this._bufferPool[size];
+};
+
+ChainDB.prototype.save = function save(entry) {
+  // Cache the past 2016 blocks in memory
+  this._cache[entry.height] = entry;
+  delete this._cache[entry.height - network.powDiffInterval];
+  assert(Object.keys(this._cache).length < network.powDiffInterval + 1);
+  return this._write(new Buffer(entry.toRaw()), entry.height * BLOCK_SIZE);
+};
+
+ChainDB.prototype._write = function _write(data, offset) {
+  var size, bytes, index, hash;
+  var self = this;
+
+  if (offset < 0 || offset == null)
+    return false;
+
+  hash = offset;
+  size = data.length;
+  bytes = 0;
+  index = 0;
+
+  // Something is already writing. Cancel it
+  // and synchronously write the data after
+  // it cancels.
+  if (this._buffer[hash]) {
+    assert(this._buffer[hash].data.length === data.length);
+    this._buffer[hash].data = data;
+    this._buffer[hash].queue.push([data, offset]);
+    return;
+  }
+
+  // Speed up writes by doing them asynchronously
+  // and keeping the data to be written in memory.
+  this._buffer[hash] = { queue: [], data: data };
+
+  (function callee() {
+    fs.write(self.fd, data, index, size, offset, function(err, bytes) {
+      // Something tried to write here but couldn't.
+      // Synchronously write it and get it over with.
+      if (self._buffer[hash].queue.length) {
+        self._buffer[hash].queue.forEach(function(chunk) {
+          self._writeSync(chunk[0], chunk[1]);
+        });
+        delete self._buffer[hash];
+        return;
+      }
+
+      index += bytes;
+      size -= bytes;
+      offset += bytes;
+
+      if (index === data.length) {
+        delete self._buffer[hash];
+        return;
+      }
+
+      callee();
+    });
+  })();
+
+  return false;
+};
+
+ChainDB.prototype._writeSync = function _writeSync(data, offset) {
+  var size, bytes, index;
+
+  if (offset < 0 || offset == null)
+    return false;
+
+  size = data.length;
+  bytes = 0;
+  index = 0;
+
+  while (bytes = fs.writeSync(this.fd, data, index, size, offset)) {
+    index += bytes;
+    size -= bytes;
+    offset += bytes;
+    if (index === data.length)
+      return true;
+  }
+
+  return false;
+};
+
+ChainDB.prototype.del = function del(height) {
+  this._write(this._nullBlock, height * BLOCK_SIZE);
+  if (height * BLOCK_SIZE + BLOCK_SIZE === this.size()) {
+    while (this.isNull(height))
+      height--;
+    fs.ftruncateSync(this.fd, height * BLOCK_SIZE + BLOCK_SIZE);
+  }
+  return true;
+};
+
+ChainDB.prototype.isNull = function isNull(height) {
+  var data = this._read(1, height * BLOCK_SIZE);
+  if (!data)
+    return false;
+  return utils.read32(data, 0) === 0;
+};
+
+ChainDB.prototype._read = function _read(size, offset) {
+  var data = this.getBuffer(size);
+  var bytes = 0;
+  var index = 0;
+
+  if (offset < 0 || offset == null)
+    return;
+
+  if (this._buffer[offset]) {
+    assert(this._buffer[offset].data.length >= size);
+    return this._buffer[offset].data.slice(0, size);
+  }
+
+  try {
+    while (bytes = fs.readSync(this.fd, data, index, size, offset)) {
+      index += bytes;
+      size -= bytes;
+      offset += bytes;
+      if (index === data.length) {
+        if (utils.read32(data, 0) === 0)
+          return;
+        return data;
+      }
+    }
+  } catch (e) {
+    return;
+  }
+};
+
+ChainDB.prototype.size = function size() {
+  try {
+    return fs.statSync(this.file).size;
+  } catch (e) {
+    return 0;
+  }
+};
+
+ChainDB.prototype.count = function count() {
+  return this.size() / BLOCK_SIZE | 0;
+};
+
+ChainDB.prototype.get = function get(height) {
+  var data;
+
+  if (this._cache[height])
+    return this._cache[height];
+
+  data = this._read(BLOCK_SIZE, height * BLOCK_SIZE);
+
+  if (!data)
+    return;
+
+  return ChainBlock.fromRaw(this.chain, height, data);
 };
 
 /**
@@ -872,19 +913,22 @@ function ChainBlock(chain, data) {
   this.chain = chain;
   this.hash = data.hash;
   this.version = data.version;
-  // this.prevBlock = data.prevBlock;
+  this.prevBlock = data.prevBlock;
+  this.merkleRoot = data.merkleRoot;
   this.ts = data.ts;
   this.bits = data.bits;
+  this.nonce = data.nonce;
   this.height = data.height;
-  this.chainwork = this.getChainwork();
+  this.chainwork = data.chainwork || new bn(0);
+  // this.chainwork = data.chainwork || this.getChainwork();
 }
 
 ChainBlock.prototype.__defineGetter__('prev', function() {
-  return this.chain.index.entries[this.height - 1];
+  return this.chain.db.get(this.height - 1);
 });
 
 ChainBlock.prototype.__defineGetter__('next', function() {
-  return this.chain.index.entries[this.height + 1];
+  return this.chain.db.get(this.height + 1);
 });
 
 ChainBlock.prototype.__defineGetter__('proof', function() {
@@ -939,36 +983,52 @@ ChainBlock.prototype.isSuperMajority = function(version, required) {
 };
 
 ChainBlock.prototype.toJSON = function() {
-  // return [
-  //   this.hash,
-  //   this.version,
-  //   this.prevBlock,
-  //   this.ts,
-  //   this.bits,
-  //   this.height
-  // };
   return {
     hash: this.hash,
     version: this.version,
-    // prevBlock: this.prevBlock,
+    prevBlock: this.prevBlock,
+    merkleRoot: this.merkleRoot,
     ts: this.ts,
     bits: this.bits,
+    nonce: this.nonce,
     height: this.height
   };
 };
 
 ChainBlock.fromJSON = function(chain, json) {
-  // return new ChainBlock(chain, {
-  //   hash: json[0],
-  //   version: json[1],
-  //   prevBlock: json[2],
-  //   ts: json[3],
-  //   bits: json[4],
-  //   height: json[5]
-  // });
   return new ChainBlock(chain, json);
 };
 
+ChainBlock.prototype.toRaw = function toRaw() {
+  var res = new Array(BLOCK_SIZE);
+
+  utils.writeU32(res, this.version, 0);
+  utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4);
+  utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36);
+  utils.writeU32(res, this.ts, 68);
+  utils.writeU32(res, this.bits, 72);
+  utils.writeU32(res, this.nonce, 76);
+  // utils.copy(utils.toArray(this.hash, 'hex'), res, 80);
+  // utils.copy(this.chainwork.toArray('be', 32), res, 112);
+
+  return res;
+};
+
+ChainBlock.fromRaw = function fromRaw(chain, height, p) {
+  return new ChainBlock(chain, {
+    hash: utils.toHex(utils.dsha256(p.slice(0, 80))),
+    version: utils.read32(p, 0),
+    prevBlock: utils.toHex(utils.toArray(p.slice(4, 36))),
+    merkleRoot: utils.toHex(utils.toArray(p.slice(36, 68))),
+    ts: utils.readU32(p, 68),
+    bits: utils.readU32(p, 72),
+    nonce: utils.readU32(p, 76),
+    height: height,
+    // hash: utils.toHex(utils.toArray(p.slice(80, 112))),
+    // chainwork: new bn(p.slice(112, 144))
+  });
+};
+
 /**
  * Expose
  */