blockdb: flat files.

lib/chain/blockdb-browser.js

@@ -0,0 +1,63 @@
+/*!
+ * blockdb.js - blockchain data management for bcoin
+ * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+/**
+ * BlockDB
+ * @constructor
+ */
+
+function BlockDB(chaindb) {
+  this.chaindb = chaindb;
+  this.db = chaindb.db;
+  this.layout = chaindb.layout;
+}
+
+BlockDB.prototype.open = function open() {
+  return Promise.resolve();
+};
+
+BlockDB.prototype.close = function close() {
+  return Promise.resolve();
+};
+
+BlockDB.prototype.sync = function sync() {
+  return Promise.resolve();
+};
+
+BlockDB.prototype.saveBlock = function saveBlock(block) {
+  this.chaindb.put(this.layout.b(block.hash()), block.toRaw());
+  return Promise.resolve();
+};
+
+BlockDB.prototype.readBlock = function readBlock(hash) {
+  return this.db.get(this.layout.b(hash));
+};
+
+BlockDB.prototype.readBlockEntry = function readBlockEntry(entry) {
+  return this.readBlock(entry.hash);
+};
+
+BlockDB.prototype.removeBlock = function removeBlock(hash) {
+  this.chaindb.del(this.layout.b(hash));
+  return Promise.resolve();
+};
+
+BlockDB.prototype.pruneBlock = function pruneBlock(hash) {
+  this.chaindb.del(this.layout.b(hash));
+  return Promise.resolve();
+};
+
+BlockDB.prototype.pruneBlockEntry = function pruneBlockEntry(entry) {
+  return this.pruneBlock(entry.hash);
+};
+
+/*
+ * Expose
+ */
+
+module.exports = BlockDB;

lib/chain/blockdb.js

@@ -0,0 +1,196 @@
+/*!
+ * blockdb.js - blockchain data management for bcoin
+ * Copyright (c) 2014-2015, Fedor Indutny (MIT License)
+ * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License).
+ * https://github.com/bcoin-org/bcoin
+ */
+
+'use strict';
+
+var assert = require('assert');
+var co = require('../utils/co');
+var Flat = require('../db/flat');
+var LRU = require('../utils/lru');
+var FileEntry = Flat.FileEntry;
+
+/**
+ * BlockDB
+ * @constructor
+ */
+
+function BlockDB(chaindb) {
+  this.chaindb = chaindb;
+  this.db = chaindb.db;
+  this.layout = chaindb.layout;
+  this.flat = new Flat(this.db);
+  this.cache = new LRU(8192);
+}
+
+BlockDB.prototype.open = function open() {
+  return this.flat.open();
+};
+
+BlockDB.prototype.close = function close() {
+  return this.flat.close();
+};
+
+BlockDB.prototype.sync = co(function* sync() {
+  var entry = yield this.chaindb.getTip();
+  var block, rollback;
+
+  assert(entry);
+
+  for (;;) {
+    try {
+      block = yield this.readBlock(entry.hash);
+    } catch (e) {
+      if (e.type !== 'ChecksumMismatch')
+        throw e;
+      block = null;
+    }
+
+    if (block)
+      break;
+
+    this.cache.remove(entry.hash);
+
+    entry = yield entry.getPrevious();
+    assert(entry);
+
+    rollback = true;
+  }
+
+  if (!rollback)
+    return;
+
+  yield this.chaindb.reset(entry.hash, true);
+});
+
+BlockDB.prototype.getEntry = co(function* getEntry(hash) {
+  var key = hash;
+  var entry, data;
+
+  if (typeof key !== 'string')
+    key = key.toString('hex');
+
+  entry = this.cache.get(key);
+
+  if (entry)
+    return entry;
+
+  data = yield this.db.get(this.layout.b(hash));
+
+  if (!data)
+    return;
+
+  entry = FileEntry.fromRaw(data);
+
+  this.cache.set(key, entry);
+
+  return entry;
+});
+
+BlockDB.prototype.saveBlock = co(function* saveBlock(block) {
+  var hash = block.hash();
+  var hex = block.hash('hex');
+  var entry = yield this.flat.write(block.toRaw());
+
+  if (block.height === 0)
+    yield this.flat.sync();
+
+  this.cache.set(hex, entry);
+
+  this.chaindb.put(this.layout.b(hash), entry.toRaw());
+});
+
+BlockDB.prototype.readBlock = co(function* readBlock(hash) {
+  var entry = yield this.getEntry(hash);
+
+  if (!entry)
+    return;
+
+  return yield this.readBlockEntry(entry);
+});
+
+BlockDB.prototype.readBlockEntry = function readBlockEntry(entry) {
+  return this.flat.read(entry.index, entry.pos);
+};
+
+BlockDB.prototype.removeBlock = co(function* removeBlock(hash) {
+  var entry = yield this.getEntry(hash);
+
+  if (!entry)
+    return;
+
+  this.chaindb.del(this.layout.b(hash));
+
+  if (entry.pos === 0)
+    yield this.flat.remove(entry.index);
+});
+
+BlockDB.prototype.pruneBlock = co(function* pruneBlock(hash) {
+  var entry = yield this.getEntry(hash);
+  if (!entry)
+    return;
+  return yield this.pruneBlockEntry(hash, entry);
+});
+
+BlockDB.prototype.pruneBlockEntry = function pruneBlockEntry(hash, entry) {
+  var index = entry.index;
+  if (index === this.current.index)
+    index -= 1;
+  this.chaindb.del(this.layout.b(hash));
+  return this.flat.remove(index);
+};
+
+/**
+ * Batch
+ * @constructor
+ */
+
+function Batch(ffdb) {
+  this.ffdb = ffdb;
+  this.ops = [];
+}
+
+Batch.prototype.put = function put(block) {
+  this.ops.push(new BatchOp(0, block));
+};
+
+Batch.prototype.del = function del(hash) {
+  this.ops.push(new BatchOp(1, hash));
+};
+
+Batch.prototype.write = co(function* write() {
+  var i, op;
+
+  for (i = 0; i < this.ops.length; i++) {
+    op = this.ops[i];
+    switch (op.type) {
+      case 0:
+        yield this.ffdb.saveBlock(op.data);
+        break;
+      case 1:
+        yield this.ffdb.removeBlock(op.data);
+        break;
+      default:
+        assert(false);
+    }
+  }
+});
+
+/**
+ * BatchOp
+ * @constructor
+ */
+
+function BatchOp(type, data) {
+  this.type = type;
+  this.data = data;
+}
+
+/*
+ * Expose
+ */
+
+module.exports = BlockDB;

lib/chain/chaindb.js

@@ -17,7 +17,7 @@ var co = require('../utils/co');
 var Network = require('../protocol/network');
 var CoinView = require('./coinview');
 var Coins = require('./coins');
-var ldb = require('../db/ldb');
+var LDB = require('../db/ldb');
 var LRU = require('../utils/lru');
 var Block = require('../primitives/block');
 var Coin = require('../primitives/coin');
@@ -27,6 +27,7 @@ var Address = require('../primitives/address');
 var ChainEntry = require('./chainentry');
 var U32 = utils.U32;
 var DUMMY = new Buffer([0]);
+var BlockDB = require('./blockdb');
 
 /*
  * Database Layout:
@@ -173,17 +174,20 @@ function ChainDB(chain) {
   this.logger = chain.logger;
   this.network = chain.network;
   this.options = new ChainOptions(chain.options);
+  this.layout = layout;
 
-  this.db = ldb({
+  this.db = LDB({
     location: chain.options.location,
     db: chain.options.db,
     maxOpenFiles: chain.options.maxFiles,
-    compression: true,
+    compression: false,
     cacheSize: 16 << 20,
     writeBufferSize: 8 << 20,
     bufferKeys: !utils.isBrowser
   });
 
+  this.blockdb = new BlockDB(this);
+
   this.state = new ChainState();
   this.pending = null;
   this.current = null;
@@ -198,6 +202,7 @@ function ChainDB(chain) {
 
   // We want to keep the last 5 blocks of unspents in memory.
   this.coinWindow = 25 << 20;
+  this.coinWindow = 100 << 20;
 
   this.coinCache = new LRU.Nil();
   this.cacheHash = new LRU(this.cacheWindow);
@@ -228,9 +233,10 @@ ChainDB.prototype._open = co(function* open() {
   this.logger.info('Starting chain load.');
 
   yield this.db.open();
-
   yield this.db.checkVersion('V', 1);
 
+  yield this.blockdb.open();
+
   state = yield this.getState();
   options = yield this.getOptions();
 
@@ -246,6 +252,7 @@ ChainDB.prototype._open = co(function* open() {
   if (state) {
     // Grab the chainstate if we have one.
     this.state = state;
+    yield this.blockdb.sync();
   } else {
     // Otherwise write the genesis block.
     // (We assume this database is fresh).
@@ -271,9 +278,10 @@ ChainDB.prototype._open = co(function* open() {
  * @returns {Promise}
  */
 
-ChainDB.prototype._close = function close() {
-  return this.db.close();
-};
+ChainDB.prototype._close = co(function* close() {
+  yield this.blockdb.close();
+  yield this.db.close();
+});
 
 /**
  * Start a batch.
@@ -841,7 +849,7 @@ ChainDB.prototype.getBlock = co(function* getBlock(hash) {
   if (!item.hash)
     return;
 
-  data = yield this.db.get(layout.b(item.hash));
+  data = yield this.blockdb.readBlock(hash);
 
   if (!data)
     return;
@@ -869,7 +877,7 @@ ChainDB.prototype.getRawBlock = co(function* getRawBlock(block) {
   if (!hash)
     return;
 
-  return yield this.db.get(layout.b(hash));
+  return yield this.blockdb.readBlock(hash);
 });
 
 /**
@@ -1369,7 +1377,7 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
  * @returns {Promise}
  */
 
-ChainDB.prototype.reset = co(function* reset(block) {
+ChainDB.prototype.reset = co(function* reset(block, noData) {
   var entry = yield this.get(block);
   var tip;
 
@@ -1417,11 +1425,15 @@ ChainDB.prototype.reset = co(function* reset(block) {
     this.del(layout.n(tip.prevBlock));
 
     // Disconnect and remove block data.
-    try {
-      yield this.removeBlock(tip.hash);
-    } catch (e) {
-      this.drop();
-      throw e;
+    if (!noData) {
+      try {
+        yield this.removeBlock(tip.hash);
+      } catch (e) {
+        this.drop();
+        throw e;
+      }
+    } else {
+      this.del(layout.b(tip.hash));
     }
 
     // Revert chain state to previous tip.
@@ -1489,7 +1501,7 @@ ChainDB.prototype._removeChain = co(function* removeChain(hash) {
     this.del(layout.p(tip.hash));
     this.del(layout.h(tip.hash));
     this.del(layout.e(tip.hash));
-    this.del(layout.b(tip.hash));
+    yield this.blockdb.removeBlock(tip.hash);
 
     // Queue up hash to be removed
     // on successful write.
@@ -1514,7 +1526,7 @@ ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) {
 
   // Write actual block data (this may be
   // better suited to flat files in the future).
-  this.put(layout.b(block.hash()), block.toRaw());
+  yield this.blockdb.saveBlock(block);
 
   if (!view)
     return;
@@ -1540,7 +1552,7 @@ ChainDB.prototype.removeBlock = co(function* removeBlock(hash) {
   if (!block)
     throw new Error('Block not found.');
 
-  this.del(layout.b(block.hash()));
+  yield this.blockdb.removeBlock(block.hash());
 
   return yield this.disconnectBlock(block);
 });
@@ -1761,7 +1773,7 @@ ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) {
   if (!hash)
     return;
 
-  this.del(layout.b(hash));
+  yield this.blockdb.removeBlock(hash);
   this.del(layout.u(hash));
 });
 

lib/db/flat.js

@@ -0,0 +1,391 @@
+'use strict';
+
+var utils = require('../utils/utils');
+var co = require('../utils/co');
+var Locker = require('../utils/locker');
+var path = require('path');
+var fs = require('fs');
+var promisify = co.promisify;
+var fsExists = promisify(fs.exists, fs);
+var fsMkdir = promisify(fs.mkdir, fs);
+var fsReaddir = promisify(fs.readdir, fs);
+var fsOpen = promisify(fs.open, fs);
+var fsStat = promisify(fs.stat, fs);
+var fsFstat = promisify(fs.fstat, fs);
+var fsWrite = promisify(fs.write, fs);
+var fsRead = promisify(fs.read, fs);
+var fsClose = promisify(fs.close, fs);
+var fsFtruncate = promisify(fs.ftruncate, fs);
+var fsFsync = promisify(fs.fsync, fs);
+var fsUnlink = promisify(fs.unlink, fs);
+var fsExists;
+var assert = utils.assert;
+var murmur3 = require('../utils/murmur3');
+
+var MAX_SIZE = 512 << 20;
+var MAX_FILES = 64;
+var MAX_ENTRY = 12 << 20;
+
+/**
+ * Flat
+ * @constructor
+ */
+
+function Flat(db) {
+  if (!(this instanceof Flat))
+    return new Flat(db);
+
+  this.dir = path.resolve(db.location, '..');
+  this.dir = path.resolve(this.dir, 'blocks');
+  this.locker = new Locker();
+
+  this.fileIndex = -1;
+  this.current = null;
+  this.files = {};
+  this.openFiles = [];
+  this.indexes = [];
+}
+
+Flat.prototype.hash = function hash(data) {
+  return murmur3(data, 0xdeedbeef);
+};
+
+Flat.prototype.open = co(function* open() {
+  var index = -1;
+  var i, list, name;
+
+  if (!(yield fsExists(this.dir)))
+    yield fsMkdir(this.dir, 493);
+
+  list = yield fsReaddir(this.dir);
+
+  for (i = 0; i < list.length; i++) {
+    name = list[i];
+
+    if (!/^\d{10}$/.test(name))
+      continue;
+
+    name = parseInt(name, 10);
+
+    utils.binaryInsert(this.indexes, name, cmp);
+
+    if (name > index)
+      index = name;
+  }
+
+  if (index === -1) {
+    yield this.allocate();
+    return;
+  }
+
+  this.fileIndex = index;
+  this.current = yield this.openFile(index);
+});
+
+Flat.prototype.close = co(function* close() {
+  var unlock = yield this.locker.lock();
+  try {
+    return yield this._close();
+  } finally {
+    unlock();
+  }
+});
+
+Flat.prototype._close = co(function* close() {
+  var i, index, file;
+
+  for (i = this.openFiles.length - 1; i >= 0; i--) {
+    index = this.openFiles[i];
+    file = this.files[index];
+    assert(file);
+    yield this._closeFile(file.index);
+  }
+
+  assert(this.current === null);
+  assert(this.openFiles.length === 0);
+
+  this.fileIndex = -1;
+  this.indexes.length = 0;
+});
+
+Flat.prototype.name = function name(index) {
+  return path.resolve(this.dir, utils.pad32(index));
+};
+
+Flat.prototype.openFile = co(function* openFile(index) {
+  var unlock = yield this.locker.lock();
+  try {
+    return yield this._openFile(index);
+  } finally {
+    unlock();
+  }
+});
+
+Flat.prototype._openFile = co(function* _openFile(index) {
+  var file = this.files[index];
+  var name, fd, stat;
+
+  if (file)
+    return file;
+
+  name = this.name(index);
+
+  fd = yield fsOpen(name, 'a+');
+  stat = yield fsFstat(fd);
+
+  file = new File(fd, index, stat.size);
+
+  this.files[index] = file;
+  utils.binaryInsert(this.openFiles, index, cmp);
+
+  yield this.evict(index);
+
+  return file;
+});
+
+Flat.prototype.closeFile = co(function* closeFile(index) {
+  var unlock = yield this.locker.lock();
+  try {
+    assert(index !== this.current.index);
+    return yield this._closeFile(index);
+  } finally {
+    unlock();
+  }
+});
+
+Flat.prototype._closeFile = co(function* _closeFile(index) {
+  var file = this.files[index];
+  var result;
+
+  if (!file)
+    return;
+
+  yield fsClose(file.fd);
+
+  result = utils.binaryRemove(this.openFiles, index, cmp);
+  assert(result);
+
+  delete this.files[index];
+
+  if (file === this.current)
+    this.current = null;
+});
+
+Flat.prototype.remove = co(function* remove(index) {
+  var unlock = yield this.locker.lock();
+  try {
+    return yield this._remove(index);
+  } finally {
+    unlock();
+  }
+});
+
+Flat.prototype._remove = co(function* remove(index) {
+  var result;
+
+  assert(index != null);
+
+  if (!this.files[index])
+    return;
+
+  yield this._closeFile(index);
+  yield fsUnlink(this.name(index));
+
+  result = utils.binaryRemove(this.indexes, index, cmp);
+  assert(result);
+
+  if (!this.current) {
+    index = this.indexes[this.indexes.length - 1];
+    assert(index != null);
+    this.current = yield this._openFile(index);
+  }
+});
+
+Flat.prototype.allocate = co(function* allocate() {
+  var index = this.fileIndex + 1;
+  var fd = yield fsOpen(this.name(index), 'a+');
+  var file = new File(fd, index, 0);
+
+  this.files[index] = file;
+  this.current = file;
+  this.fileIndex++;
+
+  utils.binaryInsert(this.openFiles, index, cmp);
+  yield this.evict(-1);
+});
+
+Flat.prototype.evict = co(function* evict(not) {
+  var i = 0;
+  var index, file;
+
+  if (this.openFiles.length <= MAX_FILES)
+    return;
+
+  for (;;) {
+    assert(i < this.openFiles.length);
+
+    index = this.openFiles[i];
+
+    if (this.current) {
+      if (index !== not && index !== this.current.index)
+        break;
+    }
+
+    i++;
+  }
+
+  index = this.openFiles[i];
+  file = this.files[index];
+  assert(file);
+
+  yield fsClose(file.fd);
+
+  this.openFiles.splice(i, 1);
+  delete this.files[index];
+});
+
+Flat.prototype.write = co(function* write(data) {
+  var unlock = yield this.locker.lock();
+  try {
+    return yield this._write(data);
+  } finally {
+    unlock();
+  }
+});
+
+Flat.prototype._write = co(function* write(data) {
+  var pos, fd, size, chk;
+  var buf = new Buffer(4);
+  var len = 4 + data.length + 4;
+
+  if (data.length > MAX_ENTRY)
+    throw new Error('Size too large.');
+
+  if (this.current.pos + len > MAX_SIZE) {
+    yield this.sync();
+    yield this.allocate();
+  }
+
+  pos = this.current.pos;
+  fd = this.current.fd;
+
+  buf.writeUInt32LE(data.length, 0, true);
+  yield fsWrite(fd, buf, 0, 4, pos);
+
+  yield fsWrite(fd, data, 0, data.length, pos + 4);
+
+  buf.writeUInt32LE(this.hash(data), 0, true);
+  yield fsWrite(fd, buf, 0, 4, pos + 4 + data.length);
+
+  this.current.pos += len;
+
+  return new FileEntry(this.current.index, pos, data.length);
+});
+
+Flat.prototype.read = co(function* read(index, offset) {
+  var file = yield this.openFile(index);
+  var buf = new Buffer(4);
+  var size, data, chk, err;
+
+  if (offset + 8 > file.pos)
+    throw new Error('Read is out of bounds.');
+
+  yield fsRead(file.fd, buf, 0, 4, offset);
+  size = buf.readUInt32LE(0, true);
+
+  if (size > MAX_ENTRY)
+    throw new Error('Size too large.');
+
+  if (offset + 4 + size + 4 > file.pos)
+    throw new Error('Read is out of bounds.');
+
+  data = new Buffer(size);
+  yield fsRead(file.fd, data, 0, data.length, offset + 4);
+
+  yield fsRead(file.fd, buf, 0, 4, offset + 4 + data.length);
+  chk = buf.readUInt32LE(0, true);
+
+  if (this.hash(data) !== chk) {
+    err = new Error('Checksum mismatch.');
+    err.type = 'ChecksumMismatch';
+    throw err;
+  }
+
+  return data;
+});
+
+Flat.prototype.sync = co(function* sync() {
+  yield fsFsync(this.current.fd);
+});
+
+/*
+ * File
+ * @constructor
+ */
+
+function File(fd, index, pos) {
+  this.fd = fd;
+  this.index = index;
+  this.pos = pos;
+}
+
+/*
+ * FileEntry
+ * @constructor
+ */
+
+function FileEntry(index, offset, size) {
+  this.index = index;
+  this.offset = offset;
+  this.size = size;
+}
+
+FileEntry.prototype.toRaw = function toRaw() {
+  var data = new Buffer(12);
+  data.writeUInt32LE(this.index, 0, true);
+  data.writeUInt32LE(this.offset, 4, true);
+  data.writeUInt32LE(this.size, 8, true);
+  return data;
+};
+
+FileEntry.fromRaw = function fromRaw(data) {
+  var entry = new FileEntry(0, 0, 0);
+  entry.index = data.readUInt32LE(0, true);
+  entry.offset = data.readUInt32LE(4, true);
+  entry.size = data.readUInt32LE(8, true);
+  return entry;
+};
+
+/*
+ * Helpers
+ */
+
+function cmp(a, b) {
+  return a - b;
+}
+
+fsExists = co(function* fsExists(name) {
+  var stat;
+
+  try {
+    stat = yield fsStat(name);
+  } catch (e) {
+    if (e.code === 'ENOENT')
+      return false;
+    throw e;
+  }
+
+  if (!stat.isDirectory())
+    throw new Error('File is not a directory.');
+
+  return true;
+});
+
+/*
+ * Expose
+ */
+
+exports = Flat;
+exports.FileEntry = FileEntry;
+
+module.exports = exports;

package.json

@@ -67,6 +67,8 @@
     "./lib/http/rpcclient": "./browser/empty.js",
     "./lib/http/server": "./browser/empty.js",
     "./lib/http/wallet": "./browser/empty.js",
+    "./lib/db/flat": "./browser/empty.js",
+    "./lib/chain/blockdb": "./lib/chain/blockdb-browser.js",
     "fs": "./browser/empty.js",
     "crypto": "./browser/empty.js",
     "child_process": "./browser/empty.js",