From 3c22d7050f670c92caa42f2befa43c59a6858d2c Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Fri, 18 Nov 2016 18:01:33 -0800 Subject: [PATCH] blockdb: flat files. --- lib/chain/blockdb-browser.js | 63 ++++++ lib/chain/blockdb.js | 196 ++++++++++++++++++ lib/chain/chaindb.js | 50 +++-- lib/db/flat.js | 391 +++++++++++++++++++++++++++++++++++ package.json | 2 + 5 files changed, 683 insertions(+), 19 deletions(-) create mode 100644 lib/chain/blockdb-browser.js create mode 100644 lib/chain/blockdb.js create mode 100644 lib/db/flat.js diff --git a/lib/chain/blockdb-browser.js b/lib/chain/blockdb-browser.js new file mode 100644 index 00000000..4662f29f --- /dev/null +++ b/lib/chain/blockdb-browser.js @@ -0,0 +1,63 @@ +/*! + * blockdb.js - blockchain data management for bcoin + * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * BlockDB + * @constructor + */ + +function BlockDB(chaindb) { + this.chaindb = chaindb; + this.db = chaindb.db; + this.layout = chaindb.layout; +} + +BlockDB.prototype.open = function open() { + return Promise.resolve(); +}; + +BlockDB.prototype.close = function close() { + return Promise.resolve(); +}; + +BlockDB.prototype.sync = function sync() { + return Promise.resolve(); +}; + +BlockDB.prototype.saveBlock = function saveBlock(block) { + this.chaindb.put(this.layout.b(block.hash()), block.toRaw()); + return Promise.resolve(); +}; + +BlockDB.prototype.readBlock = function readBlock(hash) { + return this.db.get(this.layout.b(hash)); +}; + +BlockDB.prototype.readBlockEntry = function readBlockEntry(entry) { + return this.readBlock(entry.hash); +}; + +BlockDB.prototype.removeBlock = function removeBlock(hash) { + this.chaindb.del(this.layout.b(hash)); + return Promise.resolve(); +}; + +BlockDB.prototype.pruneBlock = function pruneBlock(hash) { + this.chaindb.del(this.layout.b(hash)); + return Promise.resolve(); +}; + +BlockDB.prototype.pruneBlockEntry = function pruneBlockEntry(entry) { + return this.pruneBlock(entry.hash); +}; + +/* + * Expose + */ + +module.exports = BlockDB; diff --git a/lib/chain/blockdb.js b/lib/chain/blockdb.js new file mode 100644 index 00000000..b6b901c6 --- /dev/null +++ b/lib/chain/blockdb.js @@ -0,0 +1,196 @@ +/*! + * blockdb.js - blockchain data management for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +var assert = require('assert'); +var co = require('../utils/co'); +var Flat = require('../db/flat'); +var LRU = require('../utils/lru'); +var FileEntry = Flat.FileEntry; + +/** + * BlockDB + * @constructor + */ + +function BlockDB(chaindb) { + this.chaindb = chaindb; + this.db = chaindb.db; + this.layout = chaindb.layout; + this.flat = new Flat(this.db); + this.cache = new LRU(8192); +} + +BlockDB.prototype.open = function open() { + return this.flat.open(); +}; + +BlockDB.prototype.close = function close() { + return this.flat.close(); +}; + +BlockDB.prototype.sync = co(function* sync() { + var entry = yield this.chaindb.getTip(); + var block, rollback; + + assert(entry); + + for (;;) { + try { + block = yield this.readBlock(entry.hash); + } catch (e) { + if (e.type !== 'ChecksumMismatch') + throw e; + block = null; + } + + if (block) + break; + + this.cache.remove(entry.hash); + + entry = yield entry.getPrevious(); + assert(entry); + + rollback = true; + } + + if (!rollback) + return; + + yield this.chaindb.reset(entry.hash, true); +}); + +BlockDB.prototype.getEntry = co(function* getEntry(hash) { + var key = hash; + var entry, data; + + if (typeof key !== 'string') + key = key.toString('hex'); + + entry = this.cache.get(key); + + if (entry) + return entry; + + data = yield this.db.get(this.layout.b(hash)); + + if (!data) + return; + + entry = FileEntry.fromRaw(data); + + this.cache.set(key, entry); + + return entry; +}); + +BlockDB.prototype.saveBlock = co(function* saveBlock(block) { + var hash = block.hash(); + var hex = block.hash('hex'); + var entry = yield this.flat.write(block.toRaw()); + + if (block.height === 0) + yield this.flat.sync(); + + this.cache.set(hex, entry); + + this.chaindb.put(this.layout.b(hash), entry.toRaw()); +}); + +BlockDB.prototype.readBlock = co(function* readBlock(hash) { + var entry = yield this.getEntry(hash); + + if (!entry) + return; + + return yield this.readBlockEntry(entry); +}); + +BlockDB.prototype.readBlockEntry = function readBlockEntry(entry) { + return this.flat.read(entry.index, entry.pos); +}; + +BlockDB.prototype.removeBlock = co(function* removeBlock(hash) { + var entry = yield this.getEntry(hash); + + if (!entry) + return; + + this.chaindb.del(this.layout.b(hash)); + + if (entry.pos === 0) + yield this.flat.remove(entry.index); +}); + +BlockDB.prototype.pruneBlock = co(function* pruneBlock(hash) { + var entry = yield this.getEntry(hash); + if (!entry) + return; + return yield this.pruneBlockEntry(hash, entry); +}); + +BlockDB.prototype.pruneBlockEntry = function pruneBlockEntry(hash, entry) { + var index = entry.index; + if (index === this.current.index) + index -= 1; + this.chaindb.del(this.layout.b(hash)); + return this.flat.remove(index); +}; + +/** + * Batch + * @constructor + */ + +function Batch(ffdb) { + this.ffdb = ffdb; + this.ops = []; +} + +Batch.prototype.put = function put(block) { + this.ops.push(new BatchOp(0, block)); +}; + +Batch.prototype.del = function del(hash) { + this.ops.push(new BatchOp(1, hash)); +}; + +Batch.prototype.write = co(function* write() { + var i, op; + + for (i = 0; i < this.ops.length; i++) { + op = this.ops[i]; + switch (op.type) { + case 0: + yield this.ffdb.saveBlock(op.data); + break; + case 1: + yield this.ffdb.removeBlock(op.data); + break; + default: + assert(false); + } + } +}); + +/** + * BatchOp + * @constructor + */ + +function BatchOp(type, data) { + this.type = type; + this.data = data; +} + +/* + * Expose + */ + +module.exports = BlockDB; diff --git a/lib/chain/chaindb.js b/lib/chain/chaindb.js index 5abd4006..705f665f 100644 --- a/lib/chain/chaindb.js +++ b/lib/chain/chaindb.js @@ -17,7 +17,7 @@ var co = require('../utils/co'); var Network = require('../protocol/network'); var CoinView = require('./coinview'); var Coins = require('./coins'); -var ldb = require('../db/ldb'); +var LDB = require('../db/ldb'); var LRU = require('../utils/lru'); var Block = require('../primitives/block'); var Coin = require('../primitives/coin'); @@ -27,6 +27,7 @@ var Address = require('../primitives/address'); var ChainEntry = require('./chainentry'); var U32 = utils.U32; var DUMMY = new Buffer([0]); +var BlockDB = require('./blockdb'); /* * Database Layout: @@ -173,17 +174,20 @@ function ChainDB(chain) { this.logger = chain.logger; this.network = chain.network; this.options = new ChainOptions(chain.options); + this.layout = layout; - this.db = ldb({ + this.db = LDB({ location: chain.options.location, db: chain.options.db, maxOpenFiles: chain.options.maxFiles, - compression: true, + compression: false, cacheSize: 16 << 20, writeBufferSize: 8 << 20, bufferKeys: !utils.isBrowser }); + this.blockdb = new BlockDB(this); + this.state = new ChainState(); this.pending = null; this.current = null; @@ -198,6 +202,7 @@ function ChainDB(chain) { // We want to keep the last 5 blocks of unspents in memory. this.coinWindow = 25 << 20; + this.coinWindow = 100 << 20; this.coinCache = new LRU.Nil(); this.cacheHash = new LRU(this.cacheWindow); @@ -228,9 +233,10 @@ ChainDB.prototype._open = co(function* open() { this.logger.info('Starting chain load.'); yield this.db.open(); - yield this.db.checkVersion('V', 1); + yield this.blockdb.open(); + state = yield this.getState(); options = yield this.getOptions(); @@ -246,6 +252,7 @@ ChainDB.prototype._open = co(function* open() { if (state) { // Grab the chainstate if we have one. this.state = state; + yield this.blockdb.sync(); } else { // Otherwise write the genesis block. // (We assume this database is fresh). @@ -271,9 +278,10 @@ ChainDB.prototype._open = co(function* open() { * @returns {Promise} */ -ChainDB.prototype._close = function close() { - return this.db.close(); -}; +ChainDB.prototype._close = co(function* close() { + yield this.blockdb.close(); + yield this.db.close(); +}); /** * Start a batch. @@ -841,7 +849,7 @@ ChainDB.prototype.getBlock = co(function* getBlock(hash) { if (!item.hash) return; - data = yield this.db.get(layout.b(item.hash)); + data = yield this.blockdb.readBlock(hash); if (!data) return; @@ -869,7 +877,7 @@ ChainDB.prototype.getRawBlock = co(function* getRawBlock(block) { if (!hash) return; - return yield this.db.get(layout.b(hash)); + return yield this.blockdb.readBlock(hash); }); /** @@ -1369,7 +1377,7 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) { * @returns {Promise} */ -ChainDB.prototype.reset = co(function* reset(block) { +ChainDB.prototype.reset = co(function* reset(block, noData) { var entry = yield this.get(block); var tip; @@ -1417,11 +1425,15 @@ ChainDB.prototype.reset = co(function* reset(block) { this.del(layout.n(tip.prevBlock)); // Disconnect and remove block data. - try { - yield this.removeBlock(tip.hash); - } catch (e) { - this.drop(); - throw e; + if (!noData) { + try { + yield this.removeBlock(tip.hash); + } catch (e) { + this.drop(); + throw e; + } + } else { + this.del(layout.b(tip.hash)); } // Revert chain state to previous tip. @@ -1489,7 +1501,7 @@ ChainDB.prototype._removeChain = co(function* removeChain(hash) { this.del(layout.p(tip.hash)); this.del(layout.h(tip.hash)); this.del(layout.e(tip.hash)); - this.del(layout.b(tip.hash)); + yield this.blockdb.removeBlock(tip.hash); // Queue up hash to be removed // on successful write. @@ -1514,7 +1526,7 @@ ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) { // Write actual block data (this may be // better suited to flat files in the future). - this.put(layout.b(block.hash()), block.toRaw()); + yield this.blockdb.saveBlock(block); if (!view) return; @@ -1540,7 +1552,7 @@ ChainDB.prototype.removeBlock = co(function* removeBlock(hash) { if (!block) throw new Error('Block not found.'); - this.del(layout.b(block.hash())); + yield this.blockdb.removeBlock(block.hash()); return yield this.disconnectBlock(block); }); @@ -1761,7 +1773,7 @@ ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) { if (!hash) return; - this.del(layout.b(hash)); + yield this.blockdb.removeBlock(hash); this.del(layout.u(hash)); }); diff --git a/lib/db/flat.js b/lib/db/flat.js new file mode 100644 index 00000000..f910528c --- /dev/null +++ b/lib/db/flat.js @@ -0,0 +1,391 @@ +'use strict'; + +var utils = require('../utils/utils'); +var co = require('../utils/co'); +var Locker = require('../utils/locker'); +var path = require('path'); +var fs = require('fs'); +var promisify = co.promisify; +var fsExists = promisify(fs.exists, fs); +var fsMkdir = promisify(fs.mkdir, fs); +var fsReaddir = promisify(fs.readdir, fs); +var fsOpen = promisify(fs.open, fs); +var fsStat = promisify(fs.stat, fs); +var fsFstat = promisify(fs.fstat, fs); +var fsWrite = promisify(fs.write, fs); +var fsRead = promisify(fs.read, fs); +var fsClose = promisify(fs.close, fs); +var fsFtruncate = promisify(fs.ftruncate, fs); +var fsFsync = promisify(fs.fsync, fs); +var fsUnlink = promisify(fs.unlink, fs); +var fsExists; +var assert = utils.assert; +var murmur3 = require('../utils/murmur3'); + +var MAX_SIZE = 512 << 20; +var MAX_FILES = 64; +var MAX_ENTRY = 12 << 20; + +/** + * Flat + * @constructor + */ + +function Flat(db) { + if (!(this instanceof Flat)) + return new Flat(db); + + this.dir = path.resolve(db.location, '..'); + this.dir = path.resolve(this.dir, 'blocks'); + this.locker = new Locker(); + + this.fileIndex = -1; + this.current = null; + this.files = {}; + this.openFiles = []; + this.indexes = []; +} + +Flat.prototype.hash = function hash(data) { + return murmur3(data, 0xdeedbeef); +}; + +Flat.prototype.open = co(function* open() { + var index = -1; + var i, list, name; + + if (!(yield fsExists(this.dir))) + yield fsMkdir(this.dir, 493); + + list = yield fsReaddir(this.dir); + + for (i = 0; i < list.length; i++) { + name = list[i]; + + if (!/^\d{10}$/.test(name)) + continue; + + name = parseInt(name, 10); + + utils.binaryInsert(this.indexes, name, cmp); + + if (name > index) + index = name; + } + + if (index === -1) { + yield this.allocate(); + return; + } + + this.fileIndex = index; + this.current = yield this.openFile(index); +}); + +Flat.prototype.close = co(function* close() { + var unlock = yield this.locker.lock(); + try { + return yield this._close(); + } finally { + unlock(); + } +}); + +Flat.prototype._close = co(function* close() { + var i, index, file; + + for (i = this.openFiles.length - 1; i >= 0; i--) { + index = this.openFiles[i]; + file = this.files[index]; + assert(file); + yield this._closeFile(file.index); + } + + assert(this.current === null); + assert(this.openFiles.length === 0); + + this.fileIndex = -1; + this.indexes.length = 0; +}); + +Flat.prototype.name = function name(index) { + return path.resolve(this.dir, utils.pad32(index)); +}; + +Flat.prototype.openFile = co(function* openFile(index) { + var unlock = yield this.locker.lock(); + try { + return yield this._openFile(index); + } finally { + unlock(); + } +}); + +Flat.prototype._openFile = co(function* _openFile(index) { + var file = this.files[index]; + var name, fd, stat; + + if (file) + return file; + + name = this.name(index); + + fd = yield fsOpen(name, 'a+'); + stat = yield fsFstat(fd); + + file = new File(fd, index, stat.size); + + this.files[index] = file; + utils.binaryInsert(this.openFiles, index, cmp); + + yield this.evict(index); + + return file; +}); + +Flat.prototype.closeFile = co(function* closeFile(index) { + var unlock = yield this.locker.lock(); + try { + assert(index !== this.current.index); + return yield this._closeFile(index); + } finally { + unlock(); + } +}); + +Flat.prototype._closeFile = co(function* _closeFile(index) { + var file = this.files[index]; + var result; + + if (!file) + return; + + yield fsClose(file.fd); + + result = utils.binaryRemove(this.openFiles, index, cmp); + assert(result); + + delete this.files[index]; + + if (file === this.current) + this.current = null; +}); + +Flat.prototype.remove = co(function* remove(index) { + var unlock = yield this.locker.lock(); + try { + return yield this._remove(index); + } finally { + unlock(); + } +}); + +Flat.prototype._remove = co(function* remove(index) { + var result; + + assert(index != null); + + if (!this.files[index]) + return; + + yield this._closeFile(index); + yield fsUnlink(this.name(index)); + + result = utils.binaryRemove(this.indexes, index, cmp); + assert(result); + + if (!this.current) { + index = this.indexes[this.indexes.length - 1]; + assert(index != null); + this.current = yield this._openFile(index); + } +}); + +Flat.prototype.allocate = co(function* allocate() { + var index = this.fileIndex + 1; + var fd = yield fsOpen(this.name(index), 'a+'); + var file = new File(fd, index, 0); + + this.files[index] = file; + this.current = file; + this.fileIndex++; + + utils.binaryInsert(this.openFiles, index, cmp); + yield this.evict(-1); +}); + +Flat.prototype.evict = co(function* evict(not) { + var i = 0; + var index, file; + + if (this.openFiles.length <= MAX_FILES) + return; + + for (;;) { + assert(i < this.openFiles.length); + + index = this.openFiles[i]; + + if (this.current) { + if (index !== not && index !== this.current.index) + break; + } + + i++; + } + + index = this.openFiles[i]; + file = this.files[index]; + assert(file); + + yield fsClose(file.fd); + + this.openFiles.splice(i, 1); + delete this.files[index]; +}); + +Flat.prototype.write = co(function* write(data) { + var unlock = yield this.locker.lock(); + try { + return yield this._write(data); + } finally { + unlock(); + } +}); + +Flat.prototype._write = co(function* write(data) { + var pos, fd, size, chk; + var buf = new Buffer(4); + var len = 4 + data.length + 4; + + if (data.length > MAX_ENTRY) + throw new Error('Size too large.'); + + if (this.current.pos + len > MAX_SIZE) { + yield this.sync(); + yield this.allocate(); + } + + pos = this.current.pos; + fd = this.current.fd; + + buf.writeUInt32LE(data.length, 0, true); + yield fsWrite(fd, buf, 0, 4, pos); + + yield fsWrite(fd, data, 0, data.length, pos + 4); + + buf.writeUInt32LE(this.hash(data), 0, true); + yield fsWrite(fd, buf, 0, 4, pos + 4 + data.length); + + this.current.pos += len; + + return new FileEntry(this.current.index, pos, data.length); +}); + +Flat.prototype.read = co(function* read(index, offset) { + var file = yield this.openFile(index); + var buf = new Buffer(4); + var size, data, chk, err; + + if (offset + 8 > file.pos) + throw new Error('Read is out of bounds.'); + + yield fsRead(file.fd, buf, 0, 4, offset); + size = buf.readUInt32LE(0, true); + + if (size > MAX_ENTRY) + throw new Error('Size too large.'); + + if (offset + 4 + size + 4 > file.pos) + throw new Error('Read is out of bounds.'); + + data = new Buffer(size); + yield fsRead(file.fd, data, 0, data.length, offset + 4); + + yield fsRead(file.fd, buf, 0, 4, offset + 4 + data.length); + chk = buf.readUInt32LE(0, true); + + if (this.hash(data) !== chk) { + err = new Error('Checksum mismatch.'); + err.type = 'ChecksumMismatch'; + throw err; + } + + return data; +}); + +Flat.prototype.sync = co(function* sync() { + yield fsFsync(this.current.fd); +}); + +/* + * File + * @constructor + */ + +function File(fd, index, pos) { + this.fd = fd; + this.index = index; + this.pos = pos; +} + +/* + * FileEntry + * @constructor + */ + +function FileEntry(index, offset, size) { + this.index = index; + this.offset = offset; + this.size = size; +} + +FileEntry.prototype.toRaw = function toRaw() { + var data = new Buffer(12); + data.writeUInt32LE(this.index, 0, true); + data.writeUInt32LE(this.offset, 4, true); + data.writeUInt32LE(this.size, 8, true); + return data; +}; + +FileEntry.fromRaw = function fromRaw(data) { + var entry = new FileEntry(0, 0, 0); + entry.index = data.readUInt32LE(0, true); + entry.offset = data.readUInt32LE(4, true); + entry.size = data.readUInt32LE(8, true); + return entry; +}; + +/* + * Helpers + */ + +function cmp(a, b) { + return a - b; +} + +fsExists = co(function* fsExists(name) { + var stat; + + try { + stat = yield fsStat(name); + } catch (e) { + if (e.code === 'ENOENT') + return false; + throw e; + } + + if (!stat.isDirectory()) + throw new Error('File is not a directory.'); + + return true; +}); + +/* + * Expose + */ + +exports = Flat; +exports.FileEntry = FileEntry; + +module.exports = exports; diff --git a/package.json b/package.json index 9bd8d306..feca8e8b 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,8 @@ "./lib/http/rpcclient": "./browser/empty.js", "./lib/http/server": "./browser/empty.js", "./lib/http/wallet": "./browser/empty.js", + "./lib/db/flat": "./browser/empty.js", + "./lib/chain/blockdb": "./lib/chain/blockdb-browser.js", "fs": "./browser/empty.js", "crypto": "./browser/empty.js", "child_process": "./browser/empty.js",