From c111d673e724ecc1b9fb230999a358c22a82cb83 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 20 Jan 2016 17:40:51 -0800 Subject: [PATCH] blockchain db. --- lib/bcoin/block.js | 2 +- lib/bcoin/chain.js | 510 +++++++++++++++++++++++++-------------------- 2 files changed, 286 insertions(+), 226 deletions(-) diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js index e7ea44e6..06c741b7 100644 --- a/lib/bcoin/block.js +++ b/lib/bcoin/block.js @@ -101,7 +101,7 @@ Block.prototype.abbr = function abbr() { return this._raw.slice(); var res = new Array(80); - utils.writeU32(res, this.version, 0); + utils.write32(res, this.version, 0); utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4); utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36); utils.writeU32(res, this.ts, 68); diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 091aa532..73f66f9a 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -9,6 +9,7 @@ var EventEmitter = require('events').EventEmitter; var bcoin = require('../bcoin'); var bn = require('bn.js'); +var fs = require('fs'); var constants = bcoin.protocol.constants; var network = bcoin.protocol.network; var utils = bcoin.utils; @@ -28,6 +29,7 @@ function Chain(options) { this.prefix = 'bt/chain/'; this.storage = this.options.storage; this.strict = this.options.strict || false; + this.db = new ChainDB(this); if (this.options.debug) bcoin.debug = this.options.debug; @@ -42,12 +44,8 @@ function Chain(options) { }; this.index = { - entries: [], - // Get hash by height - hashes: [], - // Get height by hash heights: {}, - count: 0, + count: this.db.count(), lastTs: 0 }; @@ -61,9 +59,11 @@ function Chain(options) { { hash: network.genesis.hash, version: network.genesis.version, - // prevBlock: network.genesis.prevBlock, + prevBlock: network.genesis.prevBlock, + merkleRoot: network.genesis.merkleRoot, ts: network.genesis.ts, bits: network.genesis.bits, + nonce: network.genesis.nonce, height: 0 } ] @@ -76,17 +76,12 @@ function Chain(options) { else if (network.type === 'testnet') this.fromJSON(require('./protocol/preload-test-full')); this.resetHeight(+process.env.BCOIN_START_HEIGHT); - } else { - if (+process.env.BCOIN_NO_COMPACT !== 1) { - if (!this.options.fullNode) - this.fromJSON(network.preload); - } } - this.tip = this.index.entries[this.index.entries.length - 1]; + this.tip = this.db.get(this.index.count - 1); // Last TS after preload, needed for fill percent - this.index.lastTs = this.index.entries[this.index.entries.length - 1].ts; + this.index.lastTs = this.tip.ts; Chain.global = this; @@ -159,6 +154,9 @@ Chain.prototype._init = function _init() { }); }; +Chain.prototype.getEntry = function getEntry(height) { +}; + Chain.prototype._addIndex = function _addIndex(entry, save) { var self = this; @@ -169,7 +167,8 @@ Chain.prototype._addIndex = function _addIndex(entry, save) { } // Duplicate height - if (this.index.hashes[entry.height] === entry.hash) + var existing = this.db.get(entry.height); + if (existing && existing.hash === entry.hash) return Chain.codes.unchanged; // Fork at checkpoint @@ -186,16 +185,14 @@ Chain.prototype._addIndex = function _addIndex(entry, save) { } } - this.index.entries[entry.height] = entry; - this.index.hashes[entry.height] = entry.hash; + this.db.save(entry); this.index.heights[entry.hash] = entry.height; this.index.count++; - this.tip = this.index.entries[this.index.entries.length - 1]; - this.emit('tip', this.tip); + if (!this.tip || entry.height > this.tip.height) + this.tip = entry; - if (save) - this._save(entry.hash, entry); + this.emit('tip', this.tip); return Chain.codes.okay; }; @@ -219,13 +216,8 @@ Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) { Chain.prototype.resetHeight = function resetHeight(height) { var self = this; - var ahead = this.index.entries.slice(height + 1); - assert(height < this.index.entries.length); - - // Nothing to do. - if (height === this.index.entries.length - 1) - return; + assert(height < this.index.count); // Reset the orphan map completely. There may // have been some orphans on a forked chain we @@ -235,27 +227,14 @@ Chain.prototype.resetHeight = function resetHeight(height) { this.orphan.count = 0; this.orphan.size = 0; - // Rebuild the index from our new (lower) height. - this.index.entries.length = height + 1; + for (var i = height + 1; height < this.index.count; i++) { + var existing = this.db.get(i); + this.db.del(i); + delete this.index.heights[existing.hash]; + } - this.index.heights = this.index.entries.reduce(function(out, entry) { - if (!self.options.fullNode) { - if (!entry) - return out; - } - out[entry.hash] = entry.height; - return out; - }, {}); - - this.index.hashes.length = height + 1; - - if (!this.options.fullNode) - this.index.count -= this._count(ahead); - else - this.index.count = height + 1; - - // Set and emit our new (old) tip. - this.tip = this.index.entries[this.index.entries.length - 1]; + this.tip = this.db.get(height); + this.index.count = height + 1; this.emit('tip', this.tip); // The lastTs is supposed to be the last ts @@ -264,17 +243,8 @@ Chain.prototype.resetHeight = function resetHeight(height) { // be higher. Reset it if necessary. this.index.lastTs = Math.min( this.index.lastTs, - this.index.entries[this.index.entries.length - 1].ts + this.tip.ts ); - - // Delete all the blocks now above us. - ahead.forEach(function(entry) { - if (!self.options.fullNode) { - if (!entry) - return; - } - self._delete(entry.hash); - }); }; Chain.prototype.resetTime = function resetTime(ts) { @@ -349,15 +319,18 @@ Chain.prototype.add = function add(block, peer) { entry = new ChainBlock(this, { hash: hash, version: block.version, - // prevBlock: prevHash, + prevBlock: prevHash, + merkleRoot: block.merkleRoot, ts: block.ts, bits: block.bits, + nonce: block.nonce, height: prevHeight + 1 }); // Add entry if we do not have it (or if // there is another entry at its height) - if (this.index.hashes[entry.height] !== hash) { + var existing = this.db.get(entry.height); + if (!existing || existing.hash !== hash) { assert(this.index.heights[entry.hash] == null); // A valid block with an already existing @@ -365,17 +338,18 @@ Chain.prototype.add = function add(block, peer) { // don't store by hash so we can't compare // chainworks. We reset the chain, find a // new peer, and wait to see who wins. - if (this.index.hashes[entry.height]) { + if (existing) { // The tip has more chainwork, it is a // higher height than the entry. This is // not an alternate tip. Ignore it. + if (0) if (this.tip.chainwork.cmp(entry.chainwork) > 0) { code = Chain.codes.unchanged; break; } // Get _our_ tip as opposed to // the attempted alternate tip. - tip = this.index.entries[entry.height]; + tip = existing; // The block has equal chainwork (an // alternate tip). Reset the chain, find // a new peer, and wait to see who wins. @@ -466,13 +440,6 @@ Chain.prototype.add = function add(block, peer) { this.orphan.size = 0; } - // Potentially compact the chain here. A - // full chain is not necessary for spv. - // if (!this.options.fullNode) { - // if (this.size() > 100000) - // this.compact(); - // } - if (code !== Chain.codes.okay) { if (!(this.options.multiplePeers && code === Chain.codes.newOrphan)) utils.debug('Chain Error: %s', Chain.msg(code)); @@ -492,7 +459,9 @@ Chain.prototype.has = function has(hash) { }; Chain.prototype.byHeight = function byHeight(height) { - return this.index.entries[height] || null; + if (height == null) + return; + return this.db.get(height); }; Chain.prototype.byHash = function byHash(hash) { @@ -505,13 +474,10 @@ Chain.prototype.byHash = function byHash(hash) { }; Chain.prototype.byTime = function byTime(ts) { - for (var i = this.index.entries.length - 1; i >= 0; i--) { - if (!this.options.fullNode) { - if (!this.index.entries[i]) - continue; - } - if (ts >= this.index.entries[i].ts) - return this.index.entries[i]; + for (var i = this.index.count - 1; i >= 0; i--) { + var existing = this.db.get(i); + if (ts >= existing.ts) + return existing; } return null; }; @@ -540,11 +506,11 @@ Chain.prototype.getOrphan = function getOrphan(hash) { }; Chain.prototype.getTip = function getTip() { - return this.index.entries[this.index.entries.length - 1]; + return this.tip; }; Chain.prototype.isFull = function isFull() { - var last = this.index.entries[this.index.entries.length - 1].ts; + var last = this.tip.ts; var delta = utils.now() - last; return delta < 40 * 60; }; @@ -564,18 +530,15 @@ Chain.prototype.hashRange = function hashRange(start, end) { if (!start || !end) return []; - hashes = this.index.hashes.slice(start.height, end.height + 1); - - if (!this.options.fullNode) - hashes = this._filter(hashes); + for (var i = start.height; i < end.height + 1; i++) + hashes.push(this.db.get(i).hash); return hashes; }; Chain.prototype.locatorHashes = function locatorHashes(start) { - var chain = this.index.hashes; var hashes = []; - var top = chain.length - 1; + var top = this.height(); var step = 1; var i; @@ -595,22 +558,23 @@ Chain.prototype.locatorHashes = function locatorHashes(start) { // is our tip. This is useful for getheaders // when not using headers-first. hashes.push(start); - top = chain.length - 1; + top = this.index.count - 1; } } else if (typeof start === 'number') { top = start; } - assert(chain[top]); + // assert(chain[top]); i = top; for (;;) { - if (chain[i]) - hashes.push(chain[i]); + var existing = this.db.get(i); + if (existing) + hashes.push(existing.hash); i = i - step; if (i <= 0) { if (i + step !== 0) - hashes.push(chain[0]); + hashes.push(this.db.get(0).hash); break; } if (hashes.length >= 10) @@ -694,12 +658,7 @@ Chain.prototype.target = function target(last, block) { } // Back 2 weeks - // first = last; - // for (i = 0; first && i < network.powDiffInterval - 1; i++) - // first = first.prev; - - // Back 2 weeks - first = this.index.entries[last.height - (network.powDiffInterval - 1)]; + first = this.db.get(last.height - (network.powDiffInterval - 1)); assert(first); @@ -731,117 +690,9 @@ Chain.prototype.retarget = function retarget(last, first) { return utils.toCompact(target); }; -Chain.prototype.compact = function compact(keep) { - var entries; - - if (+process.env.BCOIN_NO_COMPACT === 1) - return; - - entries = this._compact(keep); - - this.index.entries = {}; - this.index.hashes = []; - this.index.heights = {}; - this.index.count = 0; - - json.entries.forEach(function(entry) { - this._addIndex(ChainBlock.fromJSON(this, entry)); - }, this); -}; - -Chain.prototype._compact = function _compact(keep) { - var entries; - var last, first, delta1, delta2, delta3, lastTs, lastHeight; - var i, ts, delta, hdelta; - - if (+process.env.BCOIN_NO_COMPACT === 1) - return this.index.entries; - - entries = this._filter(this.index.entries); - - if (!keep) - keep = network.powDiffInterval + 10; - - // Keep only last ~2016 consequent blocks, dilate others at: - // 7 day range for blocks before 2013 - // 12 hour for blocks before 2014 - // 6 hour for blocks in 2014 and after it - // (or at maximum 250 block range) - last = entries.slice(-keep); - - first = []; - - delta1 = 7 * 24 * 3600; - delta2 = 12 * 3600; - delta3 = 6 * 3600; - - lastTs = 0; - lastHeight = -1000; - - for (i = 0; i < entries.length - keep; i++) { - ts = entries[i].ts; - - delta = ts < 1356984000 - ? delta1 - : ts < 1388520000 ? delta2 : delta3; - - hdelta = entries[i].height - lastHeight; - - if (ts - lastTs < delta && hdelta < 250) - continue; - - lastTs = ts; - lastHeight = entries[i].height; - first.push(this.index.entries[i]); - } - - return first.concat(last); -}; - -Chain.prototype._save = function(hash, obj) { - var self = this; - - if (!this.storage) - return; - - this.storage.put(this.prefix + hash, obj.toJSON(), function(err) { - if (err) - self.emit('error', err); - }); -}; - -Chain.prototype._delete = function(hash) { - var self = this; - - if (!this.storage) - return; - - this.storage.del(this.prefix + hash, function(err) { - if (err) - self.emit('error', err); - }); -}; - -Chain.prototype._count = function(obj) { - for (var i = 0, c = 0; i < obj.length; i++) - if (obj[i]) - c++; - return c; -}; - -Chain.prototype._filter = function(obj) { - for (var i = 0, a = []; i < obj.length; i++) - if (obj[i]) - a.push(obj[i]); - return a; -}; - Chain.prototype.toJSON = function toJSON() { var entries = this.index.entries; - if (!this.options.fullNode) - entries = this._compact(); - return { v: 2, type: 'chain', @@ -858,10 +709,200 @@ Chain.prototype.fromJSON = function fromJSON(json) { assert.equal(json.network, network.type); json.entries.forEach(function(entry) { - this._addIndex(ChainBlock.fromJSON(this, entry)); + this._addIndex(ChainBlock.fromJSON(this, entry), true); }, this); +}; - assert(this.index.entries.length > 0); +/** + * ChainDB + */ + +var BLOCK_SIZE = 80; +// var BLOCK_SIZE = 144; + +function ChainDB(chain) { + this.file = process.env.HOME + '/bcoin-' + network.type + '.blockchain'; + try { + fs.unlinkSync(this.file); + } catch (e) { + } + var exists = false; + try { + fs.accessSync(file); + exists = true; + } catch (e) { + exists = false; + } + if (!exists) + fs.writeFileSync(this.file, new Buffer(0)); + this.fd = fs.openSync(this.file, 'r+'); + this.chain = chain; + this._buffer = []; + this._cache = {}; + this._bufferPool = {}; + this._nullBlock = new Buffer(BLOCK_SIZE); + this._nullBlock.fill(0); +} + +ChainDB.prototype.getBuffer = function(size) { + if (!this._bufferPool[size]) + this._bufferPool[size] = new Buffer(size); + return this._bufferPool[size]; +}; + +ChainDB.prototype.save = function save(entry) { + // Cache the past 2016 blocks in memory + this._cache[entry.height] = entry; + delete this._cache[entry.height - network.powDiffInterval]; + assert(Object.keys(this._cache).length < network.powDiffInterval + 1); + return this._write(new Buffer(entry.toRaw()), entry.height * BLOCK_SIZE); +}; + +ChainDB.prototype._write = function _write(data, offset) { + var size, bytes, index, hash; + var self = this; + + if (offset < 0 || offset == null) + return false; + + hash = offset; + size = data.length; + bytes = 0; + index = 0; + + // Something is already writing. Cancel it + // and synchronously write the data after + // it cancels. + if (this._buffer[hash]) { + assert(this._buffer[hash].data.length === data.length); + this._buffer[hash].data = data; + this._buffer[hash].queue.push([data, offset]); + return; + } + + // Speed up writes by doing them asynchronously + // and keeping the data to be written in memory. + this._buffer[hash] = { queue: [], data: data }; + + (function callee() { + fs.write(self.fd, data, index, size, offset, function(err, bytes) { + // Something tried to write here but couldn't. + // Synchronously write it and get it over with. + if (self._buffer[hash].queue.length) { + self._buffer[hash].queue.forEach(function(chunk) { + self._writeSync(chunk[0], chunk[1]); + }); + delete self._buffer[hash]; + return; + } + + index += bytes; + size -= bytes; + offset += bytes; + + if (index === data.length) { + delete self._buffer[hash]; + return; + } + + callee(); + }); + })(); + + return false; +}; + +ChainDB.prototype._writeSync = function _writeSync(data, offset) { + var size, bytes, index; + + if (offset < 0 || offset == null) + return false; + + size = data.length; + bytes = 0; + index = 0; + + while (bytes = fs.writeSync(this.fd, data, index, size, offset)) { + index += bytes; + size -= bytes; + offset += bytes; + if (index === data.length) + return true; + } + + return false; +}; + +ChainDB.prototype.del = function del(height) { + this._write(this._nullBlock, height * BLOCK_SIZE); + if (height * BLOCK_SIZE + BLOCK_SIZE === this.size()) { + while (this.isNull(height)) + height--; + fs.ftruncateSync(this.fd, height * BLOCK_SIZE + BLOCK_SIZE); + } + return true; +}; + +ChainDB.prototype.isNull = function isNull(height) { + var data = this._read(1, height * BLOCK_SIZE); + if (!data) + return false; + return utils.read32(data, 0) === 0; +}; + +ChainDB.prototype._read = function _read(size, offset) { + var data = this.getBuffer(size); + var bytes = 0; + var index = 0; + + if (offset < 0 || offset == null) + return; + + if (this._buffer[offset]) { + assert(this._buffer[offset].data.length >= size); + return this._buffer[offset].data.slice(0, size); + } + + try { + while (bytes = fs.readSync(this.fd, data, index, size, offset)) { + index += bytes; + size -= bytes; + offset += bytes; + if (index === data.length) { + if (utils.read32(data, 0) === 0) + return; + return data; + } + } + } catch (e) { + return; + } +}; + +ChainDB.prototype.size = function size() { + try { + return fs.statSync(this.file).size; + } catch (e) { + return 0; + } +}; + +ChainDB.prototype.count = function count() { + return this.size() / BLOCK_SIZE | 0; +}; + +ChainDB.prototype.get = function get(height) { + var data; + + if (this._cache[height]) + return this._cache[height]; + + data = this._read(BLOCK_SIZE, height * BLOCK_SIZE); + + if (!data) + return; + + return ChainBlock.fromRaw(this.chain, height, data); }; /** @@ -872,19 +913,22 @@ function ChainBlock(chain, data) { this.chain = chain; this.hash = data.hash; this.version = data.version; - // this.prevBlock = data.prevBlock; + this.prevBlock = data.prevBlock; + this.merkleRoot = data.merkleRoot; this.ts = data.ts; this.bits = data.bits; + this.nonce = data.nonce; this.height = data.height; - this.chainwork = this.getChainwork(); + this.chainwork = data.chainwork || new bn(0); + // this.chainwork = data.chainwork || this.getChainwork(); } ChainBlock.prototype.__defineGetter__('prev', function() { - return this.chain.index.entries[this.height - 1]; + return this.chain.db.get(this.height - 1); }); ChainBlock.prototype.__defineGetter__('next', function() { - return this.chain.index.entries[this.height + 1]; + return this.chain.db.get(this.height + 1); }); ChainBlock.prototype.__defineGetter__('proof', function() { @@ -939,36 +983,52 @@ ChainBlock.prototype.isSuperMajority = function(version, required) { }; ChainBlock.prototype.toJSON = function() { - // return [ - // this.hash, - // this.version, - // this.prevBlock, - // this.ts, - // this.bits, - // this.height - // }; return { hash: this.hash, version: this.version, - // prevBlock: this.prevBlock, + prevBlock: this.prevBlock, + merkleRoot: this.merkleRoot, ts: this.ts, bits: this.bits, + nonce: this.nonce, height: this.height }; }; ChainBlock.fromJSON = function(chain, json) { - // return new ChainBlock(chain, { - // hash: json[0], - // version: json[1], - // prevBlock: json[2], - // ts: json[3], - // bits: json[4], - // height: json[5] - // }); return new ChainBlock(chain, json); }; +ChainBlock.prototype.toRaw = function toRaw() { + var res = new Array(BLOCK_SIZE); + + utils.writeU32(res, this.version, 0); + utils.copy(utils.toArray(this.prevBlock, 'hex'), res, 4); + utils.copy(utils.toArray(this.merkleRoot, 'hex'), res, 36); + utils.writeU32(res, this.ts, 68); + utils.writeU32(res, this.bits, 72); + utils.writeU32(res, this.nonce, 76); + // utils.copy(utils.toArray(this.hash, 'hex'), res, 80); + // utils.copy(this.chainwork.toArray('be', 32), res, 112); + + return res; +}; + +ChainBlock.fromRaw = function fromRaw(chain, height, p) { + return new ChainBlock(chain, { + hash: utils.toHex(utils.dsha256(p.slice(0, 80))), + version: utils.read32(p, 0), + prevBlock: utils.toHex(utils.toArray(p.slice(4, 36))), + merkleRoot: utils.toHex(utils.toArray(p.slice(36, 68))), + ts: utils.readU32(p, 68), + bits: utils.readU32(p, 72), + nonce: utils.readU32(p, 76), + height: height, + // hash: utils.toHex(utils.toArray(p.slice(80, 112))), + // chainwork: new bn(p.slice(112, 144)) + }); +}; + /** * Expose */