From d371fc5d141d46d222f0ed953ef6a3507fd9741f Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 22 Oct 2016 11:23:06 -0700 Subject: [PATCH] chain: add chain options and merkle block serialization. --- lib/chain/browser.js | 1 + lib/chain/chain.js | 5 +- lib/chain/chaindb.js | 195 ++++++++++++++++++++++++++++------ lib/primitives/merkleblock.js | 82 +++++++++++++- 4 files changed, 246 insertions(+), 37 deletions(-) diff --git a/lib/chain/browser.js b/lib/chain/browser.js index c33c3bba..97ff74d7 100644 --- a/lib/chain/browser.js +++ b/lib/chain/browser.js @@ -11,6 +11,7 @@ var pad32 = utils.pad32; var layout = { R: 'R', + O: 'O', e: function e(hash) { return 'e' + hex(hash); }, diff --git a/lib/chain/chain.js b/lib/chain/chain.js index 722e6364..35c3ba49 100644 --- a/lib/chain/chain.js +++ b/lib/chain/chain.js @@ -829,10 +829,7 @@ Chain.prototype.reconnect = co(function* reconnect(entry) { var block = yield this.db.getBlock(entry.hash); var prev, view; - if (!block) { - assert(this.options.spv); - block = entry.toHeaders(); - } + assert(block); prev = yield entry.getPrevious(); assert(prev); diff --git a/lib/chain/chaindb.js b/lib/chain/chaindb.js index c51c418f..1ec96c5a 100644 --- a/lib/chain/chaindb.js +++ b/lib/chain/chaindb.js @@ -19,6 +19,7 @@ var Coins = require('./coins'); var ldb = require('../db/ldb'); var LRU = require('../utils/lru'); var Block = require('../primitives/block'); +var MerkleBlock = require('../primitives/merkleblock'); var Coin = require('../primitives/coin'); var TX = require('../primitives/tx'); var Address = require('../primitives/address'); @@ -29,6 +30,7 @@ var DUMMY = new Buffer([0]); /* * Database Layout: * R -> tip hash + * O -> chain options * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -45,6 +47,7 @@ var DUMMY = new Buffer([0]); var layout = { R: new Buffer([0x52]), + O: new Buffer([0x4f]), e: function e(hash) { return pair(0x65, hash); }, @@ -162,22 +165,20 @@ function ChainDB(chain) { AsyncObject.call(this); this.chain = chain; - this.options = chain.options; + this.options = new ChainOptions(chain.options); this.logger = chain.logger; this.network = chain.network; this.db = ldb({ - location: this.options.location, - db: this.options.db, - maxOpenFiles: this.options.maxFiles, + location: chain.options.location, + db: chain.options.db, + maxOpenFiles: chain.options.maxFiles, compression: true, cacheSize: 16 << 20, writeBufferSize: 8 << 20, bufferKeys: !utils.isBrowser }); - this.keepBlocks = this.options.keepBlocks || 288; - this.prune = !!this.options.prune; this.state = new ChainState(); this.pending = null; this.current = null; @@ -197,7 +198,7 @@ function ChainDB(chain) { this.cacheHash = new LRU(this.cacheWindow); this.cacheHeight = new LRU(this.cacheWindow); - if (this.options.coinCache) + if (chain.options.coinCache) this.coinCache = new LRU(this.coinWindow, getSize); } @@ -217,7 +218,7 @@ ChainDB.layout = layout; */ ChainDB.prototype._open = co(function* open() { - var state, block, entry; + var state, options, block, entry; this.logger.info('Starting chain load.'); @@ -226,6 +227,10 @@ ChainDB.prototype._open = co(function* open() { yield this.db.checkVersion('V', 1); state = yield this.getState(); + options = yield this.getOptions(); + + if (options) + this.options.verify(options); if (state) { // Grab the chainstate if we have one. @@ -563,7 +568,7 @@ ChainDB.prototype.getTip = function getTip() { /** * Retrieve the tip entry from the tip record. - * @returns {Promise} - Returns {@link ChainEntry}. + * @returns {Promise} - Returns {@link ChainState}. */ ChainDB.prototype.getState = co(function* getState() { @@ -575,6 +580,20 @@ ChainDB.prototype.getState = co(function* getState() { return ChainState.fromRaw(data); }); +/** + * Retrieve the tip entry from the tip record. + * @returns {Promise} - Returns {@link ChainOptions}. + */ + +ChainDB.prototype.getOptions = co(function* getOptions() { + var data = yield this.db.get(layout.O); + + if (!data) + return; + + return ChainOptions.fromRaw(data); +}); + /** * Get the _next_ block hash (does not work by height). * @param {Hash} hash @@ -779,6 +798,12 @@ ChainDB.prototype.getBlock = co(function* getBlock(hash) { if (!data) return; + if (this.options.spv) { + block = MerkleBlock.fromFull(data); + block.setHeight(item.height); + return block; + } + block = Block.fromRaw(data); block.setHeight(item.height); @@ -809,6 +834,9 @@ ChainDB.prototype.getRawBlock = co(function* getRawBlock(block) { ChainDB.prototype.getFullBlock = co(function* getFullBlock(hash) { var block = yield this.getBlock(hash); + if (this.options.spv) + return block; + if (!block) return; @@ -1047,9 +1075,6 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) { var total = 0; var i, j, entry, hashes, hash, tx, txs, block; - if (this.options.spv) - throw new Error('Cannot scan in spv mode.'); - if (start == null) start = this.network.genesis.hash; @@ -1237,11 +1262,6 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) { this.cacheHeight.remove(entry.height); - if (this.options.spv) { - this.put(layout.R, this.pending.commit(entry.prevBlock)); - return entry.toHeaders(); - } - block = yield this.getBlock(entry.hash); if (!block) @@ -1271,7 +1291,7 @@ ChainDB.prototype.reset = co(function* reset(block) { if (!(yield entry.isMainChain())) throw new Error('Cannot reset on alternate chain.'); - if (this.prune) + if (this.options.prune) throw new Error('Cannot reset when pruned.'); tip = yield this.getTip(); @@ -1320,8 +1340,11 @@ ChainDB.prototype.reset = co(function* reset(block) { */ ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) { - if (this.options.spv) + if (this.options.spv) { + this.put(layout.b(block.hash()), block.toFull()); + yield this.pruneBlock(block); return; + } this.put(layout.b(block.hash()), block.toRaw()); @@ -1339,12 +1362,7 @@ ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) { */ ChainDB.prototype.removeBlock = co(function* removeBlock(hash) { - var block; - - if (this.options.spv) - return; - - block = yield this.getBlock(hash); + var block = yield this.getBlock(hash); if (!block) throw new Error('Block not found.'); @@ -1554,13 +1572,10 @@ ChainDB.prototype.disconnectBlock = co(function* disconnectBlock(block) { ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) { var height, hash; - if (this.options.spv) + if (!this.options.prune && !this.options.spv) return; - if (!this.prune) - return; - - height = block.height - this.keepBlocks; + height = block.height - this.options.keepBlocks; if (height <= this.network.block.pruneAfterHeight) return; @@ -1574,6 +1589,126 @@ ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) { this.del(layout.u(hash)); }); +/** + * Chain Options + * @constructor + */ + +function ChainOptions(options) { + if (!(this instanceof ChainOptions)) + return new ChainOptions(options); + + this.spv = false; + this.witness = false; + this.prune = false; + this.indexTX = false; + this.indexAddress = false; + this.keepBlocks = 288; + + if (options) + this.fromOptions(options); +} + +ChainOptions.prototype.fromOptions = function fromOptions(options) { + if (options.spv != null) { + assert(typeof options.spv === 'boolean'); + this.spv = options.spv; + } + + if (options.prune != null) { + assert(typeof options.prune === 'boolean'); + this.prune = options.prune; + } + + if (options.keepBlocks != null) { + assert(utils.isUInt32(options.keepBlocks)); + this.keepBlocks = options.keepBlocks; + } + + return this; +}; + +ChainOptions.fromOptions = function fromOptions(data) { + return new ChainOptions().fromOptions(data); +}; + +ChainOptions.prototype.verify = function verify(options) { + if (this.spv && !options.spv) + throw new Error('Cannot retroactively enable SPV.'); + + if (!this.spv && options.spv) + throw new Error('Cannot retroactively disable SPV.'); + + if (this.witness && !options.witness) + throw new Error('Cannot retroactively enable witness.'); + + if (!this.witness && options.witness) + throw new Error('Cannot retroactively disable witness.'); + + if (this.prune && !options.prune) + throw new Error('Cannot retroactively prune.'); + + if (!this.prune && options.prune) + throw new Error('Cannot retroactively unprune.'); + + if (this.indexTX && !options.indexTX) + throw new Error('Cannot retroactively enable TX indexing.'); + + if (!this.indexTX && options.indexTX) + throw new Error('Cannot retroactively disable TX indexing.'); + + if (this.indexAddress && !options.indexAddress) + throw new Error('Cannot retroactively enable address indexing.'); + + if (!this.indexAddress && options.indexAddress) + throw new Error('Cannot retroactively disable address indexing.'); + + if (this.keepBlocks !== options.keepBlocks) + throw new Error('Cannot change keepBlocks option retroactively.'); +}; + +ChainOptions.prototype.toRaw = function toRaw() { + var p = new BufferWriter(); + var flags = 0; + + if (this.spv) + flags |= 1 << 0; + + if (this.witness) + flags |= 1 << 1; + + if (this.prune) + flags |= 1 << 2; + + if (this.indexTX) + flags |= 1 << 3; + + if (this.indexAddress) + flags |= 1 << 4; + + p.writeU32(flags); + p.writeU32(this.keepBlocks); + p.writeU32(0); + + return p.render(); +}; + +ChainOptions.prototype.fromRaw = function fromRaw(data) { + var p = new BufferReader(data); + var flags = p.readU32(); + this.spv = (flags & 1) !== 0; + this.witness = (flags & 2) !== 0; + this.prune = (flags & 4) !== 0; + this.indexTX = (flags & 8) !== 0; + this.indexAddress = (flags & 16) !== 0; + this.keepBlocks = p.readU32(); + return this; +}; + +ChainOptions.fromRaw = function fromRaw(data) { + return new ChainOptions().fromRaw(data); +}; + /** * Chain State * @constructor diff --git a/lib/primitives/merkleblock.js b/lib/primitives/merkleblock.js index 0f6a47fc..44f3b6e8 100644 --- a/lib/primitives/merkleblock.js +++ b/lib/primitives/merkleblock.js @@ -377,7 +377,7 @@ MerkleBlock.prototype.toRaw = function toRaw(writer) { MerkleBlock.prototype.fromRaw = function fromRaw(data) { var p = BufferReader(data); - var i, hashCount; + var i, count; this.version = p.readU32(); this.prevBlock = p.readHash('hex'); @@ -387,9 +387,9 @@ MerkleBlock.prototype.fromRaw = function fromRaw(data) { this.nonce = p.readU32(); this.totalTX = p.readU32(); - hashCount = p.readVarint(); + count = p.readVarint(); - for (i = 0; i < hashCount; i++) + for (i = 0; i < count; i++) this.hashes.push(p.readHash()); this.flags = p.readVarBytes(); @@ -410,6 +410,82 @@ MerkleBlock.fromRaw = function fromRaw(data, enc) { return new MerkleBlock().fromRaw(data); }; +/** + * Serialize the merkleblock. + * @param {String?} enc - Encoding, can be `'hex'` or null. + * @returns {Buffer|String} + */ + +MerkleBlock.prototype.toFull = function toFull(writer) { + var p = BufferWriter(writer); + var i, tx; + + this.toRaw(p); + + p.writeVarint(this.txs.length); + + for (i = 0; i < this.txs.length; i++) { + tx = this.txs[i]; + index = tx.index; + if (index === -1) + index = 0x7fffffff; + p.writeU32(index); + tx.toRaw(p); + } + + if (!writer) + p = p.render(); + + return p; +}; + +/** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + */ + +MerkleBlock.prototype.fromFull = function fromFull(data) { + var p = BufferReader(data); + var i, count, index, tx, hash, txid; + + this.fromRaw(p); + + this._validPartial = true; + + count = p.readVarint(); + + for (i = 0; i < count; i++) { + index = p.readU32(); + if (index === 0x7fffffff) + index = -1; + tx = TX.fromRaw(p); + hash = tx.hash(); + txid = tx.hash('hex'); + tx.setBlock(this, index); + this.txs.push(tx); + if (index !== -1) { + this.matches.push(hash); + this.map[txid] = index; + } + } + + return this; +}; + +/** + * Instantiate a merkleblock from a serialized data. + * @param {Buffer} data + * @param {String?} enc - Encoding, can be `'hex'` or null. + * @returns {MerkleBlock} + */ + +MerkleBlock.fromFull = function fromFull(data, enc) { + if (typeof data === 'string') + data = new Buffer(data, enc); + return new MerkleBlock().fromFull(data); +}; + /** * Convert the block to an object suitable * for JSON serialization. Note that the hashes