chain: add chain options and merkle block serialization.

lib/chain/browser.js

@@ -11,6 +11,7 @@ var pad32 = utils.pad32;
 
 var layout = {
   R: 'R',
+  O: 'O',
   e: function e(hash) {
     return 'e' + hex(hash);
   },

lib/chain/chain.js

@@ -829,10 +829,7 @@ Chain.prototype.reconnect = co(function* reconnect(entry) {
   var block = yield this.db.getBlock(entry.hash);
   var prev, view;
 
-  if (!block) {
+  assert(block);
-    assert(this.options.spv);
-    block = entry.toHeaders();
-  }
 
   prev = yield entry.getPrevious();
   assert(prev);

lib/chain/chaindb.js

@@ -19,6 +19,7 @@ var Coins = require('./coins');
 var ldb = require('../db/ldb');
 var LRU = require('../utils/lru');
 var Block = require('../primitives/block');
+var MerkleBlock = require('../primitives/merkleblock');
 var Coin = require('../primitives/coin');
 var TX = require('../primitives/tx');
 var Address = require('../primitives/address');
@@ -29,6 +30,7 @@ var DUMMY = new Buffer([0]);
 /*
  * Database Layout:
  *   R -> tip hash
+ *   O -> chain options
  *   e[hash] -> entry
  *   h[hash] -> height
  *   H[height] -> hash
@@ -45,6 +47,7 @@ var DUMMY = new Buffer([0]);
 
 var layout = {
   R: new Buffer([0x52]),
+  O: new Buffer([0x4f]),
   e: function e(hash) {
     return pair(0x65, hash);
   },
@@ -162,22 +165,20 @@ function ChainDB(chain) {
   AsyncObject.call(this);
 
   this.chain = chain;
-  this.options = chain.options;
+  this.options = new ChainOptions(chain.options);
   this.logger = chain.logger;
   this.network = chain.network;
 
   this.db = ldb({
-    location: this.options.location,
+    location: chain.options.location,
-    db: this.options.db,
+    db: chain.options.db,
-    maxOpenFiles: this.options.maxFiles,
+    maxOpenFiles: chain.options.maxFiles,
     compression: true,
     cacheSize: 16 << 20,
     writeBufferSize: 8 << 20,
     bufferKeys: !utils.isBrowser
   });
 
-  this.keepBlocks = this.options.keepBlocks || 288;
-  this.prune = !!this.options.prune;
   this.state = new ChainState();
   this.pending = null;
   this.current = null;
@@ -197,7 +198,7 @@ function ChainDB(chain) {
   this.cacheHash = new LRU(this.cacheWindow);
   this.cacheHeight = new LRU(this.cacheWindow);
 
-  if (this.options.coinCache)
+  if (chain.options.coinCache)
     this.coinCache = new LRU(this.coinWindow, getSize);
 }
 
@@ -217,7 +218,7 @@ ChainDB.layout = layout;
  */
 
 ChainDB.prototype._open = co(function* open() {
-  var state, block, entry;
+  var state, options, block, entry;
 
   this.logger.info('Starting chain load.');
 
@@ -226,6 +227,10 @@ ChainDB.prototype._open = co(function* open() {
   yield this.db.checkVersion('V', 1);
 
   state = yield this.getState();
+  options = yield this.getOptions();
+
+  if (options)
+    this.options.verify(options);
 
   if (state) {
     // Grab the chainstate if we have one.
@@ -563,7 +568,7 @@ ChainDB.prototype.getTip = function getTip() {
 
 /**
  * Retrieve the tip entry from the tip record.
- * @returns {Promise} - Returns {@link ChainEntry}.
+ * @returns {Promise} - Returns {@link ChainState}.
  */
 
 ChainDB.prototype.getState = co(function* getState() {
@@ -575,6 +580,20 @@ ChainDB.prototype.getState = co(function* getState() {
   return ChainState.fromRaw(data);
 });
 
+/**
+ * Retrieve the tip entry from the tip record.
+ * @returns {Promise} - Returns {@link ChainOptions}.
+ */
+
+ChainDB.prototype.getOptions = co(function* getOptions() {
+  var data = yield this.db.get(layout.O);
+
+  if (!data)
+    return;
+
+  return ChainOptions.fromRaw(data);
+});
+
 /**
  * Get the _next_ block hash (does not work by height).
  * @param {Hash} hash
@@ -779,6 +798,12 @@ ChainDB.prototype.getBlock = co(function* getBlock(hash) {
   if (!data)
     return;
 
+  if (this.options.spv) {
+    block = MerkleBlock.fromFull(data);
+    block.setHeight(item.height);
+    return block;
+  }
+
   block = Block.fromRaw(data);
   block.setHeight(item.height);
 
@@ -809,6 +834,9 @@ ChainDB.prototype.getRawBlock = co(function* getRawBlock(block) {
 ChainDB.prototype.getFullBlock = co(function* getFullBlock(hash) {
   var block = yield this.getBlock(hash);
 
+  if (this.options.spv)
+    return block;
+
   if (!block)
     return;
 
@@ -1047,9 +1075,6 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) {
   var total = 0;
   var i, j, entry, hashes, hash, tx, txs, block;
 
-  if (this.options.spv)
-    throw new Error('Cannot scan in spv mode.');
-
   if (start == null)
     start = this.network.genesis.hash;
 
@@ -1237,11 +1262,6 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
 
   this.cacheHeight.remove(entry.height);
 
-  if (this.options.spv) {
-    this.put(layout.R, this.pending.commit(entry.prevBlock));
-    return entry.toHeaders();
-  }
-
   block = yield this.getBlock(entry.hash);
 
   if (!block)
@@ -1271,7 +1291,7 @@ ChainDB.prototype.reset = co(function* reset(block) {
   if (!(yield entry.isMainChain()))
     throw new Error('Cannot reset on alternate chain.');
 
-  if (this.prune)
+  if (this.options.prune)
     throw new Error('Cannot reset when pruned.');
 
   tip = yield this.getTip();
@@ -1320,8 +1340,11 @@ ChainDB.prototype.reset = co(function* reset(block) {
  */
 
 ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) {
-  if (this.options.spv)
+  if (this.options.spv) {
+    this.put(layout.b(block.hash()), block.toFull());
+    yield this.pruneBlock(block);
     return;
+  }
 
   this.put(layout.b(block.hash()), block.toRaw());
 
@@ -1339,12 +1362,7 @@ ChainDB.prototype.saveBlock = co(function* saveBlock(block, view) {
  */
 
 ChainDB.prototype.removeBlock = co(function* removeBlock(hash) {
-  var block;
+  var block = yield this.getBlock(hash);
-
-  if (this.options.spv)
-    return;
-
-  block = yield this.getBlock(hash);
 
   if (!block)
     throw new Error('Block not found.');
@@ -1554,13 +1572,10 @@ ChainDB.prototype.disconnectBlock = co(function* disconnectBlock(block) {
 ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) {
   var height, hash;
 
-  if (this.options.spv)
+  if (!this.options.prune && !this.options.spv)
     return;
 
-  if (!this.prune)
+  height = block.height - this.options.keepBlocks;
-    return;
-
-  height = block.height - this.keepBlocks;
 
   if (height <= this.network.block.pruneAfterHeight)
     return;
@@ -1574,6 +1589,126 @@ ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) {
   this.del(layout.u(hash));
 });
 
+/**
+ * Chain Options
+ * @constructor
+ */
+
+function ChainOptions(options) {
+  if (!(this instanceof ChainOptions))
+    return new ChainOptions(options);
+
+  this.spv = false;
+  this.witness = false;
+  this.prune = false;
+  this.indexTX = false;
+  this.indexAddress = false;
+  this.keepBlocks = 288;
+
+  if (options)
+    this.fromOptions(options);
+}
+
+ChainOptions.prototype.fromOptions = function fromOptions(options) {
+  if (options.spv != null) {
+    assert(typeof options.spv === 'boolean');
+    this.spv = options.spv;
+  }
+
+  if (options.prune != null) {
+    assert(typeof options.prune === 'boolean');
+    this.prune = options.prune;
+  }
+
+  if (options.keepBlocks != null) {
+    assert(utils.isUInt32(options.keepBlocks));
+    this.keepBlocks = options.keepBlocks;
+  }
+
+  return this;
+};
+
+ChainOptions.fromOptions = function fromOptions(data) {
+  return new ChainOptions().fromOptions(data);
+};
+
+ChainOptions.prototype.verify = function verify(options) {
+  if (this.spv && !options.spv)
+    throw new Error('Cannot retroactively enable SPV.');
+
+  if (!this.spv && options.spv)
+    throw new Error('Cannot retroactively disable SPV.');
+
+  if (this.witness && !options.witness)
+    throw new Error('Cannot retroactively enable witness.');
+
+  if (!this.witness && options.witness)
+    throw new Error('Cannot retroactively disable witness.');
+
+  if (this.prune && !options.prune)
+    throw new Error('Cannot retroactively prune.');
+
+  if (!this.prune && options.prune)
+    throw new Error('Cannot retroactively unprune.');
+
+  if (this.indexTX && !options.indexTX)
+    throw new Error('Cannot retroactively enable TX indexing.');
+
+  if (!this.indexTX && options.indexTX)
+    throw new Error('Cannot retroactively disable TX indexing.');
+
+  if (this.indexAddress && !options.indexAddress)
+    throw new Error('Cannot retroactively enable address indexing.');
+
+  if (!this.indexAddress && options.indexAddress)
+    throw new Error('Cannot retroactively disable address indexing.');
+
+  if (this.keepBlocks !== options.keepBlocks)
+    throw new Error('Cannot change keepBlocks option retroactively.');
+};
+
+ChainOptions.prototype.toRaw = function toRaw() {
+  var p = new BufferWriter();
+  var flags = 0;
+
+  if (this.spv)
+    flags |= 1 << 0;
+
+  if (this.witness)
+    flags |= 1 << 1;
+
+  if (this.prune)
+    flags |= 1 << 2;
+
+  if (this.indexTX)
+    flags |= 1 << 3;
+
+  if (this.indexAddress)
+    flags |= 1 << 4;
+
+  p.writeU32(flags);
+  p.writeU32(this.keepBlocks);
+  p.writeU32(0);
+
+  return p.render();
+};
+
+ChainOptions.prototype.fromRaw = function fromRaw(data) {
+  var p = new BufferReader(data);
+  var flags = p.readU32();
+  this.spv = (flags & 1) !== 0;
+  this.witness = (flags & 2) !== 0;
+  this.prune = (flags & 4) !== 0;
+  this.indexTX = (flags & 8) !== 0;
+  this.indexAddress = (flags & 16) !== 0;
+  this.keepBlocks = p.readU32();
+  return this;
+};
+
+ChainOptions.fromRaw = function fromRaw(data) {
+  return new ChainOptions().fromRaw(data);
+};
+
 /**
  * Chain State
  * @constructor

lib/primitives/merkleblock.js

@@ -377,7 +377,7 @@ MerkleBlock.prototype.toRaw = function toRaw(writer) {
 
 MerkleBlock.prototype.fromRaw = function fromRaw(data) {
   var p = BufferReader(data);
-  var i, hashCount;
+  var i, count;
 
   this.version = p.readU32();
   this.prevBlock = p.readHash('hex');
@@ -387,9 +387,9 @@ MerkleBlock.prototype.fromRaw = function fromRaw(data) {
   this.nonce = p.readU32();
   this.totalTX = p.readU32();
 
-  hashCount = p.readVarint();
+  count = p.readVarint();
 
-  for (i = 0; i < hashCount; i++)
+  for (i = 0; i < count; i++)
     this.hashes.push(p.readHash());
 
   this.flags = p.readVarBytes();
@@ -410,6 +410,82 @@ MerkleBlock.fromRaw = function fromRaw(data, enc) {
   return new MerkleBlock().fromRaw(data);
 };
 
+/**
+ * Serialize the merkleblock.
+ * @param {String?} enc - Encoding, can be `'hex'` or null.
+ * @returns {Buffer|String}
+ */
+
+MerkleBlock.prototype.toFull = function toFull(writer) {
+  var p = BufferWriter(writer);
+  var i, tx;
+
+  this.toRaw(p);
+
+  p.writeVarint(this.txs.length);
+
+  for (i = 0; i < this.txs.length; i++) {
+    tx = this.txs[i];
+    index = tx.index;
+    if (index === -1)
+      index = 0x7fffffff;
+    p.writeU32(index);
+    tx.toRaw(p);
+  }
+
+  if (!writer)
+    p = p.render();
+
+  return p;
+};
+
+/**
+ * Inject properties from serialized data.
+ * @private
+ * @param {Buffer} data
+ */
+
+MerkleBlock.prototype.fromFull = function fromFull(data) {
+  var p = BufferReader(data);
+  var i, count, index, tx, hash, txid;
+
+  this.fromRaw(p);
+
+  this._validPartial = true;
+
+  count = p.readVarint();
+
+  for (i = 0; i < count; i++) {
+    index = p.readU32();
+    if (index === 0x7fffffff)
+      index = -1;
+    tx = TX.fromRaw(p);
+    hash = tx.hash();
+    txid = tx.hash('hex');
+    tx.setBlock(this, index);
+    this.txs.push(tx);
+    if (index !== -1) {
+      this.matches.push(hash);
+      this.map[txid] = index;
+    }
+  }
+
+  return this;
+};
+
+/**
+ * Instantiate a merkleblock from a serialized data.
+ * @param {Buffer} data
+ * @param {String?} enc - Encoding, can be `'hex'` or null.
+ * @returns {MerkleBlock}
+ */
+
+MerkleBlock.fromFull = function fromFull(data, enc) {
+  if (typeof data === 'string')
+    data = new Buffer(data, enc);
+  return new MerkleBlock().fromFull(data);
+};
+
 /**
  * Convert the block to an object suitable
  * for JSON serialization. Note that the hashes