From 229be344fc885a3fe73e89abe3bc0f6092babf8e Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Tue, 22 Nov 2016 20:49:17 -0800
Subject: [PATCH] chaindb: persistent versionbits state caches.

---
 lib/blockchain/chain.js          |  29 ++--
 lib/blockchain/chaindb.js        | 248 +++++++++++++++++++++++++++++++
 lib/blockchain/layout-browser.js |   8 +
 lib/blockchain/layout.js         |  13 ++
 lib/protocol/networks.js         |  22 +--
 lib/utils/co.js                  |   1 +
 lib/utils/encoding.js            |  12 ++
 lib/utils/util.js                |  41 +++++
 lib/wallet/client.js             |   6 -
 test/chain-test.js               |   7 +-
 10 files changed, 349 insertions(+), 38 deletions(-)

diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js
index 3f3c6430..9d39de52 100644
--- a/lib/blockchain/chain.js
+++ b/lib/blockchain/chain.js
@@ -87,7 +87,6 @@ function Chain(options) {
   this.height = -1;
   this.synced = false;
   this.state = new DeploymentState();
-  this.stateCache = {};
   this._time = util.hrtime();
 
   this.orphan = {
@@ -109,14 +108,6 @@ util.inherits(Chain, AsyncObject);
 
 Chain.prototype._init = function _init() {
   var self = this;
-  var keys = Object.keys(this.network.deployments);
-  var i, id;
-
-  // Setup state caches.
-  for (i = 0; i < keys.length; i++) {
-    id = keys[i];
-    this.stateCache[id] = {};
-  }
 
   this.locker.on('purge', function(total, size) {
     self.logger.warning('Warning: %dmb of pending objects. Purging.', util.mb(size));
@@ -548,9 +539,6 @@ Chain.prototype.setDeploymentState = function setDeploymentState(state) {
   if (!this.state.hasCSV() && state.hasCSV())
     this.logger.warning('CSV has been activated.');
 
-  if (!this.state.hasWitness() && state.hasWitness())
-    this.logger.warning('Segwit has been activated.');
-
   this.state = state;
 };
 
@@ -1951,8 +1939,8 @@ Chain.prototype.getState = co(function* getState(prev, id) {
   var period = this.network.minerWindow;
   var threshold = this.network.activationThreshold;
   var deployment = this.network.deployments[id];
-  var stateCache = this.stateCache[id];
   var thresholdStates = constants.thresholdStates;
+  var bit = deployment.bit;
   var timeStart, timeTimeout, compute, height;
   var i, entry, count, state, block, medianTime;
 
@@ -1979,8 +1967,8 @@ Chain.prototype.getState = co(function* getState(prev, id) {
   state = thresholdStates.DEFINED;
 
   while (entry) {
-    if (stateCache[entry.hash] != null) {
-      state = stateCache[entry.hash];
+    if (this.db.stateCache.get(bit, entry) !== -1) {
+      state = this.db.stateCache.get(bit, entry);
       break;
     }
 
@@ -1988,14 +1976,13 @@ Chain.prototype.getState = co(function* getState(prev, id) {
 
     if (medianTime < timeStart) {
       state = thresholdStates.DEFINED;
-      stateCache[entry.hash] = state;
+      this.db.stateCache.set(bit, entry, state);
       break;
     }
 
     compute.push(entry);
 
     height = entry.height - period;
-
     entry = yield entry.getAncestorByHeight(height);
   }
 
@@ -2053,7 +2040,7 @@ Chain.prototype.getState = co(function* getState(prev, id) {
         break;
     }
 
-    stateCache[entry.hash] = state;
+    this.db.stateCache.set(bit, entry, state);
   }
 
   return state;
@@ -2297,8 +2284,9 @@ DeploymentState.prototype.hasWitness = function hasWitness() {
   return (this.flags & constants.flags.VERIFY_WITNESS) !== 0;
 };
 
-/*
+/**
  * LockTimes
+ * @constructor
  */
 
 function LockTimes(height, time) {
@@ -2306,8 +2294,9 @@ function LockTimes(height, time) {
   this.time = time;
 }
 
-/*
+/**
  * ContextResult
+ * @constructor
  */
 
 function ContextResult(view, state) {
diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js
index bde2482c..9f76168d 100644
--- a/lib/blockchain/chaindb.js
+++ b/lib/blockchain/chaindb.js
@@ -28,6 +28,7 @@ var Outpoint = require('../primitives/outpoint');
 var TX = require('../primitives/tx');
 var Address = require('../primitives/address');
 var ChainEntry = require('./chainentry');
+var U8 = encoding.U8;
 var U32 = encoding.U32;
 var DUMMY = new Buffer([0]);
 
@@ -68,6 +69,7 @@ function ChainDB(chain) {
     bufferKeys: !util.isBrowser
   });
 
+  this.stateCache = new StateCache(chain.network);
   this.state = new ChainState();
   this.pending = null;
   this.current = null;
@@ -127,9 +129,15 @@ ChainDB.prototype._open = co(function* open() {
     yield this.saveOptions();
   }
 
+  // Verify deployment params have not changed.
+  yield this.verifyDeployments();
+
   if (state) {
     // Grab the chainstate if we have one.
     this.state = state;
+
+    // Load state caches.
+    this.stateCache = yield this.getStateCache();
   } else {
     // Otherwise write the genesis block.
     // (We assume this database is fresh).
@@ -226,6 +234,7 @@ ChainDB.prototype.drop = function drop() {
   this.coinCache.drop();
   this.cacheHash.drop();
   this.cacheHeight.drop();
+  this.stateCache.drop();
 
   batch.clear();
 };
@@ -264,6 +273,7 @@ ChainDB.prototype.commit = co(function* commit() {
   this.coinCache.commit();
   this.cacheHash.commit();
   this.cacheHeight.commit();
+  this.stateCache.commit();
 });
 
 /**
@@ -773,6 +783,85 @@ ChainDB.prototype.getFullBlock = co(function* getFullBlock(hash) {
   return block;
 });
 
+/**
+ * Get state caches.
+ * @returns {Promise} - Returns {@link StateCache}.
+ */
+
+ChainDB.prototype.getStateCache = co(function* getStateCache() {
+  var stateCache = new StateCache(this.network);
+  var i, items, item;
+
+  items = yield this.db.range({
+    gte: layout.s(0, constants.ZERO_HASH),
+    lte: layout.s(255, constants.MAX_HASH),
+    values: true
+  });
+
+  for (i = 0; i < items.length; i++) {
+    item = items[i];
+    stateCache.setRaw(item.key, item.value);
+  }
+
+  return stateCache;
+});
+
+/**
+ * Invalidate state cache.
+ * @private
+ * @returns {Promise}
+ */
+
+ChainDB.prototype.invalidateCache = co(function* invalidateCache(bit, batch) {
+  var i, keys, key;
+
+  keys = yield this.db.keys({
+    gte: layout.s(bit, constants.ZERO_HASH),
+    lte: layout.s(bit, constants.MAX_HASH)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+});
+
+/**
+ * Potentially invalidate state cache.
+ * @returns {Promise}
+ */
+
+ChainDB.prototype.verifyDeployments = co(function* verifyDeployments() {
+  var expected = this.stateCache.toDeployments();
+  var current = yield this.db.get(layout.v);
+  var i, invalid, bit, batch;
+
+  if (!current) {
+    yield this.db.put(layout.v, expected);
+    return true;
+  }
+
+  invalid = this.stateCache.verifyDeployments(current);
+
+  if (invalid.length === 0)
+    return true;
+
+  batch = this.db.batch();
+
+  for (i = 0; i < invalid.length; i++) {
+    bit = invalid[i];
+    this.logger.warning('Versionbit deployment params modified.');
+    this.logger.warning('Invalidating cache for bit %d.', bit);
+    yield this.invalidateCache(bit, batch);
+  }
+
+  batch.put(layout.v, expected);
+
+  yield batch.write();
+
+  return false;
+});
+
 /**
  * Fill a transaction with coins (only unspents).
  * @param {TX} tx
@@ -1124,6 +1213,9 @@ ChainDB.prototype._save = co(function* save(entry, block, view) {
   this.del(layout.p(entry.prevBlock));
   this.put(layout.p(hash), DUMMY);
 
+  // Update state caches.
+  this.saveUpdates();
+
   if (!view) {
     // Save block data.
     yield this.saveBlock(block);
@@ -1185,6 +1277,9 @@ ChainDB.prototype._reconnect = co(function* reconnect(entry, block, view) {
   // Re-insert into cache.
   this.cacheHash.push(entry.hash, entry);
 
+  // Update state caches.
+  this.saveUpdates();
+
   // Connect inputs.
   yield this.connectBlock(block, view);
 
@@ -1232,6 +1327,9 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
   this.del(layout.H(entry.height));
   this.cacheHeight.unpush(entry.height);
 
+  // Update state caches.
+  this.saveUpdates();
+
   block = yield this.getBlock(entry.hash);
 
   if (!block)
@@ -1246,6 +1344,24 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
   return block;
 });
 
+/**
+ * Save state cache updates.
+ * @private
+ */
+
+ChainDB.prototype.saveUpdates = function saveUpdates() {
+  var updates = this.stateCache.updates;
+  var i, update;
+
+  if (updates.length > 0)
+    this.logger.info('Saving %d state cache updates.', updates.length);
+
+  for (i = 0; i < updates.length; i++) {
+    update = updates[i];
+    this.put(layout.s(update.bit, update.hash), update.toRaw());
+  }
+};
+
 /**
  * Reset the chain to a height or hash. Useful for replaying
  * the blockchain download for SPV.
@@ -1879,6 +1995,138 @@ ChainState.fromRaw = function fromRaw(data) {
   return state;
 };
 
+/**
+ * StateCache
+ * @constructor
+ */
+
+function StateCache(network) {
+  this.deployments = network.deployments;
+  this.bits = {};
+  this.cache = {};
+  this.updates = [];
+  this._init();
+}
+
+StateCache.prototype._init = function _init() {
+  var keys = Object.keys(this.deployments);
+  var i, key, deployment, bit;
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    deployment = this.deployments[key];
+    bit = deployment.bit;
+    this.cache[bit] = {};
+    this.bits[bit] = deployment;
+  }
+};
+
+StateCache.prototype.toDeployments = function toDeployments() {
+  var p = new BufferWriter();
+  var keys = Object.keys(this.deployments);
+  var i, key, deployment;
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    deployment = this.deployments[key];
+    p.writeU8(deployment.bit);
+    p.writeU32(deployment.startTime);
+    p.writeU32(deployment.timeout);
+  }
+
+  return p.render();
+};
+
+StateCache.prototype.verifyDeployments = function verifyDeployments(raw) {
+  var p = new BufferReader(raw);
+  var invalid = [];
+  var deployment, bit, start, timeout;
+
+  while (p.left()) {
+    bit = p.readU8();
+    start = p.readU32();
+    timeout = p.readU32();
+    deployment = this.bits[bit];
+
+    if (deployment
+        && start === deployment.startTime
+        && timeout === deployment.timeout) {
+      continue;
+    }
+
+    invalid.push(bit);
+  }
+
+  return invalid;
+};
+
+StateCache.prototype.set = function set(bit, entry, state) {
+  var cache = this.cache[bit];
+
+  assert(cache);
+
+  if (cache[entry.hash] !== state) {
+    cache[entry.hash] = state;
+    this.updates.push(new CacheUpdate(bit, entry.hash, state));
+  }
+};
+
+StateCache.prototype.get = function get(bit, entry) {
+  var cache = this.cache[bit];
+  var state;
+
+  assert(cache);
+
+  state = cache[entry.hash];
+
+  if (state == null)
+    return -1;
+
+  return state;
+};
+
+StateCache.prototype.commit = function commit() {
+  this.updates.length = 0;
+};
+
+StateCache.prototype.drop = function drop() {
+  var i, update, cache;
+
+  for (i = 0; i < this.updates.length; i++) {
+    update = this.updates[i];
+    cache = this.cache[update.bit];
+    assert(cache);
+    delete cache[update.hash];
+  }
+
+  this.updates.length = 0;
+};
+
+StateCache.prototype.setRaw = function setRaw(key, value) {
+  var pair = layout.ss(key);
+  var bit = pair[0];
+  var hash = pair[1];
+  var state = value[0];
+  var cache = this.cache[bit];
+  assert(cache);
+  cache[hash] = state;
+};
+
+/**
+ * CacheUpdate
+ * @constructor
+ */
+
+function CacheUpdate(bit, hash, state) {
+  this.bit = bit;
+  this.hash = hash;
+  this.state = state;
+}
+
+CacheUpdate.prototype.toRaw = function toRaw() {
+  return U8(this.state);
+};
+
 /*
  * Helpers
  */
diff --git a/lib/blockchain/layout-browser.js b/lib/blockchain/layout-browser.js
index b6983efc..9b7fb06b 100644
--- a/lib/blockchain/layout-browser.js
+++ b/lib/blockchain/layout-browser.js
@@ -7,11 +7,13 @@
 'use strict';
 
 var util = require('../utils/util');
+var pad8 = util.pad8;
 var pad32 = util.pad32;
 
 var layout = {
   R: 'R',
   O: 'O',
+  v: 'v',
   e: function e(hash) {
     return 'e' + hex(hash);
   },
@@ -39,6 +41,12 @@ var layout = {
   u: function u(hash) {
     return 'u' + hex(hash);
   },
+  s: function s(bit, hash) {
+    return 's' + pad8(bit) + hex(hash);
+  },
+  ss: function ss(key) {
+    return [+key.slice(1, 4), key.slice(4, 36)];
+  },
   T: function T(address, hash) {
     address = hex(address);
 
diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js
index 76e81266..659b743f 100644
--- a/lib/blockchain/layout.js
+++ b/lib/blockchain/layout.js
@@ -19,6 +19,8 @@
  *   t[hash] -> extended tx
  *   c[hash] -> coins
  *   u[hash] -> undo coins
+ *   s[bit][hash] -> versionbits state
+ *   v -> versionbits deployments
  *   T[addr-hash][hash] -> dummy (tx by address)
  *   C[addr-hash][hash][index] -> dummy (coin by address)
  *   W+T[witaddr-hash][hash] -> dummy (tx by address)
@@ -28,6 +30,7 @@
 var layout = {
   R: new Buffer([0x52]),
   O: new Buffer([0x4f]),
+  v: new Buffer([0x76]),
   e: function e(hash) {
     return pair(0x65, hash);
   },
@@ -55,6 +58,16 @@ var layout = {
   u: function u(hash) {
     return pair(0x75, hash);
   },
+  s: function s(bit, hash) {
+    var key = new Buffer(1 + 1 + 32);
+    key[0] = 0x73;
+    key[1] = bit;
+    write(key, hash, 2);
+    return key;
+  },
+  ss: function ss(key) {
+    return [key[1], key.toString('hex', 2, 34)];
+  },
   T: function T(address, hash) {
     var len = address.length;
     var key;
diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js
index 2d02b552..b6060dba 100644
--- a/lib/protocol/networks.js
+++ b/lib/protocol/networks.js
@@ -353,8 +353,8 @@ main.deployments = {
   },
   mast: {
     bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
     force: false
   }
 };
@@ -581,8 +581,8 @@ testnet.deployments = {
   },
   mast: {
     bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
     force: false
   }
 };
@@ -711,25 +711,25 @@ regtest.deployments = {
   testdummy: {
     bit: 28,
     startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
     force: true
   },
   csv: {
     bit: 0,
     startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
     force: true
   },
   witness: {
     bit: 1,
     startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
     force: false
   },
   mast: {
     bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
     force: false
   }
 };
@@ -1139,8 +1139,8 @@ simnet.deployments = {
   },
   mast: {
     bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
     force: false
   }
 };
diff --git a/lib/utils/co.js b/lib/utils/co.js
index b35013ed..ed2f0072 100644
--- a/lib/utils/co.js
+++ b/lib/utils/co.js
@@ -249,6 +249,7 @@ function call(func) {
   for (i = 1; i < arguments.length; i++)
     args[i - 1] = arguments[i];
 
+  /* jshint validthis:true */
   return _call(this, func, args);
 }
 
diff --git a/lib/utils/encoding.js b/lib/utils/encoding.js
index 30b2f227..bf6dca23 100644
--- a/lib/utils/encoding.js
+++ b/lib/utils/encoding.js
@@ -640,6 +640,18 @@ encoding.sizeVarint2 = function sizeVarint2(num) {
   return size;
 };
 
+/**
+ * Serialize number as a u8.
+ * @param {Number} num
+ * @returns {Buffer}
+ */
+
+encoding.U8 = function U8(num) {
+  var data = new Buffer(1);
+  data[0] = num >>> 0;
+  return data;
+};
+
 /**
  * Serialize number as a u32le.
  * @param {Number} num
diff --git a/lib/utils/util.js b/lib/utils/util.js
index 4b226f93..91f9d1fb 100644
--- a/lib/utils/util.js
+++ b/lib/utils/util.js
@@ -662,6 +662,27 @@ util.indexOf = function indexOf(obj, data) {
   return -1;
 };
 
+/**
+ * Convert a number to a padded uint8
+ * string (3 digits in decimal).
+ * @param {Number} num
+ * @returns {String} Padded number.
+ */
+
+util.pad8 = function pad8(num) {
+  assert(num >= 0);
+  num = num + '';
+  switch (num.length) {
+    case 1:
+      return '00' + num;
+    case 2:
+      return '0' + num;
+    case 3:
+      return num;
+  }
+  assert(false);
+};
+
 /**
  * Convert a number to a padded uint32
  * string (10 digits in decimal).
@@ -698,6 +719,26 @@ util.pad32 = function pad32(num) {
   }
 };
 
+/**
+ * Convert a number to a padded uint8
+ * string (2 digits in hex).
+ * @param {Number} num
+ * @returns {String} Padded number.
+ */
+
+util.hex8 = function hex8(num) {
+  assert(num >= 0);
+  num = num.toString(16);
+  switch (num.length) {
+    case 1:
+      return '0' + num;
+    case 2:
+      return num;
+    default:
+      assert(false);
+  }
+};
+
 /**
  * Convert a number to a padded uint32
  * string (8 digits in hex).
diff --git a/lib/wallet/client.js b/lib/wallet/client.js
index ea4b864f..4da76e90 100644
--- a/lib/wallet/client.js
+++ b/lib/wallet/client.js
@@ -414,12 +414,6 @@ function parseTX(data) {
   return TX.fromRaw(data, 'hex');
 }
 
-function toHex(data) {
-  if (typeof data !== 'string')
-    return data.toString('hex');
-  return data;
-}
-
 function BlockResult(entry, txs) {
   this.entry = entry;
   this.txs = txs;
diff --git a/test/chain-test.js b/test/chain-test.js
index dcc30e22..9ed082ce 100644
--- a/test/chain-test.js
+++ b/test/chain-test.js
@@ -263,7 +263,7 @@ describe('Chain', function() {
   }));
 
   it('should activate csv', cob(function* () {
-    var i, block, prev, state;
+    var i, block, prev, state, cache;
 
     prev = yield chain.tip.getPrevious();
     state = yield chain.getState(prev, 'csv');
@@ -293,6 +293,11 @@ describe('Chain', function() {
 
     assert(chain.height === 432);
     assert(chain.state.hasCSV());
+
+    cache = yield chain.db.getStateCache();
+    assert.deepEqual(cache, chain.db.stateCache);
+    assert.equal(chain.db.stateCache.updates.length, 0);
+    assert(yield chain.db.verifyDeployments());
   }));
 
   var mineCSV = co(function* mineCSV(tx) {