chaindb: persistent versionbits state caches.

2016-11-22 20:49:17 -08:00 · 2016-11-22 20:49:17 -08:00 · 229be344fc
commit 229be344fc
parent 0530c8f80f
10 changed files with 349 additions and 38 deletions
--- a/lib/blockchain/chain.js
+++ b/lib/blockchain/chain.js
@ -87,7 +87,6 @@ function Chain(options) {
  this.height = -1;
  this.synced = false;
  this.state = new DeploymentState();
-  this.stateCache = {};
  this._time = util.hrtime();

  this.orphan = {
@ -109,14 +108,6 @@ util.inherits(Chain, AsyncObject);

 Chain.prototype._init = function _init() {
  var self = this;
-  var keys = Object.keys(this.network.deployments);
-  var i, id;
-
-  // Setup state caches.
-  for (i = 0; i < keys.length; i++) {
-    id = keys[i];
-    this.stateCache[id] = {};
-  }

  this.locker.on('purge', function(total, size) {
    self.logger.warning('Warning: %dmb of pending objects. Purging.', util.mb(size));
@ -548,9 +539,6 @@ Chain.prototype.setDeploymentState = function setDeploymentState(state) {
  if (!this.state.hasCSV() && state.hasCSV())
    this.logger.warning('CSV has been activated.');

-  if (!this.state.hasWitness() && state.hasWitness())
-    this.logger.warning('Segwit has been activated.');
-
  this.state = state;
 };

@ -1951,8 +1939,8 @@ Chain.prototype.getState = co(function* getState(prev, id) {
  var period = this.network.minerWindow;
  var threshold = this.network.activationThreshold;
  var deployment = this.network.deployments[id];
-  var stateCache = this.stateCache[id];
  var thresholdStates = constants.thresholdStates;
+  var bit = deployment.bit;
  var timeStart, timeTimeout, compute, height;
  var i, entry, count, state, block, medianTime;

@ -1979,8 +1967,8 @@ Chain.prototype.getState = co(function* getState(prev, id) {
  state = thresholdStates.DEFINED;

  while (entry) {
-    if (stateCache[entry.hash] != null) {
-      state = stateCache[entry.hash];
+    if (this.db.stateCache.get(bit, entry) !== -1) {
+      state = this.db.stateCache.get(bit, entry);
      break;
    }

@ -1988,14 +1976,13 @@ Chain.prototype.getState = co(function* getState(prev, id) {

    if (medianTime < timeStart) {
      state = thresholdStates.DEFINED;
-      stateCache[entry.hash] = state;
+      this.db.stateCache.set(bit, entry, state);
      break;
    }

    compute.push(entry);

    height = entry.height - period;
-
    entry = yield entry.getAncestorByHeight(height);
  }

@ -2053,7 +2040,7 @@ Chain.prototype.getState = co(function* getState(prev, id) {
        break;
    }

-    stateCache[entry.hash] = state;
+    this.db.stateCache.set(bit, entry, state);
  }

  return state;
@ -2297,8 +2284,9 @@ DeploymentState.prototype.hasWitness = function hasWitness() {
  return (this.flags & constants.flags.VERIFY_WITNESS) !== 0;
 };

-/*
+/**
 * LockTimes
+ * @constructor
 */

 function LockTimes(height, time) {
@ -2306,8 +2294,9 @@ function LockTimes(height, time) {
  this.time = time;
 }

-/*
+/**
 * ContextResult
+ * @constructor
 */

 function ContextResult(view, state) {
--- a/lib/blockchain/chaindb.js
+++ b/lib/blockchain/chaindb.js
@ -28,6 +28,7 @@ var Outpoint = require('../primitives/outpoint');
 var TX = require('../primitives/tx');
 var Address = require('../primitives/address');
 var ChainEntry = require('./chainentry');
+var U8 = encoding.U8;
 var U32 = encoding.U32;
 var DUMMY = new Buffer([0]);

@ -68,6 +69,7 @@ function ChainDB(chain) {
    bufferKeys: !util.isBrowser
  });

+  this.stateCache = new StateCache(chain.network);
  this.state = new ChainState();
  this.pending = null;
  this.current = null;
@ -127,9 +129,15 @@ ChainDB.prototype._open = co(function* open() {
    yield this.saveOptions();
  }

+  // Verify deployment params have not changed.
+  yield this.verifyDeployments();
+
  if (state) {
    // Grab the chainstate if we have one.
    this.state = state;
+
+    // Load state caches.
+    this.stateCache = yield this.getStateCache();
  } else {
    // Otherwise write the genesis block.
    // (We assume this database is fresh).
@ -226,6 +234,7 @@ ChainDB.prototype.drop = function drop() {
  this.coinCache.drop();
  this.cacheHash.drop();
  this.cacheHeight.drop();
+  this.stateCache.drop();

  batch.clear();
 };
@ -264,6 +273,7 @@ ChainDB.prototype.commit = co(function* commit() {
  this.coinCache.commit();
  this.cacheHash.commit();
  this.cacheHeight.commit();
+  this.stateCache.commit();
 });

 /**
@ -773,6 +783,85 @@ ChainDB.prototype.getFullBlock = co(function* getFullBlock(hash) {
  return block;
 });

+/**
+ * Get state caches.
+ * @returns {Promise} - Returns {@link StateCache}.
+ */
+
+ChainDB.prototype.getStateCache = co(function* getStateCache() {
+  var stateCache = new StateCache(this.network);
+  var i, items, item;
+
+  items = yield this.db.range({
+    gte: layout.s(0, constants.ZERO_HASH),
+    lte: layout.s(255, constants.MAX_HASH),
+    values: true
+  });
+
+  for (i = 0; i < items.length; i++) {
+    item = items[i];
+    stateCache.setRaw(item.key, item.value);
+  }
+
+  return stateCache;
+});
+
+/**
+ * Invalidate state cache.
+ * @private
+ * @returns {Promise}
+ */
+
+ChainDB.prototype.invalidateCache = co(function* invalidateCache(bit, batch) {
+  var i, keys, key;
+
+  keys = yield this.db.keys({
+    gte: layout.s(bit, constants.ZERO_HASH),
+    lte: layout.s(bit, constants.MAX_HASH)
+  });
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    batch.del(key);
+  }
+});
+
+/**
+ * Potentially invalidate state cache.
+ * @returns {Promise}
+ */
+
+ChainDB.prototype.verifyDeployments = co(function* verifyDeployments() {
+  var expected = this.stateCache.toDeployments();
+  var current = yield this.db.get(layout.v);
+  var i, invalid, bit, batch;
+
+  if (!current) {
+    yield this.db.put(layout.v, expected);
+    return true;
+  }
+
+  invalid = this.stateCache.verifyDeployments(current);
+
+  if (invalid.length === 0)
+    return true;
+
+  batch = this.db.batch();
+
+  for (i = 0; i < invalid.length; i++) {
+    bit = invalid[i];
+    this.logger.warning('Versionbit deployment params modified.');
+    this.logger.warning('Invalidating cache for bit %d.', bit);
+    yield this.invalidateCache(bit, batch);
+  }
+
+  batch.put(layout.v, expected);
+
+  yield batch.write();
+
+  return false;
+});
+
 /**
 * Fill a transaction with coins (only unspents).
 * @param {TX} tx
@ -1124,6 +1213,9 @@ ChainDB.prototype._save = co(function* save(entry, block, view) {
  this.del(layout.p(entry.prevBlock));
  this.put(layout.p(hash), DUMMY);

+  // Update state caches.
+  this.saveUpdates();
+
  if (!view) {
    // Save block data.
    yield this.saveBlock(block);
@ -1185,6 +1277,9 @@ ChainDB.prototype._reconnect = co(function* reconnect(entry, block, view) {
  // Re-insert into cache.
  this.cacheHash.push(entry.hash, entry);

+  // Update state caches.
+  this.saveUpdates();
+
  // Connect inputs.
  yield this.connectBlock(block, view);

@ -1232,6 +1327,9 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
  this.del(layout.H(entry.height));
  this.cacheHeight.unpush(entry.height);

+  // Update state caches.
+  this.saveUpdates();
+
  block = yield this.getBlock(entry.hash);

  if (!block)
@ -1246,6 +1344,24 @@ ChainDB.prototype._disconnect = co(function* disconnect(entry) {
  return block;
 });

+/**
+ * Save state cache updates.
+ * @private
+ */
+
+ChainDB.prototype.saveUpdates = function saveUpdates() {
+  var updates = this.stateCache.updates;
+  var i, update;
+
+  if (updates.length > 0)
+    this.logger.info('Saving %d state cache updates.', updates.length);
+
+  for (i = 0; i < updates.length; i++) {
+    update = updates[i];
+    this.put(layout.s(update.bit, update.hash), update.toRaw());
+  }
+};
+
 /**
 * Reset the chain to a height or hash. Useful for replaying
 * the blockchain download for SPV.
@ -1879,6 +1995,138 @@ ChainState.fromRaw = function fromRaw(data) {
  return state;
 };

+/**
+ * StateCache
+ * @constructor
+ */
+
+function StateCache(network) {
+  this.deployments = network.deployments;
+  this.bits = {};
+  this.cache = {};
+  this.updates = [];
+  this._init();
+}
+
+StateCache.prototype._init = function _init() {
+  var keys = Object.keys(this.deployments);
+  var i, key, deployment, bit;
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    deployment = this.deployments[key];
+    bit = deployment.bit;
+    this.cache[bit] = {};
+    this.bits[bit] = deployment;
+  }
+};
+
+StateCache.prototype.toDeployments = function toDeployments() {
+  var p = new BufferWriter();
+  var keys = Object.keys(this.deployments);
+  var i, key, deployment;
+
+  for (i = 0; i < keys.length; i++) {
+    key = keys[i];
+    deployment = this.deployments[key];
+    p.writeU8(deployment.bit);
+    p.writeU32(deployment.startTime);
+    p.writeU32(deployment.timeout);
+  }
+
+  return p.render();
+};
+
+StateCache.prototype.verifyDeployments = function verifyDeployments(raw) {
+  var p = new BufferReader(raw);
+  var invalid = [];
+  var deployment, bit, start, timeout;
+
+  while (p.left()) {
+    bit = p.readU8();
+    start = p.readU32();
+    timeout = p.readU32();
+    deployment = this.bits[bit];
+
+    if (deployment
+        && start === deployment.startTime
+        && timeout === deployment.timeout) {
+      continue;
+    }
+
+    invalid.push(bit);
+  }
+
+  return invalid;
+};
+
+StateCache.prototype.set = function set(bit, entry, state) {
+  var cache = this.cache[bit];
+
+  assert(cache);
+
+  if (cache[entry.hash] !== state) {
+    cache[entry.hash] = state;
+    this.updates.push(new CacheUpdate(bit, entry.hash, state));
+  }
+};
+
+StateCache.prototype.get = function get(bit, entry) {
+  var cache = this.cache[bit];
+  var state;
+
+  assert(cache);
+
+  state = cache[entry.hash];
+
+  if (state == null)
+    return -1;
+
+  return state;
+};
+
+StateCache.prototype.commit = function commit() {
+  this.updates.length = 0;
+};
+
+StateCache.prototype.drop = function drop() {
+  var i, update, cache;
+
+  for (i = 0; i < this.updates.length; i++) {
+    update = this.updates[i];
+    cache = this.cache[update.bit];
+    assert(cache);
+    delete cache[update.hash];
+  }
+
+  this.updates.length = 0;
+};
+
+StateCache.prototype.setRaw = function setRaw(key, value) {
+  var pair = layout.ss(key);
+  var bit = pair[0];
+  var hash = pair[1];
+  var state = value[0];
+  var cache = this.cache[bit];
+  assert(cache);
+  cache[hash] = state;
+};
+
+/**
+ * CacheUpdate
+ * @constructor
+ */
+
+function CacheUpdate(bit, hash, state) {
+  this.bit = bit;
+  this.hash = hash;
+  this.state = state;
+}
+
+CacheUpdate.prototype.toRaw = function toRaw() {
+  return U8(this.state);
+};
+
 /*
 * Helpers
 */
--- a/lib/blockchain/layout-browser.js
+++ b/lib/blockchain/layout-browser.js
@ -7,11 +7,13 @@
 'use strict';

 var util = require('../utils/util');
+var pad8 = util.pad8;
 var pad32 = util.pad32;

 var layout = {
  R: 'R',
  O: 'O',
+  v: 'v',
  e: function e(hash) {
    return 'e' + hex(hash);
  },
@ -39,6 +41,12 @@ var layout = {
  u: function u(hash) {
    return 'u' + hex(hash);
  },
+  s: function s(bit, hash) {
+    return 's' + pad8(bit) + hex(hash);
+  },
+  ss: function ss(key) {
+    return [+key.slice(1, 4), key.slice(4, 36)];
+  },
  T: function T(address, hash) {
    address = hex(address);

--- a/lib/blockchain/layout.js
+++ b/lib/blockchain/layout.js
@ -19,6 +19,8 @@
 *   t[hash] -> extended tx
 *   c[hash] -> coins
 *   u[hash] -> undo coins
+ *   s[bit][hash] -> versionbits state
+ *   v -> versionbits deployments
 *   T[addr-hash][hash] -> dummy (tx by address)
 *   C[addr-hash][hash][index] -> dummy (coin by address)
 *   W+T[witaddr-hash][hash] -> dummy (tx by address)
@ -28,6 +30,7 @@
 var layout = {
  R: new Buffer([0x52]),
  O: new Buffer([0x4f]),
+  v: new Buffer([0x76]),
  e: function e(hash) {
    return pair(0x65, hash);
  },
@ -55,6 +58,16 @@ var layout = {
  u: function u(hash) {
    return pair(0x75, hash);
  },
+  s: function s(bit, hash) {
+    var key = new Buffer(1 + 1 + 32);
+    key[0] = 0x73;
+    key[1] = bit;
+    write(key, hash, 2);
+    return key;
+  },
+  ss: function ss(key) {
+    return [key[1], key.toString('hex', 2, 34)];
+  },
  T: function T(address, hash) {
    var len = address.length;
    var key;
--- a/lib/protocol/networks.js
+++ b/lib/protocol/networks.js
@ -353,8 +353,8 @@ main.deployments = {
  },
  mast: {
    bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
    force: false
  }
 };
@ -581,8 +581,8 @@ testnet.deployments = {
  },
  mast: {
    bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
    force: false
  }
 };
@ -711,25 +711,25 @@ regtest.deployments = {
  testdummy: {
    bit: 28,
    startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
    force: true
  },
  csv: {
    bit: 0,
    startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
    force: true
  },
  witness: {
    bit: 1,
    startTime: 0,
-    timeout: 999999999999,
+    timeout: 0xffffffff,
    force: false
  },
  mast: {
    bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
    force: false
  }
 };
@ -1139,8 +1139,8 @@ simnet.deployments = {
  },
  mast: {
    bit: 2,
-    startTime: 2000000000, // Far in the future
-    timeout: 2100000000,
+    startTime: 0xffffffff, // Far in the future
+    timeout: 0xffffffff,
    force: false
  }
 };
--- a/lib/utils/co.js
+++ b/lib/utils/co.js
@ -249,6 +249,7 @@ function call(func) {
  for (i = 1; i < arguments.length; i++)
    args[i - 1] = arguments[i];

+  /* jshint validthis:true */
  return _call(this, func, args);
 }

--- a/lib/utils/encoding.js
+++ b/lib/utils/encoding.js
@ -640,6 +640,18 @@ encoding.sizeVarint2 = function sizeVarint2(num) {
  return size;
 };

+/**
+ * Serialize number as a u8.
+ * @param {Number} num
+ * @returns {Buffer}
+ */
+
+encoding.U8 = function U8(num) {
+  var data = new Buffer(1);
+  data[0] = num >>> 0;
+  return data;
+};
+
 /**
 * Serialize number as a u32le.
 * @param {Number} num
--- a/lib/utils/util.js
+++ b/lib/utils/util.js
@ -662,6 +662,27 @@ util.indexOf = function indexOf(obj, data) {
  return -1;
 };

+/**
+ * Convert a number to a padded uint8
+ * string (3 digits in decimal).
+ * @param {Number} num
+ * @returns {String} Padded number.
+ */
+
+util.pad8 = function pad8(num) {
+  assert(num >= 0);
+  num = num + '';
+  switch (num.length) {
+    case 1:
+      return '00' + num;
+    case 2:
+      return '0' + num;
+    case 3:
+      return num;
+  }
+  assert(false);
+};
+
 /**
 * Convert a number to a padded uint32
 * string (10 digits in decimal).
@ -698,6 +719,26 @@ util.pad32 = function pad32(num) {
  }
 };

+/**
+ * Convert a number to a padded uint8
+ * string (2 digits in hex).
+ * @param {Number} num
+ * @returns {String} Padded number.
+ */
+
+util.hex8 = function hex8(num) {
+  assert(num >= 0);
+  num = num.toString(16);
+  switch (num.length) {
+    case 1:
+      return '0' + num;
+    case 2:
+      return num;
+    default:
+      assert(false);
+  }
+};
+
 /**
 * Convert a number to a padded uint32
 * string (8 digits in hex).
--- a/lib/wallet/client.js
+++ b/lib/wallet/client.js
@ -414,12 +414,6 @@ function parseTX(data) {
  return TX.fromRaw(data, 'hex');
 }

-function toHex(data) {
-  if (typeof data !== 'string')
-    return data.toString('hex');
-  return data;
-}
-
 function BlockResult(entry, txs) {
  this.entry = entry;
  this.txs = txs;
--- a/test/chain-test.js
+++ b/test/chain-test.js
@ -263,7 +263,7 @@ describe('Chain', function() {
  }));

  it('should activate csv', cob(function* () {
-    var i, block, prev, state;
+    var i, block, prev, state, cache;

    prev = yield chain.tip.getPrevious();
    state = yield chain.getState(prev, 'csv');
@ -293,6 +293,11 @@ describe('Chain', function() {

    assert(chain.height === 432);
    assert(chain.state.hasCSV());
+
+    cache = yield chain.db.getStateCache();
+    assert.deepEqual(cache, chain.db.stateCache);
+    assert.equal(chain.db.stateCache.updates.length, 0);
+    assert(yield chain.db.verifyDeployments());
  }));

  var mineCSV = co(function* mineCSV(tx) {