chaindb: add migration for tip index.

lib/chain/browser.js

@@ -24,8 +24,8 @@ var layout = {
   n: function n(hash) {
     return 'n' + hex(hash);
   },
-  a: function a(hash) {
-    return 'a' + hex(hash);
+  p: function p(hash) {
+    return 'p' + hex(hash);
   },
   b: function b(hash) {
     return 'b' + hex(hash);
@@ -55,7 +55,7 @@ var layout = {
 
     return 'C' + address + hex(hash) + pad32(index);
   },
-  aa: function aa(key) {
+  pp: function aa(key) {
     return key.slice(1, 65);
   },
   Cc: function Cc(key) {

lib/chain/chaindb.js

@@ -36,7 +36,7 @@ var DUMMY = new Buffer([0]);
  *   h[hash] -> height
  *   H[height] -> hash
  *   n[hash] -> next hash
- *   a[hash] -> tip index
+ *   p[hash] -> tip index
  *   b[hash] -> block
  *   t[hash] -> extended tx
  *   c[hash] -> coins
@@ -62,8 +62,8 @@ var layout = {
   n: function n(hash) {
     return pair(0x6e, hash);
   },
-  a: function a(hash) {
-    return pair(0x61, hash);
+  p: function p(hash) {
+    return pair(0x70, hash);
   },
   b: function b(hash) {
     return pair(0x62, hash);
@@ -121,7 +121,7 @@ var layout = {
 
     return key;
   },
-  aa: function aa(key) {
+  pp: function aa(key) {
     return key.toString('hex', 1, 33);
   },
   Cc: function Cc(key) {
@@ -674,9 +674,9 @@ ChainDB.prototype.getEntries = function getEntries() {
 
 ChainDB.prototype.getTips = function getTips() {
   return this.db.keys({
-    gte: layout.a(constants.ZERO_HASH),
-    lte: layout.a(constants.MAX_HASH),
-    parse: layout.aa
+    gte: layout.p(constants.ZERO_HASH),
+    lte: layout.p(constants.MAX_HASH),
+    parse: layout.pp
   });
 };
 
@@ -1237,8 +1237,8 @@ ChainDB.prototype._save = co(function* save(entry, block, view) {
   this.cacheHash.push(entry.hash, entry);
 
   // Tip index.
-  this.del(layout.a(entry.prevBlock));
-  this.put(layout.a(hash), DUMMY);
+  this.del(layout.p(entry.prevBlock));
+  this.put(layout.p(hash), DUMMY);
 
   if (!view) {
     // Save block data.
@@ -1406,8 +1406,8 @@ ChainDB.prototype.reset = co(function* reset(block) {
     assert(!tip.isGenesis());
 
     // Revert the tip index.
-    this.del(layout.a(tip.hash));
-    this.put(layout.a(tip.prevBlock), DUMMY);
+    this.del(layout.p(tip.hash));
+    this.put(layout.p(tip.prevBlock), DUMMY);
 
     // Remove all records (including
     // main-chain-only records).
@@ -1486,7 +1486,7 @@ ChainDB.prototype._removeChain = co(function* removeChain(hash) {
     assert(!tip.isGenesis());
 
     // Remove all non-main-chain records.
-    this.del(layout.a(tip.hash));
+    this.del(layout.p(tip.hash));
     this.del(layout.h(tip.hash));
     this.del(layout.e(tip.hash));
     this.del(layout.b(tip.hash));

migrate/ensure-tip-index.js

@@ -0,0 +1,172 @@
+var assert = require('assert');
+var constants = require('../lib/protocol/constants');
+var co = require('../lib/utils/co');
+var BufferWriter = require('../lib/utils/writer');
+var BufferReader = require('../lib/utils/reader');
+var crypto = require('../lib/crypto/crypto');
+var utils = require('../lib/utils/utils');
+var LDB = require('../lib/db/ldb');
+var BN = require('bn.js');
+var DUMMY = new Buffer([0]);
+var file = process.argv[2];
+var db, batch;
+
+assert(typeof file === 'string', 'Please pass in a database path.');
+
+file = file.replace(/\.ldb\/?$/, '');
+
+db = LDB({
+  location: file,
+  db: 'leveldb',
+  compression: true,
+  cacheSize: 16 << 20,
+  writeBufferSize: 8 << 20,
+  createIfMissing: false,
+  bufferKeys: true
+});
+
+var checkVersion = co(function* checkVersion() {
+  var data, ver;
+
+  console.log('Checking version.');
+
+  data = yield db.get('V');
+
+  if (!data)
+    return;
+
+  ver = data.readUInt32LE(0, true);
+
+  if (ver !== 1)
+    throw Error('DB is version ' + ver + '.');
+});
+
+function entryFromRaw(data) {
+  var p = new BufferReader(data, true);
+  var hash = crypto.hash256(p.readBytes(80));
+  var entry = {};
+
+  p.seek(-80);
+
+  entry.hash = hash.toString('hex');
+  entry.version = p.readU32(); // Technically signed
+  entry.prevBlock = p.readHash('hex');
+  entry.merkleRoot = p.readHash('hex');
+  entry.ts = p.readU32();
+  entry.bits = p.readU32();
+  entry.nonce = p.readU32();
+  entry.height = p.readU32();
+  entry.chainwork = new BN(p.readBytes(32), 'le');
+
+  return entry;
+}
+
+function getEntries() {
+  return db.values({
+    gte: pair('e', constants.ZERO_HASH),
+    lte: pair('e', constants.MAX_HASH),
+    parse: entryFromRaw
+  });
+}
+
+var getTip = co(function* getTip(entry) {
+  var state = yield db.get('R');
+  assert(state);
+  var tip = state.toString('hex', 0, 32);
+  var data = yield db.get(pair('e', tip));
+  assert(data);
+  return entryFromRaw(data);
+});
+
+var isMainChain = co(function* isMainChain(entry, tip) {
+  if (entry.hash === tip)
+    return true;
+
+  if (yield db.get(pair('n', entry.hash)))
+    return true;
+
+  return false;
+});
+
+var removeOld = co(function* removeOld() {
+  var i, keys, key;
+
+  keys = yield db.keys({
+    gte: pair('a', constants.ZERO_HASH),
+    lte: pair('a', constants.MAX_HASH)
+  });
+
+  for (i = 0; i < keys.length; i++)
+    batch.del(keys[i]);
+});
+
+// And this insane function is why we should
+// be indexing tips in the first place!
+var indexTips = co(function* indexTips() {
+  var entries = yield getEntries();
+  var tip = yield getTip();
+  var tips = [];
+  var orphans = [];
+  var prevs = {};
+  var i, orphan, entry, main;
+
+  for (i = 0; i < entries.length; i++) {
+    entry = entries[i];
+    main = yield isMainChain(entry, tip.hash);
+    if (!main) {
+      orphans.push(entry);
+      prevs[entry.prevBlock] = true;
+    }
+  }
+
+  for (i = 0; i < orphans.length; i++) {
+    orphan = orphans[i];
+    if (!prevs[orphan.hash])
+      tips.push(orphan.hash);
+  }
+
+  tips.push(tip.hash);
+
+  for (i = 0; i < tips.length; i++) {
+    tip = tips[i];
+    console.log('Indexing chain tip: %s.', utils.revHex(tip));
+    batch.put(pair('p', tip), DUMMY);
+  }
+});
+
+function write(data, str, off) {
+  if (Buffer.isBuffer(str))
+    return str.copy(data, off);
+  data.write(str, off, 'hex');
+}
+
+function pair(prefix, hash) {
+  var key = new Buffer(33);
+  if (typeof prefix === 'string')
+    prefix = prefix.charCodeAt(0);
+  key[0] = prefix;
+  write(key, hash, 1);
+  return key;
+}
+
+function ipair(prefix, num) {
+  var key = new Buffer(5);
+  if (typeof prefix === 'string')
+    prefix = prefix.charCodeAt(0);
+  key[0] = prefix;
+  key.writeUInt32BE(num, 1, true);
+  return key;
+}
+
+co.spawn(function* () {
+  yield db.open();
+  console.log('Opened %s.', file);
+  batch = db.batch();
+  yield checkVersion();
+  yield removeOld();
+  yield indexTips();
+  yield batch.write();
+}).then(function() {
+  console.log('Migration complete.');
+  process.exit(0);
+});