From 14ae1eb29c3ccadf46c308b6ea239282823a9bf9 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Fri, 11 Nov 2016 15:37:28 -0800 Subject: [PATCH] chaindb: add migration for tip index. --- lib/chain/browser.js | 6 +- lib/chain/chaindb.js | 24 ++--- migrate/ensure-tip-index.js | 172 ++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 15 deletions(-) create mode 100644 migrate/ensure-tip-index.js diff --git a/lib/chain/browser.js b/lib/chain/browser.js index ff6f7b3f..fb30592f 100644 --- a/lib/chain/browser.js +++ b/lib/chain/browser.js @@ -24,8 +24,8 @@ var layout = { n: function n(hash) { return 'n' + hex(hash); }, - a: function a(hash) { - return 'a' + hex(hash); + p: function p(hash) { + return 'p' + hex(hash); }, b: function b(hash) { return 'b' + hex(hash); @@ -55,7 +55,7 @@ var layout = { return 'C' + address + hex(hash) + pad32(index); }, - aa: function aa(key) { + pp: function aa(key) { return key.slice(1, 65); }, Cc: function Cc(key) { diff --git a/lib/chain/chaindb.js b/lib/chain/chaindb.js index 67077333..5abd4006 100644 --- a/lib/chain/chaindb.js +++ b/lib/chain/chaindb.js @@ -36,7 +36,7 @@ var DUMMY = new Buffer([0]); * h[hash] -> height * H[height] -> hash * n[hash] -> next hash - * a[hash] -> tip index + * p[hash] -> tip index * b[hash] -> block * t[hash] -> extended tx * c[hash] -> coins @@ -62,8 +62,8 @@ var layout = { n: function n(hash) { return pair(0x6e, hash); }, - a: function a(hash) { - return pair(0x61, hash); + p: function p(hash) { + return pair(0x70, hash); }, b: function b(hash) { return pair(0x62, hash); @@ -121,7 +121,7 @@ var layout = { return key; }, - aa: function aa(key) { + pp: function aa(key) { return key.toString('hex', 1, 33); }, Cc: function Cc(key) { @@ -674,9 +674,9 @@ ChainDB.prototype.getEntries = function getEntries() { ChainDB.prototype.getTips = function getTips() { return this.db.keys({ - gte: layout.a(constants.ZERO_HASH), - lte: layout.a(constants.MAX_HASH), - parse: layout.aa + gte: layout.p(constants.ZERO_HASH), + lte: layout.p(constants.MAX_HASH), + parse: layout.pp }); }; @@ -1237,8 +1237,8 @@ ChainDB.prototype._save = co(function* save(entry, block, view) { this.cacheHash.push(entry.hash, entry); // Tip index. - this.del(layout.a(entry.prevBlock)); - this.put(layout.a(hash), DUMMY); + this.del(layout.p(entry.prevBlock)); + this.put(layout.p(hash), DUMMY); if (!view) { // Save block data. @@ -1406,8 +1406,8 @@ ChainDB.prototype.reset = co(function* reset(block) { assert(!tip.isGenesis()); // Revert the tip index. - this.del(layout.a(tip.hash)); - this.put(layout.a(tip.prevBlock), DUMMY); + this.del(layout.p(tip.hash)); + this.put(layout.p(tip.prevBlock), DUMMY); // Remove all records (including // main-chain-only records). @@ -1486,7 +1486,7 @@ ChainDB.prototype._removeChain = co(function* removeChain(hash) { assert(!tip.isGenesis()); // Remove all non-main-chain records. - this.del(layout.a(tip.hash)); + this.del(layout.p(tip.hash)); this.del(layout.h(tip.hash)); this.del(layout.e(tip.hash)); this.del(layout.b(tip.hash)); diff --git a/migrate/ensure-tip-index.js b/migrate/ensure-tip-index.js new file mode 100644 index 00000000..ad920b64 --- /dev/null +++ b/migrate/ensure-tip-index.js @@ -0,0 +1,172 @@ +var assert = require('assert'); +var constants = require('../lib/protocol/constants'); +var co = require('../lib/utils/co'); +var BufferWriter = require('../lib/utils/writer'); +var BufferReader = require('../lib/utils/reader'); +var crypto = require('../lib/crypto/crypto'); +var utils = require('../lib/utils/utils'); +var LDB = require('../lib/db/ldb'); +var BN = require('bn.js'); +var DUMMY = new Buffer([0]); +var file = process.argv[2]; +var db, batch; + +assert(typeof file === 'string', 'Please pass in a database path.'); + +file = file.replace(/\.ldb\/?$/, ''); + +db = LDB({ + location: file, + db: 'leveldb', + compression: true, + cacheSize: 16 << 20, + writeBufferSize: 8 << 20, + createIfMissing: false, + bufferKeys: true +}); + +var checkVersion = co(function* checkVersion() { + var data, ver; + + console.log('Checking version.'); + + data = yield db.get('V'); + + if (!data) + return; + + ver = data.readUInt32LE(0, true); + + if (ver !== 1) + throw Error('DB is version ' + ver + '.'); +}); + +function entryFromRaw(data) { + var p = new BufferReader(data, true); + var hash = crypto.hash256(p.readBytes(80)); + var entry = {}; + + p.seek(-80); + + entry.hash = hash.toString('hex'); + entry.version = p.readU32(); // Technically signed + entry.prevBlock = p.readHash('hex'); + entry.merkleRoot = p.readHash('hex'); + entry.ts = p.readU32(); + entry.bits = p.readU32(); + entry.nonce = p.readU32(); + entry.height = p.readU32(); + entry.chainwork = new BN(p.readBytes(32), 'le'); + + return entry; +} + +function getEntries() { + return db.values({ + gte: pair('e', constants.ZERO_HASH), + lte: pair('e', constants.MAX_HASH), + parse: entryFromRaw + }); +} + +var getTip = co(function* getTip(entry) { + var state = yield db.get('R'); + assert(state); + var tip = state.toString('hex', 0, 32); + var data = yield db.get(pair('e', tip)); + assert(data); + return entryFromRaw(data); +}); + +var isMainChain = co(function* isMainChain(entry, tip) { + if (entry.hash === tip) + return true; + + if (yield db.get(pair('n', entry.hash))) + return true; + + return false; +}); + +var removeOld = co(function* removeOld() { + var i, keys, key; + + keys = yield db.keys({ + gte: pair('a', constants.ZERO_HASH), + lte: pair('a', constants.MAX_HASH) + }); + + for (i = 0; i < keys.length; i++) + batch.del(keys[i]); +}); + +// And this insane function is why we should +// be indexing tips in the first place! +var indexTips = co(function* indexTips() { + var entries = yield getEntries(); + var tip = yield getTip(); + var tips = []; + var orphans = []; + var prevs = {}; + var i, orphan, entry, main; + + for (i = 0; i < entries.length; i++) { + entry = entries[i]; + main = yield isMainChain(entry, tip.hash); + if (!main) { + orphans.push(entry); + prevs[entry.prevBlock] = true; + } + } + + for (i = 0; i < orphans.length; i++) { + orphan = orphans[i]; + if (!prevs[orphan.hash]) + tips.push(orphan.hash); + } + + tips.push(tip.hash); + + for (i = 0; i < tips.length; i++) { + tip = tips[i]; + console.log('Indexing chain tip: %s.', utils.revHex(tip)); + batch.put(pair('p', tip), DUMMY); + } +}); + +function write(data, str, off) { + if (Buffer.isBuffer(str)) + return str.copy(data, off); + data.write(str, off, 'hex'); +} + +function pair(prefix, hash) { + var key = new Buffer(33); + if (typeof prefix === 'string') + prefix = prefix.charCodeAt(0); + key[0] = prefix; + write(key, hash, 1); + return key; +} + +function ipair(prefix, num) { + var key = new Buffer(5); + if (typeof prefix === 'string') + prefix = prefix.charCodeAt(0); + key[0] = prefix; + key.writeUInt32BE(num, 1, true); + return key; +} + +co.spawn(function* () { + yield db.open(); + console.log('Opened %s.', file); + batch = db.batch(); + yield checkVersion(); + yield removeOld(); + yield indexTips(); + yield batch.write(); +}).then(function() { + console.log('Migration complete.'); + process.exit(0); +});