From dbde501444f91bb7a41df1efa0e34003a9a081a8 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 4 Dec 2017 12:11:26 -0800 Subject: [PATCH] db: use bdb key object. --- bin/cli | 2 +- lib/blockchain/chain.js | 1 - lib/blockchain/chaindb.js | 167 ++++++++--------- lib/blockchain/layout-browser.js | 157 ---------------- lib/blockchain/layout.js | 86 +++------ lib/hd/mnemonic.js | 1 - lib/mempool/layout-browser.js | 41 ----- lib/mempool/layout.js | 22 +-- lib/mempool/mempool.js | 41 +++-- lib/wallet/layout-browser.js | 269 --------------------------- lib/wallet/layout.js | 142 +++++---------- lib/wallet/txdb.js | 209 ++++++++++----------- lib/wallet/wallet.js | 16 +- lib/wallet/walletdb.js | 298 +++++++++++++++--------------- migrate/chaindb3to4.js | 163 +++++++++++++++++ migrate/walletdb6to7.js | 299 +++++++++++++++++++++---------- 16 files changed, 818 insertions(+), 1096 deletions(-) delete mode 100644 lib/blockchain/layout-browser.js delete mode 100644 lib/mempool/layout-browser.js delete mode 100644 lib/wallet/layout-browser.js create mode 100644 migrate/chaindb3to4.js diff --git a/bin/cli b/bin/cli index 7407aeb3..5651747d 100755 --- a/bin/cli +++ b/bin/cli @@ -541,7 +541,7 @@ class CLI { let result; try { - result = await this.client.rpc.execute(method, params); + result = await this.client.execute(method, params); } catch (e) { if (e.type === 'RPCError') { this.log(e.message); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index 7fec51d6..d302c72d 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -2674,7 +2674,6 @@ class ChainOptions { this.maxFiles = 64; this.cacheSize = 32 << 20; this.compression = true; - this.bufferKeys = ChainDB.layout.binary; this.spv = false; this.bip91 = false; diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 580bffdd..f5fda9e8 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -8,7 +8,7 @@ 'use strict'; const assert = require('assert'); -const BDB = require('bdb'); +const bdb = require('bdb'); const bio = require('bufio'); const Amount = require('../btc/amount'); const Network = require('../protocol/network'); @@ -43,7 +43,7 @@ class ChainDB { this.network = this.options.network; this.logger = this.options.logger.context('chaindb'); - this.db = new BDB(this.options); + this.db = bdb.create(this.options); this.stateCache = new StateCache(this.network); this.state = new ChainState(); this.pending = null; @@ -63,7 +63,7 @@ class ChainDB { this.logger.info('Opening ChainDB...'); await this.db.open(); - await this.db.checkVersion('V', 3); + await this.db.verify(layout.V.build(), 'chain', 4); const state = await this.getState(); @@ -265,7 +265,7 @@ class ChainDB { if (entry) return entry.height; - const height = await this.db.get(layout.h(hash)); + const height = await this.db.get(layout.h.build(hash)); if (!height) return -1; @@ -294,7 +294,7 @@ class ChainDB { if (entry) return entry.hash; - const hash = await this.db.get(layout.H(height)); + const hash = await this.db.get(layout.H.build(height)); if (!hash) return null; @@ -319,7 +319,7 @@ class ChainDB { if (cache) return cache; - const data = await this.db.get(layout.H(height)); + const data = await this.db.get(layout.H.build(height)); if (!data) return null; @@ -358,7 +358,7 @@ class ChainDB { if (cache) return cache; - const raw = await this.db.get(layout.e(hash)); + const raw = await this.db.get(layout.e.build(hash)); if (!raw) return null; @@ -496,7 +496,7 @@ class ChainDB { */ async getState() { - const data = await this.db.get(layout.R()); + const data = await this.db.get(layout.R.build()); if (!data) return null; @@ -525,7 +525,7 @@ class ChainDB { */ async getFlags() { - const data = await this.db.get(layout.O()); + const data = await this.db.get(layout.O.build()); if (!data) return null; @@ -617,13 +617,13 @@ class ChainDB { const stateCache = new StateCache(this.network); const items = await this.db.range({ - gte: layout.v(0, encoding.ZERO_HASH), - lte: layout.v(255, encoding.MAX_HASH), + gte: layout.v.min(), + lte: layout.v.max(), values: true }); for (const item of items) { - const [bit, hash] = layout.vv(item.key); + const [bit, hash] = layout.v.parse(item.key); const state = item.value[0]; stateCache.insert(bit, hash, state); } @@ -660,7 +660,7 @@ class ChainDB { bw.writeI32(deployment.window); } - batch.put(layout.V(), bw.render()); + batch.put(layout.D.build(), bw.render()); } /** @@ -670,7 +670,7 @@ class ChainDB { */ async checkDeployments() { - const raw = await this.db.get(layout.V()); + const raw = await this.db.get(layout.D.build()); assert(raw, 'No deployment table found.'); @@ -744,8 +744,8 @@ class ChainDB { async invalidateCache(bit, batch) { const keys = await this.db.keys({ - gte: layout.v(bit, encoding.ZERO_HASH), - lte: layout.v(bit, encoding.MAX_HASH) + gte: layout.v.min(bit), + lte: layout.v.max(bit) }); for (const key of keys) @@ -782,8 +782,8 @@ class ChainDB { if (!hash) throw new Error(`Cannot find hash for ${i}.`); - batch.del(layout.b(hash)); - batch.del(layout.u(hash)); + batch.del(layout.b.build(hash)); + batch.del(layout.u.build(hash)); } try { @@ -792,7 +792,7 @@ class ChainDB { const flags = ChainFlags.fromOptions(options); assert(flags.prune); - batch.put(layout.O(), flags.toRaw()); + batch.put(layout.O.build(), flags.toRaw()); await batch.write(); } catch (e) { @@ -812,7 +812,7 @@ class ChainDB { */ async getNextHash(hash) { - const data = await this.db.get(layout.n(hash)); + const data = await this.db.get(layout.n.build(hash)); if (!data) return null; @@ -894,8 +894,8 @@ class ChainDB { assert((end >>> 0) === end); return this.db.values({ - gte: layout.H(start), - lte: layout.H(end), + gte: layout.H.min(start), + lte: layout.H.max(end), parse: data => data.toString('hex') }); } @@ -907,8 +907,8 @@ class ChainDB { getEntries() { return this.db.values({ - gte: layout.e(encoding.ZERO_HASH), - lte: layout.e(encoding.MAX_HASH), + gte: layout.e.min(), + lte: layout.e.max(), parse: data => ChainEntry.fromRaw(data) }); } @@ -920,9 +920,9 @@ class ChainDB { getTips() { return this.db.keys({ - gte: layout.p(encoding.ZERO_HASH), - lte: layout.p(encoding.MAX_HASH), - parse: layout.pp + gte: layout.p.min(), + lte: layout.p.max(), + parse: key => layout.p.parse(key) }); } @@ -946,7 +946,7 @@ class ChainDB { if (cache) return CoinEntry.fromRaw(cache); - const raw = await this.db.get(layout.c(hash, index)); + const raw = await this.db.get(layout.c.build(hash, index)); if (!raw) return null; @@ -983,7 +983,7 @@ class ChainDB { async hasCoins(tx) { for (let i = 0; i < tx.outputs.length; i++) { - const key = layout.c(tx.hash(), i); + const key = layout.c.build(tx.hash(), i); if (await this.db.has(key)) return true; } @@ -1044,7 +1044,7 @@ class ChainDB { */ async getUndoCoins(hash) { - const data = await this.db.get(layout.u(hash)); + const data = await this.db.get(layout.u.build(hash)); if (!data) return new UndoCoins(); @@ -1082,7 +1082,7 @@ class ChainDB { if (!hash) return null; - return this.db.get(layout.b(hash)); + return this.db.get(layout.b.build(hash)); } /** @@ -1123,7 +1123,7 @@ class ChainDB { if (!this.options.indexTX) return null; - const data = await this.db.get(layout.t(hash)); + const data = await this.db.get(layout.t.build(hash)); if (!data) return null; @@ -1155,7 +1155,7 @@ class ChainDB { if (!this.options.indexTX) return false; - return this.db.has(layout.t(hash)); + return this.db.has(layout.t.build(hash)); } /** @@ -1177,9 +1177,12 @@ class ChainDB { const hash = Address.getHash(addr); const keys = await this.db.keys({ - gte: layout.C(hash, encoding.ZERO_HASH, 0), - lte: layout.C(hash, encoding.MAX_HASH, 0xffffffff), - parse: layout.Cc + gte: layout.C.min(hash), + lte: layout.C.max(hash), + parse: (key) => { + const [, txid, index] = layout.C.parse(hash); + return [txid, index]; + } }); for (const [hash, index] of keys) { @@ -1208,10 +1211,10 @@ class ChainDB { const hash = Address.getHash(addr); await this.db.keys({ - gte: layout.T(hash, encoding.ZERO_HASH), - lte: layout.T(hash, encoding.MAX_HASH), + gte: layout.T.min(hash), + lte: layout.T.max(hash), parse: (key) => { - const hash = layout.Tt(key); + const [, hash] = layout.T.parse(key); hashes[hash] = true; } }); @@ -1383,15 +1386,15 @@ class ChainDB { const hash = block.hash(); // Hash->height index. - this.put(layout.h(hash), u32(entry.height)); + this.put(layout.h.build(hash), u32(entry.height)); // Entry data. - this.put(layout.e(hash), entry.toRaw()); + this.put(layout.e.build(hash), entry.toRaw()); this.cacheHash.push(entry.hash, entry); // Tip index. - this.del(layout.p(entry.prevBlock)); - this.put(layout.p(hash), null); + this.del(layout.p.build(entry.prevBlock)); + this.put(layout.p.build(hash), null); // Update state caches. this.saveUpdates(); @@ -1404,17 +1407,17 @@ class ChainDB { // Hash->next-block index. if (!entry.isGenesis()) - this.put(layout.n(entry.prevBlock), hash); + this.put(layout.n.build(entry.prevBlock), hash); // Height->hash index. - this.put(layout.H(entry.height), hash); + this.put(layout.H.build(entry.height), hash); this.cacheHeight.push(entry.height, entry); // Connect block and save data. await this.saveBlock(entry, block, view); // Commit new chain state. - this.put(layout.R(), this.pending.commit(hash)); + this.put(layout.R.build(), this.pending.commit(hash)); } /** @@ -1451,10 +1454,10 @@ class ChainDB { assert(!entry.isGenesis()); // We can now add a hash->next-block index. - this.put(layout.n(entry.prevBlock), hash); + this.put(layout.n.build(entry.prevBlock), hash); // We can now add a height->hash index. - this.put(layout.H(entry.height), hash); + this.put(layout.H.build(entry.height), hash); this.cacheHeight.push(entry.height, entry); // Re-insert into cache. @@ -1467,7 +1470,7 @@ class ChainDB { await this.connectBlock(entry, block, view); // Update chain state. - this.put(layout.R(), this.pending.commit(hash)); + this.put(layout.R.build(), this.pending.commit(hash)); } /** @@ -1503,10 +1506,10 @@ class ChainDB { async _disconnect(entry, block) { // Remove hash->next-block index. - this.del(layout.n(entry.prevBlock)); + this.del(layout.n.build(entry.prevBlock)); // Remove height->hash index. - this.del(layout.H(entry.height)); + this.del(layout.H.build(entry.height)); this.cacheHeight.unpush(entry.height); // Update state caches. @@ -1516,7 +1519,7 @@ class ChainDB { const view = await this.disconnectBlock(entry, block); // Revert chain state to previous tip. - this.put(layout.R(), this.pending.commit(entry.prevBlock)); + this.put(layout.R.build(), this.pending.commit(entry.prevBlock)); return view; } @@ -1536,7 +1539,7 @@ class ChainDB { for (const update of updates) { const {bit, hash} = update; - this.put(layout.v(bit, hash), update.toRaw()); + this.put(layout.v.build(bit, hash), update.toRaw()); } } @@ -1575,7 +1578,7 @@ class ChainDB { // Stop once we hit our target tip. if (tip.hash === entry.hash) { - this.put(layout.R(), this.pending.commit(tip.hash)); + this.put(layout.R.build(), this.pending.commit(tip.hash)); await this.commit(); break; } @@ -1583,15 +1586,15 @@ class ChainDB { assert(!tip.isGenesis()); // Revert the tip index. - this.del(layout.p(tip.hash)); - this.put(layout.p(tip.prevBlock), null); + this.del(layout.p.build(tip.hash)); + this.put(layout.p.build(tip.prevBlock), null); // Remove all records (including // main-chain-only records). - this.del(layout.H(tip.height)); - this.del(layout.h(tip.hash)); - this.del(layout.e(tip.hash)); - this.del(layout.n(tip.prevBlock)); + this.del(layout.H.build(tip.height)); + this.del(layout.h.build(tip.hash)); + this.del(layout.e.build(tip.hash)); + this.del(layout.n.build(tip.prevBlock)); // Disconnect and remove block data. try { @@ -1602,7 +1605,7 @@ class ChainDB { } // Revert chain state to previous tip. - this.put(layout.R(), this.pending.commit(tip.prevBlock)); + this.put(layout.R.build(), this.pending.commit(tip.prevBlock)); await this.commit(); @@ -1662,10 +1665,10 @@ class ChainDB { assert(!tip.isGenesis()); // Remove all non-main-chain records. - this.del(layout.p(tip.hash)); - this.del(layout.h(tip.hash)); - this.del(layout.e(tip.hash)); - this.del(layout.b(tip.hash)); + this.del(layout.p.build(tip.hash)); + this.del(layout.h.build(tip.hash)); + this.del(layout.e.build(tip.hash)); + this.del(layout.b.build(tip.hash)); // Queue up hash to be removed // on successful write. @@ -1693,7 +1696,7 @@ class ChainDB { // Write actual block data (this may be // better suited to flat files in the future). - this.put(layout.b(hash), block.toRaw()); + this.put(layout.b.build(hash), block.toRaw()); if (!view) return; @@ -1717,7 +1720,7 @@ class ChainDB { if (!block) throw new Error('Block not found.'); - this.del(layout.b(block.hash())); + this.del(layout.b.build(block.hash())); return this.disconnectBlock(entry, block); } @@ -1732,14 +1735,14 @@ class ChainDB { for (const [hash, coins] of view.map) { for (const [index, coin] of coins.outputs) { if (coin.spent) { - this.del(layout.c(hash, index)); + this.del(layout.c.build(hash, index)); this.coinCache.unpush(hash + index); continue; } const raw = coin.toRaw(); - this.put(layout.c(hash, index), raw); + this.put(layout.c.build(hash, index), raw); this.coinCache.push(hash + index, raw); } } @@ -1790,7 +1793,7 @@ class ChainDB { // Write undo coins (if there are any). if (!view.undo.isEmpty()) - this.put(layout.u(hash), view.undo.commit()); + this.put(layout.u.build(hash), view.undo.commit()); // Prune height-288 if pruning is enabled. await this.pruneBlock(entry); @@ -1849,7 +1852,7 @@ class ChainDB { this.saveView(view); // Remove undo coins. - this.del(layout.u(hash)); + this.del(layout.u.build(hash)); return view; } @@ -1879,8 +1882,8 @@ class ChainDB { if (!hash) return; - this.del(layout.b(hash)); - this.del(layout.u(hash)); + this.del(layout.b.build(hash)); + this.del(layout.u.build(hash)); } /** @@ -1891,7 +1894,7 @@ class ChainDB { saveFlags() { const flags = ChainFlags.fromOptions(this.options); const batch = this.db.batch(); - batch.put(layout.O(), flags.toRaw()); + batch.put(layout.O.build(), flags.toRaw()); return batch.write(); } @@ -1910,12 +1913,12 @@ class ChainDB { if (this.options.indexTX) { const meta = TXMeta.fromTX(tx, entry, index); - this.put(layout.t(hash), meta.toRaw()); + this.put(layout.t.build(hash), meta.toRaw()); if (this.options.indexAddress) { const hashes = tx.getHashes(view); for (const addr of hashes) - this.put(layout.T(addr, hash), null); + this.put(layout.T.build(addr, hash), null); } } @@ -1929,7 +1932,7 @@ class ChainDB { if (!addr) continue; - this.del(layout.C(addr, prevout.hash, prevout.index)); + this.del(layout.C.build(addr, prevout.hash, prevout.index)); } } @@ -1940,7 +1943,7 @@ class ChainDB { if (!addr) continue; - this.put(layout.C(addr, hash, i), null); + this.put(layout.C.build(addr, hash, i), null); } } @@ -1955,11 +1958,11 @@ class ChainDB { const hash = tx.hash(); if (this.options.indexTX) { - this.del(layout.t(hash)); + this.del(layout.t.build(hash)); if (this.options.indexAddress) { const hashes = tx.getHashes(view); for (const addr of hashes) - this.del(layout.T(addr, hash)); + this.del(layout.T.build(addr, hash)); } } @@ -1973,7 +1976,7 @@ class ChainDB { if (!addr) continue; - this.put(layout.C(addr, prevout.hash, prevout.index), null); + this.put(layout.C.build(addr, prevout.hash, prevout.index), null); } } @@ -1984,7 +1987,7 @@ class ChainDB { if (!addr) continue; - this.del(layout.C(addr, hash, i)); + this.del(layout.C.build(addr, hash, i)); } } } diff --git a/lib/blockchain/layout-browser.js b/lib/blockchain/layout-browser.js deleted file mode 100644 index eaeba263..00000000 --- a/lib/blockchain/layout-browser.js +++ /dev/null @@ -1,157 +0,0 @@ -/*! - * layout-browser.js - chaindb layout for browser. - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); - -const layout = { - binary: false, - R: 'R', - O: 'O', - V: 'v', - e: function e(hash) { - return 'e' + hex(hash); - }, - h: function h(hash) { - return 'h' + hex(hash); - }, - H: function H(height) { - return 'H' + hex32(height); - }, - n: function n(hash) { - return 'n' + hex(hash); - }, - p: function p(hash) { - return 'p' + hex(hash); - }, - b: function b(hash) { - return 'b' + hex(hash); - }, - t: function t(hash) { - return 't' + hex(hash); - }, - c: function c(hash, index) { - return 'c' + hex(hash) + hex32(index); - }, - u: function u(hash) { - return 'u' + hex(hash); - }, - v: function v(bit, hash) { - return 'v' + hex8(bit) + hex(hash); - }, - vv: function vv(key) { - assert(typeof key === 'string'); - assert(key.length === 36); - return [parseInt(key.slice(1, 3), 16), key.slice(3, 35)]; - }, - T: function T(addr, hash) { - addr = hex(addr); - - if (addr.length === 64) - return 'W' + addr + hex(hash); - - assert(addr.length === 40); - return 'T' + addr + hex(hash); - }, - C: function C(addr, hash, index) { - addr = hex(addr); - - if (addr.length === 64) - return 'X' + addr + hex(hash) + hex32(index); - - assert(addr.length === 40); - return 'C' + addr + hex(hash) + hex32(index); - }, - pp: function pp(key) { - assert(typeof key === 'string'); - assert(key.length === 65); - return key.slice(1, 65); - }, - Cc: function Cc(key) { - assert(typeof key === 'string'); - - let hash, index; - if (key.length === 137) { - hash = key.slice(65, 129); - index = parseInt(key.slice(129), 16); - } else if (key.length === 113) { - hash = key.slice(41, 105); - index = parseInt(key.slice(105), 16); - } else { - assert(false); - } - - return [hash, index]; - }, - Tt: function Tt(key) { - assert(typeof key === 'string'); - - if (key.length === 129) - return key.slice(64); - - assert(key.length === 105); - return key.slice(41); - } -}; - -/* - * Helpers - */ - -function hex(hash) { - if (Buffer.isBuffer(hash)) - hash = hash.toString('hex'); - assert(typeof hash === 'string'); - return hash; -} - -function hex8(num) { - assert(typeof num === 'number'); - assert(num >= 0 && num <= 255); - - if (num <= 0x0f) - return '0' + num.toString(16); - - if (num <= 0xff) - return num.toString(16); - - throw new Error('Number too big.'); -} - -function hex32(num) { - assert(typeof num === 'number'); - assert(num >= 0); - - num = num.toString(16); - - switch (num.length) { - case 1: - return '0000000' + num; - case 2: - return '000000' + num; - case 3: - return '00000' + num; - case 4: - return '0000' + num; - case 5: - return '000' + num; - case 6: - return '00' + num; - case 7: - return '0' + num; - case 8: - return num; - } - - throw new Error('Number too big.'); -} - -/* - * Expose - */ - -module.exports = layout; diff --git a/lib/blockchain/layout.js b/lib/blockchain/layout.js index dee4c509..1bc801d5 100644 --- a/lib/blockchain/layout.js +++ b/lib/blockchain/layout.js @@ -6,13 +6,14 @@ 'use strict'; -const assert = require('assert'); -const Key = require('bdb/lib/key'); +const bdb = require('bdb'); /* * Database Layout: - * R -> tip hash + * V -> db version * O -> chain options + * R -> tip hash + * D -> versionbits deployments * e[hash] -> entry * h[hash] -> height * H[height] -> hash @@ -22,75 +23,28 @@ const Key = require('bdb/lib/key'); * t[hash] -> extended tx * c[hash] -> coins * u[hash] -> undo coins - * v -> versionbits deployments * v[bit][hash] -> versionbits state * T[addr-hash][hash] -> dummy (tx by address) * C[addr-hash][hash][index] -> dummy (coin by address) - * W+T[witaddr-hash][hash] -> dummy (tx by address) - * W+C[witaddr-hash][hash][index] -> dummy (coin by address) */ -const R = new Key('R'); -const O = new Key('O'); -const V = new Key('v'); -const e = new Key('e', ['hash256']); -const h = new Key('h', ['hash256']); -const H = new Key('H', ['uint32']); -const n = new Key('n', ['hash256']); -const p = new Key('p', ['hash256']); -const b = new Key('b', ['hash256']); -const t = new Key('t', ['hash256']); -const c = new Key('c', ['hash256', 'uint32']); -const u = new Key('u', ['hash256']); -const v = new Key('v', ['uint8', 'hash256']); -const T160 = new Key('T', ['hash160', 'hash256']); -const T256 = new Key(0xab, ['hash256', 'hash256']); -const C160 = new Key('C', ['hash160', 'hash256', 'uint32']); -const C256 = new Key(0x9a, ['hash256', 'hash256', 'uint32']); - const layout = { - binary: true, - R: R.build.bind(R), - O: O.build.bind(O), - V: V.build.bind(V), - e: e.build.bind(e), - h: h.build.bind(h), - H: H.build.bind(H), - n: n.build.bind(n), - p: p.build.bind(p), - pp: p.parse.bind(p), - b: b.build.bind(b), - t: t.build.bind(t), - c: c.build.bind(c), - u: u.build.bind(u), - v: v.build.bind(v), - vv: v.parse.bind(v), - T: function T(addr, hash) { - let len = addr ? addr.length : 0; - if (typeof addr === 'string') - len >>>= 1; - if (len === 32) - return T256.build(addr, hash); - return T160.build(addr, hash); - }, - Tt: function Tt(key) { - if (key && key[0] === 0xab) - return T256.parse(key); - return T160.parse(key); - }, - C: function C(addr, hash, index) { - let len = addr ? addr.length : 0; - if (typeof addr === 'string') - len >>>= 1; - if (len === 32) - return C256.build(addr, hash, index); - return C160.build(addr, hash, index); - }, - Cc: function Cc(key) { - if (key && key[0] === 0x9a) - return C256.parse(key); - return C160.parse(key); - } + V: bdb.key('V'), + O: bdb.key('O'), + R: bdb.key('R'), + D: bdb.key('D'), + e: bdb.key('e', ['hash256']), + h: bdb.key('h', ['hash256']), + H: bdb.key('H', ['uint32']), + n: bdb.key('n', ['hash256']), + p: bdb.key('p', ['hash256']), + b: bdb.key('b', ['hash256']), + t: bdb.key('t', ['hash256']), + c: bdb.key('c', ['hash256', 'uint32']), + u: bdb.key('u', ['hash256']), + v: bdb.key('v', ['uint8', 'hash256']), + T: bdb.key('T', ['hash', 'hash256']), + C: bdb.key('C', ['hash', 'hash256', 'uint32']) }; /* diff --git a/lib/hd/mnemonic.js b/lib/hd/mnemonic.js index 98c077bd..ab056da3 100644 --- a/lib/hd/mnemonic.js +++ b/lib/hd/mnemonic.js @@ -16,7 +16,6 @@ const sha512 = require('bcrypto/lib/sha512'); const wordlist = require('./wordlist'); const common = require('./common'); const nfkd = require('./nfkd'); -const {encoding} = bio; /* * Constants diff --git a/lib/mempool/layout-browser.js b/lib/mempool/layout-browser.js deleted file mode 100644 index b7da4bb4..00000000 --- a/lib/mempool/layout-browser.js +++ /dev/null @@ -1,41 +0,0 @@ -/*! - * layout-browser.js - mempooldb layout for browser. - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); - -const layout = { - binary: false, - R: 'R', - V: 'V', - F: 'F', - e: function e(hash) { - return 'e' + hex(hash); - }, - ee: function ee(key) { - assert(typeof key === 'string'); - assert(key.length === 65); - return key.slice(1, 65); - } -}; - -/* - * Helpers - */ - -function hex(hash) { - if (Buffer.isBuffer(hash)) - hash = hash.toString('hex'); - assert(typeof hash === 'string'); - return hash; -} - -/* - * Expose - */ - -module.exports = layout; diff --git a/lib/mempool/layout.js b/lib/mempool/layout.js index fe96a120..1914595c 100644 --- a/lib/mempool/layout.js +++ b/lib/mempool/layout.js @@ -6,28 +6,20 @@ 'use strict'; -const assert = require('assert'); -const Key = require('bdb/lib/key'); +const bdb = require('bdb'); /* * Database Layout: - * R -> tip hash * V -> db version - * e[id][hash] -> entry + * R -> tip hash + * e[hash] -> entry */ -const R = new Key('R'); -const V = new Key('v'); -const F = new Key('F'); -const e = new Key('e', ['hash256']); - const layout = { - binary: true, - R: R.build.bind(R), - V: V.build.bind(V), - F: F.build.bind(F), - e: e.build.bind(e), - ee: e.parse.bind(e) + V: bdb.key('v'), + R: bdb.key('R'), + F: bdb.key('F'), + e: bdb.key('e', ['hash256']) }; /* diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index a0bc5572..28c3f802 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -9,7 +9,7 @@ const assert = require('assert'); const path = require('path'); const EventEmitter = require('events'); -const BDB = require('bdb'); +const bdb = require('bdb'); const {encoding} = require('bufio'); const {RollingFilter} = require('bfilter'); const common = require('../blockchain/common'); @@ -1974,7 +1974,6 @@ class MempoolOptions { this.maxFiles = 64; this.cacheSize = 32 << 20; this.compression = true; - this.bufferKeys = layout.binary; this.persistent = false; @@ -2386,11 +2385,11 @@ class MempoolCache { this.batch = null; if (options.persistent) - this.db = new BDB(options); + this.db = bdb.create(options); } async getVersion() { - const data = await this.db.get(layout.V); + const data = await this.db.get(layout.V.build()); if (!data) return -1; @@ -2399,7 +2398,7 @@ class MempoolCache { } async getTip() { - const hash = await this.db.get(layout.R); + const hash = await this.db.get(layout.R.build()); if (!hash) return null; @@ -2408,7 +2407,7 @@ class MempoolCache { } async getFees() { - const data = await this.db.get(layout.F); + const data = await this.db.get(layout.F.build()); if (!data) return null; @@ -2427,16 +2426,16 @@ class MempoolCache { getEntries() { return this.db.values({ - gte: layout.e(encoding.ZERO_HASH), - lte: layout.e(encoding.MAX_HASH), - parse: MempoolEntry.fromRaw + gte: layout.e.min(), + lte: layout.e.max(), + parse: data => MempoolEntry.fromRaw(data) }); } getKeys() { return this.db.keys({ - gte: layout.e(encoding.ZERO_HASH), - lte: layout.e(encoding.MAX_HASH) + gte: layout.e.min(), + lte: layout.e.max() }); } @@ -2463,28 +2462,28 @@ class MempoolCache { if (!this.db) return; - this.batch.put(layout.e(entry.tx.hash()), entry.toRaw()); + this.batch.put(layout.e.build(entry.hash()), entry.toRaw()); } remove(hash) { if (!this.db) return; - this.batch.del(layout.e(hash)); + this.batch.del(layout.e.build(hash)); } - sync(hash) { + sync(tip) { if (!this.db) return; - this.batch.put(layout.R, Buffer.from(hash, 'hex')); + this.batch.put(layout.R.build(), Buffer.from(tip, 'hex')); } writeFees(fees) { if (!this.db) return; - this.batch.put(layout.F, fees.toRaw()); + this.batch.put(layout.F.build(), fees.toRaw()); } clear() { @@ -2503,8 +2502,8 @@ class MempoolCache { async init(hash) { const batch = this.db.batch(); - batch.put(layout.V, encoding.u32(MempoolCache.VERSION)); - batch.put(layout.R, Buffer.from(hash, 'hex')); + batch.put(layout.V.build(), encoding.u32(MempoolCache.VERSION)); + batch.put(layout.R.build(), Buffer.from(hash, 'hex')); await batch.write(); } @@ -2555,9 +2554,9 @@ class MempoolCache { for (const key of keys) batch.del(key); - batch.put(layout.V, encoding.u32(MempoolCache.VERSION)); - batch.put(layout.R, Buffer.from(this.chain.tip.hash, 'hex')); - batch.del(layout.F); + batch.put(layout.V.build(), encoding.u32(MempoolCache.VERSION)); + batch.put(layout.R.build(), Buffer.from(this.chain.tip.hash, 'hex')); + batch.del(layout.F.build()); await batch.write(); diff --git a/lib/wallet/layout-browser.js b/lib/wallet/layout-browser.js deleted file mode 100644 index b932df39..00000000 --- a/lib/wallet/layout-browser.js +++ /dev/null @@ -1,269 +0,0 @@ -/*! - * layout-browser.js - walletdb and txdb layout for browser. - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); -const layouts = exports; - -layouts.walletdb = { - binary: false, - p: function p(hash) { - assert(typeof hash === 'string'); - return 'p' + hash; - }, - pp: function pp(key) { - assert(typeof key === 'string'); - return key.slice(1); - }, - P: function P(wid, hash) { - assert(typeof hash === 'string'); - return 'p' + hex32(wid) + hash; - }, - Pp: function Pp(key) { - assert(typeof key === 'string'); - return key.slice(11); - }, - r: function r(wid, index, hash) { - assert(typeof hash === 'string'); - return 'r' + hex32(wid) + hex32(index) + hash; - }, - rr: function rr(key) { - assert(typeof key === 'string'); - return key.slice(21); - }, - w: function w(wid) { - return 'w' + hex32(wid); - }, - ww: function ww(key) { - assert(typeof key === 'string'); - return parseInt(key.slice(1), 16); - }, - l: function l(id) { - assert(typeof id === 'string'); - return 'l' + id; - }, - ll: function ll(key) { - assert(typeof key === 'string'); - return key.slice(1); - }, - a: function a(wid, index) { - return 'a' + hex32(wid) + hex32(index); - }, - i: function i(wid, name) { - assert(typeof name === 'string'); - return 'i' + hex32(wid) + name; - }, - ii: function ii(key) { - assert(typeof key === 'string'); - return [parseInt(key.slice(1, 9), 16), key.slice(9)]; - }, - n: function n(wid, index) { - return 'n' + hex32(wid) + hex32(index); - }, - R: 'R', - h: function h(height) { - return 'h' + hex32(height); - }, - b: function b(height) { - return 'b' + hex32(height); - }, - bb: function bb(key) { - assert(typeof key === 'string'); - return parseInt(key.slice(1), 16); - }, - o: function o(hash, index) { - assert(typeof hash === 'string'); - return 'o' + hash + hex32(index); - }, - oo: function oo(key) { - return [key.slice(1, 65), parseInt(key.slice(65), 16)]; - }, - T: function T(hash) { - assert(typeof hash === 'string'); - return 'T' + hash; - }, - Tt: function Tt(key) { - return [key.slice(1, 65)]; - } -}; - -layouts.txdb = { - binary: false, - prefix: function prefix(wid) { - return 't' + hex32(wid); - }, - R: 'R', - r: function r(acct) { - assert(typeof acct === 'number'); - return 'r' + hex32(acct); - }, - rr: function rr(key) { - assert(typeof key === 'string'); - return parseInt(key.slice(1), 16); - }, - hi: function hi(ch, hash, index) { - assert(typeof hash === 'string'); - return ch + hash + hex32(index); - }, - hii: function hii(key) { - assert(typeof key === 'string'); - return [key.slice(1, 65), parseInt(key.slice(65), 16)]; - }, - ih: function ih(ch, index, hash) { - assert(typeof hash === 'string'); - return ch + hex32(index) + hash; - }, - ihh: function ihh(key) { - assert(typeof key === 'string'); - return [parseInt(key.slice(1, 9), 16), key.slice(9)]; - }, - iih: function iih(ch, index, num, hash) { - assert(typeof hash === 'string'); - return ch + hex32(index) + hex32(num) + hash; - }, - iihh: function iihh(key) { - assert(typeof key === 'string'); - return [ - parseInt(key.slice(1, 9), 16), - parseInt(key.slice(9, 17), 16), - key.slice(17) - ]; - }, - ihi: function ihi(ch, index, hash, num) { - assert(typeof hash === 'string'); - return ch + hex32(index) + hash + hex32(num); - }, - ihii: function ihii(key) { - assert(typeof key === 'string'); - return [ - parseInt(key.slice(1, 9), 16), - key.slice(9, 73), - parseInt(key.slice(73), 16) - ]; - }, - ha: function ha(ch, hash) { - assert(typeof hash === 'string'); - return ch + hash; - }, - haa: function haa(key) { - assert(typeof key === 'string'); - return key.slice(1); - }, - t: function t(hash) { - return this.ha('t', hash); - }, - tt: function tt(key) { - return this.haa(key); - }, - c: function c(hash, index) { - return this.hi('c', hash, index); - }, - cc: function cc(key) { - return this.hii(key); - }, - d: function d(hash, index) { - return this.hi('d', hash, index); - }, - dd: function dd(key) { - return this.hii(key); - }, - s: function s(hash, index) { - return this.hi('s', hash, index); - }, - ss: function ss(key) { - return this.hii(key); - }, - S: function S(hash, index) { - return this.hi('S', hash, index); - }, - Ss: function Ss(key) { - return this.hii(key); - }, - p: function p(hash) { - return this.ha('p', hash); - }, - pp: function pp(key) { - return this.haa(key); - }, - m: function m(time, hash) { - return this.ih('m', time, hash); - }, - mm: function mm(key) { - return this.ihh(key); - }, - h: function h(height, hash) { - return this.ih('h', height, hash); - }, - hh: function hh(key) { - return this.ihh(key); - }, - T: function T(account, hash) { - return this.ih('T', account, hash); - }, - Tt: function Tt(key) { - return this.ihh(key); - }, - P: function P(account, hash) { - return this.ih('P', account, hash); - }, - Pp: function Pp(key) { - return this.ihh(key); - }, - M: function M(account, time, hash) { - return this.iih('M', account, time, hash); - }, - Mm: function Mm(key) { - return this.iihh(key); - }, - H: function H(account, height, hash) { - return this.iih('H', account, height, hash); - }, - Hh: function Hh(key) { - return this.iihh(key); - }, - C: function C(account, hash, index) { - return this.ihi('C', account, hash, index); - }, - Cc: function Cc(key) { - return this.ihii(key); - }, - b: function b(height) { - return 'b' + hex32(height); - }, - bb: function bb(key) { - assert(typeof key === 'string'); - return parseInt(key.slice(1), 16); - } -}; - -function hex32(num) { - assert(typeof num === 'number'); - assert(num >= 0); - - num = num.toString(16); - - switch (num.length) { - case 1: - return '0000000' + num; - case 2: - return '000000' + num; - case 3: - return '00000' + num; - case 4: - return '0000' + num; - case 5: - return '000' + num; - case 6: - return '00' + num; - case 7: - return '0' + num; - case 8: - return num; - } - - throw new Error('Number too big.'); -} diff --git a/lib/wallet/layout.js b/lib/wallet/layout.js index 8a4b9570..17875616 100644 --- a/lib/wallet/layout.js +++ b/lib/wallet/layout.js @@ -6,13 +6,14 @@ 'use strict'; -const assert = require('assert'); -const layouts = exports; - -const Key = require('bdb/lib/key'); +const bdb = require('bdb'); /* * Wallet Database Layout: + * V -> db version + * O -> flags + * R -> chain sync state + * D -> wallet id depth * p[addr-hash] -> wallet ids * P[wid][addr-hash] -> path data * r[wid][index][hash] -> path account index @@ -21,58 +22,37 @@ const Key = require('bdb/lib/key'); * a[wid][index] -> account * i[wid][name] -> account index * n[wid][index] -> account name - * t[wid]* -> txdb - * R -> chain sync state * h[height] -> recent block hash * b[height] -> block->wid map * o[hash][index] -> outpoint->wid map * T[hash] -> tx->wid map + * t[wid]* -> txdb */ -const p = new Key('p', ['hash']); -const P = new Key('P', ['uint32', 'hash']); -const r = new Key('r', ['uint32', 'uint32', 'hash']); -const w = new Key('w', ['uint32']); -const l = new Key('l', ['ascii']); -const a = new Key('a', ['uint32', 'uint32']); -const i = new Key('i', ['uint32', 'ascii']); -const n = new Key('n', ['uint32', 'uint32']); -const R = new Key('R'); -const h = new Key('h', ['uint32']); -const b = new Key('b', ['uint32']); -const o = new Key('o', ['hash256', 'uint32']); -const T = new Key('T', ['hash256']); - -// Pp -// rr -layouts.walletdb = { - binary: true, - p: p.build.bind(p), - pp: p.parse.bind(p), - P: P.build.bind(P), - Pp: P.parse.bind(P), - r: r.build.bind(r), - rr: r.parse.bind(r), - w: w.build.bind(w), - ww: w.parse.bind(w), - l: l.build.bind(l), - ll: l.parse.bind(l), - a: a.build.bind(a), - i: i.build.bind(i), - ii: i.parse.bind(i), - n: n.build.bind(n), - R: R.build.bind(R), - h: h.build.bind(h), - b: b.build.bind(b), - bb: b.parse.bind(b), - o: o.build.bind(o), - oo: o.parse.bind(o), - T: T.build.bind(T), - Tt: T.parse.bind(T) +exports.wdb = { + V: bdb.key('V'), + O: bdb.key('O'), + R: bdb.key('R'), + D: bdb.key('D'), + p: bdb.key('p', ['hash']), + P: bdb.key('P', ['uint32', 'hash']), + r: bdb.key('r', ['uint32', 'uint32', 'hash']), + w: bdb.key('w', ['uint32']), + l: bdb.key('l', ['ascii']), + a: bdb.key('a', ['uint32', 'uint32']), + i: bdb.key('i', ['uint32', 'ascii']), + n: bdb.key('n', ['uint32', 'uint32']), + h: bdb.key('h', ['uint32']), + b: bdb.key('b', ['uint32']), + o: bdb.key('o', ['hash256', 'uint32']), + T: bdb.key('T', ['hash256']), + t: bdb.key('t', ['uint32']) }; /* * TXDB Database Layout: + * R -> wallet balance + * r[account] -> account balance * t[hash] -> extended tx * c[hash][index] -> coin * d[hash][index] -> undo coin @@ -85,58 +65,24 @@ layouts.walletdb = { * M[account][time][hash] -> dummy (tx by time + account) * H[account][height][hash] -> dummy (tx by height + account) * C[account][hash][index] -> dummy (coin by account) - * r[hash] -> dummy (replace by fee chain) + * b[height] -> block record */ -{ -const prefix = new Key('t', ['uint32']); -const R = new Key('R'); -const r = new Key('r', ['uint32']); -const t = new Key('t', ['hash256']); -const c = new Key('c', ['hash256', 'uint32']); -const d = new Key('d', ['hash256', 'uint32']); -const s = new Key('s', ['hash256', 'uint32']); -const p = new Key('p', ['hash256']); -const m = new Key('m', ['uint32', 'hash256']); -const h = new Key('h', ['uint32', 'hash256']); -const T = new Key('T', ['uint32', 'hash256']); -const P = new Key('P', ['uint32', 'hash256']); -const M = new Key('M', ['uint32', 'uint32', 'hash256']); -const H = new Key('H', ['uint32', 'uint32', 'hash256']); -const C = new Key('C', ['uint32', 'hash256', 'uint32']); -const b = new Key('b', ['uint32']); - -layouts.txdb = { - binary: true, - prefix: prefix.build.bind(prefix), - R: R.build.bind(R), - r: r.build.bind(r), - rr: r.parse.bind(r), - t: t.build.bind(t), - tt: t.parse.bind(t), - c: c.build.bind(c), - cc: c.parse.bind(c), - d: d.build.bind(d), - dd: d.parse.bind(d), - s: s.build.bind(s), - ss: s.parse.bind(s), - p: p.build.bind(p), - pp: p.parse.bind(p), - m: m.build.bind(m), - mm: m.parse.bind(m), - h: h.build.bind(h), - hh: h.parse.bind(h), - T: T.build.bind(T), - Tt: T.parse.bind(T), - P: P.build.bind(P), - Pp: P.parse.bind(P), - M: M.build.bind(M), - Mm: M.parse.bind(M), - H: H.build.bind(H), - Hh: H.parse.bind(H), - C: C.build.bind(C), - Cc: C.parse.bind(C), - b: b.build.bind(b), - bb: b.parse.bind(b) +exports.txdb = { + prefix: bdb.key('t', ['uint32']), + R: bdb.key('R'), + r: bdb.key('r', ['uint32']), + t: bdb.key('t', ['hash256']), + c: bdb.key('c', ['hash256', 'uint32']), + d: bdb.key('d', ['hash256', 'uint32']), + s: bdb.key('s', ['hash256', 'uint32']), + p: bdb.key('p', ['hash256']), + m: bdb.key('m', ['uint32', 'hash256']), + h: bdb.key('h', ['uint32', 'hash256']), + T: bdb.key('T', ['uint32', 'hash256']), + P: bdb.key('P', ['uint32', 'hash256']), + M: bdb.key('M', ['uint32', 'uint32', 'hash256']), + H: bdb.key('H', ['uint32', 'uint32', 'hash256']), + C: bdb.key('C', ['uint32', 'hash256', 'uint32']), + b: bdb.key('b', ['uint32']) }; -} diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index 1c7a44b4..8fcd4020 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -49,8 +49,10 @@ class TXDB { */ async open(wallet) { + const prefix = layout.prefix.build(wallet.wid); + this.wid = wallet.wid; - this.bucket = this.db.bucket(layout.prefix(this.wid)); + this.bucket = this.db.bucket(prefix); this.wallet = wallet; } @@ -106,8 +108,8 @@ class TXDB { async saveCredit(b, credit, path) { const {coin} = credit; - b.put(layout.c(coin.hash, coin.index), credit.toRaw()); - b.put(layout.C(path.account, coin.hash, coin.index), null); + b.put(layout.c.build(coin.hash, coin.index), credit.toRaw()); + b.put(layout.C.build(path.account, coin.hash, coin.index), null); return this.addOutpointMap(b, coin.hash, coin.index); } @@ -121,8 +123,8 @@ class TXDB { async removeCredit(b, credit, path) { const {coin} = credit; - b.del(layout.c(coin.hash, coin.index)); - b.del(layout.C(path.account, coin.hash, coin.index)); + b.del(layout.c.build(coin.hash, coin.index)); + b.del(layout.C.build(path.account, coin.hash, coin.index)); return this.removeOutpointMap(b, coin.hash, coin.index); } @@ -137,8 +139,8 @@ class TXDB { spendCredit(b, credit, tx, index) { const prevout = tx.inputs[index].prevout; const spender = Outpoint.fromTX(tx, index); - b.put(layout.s(prevout.hash, prevout.index), spender.toRaw()); - b.put(layout.d(spender.hash, spender.index), credit.coin.toRaw()); + b.put(layout.s.build(prevout.hash, prevout.index), spender.toRaw()); + b.put(layout.d.build(spender.hash, spender.index), credit.coin.toRaw()); } /** @@ -150,8 +152,8 @@ class TXDB { unspendCredit(b, tx, index) { const prevout = tx.inputs[index].prevout; const spender = Outpoint.fromTX(tx, index); - b.del(layout.s(prevout.hash, prevout.index)); - b.del(layout.d(spender.hash, spender.index)); + b.del(layout.s.build(prevout.hash, prevout.index)); + b.del(layout.d.build(spender.hash, spender.index)); } /** @@ -163,7 +165,7 @@ class TXDB { async writeInput(b, tx, index) { const prevout = tx.inputs[index].prevout; const spender = Outpoint.fromTX(tx, index); - b.put(layout.s(prevout.hash, prevout.index), spender.toRaw()); + b.put(layout.s.build(prevout.hash, prevout.index), spender.toRaw()); return this.addOutpointMap(b, prevout.hash, prevout.index); } @@ -175,7 +177,7 @@ class TXDB { async removeInput(b, tx, index) { const prevout = tx.inputs[index].prevout; - b.del(layout.s(prevout.hash, prevout.index)); + b.del(layout.s.build(prevout.hash, prevout.index)); return this.removeOutpointMap(b, prevout.hash, prevout.index); } @@ -187,7 +189,7 @@ class TXDB { async updateBalance(b, state) { const balance = await this.getWalletBalance(); state.applyTo(balance); - b.put(layout.R(), balance.toRaw()); + b.put(layout.R.build(), balance.toRaw()); return balance; } @@ -200,7 +202,7 @@ class TXDB { async updateAccountBalance(b, acct, delta) { const balance = await this.getAccountBalance(acct); delta.applyTo(balance); - b.put(layout.r(acct), balance.toRaw()); + b.put(layout.r.build(acct), balance.toRaw()); return balance; } @@ -212,7 +214,7 @@ class TXDB { */ async getSpent(hash, index) { - const data = await this.bucket.get(layout.s(hash, index)); + const data = await this.bucket.get(layout.s.build(hash, index)); if (!data) return null; @@ -228,7 +230,7 @@ class TXDB { */ isSpent(hash, index) { - return this.bucket.has(layout.s(hash, index)); + return this.bucket.has(layout.s.build(hash, index)); } /** @@ -238,7 +240,7 @@ class TXDB { */ addBlockMap(b, height) { - return this.wdb.addBlockMap(b.batch, height, this.wid); + return this.wdb.addBlockMap(b.root(), height, this.wid); } /** @@ -248,7 +250,7 @@ class TXDB { */ removeBlockMap(b, height) { - return this.wdb.removeBlockMap(b.batch, height, this.wid); + return this.wdb.removeBlockMap(b.root(), height, this.wid); } /** @@ -258,7 +260,7 @@ class TXDB { */ addTXMap(b, hash) { - return this.wdb.addTXMap(b.batch, hash, this.wid); + return this.wdb.addTXMap(b.root(), hash, this.wid); } /** @@ -268,7 +270,7 @@ class TXDB { */ removeTXMap(b, hash) { - return this.wdb.removeTXMap(b.batch, hash, this.wid); + return this.wdb.removeTXMap(b.root(), hash, this.wid); } /** @@ -279,7 +281,7 @@ class TXDB { */ addOutpointMap(b, hash, index) { - return this.wdb.addOutpointMap(b.batch, hash, index, this.wid); + return this.wdb.addOutpointMap(b.root(), hash, index, this.wid); } /** @@ -290,7 +292,7 @@ class TXDB { */ removeOutpointMap(b, hash, index) { - return this.wdb.removeOutpointMap(b.batch, hash, index, this.wid); + return this.wdb.removeOutpointMap(b.root(), hash, index, this.wid); } /** @@ -300,9 +302,9 @@ class TXDB { getBlocks() { return this.bucket.keys({ - gte: layout.b(0), - lte: layout.b(0xffffffff), - parse: key => layout.bb(key) + gte: layout.b.min(), + lte: layout.b.max(), + parse: key => layout.b.parse(key) }); } @@ -313,7 +315,7 @@ class TXDB { */ async getBlock(height) { - const data = await this.bucket.get(layout.b(height)); + const data = await this.bucket.get(layout.b.build(height)); if (!data) return null; @@ -329,7 +331,7 @@ class TXDB { */ async addBlock(b, hash, block) { - const key = layout.b(block.height); + const key = layout.b.build(block.height); const data = await this.bucket.get(key); if (!data) { @@ -357,7 +359,7 @@ class TXDB { */ async removeBlock(b, hash, height) { - const key = layout.b(height); + const key = layout.b.build(height); const data = await this.bucket.get(key); if (!data) @@ -396,11 +398,11 @@ class TXDB { return; if (block.hashes.size === 0) { - b.del(layout.b(height)); + b.del(layout.b.build(height)); return; } - b.put(layout.b(height), block.toRaw()); + b.put(layout.b.build(height), block.toRaw()); } /** @@ -552,13 +554,13 @@ class TXDB { return null; // Save and index the transaction record. - b.put(layout.t(hash), wtx.toRaw()); - b.put(layout.m(wtx.mtime, hash), null); + b.put(layout.t.build(hash), wtx.toRaw()); + b.put(layout.m.build(wtx.mtime, hash), null); if (!block) - b.put(layout.p(hash), null); + b.put(layout.p.build(hash), null); else - b.put(layout.h(height, hash), null); + b.put(layout.h.build(height, hash), null); // Do some secondary indexing for account-based // queries. This saves us a lot of time for @@ -566,13 +568,13 @@ class TXDB { for (const [acct, delta] of state.accounts) { await this.updateAccountBalance(b, acct, delta); - b.put(layout.T(acct, hash), null); - b.put(layout.M(acct, wtx.mtime, hash), null); + b.put(layout.T.build(acct, hash), null); + b.put(layout.M.build(acct, wtx.mtime, hash), null); if (!block) - b.put(layout.P(acct, hash), null); + b.put(layout.P.build(acct, hash), null); else - b.put(layout.H(acct, height, hash), null); + b.put(layout.H.build(acct, height, hash), null); } // Update block records. @@ -707,15 +709,15 @@ class TXDB { // Save the new serialized transaction as // the block-related properties have been // updated. Also reindex for height. - b.put(layout.t(hash), wtx.toRaw()); - b.del(layout.p(hash)); - b.put(layout.h(height, hash), null); + b.put(layout.t.build(hash), wtx.toRaw()); + b.del(layout.p.build(hash)); + b.put(layout.h.build(height, hash), null); // Secondary indexing also needs to change. for (const [acct, delta] of state.accounts) { await this.updateAccountBalance(b, acct, delta); - b.del(layout.P(acct, hash)); - b.put(layout.H(acct, height, hash), null); + b.del(layout.P.build(acct, hash)); + b.put(layout.H.build(acct, height, hash), null); } await this.removeTXMap(b, hash); @@ -829,25 +831,25 @@ class TXDB { // Remove the transaction data // itself as well as unindex. - b.del(layout.t(hash)); - b.del(layout.m(wtx.mtime, hash)); + b.del(layout.t.build(hash)); + b.del(layout.m.build(wtx.mtime, hash)); if (!block) - b.del(layout.p(hash)); + b.del(layout.p.build(hash)); else - b.del(layout.h(height, hash)); + b.del(layout.h.build(height, hash)); // Remove all secondary indexing. for (const [acct, delta] of state.accounts) { await this.updateAccountBalance(b, acct, delta); - b.del(layout.T(acct, hash)); - b.del(layout.M(acct, wtx.mtime, hash)); + b.del(layout.T.build(acct, hash)); + b.del(layout.M.build(acct, wtx.mtime, hash)); if (!block) - b.del(layout.P(acct, hash)); + b.del(layout.P.build(acct, hash)); else - b.del(layout.H(acct, height, hash)); + b.del(layout.H.build(acct, height, hash)); } // Update block records. @@ -1027,15 +1029,15 @@ class TXDB { // We need to update the now-removed // block properties and reindex due // to the height change. - b.put(layout.t(hash), wtx.toRaw()); - b.put(layout.p(hash), null); - b.del(layout.h(height, hash)); + b.put(layout.t.build(hash), wtx.toRaw()); + b.put(layout.p.build(hash), null); + b.del(layout.h.build(height, hash)); // Secondary indexing also needs to change. for (const [acct, delta] of state.accounts) { await this.updateAccountBalance(b, acct, delta); - b.put(layout.P(acct, hash), null); - b.del(layout.H(acct, height, hash)); + b.put(layout.P.build(acct, hash), null); + b.del(layout.H.build(acct, height, hash)); } // Commit state due to unconfirmed @@ -1223,10 +1225,10 @@ class TXDB { getAccountHistoryHashes(acct) { assert(typeof acct === 'number'); return this.bucket.keys({ - gte: layout.T(acct, encoding.NULL_HASH), - lte: layout.T(acct, encoding.HIGH_HASH), + gte: layout.T.min(acct), + lte: layout.T.max(acct), parse: (key) => { - const [, hash] = layout.Tt(key); + const [, hash] = layout.T.parse(key); return hash; } }); @@ -1245,9 +1247,9 @@ class TXDB { return this.getAccountHistoryHashes(acct); return this.bucket.keys({ - gte: layout.t(encoding.NULL_HASH), - lte: layout.t(encoding.HIGH_HASH), - parse: key => layout.tt(key) + gte: layout.t.min(), + lte: layout.t.max(), + parse: key => layout.t.parse(key) }); } @@ -1260,10 +1262,10 @@ class TXDB { getAccountPendingHashes(acct) { assert(typeof acct === 'number'); return this.bucket.keys({ - gte: layout.P(acct, encoding.NULL_HASH), - lte: layout.P(acct, encoding.HIGH_HASH), + gte: layout.P.min(acct), + lte: layout.P.max(acct), parse: (key) => { - const [, hash] = layout.Pp(key); + const [, hash] = layout.P.parse(key); return hash; } }); @@ -1282,9 +1284,9 @@ class TXDB { return this.getAccountPendingHashes(acct); return this.bucket.keys({ - gte: layout.p(encoding.NULL_HASH), - lte: layout.p(encoding.HIGH_HASH), - parse: key => layout.pp(key) + gte: layout.p.min(), + lte: layout.p.max(), + parse: key => layout.p.parse(key) }); } @@ -1297,10 +1299,10 @@ class TXDB { getAccountOutpoints(acct) { assert(typeof acct === 'number'); return this.bucket.keys({ - gte: layout.C(acct, encoding.NULL_HASH, 0), - lte: layout.C(acct, encoding.HIGH_HASH, 0xffffffff), + gte: layout.C.min(acct), + lte: layout.C.max(acct), parse: (key) => { - const [, hash, index] = layout.Cc(key); + const [, hash, index] = layout.C.parse(key); return new Outpoint(hash, index); } }); @@ -1319,10 +1321,10 @@ class TXDB { return this.getAccountOutpoints(acct); return this.bucket.keys({ - gte: layout.c(encoding.NULL_HASH, 0), - lte: layout.c(encoding.HIGH_HASH, 0xffffffff), + gte: layout.c.min(), + lte: layout.c.max(), parse: (key) => { - const [hash, index] = layout.cc(key); + const [hash, index] = layout.c.parse(key); return new Outpoint(hash, index); } }); @@ -1346,12 +1348,12 @@ class TXDB { const end = options.end || 0xffffffff; return this.bucket.keys({ - gte: layout.H(acct, start, encoding.NULL_HASH), - lte: layout.H(acct, end, encoding.HIGH_HASH), + gte: layout.H.min(acct, start), + lte: layout.H.max(acct, end), limit: options.limit, reverse: options.reverse, parse: (key) => { - const [,, hash] = layout.Hh(key); + const [,, hash] = layout.H.parse(key); return hash; } }); @@ -1378,12 +1380,12 @@ class TXDB { const end = options.end || 0xffffffff; return this.bucket.keys({ - gte: layout.h(start, encoding.NULL_HASH), - lte: layout.h(end, encoding.HIGH_HASH), + gte: layout.h.min(start), + lte: layout.h.max(end), limit: options.limit, reverse: options.reverse, parse: (key) => { - const [, hash] = layout.hh(key); + const [, hash] = layout.h.parse(key); return hash; } }); @@ -1417,12 +1419,12 @@ class TXDB { const end = options.end || 0xffffffff; return this.bucket.keys({ - gte: layout.M(acct, start, encoding.NULL_HASH), - lte: layout.M(acct, end, encoding.HIGH_HASH), + gte: layout.M.min(acct, start), + lte: layout.M.max(acct, end), limit: options.limit, reverse: options.reverse, parse: (key) => { - const [,, hash] = layout.Mm(key); + const [,, hash] = layout.M.parse(key); return hash; } }); @@ -1449,12 +1451,12 @@ class TXDB { const end = options.end || 0xffffffff; return this.bucket.keys({ - gte: layout.m(start, encoding.NULL_HASH), - lte: layout.m(end, encoding.HIGH_HASH), + gte: layout.m.min(start), + lte: layout.m.max(end), limit: options.limit, reverse: options.reverse, parse: (key) => { - const [, hash] = layout.mm(key); + const [, hash] = layout.m.parse(key); return hash; } }); @@ -1515,8 +1517,8 @@ class TXDB { // Fast case return this.bucket.values({ - gte: layout.t(encoding.NULL_HASH), - lte: layout.t(encoding.HIGH_HASH), + gte: layout.t.min(), + lte: layout.t.max(), parse: data => TXRecord.fromRaw(data) }); } @@ -1574,10 +1576,10 @@ class TXDB { // Fast case return this.bucket.range({ - gte: layout.c(encoding.NULL_HASH, 0x00000000), - lte: layout.c(encoding.HIGH_HASH, 0xffffffff), + gte: layout.c.min(), + lte: layout.c.max(), parse: (key, value) => { - const [hash, index] = layout.cc(key); + const [hash, index] = layout.c.parse(key); const credit = Credit.fromRaw(value); credit.coin.hash = hash; credit.coin.index = index; @@ -1622,10 +1624,10 @@ class TXDB { credits.push(null); await this.bucket.range({ - gte: layout.d(hash, 0x00000000), - lte: layout.d(hash, 0xffffffff), + gte: layout.d.min(hash), + lte: layout.d.max(hash), parse: (key, value) => { - const [, index] = layout.dd(key); + const [, index] = layout.d.parse(key); const coin = Coin.fromRaw(value); const input = tx.inputs[index]; assert(input); @@ -1759,7 +1761,7 @@ class TXDB { */ async getTX(hash) { - const raw = await this.bucket.get(layout.t(hash)); + const raw = await this.bucket.get(layout.t.build(hash)); if (!raw) return null; @@ -1821,6 +1823,7 @@ class TXDB { for (let i = 0; i < tx.inputs.length; i++) { const coin = coins[i]; + let path = null; if (coin) @@ -1845,7 +1848,7 @@ class TXDB { */ hasTX(hash) { - return this.bucket.has(layout.t(hash)); + return this.bucket.has(layout.t.build(hash)); } /** @@ -1872,7 +1875,7 @@ class TXDB { */ async getCredit(hash, index) { - const data = await this.bucket.get(layout.c(hash, index)); + const data = await this.bucket.get(layout.c.build(hash, index)); if (!data) return null; @@ -1892,7 +1895,7 @@ class TXDB { */ async getSpentCoin(spent, prevout) { - const data = await this.bucket.get(layout.d(spent.hash, spent.index)); + const data = await this.bucket.get(layout.d.build(spent.hash, spent.index)); if (!data) return null; @@ -1911,7 +1914,7 @@ class TXDB { */ hasSpentCoin(spent) { - return this.bucket.has(layout.d(spent.hash, spent.index)); + return this.bucket.has(layout.d.build(spent.hash, spent.index)); } /** @@ -1936,7 +1939,7 @@ class TXDB { coin.height = height; - b.put(layout.d(spent.hash, spent.index), coin.toRaw()); + b.put(layout.d.build(spent.hash, spent.index), coin.toRaw()); } /** @@ -1946,7 +1949,7 @@ class TXDB { */ async hasCoin(hash, index) { - return this.bucket.has(layout.c(hash, index)); + return this.bucket.has(layout.c.build(hash, index)); } /** @@ -1970,7 +1973,7 @@ class TXDB { */ async getWalletBalance() { - const data = await this.bucket.get(layout.R()); + const data = await this.bucket.get(layout.R.build()); if (!data) return new Balance(); @@ -1985,7 +1988,7 @@ class TXDB { */ async getAccountBalance(acct) { - const data = await this.bucket.get(layout.r(acct)); + const data = await this.bucket.get(layout.r.build(acct)); if (!data) return new Balance(acct); @@ -2036,7 +2039,7 @@ class TXDB { */ async abandon(hash) { - const result = await this.bucket.has(layout.p(hash)); + const result = await this.bucket.has(layout.p.build(hash)); if (!result) throw new Error('TX not eligible.'); diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index ab53d67c..918d3f15 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -15,7 +15,6 @@ const bio = require('bufio'); const hash160 = require('bcrypto/lib/hash160'); const hash256 = require('bcrypto/lib/hash256'); const cleanse = require('bcrypto/lib/cleanse'); -const Network = require('../protocol/network'); const TXDB = require('./txdb'); const Path = require('./path'); const common = require('./common'); @@ -78,6 +77,9 @@ class Wallet extends EventEmitter { */ fromOptions(options) { + if (!options) + return this; + let key = options.master; let id, token, mnemonic; @@ -618,6 +620,9 @@ class Wallet extends EventEmitter { this.accountDepth += 1; this.save(b); + if (this.accountDepth === 0) + this.increment(b); + await b.write(); return account; @@ -840,6 +845,15 @@ class Wallet extends EventEmitter { return this.wdb.save(b, this); } + /** + * Increment the wid depth. + * @returns {Promise} + */ + + increment(b) { + return this.wdb.increment(b, this.wid); + } + /** * Test whether the wallet possesses an address. * @param {Address|Hash} address diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index c57f996e..5c5eaaee 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -13,7 +13,7 @@ const EventEmitter = require('events'); const bio = require('bufio'); const {BloomFilter} = require('bfilter'); const {Lock, MapLock} = require('bmutex'); -const BDB = require('bdb'); +const bdb = require('bdb'); const Logger = require('blgr'); const ccmp = require('bcrypto/lib/ccmp'); const aes = require('bcrypto/lib/aes'); @@ -29,7 +29,8 @@ const records = require('./records'); const NullClient = require('./nullclient'); const {encoding} = bio; const {u32} = encoding; -const layout = layouts.walletdb; +const layout = layouts.wdb; +const tlayout = layouts.txdb; const { ChainState, @@ -61,7 +62,7 @@ class WalletDB extends EventEmitter { this.workers = this.options.workers; this.client = this.options.client || new NullClient(this); this.feeRate = this.options.feeRate; - this.db = new BDB(this.options); + this.db = bdb.create(this.options); this.primary = null; this.state = new ChainState(); @@ -174,7 +175,9 @@ class WalletDB extends EventEmitter { async open() { await this.db.open(); - await this.db.checkVersion('V', 7); + await this.db.verify(layout.V.build(), 'wallet', 7); + + await this.verifyNetwork(); this.depth = await this.getDepth(); @@ -203,6 +206,29 @@ class WalletDB extends EventEmitter { this.primary = wallet; } + /** + * Verify network. + * @returns {Promise} + */ + + async verifyNetwork() { + const raw = await this.db.get(layout.O.build()); + + if (!raw) { + const b = this.db.batch(); + b.put(layout.O.build(), u32(this.network.magic)); + return b.write(); + } + + const magic = raw.readUInt32LE(0, true); + const network = Network.fromMagic(magic); + + if (network !== this.network) + throw new Error('Network mismatch for WalletDB.'); + + return undefined; + } + /** * Close the walletdb, wait for the database to close. * @returns {Promise} @@ -227,14 +253,14 @@ class WalletDB extends EventEmitter { async watch() { const piter = this.db.iterator({ - gte: layout.p(encoding.NULL_HASH), - lte: layout.p(encoding.HIGH_HASH) + gte: layout.p.min(), + lte: layout.p.max() }); let hashes = 0; await piter.each((key) => { - const data = layout.pp(key); + const data = layout.p.parse(key); this.filter.add(data, 'hex'); @@ -244,14 +270,14 @@ class WalletDB extends EventEmitter { this.logger.info('Added %d hashes to WalletDB filter.', hashes); const oiter = this.db.iterator({ - gte: layout.o(encoding.NULL_HASH, 0), - lte: layout.o(encoding.HIGH_HASH, 0xffffffff) + gte: layout.o.min(), + lte: layout.o.max() }); let outpoints = 0; await oiter.each((key) => { - const [hash, index] = layout.oo(key); + const [hash, index] = layout.o.parse(key); const outpoint = new Outpoint(hash, index); const data = outpoint.toRaw(); @@ -314,7 +340,7 @@ class WalletDB extends EventEmitter { this.state = cache; this.height = cache.height; - return; + return undefined; } this.logger.info('Initializing database state from server.'); @@ -327,7 +353,7 @@ class WalletDB extends EventEmitter { for (let height = 0; height < hashes.length; height++) { const hash = hashes[height]; const meta = new BlockMeta(hash, height); - b.put(layout.h(height), meta.toHash()); + b.put(layout.h.build(height), meta.toHash()); tip = meta; } @@ -339,12 +365,14 @@ class WalletDB extends EventEmitter { state.height = tip.height; state.marked = false; - b.put(layout.R(), state.toRaw()); + b.put(layout.R.build(), state.toRaw()); await b.write(); this.state = state; this.height = state.height; + + return undefined; } /** @@ -364,7 +392,7 @@ class WalletDB extends EventEmitter { for (let height = 0; height < hashes.length; height++) { const hash = hashes[height]; const meta = new BlockMeta(hash, height); - b.put(layout.h(height), meta.toHash()); + b.put(layout.h.build(height), meta.toHash()); } await b.write(); @@ -537,11 +565,7 @@ class WalletDB extends EventEmitter { this.logger.warning('Wiping WalletDB TXDB...'); this.logger.warning('I hope you know what you\'re doing.'); - const iter = this.db.iterator({ - gte: Buffer.from([0x00]), - lte: Buffer.from([0xff]) - }); - + const iter = this.db.iterator(); const b = this.db.batch(); let total = 0; @@ -573,32 +597,12 @@ class WalletDB extends EventEmitter { */ async getDepth() { - // This may seem like a strange way to do - // this, but updating a global state when - // creating a new wallet is actually pretty - // damn tricky. There would be major atomicity - // issues if updating a global state inside - // a "scoped" state. So, we avoid all the - // nonsense of adding a global lock to - // walletdb.create by simply seeking to the - // highest wallet wid. - const iter = this.db.iterator({ - gte: layout.w(0x00000000), - lte: layout.w(0xffffffff), - reverse: true, - limit: 1 - }); + const raw = await this.db.get(layout.D.build()); - if (!await iter.next()) + if (!raw) return 1; - const {key} = iter; - - await iter.end(); - - const depth = layout.ww(key); - - return depth + 1; + return raw.readUInt32LE(0, true); } /** @@ -678,7 +682,7 @@ class WalletDB extends EventEmitter { if (typeof id === 'number') return id; - const data = await this.db.get(layout.l(id)); + const data = await this.db.get(layout.l.build(id)); if (!data) return null; @@ -722,7 +726,7 @@ class WalletDB extends EventEmitter { if (cache) return cache; - const data = await this.db.get(layout.w(wid)); + const data = await this.db.get(layout.w.build(wid)); if (!data) return null; @@ -745,8 +749,18 @@ class WalletDB extends EventEmitter { const wid = wallet.wid; const id = wallet.id; - b.put(layout.w(wid), wallet.toRaw()); - b.put(layout.l(id), u32(wid)); + b.put(layout.w.build(wid), wallet.toRaw()); + b.put(layout.l.build(id), u32(wid)); + } + + /** + * Increment the wid depth. + * @param {Batch} b + * @param {Number} wid + */ + + increment(b, wid) { + b.put(layout.D.build(), u32(wid + 1)); } /** @@ -783,7 +797,7 @@ class WalletDB extends EventEmitter { const old = wallet.id; const b = this.db.batch(); - b.del(layout.l(old)); + b.del(layout.l.build(old)); wallet.id = id; @@ -800,7 +814,7 @@ class WalletDB extends EventEmitter { renameAccount(b, account, name) { // Remove old wid/name->account index. - b.del(layout.i(account.wid, account.name)); + b.del(layout.i.build(account.wid, account.name)); account.name = name; @@ -841,12 +855,10 @@ class WalletDB extends EventEmitter { */ async _remove(wid) { - const tlayout = layouts.txdb; - if (wid === 1) throw new Error('Cannot remove primary wallet.'); - const data = await this.db.get(layout.w(wid)); + const data = await this.db.get(layout.w.build(wid)); if (!data) return false; @@ -855,16 +867,16 @@ class WalletDB extends EventEmitter { const b = this.db.batch(); - b.del(layout.w(wid)); - b.del(layout.l(id)); + b.del(layout.w.build(wid)); + b.del(layout.l.build(id)); const piter = this.db.iterator({ - gte: layout.P(wid, encoding.NULL_HASH), - lte: layout.P(wid, encoding.HIGH_HASH) + gte: layout.P.min(wid), + lte: layout.P.max(wid) }); await piter.each((key, value) => { - const [, hash] = layout.Pp(key); + const [, hash] = layout.P.parse(key); b.del(key); return this.removePathMap(b, hash, wid); }); @@ -874,61 +886,61 @@ class WalletDB extends EventEmitter { }; await removeRange({ - gte: layout.r(wid, 0x00000000, encoding.NULL_HASH), - lte: layout.r(wid, 0xffffffff, encoding.HIGH_HASH) + gte: layout.r.min(wid), + lte: layout.r.max(wid) }); await removeRange({ - gte: layout.a(wid, 0x00000000), - lte: layout.a(wid, 0xffffffff) + gte: layout.a.min(wid), + lte: layout.a.max(wid) }); await removeRange({ - gte: layout.i(wid, '\x00'), - lte: layout.i(wid, '\xff') + gte: layout.i.min(wid), + lte: layout.i.max(wid) }); await removeRange({ - gte: layout.n(wid, 0x00000000), - lte: layout.n(wid, 0xffffffff) + gte: layout.n.min(wid), + lte: layout.n.max(wid) }); await removeRange({ - gte: tlayout.prefix(wid), - lt: tlayout.prefix(wid + 1) + gt: layout.t.build(wid), + lt: layout.t.build(wid + 1) }); - const bucket = this.db.bucket(tlayout.prefix(wid)); + const bucket = this.db.bucket(layout.t.build(wid)); const biter = bucket.iterator({ - gte: tlayout.b(0x00000000), - lte: tlayout.b(0xffffffff) + gte: tlayout.b.min(), + lte: tlayout.b.max() }); await biter.each((key, value) => { - const height = layout.bb(key); + const height = tlayout.b.parse(key); return this.removeBlockMap(b, height, wid); }); const siter = bucket.iterator({ - gte: tlayout.s(encoding.NULL_HASH, 0x00000000), - lte: tlayout.s(encoding.HIGH_HASH, 0xffffffff), + gte: tlayout.s.min(), + lte: tlayout.s.max(), keys: true }); await siter.each((key, value) => { - const [hash, index] = layout.ss(key); + const [hash, index] = tlayout.s.parse(key); return this.removeOutpointMap(b, hash, index, wid); }); const uiter = bucket.iterator({ - gte: tlayout.p(encoding.NULL_HASH), - lte: tlayout.p(encoding.HIGH_HASH), + gte: tlayout.p.min(), + lte: tlayout.p.max(), keys: true }); await uiter.each((key, value) => { - const hash = layout.pp(key); + const hash = tlayout.p.parse(key); return this.removeTXMap(b, hash, wid); }); @@ -1044,7 +1056,7 @@ class WalletDB extends EventEmitter { */ async getAccount(wid, index) { - const data = await this.db.get(layout.a(wid, index)); + const data = await this.db.get(layout.a.build(wid, index)); if (!data) return null; @@ -1060,8 +1072,8 @@ class WalletDB extends EventEmitter { getAccounts(wid) { return this.db.values({ - gte: layout.n(wid, 0x00000000), - lte: layout.n(wid, 0xffffffff), + gte: layout.n.min(wid), + lte: layout.n.max(wid), parse: data => data.toString('ascii') }); } @@ -1074,7 +1086,7 @@ class WalletDB extends EventEmitter { */ async getAccountIndex(wid, name) { - const index = await this.db.get(layout.i(wid, name)); + const index = await this.db.get(layout.i.build(wid, name)); if (!index) return -1; @@ -1090,7 +1102,7 @@ class WalletDB extends EventEmitter { */ async getAccountName(wid, index) { - const name = await this.db.get(layout.n(wid, index)); + const name = await this.db.get(layout.n.build(wid, index)); if (!name) return null; @@ -1110,13 +1122,13 @@ class WalletDB extends EventEmitter { const name = account.name; // Account data - b.put(layout.a(wid, index), account.toRaw()); + b.put(layout.a.build(wid, index), account.toRaw()); // Name->Index lookups - b.put(layout.i(wid, name), u32(index)); + b.put(layout.i.build(wid, name), u32(index)); // Index->Name lookups - b.put(layout.n(wid, index), Buffer.from(name, 'ascii')); + b.put(layout.n.build(wid, index), Buffer.from(name, 'ascii')); } /** @@ -1127,7 +1139,7 @@ class WalletDB extends EventEmitter { */ hasAccount(wid, index) { - return this.db.has(layout.a(wid, index)); + return this.db.has(layout.a.build(wid, index)); } /** @@ -1159,10 +1171,10 @@ class WalletDB extends EventEmitter { await this.addPathMap(b, path.hash, wid); // Wallet ID + Address Hash -> Path Data - b.put(layout.P(wid, path.hash), path.toRaw()); + b.put(layout.P.build(wid, path.hash), path.toRaw()); // Wallet ID + Account Index + Address Hash -> Dummy - b.put(layout.r(wid, path.account, path.hash), null); + b.put(layout.r.build(wid, path.account, path.hash), null); } /** @@ -1192,7 +1204,7 @@ class WalletDB extends EventEmitter { */ async readPath(wid, hash) { - const data = await this.db.get(layout.P(wid, hash)); + const data = await this.db.get(layout.P.build(wid, hash)); if (!data) return null; @@ -1211,7 +1223,7 @@ class WalletDB extends EventEmitter { */ hasPath(wid, hash) { - return this.db.has(layout.P(wid, hash)); + return this.db.has(layout.P.build(wid, hash)); } /** @@ -1221,9 +1233,9 @@ class WalletDB extends EventEmitter { getHashes() { return this.db.keys({ - gte: layout.p(encoding.NULL_HASH), - lte: layout.p(encoding.HIGH_HASH), - parse: layout.pp + gte: layout.p.min(), + lte: layout.p.max(), + parse: key => layout.p.parse(key) }); } @@ -1234,10 +1246,10 @@ class WalletDB extends EventEmitter { getOutpoints() { return this.db.keys({ - gte: layout.o(encoding.NULL_HASH, 0), - lte: layout.o(encoding.HIGH_HASH, 0xffffffff), + gte: layout.o.min(), + lte: layout.o.max(), parse: (key) => { - const [hash, index] = layout.oo(key); + const [hash, index] = layout.o.parse(key); return new Outpoint(hash, index); } }); @@ -1251,9 +1263,9 @@ class WalletDB extends EventEmitter { getWalletHashes(wid) { return this.db.keys({ - gte: layout.P(wid, encoding.NULL_HASH), - lte: layout.P(wid, encoding.HIGH_HASH), - parse: key => layout.Pp(key)[1] + gte: layout.P.min(wid), + lte: layout.P.max(wid), + parse: key => layout.P.parse(key)[1] }); } @@ -1266,9 +1278,9 @@ class WalletDB extends EventEmitter { getAccountHashes(wid, account) { return this.db.keys({ - gte: layout.r(wid, account, encoding.NULL_HASH), - lte: layout.r(wid, account, encoding.HIGH_HASH), - parse: key => layout.rr(key)[2] + gte: layout.r.min(wid, account), + lte: layout.r.max(wid, account), + parse: key => layout.r.parse(key)[2] }); } @@ -1280,14 +1292,14 @@ class WalletDB extends EventEmitter { async getWalletPaths(wid) { const items = await this.db.range({ - gte: layout.P(wid, encoding.NULL_HASH), - lte: layout.P(wid, encoding.HIGH_HASH) + gte: layout.P.min(wid), + lte: layout.P.min(wid) }); const paths = []; for (const item of items) { - const [, hash] = layout.Pp(item.key); + const [, hash] = layout.P.parse(item.key); const path = Path.fromRaw(item.value); path.hash = hash; @@ -1307,9 +1319,9 @@ class WalletDB extends EventEmitter { getWallets() { return this.db.keys({ - gte: layout.l('\x00'), - lte: layout.l('\xff'), - parse: layout.ll + gte: layout.l.min(), + lte: layout.l.max(), + parse: key => layout.l.parse(key) }); } @@ -1322,13 +1334,13 @@ class WalletDB extends EventEmitter { async encryptKeys(b, wid, key) { const iter = this.db.iterator({ - gte: layout.P(wid, encoding.NULL_HASH), - lte: layout.P(wid, encoding.HIGH_HASH), + gte: layout.P.min(wid), + lte: layout.P.max(wid), values: true }); await iter.each((k, value) => { - const [, hash] = layout.Pp(k); + const [, hash] = layout.P.parse(k); const path = Path.fromRaw(value); if (!path.data) @@ -1355,13 +1367,13 @@ class WalletDB extends EventEmitter { async decryptKeys(b, wid, key) { const iter = this.db.iterator({ - gte: layout.P(wid, encoding.NULL_HASH), - lte: layout.P(wid, encoding.HIGH_HASH), + gte: layout.P.min(wid), + lte: layout.P.max(wid), values: true }); await iter.each((k, value) => { - const [, hash] = layout.Pp(k); + const [, hash] = layout.P.parse(k); const path = Path.fromRaw(value); if (!path.data) @@ -1386,9 +1398,9 @@ class WalletDB extends EventEmitter { async resend() { const wids = await this.db.keys({ - gte: layout.w(0x00000000), - lte: layout.w(0xffffffff), - parse: key => layout.ww(key) + gte: layout.w.min(), + lte: layout.w.max(), + parse: key => layout.w.parse(key) }); this.logger.info('Resending from %d wallets.', wids.length); @@ -1405,14 +1417,13 @@ class WalletDB extends EventEmitter { */ async resendPending(wid) { - const layout = layouts.txdb; - const prefix = layout.prefix(wid); + const prefix = layout.t.build(wid); const b = this.db.bucket(prefix); const hashes = await b.keys({ - gte: layout.p(encoding.NULL_HASH), - lte: layout.p(encoding.HIGH_HASH), - parse: key => layout.pp(key) + gte: tlayout.p.min(), + lte: tlayout.p.max(), + parse: key => tlayout.p.parse(key) }); if (hashes.length === 0) @@ -1426,7 +1437,7 @@ class WalletDB extends EventEmitter { const txs = []; for (const hash of hashes) { - const data = await b.get(layout.t(hash)); + const data = await b.get(tlayout.t.build(hash)); if (!data) continue; @@ -1496,7 +1507,7 @@ class WalletDB extends EventEmitter { */ async getState() { - const data = await this.db.get(layout.R()); + const data = await this.db.get(layout.R.build()); if (!data) return null; @@ -1518,7 +1529,7 @@ class WalletDB extends EventEmitter { // Hashes ahead of our new tip // that we need to delete. while (state.height !== tip.height) { - b.del(layout.h(state.height)); + b.del(layout.h.build(state.height)); state.height -= 1; } } else if (tip.height > state.height) { @@ -1533,8 +1544,8 @@ class WalletDB extends EventEmitter { } // Save tip and state. - b.put(layout.h(tip.height), tip.toHash()); - b.put(layout.R(), state.toRaw()); + b.put(layout.h.build(tip.height), tip.toHash()); + b.put(layout.R.build(), state.toRaw()); await b.write(); @@ -1555,7 +1566,7 @@ class WalletDB extends EventEmitter { state.marked = true; const b = this.db.batch(); - b.put(layout.R(), state.toRaw()); + b.put(layout.R.build(), state.toRaw()); await b.write(); this.state = state; @@ -1637,7 +1648,7 @@ class WalletDB extends EventEmitter { */ getPathMap(hash) { - return this.getMap(layout.p(hash)); + return this.getMap(layout.p.build(hash)); } /** @@ -1649,7 +1660,7 @@ class WalletDB extends EventEmitter { async addPathMap(b, hash, wid) { await this.addHash(hash); - return this.addMap(b, layout.p(hash), wid); + return this.addMap(b, layout.p.build(hash), wid); } /** @@ -1660,7 +1671,7 @@ class WalletDB extends EventEmitter { */ removePathMap(b, hash, wid) { - return this.removeMap(b, layout.p(hash), wid); + return this.removeMap(b, layout.p.build(hash), wid); } /** @@ -1670,7 +1681,7 @@ class WalletDB extends EventEmitter { */ async getBlockMap(height) { - return this.getMap(layout.b(height)); + return this.getMap(layout.b.build(height)); } /** @@ -1681,7 +1692,7 @@ class WalletDB extends EventEmitter { */ addBlockMap(b, height, wid) { - return this.addMap(b, layout.b(height), wid); + return this.addMap(b, layout.b.build(height), wid); } /** @@ -1692,7 +1703,7 @@ class WalletDB extends EventEmitter { */ removeBlockMap(b, height, wid) { - return this.removeMap(b, layout.b(height), wid); + return this.removeMap(b, layout.b.build(height), wid); } /** @@ -1702,7 +1713,7 @@ class WalletDB extends EventEmitter { */ getTXMap(hash) { - return this.getMap(layout.T(hash)); + return this.getMap(layout.T.build(hash)); } /** @@ -1713,7 +1724,7 @@ class WalletDB extends EventEmitter { */ addTXMap(b, hash, wid) { - return this.addMap(b, layout.T(hash), wid); + return this.addMap(b, layout.T.build(hash), wid); } /** @@ -1724,7 +1735,7 @@ class WalletDB extends EventEmitter { */ removeTXMap(b, hash, wid) { - return this.removeMap(b, layout.T(hash), wid); + return this.removeMap(b, layout.T.build(hash), wid); } /** @@ -1734,7 +1745,7 @@ class WalletDB extends EventEmitter { */ getOutpointMap(hash, index) { - return this.getMap(layout.o(hash, index)); + return this.getMap(layout.o.build(hash, index)); } /** @@ -1746,7 +1757,7 @@ class WalletDB extends EventEmitter { async addOutpointMap(b, hash, index, wid) { await this.addOutpoint(hash, index); - return this.addMap(b, layout.o(hash, index), wid); + return this.addMap(b, layout.o.build(hash, index), wid); } /** @@ -1757,7 +1768,7 @@ class WalletDB extends EventEmitter { */ removeOutpointMap(b, hash, index, wid) { - return this.removeMap(b, layout.o(hash, index), wid); + return this.removeMap(b, layout.o.build(hash, index), wid); } /** @@ -1767,7 +1778,7 @@ class WalletDB extends EventEmitter { */ async getBlock(height) { - const data = await this.db.get(layout.h(height)); + const data = await this.db.get(layout.h.build(height)); if (!data) return null; @@ -1828,8 +1839,8 @@ class WalletDB extends EventEmitter { async revert(target) { const iter = this.db.iterator({ - gte: layout.b(target + 1), - lte: layout.b(0xffffffff), + gte: layout.b.build(target + 1), + lte: layout.b.max(), reverse: true, values: true }); @@ -1837,7 +1848,7 @@ class WalletDB extends EventEmitter { let total = 0; await iter.each(async (key, value) => { - const height = layout.bb(key); + const height = layout.b.parse(key); const block = MapRecord.fromRaw(value); for (const wid of block.wids) { @@ -2140,7 +2151,6 @@ class WalletOptions { this.maxFiles = 64; this.cacheSize = 16 << 20; this.compression = true; - this.bufferKeys = layout.binary; this.spv = false; this.witness = false; diff --git a/migrate/chaindb3to4.js b/migrate/chaindb3to4.js new file mode 100644 index 00000000..5f2487a5 --- /dev/null +++ b/migrate/chaindb3to4.js @@ -0,0 +1,163 @@ +'use strict'; + +const assert = require('assert'); +const bdb = require('bdb'); +const layout = require('../lib/blockchain/layout'); +const DUMMY = Buffer.alloc(1, 0x00); + +// changes: +// db version record +// deployment table v->D +// C/T key format + +let file = process.argv[2]; +let parent = null; + +assert(typeof file === 'string', 'Please pass in a database path.'); + +file = file.replace(/\.ldb\/?$/, ''); + +const db = bdb.create({ + location: file, + db: 'leveldb', + compression: true, + cacheSize: 32 << 20, + createIfMissing: false +}); + +async function updateVersion() { + console.log('Checking version.'); + + const data = await db.get(layout.V.build()); + assert(data, 'No version.'); + + const ver = data.readUInt32LE(0, true); + + if (ver !== 3) + throw Error(`DB is version ${ver}.`); + + console.log('Updating version to %d.', ver + 1); + + const buf = Buffer.allocUnsafe(5 + 4); + buf.write('chain', 0, 'ascii'); + buf.writeUInt32LE(4, 5, true); + + parent.put(layout.V.build(), buf); +} + +async function migrateKeys(id, from, to) { + console.log('Migrating keys for %s.', String.fromCharCode(id)); + + const iter = db.iterator({ + gt: Buffer.from([id]), + lt: Buffer.from([id + 1]), + keys: true + }); + + let batch = db.batch(); + let total = 0; + let items = 0; + + await iter.each(async (key) => { + batch.put(to.build(...from(key)), DUMMY); + batch.del(key); + + total += (key.length + 80) * 2; + items += 1; + + if (total >= (128 << 20)) { + await batch.write(); + batch = db.batch(); + total = 0; + } + }); + + console.log('Migrated %d keys for %s.', items, String.fromCharCode(id)); + + return batch.write(); +} + +async function updateKeys() { + console.log('Updating keys...'); + + const v = Buffer.from('v', 'ascii'); + + const table = await db.get(v); + assert(table); + + parent.put(layout.D.build(), table); + parent.del(v); + + const raw = await db.get(layout.O.build()); + assert(raw); + + const flags = raw.readUInt32LE(8, true); + + if (!(flags & 16)) { + console.log('Updated keys.'); + return; + } + + console.log('Updating address index keys...'); + + await migrateKeys(0x54, parseT, layout.T); // T + await migrateKeys(0xab, parseT, layout.T); // W + T + await migrateKeys(0x43, parseC, layout.C); // C + await migrateKeys(0x9a, parseC, layout.C); // W + C + + console.log('Updated keys.'); +} + +function parseT(key) { + assert(Buffer.isBuffer(key)); + + if (key.length === 65) + return [key.slice(1, 33), key.slice(33, 65)]; + + assert(key.length === 53); + return [key.slice(1, 21), key.slice(21, 53)]; +} + +function parseC(key) { + assert(Buffer.isBuffer(key)); + + let addr, hash, index; + + if (key.length === 69) { + addr = key.slice(1, 33); + hash = key.slice(33, 65); + index = key.readUInt32BE(65, 0); + } else if (key.length === 57) { + addr = key.slice(1, 21); + hash = key.slice(21, 53); + index = key.readUInt32BE(53, 0); + } else { + assert(false); + } + + return [addr, hash, index]; +} + +/* + * Execute + */ + +(async () => { + await db.open(); + + console.log('Opened %s.', file); + + parent = db.batch(); + + await updateVersion(); + await updateKeys(); + + await parent.write(); + await db.close(); +})().then(() => { + console.log('Migration complete.'); + process.exit(0); +}).catch((err) => { + console.error(err.stack); + process.exit(1); +}); diff --git a/migrate/walletdb6to7.js b/migrate/walletdb6to7.js index 3257bd53..b2372658 100644 --- a/migrate/walletdb6to7.js +++ b/migrate/walletdb6to7.js @@ -1,16 +1,16 @@ 'use strict'; const assert = require('assert'); -const BDB = require('bdb'); +const bdb = require('bdb'); const bio = require('bufio'); const layouts = require('../lib/wallet/layout'); const TX = require('../lib/primitives/tx'); const Coin = require('../lib/primitives/coin'); -const layout = layouts.walletdb; +const layout = layouts.wdb; const tlayout = layouts.txdb; -const {encoding} = bio; // changes: +// db version record // headers - all headers // block map - just a map // input map - only on unconfirmed @@ -21,21 +21,22 @@ const {encoding} = bio; // wallet - serialization // account - serialization // path - serialization +// depth - counter record +// hash/ascii - variable length key prefixes let file = process.argv[2]; -let batch; +let parent = null; assert(typeof file === 'string', 'Please pass in a database path.'); file = file.replace(/\.ldb\/?$/, ''); -const db = new BDB({ +const db = bdb.create({ location: file, db: 'leveldb', compression: true, cacheSize: 32 << 20, - createIfMissing: false, - bufferKeys: true + createIfMissing: false }); async function updateVersion() { @@ -43,7 +44,7 @@ async function updateVersion() { console.log('Checking version.'); - const data = await db.get('V'); + const data = await db.get(layout.V.build()); assert(data, 'No version.'); const ver = data.readUInt32LE(0, true); @@ -52,16 +53,65 @@ async function updateVersion() { throw Error(`DB is version ${ver}.`); console.log('Backing up DB to: %s.', bak); + console.log('Updating version to %d.', ver + 1); await db.backup(bak); - const buf = Buffer.allocUnsafe(4); - buf.writeUInt32LE(7, 0, true); - batch.put('V', buf); + const buf = Buffer.allocUnsafe(6 + 4); + buf.write('wallet', 0, 'ascii'); + buf.writeUInt32LE(7, 6, true); + + parent.put(layout.V.build(), buf); +} + +async function migrateKeys(id, from, to) { + console.log('Migrating keys for %s.', String.fromCharCode(id)); + + const iter = db.iterator({ + gt: Buffer.from([id]), + lt: Buffer.from([id + 1]), + keys: true, + values: true + }); + + let batch = db.batch(); + let total = 0; + let items = 0; + + await iter.each(async (key, value) => { + batch.put(to.build(...from(key)), value); + batch.del(key); + + total += (key.length + 80) * 2; + total += value.length + 80; + items += 1; + + if (total >= (128 << 20)) { + await batch.write(); + batch = db.batch(); + total = 0; + } + }); + + console.log('Migrated %d keys for %s.', items, String.fromCharCode(id)); + + return batch.write(); +} + +async function updateKeys() { + console.log('Updating keys...'); + + await migrateKeys(0x70, parsep, layout.p); // p + await migrateKeys(0x50, parseP, layout.P); // P + await migrateKeys(0x72, parser, layout.r); // r + await migrateKeys(0x6c, parsel, layout.l); // l + await migrateKeys(0x69, parsei, layout.i); // i + + console.log('Updated keys.'); } async function updateState() { - const raw = await db.get(layout.R); + const raw = await db.get(layout.R.build()); if (!raw) return; @@ -69,15 +119,25 @@ async function updateState() { console.log('Updating state...'); if (raw.length === 40) { - batch.put(layout.R, c(raw, Buffer.from([1]))); + const bw = bio.write(41); + bw.writeBytes(raw); + bw.writeU8(1); + parent.put(layout.R.build(), bw.render()); console.log('State updated.'); } + + const depth = await getDepth(); + + const buf = Buffer.allocUnsafe(4); + buf.writeUInt32LE(depth, 0, true); + + parent.put(layout.D.build(), buf); } async function updateBlockMap() { const iter = db.iterator({ - gte: layout.b(0), - lte: layout.b(0xffffffff), + gte: layout.b.min(), + lte: layout.b.max(), keys: true, values: true }); @@ -87,7 +147,7 @@ async function updateBlockMap() { let total = 0; await iter.each((key, value) => { - const height = layout.bb(key); + const height = layout.b.parse(key); const block = BlockMapRecord.fromRaw(height, value); const map = new Set(); @@ -99,7 +159,7 @@ async function updateBlockMap() { const bw = bio.write(sizeMap(map)); serializeMap(bw, map); - batch.put(key, bw.render()); + parent.put(key, bw.render()); total += 1; }); @@ -109,10 +169,10 @@ async function updateBlockMap() { async function updateTXDB() { const wids = await db.keys({ - gte: layout.w(0), - lte: layout.w(0xffffffff), + gte: layout.w.min(), + lte: layout.w.max(), keys: true, - parse: k => layout.ww(k) + parse: key => layout.w.parse(key) }); console.log('Updating wallets...'); @@ -120,41 +180,42 @@ async function updateTXDB() { let total = 0; for (const wid of wids) { - await updateInputs(wid); - await updateCoins(wid); - await updateTX(wid); - await updateWalletBalance(wid); - await updateAccountBalances(wid); + const bucket = db.bucket(layout.t.build(wid)); + const batch = bucket.wrap(parent); + + await updateInputs(wid, bucket, batch); + await updateCoins(wid, bucket, batch); + await updateTX(wid, bucket, batch); + await updateWalletBalance(wid, bucket, batch); + await updateAccountBalances(wid, bucket, batch); await updateWallet(wid); + total += 1; } console.log('Updated %d wallets.', total); } -async function updateInputs(wid) { - const pre = tlayout.prefix(wid); - - const iter = db.iterator({ - gte: c(pre, tlayout.h(0, encoding.NULL_HASH)), - lte: c(pre, tlayout.h(0xffffffff, encoding.HIGH_HASH)), +async function updateInputs(wid, bucket, batch) { + const iter = bucket.iterator({ + gte: tlayout.h.min(), + lte: tlayout.h.max(), keys: true }); - console.log('Updating inputs for %d.', wid); + console.log('Updating inputs for %d...', wid); let total = 0; - await iter.each(async (k, value) => { - const key = k.slice(pre.length); - const [height, hash] = tlayout.hh(key); - const data = await db.get(c(pre, tlayout.t(hash))); + await iter.each(async (key, value) => { + const [, hash] = tlayout.h.parse(key); + const data = await bucket.get(tlayout.t.build(hash)); assert(data); const tx = TX.fromRaw(data); for (const {prevout} of tx.inputs) { const {hash, index} = prevout; - batch.del(c(pre, tlayout.s(hash, index))); + batch.del(tlayout.s.build(hash, index)); total += 1; } }); @@ -162,12 +223,10 @@ async function updateInputs(wid) { console.log('Updated %d inputs for %d.', total, wid); } -async function updateCoins(wid) { - const pre = tlayout.prefix(wid); - - const iter = db.iterator({ - gte: c(pre, tlayout.c(encoding.NULL_HASH, 0)), - lte: c(pre, tlayout.c(encoding.HIGH_HASH, 0xffffffff)), +async function updateCoins(wid, bucket, batch) { + const iter = bucket.iterator({ + gte: tlayout.c.min(), + lte: tlayout.c.max(), keys: true, values: true }); @@ -183,7 +242,10 @@ async function updateCoins(wid) { br.readU8(); if (br.left() === 0) { - batch.put(key, c(value, Buffer.from([0]))); + const bw = bio.write(value.length + 1); + bw.writeBytes(value); + bw.writeU8(0); + batch.put(key, bw.render()); total += 1; } }); @@ -191,12 +253,10 @@ async function updateCoins(wid) { console.log('Updated %d coins for %d.', total, wid); } -async function updateTX(wid) { - const pre = tlayout.prefix(wid); - - const iter = db.iterator({ - gte: c(pre, tlayout.p(encoding.NULL_HASH)), - lte: c(pre, tlayout.p(encoding.HIGH_HASH)), +async function updateTX(wid, bucket, batch) { + const iter = bucket.iterator({ + gte: tlayout.p.min(), + lte: tlayout.p.max(), keys: true }); @@ -204,10 +264,9 @@ async function updateTX(wid) { let total = 0; - await iter.each(async (k, value) => { - const key = k.slice(pre.length); - const hash = tlayout.pp(key); - const raw = await db.get(layout.T(hash)); + await iter.each(async (key, value) => { + const hash = tlayout.p.parse(key); + const raw = await db.get(layout.T.build(hash)); let map = null; @@ -222,7 +281,7 @@ async function updateTX(wid) { const bw = bio.write(sizeMap(map)); serializeMap(bw, map); - batch.put(layout.T(hash), bw.render()); + batch.put(layout.T.build(hash), bw.render()); total += 1; }); @@ -230,26 +289,25 @@ async function updateTX(wid) { console.log('Added %d TX maps for %d.', total, wid); } -async function updateWalletBalance(wid) { - const pre = tlayout.prefix(wid); +async function updateWalletBalance(wid, bucket, batch) { const bal = newBalance(); - const keys = await db.keys({ - gte: c(pre, tlayout.t(encoding.NULL_HASH)), - lte: c(pre, tlayout.t(encoding.HIGH_HASH)), + const keys = await bucket.keys({ + gte: tlayout.t.min(), + lte: tlayout.t.max(), keys: true }); bal.tx = keys.length; - const iter = db.iterator({ - gte: c(pre, tlayout.c(encoding.NULL_HASH, 0)), - lte: c(pre, tlayout.c(encoding.HIGH_HASH, 0xffffffff)), + const iter = bucket.iterator({ + gte: tlayout.c.min(), + lte: tlayout.c.max(), keys: true, values: true }); - console.log('Updating wallet balance for %d.', wid); + console.log('Updating wallet balance for %d...', wid); await iter.each((key, value) => { const br = bio.read(value, true); @@ -265,13 +323,13 @@ async function updateWalletBalance(wid) { bal.unconfirmed += coin.value; }); - batch.put(c(pre, tlayout.R), serializeBalance(bal)); + batch.put(tlayout.R.build(), serializeBalance(bal)); console.log('Updated wallet balance for %d.', wid); } -async function updateAccountBalances(wid) { - const raw = await db.get(layout.w(wid)); +async function updateAccountBalances(wid, bucket, batch) { + const raw = await db.get(layout.w.build(wid)); assert(raw); const br = bio.read(raw, true); @@ -287,35 +345,33 @@ async function updateAccountBalances(wid) { console.log('Updating account balances for %d...', wid); for (let acct = 0; acct < depth; acct++) - await updateAccountBalance(wid, acct); + await updateAccountBalance(wid, acct, bucket, batch); console.log('Updated %d account balances for %d.', depth, wid); } -async function updateAccountBalance(wid, acct) { - const pre = tlayout.prefix(wid); +async function updateAccountBalance(wid, acct, bucket, batch) { const bal = newBalance(); - const keys = await db.keys({ - gte: c(pre, tlayout.T(acct, encoding.NULL_HASH)), - lte: c(pre, tlayout.T(acct, encoding.HIGH_HASH)), + const keys = await bucket.keys({ + gte: tlayout.T.min(acct), + lte: tlayout.T.max(acct), keys: true }); bal.tx = keys.length; - const iter = db.iterator({ - gte: c(pre, tlayout.C(acct, encoding.NULL_HASH, 0)), - lte: c(pre, tlayout.C(acct, encoding.HIGH_HASH, 0xffffffff)), + const iter = bucket.iterator({ + gte: tlayout.C.min(acct), + lte: tlayout.C.max(acct), keys: true }); console.log('Updating account balance for %d/%d...', wid, acct); - await iter.each(async (k, value) => { - const key = k.slice(pre.length); - const [, hash, index] = tlayout.Cc(key); - const raw = await db.get(c(pre, tlayout.c(hash, index))); + await iter.each(async (key, value) => { + const [, hash, index] = tlayout.C.parse(key); + const raw = await bucket.get(tlayout.c.build(hash, index)); assert(raw); const br = bio.read(raw, true); const coin = Coin.fromReader(br); @@ -330,13 +386,13 @@ async function updateAccountBalance(wid, acct) { bal.unconfirmed += coin.value; }); - batch.put(c(pre, tlayout.r(acct)), serializeBalance(bal)); + batch.put(tlayout.r.build(acct), serializeBalance(bal)); console.log('Updated account balance for %d/%d.', wid, acct); } async function updateWallet(wid) { - const raw = await db.get(layout.w(wid)); + const raw = await db.get(layout.w.build(wid)); assert(raw); console.log('Updating wallet: %d.', wid); @@ -346,7 +402,7 @@ async function updateWallet(wid) { br.readU32(); // Skip network. br.readU32(); // Skip wid. const id = br.readVarString('ascii'); - const initialized = br.readU8() === 1; + br.readU8(); // Skip initialized. const watchOnly = br.readU8() === 1; const accountDepth = br.readU32(); const token = br.readBytes(32); @@ -414,7 +470,7 @@ async function updateWallet(wid) { bw.writeU32(tokenDepth); bw.writeBytes(key); - batch.put(layout.w(wid), bw.render()); + parent.put(layout.w.build(wid), bw.render()); console.log('Updating accounts for %d...', wid); @@ -427,7 +483,7 @@ async function updateWallet(wid) { } async function updateAccount(wid, acct) { - const raw = await db.get(layout.a(wid, acct)); + const raw = await db.get(layout.a.build(wid, acct)); assert(raw); console.log('Updating account: %d/%d...', wid, acct); @@ -508,15 +564,15 @@ async function updateAccount(wid, acct) { bw.writeBytes(key.publicKey); } - batch.put(layout.a(wid, acct), bw.render()); + parent.put(layout.a.build(wid, acct), bw.render()); console.log('Updated account: %d/%d.', wid, acct); } async function updatePaths() { const iter = db.iterator({ - gte: layout.P(0, encoding.NULL_HASH), - lte: layout.P(0xffffffff, encoding.HIGH_HASH), + gte: layout.P.min(), + lte: layout.P.max(), keys: true, values: true }); @@ -595,7 +651,7 @@ async function updatePaths() { break; } - batch.put(key, bw.render()); + parent.put(key, bw.render()); total += 1; }); @@ -603,6 +659,26 @@ async function updatePaths() { console.log('Updated %d paths.', total); } +async function getDepth() { + const iter = db.iterator({ + gte: layout.w.min(), + lte: layout.w.max(), + reverse: true, + limit: 1 + }); + + if (!await iter.next()) + return 1; + + const {key} = iter; + + await iter.end(); + + const depth = layout.w.parse(key); + + return depth + 1; +} + /* * Old Records */ @@ -695,7 +771,7 @@ class BlockMapRecord { class TXMapRecord { constructor(hash, wids) { - this.hash = hash || encoding.NULL_HASH; + this.hash = hash || null; this.wids = wids || new Set(); } @@ -769,10 +845,6 @@ function serializeMap(bw, wids) { * Helpers */ -function c(a, b) { - return Buffer.concat([a, b]); -} - function newBalance() { return { tx: 0, @@ -793,6 +865,40 @@ function serializeBalance(bal) { return bw.render(); } +function parsep(key) { // p[hash] + assert(Buffer.isBuffer(key)); + assert(key.length >= 21); + return [key.toString('hex', 1)]; +} + +function parseP(key) { // P[wid][hash] + assert(Buffer.isBuffer(key)); + assert(key.length >= 25); + return [key.readUInt32BE(1, true), key.toString('hex', 5)]; +} + +function parser(key) { // r[wid][index][hash] + assert(Buffer.isBuffer(key)); + assert(key.length >= 29); + return [ + key.readUInt32BE(1, true), + key.readUInt32BE(5, true), + key.toString('hex', 9) + ]; +} + +function parsel(key) { // l[id] + assert(Buffer.isBuffer(key)); + assert(key.length >= 1); + return [key.toString('ascii', 1)]; +} + +function parsei(key) { // i[wid][name] + assert(Buffer.isBuffer(key)); + assert(key.length >= 5); + return [key.readUInt32BE(1, true), key.toString('ascii', 5)]; +} + /* * Execute */ @@ -802,15 +908,16 @@ function serializeBalance(bal) { console.log('Opened %s.', file); - batch = db.batch(); + parent = db.batch(); await updateVersion(); + await updateKeys(); await updateState(); await updateBlockMap(); await updateTXDB(); await updatePaths(); - await batch.write(); + await parent.write(); await db.close(); })().then(() => { console.log('Migration complete.');