From 969fd8f704ab121706c2830802b67d3bf94480df Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 16 Oct 2017 04:35:24 -0700 Subject: [PATCH] wallet-rewrite --- lib/blockchain/chaindb.js | 24 ++ lib/wallet/account.js | 31 +- lib/wallet/layout-browser.js | 7 + lib/wallet/layout.js | 13 + lib/wallet/nodeclient.js | 11 + lib/wallet/records.js | 378 ++++----------------- lib/wallet/txdb.js | 258 ++++++--------- lib/wallet/wallet.js | 109 +++--- lib/wallet/walletdb.js | 623 +++++++++++++++++------------------ test/node-test.js | 4 +- 10 files changed, 586 insertions(+), 872 deletions(-) diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 13117903..94e9d19a 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -891,6 +891,30 @@ ChainDB.prototype.isMainChain = async function isMainChain(entry) { return false; }; +/** + * Get hash range. + * @param {Number} [start=-1] + * @param {Number} [end=-1] + * @returns {Promise} + */ + +ChainDB.prototype.getHashes = function getHashes(start = -1, end = -1) { + if (start === -1) + start = 0; + + if (end === -1) + end >>>= 0; + + assert((start >>> 0) === start); + assert((end >>> 0) === end); + + return this.db.values({ + gte: layout.H(start), + lte: layout.H(end), + parse: data => data.toString('hex') + }); +}; + /** * Get all entries. * @returns {Promise} - Returns {@link ChainEntry}[]. diff --git a/lib/wallet/account.js b/lib/wallet/account.js index 4d7dd7ed..b0f83804 100644 --- a/lib/wallet/account.js +++ b/lib/wallet/account.js @@ -25,7 +25,6 @@ const HD = require('../hd/hd'); * @alias module:wallet.Account * @constructor * @param {Object} options - * @param {WalletDB} options.db * @param {HDPublicKey} options.accountKey * @param {Boolean?} options.witness - Whether to use witness programs. * @param {Number} options.accountIndex - The BIP44 account index. @@ -41,14 +40,14 @@ const HD = require('../hd/hd'); * @param {String?} options.name - Account name */ -function Account(db, options) { +function Account(wdb, options) { if (!(this instanceof Account)) - return new Account(db, options); + return new Account(wdb, options); - assert(db, 'Database is required.'); + assert(wdb, 'Database is required.'); - this.db = db; - this.network = db.network; + this.wdb = wdb; + this.network = wdb.network; this.wallet = null; this.receive = null; @@ -59,7 +58,7 @@ function Account(db, options) { this.id = null; this.name = null; this.initialized = false; - this.witness = this.db.options.witness === true; + this.witness = wdb.options.witness === true; this.watchOnly = false; this.type = Account.types.PUBKEYHASH; this.m = 1; @@ -203,13 +202,13 @@ Account.prototype.fromOptions = function fromOptions(options) { /** * Instantiate account from options. - * @param {WalletDB} db + * @param {WalletDB} wdb * @param {Object} options * @returns {Account} */ -Account.fromOptions = function fromOptions(db, options) { - return new Account(db).fromOptions(options); +Account.fromOptions = function fromOptions(wdb, options) { + return new Account(wdb).fromOptions(options); }; /* @@ -572,7 +571,7 @@ Account.prototype.deriveKey = function deriveKey(branch, index, master) { */ Account.prototype.save = function save() { - return this.db.saveAccount(this); + return this.wdb.saveAccount(this); }; /** @@ -582,7 +581,7 @@ Account.prototype.save = function save() { */ Account.prototype.saveKey = function saveKey(ring) { - return this.db.saveKey(this.wallet, ring); + return this.wdb.saveKey(this.wallet, ring); }; /** @@ -592,7 +591,7 @@ Account.prototype.saveKey = function saveKey(ring) { */ Account.prototype.savePath = function savePath(path) { - return this.db.savePath(this.wallet, path); + return this.wdb.savePath(this.wallet, path); }; /** @@ -718,7 +717,7 @@ Account.prototype.syncDepth = async function syncDepth(receive, change, nested) Account.prototype.setLookahead = async function setLookahead(lookahead) { if (lookahead === this.lookahead) { - this.db.logger.warning( + this.wdb.logger.warning( 'Lookahead is not changing for: %s/%s.', this.id, this.name); return; @@ -983,8 +982,8 @@ Account.prototype.fromRaw = function fromRaw(data) { * @returns {Account} */ -Account.fromRaw = function fromRaw(db, data) { - return new Account(db).fromRaw(data); +Account.fromRaw = function fromRaw(wdb, data) { + return new Account(wdb).fromRaw(data); }; /** diff --git a/lib/wallet/layout-browser.js b/lib/wallet/layout-browser.js index cbaa13ff..f0c47c89 100644 --- a/lib/wallet/layout-browser.js +++ b/lib/wallet/layout-browser.js @@ -83,6 +83,13 @@ layouts.walletdb = { }, oo: function oo(key) { return [key.slice(1, 65), parseInt(key.slice(65), 10)]; + }, + T: function T(hash) { + assert(typeof hash === 'string'); + return 'T' + hash; + }, + Tt: function Tt(key) { + return [key.slice(1, 65)]; } }; diff --git a/lib/wallet/layout.js b/lib/wallet/layout.js index 3f888752..4dc42e42 100644 --- a/lib/wallet/layout.js +++ b/lib/wallet/layout.js @@ -24,6 +24,7 @@ const layouts = exports; * h[height] -> recent block hash * b[height] -> block->wid map * o[hash][index] -> outpoint->wid map + * T[hash] -> tx->wid map */ layouts.walletdb = { @@ -163,6 +164,18 @@ layouts.walletdb = { assert(Buffer.isBuffer(key)); assert(key.length === 37); return [key.toString('hex', 1, 33), key.readUInt32BE(33, true)]; + }, + T: function T(hash, index) { + assert(typeof hash === 'string'); + const key = Buffer.allocUnsafe(33); + key[0] = 0x54; + key.write(hash, 1, 'hex'); + return key; + }, + Tt: function Tt(key) { + assert(Buffer.isBuffer(key)); + assert(key.length === 33); + return key.toString('hex', 1, 33); } }; diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index f22dca2e..145e8d89 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -170,6 +170,17 @@ NodeClient.prototype.estimateFee = async function estimateFee(blocks) { return this.node.fees.estimateFee(blocks); }; +/** + * Get hash range. + * @param {Number} start + * @param {Number} end + * @returns {Promise} + */ + +NodeClient.prototype.getHashes = async function getHashes(start = -1, end = -1) { + return this.node.chain.db.getHashes(start, end); +}; + /** * Rescan for any missed transactions. * @param {Number|Hash} start - Start block. diff --git a/lib/wallet/records.js b/lib/wallet/records.js index 2b78c808..3e28a747 100644 --- a/lib/wallet/records.js +++ b/lib/wallet/records.js @@ -26,9 +26,9 @@ function ChainState() { if (!(this instanceof ChainState)) return new ChainState(); - this.startHeight = -1; + this.startHeight = 0; this.startHash = encoding.NULL_HASH; - this.height = -1; + this.height = 0; this.marked = false; } @@ -58,10 +58,7 @@ ChainState.prototype.fromRaw = function fromRaw(data) { this.startHeight = br.readU32(); this.startHash = br.readHash('hex'); this.height = br.readU32(); - this.marked = true; - - if (br.left() > 0) - this.marked = br.readU8() === 1; + this.marked = br.readU8() === 1; return this; }; @@ -225,301 +222,6 @@ BlockMeta.prototype.toJSON = function toJSON() { }; }; -/** - * Wallet Block - * @constructor - * @param {Hash} hash - * @param {Number} height - */ - -function BlockMapRecord(height) { - if (!(this instanceof BlockMapRecord)) - return new BlockMapRecord(height); - - this.height = height != null ? height : -1; - this.txs = new Map(); -} - -/** - * Instantiate wallet block from serialized data. - * @private - * @param {Hash} hash - * @param {Buffer} data - */ - -BlockMapRecord.prototype.fromRaw = function fromRaw(data) { - const br = new BufferReader(data); - const count = br.readU32(); - - for (let i = 0; i < count; i++) { - const hash = br.readHash('hex'); - const tx = TXMapRecord.fromReader(hash, br); - this.txs.set(tx.hash, tx); - } - - return this; -}; - -/** - * Instantiate wallet block from serialized data. - * @param {Hash} hash - * @param {Buffer} data - * @returns {BlockMapRecord} - */ - -BlockMapRecord.fromRaw = function fromRaw(height, data) { - return new BlockMapRecord(height).fromRaw(data); -}; - -/** - * Calculate serialization size. - * @returns {Number} - */ - -BlockMapRecord.prototype.getSize = function getSize() { - let size = 0; - - size += 4; - - for (const tx of this.txs.values()) { - size += 32; - size += tx.getSize(); - } - - return size; -}; - -/** - * Serialize the wallet block as a block. - * Contains matching transaction hashes. - * @returns {Buffer} - */ - -BlockMapRecord.prototype.toRaw = function toRaw() { - const size = this.getSize(); - const bw = new StaticWriter(size); - - bw.writeU32(this.txs.size); - - for (const [hash, tx] of this.txs) { - bw.writeHash(hash); - tx.toWriter(bw); - } - - return bw.render(); -}; - -/** - * Add a hash and wid pair to the block. - * @param {Hash} hash - * @param {WalletID} wid - * @returns {Boolean} - */ - -BlockMapRecord.prototype.add = function add(hash, wid) { - let tx = this.txs.get(hash); - - if (!tx) { - tx = new TXMapRecord(hash); - this.txs.set(hash, tx); - } - - return tx.add(wid); -}; - -/** - * Remove a hash and wid pair from the block. - * @param {Hash} hash - * @param {WalletID} wid - * @returns {Boolean} - */ - -BlockMapRecord.prototype.remove = function remove(hash, wid) { - const tx = this.txs.get(hash); - - if (!tx) - return false; - - if (!tx.remove(wid)) - return false; - - if (tx.wids.size === 0) - this.txs.delete(tx.hash); - - return true; -}; - -/** - * Convert tx map to an array. - * @returns {Array} - */ - -BlockMapRecord.prototype.toArray = function toArray() { - const txs = []; - - for (const tx of this.txs.values()) - txs.push(tx); - - return txs; -}; - -/** - * TX Hash - * @constructor - */ - -function TXMapRecord(hash, wids) { - this.hash = hash || encoding.NULL_HASH; - this.wids = wids || new Set(); -} - -TXMapRecord.prototype.add = function add(wid) { - if (this.wids.has(wid)) - return false; - - this.wids.add(wid); - return true; -}; - -TXMapRecord.prototype.remove = function remove(wid) { - return this.wids.delete(wid); -}; - -TXMapRecord.prototype.toWriter = function toWriter(bw) { - return serializeWallets(bw, this.wids); -}; - -TXMapRecord.prototype.getSize = function getSize() { - return sizeWallets(this.wids); -}; - -TXMapRecord.prototype.toRaw = function toRaw() { - const size = this.getSize(); - return this.toWriter(new StaticWriter(size)).render(); -}; - -TXMapRecord.prototype.fromReader = function fromReader(br) { - this.wids = parseWallets(br); - return this; -}; - -TXMapRecord.prototype.fromRaw = function fromRaw(data) { - return this.fromReader(new BufferReader(data)); -}; - -TXMapRecord.fromReader = function fromReader(hash, br) { - return new TXMapRecord(hash).fromReader(br); -}; - -TXMapRecord.fromRaw = function fromRaw(hash, data) { - return new TXMapRecord(hash).fromRaw(data); -}; - -/** - * Outpoint Map - * @constructor - */ - -function OutpointMapRecord(hash, index, wids) { - this.hash = hash || encoding.NULL_HASH; - this.index = index != null ? index : -1; - this.wids = wids || new Set(); -} - -OutpointMapRecord.prototype.add = function add(wid) { - if (this.wids.has(wid)) - return false; - - this.wids.add(wid); - return true; -}; - -OutpointMapRecord.prototype.remove = function remove(wid) { - return this.wids.delete(wid); -}; - -OutpointMapRecord.prototype.toWriter = function toWriter(bw) { - return serializeWallets(bw, this.wids); -}; - -OutpointMapRecord.prototype.getSize = function getSize() { - return sizeWallets(this.wids); -}; - -OutpointMapRecord.prototype.toRaw = function toRaw() { - const size = this.getSize(); - return this.toWriter(new StaticWriter(size)).render(); -}; - -OutpointMapRecord.prototype.fromReader = function fromReader(br) { - this.wids = parseWallets(br); - return this; -}; - -OutpointMapRecord.prototype.fromRaw = function fromRaw(data) { - return this.fromReader(new BufferReader(data)); -}; - -OutpointMapRecord.fromReader = function fromReader(hash, index, br) { - return new OutpointMapRecord(hash, index).fromReader(br); -}; - -OutpointMapRecord.fromRaw = function fromRaw(hash, index, data) { - return new OutpointMapRecord(hash, index).fromRaw(data); -}; - -/** - * Path Record - * @constructor - */ - -function PathMapRecord(hash, wids) { - this.hash = hash || encoding.NULL_HASH; - this.wids = wids || new Set(); -} - -PathMapRecord.prototype.add = function add(wid) { - if (this.wids.has(wid)) - return false; - - this.wids.add(wid); - return true; -}; - -PathMapRecord.prototype.remove = function remove(wid) { - return this.wids.delete(wid); -}; - -PathMapRecord.prototype.toWriter = function toWriter(bw) { - return serializeWallets(bw, this.wids); -}; - -PathMapRecord.prototype.getSize = function getSize() { - return sizeWallets(this.wids); -}; - -PathMapRecord.prototype.toRaw = function toRaw() { - const size = this.getSize(); - return this.toWriter(new StaticWriter(size)).render(); -}; - -PathMapRecord.prototype.fromReader = function fromReader(br) { - this.wids = parseWallets(br); - return this; -}; - -PathMapRecord.prototype.fromRaw = function fromRaw(data) { - return this.fromReader(new BufferReader(data)); -}; - -PathMapRecord.fromReader = function fromReader(hash, br) { - return new PathMapRecord(hash).fromReader(br); -}; - -PathMapRecord.fromRaw = function fromRaw(hash, data) { - return new PathMapRecord(hash).fromRaw(data); -}; - /** * TXRecord * @constructor @@ -653,6 +355,7 @@ TXRecord.prototype.getSize = function getSize() { TXRecord.prototype.toRaw = function toRaw() { const size = this.getSize(); const bw = new StaticWriter(size); + let index = this.index; this.tx.toWriter(bw); @@ -713,32 +416,66 @@ TXRecord.fromRaw = function fromRaw(data) { return new TXRecord().fromRaw(data); }; -/* - * Helpers +/** + * Map Record + * @constructor */ -function parseWallets(br) { - const count = br.readU32(); - const wids = new Set(); - - for (let i = 0; i < count; i++) - wids.add(br.readU32()); - - return wids; +function MapRecord() { + this.wids = new Set(); } -function sizeWallets(wids) { - return 4 + wids.size * 4; -} +MapRecord.prototype.add = function add(wid) { + if (this.wids.has(wid)) + return false; -function serializeWallets(bw, wids) { - bw.writeU32(wids.size); + this.wids.add(wid); - for (const wid of wids) + return true; +}; + +MapRecord.prototype.remove = function remove(wid) { + return this.wids.delete(wid); +}; + +MapRecord.prototype.toWriter = function toWriter(bw) { + bw.writeU32(this.wids.size); + + for (const wid of this.wids) bw.writeU32(wid); return bw; -} +}; + +MapRecord.prototype.getSize = function getSize() { + return 4 + this.wids.size * 4; +}; + +MapRecord.prototype.toRaw = function toRaw() { + const size = this.getSize(); + return this.toWriter(new StaticWriter(size)).render(); +}; + +MapRecord.prototype.fromReader = function fromReader(br) { + const count = br.readU32(); + + for (let i = 0; i < count; i++) + this.wids.add(br.readU32()); + + return this; +}; + +MapRecord.prototype.fromRaw = function fromRaw(data) { + return this.fromReader(new BufferReader(data)); +}; + +MapRecord.fromReader = function fromReader(br) { + return new MapRecord().fromReader(br); +}; + +MapRecord.fromRaw = function fromRaw(data) { + return new MapRecord().fromRaw(data); +}; /* * Expose @@ -746,10 +483,7 @@ function serializeWallets(bw, wids) { exports.ChainState = ChainState; exports.BlockMeta = BlockMeta; -exports.BlockMapRecord = BlockMapRecord; -exports.TXMapRecord = TXMapRecord; -exports.OutpointMapRecord = OutpointMapRecord; -exports.PathMapRecord = PathMapRecord; exports.TXRecord = TXRecord; +exports.MapRecord = MapRecord; module.exports = exports; diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index 079480e4..f6efd1e3 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -7,9 +7,8 @@ 'use strict'; -const util = require('../utils/util'); -const LRU = require('../utils/lru'); const assert = require('assert'); +const util = require('../utils/util'); const BufferReader = require('../utils/reader'); const StaticWriter = require('../utils/staticwriter'); const Amount = require('../btc/amount'); @@ -21,8 +20,6 @@ const layout = require('./layout').txdb; const encoding = require('../utils/encoding'); const policy = require('../protocol/policy'); const Script = require('../script/script'); -const BlockMapRecord = records.BlockMapRecord; -const OutpointMapRecord = records.OutpointMapRecord; const TXRecord = records.TXRecord; /** @@ -37,12 +34,11 @@ function TXDB(wallet) { return new TXDB(wallet); this.wallet = wallet; - this.walletdb = wallet.db; - this.db = wallet.db.db; - this.logger = wallet.db.logger; - this.network = wallet.db.network; - this.options = wallet.db.options; - this.coinCache = new LRU(10000); + this.wdb = wallet.wdb; + this.db = this.wdb.db; + this.logger = this.wdb.logger; + this.network = this.wdb.network; + this.options = this.wdb.options; this.locked = new Set(); this.state = null; @@ -89,7 +85,6 @@ TXDB.prototype.open = async function open() { TXDB.prototype.start = function start() { this.pending = this.state.clone(); - this.coinCache.start(); return this.wallet.start(); }; @@ -101,7 +96,6 @@ TXDB.prototype.start = function start() { TXDB.prototype.drop = function drop() { this.pending = null; this.events.length = 0; - this.coinCache.drop(); return this.wallet.drop(); }; @@ -113,7 +107,6 @@ TXDB.prototype.drop = function drop() { TXDB.prototype.clear = function clear() { this.pending = this.state.clone(); this.events.length = 0; - this.coinCache.clear(); return this.wallet.clear(); }; @@ -128,7 +121,6 @@ TXDB.prototype.commit = async function commit() { } catch (e) { this.pending = null; this.events.length = 0; - this.coinCache.drop(); throw e; } @@ -140,14 +132,13 @@ TXDB.prototype.commit = async function commit() { // Emit buffered events now that // we know everything is written. for (const [event, data, details] of this.events) { - this.walletdb.emit(event, this.wallet.id, data, details); + this.wdb.emit(event, this.wallet.id, data, details); this.wallet.emit(event, data, details); } } this.pending = null; this.events.length = 0; - this.coinCache.commit(); }; /** @@ -298,15 +289,12 @@ TXDB.prototype.hasPath = async function hasPath(output) { TXDB.prototype.saveCredit = async function saveCredit(credit, path) { const coin = credit.coin; - const key = coin.toKey(); const raw = credit.toRaw(); await this.addOutpointMap(coin.hash, coin.index); this.put(layout.c(coin.hash, coin.index), raw); this.put(layout.C(path.account, coin.hash, coin.index), null); - - this.coinCache.push(key, raw); }; /** @@ -317,14 +305,11 @@ TXDB.prototype.saveCredit = async function saveCredit(credit, path) { TXDB.prototype.removeCredit = async function removeCredit(credit, path) { const coin = credit.coin; - const key = coin.toKey(); await this.removeOutpointMap(coin.hash, coin.index); this.del(layout.c(coin.hash, coin.index)); this.del(layout.C(path.account, coin.hash, coin.index)); - - this.coinCache.unpush(key); }; /** @@ -490,89 +475,65 @@ TXDB.prototype.isSpent = function isSpent(hash, index) { }; /** - * Append to the global unspent record. + * Append to global map. + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.addBlockMap = function addBlockMap(height) { + return this.wdb.addBlockMap(this.wallet, height); +}; + +/** + * Remove from global map. + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.removeBlockMap = function removeBlockMap(height) { + return this.wdb.removeBlockMap(this.wallet, height); +}; + +/** + * Append to global map. + * @param {Hash} hash + * @returns {Promise} + */ + +TXDB.prototype.addTXMap = function addTXMap(hash) { + return this.wdb.addTXMap(this.wallet, hash); +}; + +/** + * Remove from global map. + * @param {Hash} hash + * @returns {Promise} + */ + +TXDB.prototype.removeTXMap = function removeTXMap(hash) { + return this.wdb.removeTXMap(this.wallet, hash); +}; + +/** + * Append to global map. * @param {Hash} hash * @param {Number} index * @returns {Promise} */ -TXDB.prototype.addOutpointMap = async function addOutpointMap(hash, i) { - let map = await this.walletdb.getOutpointMap(hash, i); - - if (!map) - map = new OutpointMapRecord(hash, i); - - if (!map.add(this.wallet.wid)) - return; - - this.walletdb.writeOutpointMap(this.wallet, hash, i, map); +TXDB.prototype.addOutpointMap = function addOutpointMap(hash, index) { + return this.wdb.addOutpointMap(this.wallet, hash, index); }; /** - * Remove from the global unspent record. + * Remove from global map. * @param {Hash} hash * @param {Number} index * @returns {Promise} */ -TXDB.prototype.removeOutpointMap = async function removeOutpointMap(hash, i) { - const map = await this.walletdb.getOutpointMap(hash, i); - - if (!map) - return; - - if (!map.remove(this.wallet.wid)) - return; - - if (map.wids.size === 0) { - this.walletdb.unwriteOutpointMap(this.wallet, hash, i); - return; - } - - this.walletdb.writeOutpointMap(this.wallet, hash, i, map); -}; - -/** - * Append to the global block record. - * @param {Hash} hash - * @param {Number} height - * @returns {Promise} - */ - -TXDB.prototype.addBlockMap = async function addBlockMap(hash, height) { - let block = await this.walletdb.getBlockMap(height); - - if (!block) - block = new BlockMapRecord(height); - - if (!block.add(hash, this.wallet.wid)) - return; - - this.walletdb.writeBlockMap(this.wallet, height, block); -}; - -/** - * Remove from the global block record. - * @param {Hash} hash - * @param {Number} height - * @returns {Promise} - */ - -TXDB.prototype.removeBlockMap = async function removeBlockMap(hash, height) { - const block = await this.walletdb.getBlockMap(height); - - if (!block) - return; - - if (!block.remove(hash, this.wallet.wid)) - return; - - if (block.txs.size === 0) { - this.walletdb.unwriteBlockMap(this.wallet, height); - return; - } - - this.walletdb.writeBlockMap(this.wallet, height, block); +TXDB.prototype.removeOutpointMap = function removeOutpointMap(hash, index) { + return this.wdb.removeOutpointMap(this.wallet, hash, index); }; /** @@ -695,7 +656,7 @@ TXDB.prototype.removeBlockSlow = async function removeBlockSlow(hash, height) { if (!block.remove(hash)) return; - if (block.hashes.length === 0) { + if (block.hashes.size === 0) { this.del(layout.b(height)); return; } @@ -797,6 +758,7 @@ TXDB.prototype.insert = async function insert(wtx, block) { const height = block ? block.height : -1; const details = new Details(this, wtx, block); const accounts = new Set(); + let own = false; let updated = false; @@ -940,9 +902,11 @@ TXDB.prototype.insert = async function insert(wtx, block) { this.put(layout.H(account, height, hash), null); } + await this.addTXMap(hash); + // Update block records. if (block) { - await this.addBlockMap(hash, height); + await this.addBlockMap(height); await this.addBlock(tx.hash(), block); } @@ -1029,6 +993,7 @@ TXDB.prototype._confirm = async function _confirm(wtx, block) { for (let i = 0; i < tx.inputs.length; i++) { const input = tx.inputs[i]; const prevout = input.prevout; + let credit = credits[i]; // There may be new credits available @@ -1116,10 +1081,8 @@ TXDB.prototype._confirm = async function _confirm(wtx, block) { this.put(layout.H(account, height, hash), null); } - if (block) { - await this.addBlockMap(hash, height); - await this.addBlock(tx.hash(), block); - } + await this.addBlockMap(height); + await this.addBlock(tx.hash(), block); // Commit the new state. The balance has updated. this.put(layout.R, this.pending.commit()); @@ -1248,9 +1211,11 @@ TXDB.prototype.erase = async function erase(wtx, block) { this.del(layout.H(account, height, hash)); } + await this.removeTXMap(hash); + // Update block records. if (block) { - await this.removeBlockMap(hash, height); + await this.removeBlockMap(height); await this.removeBlockSlow(hash, height); } @@ -1304,6 +1269,28 @@ TXDB.prototype.removeRecursive = async function removeRecursive(wtx) { return details; }; +/** + * Revert a block. + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.revert = async function revert(height) { + const block = await this.getBlock(height); + + if (!block) + return 0; + + const hashes = block.toArray(); + + for (let i = hashes.length - 1; i >= 0; i--) { + const hash = hashes[i]; + await this.unconfirm(hash); + } + + return block.hashes.length; +}; + /** * Unconfirm a transaction. Necessary after a reorg. * @param {Hash} hash @@ -1426,7 +1413,7 @@ TXDB.prototype.disconnect = async function disconnect(wtx, block) { await this.saveCredit(credit, path); } - await this.removeBlockMap(hash, height); + await this.removeBlockMap(height); await this.removeBlock(tx.hash(), height); // We need to update the now-removed @@ -2004,10 +1991,8 @@ TXDB.prototype.getCredits = function getCredits(account) { parse: (key, value) => { const [hash, index] = layout.cc(key); const credit = Credit.fromRaw(value); - const ckey = Outpoint.toKey(hash, index); credit.coin.hash = hash; credit.coin.index = index; - this.coinCache.set(ckey, value); return credit; } }); @@ -2313,17 +2298,6 @@ TXDB.prototype.getCoin = async function getCoin(hash, index) { */ TXDB.prototype.getCredit = async function getCredit(hash, index) { - const state = this.state; - const key = Outpoint.toKey(hash, index); - const cache = this.coinCache.get(key); - - if (cache) { - const credit = Credit.fromRaw(cache); - credit.coin.hash = hash; - credit.coin.index = index; - return credit; - } - const data = await this.get(layout.c(hash, index)); if (!data) @@ -2333,9 +2307,6 @@ TXDB.prototype.getCredit = async function getCredit(hash, index) { credit.coin.hash = hash; credit.coin.index = index; - if (state === this.state) - this.coinCache.set(key, data); - return credit; }; @@ -2401,11 +2372,6 @@ TXDB.prototype.updateSpentCoin = async function updateSpentCoin(tx, index, heigh */ TXDB.prototype.hasCoin = async function hasCoin(hash, index) { - const key = Outpoint.toKey(hash, index); - - if (this.coinCache.has(key)) - return true; - return await this.has(layout.c(hash, index)); }; @@ -2826,7 +2792,7 @@ function Details(txdb, wtx, block) { this.wid = this.wallet.wid; this.id = this.wallet.id; - this.chainHeight = txdb.walletdb.state.height; + this.chainHeight = txdb.wdb.state.height; this.hash = wtx.hash; this.tx = wtx.tx; @@ -3051,8 +3017,7 @@ function BlockRecord(hash, height, time) { this.hash = hash || encoding.NULL_HASH; this.height = height != null ? height : -1; this.time = time || 0; - this.hashes = []; - this.index = new Set(); + this.hashes = new Set(); } /** @@ -3062,11 +3027,10 @@ function BlockRecord(hash, height, time) { */ BlockRecord.prototype.add = function add(hash) { - if (this.index.has(hash)) + if (this.hashes.has(hash)) return false; - this.index.add(hash); - this.hashes.push(hash); + this.hashes.add(hash); return true; }; @@ -3078,24 +3042,7 @@ BlockRecord.prototype.add = function add(hash) { */ BlockRecord.prototype.remove = function remove(hash) { - if (!this.index.has(hash)) - return false; - - this.index.delete(hash); - - // Fast case - if (this.hashes[this.hashes.length - 1] === hash) { - this.hashes.pop(); - return true; - } - - const index = this.hashes.indexOf(hash); - - assert(index !== -1); - - this.hashes.splice(index, 1); - - return true; + return this.hashes.delete(hash); }; /** @@ -3115,8 +3062,7 @@ BlockRecord.prototype.fromRaw = function fromRaw(data) { for (let i = 0; i < count; i++) { const hash = br.readHash('hex'); - this.index.add(hash); - this.hashes.push(hash); + this.hashes.add(hash); } return this; @@ -3138,7 +3084,7 @@ BlockRecord.fromRaw = function fromRaw(data) { */ BlockRecord.prototype.getSize = function getSize() { - return 44 + this.hashes.length * 32; + return 44 + this.hashes.size * 32; }; /** @@ -3154,7 +3100,7 @@ BlockRecord.prototype.toRaw = function toRaw() { bw.writeU32(this.height); bw.writeU32(this.time); - bw.writeU32(this.hashes.length); + bw.writeU32(this.hashes.size); for (const hash of this.hashes) bw.writeHash(hash); @@ -3162,6 +3108,18 @@ BlockRecord.prototype.toRaw = function toRaw() { return bw.render(); }; +/** + * Convert hashes set to an array. + * @returns {Hash[]} + */ + +BlockRecord.prototype.toArray = function toArray() { + const hashes = []; + for (const hash of this.hashes) + hashes.push(hash); + return hashes; +}; + /** * Convert the block to a more json-friendly object. * @returns {Object} @@ -3172,7 +3130,7 @@ BlockRecord.prototype.toJSON = function toJSON() { hash: util.revHex(this.hash), height: this.height, time: this.time, - hashes: this.hashes.map(util.revHex) + hashes: this.toArray().map(util.revHex) }; }; diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 471eb066..5c5ca53f 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -40,7 +40,6 @@ const Mnemonic = HD.Mnemonic; * @alias module:wallet.Wallet * @constructor * @param {Object} options - * @param {WalletDB} options.db * present, no coins will be available. * @param {(HDPrivateKey|HDPublicKey)?} options.master - Master HD key. If not * present, it will be generated. @@ -62,23 +61,23 @@ const Mnemonic = HD.Mnemonic; * (default=account key "address"). */ -function Wallet(db, options) { +function Wallet(wdb, options) { if (!(this instanceof Wallet)) - return new Wallet(db, options); + return new Wallet(wdb, options); EventEmitter.call(this); - assert(db, 'DB required.'); + assert(wdb, 'WDB required.'); - this.db = db; - this.network = db.network; - this.logger = db.logger; + this.wdb = wdb; + this.network = wdb.network; + this.logger = wdb.logger; this.readLock = new MappedLock(); this.writeLock = new Lock(); this.fundLock = new Lock(); - this.indexCache = new LRU(10000); - this.accountCache = new LRU(10000); - this.pathCache = new LRU(100000); + this.indexCache = new LRU(1000); + this.accountCache = new LRU(1000); + this.pathCache = new LRU(1000); this.current = null; this.wid = 0; @@ -175,13 +174,13 @@ Wallet.prototype.fromOptions = function fromOptions(options) { /** * Instantiate wallet from options. - * @param {WalletDB} db + * @param {WalletDB} wdb * @param {Object} options * @returns {Wallet} */ -Wallet.fromOptions = function fromOptions(db, options) { - return new Wallet(db).fromOptions(options); +Wallet.fromOptions = function fromOptions(wdb, options) { + return new Wallet(wdb).fromOptions(options); }; /** @@ -240,7 +239,7 @@ Wallet.prototype.destroy = async function destroy() { const unlock1 = await this.writeLock.lock(); const unlock2 = await this.fundLock.lock(); try { - this.db.unregister(this); + this.wdb.unregister(this); await this.master.destroy(); this.readLock.destroy(); this.writeLock.destroy(); @@ -401,7 +400,7 @@ Wallet.prototype._encrypt = async function _encrypt(passphrase) { this.start(); try { - await this.db.encryptKeys(this, key); + await this.wdb.encryptKeys(this, key); } catch (e) { cleanse(key); this.drop(); @@ -443,7 +442,7 @@ Wallet.prototype._decrypt = async function _decrypt(passphrase) { this.start(); try { - await this.db.decryptKeys(this, key); + await this.wdb.decryptKeys(this, key); } catch (e) { cleanse(key); this.drop(); @@ -502,7 +501,7 @@ Wallet.prototype._retoken = async function _retoken(passphrase) { Wallet.prototype.rename = async function rename(id) { const unlock = await this.writeLock.lock(); try { - return await this.db.rename(this, id); + return await this.wdb.rename(this, id); } finally { unlock(); } @@ -551,7 +550,7 @@ Wallet.prototype._renameAccount = async function _renameAccount(acct, name) { this.start(); - this.db.renameAccount(account, name); + this.wdb.renameAccount(account, name); await this.commit(); @@ -711,7 +710,7 @@ Wallet.prototype._createAccount = async function _createAccount(options, passphr let account; try { - account = Account.fromOptions(this.db, opt); + account = Account.fromOptions(this.wdb, opt); account.wallet = this; await account.init(); } catch (e) { @@ -754,7 +753,7 @@ Wallet.prototype.ensureAccount = async function ensureAccount(options, passphras */ Wallet.prototype.getAccounts = function getAccounts() { - return this.db.getAccounts(this.wid); + return this.wdb.getAccounts(this.wid); }; /** @@ -766,7 +765,7 @@ Wallet.prototype.getAccounts = function getAccounts() { Wallet.prototype.getAddressHashes = function getAddressHashes(acct) { if (acct != null) return this.getAccountHashes(acct); - return this.db.getWalletHashes(this.wid); + return this.wdb.getWalletHashes(this.wid); }; /** @@ -777,7 +776,7 @@ Wallet.prototype.getAddressHashes = function getAddressHashes(acct) { Wallet.prototype.getAccountHashes = async function getAccountHashes(acct) { const index = await this.ensureIndex(acct, true); - return await this.db.getAccountHashes(this.wid, index); + return await this.wdb.getAccountHashes(this.wid, index); }; /** @@ -818,7 +817,7 @@ Wallet.prototype._getAccount = async function _getAccount(index) { if (cache) return cache; - const account = await this.db.getAccount(this.wid, index); + const account = await this.wdb.getAccount(this.wid, index); if (!account) return null; @@ -854,7 +853,7 @@ Wallet.prototype.getAccountIndex = async function getAccountIndex(name) { if (cache != null) return cache; - const index = await this.db.getAccountIndex(this.wid, name); + const index = await this.wdb.getAccountIndex(this.wid, name); if (index === -1) return -1; @@ -880,7 +879,7 @@ Wallet.prototype.getAccountName = async function getAccountName(index) { if (account) return account.name; - return await this.db.getAccountName(this.wid, index); + return await this.wdb.getAccountName(this.wid, index); }; /** @@ -991,7 +990,7 @@ Wallet.prototype._createKey = async function _createKey(acct, branch) { */ Wallet.prototype.save = function save() { - return this.db.save(this); + return this.wdb.save(this); }; /** @@ -1000,7 +999,7 @@ Wallet.prototype.save = function save() { */ Wallet.prototype.start = function start() { - return this.db.start(this); + return this.wdb.start(this); }; /** @@ -1009,7 +1008,7 @@ Wallet.prototype.start = function start() { */ Wallet.prototype.drop = function drop() { - return this.db.drop(this); + return this.wdb.drop(this); }; /** @@ -1018,7 +1017,7 @@ Wallet.prototype.drop = function drop() { */ Wallet.prototype.clear = function clear() { - return this.db.clear(this); + return this.wdb.clear(this); }; /** @@ -1027,7 +1026,7 @@ Wallet.prototype.clear = function clear() { */ Wallet.prototype.commit = function commit() { - return this.db.commit(this); + return this.wdb.commit(this); }; /** @@ -1077,7 +1076,7 @@ Wallet.prototype.readPath = async function readPath(address) { if (cache) return cache; - const path = await this.db.getPath(this.wid, hash); + const path = await this.wdb.getPath(this.wid, hash); if (!path) return null; @@ -1099,7 +1098,7 @@ Wallet.prototype.hasPath = async function hasPath(address) { if (this.pathCache.has(hash)) return true; - return await this.db.hasPath(this.wid, hash); + return await this.wdb.hasPath(this.wid, hash); }; /** @@ -1112,7 +1111,7 @@ Wallet.prototype.getPaths = async function getPaths(acct) { if (acct != null) return await this.getAccountPaths(acct); - const paths = await this.db.getWalletPaths(this.wid); + const paths = await this.wdb.getWalletPaths(this.wid); const result = []; for (const path of paths) { @@ -1375,7 +1374,7 @@ Wallet.prototype._fund = async function _fund(mtx, options) { let rate = options.rate; if (rate == null) - rate = await this.db.estimateFee(options.blocks); + rate = await this.wdb.estimateFee(options.blocks); let coins; if (options.smart) { @@ -1393,7 +1392,7 @@ Wallet.prototype._fund = async function _fund(mtx, options) { subtractFee: options.subtractFee, subtractIndex: options.subtractIndex, changeAddress: account.change.getAddress(), - height: this.db.state.height, + height: this.wdb.state.height, rate: rate, maxFee: options.maxFee, estimate: prev => this.estimateSize(prev) @@ -1544,7 +1543,7 @@ Wallet.prototype.createTX = async function createTX(options, force) { // Consensus sanity checks. assert(mtx.isSane(), 'TX failed sanity check.'); - assert(mtx.verifyInputs(this.db.state.height + 1), + assert(mtx.verifyInputs(this.wdb.state.height + 1), 'TX failed context check.'); const total = await this.template(mtx); @@ -1599,11 +1598,11 @@ Wallet.prototype._send = async function _send(options, passphrase) { if (tx.getWeight() > policy.MAX_TX_WEIGHT) throw new Error('TX exceeds policy weight.'); - await this.db.addTX(tx); + await this.wdb.addTX(tx); this.logger.debug('Sending wallet tx (%s): %s', this.id, tx.txid()); - await this.db.send(tx); + await this.wdb.send(tx); return tx; }; @@ -1705,8 +1704,8 @@ Wallet.prototype.increaseFee = async function increaseFee(hash, rate, passphrase 'Increasing fee for wallet tx (%s): %s', this.id, ntx.txid()); - await this.db.addTX(ntx); - await this.db.send(ntx); + await this.wdb.addTX(ntx); + await this.wdb.send(ntx); return ntx; }; @@ -1730,7 +1729,7 @@ Wallet.prototype.resend = async function resend() { const sorted = common.sortDeps(txs); for (const tx of sorted) - await this.db.send(tx); + await this.wdb.send(tx); return txs; }; @@ -1911,15 +1910,15 @@ Wallet.prototype._setLookahead = async function _setLookahead(acct, lookahead) { * Sync address depths based on a transaction's outputs. * This is used for deriving new addresses when * a confirmed transaction is seen. - * @param {Details} details + * @param {TX} tx * @returns {Promise} */ -Wallet.prototype.syncOutputDepth = async function syncOutputDepth(details) { +Wallet.prototype.syncOutputDepth = async function syncOutputDepth(tx) { const map = new Map(); - for (const output of details.outputs) { - const path = output.path; + for (const hash of tx.getOutputHashes('hex')) { + const path = await this.readPath(hash); if (!path) continue; @@ -2022,7 +2021,7 @@ Wallet.prototype.sign = async function sign(mtx, passphrase) { const rings = await this.deriveInputs(mtx); - return await mtx.signAsync(rings, Script.hashType.ALL, this.db.workers); + return await mtx.signAsync(rings, Script.hashType.ALL, this.wdb.workers); }; /** @@ -2135,7 +2134,7 @@ Wallet.prototype._add = async function _add(tx, block) { try { details = await this.txdb._add(tx, block); if (details) - derived = await this.syncOutputDepth(details); + derived = await this.syncOutputDepth(tx); } catch (e) { this.txdb.drop(); throw e; @@ -2144,7 +2143,7 @@ Wallet.prototype._add = async function _add(tx, block) { await this.txdb.commit(); if (derived && derived.length > 0) { - this.db.emit('address', this.id, derived); + this.wdb.emit('address', this.id, derived); this.emit('address', derived); } @@ -2152,15 +2151,15 @@ Wallet.prototype._add = async function _add(tx, block) { }; /** - * Unconfirm a wallet transcation. - * @param {Hash} hash + * Revert a block. + * @param {Number} height * @returns {Promise} */ -Wallet.prototype.unconfirm = async function unconfirm(hash) { +Wallet.prototype.revert = async function revert(height) { const unlock = await this.writeLock.lock(); try { - return await this.txdb.unconfirm(hash); + return await this.txdb.revert(height); } finally { unlock(); } @@ -2557,7 +2556,7 @@ Wallet.prototype.fromRaw = function fromRaw(data) { this.tokenDepth = br.readU32(); this.master.fromRaw(br.readVarBytes()); - assert(network === this.db.network, 'Wallet network mismatch.'); + assert(network === this.wdb.network, 'Wallet network mismatch.'); return this; }; @@ -2568,8 +2567,8 @@ Wallet.prototype.fromRaw = function fromRaw(data) { * @returns {Wallet} */ -Wallet.fromRaw = function fromRaw(db, data) { - return new Wallet(db).fromRaw(data); +Wallet.fromRaw = function fromRaw(wdb, data) { + return new Wallet(wdb).fromRaw(data); }; /** diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 04fa8132..963101f1 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -13,7 +13,6 @@ const AsyncObject = require('../utils/asyncobject'); const util = require('../utils/util'); const Lock = require('../utils/lock'); const MappedLock = require('../utils/mappedlock'); -const LRU = require('../utils/lru'); const encoding = require('../utils/encoding'); const ccmp = require('../crypto/ccmp'); const aes = require('../crypto/aes'); @@ -30,13 +29,12 @@ const layouts = require('./layout'); const records = require('./records'); const HTTPServer = require('./http'); const RPC = require('./rpc'); +const StaticWriter = require('../utils/staticwriter'); const layout = layouts.walletdb; const ChainState = records.ChainState; -const BlockMapRecord = records.BlockMapRecord; const BlockMeta = records.BlockMeta; -const PathMapRecord = records.PathMapRecord; -const OutpointMapRecord = records.OutpointMapRecord; const TXRecord = records.TXRecord; +const MapRecord = records.MapRecord; const U32 = encoding.U32; /** @@ -97,9 +95,6 @@ function WalletDB(options) { this.writeLock = new Lock(); this.txLock = new Lock(); - this.widCache = new LRU(10000); - this.pathMapCache = new LRU(100000); - this.filter = new Bloom(); this._init(); @@ -309,32 +304,43 @@ WalletDB.prototype.disconnect = async function disconnect() { WalletDB.prototype.init = async function init() { const state = await this.getState(); - const startHeight = this.options.startHeight; if (state) { this.state = state; return; } - let tip; + const batch = this.db.batch(); + + let tip = null; + if (this.client) { - if (startHeight != null) { - tip = await this.client.getEntry(startHeight); - if (!tip) - throw new Error('WDB: Could not find start block.'); - } else { - tip = await this.client.getTip(); + const hashes = await this.client.getHashes(); + + for (let height = 0; height < hashes.length; height++) { + const hash = hashes[height]; + const meta = new BlockMeta(hash, height); + batch.put(layout.h(height), meta.toHash()); + tip = meta; } - tip = BlockMeta.fromEntry(tip); } else { - tip = BlockMeta.fromEntry(this.network.genesis); + tip = new BlockMeta(this.network.genesis.hash, 0); + batch.put(layout.h(0), tip.toHash()); } - this.logger.info( - 'Initializing WalletDB chain state at %s (%d).', - util.revHex(tip.hash), tip.height); + assert(tip); - await this.resetState(tip, false); + const pending = this.state.clone(); + pending.startHeight = tip.height; + pending.startHash = tip.hash; + pending.height = tip.height; + pending.marked = false; + + batch.put(layout.R, pending.toRaw()); + + await batch.write(); + + this.state = pending; }; /** @@ -344,50 +350,40 @@ WalletDB.prototype.init = async function init() { */ WalletDB.prototype.watch = async function watch() { - let iter = this.db.iterator({ + const piter = this.db.iterator({ gte: layout.p(encoding.NULL_HASH), lte: layout.p(encoding.HIGH_HASH) }); let hashes = 0; - let outpoints = 0; - while (await iter.next()) { - const {key} = iter; + await piter.each((key) => { + const data = layout.pp(key); - try { - const data = layout.pp(key); - this.filter.add(data, 'hex'); - } catch (e) { - await iter.end(); - throw e; - } + this.filter.add(data, 'hex'); - hashes++; - } + hashes += 1; + }); - iter = this.db.iterator({ + this.logger.info('Added %d hashes to WalletDB filter.', hashes); + + const oiter = this.db.iterator({ gte: layout.o(encoding.NULL_HASH, 0), lte: layout.o(encoding.HIGH_HASH, 0xffffffff) }); - while (await iter.next()) { - const {key} = iter; + let outpoints = 0; - try { - const [hash, index] = layout.oo(key); - const outpoint = new Outpoint(hash, index); - const data = outpoint.toRaw(); - this.filter.add(data); - } catch (e) { - await iter.end(); - throw e; - } + await oiter.each((key) => { + const [hash, index] = layout.oo(key); + const outpoint = new Outpoint(hash, index); + const data = outpoint.toRaw(); - outpoints++; - } + this.filter.add(data); + + outpoints += 1; + }); - this.logger.info('Added %d hashes to WalletDB filter.', hashes); this.logger.info('Added %d outpoints to WalletDB filter.', outpoints); await this.setFilter(); @@ -404,29 +400,22 @@ WalletDB.prototype.sync = async function sync() { return; let height = this.state.height; - let entry; + let entry = null; - while (height >= 0) { + for (;;) { const tip = await this.getBlock(height); - - if (!tip) - break; + assert(tip); entry = await this.client.getEntry(tip.hash); if (entry) break; - height--; + assert(height !== 0); + height -= 1; } - if (!entry) { - height = this.state.startHeight; - entry = await this.client.getEntry(this.state.startHash); - - if (!entry) - height = 0; - } + assert(entry); await this.scan(height); }; @@ -599,29 +588,23 @@ WalletDB.prototype.wipe = async function wipe() { }); const batch = this.db.batch(); + let total = 0; - while (await iter.next()) { - const {key} = iter; - - try { - switch (key[0]) { - case 0x62: // b - case 0x63: // c - case 0x65: // e - case 0x74: // t - case 0x6f: // o - case 0x68: // h - case 0x52: // R - batch.del(key); - total++; - break; - } - } catch (e) { - await iter.end(); - throw e; + await iter.each((key) => { + switch (key[0]) { + case 0x62: // b + case 0x63: // c + case 0x65: // e + case 0x74: // t + case 0x6f: // o + case 0x68: // h + case 0x52: // R + batch.del(key); + total += 1; + break; } - } + }); this.logger.warning('Wiped %d txdb records.', total); @@ -817,21 +800,14 @@ WalletDB.prototype.getWalletID = async function getWalletID(id) { if (typeof id === 'number') return id; - const cache = this.widCache.get(id); - - if (cache) - return cache; - const data = await this.db.get(layout.l(id)); if (!data) return null; - const wid = data.readUInt32LE(0, true); + assert(data.length === 4); - this.widCache.set(id, wid); - - return wid; + return data.readUInt32LE(0, true); }; /** @@ -892,8 +868,6 @@ WalletDB.prototype.save = function save(wallet) { const id = wallet.id; const batch = this.batch(wallet); - this.widCache.set(id, wid); - batch.put(layout.w(wid), wallet.toRaw()); batch.put(layout.l(id), U32(wid)); }; @@ -940,8 +914,6 @@ WalletDB.prototype._rename = async function _rename(wallet, id) { await this.commit(wallet); - this.widCache.remove(old); - const paths = wallet.pathCache.values(); for (const path of paths) @@ -1025,10 +997,13 @@ WalletDB.prototype._create = async function _create(options) { throw new Error('WDB: Wallet already exists.'); const wallet = Wallet.fromOptions(this, options); - wallet.wid = this.depth++; + + wallet.wid = this.depth; await wallet.init(options); + this.depth += 1; + this.register(wallet); this.logger.info('Created wallet %s in WalletDB.', wallet.id); @@ -1161,31 +1136,6 @@ WalletDB.prototype.hasAccount = function hasAccount(wid, index) { return this.db.has(layout.a(wid, index)); }; -/** - * Lookup the corresponding account name's index. - * @param {WalletID} wid - * @param {String|Number} name - Account name/index. - * @returns {Promise} - Returns Number. - */ - -WalletDB.prototype.getPathMap = async function getPathMap(hash) { - const cache = this.pathMapCache.get(hash); - - if (cache) - return cache; - - const data = await this.db.get(layout.p(hash)); - - if (!data) - return null; - - const map = PathMapRecord.fromRaw(hash, data); - - this.pathMapCache.set(hash, map); - - return map; -}; - /** * Save an address to the path map. * @param {Wallet} wallet @@ -1215,21 +1165,10 @@ WalletDB.prototype.savePath = async function savePath(wallet, path) { const hash = path.hash; const batch = this.batch(wallet); - await this.addHash(hash); - - let map = await this.getPathMap(hash); - - if (!map) - map = new PathMapRecord(hash); - - if (!map.add(wid)) - return; - - this.pathMapCache.set(hash, map); wallet.pathCache.push(hash, path); // Address Hash -> Wallet Map - batch.put(layout.p(hash), map.toRaw()); + await this.addPathMap(wallet, hash); // Wallet ID + Address Hash -> Path Data batch.put(layout.P(wid, hash), path.toRaw()); @@ -1500,17 +1439,16 @@ WalletDB.prototype.resendPending = async function resendPending(wid) { */ WalletDB.prototype.getWalletsByTX = async function getWalletsByTX(tx) { - const hashes = tx.getOutputHashes('hex'); const result = new Set(); if (!tx.isCoinbase()) { - for (const input of tx.inputs) { - const prevout = input.prevout; + for (const {prevout} of tx.inputs) { + const {hash, index} = prevout; if (!this.testFilter(prevout.toRaw())) continue; - const map = await this.getOutpointMap(prevout.hash, prevout.index); + const map = await this.getOutpointMap(hash, index); if (!map) continue; @@ -1520,6 +1458,8 @@ WalletDB.prototype.getWalletsByTX = async function getWalletsByTX(tx) { } } + const hashes = tx.getOutputHashes('hex'); + for (const hash of hashes) { if (!this.testFilter(hash)) continue; @@ -1553,44 +1493,6 @@ WalletDB.prototype.getState = async function getState() { return ChainState.fromRaw(data); }; -/** - * Reset the chain state to a tip/start-block. - * @param {BlockMeta} tip - * @returns {Promise} - */ - -WalletDB.prototype.resetState = async function resetState(tip, marked) { - const batch = this.db.batch(); - const state = this.state.clone(); - - const iter = this.db.iterator({ - gte: layout.h(0), - lte: layout.h(0xffffffff), - values: false - }); - - while (await iter.next()) { - try { - batch.del(iter.key); - } catch (e) { - await iter.end(); - throw e; - } - } - - state.startHeight = tip.height; - state.startHash = tip.hash; - state.height = tip.height; - state.marked = marked; - - batch.put(layout.h(tip.height), tip.toHash()); - batch.put(layout.R, state.toRaw()); - - await batch.write(); - - this.state = state; -}; - /** * Sync the current chain state to tip. * @param {BlockMeta} tip @@ -1604,27 +1506,20 @@ WalletDB.prototype.syncState = async function syncState(tip) { if (tip.height < state.height) { // Hashes ahead of our new tip // that we need to delete. - let height = state.height; - let blocks = height - tip.height; - - if (blocks > this.options.keepBlocks) - blocks = this.options.keepBlocks; - - for (let i = 0; i < blocks; i++) { - batch.del(layout.h(height)); - height--; + while (state.height !== tip.height) { + batch.del(layout.h(state.height)); + state.height -= 1; } } else if (tip.height > state.height) { - // Prune old hashes. - const height = tip.height - this.options.keepBlocks; - assert(tip.height === state.height + 1, 'Bad chain sync.'); - - if (height >= 0) - batch.del(layout.h(height)); + state.height += 1; } - state.height = tip.height; + if (tip.height < state.startHeight) { + state.startHeight = tip.height; + state.startHash = tip.hash; + state.marked = false; + } // Save tip and state. batch.put(layout.h(tip.height), tip.toHash()); @@ -1636,101 +1531,224 @@ WalletDB.prototype.syncState = async function syncState(tip) { }; /** - * Mark the start block once a confirmed tx is seen. - * @param {BlockMeta} tip + * Mark current state. + * @param {BlockMeta} block * @returns {Promise} */ -WalletDB.prototype.maybeMark = async function maybeMark(tip) { - if (this.state.marked) - return; +WalletDB.prototype.markState = async function markState(block) { + const state = this.state.clone(); + state.startHeight = block.height; + state.startHash = block.hash; + state.marked = true; - this.logger.info('Marking WalletDB start block at %s (%d).', - util.revHex(tip.hash), tip.height); + const batch = this.db.batch(); + batch.put(layout.R, state.toRaw()); + await batch.write(); - await this.resetState(tip, true); + this.state = state; }; /** - * Get a block->wallet map. - * @param {Number} height + * Get a wallet map. + * @param {Buffer} key + * @returns {Promise} + */ + +WalletDB.prototype.getMap = async function getMap(key) { + const data = await this.db.get(key); + + if (!data) + return null; + + return MapRecord.fromRaw(data); +}; + +/** + * Add wid to a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.addMap = async function addMap(wallet, key) { + const wid = wallet.wid; + const batch = this.batch(wallet); + const data = await this.db.get(key); + + if (!data) { + const map = new MapRecord(); + map.add(wid); + batch.put(key, map.toRaw()); + return; + } + + assert(data.length >= 4); + + const len = data.readUInt32LE(0, true); + const bw = new StaticWriter(data.length + 4); + + bw.writeU32(len + 1); + bw.copy(data, 4, data.length); + bw.writeU32(wid); + + batch.put(key, bw.render()); +}; + +/** + * Remove wid from a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.removeMap = async function removeMap(wallet, key) { + const wid = wallet.wid; + const batch = this.batch(wallet); + const map = await this.getMap(key); + + if (!map) + return; + + if (!map.remove(wid)) + return; + + if (map.size === 0) { + batch.del(key); + return; + } + + batch.put(key, map.toRaw()); +}; + +/** + * Get a wallet map. + * @param {Buffer} key + * @returns {Promise} + */ + +WalletDB.prototype.getPathMap = function getPathMap(hash) { + return this.getMap(layout.p(hash)); +}; + +/** + * Add wid to a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.addPathMap = function addPathMap(wallet, hash) { + this.addHash(hash); + return this.addMap(wallet, layout.p(hash)); +}; + +/** + * Remove wid from a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.removePathMap = function removePathMap(wallet, hash) { + return this.removeMap(wallet, layout.p(hash)); +}; + +/** + * Get a wallet map. + * @param {Buffer} key * @returns {Promise} */ WalletDB.prototype.getBlockMap = async function getBlockMap(height) { - const data = await this.db.get(layout.b(height)); - - if (!data) - return null; - - return BlockMapRecord.fromRaw(height, data); + return this.getMap(layout.b(height)); }; /** - * Add block to the global block map. + * Add wid to a wallet map. * @param {Wallet} wallet - * @param {Number} height - * @param {BlockMapRecord} block + * @param {Buffer} key + * @param {Number} wid */ -WalletDB.prototype.writeBlockMap = function writeBlockMap(wallet, height, block) { - const batch = this.batch(wallet); - batch.put(layout.b(height), block.toRaw()); +WalletDB.prototype.addBlockMap = function addBlockMap(wallet, height) { + return this.addMap(wallet, layout.b(height)); }; /** - * Remove a block from the global block map. + * Remove wid from a wallet map. * @param {Wallet} wallet - * @param {Number} height + * @param {Buffer} key + * @param {Number} wid */ -WalletDB.prototype.unwriteBlockMap = function unwriteBlockMap(wallet, height) { - const batch = this.batch(wallet); - batch.del(layout.b(height)); +WalletDB.prototype.removeBlockMap = function removeBlockMap(wallet, height) { + return this.removeMap(wallet, layout.b(height)); }; /** - * Get a Unspent->Wallet map. - * @param {Hash} hash - * @param {Number} index + * Get a wallet map. + * @param {Buffer} key + * @returns {Promise} + */ + +WalletDB.prototype.getTXMap = function getTXMap(hash) { + return this.getMap(layout.T(hash)); +}; + +/** + * Add wid to a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.addTXMap = function addTXMap(wallet, hash) { + return this.addMap(wallet, layout.T(hash)); +}; + +/** + * Remove wid from a wallet map. + * @param {Wallet} wallet + * @param {Buffer} key + * @param {Number} wid + */ + +WalletDB.prototype.removeTXMap = function removeTXMap(wallet, hash) { + return this.removeMap(wallet, layout.T(hash)); +}; + +/** + * Get a wallet map. + * @param {Buffer} key * @returns {Promise} */ WalletDB.prototype.getOutpointMap = async function getOutpointMap(hash, index) { - const data = await this.db.get(layout.o(hash, index)); - - if (!data) - return null; - - return OutpointMapRecord.fromRaw(hash, index, data); + return this.getMap(layout.o(hash, index)); }; /** - * Add an outpoint to global unspent map. + * Add wid to a wallet map. * @param {Wallet} wallet - * @param {Hash} hash - * @param {Number} index - * @param {OutpointMapRecord} map + * @param {Buffer} key + * @param {Number} wid */ -WalletDB.prototype.writeOutpointMap = function writeOutpointMap(wallet, hash, index, map) { - const batch = this.batch(wallet); - +WalletDB.prototype.addOutpointMap = async function addOutpointMap(wallet, hash, index) { this.addOutpoint(hash, index); - - batch.put(layout.o(hash, index), map.toRaw()); + return this.addMap(wallet, layout.o(hash, index)); }; /** - * Remove an outpoint from global unspent map. + * Remove wid from a wallet map. * @param {Wallet} wallet - * @param {Hash} hash - * @param {Number} index + * @param {Buffer} key + * @param {Number} wid */ -WalletDB.prototype.unwriteOutpointMap = function unwriteOutpointMap(wallet, hash, index) { - const batch = this.batch(wallet); - batch.del(layout.o(hash, index)); +WalletDB.prototype.removeOutpointMap = async function removeOutpointMap(wallet, hash, index) { + return this.removeMap(wallet, layout.o(hash, index)); }; /** @@ -1786,37 +1804,13 @@ WalletDB.prototype.rollback = async function rollback(height) { 'Rolling back %d WalletDB blocks to height %d.', this.state.height - height, height); - let tip = await this.getBlock(height); - let marked = false; - - if (tip) { - await this.revert(tip.height); - await this.syncState(tip); - return true; - } - - tip = new BlockMeta(); - - if (height >= this.state.startHeight) { - tip.height = this.state.startHeight; - tip.hash = this.state.startHash; - marked = this.state.marked; - - this.logger.warning( - 'Rolling back WalletDB to start block (%d).', - tip.height); - } else { - tip.height = 0; - tip.hash = this.network.genesis.hash; - marked = false; - - this.logger.warning('Rolling back WalletDB to genesis block.'); - } + const tip = await this.getBlock(height); + assert(tip); await this.revert(tip.height); - await this.resetState(tip, marked); + await this.syncState(tip); - return false; + return true; }; /** @@ -1835,27 +1829,18 @@ WalletDB.prototype.revert = async function revert(target) { let total = 0; - while (await iter.next()) { - const {key, value} = iter; + await iter.each(async (key, value) => { + const height = layout.bb(key); + const block = MapRecord.fromRaw(value); - try { - const height = layout.bb(key); - const block = BlockMapRecord.fromRaw(height, value); - const txs = block.toArray(); - - total += txs.length; - - for (let i = txs.length - 1; i >= 0; i--) { - const tx = txs[i]; - await this._unconfirm(tx); - } - } catch (e) { - await iter.end(); - throw e; + for (const wid of block.wids) { + const wallet = await this.get(wid); + assert(wallet); + total += await wallet.revert(height); } - } + }); - this.logger.info('Rolled back %d WalletDB transactions.', total); + this.logger.info('Rolled back WalletDB %d transactions.', total); }; /** @@ -1883,13 +1868,12 @@ WalletDB.prototype.addBlock = async function addBlock(entry, txs) { WalletDB.prototype._addBlock = async function _addBlock(entry, txs) { const tip = BlockMeta.fromEntry(entry); - let total = 0; if (tip.height < this.state.height) { this.logger.warning( 'WalletDB is connecting low blocks (%d).', tip.height); - return total; + return 0; } if (tip.height === this.state.height) { @@ -1909,12 +1893,14 @@ WalletDB.prototype._addBlock = async function _addBlock(entry, txs) { if (this.options.checkpoints) { if (tip.height <= this.network.lastCheckpoint) - return total; + return 0; } + let total = 0; + for (const tx of txs) { if (await this._insert(tx, tip)) - total++; + total += 1; } if (total > 0) { @@ -1951,6 +1937,9 @@ WalletDB.prototype.removeBlock = async function removeBlock(entry) { WalletDB.prototype._removeBlock = async function _removeBlock(entry) { const tip = BlockMeta.fromEntry(entry); + if (tip.height === 0) + throw new Error('WDB: Bad disconnection (genesis block).'); + if (tip.height > this.state.height) { this.logger.warning( 'WalletDB is disconnecting high blocks (%d).', @@ -1962,32 +1951,31 @@ WalletDB.prototype._removeBlock = async function _removeBlock(entry) { throw new Error('WDB: Bad disconnection (height mismatch).'); const prev = await this.getBlock(tip.height - 1); + assert(prev); - if (!prev) - throw new Error('WDB: Bad disconnection (no previous block).'); + // Get the map of block->wids. + const map = await this.getBlockMap(tip.height); - // Get the map of txids->wids. - const block = await this.getBlockMap(tip.height); - - if (!block) { + if (!map) { await this.syncState(prev); return 0; } - const txs = block.toArray(); + let total = 0; - for (let i = txs.length - 1; i >= 0; i--) { - const tx = txs[i]; - await this._unconfirm(tx); + for (const wid of map.wids) { + const wallet = await this.get(wid); + assert(wallet); + total += await wallet.revert(tip.height); } // Sync the state to the previous tip. await this.syncState(prev); this.logger.warning('Disconnected wallet block %s (tx=%d).', - util.revHex(tip.hash), block.txs.size); + util.revHex(tip.hash), total); - return block.txs.size; + return total; }; /** @@ -2040,21 +2028,20 @@ WalletDB.prototype.addTX = async function addTX(tx) { WalletDB.prototype._insert = async function _insert(tx, block) { const wids = await this.getWalletsByTX(tx); - let result = false; assert(!tx.mutable, 'WDB: Cannot add mutable TX.'); if (!wids) return null; + if (block && !this.state.marked) + await this.markState(block); + this.logger.info( 'Incoming transaction for %d wallets in WalletDB (%s).', wids.size, tx.txid()); - // If this is our first transaction - // in a block, set the start block here. - if (block) - await this.maybeMark(block); + let result = false; // Insert the transaction // into every matching wallet. @@ -2077,22 +2064,6 @@ WalletDB.prototype._insert = async function _insert(tx, block) { return wids; }; -/** - * Unconfirm a transaction from all - * relevant wallets without a lock. - * @private - * @param {TXMapRecord} tx - * @returns {Promise} - */ - -WalletDB.prototype._unconfirm = async function _unconfirm(tx) { - for (const wid of tx.wids) { - const wallet = await this.get(wid); - assert(wallet); - await wallet.unconfirm(tx.hash); - } -}; - /** * Handle a chain reset. * @param {ChainEntry} entry @@ -2157,7 +2128,6 @@ function WalletOptions(options) { this.witness = false; this.checkpoints = false; this.startHeight = 0; - this.keepBlocks = this.network.block.keepBlocks; this.wipeNoReally = false; this.apiKey = null; this.walletAuth = false; @@ -2181,7 +2151,6 @@ function WalletOptions(options) { WalletOptions.prototype.fromOptions = function fromOptions(options) { if (options.network != null) { this.network = Network.get(options.network); - this.keepBlocks = this.network.block.keepBlocks; this.port = this.network.rpcPort + 2; } diff --git a/test/node-test.js b/test/node-test.js index 73d0021c..f2551107 100644 --- a/test/node-test.js +++ b/test/node-test.js @@ -582,7 +582,7 @@ describe('Node', function() { const tx = mtx.toTX(); - await wallet.db.addTX(tx); + await wallet.wdb.addTX(tx); const missing = await node.mempool.addTX(tx); assert(!missing); @@ -607,7 +607,7 @@ describe('Node', function() { const tx = mtx.toTX(); - await wallet.db.addTX(tx); + await wallet.wdb.addTX(tx); const missing = await node.mempool.addTX(tx); assert(!missing);