From 45ad99c8f57ef54d8cf49c577add14128eb99cb0 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Fri, 28 Oct 2016 10:28:42 -0700 Subject: [PATCH] walletdb: add sync state object. --- lib/wallet/records.js | 69 +++++++++++ lib/wallet/txdb.js | 10 +- lib/wallet/wallet.js | 4 +- lib/wallet/walletdb.js | 254 ++++++++++++++++++++++------------------- test/chain-test.js | 2 +- 5 files changed, 213 insertions(+), 126 deletions(-) diff --git a/lib/wallet/records.js b/lib/wallet/records.js index 5a577d72..148e5e56 100644 --- a/lib/wallet/records.js +++ b/lib/wallet/records.js @@ -12,6 +12,74 @@ var constants = require('../protocol/constants'); var BufferReader = require('../utils/reader'); var BufferWriter = require('../utils/writer'); +/** + * Wallet Tip + * @constructor + * @param {Hash} hash + * @param {Number} height + */ + +function SyncState() { + if (!(this instanceof SyncState)) + return new SyncState(); + + this.start = new HeaderRecord(); + this.tip = new HeaderRecord(); +} + +/** + * Clone the block. + * @returns {SyncState} + */ + +SyncState.prototype.clone = function clone() { + var state = new SyncState(); + state.start = this.start.clone(); + state.tip = this.tip.clone(); + return state; +}; + +/** + * Instantiate wallet block from serialized tip data. + * @private + * @param {Buffer} data + */ + +SyncState.prototype.fromRaw = function fromRaw(data) { + var p = new BufferReader(data); + this.start.fromRaw(p); + this.tip.fromRaw(p); + return this; +}; + +/** + * Instantiate wallet block from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {SyncState} + */ + +SyncState.fromRaw = function fromRaw(data) { + return new SyncState().fromRaw(data); +}; + +/** + * Serialize the wallet block as a tip (hash and height). + * @returns {Buffer} + */ + +SyncState.prototype.toRaw = function toRaw(writer) { + var p = new BufferWriter(writer); + + this.start.toRaw(p); + this.tip.toRaw(p); + + if (!writer) + p = p.render(); + + return p; +}; + /** * Wallet Tip * @constructor @@ -364,6 +432,7 @@ function serializeWallets(wids) { * Expose */ +exports.SyncState = SyncState; exports.HeaderRecord = HeaderRecord; exports.BlockMapRecord = BlockMapRecord; exports.TXMapRecord = TXMapRecord; diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index b92ac301..e7df794e 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -1550,13 +1550,13 @@ TXDB.prototype.removeRecursive = co(function* removeRecursive(tx) { * @returns {Promise} */ -TXDB.prototype.unconfirm = co(function* unconfirm(hash, block) { +TXDB.prototype.unconfirm = co(function* unconfirm(hash) { var details; this.start(); try { - details = yield this._unconfirm(hash, block); + details = yield this._unconfirm(hash); } catch (e) { this.drop(); throw e; @@ -1574,13 +1574,13 @@ TXDB.prototype.unconfirm = co(function* unconfirm(hash, block) { * @returns {Promise} */ -TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) { +TXDB.prototype._unconfirm = co(function* unconfirm(hash) { var tx = yield this.getTX(hash); if (!tx) return; - return yield this.disconnect(tx, block); + return yield this.disconnect(tx); }); /** @@ -1589,7 +1589,7 @@ TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) { * @returns {Promise} */ -TXDB.prototype.disconnect = co(function* disconnect(tx, block) { +TXDB.prototype.disconnect = co(function* disconnect(tx) { var hash = tx.hash('hex'); var details = new Details(this, tx); var height = tx.height; diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 66d746ef..5e21ea0e 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -1965,10 +1965,10 @@ Wallet.prototype._insert = co(function* insert(tx, block) { * @returns {Promise} */ -Wallet.prototype.unconfirm = co(function* unconfirm(hash, block) { +Wallet.prototype.unconfirm = co(function* unconfirm(hash) { var unlock = yield this.writeLock.lock(); try { - return yield this.txdb.unconfirm(hash, block); + return yield this.txdb.unconfirm(hash); } finally { unlock(); } diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 9f9dee5a..f7a39f55 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -24,6 +24,7 @@ var Bloom = require('../utils/bloom'); var Logger = require('../node/logger'); var TX = require('../primitives/tx'); var records = require('./records'); +var SyncState = records.SyncState; var BlockMapRecord = records.BlockMapRecord; var HeaderRecord = records.HeaderRecord; var PathMapRecord = records.PathMapRecord; @@ -165,10 +166,11 @@ function WalletDB(options) { this.logger = options.logger || Logger.global; this.client = options.client; - this.tip = null; - this.height = -1; + this.state = new SyncState(); this.depth = 0; this.wallets = {}; + this.genesis = HeaderRecord.fromEntry(this.network.genesis); + this.keepBlocks = this.network.block.keepBlocks; // We need one read lock for `get` and `create`. // It will hold locks specific to wallet ids. @@ -195,12 +197,14 @@ function WalletDB(options) { writeBufferSize: 4 << 20, bufferKeys: !utils.isBrowser }); - - this._init(); } utils.inherits(WalletDB, AsyncObject); +WalletDB.prototype.__defineGetter__('height', function() { + return this.state.tip.height; +}); + /** * Database layout. * @type {Object} @@ -208,15 +212,6 @@ utils.inherits(WalletDB, AsyncObject); WalletDB.layout = layout; -/** - * Initialize wallet db. - * @private - */ - -WalletDB.prototype._init = function _init() { - ; -}; - /** * Open the walletdb, wait for the database to load. * @alias WalletDB#open @@ -226,19 +221,20 @@ WalletDB.prototype._init = function _init() { WalletDB.prototype._open = co(function* open() { yield this.db.open(); yield this.db.checkVersion('V', 5); - yield this.writeGenesis(); + + this.depth = yield this.getDepth(); if (this.options.wipeNoReally) yield this.wipe(); - this.depth = yield this.getDepth(); + yield this.init(); + yield this.watch(); + yield this.sync(); + yield this.resend(); this.logger.info( 'WalletDB loaded (depth=%d, height=%d).', - this.depth, this.height); - - yield this.connect(); - yield this.resend(); + this.depth, this.state.tip.height); }); /** @@ -260,15 +256,29 @@ WalletDB.prototype._close = co(function* close() { yield this.db.close(); }); +/** + * Connect and sync with the chain server (without a lock). + * @private + * @returns {Promise} + */ + +WalletDB.prototype.watch = co(function* watch() { + var hashes = yield this.getFilterHashes(); + + this.logger.info('Adding %d hashes to filter.', hashes.length); + + this.addFilter(hashes); +}); + /** * Connect and sync with the chain server. * @returns {Promise} */ -WalletDB.prototype.connect = co(function* connect() { +WalletDB.prototype.sync = co(function* sync() { var unlock = yield this.txLock.lock(); try { - return yield this._connect(); + return yield this._sync(); } finally { unlock(); } @@ -280,36 +290,36 @@ WalletDB.prototype.connect = co(function* connect() { * @returns {Promise} */ -WalletDB.prototype._connect = co(function* connect() { - var hashes = yield this.getFilterHashes(); - var tip, height; - - this.logger.info('Adding %d hashes to filter.', hashes.length); - - this.addFilter(hashes); +WalletDB.prototype._sync = co(function* connect() { + var height = this.state.tip.height; + var tip, entry; if (!this.client) return; - if (this.options.noScan) { - tip = yield this.client.getTip(); + while (height >= 0) { + tip = yield this.getHeader(height); if (!tip) - throw new Error('Could not get chain tip.'); + break; - yield this.forceTip(tip); + entry = yield this.client.getEntry(tip.hash); - return; + if (entry) + break; + + height--; } - assert(this.network.block.keepBlocks > 36); + if (!entry) { + height = this.state.start.height; + entry = yield this.client.getEntry(this.state.start.hash); - height = this.height - 36; + if (!entry) + height = 0; + } - if (height < 0) - height = 0; - - yield this.scan(height, hashes); + yield this.scan(height); }); /** @@ -336,10 +346,7 @@ WalletDB.prototype.rescan = co(function* rescan(height) { */ WalletDB.prototype._rescan = co(function* rescan(height) { - if (!this.client) - return; - - yield this.scan(height); + return yield this.scan(height); }); /** @@ -352,26 +359,20 @@ WalletDB.prototype._rescan = co(function* rescan(height) { WalletDB.prototype.scan = co(function* scan(height) { var self = this; - var blocks; if (!this.client) return; - assert(utils.isNumber(height), 'Must pass in a height.'); + assert(utils.isUInt32(height), 'Must pass in a height.'); - blocks = this.height - height; - - if (blocks < 0) + if (height > this.state.tip.height) throw new Error('Cannot rescan future blocks.'); - if (blocks > this.network.block.keepBlocks) - throw new Error('Cannot roll back beyond keepBlocks.'); - yield this.rollback(height); - this.logger.info('Scanning for blocks.'); + this.logger.info('Scanning %d blocks.', this.state.tip.height - height); - yield this.client.scan(this.tip.hash, this.filter, function(block, txs) { + yield this.client.scan(this.state.tip.hash, this.filter, function(block, txs) { return self._addBlock(block, txs); }); }); @@ -503,7 +504,6 @@ WalletDB.prototype.wipe = co(function* wipe() { batch.del(layout.R); yield batch.write(); - yield this.writeGenesis(); }); /** @@ -542,21 +542,6 @@ WalletDB.prototype.getDepth = co(function* getDepth() { return depth + 1; }); -/** - * Get current block height. - * @private - * @returns {Promise} - */ - -WalletDB.prototype.getHeight = co(function* getHeight() { - var data = yield this.db.get(layout.R); - - if (!data) - return -1; - - return data.readUInt32LE(0, true); -}); - /** * Start batch. * @private @@ -1534,26 +1519,24 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { * @returns {Promise} */ -WalletDB.prototype.writeGenesis = co(function* writeGenesis() { - var tip = yield this.getTip(); +WalletDB.prototype.init = co(function* init() { + var state = yield this.getState(); + var tip; - if (tip) { - this.tip = tip; - this.height = tip.height; + if (state) { + this.state = state; return; } - yield this.forceTip(this.network.genesis); -}); + if (this.client) { + tip = yield this.client.getTip(); + assert(tip); + tip = HeaderRecord.fromEntry(tip); + } else { + tip = this.genesis; + } -/** - * Write the genesis block as the best hash. - * @returns {Promise} - */ - -WalletDB.prototype.forceTip = co(function* forceTip(entry) { - var tip = HeaderRecord.fromEntry(entry); - yield this.setTip(tip); + yield this.syncState(tip, true); }); /** @@ -1561,13 +1544,13 @@ WalletDB.prototype.forceTip = co(function* forceTip(entry) { * @returns {Promise} */ -WalletDB.prototype.getTip = co(function* getTip() { - var height = yield this.getHeight(); +WalletDB.prototype.getState = co(function* getState() { + var data = yield this.db.get(layout.R); - if (height === -1) + if (!data) return; - return yield this.getHeader(height); + return SyncState.fromRaw(data); }); /** @@ -1576,23 +1559,41 @@ WalletDB.prototype.getTip = co(function* getTip() { * @returns {Promise} */ -WalletDB.prototype.setTip = co(function* setTip(tip) { +WalletDB.prototype.syncState = co(function* syncState(tip, start) { var batch = this.db.batch(); - var height; + var state = this.state.clone(); + var height = this.state.tip.height; + var i, blocks; - batch.del(layout.c(tip.height + 1)); - batch.put(layout.c(tip.height), tip.toRaw()); - batch.put(layout.R, U32(tip.height)); + if (start) + state.start = tip; - height = tip.height - this.network.block.keepBlocks; + state.tip = tip; + // Blocks ahead of our new tip that we need to delete. + if (height !== -1) { + blocks = height - tip.height; + if (blocks > 0) { + blocks = Math.min(blocks, this.keepBlocks); + for (i = 0; i < blocks; i++) { + batch.del(layout.c(height)); + height--; + } + } + } + + // Prune old blocks. + height = tip.height - this.keepBlocks; if (height >= 0) batch.del(layout.c(height)); + // Save tip and state. + batch.put(layout.c(tip.height), tip.toRaw()); + batch.put(layout.R, state.toRaw()); + yield batch.write(); - this.tip = tip; - this.height = tip.height; + this.state = state; }); /** @@ -1697,24 +1698,34 @@ WalletDB.prototype.getTXMap = co(function* getTXMap(hash) { */ WalletDB.prototype.rollback = co(function* rollback(height) { - var tip; + var tip, start; - if (this.height > height) { - this.logger.info( - 'Rolling back %d blocks to height %d.', - this.height - height, height); - } + if (this.state.tip.height <= height) + return; - while (this.height > height) { - tip = yield this.getHeader(this.height); + this.logger.info( + 'Rolling back %d blocks to height %d.', + this.state.tip.height - height, height); - if (!tip) { - yield this.forceTip(this.network.genesis); - throw new Error('Wallet reorgd beyond safe height. Rescan required.'); + tip = yield this.getHeader(height); + + if (!tip) { + if (height >= this.state.start.height) { + yield this.revert(this.state.start.height); + yield this.syncState(this.state.start, true); + this.logger.warning( + 'Wallet rolled back to start block (%d).', + this.state.tip.height); + } else { + yield this.revert(0); + yield this.syncState(this.genesis, true); + this.logger.warning('Wallet rolled back to genesis block.'); } - - yield this._removeBlock(tip); + return; } + + yield this.revert(height); + yield this.syncState(tip); }); /** @@ -1724,6 +1735,7 @@ WalletDB.prototype.rollback = co(function* rollback(height) { */ WalletDB.prototype.revert = co(function* revert(height) { + var total = 0; var i, iter, item, block, tx; iter = this.db.iterator({ @@ -1740,6 +1752,7 @@ WalletDB.prototype.revert = co(function* revert(height) { try { block = BlockMapRecord.fromRaw(item.value); + total += block.txs.length; for (i = block.txs.length - 1; i >= 0; i--) { tx = block.txs[i]; @@ -1750,6 +1763,8 @@ WalletDB.prototype.revert = co(function* revert(height) { throw e; } } + + this.logger.info('Rolled back %d transactions.', total); }); /** @@ -1778,17 +1793,20 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) { var total = 0; var i, tip, tx; - if (entry.height <= this.height) { + if (entry.height < this.state.tip.height) { this.logger.warning('Wallet is connecting low blocks.'); return total; } - if (entry.height !== this.height + 1) + if (entry.height === this.state.tip.height) { + this.logger.warning('Wallet is connecting low blocks.'); + } else if (entry.height !== this.state.tip.height + 1) { throw new Error('Bad connection (height mismatch).'); + } tip = HeaderRecord.fromEntry(entry); - yield this.setTip(tip); + yield this.syncState(tip); if (this.options.useCheckpoints) { if (tip.height <= this.network.checkpoints.lastHeight) @@ -1835,12 +1853,12 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) { WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { var i, tx, tip, prev, block; - if (entry.height > this.height) { + if (entry.height > this.state.tip.height) { this.logger.warning('Wallet is disconnecting high blocks.'); return 0; } - if (entry.height !== this.height) + if (entry.height !== this.state.tip.height) throw new Error('Bad disconnection (height mismatch).'); tip = yield this.getHeader(entry.height); @@ -1856,7 +1874,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { block = yield this.getBlockMap(tip.height); if (!block) { - yield this.setTip(prev); + yield this.syncState(prev); return 0; } @@ -1865,7 +1883,7 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { yield this._unconfirm(tx, tip); } - yield this.setTip(prev); + yield this.syncState(prev); this.logger.warning('Disconnected block %s (tx=%d).', utils.revHex(tip.hash), block.txs.length); @@ -1956,14 +1974,14 @@ WalletDB.prototype._insert = co(function* insert(tx, block) { * @returns {Promise} */ -WalletDB.prototype._unconfirm = co(function* unconfirm(tx, block) { +WalletDB.prototype._unconfirm = co(function* unconfirm(tx) { var i, wid, wallet; for (i = 0; i < tx.wids.length; i++) { wid = tx.wids[i]; wallet = yield this.get(wid); assert(wallet); - yield wallet.unconfirm(tx.hash, block); + yield wallet.unconfirm(tx.hash); } }); diff --git a/test/chain-test.js b/test/chain-test.js index b0dff3e7..9c9e4a51 100644 --- a/test/chain-test.js +++ b/test/chain-test.js @@ -228,7 +228,7 @@ describe('Chain', function() { assert(wallet.account.changeDepth >= 7); assert.equal(walletdb.height, chain.height); - assert.equal(walletdb.tip.hash, chain.tip.hash); + assert.equal(walletdb.state.tip.hash, chain.tip.hash); txs = yield wallet.getHistory(); assert.equal(txs.length, 44);