From b74c32cc4b7099e2eeffa8f7da4d41e22cb80211 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Fri, 28 Oct 2016 22:34:01 -0700 Subject: [PATCH] walletdb: sync state object refactor. --- lib/wallet/browser.js | 6 ++ lib/wallet/records.js | 28 ++++---- lib/wallet/txdb.js | 157 +++++++++++++++++++++++++++++++++++++++++ lib/wallet/wallet.js | 15 ---- lib/wallet/walletdb.js | 115 +++++++++++++++++------------- 5 files changed, 244 insertions(+), 77 deletions(-) diff --git a/lib/wallet/browser.js b/lib/wallet/browser.js index 99add81b..04bd52be 100644 --- a/lib/wallet/browser.js +++ b/lib/wallet/browser.js @@ -188,6 +188,12 @@ layout.txdb = { }, r: function r(hash) { return this.ha('r', hash); + }, + b: function b(height) { + return 'b' + pad32(height); + }, + bb: function bb(key) { + return +key.slice(1); } }; diff --git a/lib/wallet/records.js b/lib/wallet/records.js index 148e5e56..69d58897 100644 --- a/lib/wallet/records.js +++ b/lib/wallet/records.js @@ -19,9 +19,9 @@ var BufferWriter = require('../utils/writer'); * @param {Number} height */ -function SyncState() { - if (!(this instanceof SyncState)) - return new SyncState(); +function ChainState() { + if (!(this instanceof ChainState)) + return new ChainState(); this.start = new HeaderRecord(); this.tip = new HeaderRecord(); @@ -29,13 +29,13 @@ function SyncState() { /** * Clone the block. - * @returns {SyncState} + * @returns {ChainState} */ -SyncState.prototype.clone = function clone() { - var state = new SyncState(); - state.start = this.start.clone(); - state.tip = this.tip.clone(); +ChainState.prototype.clone = function clone() { + var state = new ChainState(); + state.start = this.start; + state.tip = this.tip; return state; }; @@ -45,7 +45,7 @@ SyncState.prototype.clone = function clone() { * @param {Buffer} data */ -SyncState.prototype.fromRaw = function fromRaw(data) { +ChainState.prototype.fromRaw = function fromRaw(data) { var p = new BufferReader(data); this.start.fromRaw(p); this.tip.fromRaw(p); @@ -56,11 +56,11 @@ SyncState.prototype.fromRaw = function fromRaw(data) { * Instantiate wallet block from serialized data. * @param {Hash} hash * @param {Buffer} data - * @returns {SyncState} + * @returns {ChainState} */ -SyncState.fromRaw = function fromRaw(data) { - return new SyncState().fromRaw(data); +ChainState.fromRaw = function fromRaw(data) { + return new ChainState().fromRaw(data); }; /** @@ -68,7 +68,7 @@ SyncState.fromRaw = function fromRaw(data) { * @returns {Buffer} */ -SyncState.prototype.toRaw = function toRaw(writer) { +ChainState.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(writer); this.start.toRaw(p); @@ -432,7 +432,7 @@ function serializeWallets(wids) { * Expose */ -exports.SyncState = SyncState; +exports.ChainState = ChainState; exports.HeaderRecord = HeaderRecord; exports.BlockMapRecord = BlockMapRecord; exports.TXMapRecord = TXMapRecord; diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index e7df794e..107d4656 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -190,6 +190,15 @@ var layout = { }, r: function r(hash) { return layout.ha(0x72, hash); + }, + b: function b(height) { + var key = new Buffer(5); + key[0] = 0x62; + key.writeUInt32BE(height, 1, true); + return key; + }, + bb: function bb(key) { + return key.readUInt32BE(1, true); } }; @@ -957,6 +966,71 @@ TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) { this.walletdb.writeBlockMap(this.wallet, height, block); }); +/** + * Get block record. + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.getBlock = co(function* getBlock(height) { + var data = yield this.db.get(layout.b(height)); + if (!data) + return; + return BlockRecord.fromRaw(data); +}); + +/** + * Append to the global block record. + * @param {TX} tx + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.addBlock = co(function* addBlock(tx, entry) { + var hash = tx.hash('hex'); + var block = yield this.getBlock(entry.height); + + if (!block) + block = new BlockRecord(entry.hash, entry.height, entry.ts); + + if (block.hashes.indexOf(hash) !== -1) + return; + + block.hashes.push(hash); + + this.put(layout.b(entry.height), block.toRaw()); +}); + +/** + * Remove from the global block record. + * @param {TX} + * @param {Number} height + * @returns {Promise} + */ + +TXDB.prototype.removeBlock = co(function* removeBlock(tx, height) { + var hash = tx.hash('hex'); + var block = yield this.getBlock(height); + var index; + + if (!block) + return; + + index = block.hashes.indexOf(hash); + + if (index === -1) + return; + + block.hashes.splice(index, 1); + + if (block.hashes.length === 0) { + this.del(layout.b(height)); + return; + } + + this.put(layout.b(height), block.toRaw()); +}); + /** * Add transaction, potentially runs * `confirm()` and `removeConflicts()`. @@ -3268,6 +3342,89 @@ DetailsMember.prototype.toJSON = function toJSON(network) { }; }; +/** + * Block Record + * @constructor + * @param {Hash} hash + * @param {Number} height + * @param {Number} ts + */ + +function BlockRecord(hash, height, ts) { + if (!(this instanceof BlockRecord)) + return new BlockRecord(hash, height, ts); + + this.hash = hash || constants.NULL_HASH; + this.height = height != null ? height : -1; + this.ts = ts || 0; + this.hashes = []; +} + +/** + * Instantiate wallet block from serialized tip data. + * @private + * @param {Buffer} data + */ + +BlockRecord.prototype.fromRaw = function fromRaw(data) { + var p = new BufferReader(data); + this.hash = p.readHash('hex'); + this.height = p.readU32(); + this.ts = p.readU32(); + + while (p.left()) + this.hashes.push(p.readHash('hex')); + + return this; +}; + +/** + * Instantiate wallet block from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {BlockRecord} + */ + +BlockRecord.fromRaw = function fromRaw(data) { + return new BlockRecord().fromRaw(data); +}; + +/** + * Serialize the wallet block as a tip (hash and height). + * @returns {Buffer} + */ + +BlockRecord.prototype.toRaw = function toRaw(writer) { + var p = new BufferWriter(writer); + var i; + + p.writeHash(this.hash); + p.writeU32(this.height); + p.writeU32(this.ts); + + for (i = 0; i < this.hashes.length; i++) + p.writeHash(this.hashes[i]); + + if (!writer) + p = p.render(); + + return p; +}; + +/** + * Convert the block to a more json-friendly object. + * @returns {Object} + */ + +BlockRecord.prototype.toJSON = function toJSON() { + return { + hash: utils.revHex(this.hash), + height: this.height, + ts: this.ts, + hashes: this.hashes.map(utils.revHex) + }; +}; + /* * Helpers */ diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index 5e21ea0e..098c2e8f 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -1974,21 +1974,6 @@ Wallet.prototype.unconfirm = co(function* unconfirm(hash) { } }); -/** - * Confirm a wallet transcation. - * @param {Hash} hash - * @returns {Promise} - */ - -Wallet.prototype.confirm = co(function* confirm(hash, block) { - var unlock = yield this.writeLock.lock(); - try { - return yield this.txdb.confirm(hash, block); - } finally { - unlock(); - } -}); - /** * Remove a wallet transaction. * @param {Hash} hash diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index f7a39f55..5625114e 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -24,7 +24,7 @@ var Bloom = require('../utils/bloom'); var Logger = require('../node/logger'); var TX = require('../primitives/tx'); var records = require('./records'); -var SyncState = records.SyncState; +var ChainState = records.ChainState; var BlockMapRecord = records.BlockMapRecord; var HeaderRecord = records.HeaderRecord; var PathMapRecord = records.PathMapRecord; @@ -166,7 +166,7 @@ function WalletDB(options) { this.logger = options.logger || Logger.global; this.client = options.client; - this.state = new SyncState(); + this.state = new ChainState(); this.depth = 0; this.wallets = {}; this.genesis = HeaderRecord.fromEntry(this.network.genesis); @@ -233,8 +233,10 @@ WalletDB.prototype._open = co(function* open() { yield this.resend(); this.logger.info( - 'WalletDB loaded (depth=%d, height=%d).', - this.depth, this.state.tip.height); + 'WalletDB loaded (depth=%d, height=%d, start=%d).', + this.depth, + this.state.tip.height, + this.state.start.height); }); /** @@ -265,7 +267,7 @@ WalletDB.prototype._close = co(function* close() { WalletDB.prototype.watch = co(function* watch() { var hashes = yield this.getFilterHashes(); - this.logger.info('Adding %d hashes to filter.', hashes.length); + this.logger.info('Adding %d hashes to WalletDB filter.', hashes.length); this.addFilter(hashes); }); @@ -363,14 +365,16 @@ WalletDB.prototype.scan = co(function* scan(height) { if (!this.client) return; - assert(utils.isUInt32(height), 'Must pass in a height.'); + assert(utils.isUInt32(height), 'WDB: Must pass in a height.'); if (height > this.state.tip.height) - throw new Error('Cannot rescan future blocks.'); + throw new Error('WDB: Cannot rescan future blocks.'); yield this.rollback(height); - this.logger.info('Scanning %d blocks.', this.state.tip.height - height); + this.logger.info( + 'WalletDB is scanning %d blocks.', + this.state.tip.height - height + 1); yield this.client.scan(this.state.tip.hash, this.filter, function(block, txs) { return self._addBlock(block, txs); @@ -440,7 +444,7 @@ WalletDB.prototype.wipe = co(function* wipe() { var dummy = new Buffer(0); var i, keys, key, gte, lte; - this.logger.warning('Wiping txdb...'); + this.logger.warning('Wiping WalletDB TXDB...'); this.logger.warning('I hope you know what you\'re doing.'); keys = yield this.db.keys({ @@ -549,7 +553,7 @@ WalletDB.prototype.getDepth = co(function* getDepth() { */ WalletDB.prototype.start = function start(wallet) { - assert(!wallet.current, 'Batch already started.'); + assert(!wallet.current, 'WDB: Batch already started.'); wallet.current = this.db.batch(); wallet.accountCache.start(); wallet.pathCache.start(); @@ -591,7 +595,7 @@ WalletDB.prototype.clear = function clear(wallet) { */ WalletDB.prototype.batch = function batch(wallet) { - assert(wallet.current, 'Batch does not exist.'); + assert(wallet.current, 'WDB: Batch does not exist.'); return wallet.current; }; @@ -808,10 +812,10 @@ WalletDB.prototype._rename = co(function* _rename(wallet, id) { var i, paths, path, batch; if (!utils.isName(id)) - throw new Error('Bad wallet ID.'); + throw new Error('WDB: Bad wallet ID.'); if (yield this.has(id)) - throw new Error('ID not available.'); + throw new Error('WDB: ID not available.'); batch = this.start(wallet); batch.del(layout.l(old)); @@ -861,13 +865,13 @@ WalletDB.prototype.auth = co(function* auth(wid, token) { if (typeof token === 'string') { if (!utils.isHex256(token)) - throw new Error('Authentication error.'); + throw new Error('WDB: Authentication error.'); token = new Buffer(token, 'hex'); } // Compare in constant time: if (!crypto.ccmp(token, wallet.token)) - throw new Error('Authentication error.'); + throw new Error('WDB: Authentication error.'); return wallet; }); @@ -903,7 +907,7 @@ WalletDB.prototype._create = co(function* create(options) { var wallet; if (exists) - throw new Error('Wallet already exists.'); + throw new Error('WDB: Wallet already exists.'); wallet = Wallet.fromOptions(this, options); wallet.wid = this.depth++; @@ -912,7 +916,7 @@ WalletDB.prototype._create = co(function* create(options) { this.register(wallet); - this.logger.info('Created wallet %s.', wallet.id); + this.logger.info('Created wallet %s in WalletDB.', wallet.id); return wallet; }); @@ -1412,7 +1416,7 @@ WalletDB.prototype.resend = co(function* resend() { var i, key, data, tx; if (keys.length > 0) - this.logger.info('Rebroadcasting %d transactions.', keys.length); + this.logger.info('Rebroadcasting %d WalletDB transactions.', keys.length); for (i = 0; i < keys.length; i++) { key = keys[i]; @@ -1536,6 +1540,10 @@ WalletDB.prototype.init = co(function* init() { tip = this.genesis; } + this.logger.info( + 'Initializing WalletDB chain state at %s (%d).', + utils.revHex(tip.hash), tip.height); + yield this.syncState(tip, true); }); @@ -1550,7 +1558,7 @@ WalletDB.prototype.getState = co(function* getState() { if (!data) return; - return SyncState.fromRaw(data); + return ChainState.fromRaw(data); }); /** @@ -1698,29 +1706,35 @@ WalletDB.prototype.getTXMap = co(function* getTXMap(hash) { */ WalletDB.prototype.rollback = co(function* rollback(height) { - var tip, start; + var tip, blocks; if (this.state.tip.height <= height) return; this.logger.info( - 'Rolling back %d blocks to height %d.', + 'Rolling back %d WalletDB blocks to height %d.', this.state.tip.height - height, height); tip = yield this.getHeader(height); if (!tip) { + blocks = this.state.tip.height - height; + + if (blocks < this.keepBlocks) + throw new Error('WDB: Block not found for rollback.'); + if (height >= this.state.start.height) { yield this.revert(this.state.start.height); yield this.syncState(this.state.start, true); this.logger.warning( - 'Wallet rolled back to start block (%d).', + 'WalletDB rolled back to start block (%d).', this.state.tip.height); } else { yield this.revert(0); yield this.syncState(this.genesis, true); - this.logger.warning('Wallet rolled back to genesis block.'); + this.logger.warning('WalletDB rolled back to genesis block.'); } + return; } @@ -1729,7 +1743,7 @@ WalletDB.prototype.rollback = co(function* rollback(height) { }); /** - * Sync with chain height. + * Revert TXDB to an older state. * @param {Number} height * @returns {Promise} */ @@ -1764,7 +1778,7 @@ WalletDB.prototype.revert = co(function* revert(height) { } } - this.logger.info('Rolled back %d transactions.', total); + this.logger.info('Rolled back %d WalletDB transactions.', total); }); /** @@ -1794,14 +1808,22 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) { var i, tip, tx; if (entry.height < this.state.tip.height) { - this.logger.warning('Wallet is connecting low blocks.'); + this.logger.warning( + 'WalletDB is connecting low blocks (%d).', + entry.height); return total; } if (entry.height === this.state.tip.height) { - this.logger.warning('Wallet is connecting low blocks.'); + // We let blocks of the same height + // through specifically for rescans: + // we always want to rescan the last + // block since the state may have + // updated before the block was fully + // processed (in the case of a crash). + this.logger.warning('Duplicate connection for %d.', entry.height); } else if (entry.height !== this.state.tip.height + 1) { - throw new Error('Bad connection (height mismatch).'); + throw new Error('WDB: Bad connection (height mismatch).'); } tip = HeaderRecord.fromEntry(entry); @@ -1820,7 +1842,7 @@ WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) { } if (total > 0) { - this.logger.info('Connected block %s (tx=%d).', + this.logger.info('Connected WalletDB block %s (tx=%d).', utils.revHex(tip.hash), total); } @@ -1851,27 +1873,24 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) { */ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { - var i, tx, tip, prev, block; + var i, tx, prev, block; if (entry.height > this.state.tip.height) { - this.logger.warning('Wallet is disconnecting high blocks.'); + this.logger.warning( + 'WalletDB is disconnecting high blocks (%d).', + entry.height); return 0; } if (entry.height !== this.state.tip.height) - throw new Error('Bad disconnection (height mismatch).'); - - tip = yield this.getHeader(entry.height); - - if (!tip) - throw new Error('Bad disconnection (block not found).'); + throw new Error('WDB: Bad disconnection (height mismatch).'); prev = yield this.getHeader(entry.height - 1); if (!prev) - throw new Error('Bad disconnection (no previous block).'); + throw new Error('WDB: Bad disconnection (no previous block).'); - block = yield this.getBlockMap(tip.height); + block = yield this.getBlockMap(entry.height); if (!block) { yield this.syncState(prev); @@ -1880,13 +1899,13 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { for (i = block.txs.length - 1; i >= 0; i--) { tx = block.txs[i]; - yield this._unconfirm(tx, tip); + yield this._unconfirm(tx); } yield this.syncState(prev); - this.logger.warning('Disconnected block %s (tx=%d).', - utils.revHex(tip.hash), block.txs.length); + this.logger.warning('Disconnected wallet block %s (tx=%d).', + utils.revHex(entry.hash), block.txs.length); return block.txs.length; }); @@ -1908,12 +1927,12 @@ WalletDB.prototype.addTX = co(function* addTX(tx) { entry = yield this.getHeader(tx.height); if (!entry) - throw new Error('Inserting unconfirmed transaction.'); + throw new Error('WDB: Inserting unconfirmed transaction.'); if (tx.block !== entry.hash) - throw new Error('Inserting unconfirmed transaction.'); + throw new Error('WDB: Inserting unconfirmed transaction.'); - this.logger.warning('Retroactively inserting confirmed transaction.'); + this.logger.warning('WalletDB is inserting confirmed transaction.'); } return yield this._insert(tx); @@ -1934,7 +1953,7 @@ WalletDB.prototype._insert = co(function* insert(tx, block) { var result = false; var i, wids, wid, wallet; - assert(!tx.mutable, 'Cannot add mutable TX to wallet.'); + assert(!tx.mutable, 'WDB: Cannot add mutable TX.'); wids = yield this.getWalletsByInsert(tx); @@ -1942,7 +1961,7 @@ WalletDB.prototype._insert = co(function* insert(tx, block) { return; this.logger.info( - 'Incoming transaction for %d wallets (%s).', + 'Incoming transaction for %d wallets in WalletDB (%s).', wids.length, tx.rhash); for (i = 0; i < wids.length; i++) { @@ -1953,7 +1972,7 @@ WalletDB.prototype._insert = co(function* insert(tx, block) { if (yield wallet.add(tx, block)) { this.logger.info( - 'Added transaction to wallet: %s (%d).', + 'Added transaction to wallet in WalletDB: %s (%d).', wallet.id, wid); result = true; }