From c147e5bdf38d1e812f8ce591ae2222cbc11356e1 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 15 Aug 2016 00:41:19 -0700 Subject: [PATCH] walletdb: better block management. --- lib/bcoin/chaindb.js | 4 +- lib/bcoin/fullnode.js | 2 +- lib/bcoin/http/server.js | 6 +- lib/bcoin/txdb.js | 11 +- lib/bcoin/utils.js | 4 +- lib/bcoin/wallet.js | 14 +- lib/bcoin/walletdb.js | 333 ++++++++++++++++++++++++++++++++------- 7 files changed, 297 insertions(+), 77 deletions(-) diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index cf04a80f..b1b4b489 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -1138,7 +1138,7 @@ ChainDB.prototype.scan = function scan(start, filter, iter, callback) { if (err) return callback(err); - if (!hash) { + if (hash == null) { self.logger.info('Finished scanning %d blocks.', total); return callback(); } @@ -1181,7 +1181,7 @@ ChainDB.prototype.scan = function scan(start, filter, iter, callback) { if (txs.length === 0) return self.getNextHash(hash, next); - iter(txs, entry, function(err) { + iter(entry, txs, function(err) { if (err) return next(err); self.getNextHash(hash, next); diff --git a/lib/bcoin/fullnode.js b/lib/bcoin/fullnode.js index d6f07e9f..cfad931a 100644 --- a/lib/bcoin/fullnode.js +++ b/lib/bcoin/fullnode.js @@ -298,7 +298,7 @@ Fullnode.prototype._open = function open(callback) { }, function(next) { if (self.options.noScan) { - self.walletdb.setTip(self.chain.tip.hash, 0, next); + self.walletdb.setTip(self.chain.tip.hash, self.chain.height, next); return next(); } // Always rescan to make sure we didn't miss anything: diff --git a/lib/bcoin/http/server.js b/lib/bcoin/http/server.js index ab9450f3..4e513a03 100644 --- a/lib/bcoin/http/server.js +++ b/lib/bcoin/http/server.js @@ -1294,7 +1294,7 @@ ClientSocket.prototype.watchChain = function watchChain() { txs = self.testBlock(block); if (txs) - self.emit('block tx', txs, entry.toJSON()); + self.emit('block tx', entry.toJSON(), txs); }); this.bind(this.chain, 'disconnect', function(entry, block) { @@ -1353,11 +1353,11 @@ ClientSocket.prototype.scan = function scan(start, callback) { start = utils.revHex(start); - this.chain.db.scan(start, this.filter, function(txs, entry, next) { + this.chain.db.scan(start, this.filter, function(entry, txs, next) { for (i = 0; i < txs.length; i++) txs[i] = txs[i].toJSON(); - self.emit('block tx', txs, entry.toJSON()); + self.emit('block tx', entry.toJSON(), txs); next(); }, function(err) { if (err) diff --git a/lib/bcoin/txdb.js b/lib/bcoin/txdb.js index 67e9719f..4b17c83d 100644 --- a/lib/bcoin/txdb.js +++ b/lib/bcoin/txdb.js @@ -11,16 +11,17 @@ * Database Layout: * t/[hash] -> extended tx * c/[hash]/[index] -> coin + * d/[hash]/[index] -> undo coin * s/[hash]/[index] -> spent by hash * o/[hash]/[index] -> orphan inputs * p/[hash] -> dummy (pending flag) * m/[time]/[hash] -> dummy (tx by time) * h/[height]/[hash] -> dummy (tx by height) - * T/[id]/[name]/[hash] -> dummy (tx by wallet id) - * P/[id]/[name]/[hash] -> dummy (pending tx by wallet/account id) - * M/[id]/[name]/[time]/[hash] -> dummy (tx by time + id/account) - * H/[id]/[name]/[height]/[hash] -> dummy (tx by height + id/account) - * C/[id]/[name]/[hash]/[index] -> dummy (coin by id/account) + * T/[account]/[hash] -> dummy (tx by account) + * P/[account]/[hash] -> dummy (pending tx by account) + * M/[account]/[time]/[hash] -> dummy (tx by time + account) + * H/[account]/[height]/[hash] -> dummy (tx by height + account) + * C/[account]/[hash]/[index] -> dummy (coin by account) */ var bcoin = require('./env'); diff --git a/lib/bcoin/utils.js b/lib/bcoin/utils.js index 49a77f9f..d2c730c3 100644 --- a/lib/bcoin/utils.js +++ b/lib/bcoin/utils.js @@ -2411,8 +2411,8 @@ utils.isAlpha = function isAlpha(key) { if (typeof key !== 'string') return false; // We allow /-~ (exclusive), 0-} (inclusive) - return key.length > 0 - && key.length <= 64 + return key.length >= 1 + && key.length <= 40 && /^[\u0030-\u007d]+$/.test(key); }; diff --git a/lib/bcoin/wallet.js b/lib/bcoin/wallet.js index 5b395255..6ed4fa8d 100644 --- a/lib/bcoin/wallet.js +++ b/lib/bcoin/wallet.js @@ -413,25 +413,31 @@ Wallet.prototype.unlock = function unlock(passphrase, timeout, callback) { /** * Generate the wallet ID if none was passed in. - * It is represented as `m/44` (public) hashed - * and converted to an address with a prefix + * It is represented as HASH160(m/44->public|magic) + * converted to an "address" with a prefix * of `0x03be04` (`WLT` in base58). * @private * @returns {Base58String} */ Wallet.prototype.getID = function getID() { - var key, p; + var key, p, hash; assert(this.master.key, 'Cannot derive id.'); key = this.master.key.derive(44); + p = new BufferWriter(); + p.writeBytes(key.publicKey); + p.writeU32(this.network.magic); + + hash = utils.hash160(p.render()); + p = new BufferWriter(); p.writeU8(0x03); p.writeU8(0xbe); p.writeU8(0x04); - p.writeBytes(utils.hash160(key.publicKey)); + p.writeBytes(hash); p.writeChecksum(); return utils.toBase58(p.render()); diff --git a/lib/bcoin/walletdb.js b/lib/bcoin/walletdb.js index c8f0bdf0..9aaa3583 100644 --- a/lib/bcoin/walletdb.js +++ b/lib/bcoin/walletdb.js @@ -14,6 +14,9 @@ * w/[id] -> wallet * a/[id]/[index] -> account * i/[id]/[name] -> account index + * R -> tip + * b/[hash] -> wallet block + * e/[hash] -> tx->wallet-id map */ var bcoin = require('./env'); @@ -928,7 +931,7 @@ WalletDB.prototype.rescan = function rescan(chaindb, callback) { self.logger.info('Scanning for %d addresses.', hashes.length); - chaindb.scan(self.tip, hashes, function(txs, block, next) { + chaindb.scan(self.height, hashes, function(block, txs, next) { self.addBlock(block, txs, next); }, callback); }); @@ -1064,17 +1067,17 @@ WalletDB.prototype.getTable = function getTable(addresses, callback) { WalletDB.prototype.writeGenesis = function writeGenesis(callback) { var self = this; - this.getTip(function(err, hash, height) { + this.getTip(function(err, block) { if (err) return callback(err); - if (hash) { - self.tip = hash; - self.height = height; + if (block) { + self.tip = block.hash; + self.height = block.height; return callback(); } - self.setTip(self.tip, self.height, callback); + self.setTip(self.network.genesis.hash, 0, callback); }); }; @@ -1085,54 +1088,121 @@ WalletDB.prototype.writeGenesis = function writeGenesis(callback) { WalletDB.prototype.getTip = function getTip(callback) { this.db.fetch('R', function(data) { - var p = new BufferReader(data); - return [p.readHash('hex'), p.readU32()]; - }, function(err, items) { - if (err) - return callback(err); - - if (!items) - return callback(null, null, -1); - - return callback(null, items[0], items[1]); - }); + return WalletBlock.fromTip(data); + }, callback); }; /** * Write the best block hash. * @param {Hash} hash + * @param {Number} height * @param {Function} callback */ WalletDB.prototype.setTip = function setTip(hash, height, callback) { var self = this; - var p = new BufferWriter(); - - p.writeHash(hash); - p.writeU32(height); - - this.db.put('R', p.render(), function(err) { + var block = new WalletBlock(hash, height); + this.db.put('R', block.toTip(), function(err) { if (err) return callback(err); - self.tip = hash; - self.height = height; + self.tip = block.hash; + self.height = block.height; return callback(); }); }; /** - * Add a block's transactions and write the new best hash. - * @param {Block} block + * Connect a block. + * @param {WalletBlock} block * @param {Function} callback */ -WalletDB.prototype.addBlock = function addBlock(block, txs, callback, force) { - var self = this; - var unlock; +WalletDB.prototype.writeBlock = function writeBlock(block, matches, callback) { + var batch = this.db.batch(); + var i, hash, wallets; - unlock = this.locker.lock(addBlock, [block, txs, callback], force); + batch.put('R', block.toTip()); + + if (block.hashes.length > 0) { + batch.put('b/' + block.hash, block.toRaw()); + + for (i = 0; i < block.hashes.length; i++) { + hash = block.hashes[i]; + wallets = matches[i]; + batch.put('e/' + hash, serializeWallets(wallets)); + } + } + + batch.write(function(err) { + if (err) + return callback(err); + + self.tip = block.hash; + self.height = block.height; + + return callback(); + }); +}; + +/** + * Disconnect a block. + * @param {WalletBlock} block + * @param {Function} callback + */ + +WalletDB.prototype.unwriteBlock = function unwriteBlock(block, callback) { + var batch = this.db.batch(); + var prev = new WalletBlock(block.prevBlock, block.height - 1); + + batch.put('R', prev.toTip()); + batch.del('b/' + block.hash); + + batch.write(function(err) { + if (err) + return callback(err); + + self.tip = prev.hash; + self.height = prev.height; + + return callback(); + }); +}; + +/** + * Get a wallet block (with hashes). + * @param {Hash} hash + * @param {Function} callback + */ + +WalletDB.prototype.getBlock = function getBlock(hash, callback) { + this.db.fetch('b/' + hash, function(err, data) { + return WalletBlock.fromRaw(hash, data); + }, callback); +}; + +/** + * Get a TX->Wallet map. + * @param {Hash} hash + * @param {Function} callback + */ + +WalletDB.prototype.getWalletsByTX = function getWalletsByTX(hash, callback) { + this.db.fetch('e/' + hash, parseWallets, callback); +}; + +/** + * Add a block's transactions and write the new best hash. + * @param {ChainEntry} entry + * @param {Function} callback + */ + +WalletDB.prototype.addBlock = function addBlock(entry, txs, callback, force) { + var self = this; + var i, block, matches, hash, unlock; + + unlock = this.locker.lock(addBlock, [entry, txs, callback], force); if (!unlock) return; @@ -1140,66 +1210,98 @@ WalletDB.prototype.addBlock = function addBlock(block, txs, callback, force) { callback = utils.wrap(callback, unlock); if (this.options.useCheckpoints) { - if (block.height < this.network.checkpoints.lastHeight) - return this.setTip(block.hash, block.height, callback); + if (entry.height <= this.network.checkpoints.lastHeight) + return this.setTip(entry.hash, entry.height, callback); } - if (!Array.isArray(txs)) - txs = [txs]; + block = WalletBlock.fromEntry(entry); + matches = []; + // NOTE: Atomicity doesn't matter here. If we crash + // during this loop, the automatic rescan will get + // the database back into the correct state. utils.forEachSerial(txs, function(tx, next) { - self.addTX(tx, next, true); + self.addTX(tx, function(err, wallets) { + if (err) + return next(err); + + if (!wallets) + return next(); + + hash = tx.hash('hex'); + block.hashes.push(hash); + matches.push(wallets); + + next(); + }, true); }, function(err) { if (err) return callback(err); - self.setTip(block.hash, block.height, callback); + self.writeBlock(block, matches, callback); }); }; /** * Unconfirm a block's transactions * and write the new best hash (SPV version). - * @param {Block} block + * @param {ChainEntry} entry * @param {Function} callback */ -WalletDB.prototype.removeBlock = function removeBlock(block, callback, force) { +WalletDB.prototype.removeBlock = function removeBlock(entry, callback, force) { var self = this; var unlock; - unlock = this.locker.lock(removeBlock, [block, callback], force); + unlock = this.locker.lock(removeBlock, [entry, callback], force); if (!unlock) return; callback = utils.wrap(callback, unlock); - this.getWallets(function(err, wallets) { + // Note: + // If we crash during a reorg, there's not much to do. + // Reorgs cannot be rescanned. The database will be + // in an odd state, with some txs being confirmed + // when they shouldn't be. That being said, this + // should eventually resolve itself when a new block + // comes in. + this.getBlock(entry.hash, function(err, block) { if (err) return callback(err); - utils.forEachSerial(wallets, function(id, next) { - self.get(id, function(err, wallet) { - if (err) - return next(err); + // Not a saved block, but we + // still want to reset the tip. + if (!block) + block = WalletBlock.fromEntry(entry); - if (!wallet) - return next(); - - wallet.tx.getHeightHashes(block.height, function(err, hashes) { - if (err) - return callback(err); - - utils.forEachSerial(hashes, function(hash, next) { - wallet.tx.unconfirm(hash, next); - }, next); - }); - }); - }, function(err) { + // Unwrite the tip as fast as we can. + self.unwriteBlock(block, function(err) { if (err) return callback(err); - self.setTip(block.prevBlock, block.height - 1, callback); + + utils.forEachSerial(block.hashes, function(hash, next) { + self.getWalletsByTX(hash, function(err, wallets) { + if (err) + return next(err); + + if (!wallets) + return next(); + + utils.forEachSerial(wallets, function(id, next) { + self.get(id, function(err, wallet) { + if (err) + return next(err); + + if (!wallet) + return next(); + + wallet.tx.unconfirm(hash, next); + }); + }, callback); + }); + }); }); }); }; @@ -1214,6 +1316,11 @@ WalletDB.prototype.removeBlock = function removeBlock(block, callback, force) { WalletDB.prototype.addTX = function addTX(tx, callback, force) { var self = this; + + // Note: + // Atomicity doesn't matter here. If we crash, + // the automatic rescan will get the database + // back in the correct state. this.mapWallets(tx, function(err, wallets) { if (err) return callback(err); @@ -1242,7 +1349,11 @@ WalletDB.prototype.addTX = function addTX(tx, callback, force) { wallet.handleTX(info, next); }); }); - }, callback); + }, function(err) { + if (err) + return callback(err); + return callback(null, wallets); + }); }); }; @@ -1630,6 +1741,108 @@ function serializePaths(out) { return p.render(); } +function serializeWallets(wallets) { + var p = new BufferWriter(); + var i, info; + + for (i = 0; i < wallets.length; i++) { + info = wallets[i]; + p.writeVarBytes(info.id, 'ascii'); + } + + return p.render(); +} + +function parseWallets(data) { + var p = new BufferReader(data); + var wallets = []; + while (p.left()) + wallets.push(p.readVarString('ascii')); + return wallets; +} + +function WalletBlock(hash, height) { + if (!(this instanceof WalletBlock)) + return new WalletBlock(hash, height); + + this.hash = hash || constants.NULL_HASH; + this.height = height != null ? height : -1; + this.prevBlock = constants.NULL_HASH; + this.hashes = []; +} + +WalletBlock.prototype.fromEntry = function fromEntry(entry) { + this.hash = entry.hash; + this.height = entry.height; + this.prevBlock = entry.prevBlock; + return this; +}; + +WalletBlock.prototype.fromJSON = function fromJSON(json) { + this.hash = utils.revHex(json.hash); + this.height = entry.height; + if (entry.prevBlock) + this.prevBlock = utils.revHex(entry.prevBlock); + return this; +}; + +WalletBlock.prototype.fromRaw = function fromRaw(hash, data) { + var p = new BufferReader(data); + this.hash = hash; + this.height = p.readU32(); + while (p.left()) + this.hashes.push(p.readHash('hex')); + return this; +}; + +WalletBlock.prototype.fromTip = function fromTip(data) { + this.hash = p.readHash('hex'); + this.height = p.readU32(); + return this; +}; + +WalletBlock.fromEntry = function fromEntry(entry) { + return new WalletBlock().fromEntry(entry); +}; + +WalletBlock.fromJSON = function fromJSON(json) { + return new WalletBlock().fromJSON(json); +}; + +WalletBlock.fromRaw = function fromRaw(hash, data) { + return new WalletBlock().fromTip(hash, data); +}; + +WalletBlock.fromTip = function fromTip(data) { + return new WalletBlock().fromTip(data); +}; + +WalletBlock.prototype.toTip = function toTip(data) { + var p = new BufferWriter(); + p.writeHash(this.hash); + p.writeU32(this.height); + return p.render(); +}; + +WalletBlock.prototype.toRaw = function toRaw(data) { + var p = new BufferWriter(); + var i; + + p.writeU32(this.height); + + for (i = 0; i < this.hashes.length; i++) + p.writeHash(this.hashes[i]); + + return p.render(); +}; + +WalletBlock.prototype.toJSON = function toJSON() { + return { + hash: utils.revHex(this.hash), + height: this.height + }; +}; + function ReadLock(parent) { if (!(this instanceof ReadLock)) return new ReadLock(parent);