From 45464c412f8febe1b7486f80312643ecc7d94dcc Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 24 Oct 2016 10:12:35 -0700 Subject: [PATCH] walletdb: refactor block handling. --- lib/chain/chaindb.js | 7 +- lib/node/config.js | 1 + lib/node/fullnode.js | 1 + lib/node/spvnode.js | 1 + lib/wallet/browser.js | 6 + lib/wallet/{walletblock.js => records.js} | 241 ++++++++------ lib/wallet/txdb.js | 67 ++-- lib/wallet/walletdb.js | 388 ++++++++++++---------- test/wallet-test.js | 1 - 9 files changed, 398 insertions(+), 315 deletions(-) rename lib/wallet/{walletblock.js => records.js} (56%) diff --git a/lib/chain/chaindb.js b/lib/chain/chaindb.js index f161c1cb..11038d39 100644 --- a/lib/chain/chaindb.js +++ b/lib/chain/chaindb.js @@ -1121,8 +1121,11 @@ ChainDB.prototype.scan = co(function* scan(start, filter, iter) { block = yield this.getBlock(entry.hash); total++; - if (!block) - throw new Error('Block not found.'); + if (!block) { + if (!this.options.spv && !this.options.prune) + throw new Error('Block not found.'); + continue; + } this.logger.info( 'Scanning block %s (%d).', diff --git a/lib/node/config.js b/lib/node/config.js index d2c03c25..bc575290 100644 --- a/lib/node/config.js +++ b/lib/node/config.js @@ -203,6 +203,7 @@ config.parseData = function parseData(data, prefix, dirname) { options.noAuth = bool(data.noauth); // Wallet + options.noScan = bool(data.noscan); options.wipeNoReally = bool(data.wipenoreally); options.data = data; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index 797532c5..9f37eccb 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -149,6 +149,7 @@ function Fullnode(options) { witness: this.options.witness, useCheckpoints: this.options.useCheckpoints, maxFiles: this.options.maxFiles, + noScan: this.options.noScan, wipeNoReally: this.options.wipeNoReally, resolution: false, verify: false diff --git a/lib/node/spvnode.js b/lib/node/spvnode.js index a53256c5..44ddee51 100644 --- a/lib/node/spvnode.js +++ b/lib/node/spvnode.js @@ -87,6 +87,7 @@ function SPVNode(options) { location: this.location('walletdb'), witness: this.options.witness, maxFiles: this.options.maxFiles, + noScan: this.options.noScan, wipeNoReally: this.options.wipeNoReally, resolution: true, verify: true diff --git a/lib/wallet/browser.js b/lib/wallet/browser.js index 369821b6..9697375a 100644 --- a/lib/wallet/browser.js +++ b/lib/wallet/browser.js @@ -45,6 +45,12 @@ layout.walletdb = { return [+key.slice(1, 11), key.slice(11)]; }, R: 'R', + c: function c(height) { + return 'b' + pad32(height); + }, + cc: function cc(key) { + return +key.slice(1); + }, b: function b(height) { return 'b' + pad32(height); }, diff --git a/lib/wallet/walletblock.js b/lib/wallet/records.js similarity index 56% rename from lib/wallet/walletblock.js rename to lib/wallet/records.js index 1a38e851..5a577d72 100644 --- a/lib/wallet/walletblock.js +++ b/lib/wallet/records.js @@ -1,6 +1,5 @@ /*! - * walletdb.js - storage for wallets - * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * records.js - walletdb records * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License). * https://github.com/bcoin-org/bcoin */ @@ -14,30 +13,28 @@ var BufferReader = require('../utils/reader'); var BufferWriter = require('../utils/writer'); /** - * Wallet Block + * Wallet Tip * @constructor * @param {Hash} hash * @param {Number} height */ -function WalletBlock(hash, height, ts) { - if (!(this instanceof WalletBlock)) - return new WalletBlock(hash, height, ts); +function HeaderRecord(hash, height, ts) { + if (!(this instanceof HeaderRecord)) + return new HeaderRecord(hash, height, ts); this.hash = hash || constants.NULL_HASH; this.height = height != null ? height : -1; this.ts = ts || 0; - this.txs = []; - this.index = {}; } /** * Clone the block. - * @returns {WalletBlock} + * @returns {HeaderRecord} */ -WalletBlock.prototype.clone = function clone() { - return new WalletBlock(this.hash, this.height, this.ts); +HeaderRecord.prototype.clone = function clone() { + return new HeaderRecord(this.hash, this.height, this.ts); }; /** @@ -46,7 +43,7 @@ WalletBlock.prototype.clone = function clone() { * @param {ChainEntry} entry */ -WalletBlock.prototype.fromEntry = function fromEntry(entry) { +HeaderRecord.prototype.fromEntry = function fromEntry(entry) { this.hash = entry.hash; this.height = entry.height; this.ts = entry.ts; @@ -59,48 +56,20 @@ WalletBlock.prototype.fromEntry = function fromEntry(entry) { * @param {Object} json */ -WalletBlock.prototype.fromJSON = function fromJSON(json) { +HeaderRecord.prototype.fromJSON = function fromJSON(json) { this.hash = utils.revHex(json.hash); this.height = json.height; this.ts = json.ts; return this; }; -/** - * Instantiate wallet block from serialized data. - * @private - * @param {Hash} hash - * @param {Buffer} data - */ - -WalletBlock.prototype.fromRaw = function fromRaw(data) { - var p = new BufferReader(data); - var i, hash, tx, count; - - this.hash = p.readHash('hex'); - this.height = p.readU32(); - this.ts = p.readU32(); - - while (p.left()) { - hash = p.readHash('hex'); - tx = new TXHash(hash); - count = p.readVarint(); - for (i = 0; i < count; i++) - tx.wids.push(p.readU32()); - this.txs.push(tx); - this.index[tx.hash] = tx; - } - - return this; -}; - /** * Instantiate wallet block from serialized tip data. * @private * @param {Buffer} data */ -WalletBlock.prototype.fromTip = function fromTip(data) { +HeaderRecord.prototype.fromRaw = function fromRaw(data) { var p = new BufferReader(data); this.hash = p.readHash('hex'); this.height = p.readU32(); @@ -111,42 +80,32 @@ WalletBlock.prototype.fromTip = function fromTip(data) { /** * Instantiate wallet block from chain entry. * @param {ChainEntry} entry - * @returns {WalletBlock} + * @returns {HeaderRecord} */ -WalletBlock.fromEntry = function fromEntry(entry) { - return new WalletBlock().fromEntry(entry); +HeaderRecord.fromEntry = function fromEntry(entry) { + return new HeaderRecord().fromEntry(entry); }; /** * Instantiate wallet block from json object. * @param {Object} json - * @returns {WalletBlock} + * @returns {HeaderRecord} */ -WalletBlock.fromJSON = function fromJSON(json) { - return new WalletBlock().fromJSON(json); +HeaderRecord.fromJSON = function fromJSON(json) { + return new HeaderRecord().fromJSON(json); }; /** * Instantiate wallet block from serialized data. * @param {Hash} hash * @param {Buffer} data - * @returns {WalletBlock} + * @returns {HeaderRecord} */ -WalletBlock.fromRaw = function fromRaw(data) { - return new WalletBlock().fromRaw(data); -}; - -/** - * Instantiate wallet block from serialized tip data. - * @private - * @param {Buffer} data - */ - -WalletBlock.fromTip = function fromTip(data) { - return new WalletBlock().fromTip(data); +HeaderRecord.fromRaw = function fromRaw(data) { + return new HeaderRecord().fromRaw(data); }; /** @@ -154,7 +113,7 @@ WalletBlock.fromTip = function fromTip(data) { * @returns {Buffer} */ -WalletBlock.prototype.toTip = function toTip(writer) { +HeaderRecord.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(writer); p.writeHash(this.hash); @@ -167,20 +126,80 @@ WalletBlock.prototype.toTip = function toTip(writer) { return p; }; +/** + * Convert the block to a more json-friendly object. + * @returns {Object} + */ + +HeaderRecord.prototype.toJSON = function toJSON() { + return { + hash: utils.revHex(this.hash), + height: this.height, + ts: this.ts + }; +}; + +/** + * Wallet Block + * @constructor + * @param {Hash} hash + * @param {Number} height + */ + +function BlockMapRecord(height) { + if (!(this instanceof BlockMapRecord)) + return new BlockMapRecord(height); + + this.height = height != null ? height : -1; + this.txs = []; + this.index = {}; +} + +/** + * Instantiate wallet block from serialized data. + * @private + * @param {Hash} hash + * @param {Buffer} data + */ + +BlockMapRecord.prototype.fromRaw = function fromRaw(data) { + var p = new BufferReader(data); + var i, hash, tx, count; + + while (p.left()) { + hash = p.readHash('hex'); + tx = new TXMapRecord(hash); + count = p.readVarint(); + for (i = 0; i < count; i++) + tx.wids.push(p.readU32()); + this.txs.push(tx); + this.index[tx.hash] = tx; + } + + return this; +}; + +/** + * Instantiate wallet block from serialized data. + * @param {Hash} hash + * @param {Buffer} data + * @returns {BlockMapRecord} + */ + +BlockMapRecord.fromRaw = function fromRaw(height, data) { + return new BlockMapRecord(height).fromRaw(data); +}; + /** * Serialize the wallet block as a block. * Contains matching transaction hashes. * @returns {Buffer} */ -WalletBlock.prototype.toRaw = function toRaw(writer) { +BlockMapRecord.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(writer); var i, j, tx; - p.writeHash(this.hash); - p.writeU32(this.height); - p.writeU32(this.ts); - for (i = 0; i < this.txs.length; i++) { tx = this.txs[i]; p.writeHash(tx.hash); @@ -202,11 +221,11 @@ WalletBlock.prototype.toRaw = function toRaw(writer) { * @returns {Boolean} */ -WalletBlock.prototype.add = function add(hash, wid) { +BlockMapRecord.prototype.add = function add(hash, wid) { var tx = this.index[hash]; if (!tx) { - tx = new TXHash(hash); + tx = new TXMapRecord(hash); tx.wids.push(wid); this.txs.push(tx); this.index[tx.hash] = tx; @@ -223,7 +242,7 @@ WalletBlock.prototype.add = function add(hash, wid) { * @returns {Boolean} */ -WalletBlock.prototype.remove = function remove(hash, wid) { +BlockMapRecord.prototype.remove = function remove(hash, wid) { var tx = this.index[hash]; var result; @@ -242,49 +261,69 @@ WalletBlock.prototype.remove = function remove(hash, wid) { return true; }; -/** - * Convert the block to a more json-friendly object. - * @returns {Object} - */ - -WalletBlock.prototype.toJSON = function toJSON() { - return { - hash: utils.revHex(this.hash), - height: this.height - }; -}; - /** * TX Hash * @constructor */ -function TXHash(hash, wids) { +function TXMapRecord(hash, wids) { this.hash = hash || constants.NULL_HASH; this.wids = wids || []; - this.id = TXHash.id++; + this.id = TXMapRecord.id++; } -TXHash.id = 0; +TXMapRecord.id = 0; -TXHash.prototype.add = function add(wid) { +TXMapRecord.prototype.add = function add(wid) { return utils.binaryInsert(this.wids, wid, cmp, true) !== -1; }; -TXHash.prototype.remove = function remove(wid) { +TXMapRecord.prototype.remove = function remove(wid) { return utils.binaryRemove(this.wids, wid, cmp); }; -TXHash.prototype.toRaw = function toRaw(writer) { - return serializeWallets(this.wids, writer); +TXMapRecord.prototype.toRaw = function toRaw() { + return serializeWallets(this.wids); }; -TXHash.prototype.fromRaw = function fromRaw(data) { - return parseWallets(data); +TXMapRecord.prototype.fromRaw = function fromRaw(data) { + this.wids = parseWallets(data); + return this; }; -TXHash.fromRaw = function fromRaw(hash, data) { - return new TXHash(hash).fromRaw(data); +TXMapRecord.fromRaw = function fromRaw(hash, data) { + return new TXMapRecord(hash).fromRaw(data); +}; + +/** + * Path Record + * @constructor + */ + +function PathMapRecord(hash, wids) { + this.hash = hash || constants.NULL_HASH; + this.wids = wids || []; +} + +PathMapRecord.prototype.add = function add(wid) { + return utils.binaryInsert(this.wids, wid, cmp, true) !== -1; +}; + +PathMapRecord.prototype.remove = function remove(wid) { + return utils.binaryRemove(this.wids, wid, cmp); +}; + +PathMapRecord.prototype.toRaw = function toRaw() { + return serializeWallets(this.wids); +}; + +PathMapRecord.prototype.fromRaw = function fromRaw(data) { + this.wids = parseWallets(data); + return this; +}; + +PathMapRecord.fromRaw = function fromRaw(hash, data) { + return new PathMapRecord(hash).fromRaw(data); }; /* @@ -309,8 +348,8 @@ function parseWallets(data) { return wids; } -function serializeWallets(wids, writer) { - var p = new BufferWriter(writer); +function serializeWallets(wids) { + var p = new BufferWriter(); var i, wid; for (i = 0; i < wids.length; i++) { @@ -318,18 +357,16 @@ function serializeWallets(wids, writer) { p.writeU32(wid); } - if (!writer) - p = p.render(); - - return p; + return p.render(); } /* * Expose */ -exports = WalletBlock; -exports.WalletBlock = WalletBlock; -exports.TXHash = TXHash; +exports.HeaderRecord = HeaderRecord; +exports.BlockMapRecord = BlockMapRecord; +exports.TXMapRecord = TXMapRecord; +exports.PathMapRecord = PathMapRecord; module.exports = exports; diff --git a/lib/wallet/txdb.js b/lib/wallet/txdb.js index 0a4fa213..b6adf01a 100644 --- a/lib/wallet/txdb.js +++ b/lib/wallet/txdb.js @@ -17,7 +17,9 @@ var BufferWriter = require('../utils/writer'); var TX = require('../primitives/tx'); var Coin = require('../primitives/coin'); var Outpoint = require('../primitives/outpoint'); -var WalletBlock = require('./walletblock'); +var records = require('./records'); +var BlockMapRecord = records.BlockMapRecord; +var TXMapRecord = records.TXMapRecord; var DUMMY = new Buffer([0]); /* @@ -875,18 +877,15 @@ TXDB.prototype.isSpent = function isSpent(hash, index) { TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) { var hash = tx.hash('hex'); - var wids = yield this.walletdb.getWalletsByTX(hash); - var result; + var map = yield this.walletdb.getTXMap(hash); - if (!wids) - wids = []; + if (!map) + map = new TXMapRecord(hash); - result = utils.binaryInsert(wids, this.wallet.wid, cmp, true); - - if (result === -1) + if (!map.add(this.wallet.wid)) return; - this.walletdb.writeTX(this.wallet, hash, wids); + this.walletdb.writeTXMap(this.wallet, hash, map); }); /** @@ -897,31 +896,29 @@ TXDB.prototype.addTXRecord = co(function* addTXRecord(tx) { TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) { var hash = tx.hash('hex'); - var wids = yield this.walletdb.getWalletsByTX(hash); - var result, block; + var map = yield this.walletdb.getTXMap(hash); + var block; - if (!wids) + if (!map) return; - result = utils.binaryRemove(wids, this.wallet.wid, cmp); - - if (!result) + if (!map.remove(this.wallet.wid)) return; if (tx.height !== -1) { - block = yield this.walletdb.getBlock(tx.height); + block = yield this.walletdb.getBlockMap(tx.height); assert(block); if (block.remove(hash, this.wallet.wid)) - this.walletdb.writeBlock(this.wallet, block); + this.walletdb.writeBlockMap(this.wallet, tx.height, block); } - if (wids.length === 0) { - this.walletdb.unwriteTX(this.wallet, hash); + if (map.wids.length === 0) { + this.walletdb.unwriteTXMap(this.wallet, hash); return; } - this.walletdb.writeTX(this.wallet, hash, wids); + this.walletdb.writeTXMap(this.wallet, hash, map); }); /** @@ -933,15 +930,15 @@ TXDB.prototype.removeTXRecord = co(function* removeTXRecord(tx) { TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, height) { var hash = tx.hash('hex'); - var block = yield this.walletdb.getBlock(height); + var block = yield this.walletdb.getBlockMap(height); if (!block) - block = new WalletBlock(tx.block, tx.height, tx.ts); + block = new BlockMapRecord(height); if (!block.add(hash, this.wallet.wid)) return; - this.walletdb.writeBlock(this.wallet, block); + this.walletdb.writeBlockMap(this.wallet, height, block); }); /** @@ -953,7 +950,7 @@ TXDB.prototype.addBlockRecord = co(function* addBlockRecord(tx, height) { TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) { var hash = tx.hash('hex'); - var block = yield this.walletdb.getBlock(height); + var block = yield this.walletdb.getBlockMap(height); if (!block) return; @@ -962,11 +959,11 @@ TXDB.prototype.removeBlockRecord = co(function* removeBlockRecord(tx, height) { return; if (block.txs.length === 0) { - this.walletdb.unwriteBlock(this.wallet, block); + this.walletdb.unwriteBlockMap(this.wallet, height); return; } - this.walletdb.writeBlock(this.wallet, block); + this.walletdb.writeBlockMap(this.wallet, height, block); }); /** @@ -1062,8 +1059,7 @@ TXDB.prototype._add = co(function* add(tx, block) { TXDB.prototype.insert = co(function* insert(tx, block) { var hash = tx.hash('hex'); - var height = this.walletdb.height + 1; - var details = new Details(this, tx, height); + var details = new Details(this, tx); var updated = false; var i, input, output, coin; var prevout, credit, path, account; @@ -1271,8 +1267,7 @@ TXDB.prototype.confirm = co(function* confirm(hash, block) { TXDB.prototype._confirm = co(function* confirm(tx, block) { var hash = tx.hash('hex'); - var height = this.walletdb.height + 1; - var details = new Details(this, tx, height); + var details = new Details(this, tx); var i, account, output, coin, input, prevout; var path, credit, credits; @@ -1413,7 +1408,7 @@ TXDB.prototype.remove = co(function* remove(hash) { TXDB.prototype.erase = co(function* erase(tx) { var hash = tx.hash('hex'); - var details = new Details(this, tx, this.walletdb.height); + var details = new Details(this, tx); var i, path, account, credits; var input, output, coin, credit; @@ -1602,7 +1597,7 @@ TXDB.prototype._unconfirm = co(function* unconfirm(hash, block) { TXDB.prototype.disconnect = co(function* disconnect(tx, block) { var hash = tx.hash('hex'); - var details = new Details(this, tx, this.walletdb.height - 1); + var details = new Details(this, tx); var height = tx.height; var i, account, output, coin, credits; var input, path, credit; @@ -2531,7 +2526,7 @@ TXDB.prototype.toDetails = co(function* toDetails(txs) { */ TXDB.prototype._toDetails = co(function* _toDetails(tx) { - var details = new Details(this, tx, this.walletdb.height); + var details = new Details(this, tx); var coins = yield this.fillHistory(tx); var i, coin, path, output; @@ -3080,16 +3075,16 @@ Credit.fromTX = function fromTX(tx, index) { * @param {TX} tx */ -function Details(txdb, tx, height) { +function Details(txdb, tx) { if (!(this instanceof Details)) - return new Details(txdb, tx, height); + return new Details(txdb, tx); this.wallet = txdb.wallet; this.network = this.wallet.network; this.wid = this.wallet.wid; this.id = this.wallet.id; - this.chainHeight = height; + this.chainHeight = txdb.walletdb.height; this.hash = tx.hash('hex'); this.tx = tx; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index b3934ecf..6bfbaaea 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -25,7 +25,11 @@ var ldb = require('../db/ldb'); var Bloom = require('../utils/bloom'); var Logger = require('../node/logger'); var TX = require('../primitives/tx'); -var WalletBlock = require('./walletblock'); +var records = require('./records'); +var BlockMapRecord = records.BlockMapRecord; +var HeaderRecord = records.HeaderRecord; +var PathMapRecord = records.PathMapRecord; +var TXMapRecord = records.TXMapRecord; var TXDB = require('./txdb'); var U32 = utils.U32; @@ -103,6 +107,15 @@ var layout = { return [key.readUInt32BE(1, true), key.toString('ascii', 5)]; }, R: new Buffer([0x52]), + c: function c(height) { + var key = new Buffer(5); + key[0] = 0x63; + key.writeUInt32BE(height, 1, true); + return key; + }, + cc: function cc(key) { + return key.readUInt32BE(1, true); + }, b: function b(height) { var key = new Buffer(5); key[0] = 0x62; @@ -268,7 +281,7 @@ WalletDB.prototype.backup = function backup(path) { WalletDB.prototype.wipe = co(function* wipe() { var batch = this.db.batch(); var dummy = new Buffer(0); - var i, keys, key; + var i, keys, key, gte, lte; this.logger.warning('Wiping txdb...'); this.logger.warning('I hope you know what you\'re doing.'); @@ -283,9 +296,37 @@ WalletDB.prototype.wipe = co(function* wipe() { batch.del(key); } + gte = new Buffer(33); + key.fill(0); + key[0] = 0x62; + + lte = new Buffer(33); + key.fill(255); + key[0] = 0x62; + keys = yield this.db.keys({ - gte: layout.b(constants.NULL_HASH), - lte: layout.b(constants.HIGH_HASH) + gte: gte, + lte: lte + }); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + batch.del(key); + } + + keys = yield this.db.keys({ + gte: layout.b(0), + lte: layout.b(0xffffffff) + }); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + batch.del(key); + } + + keys = yield this.db.keys({ + gte: layout.c(0), + lte: layout.c(0xffffffff) }); for (i = 0; i < keys.length; i++) { @@ -900,23 +941,23 @@ WalletDB.prototype.hasAccount = function hasAccount(wid, index) { * @returns {Promise} - Returns Number. */ -WalletDB.prototype.getWalletsByHash = co(function* getWalletsByHash(hash) { - var wids = this.pathMapCache.get(hash); +WalletDB.prototype.getPathMap = co(function* getPathMap(hash) { + var map = this.pathMapCache.get(hash); var data; - if (wids) - return wids; + if (map) + return map; data = yield this.db.get(layout.p(hash)); if (!data) return; - wids = parseWallets(data); + map = PathMapRecord.fromRaw(hash, data); - this.pathMapCache.set(hash, wids); + this.pathMapCache.set(hash, map); - return wids; + return map; }); /** @@ -946,27 +987,24 @@ WalletDB.prototype.savePath = co(function* savePath(wallet, path) { var wid = wallet.wid; var hash = path.hash; var batch = this.batch(wallet); - var wids, result; + var map, result; this.addFilter(hash); this.emit('path', path); - wids = yield this.getWalletsByHash(hash); + map = yield this.getPathMap(hash); - if (!wids) - wids = []; + if (!map) + map = new PathMapRecord(hash); - // Keep these motherfuckers sorted. - result = utils.binaryInsert(wids, wid, cmp, true); - - if (result === -1) + if (!map.add(wid)) return; - this.pathMapCache.set(hash, wids); + this.pathMapCache.set(hash, map); wallet.pathCache.push(hash, path); - batch.put(layout.p(hash), serializeWallets(wids)); + batch.put(layout.p(hash), map.toRaw()); batch.put(layout.P(wid, hash), path.toRaw()); }); @@ -1161,7 +1199,18 @@ WalletDB.prototype.rescan = co(function* rescan(chaindb, height) { WalletDB.prototype._rescan = co(function* rescan(chaindb, height) { var self = this; - var hashes, blocks; + var tip, hashes, blocks; + + if (this.options.noScan) { + tip = yield chaindb.getTip(); + + if (!tip) + throw new Error('Could not get chain tip.'); + + yield this.forceTip(tip); + + return; + } if (height == null) { height = this.height - 36; @@ -1174,10 +1223,8 @@ WalletDB.prototype._rescan = co(function* rescan(chaindb, height) { if (blocks < 0) throw new Error('Cannot rescan future blocks.'); - if (this.prune) { - if (blocks > this.network.block.keepBlocks) - throw new Error('Cannot roll back beyond keepBlocks.'); - } + if (blocks > this.network.block.keepBlocks) + throw new Error('Cannot roll back beyond keepBlocks.'); yield this.rollback(height); @@ -1316,7 +1363,7 @@ WalletDB.prototype.resend = co(function* resend() { WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) { var result = []; var hashes = tx.getHashes('hex'); - var i, j, hash, wids; + var i, j, hash, map; for (i = 0; i < hashes.length; i++) { hash = hashes[i]; @@ -1324,13 +1371,13 @@ WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) { if (!this.testFilter(hash)) continue; - wids = yield this.getWalletsByHash(hash); + map = yield this.getPathMap(hash); - if (!wids) + if (!map) continue; - for (j = 0; j < wids.length; j++) - utils.binaryInsert(result, wids[j], cmp, true); + for (j = 0; j < map.wids.length; j++) + utils.binaryInsert(result, map.wids[j], cmp, true); } if (result.length === 0) @@ -1346,7 +1393,7 @@ WalletDB.prototype.getWalletsByHashes = co(function* getWalletsByHashes(tx) { */ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { - var i, j, result, hashes, input, prevout, hash, wids; + var i, j, result, hashes, input, prevout, hash, map; if (this.options.resolution) return yield this.getWalletsByHashes(tx); @@ -1361,13 +1408,13 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { if (!this.testFilter(prevout.hash)) continue; - wids = yield this.getWalletsByTX(prevout.hash); + map = yield this.getTXMap(prevout.hash); - if (!wids) + if (!map) continue; - for (j = 0; j < wids.length; j++) - utils.binaryInsert(result, wids[j], cmp, true); + for (j = 0; j < map.wids.length; j++) + utils.binaryInsert(result, map.wids[j], cmp, true); } for (i = 0; i < hashes.length; i++) { @@ -1376,13 +1423,13 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { if (!this.testFilter(hash)) continue; - wids = yield this.getWalletsByHash(hash); + map = yield this.getPathMap(hash); - if (!wids) + if (!map) continue; - for (j = 0; j < wids.length; j++) - utils.binaryInsert(result, wids[j], cmp, true); + for (j = 0; j < map.wids.length; j++) + utils.binaryInsert(result, map.wids[j], cmp, true); } if (result.length === 0) @@ -1397,16 +1444,29 @@ WalletDB.prototype.getWalletsByInsert = co(function* getWalletsByInsert(tx) { */ WalletDB.prototype.writeGenesis = co(function* writeGenesis() { - var block = yield this.getTip(); - var genesis = this.network.genesis; + var tip = yield this.getTip(); - if (block) { - this.tip = block; - this.height = block.height; + if (tip) { + this.tip = tip; + this.height = tip.height; return; } - yield this.setTip(genesis.hash, 0, genesis.ts); + yield this.forceTip(this.network.genesis); +}); + +/** + * Write the genesis block as the best hash. + * @returns {Promise} + */ + +WalletDB.prototype.forceTip = co(function* forceTip(entry) { + var tip = HeaderRecord.fromEntry(entry); + + if (entry === this.network.genesis) + tip.height = 0; + + yield this.setTip(tip); }); /** @@ -1416,104 +1476,82 @@ WalletDB.prototype.writeGenesis = co(function* writeGenesis() { WalletDB.prototype.getTip = co(function* getTip() { var height = yield this.getHeight(); - var data = yield this.db.get(layout.b(height)); - - if (!data) + if (height === -1) return; - - return WalletBlock.fromTip(data); + return yield this.getHeader(height); }); -/** - * Write the tip immediately. - * @param {Hash} hash - * @param {Number} height - * @returns {Promise} - */ - -WalletDB.prototype.setTip = function setTip(hash, height, ts) { - var block = new WalletBlock(hash, height, ts); - return this.setBlock(block); -}; - /** * Write the connecting block immediately. - * @param {Hash} hash - * @param {Number} height + * @param {HeaderRecord} tip * @returns {Promise} */ -WalletDB.prototype.setBlock = co(function* setBlock(block) { +WalletDB.prototype.setTip = co(function* setTip(tip) { var batch = this.db.batch(); var height; - batch.del(layout.b(block.height + 1)); - batch.put(layout.b(block.height), block.toRaw()); - batch.put(layout.R, U32(block.height)); + batch.del(layout.c(tip.height + 1)); + batch.put(layout.c(tip.height), tip.toRaw()); + batch.put(layout.R, U32(tip.height)); - if (this.prune) { - height = block.height - this.network.block.keepBlocks; - if (height > this.network.block.pruneAfterHeight) - batch.del(layout.b(height)); - } + height = tip.height - this.network.block.keepBlocks; + + if (height >= 0) + batch.del(layout.c(height)); yield batch.write(); - this.tip = block; - this.height = block.height; + this.tip = tip; + this.height = tip.height; }); /** * Connect a block. - * @param {WalletBlock} block + * @param {Wallet} wallet + * @param {BlockMapRecord} block * @returns {Promise} */ -WalletDB.prototype.writeBlock = function writeBlock(wallet, block) { +WalletDB.prototype.writeBlockMap = function writeBlockMap(wallet, height, block) { var batch = this.batch(wallet); - var height = block.height - this.network.block.keepBlocks; - - batch.put(layout.b(block.height), block.toRaw()); - batch.put(layout.R, U32(block.height)); - - if (this.prune) { - height = block.height - this.network.block.keepBlocks; - if (height > this.network.block.pruneAfterHeight) - batch.del(layout.b(height)); - } + batch.put(layout.b(height), block.toRaw()); }; /** * Connect a block. - * @param {WalletBlock} block + * @param {Wallet} wallet + * @param {BlockMapRecord} block * @returns {Promise} */ -WalletDB.prototype.unwriteBlock = function unwriteBlock(wallet, block) { +WalletDB.prototype.unwriteBlockMap = function unwriteBlockMap(wallet, height) { var batch = this.batch(wallet); - batch.del(layout.b(block.height)); - batch.put(layout.R, U32(block.height - 1)); + batch.del(layout.b(height)); }; /** * Connect a transaction. - * @param {WalletBlock} block + * @param {Wallet} wallet + * @param {Hash} hash + * @param {TXMapRecord} map * @returns {Promise} */ -WalletDB.prototype.writeTX = function writeTX(wallet, hash, wids) { +WalletDB.prototype.writeTXMap = function writeTXMap(wallet, hash, map) { var batch = this.batch(wallet); - batch.put(layout.e(hash), serializeWallets(wids)); + batch.put(layout.e(hash), map.toRaw()); this.addFilter(hash); }; /** * Connect a transaction. - * @param {WalletBlock} block + * @param {Wallet} wallet + * @param {Hash} hash * @returns {Promise} */ -WalletDB.prototype.unwriteTX = function unwriteTX(wallet, hash) { +WalletDB.prototype.unwriteTXMap = function unwriteTXMap(wallet, hash) { var batch = this.batch(wallet); batch.del(layout.e(hash)); }; @@ -1524,13 +1562,28 @@ WalletDB.prototype.unwriteTX = function unwriteTX(wallet, hash) { * @returns {Promise} */ -WalletDB.prototype.getBlock = co(function* getBlock(height) { +WalletDB.prototype.getBlockMap = co(function* getBlockMap(height) { var data = yield this.db.get(layout.b(height)); if (!data) return; - return WalletBlock.fromRaw(data); + return BlockMapRecord.fromRaw(height, data); +}); + +/** + * Get a wallet block (with hashes). + * @param {Hash} hash + * @returns {Promise} + */ + +WalletDB.prototype.getHeader = co(function* getHeader(height) { + var data = yield this.db.get(layout.c(height)); + + if (!data) + return; + + return HeaderRecord.fromRaw(data); }); /** @@ -1539,13 +1592,13 @@ WalletDB.prototype.getBlock = co(function* getBlock(height) { * @returns {Promise} */ -WalletDB.prototype.getWalletsByTX = co(function* getWalletsByTX(hash) { +WalletDB.prototype.getTXMap = co(function* getTXMap(hash) { var data = yield this.db.get(layout.e(hash)); if (!data) return; - return parseWallets(data); + return TXMapRecord.fromRaw(hash, data); }); /** @@ -1555,7 +1608,7 @@ WalletDB.prototype.getWalletsByTX = co(function* getWalletsByTX(hash) { */ WalletDB.prototype.rollback = co(function* rollback(height) { - var block; + var tip; if (this.height > height) { this.logger.info( @@ -1564,20 +1617,14 @@ WalletDB.prototype.rollback = co(function* rollback(height) { } while (this.height > height) { - block = yield this.getBlock(this.height); + tip = yield this.getHeader(this.height); - if (!block) { - if (this.prune) { - height = this.network.blocks.pruneAfterHeight; - block = yield this.getBlock(height); - assert(block); - yield this.setBlock(block); - throw new Error('Wallet reorgd beyond safe height. Rescan required.'); - } - assert(false, 'Database corruption.'); + if (!tip) { + yield this.forceTip(this.network.genesis); + throw new Error('Wallet reorgd beyond safe height. Rescan required.'); } - yield this._removeBlock(block); + yield this._removeBlock(tip); } }); @@ -1605,42 +1652,36 @@ WalletDB.prototype.addBlock = co(function* addBlock(entry, txs) { WalletDB.prototype._addBlock = co(function* addBlock(entry, txs) { var total = 0; - var i, block, tx; + var i, tip, tx; - // Special case for the guh-naysis block. - if (entry.hash === this.network.genesis.hash) - return; + if (entry.height <= this.height) { + this.logger.warning('Connecting low blocks.'); + return total; + } if (entry.height !== this.height + 1) throw new Error('Bad connection (height mismatch).'); - if (this.options.useCheckpoints) { - if (entry.height <= this.network.checkpoints.lastHeight) { - block = WalletBlock.fromEntry(entry); - yield this.setBlock(block); - return 0; - } - } + tip = HeaderRecord.fromEntry(entry); - block = WalletBlock.fromEntry(entry); + yield this.setTip(tip); + + if (this.options.useCheckpoints) { + if (tip.height <= this.network.checkpoints.lastHeight) + return 0; + } for (i = 0; i < txs.length; i++) { tx = txs[i]; - if (yield this._addTX(tx, block)) + if (yield this._addTX(tx, tip)) total++; } - if (total === 0) { - yield this.setBlock(block); - return total; + if (total > 0) { + this.logger.info('Connected block %s (tx=%d).', + utils.revHex(tip.hash), total); } - this.height = block.height; - this.tip = block; - - this.logger.info('Connected block %s (tx=%d).', - utils.revHex(block.hash), total); - return total; }); @@ -1668,31 +1709,39 @@ WalletDB.prototype.removeBlock = co(function* removeBlock(entry) { */ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { - var block = yield this.getBlock(entry.height); - var prev = yield this.getBlock(entry.height - 1); - var i, tx; + var i, tx, tip, prev, block; - if (!block) - throw new Error('Bad disconnection (block not found).'); - - if (!prev) - throw new Error('Bad disconnection (no previous block).'); + if (entry.height > this.height) { + this.logger.warning('Disconnecting high blocks.'); + return 0; + } if (entry.height !== this.height) throw new Error('Bad disconnection (height mismatch).'); - if (block.txs.length === 0) { - yield this.setBlock(prev); - return block.txs.length; + tip = yield this.getHeader(entry.height); + + if (!tip) + throw new Error('Bad disconnection (block not found).'); + + prev = yield this.getHeader(entry.height - 1); + + if (!prev) + throw new Error('Bad disconnection (no previous block).'); + + block = yield this.getBlockMap(tip.height); + + if (!block) { + yield this.setTip(prev); + return 0; } for (i = block.txs.length - 1; i >= 0; i--) { tx = block.txs[i]; - yield this._unconfirmTX(tx, block); + yield this._unconfirmTX(tx, tip); } - this.height = prev.height; - this.tip = prev; + yield this.setTip(prev); this.logger.warning('Disconnected block %s (tx=%d).', utils.revHex(block.hash), block.txs.length); @@ -1710,7 +1759,21 @@ WalletDB.prototype._removeBlock = co(function* removeBlock(entry) { WalletDB.prototype.addTX = co(function* addTX(tx) { var unlock = yield this.txLock.lock(); + var entry; + try { + if (tx.height !== -1) { + entry = yield this.getHeader(tx.height); + + if (!entry) + throw new Error('Inserting unconfirmed transaction.'); + + if (tx.block !== entry.hash) + throw new Error('Inserting unconfirmed transaction.'); + + this.logger.warning('Retroactively inserting confirmed transaction.'); + } + return yield this._addTX(tx); } finally { unlock(); @@ -1721,6 +1784,7 @@ WalletDB.prototype.addTX = co(function* addTX(tx) { * Add a transaction to the database without a lock. * @private * @param {TX} tx + * @param {HeaderRecord} block * @returns {Promise} */ @@ -1764,6 +1828,7 @@ WalletDB.prototype._addTX = co(function* addTX(tx, block) { * relevant wallets without a lock. * @private * @param {TXHash} hash + * @param {HeaderRecord} block * @returns {Promise} */ @@ -1816,31 +1881,6 @@ WalletDB.prototype._zap = co(function* zap(age) { * Helpers */ -function parseWallets(data) { - var p = new BufferReader(data); - var wids = []; - - while (p.left()) - wids.push(p.readU32()); - - return wids; -} - -function serializeWallets(wids, writer) { - var p = new BufferWriter(writer); - var i, wid; - - for (i = 0; i < wids.length; i++) { - wid = wids[i]; - p.writeU32(wid); - } - - if (!writer) - p = p.render(); - - return p; -} - function cmp(a, b) { return a - b; } diff --git a/test/wallet-test.js b/test/wallet-test.js index 65704b06..6633ac97 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -10,7 +10,6 @@ var assert = require('assert'); var scriptTypes = constants.scriptTypes; var co = require('../lib/utils/co'); var cob = co.cob; -var WalletBlock = require('../lib/wallet/walletblock'); var KEY1 = 'xprv9s21ZrQH143K3Aj6xQBymM31Zb4BVc7wxqfUhMZrzewdDVCt' + 'qUP9iWfcHgJofs25xbaUpCps9GDXj83NiWvQCAkWQhVj5J4CorfnpKX94AZ';