From debe2e4ec7384bffc3a7613fca6271c64e632f63 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Sat, 22 Oct 2016 14:17:27 -0700 Subject: [PATCH] walletdb: pruning. --- lib/chain/chaindb.js | 34 +++++++++++++++++++-------------- lib/protocol/networks.js | 13 ++++++++++++- lib/wallet/walletdb.js | 41 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 70 insertions(+), 18 deletions(-) diff --git a/lib/chain/chaindb.js b/lib/chain/chaindb.js index b54e5e9e..ec929024 100644 --- a/lib/chain/chaindb.js +++ b/lib/chain/chaindb.js @@ -147,13 +147,10 @@ if (utils.isBrowser) * @param {Boolean?} options.prune - Whether to prune the chain. * @param {Boolean?} options.spv - SPV-mode, will not save block * data, only entries. - * @param {Number?} [options.keepBlocks=288] - Number of - * blocks to keep when pruning. * @param {String?} options.name - Database name * @param {String?} options.location - Database location * @param {String?} options.db - Database backend name * @property {Boolean} prune - * @property {Number} keepBlocks * @emits ChainDB#open * @emits ChainDB#error */ @@ -229,8 +226,13 @@ ChainDB.prototype._open = co(function* open() { state = yield this.getState(); options = yield this.getOptions(); - if (options) + if (options) { + // Verify the options haven't changed. this.options.verify(options); + this.options = options; + } else { + yield this.db.put(layout.O, this.options.toRaw()); + } if (state) { // Grab the chainstate if we have one. @@ -1575,7 +1577,7 @@ ChainDB.prototype.pruneBlock = co(function* pruneBlock(block) { if (!this.options.prune && !this.options.spv) return; - height = block.height - this.options.keepBlocks; + height = block.height - this.network.block.keepBlocks; if (height <= this.network.block.pruneAfterHeight) return; @@ -1603,7 +1605,6 @@ function ChainOptions(options) { this.prune = false; this.indexTX = false; this.indexAddress = false; - this.keepBlocks = 288; if (options) this.fromOptions(options); @@ -1615,14 +1616,24 @@ ChainOptions.prototype.fromOptions = function fromOptions(options) { this.spv = options.spv; } + if (options.witness != null) { + assert(typeof options.witness === 'boolean'); + this.witness = options.witness; + } + if (options.prune != null) { assert(typeof options.prune === 'boolean'); this.prune = options.prune; } - if (options.keepBlocks != null) { - assert(utils.isUInt32(options.keepBlocks)); - this.keepBlocks = options.keepBlocks; + if (options.indexTX != null) { + assert(typeof options.indexTX === 'boolean'); + this.indexTX = options.indexTX; + } + + if (options.indexAddress != null) { + assert(typeof options.indexAddress === 'boolean'); + this.indexAddress = options.indexAddress; } return this; @@ -1662,9 +1673,6 @@ ChainOptions.prototype.verify = function verify(options) { if (!this.indexAddress && options.indexAddress) throw new Error('Cannot retroactively disable address indexing.'); - - if (this.keepBlocks !== options.keepBlocks) - throw new Error('Cannot change keepBlocks option retroactively.'); }; ChainOptions.prototype.toRaw = function toRaw() { @@ -1687,7 +1695,6 @@ ChainOptions.prototype.toRaw = function toRaw() { flags |= 1 << 4; p.writeU32(flags); - p.writeU32(this.keepBlocks); p.writeU32(0); return p.render(); @@ -1701,7 +1708,6 @@ ChainOptions.prototype.fromRaw = function fromRaw(data) { this.prune = (flags & 4) !== 0; this.indexTX = (flags & 8) !== 0; this.indexAddress = (flags & 16) !== 0; - this.keepBlocks = p.readU32(); return this; }; diff --git a/lib/protocol/networks.js b/lib/protocol/networks.js index addd2434..502d4deb 100644 --- a/lib/protocol/networks.js +++ b/lib/protocol/networks.js @@ -269,7 +269,13 @@ main.block = { * Safe height to start pruning. */ - pruneAfterHeight: 100000, + pruneAfterHeight: 1000, + + /** + * Safe number of blocks to keep. + */ + + keepBlocks: 288, /** * Age used for the time delta to @@ -530,6 +536,7 @@ testnet.block = { bip66height: 330776, bip66hash: '82a14b9e5ea81d4832b8e2cd3c2a6092b5a3853285a8995ec4c8042100000000', pruneAfterHeight: 1000, + keepBlocks: 10000, // maxTipAge: 0x7fffffff maxTipAge: 24 * 60 * 60, slowHeight: 750000 @@ -674,6 +681,7 @@ regtest.block = { bip66height: 1251, bip66hash: null, pruneAfterHeight: 1000, + keepBlocks: 10000, maxTipAge: 24 * 60 * 60, slowHeight: 0x7fffffff }; @@ -817,6 +825,7 @@ segnet3.block = { bip141height: 8, bip141hash: '1c2a2898cebca152f872fa71b756903711ad778c7d63ba6b73c140f800000000', pruneAfterHeight: 1000, + keepBlocks: 10000, // maxTipAge: 0x7fffffff, maxTipAge: 24 * 60 * 60, slowHeight: 0x7fffffff @@ -934,6 +943,7 @@ segnet4.block = { bip66height: 8, bip66hash: '6c48386dc7c460defabb5640e28b6510a5f238cdbe6756c2976a7e0913000000', pruneAfterHeight: 1000, + keepBlocks: 10000, // maxTipAge: 0x7fffffff, maxTipAge: 24 * 60 * 60, slowHeight: 0x7fffffff @@ -1073,6 +1083,7 @@ simnet.block = { bip66height: 0, bip66hash: 'f67ad7695d9b662a72ff3d8edbbb2de0bfa67b13974bb9910d116d5cbd863e68', pruneAfterHeight: 1000, + keepBlocks: 10000, maxTipAge: 0x7fffffff, slowHeight: 0 }; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 4fe60557..e0043dcf 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -148,6 +148,7 @@ function WalletDB(options) { AsyncObject.call(this); this.options = options; + this.prune = options.prune !== false; this.network = Network.get(options.network); this.fees = options.fees; this.logger = options.logger || Logger.global; @@ -1142,7 +1143,7 @@ WalletDB.prototype.rescan = co(function* rescan(chaindb, height) { WalletDB.prototype._rescan = co(function* rescan(chaindb, height) { var self = this; - var hashes; + var hashes, blocks; if (height == null) { height = this.height - 36; @@ -1150,7 +1151,15 @@ WalletDB.prototype._rescan = co(function* rescan(chaindb, height) { height = 0; } - yield this.rollback(height); + blocks = this.height - height; + assert(blocks >= 0); + + if (this.prune) { + if (blocks <= this.network.block.keepBlocks) + yield this.rollback(height); + } else { + yield this.rollback(height); + } hashes = yield this.getHashes(); @@ -1416,11 +1425,18 @@ WalletDB.prototype.setTip = function setTip(hash, height, ts) { WalletDB.prototype.setBlock = co(function* setBlock(block) { var batch = this.db.batch(); + var height; batch.del(layout.b(block.height + 1)); batch.put(layout.b(block.height), block.toRaw()); batch.put(layout.R, U32(block.height)); + if (this.prune) { + height = block.height - this.network.block.keepBlocks; + if (height > this.network.block.pruneAfterHeight) + batch.del(layout.b(height)); + } + yield batch.write(); this.tip = block; @@ -1435,8 +1451,16 @@ WalletDB.prototype.setBlock = co(function* setBlock(block) { WalletDB.prototype.writeBlock = function writeBlock(wallet, block) { var batch = this.batch(wallet); + var height = block.height - this.network.block.keepBlocks; + batch.put(layout.b(block.height), block.toRaw()); batch.put(layout.R, U32(block.height)); + + if (this.prune) { + height = block.height - this.network.block.keepBlocks; + if (height > this.network.block.pruneAfterHeight) + batch.del(layout.b(height)); + } }; /** @@ -1521,7 +1545,18 @@ WalletDB.prototype.rollback = co(function* rollback(height) { while (this.height > height) { block = yield this.getBlock(this.height); - assert(block); + + if (!block) { + if (this.prune) { + height = this.network.blocks.pruneAfterHeight; + block = yield this.getBlock(height); + assert(block); + yield this.setBlock(block); + throw new Error('Wallet reorgd beyond safe height. Rescan required.'); + } + assert(false, 'Database corruption.'); + } + yield this._removeBlock(block); } });