From 9f44ddc22fac628e1c17121d0d4eadd29bf3a9ca Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 30 Nov 2016 16:03:19 -0800 Subject: [PATCH] chain: add undocoins object. --- lib/blockchain/chaindb.js | 51 ++------ lib/blockchain/coins-old.js | 2 +- lib/blockchain/coins.js | 130 ++++++++++--------- lib/blockchain/coinview.js | 40 +++--- lib/blockchain/compress.js | 30 ++++- lib/blockchain/undocoins.js | 248 ++++++++++++++++++++++++++++++++++++ migrate/chaindb1to2.js | 63 ++++++++- 7 files changed, 443 insertions(+), 121 deletions(-) create mode 100644 lib/blockchain/undocoins.js diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index fc351358..205abc56 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -19,6 +19,7 @@ var co = require('../utils/co'); var Network = require('../protocol/network'); var CoinView = require('./coinview'); var Coins = require('./coins'); +var UndoCoins = require('./undocoins'); var LDB = require('../db/ldb'); var layout = require('./layout'); var LRU = require('../utils/lru'); @@ -715,18 +716,9 @@ ChainDB.prototype.getCoinView = co(function* getCoinView(block, callback) { ChainDB.prototype.getUndoCoins = co(function* getUndoCoins(hash) { var data = yield this.db.get(layout.u(hash)); - var br, coins; - if (!data) - return; - - br = new BufferReader(data); - coins = []; - - while (br.left()) - coins.push(Coin.fromRaw(br)); - - return coins; + return new UndoCoins(); + return UndoCoins.fromRaw(data); }); /** @@ -739,25 +731,15 @@ ChainDB.prototype.getUndoCoins = co(function* getUndoCoins(hash) { ChainDB.prototype.getUndoView = co(function* getUndoView(block) { var view = yield this.getCoinView(block); - var coins = yield this.getUndoCoins(block.hash()); - var i, j, k, tx, input, coin; + var undo = yield this.getUndoCoins(block.hash()); + var index = 0; + var i, j, tx, input; - if (!coins) - return view; - - for (i = 0, k = 0; i < block.txs.length; i++) { + for (i = 1; i < block.txs.length; i++) { tx = block.txs[i]; - - if (tx.isCoinbase()) - continue; - for (j = 0; j < tx.inputs.length; j++) { input = tx.inputs[j]; - coin = coins[k++]; - coin.hash = input.prevout.hash; - coin.index = input.prevout.index; - input.coin = coin; - view.addCoin(coin); + input.coin = undo.apply(index++, view, input.prevout); } } @@ -1599,7 +1581,6 @@ ChainDB.prototype.removeBlock = co(function* removeBlock(hash) { */ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) { - var undo = new BufferWriter(); var i, j, tx, input, output, prev; var hashes, address, hash, coins, raw; @@ -1643,10 +1624,6 @@ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) { } } - // Add coin to set of undo - // coins for the block. - input.coin.toRaw(undo); - this.pending.spend(input.coin); } } @@ -1667,12 +1644,16 @@ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) { } } + // Write undo coins (if there are any). + if (!view.undo.isEmpty()) + this.put(layout.u(block.hash()), view.undo.toRaw()); + // Commit new coin state. view = view.toArray(); for (i = 0; i < view.length; i++) { coins = view[i]; - if (coins.isFullySpent()) { + if (coins.isEmpty()) { this.del(layout.c(coins.hash)); this.coinCache.unpush(coins.hash); } else { @@ -1682,10 +1663,6 @@ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) { } } - // Write undo coins (if there are any). - if (undo.written > 0) - this.put(layout.u(block.hash()), undo.render()); - yield this.pruneBlock(block); }); @@ -1768,7 +1745,7 @@ ChainDB.prototype.disconnectBlock = co(function* disconnectBlock(block) { for (i = 0; i < view.length; i++) { coins = view[i]; - if (coins.isFullySpent()) { + if (coins.isEmpty()) { this.del(layout.c(coins.hash)); this.coinCache.unpush(coins.hash); } else { diff --git a/lib/blockchain/coins-old.js b/lib/blockchain/coins-old.js index b54d4cf2..8fff1bc6 100644 --- a/lib/blockchain/coins-old.js +++ b/lib/blockchain/coins-old.js @@ -188,7 +188,7 @@ Coins.prototype.size = function size() { * @returns {Boolean} */ -Coins.prototype.isFullySpent = function isFullySpent() { +Coins.prototype.isEmpty = function isEmpty() { return this.size() === 0; }; diff --git a/lib/blockchain/coins.js b/lib/blockchain/coins.js index c8508bec..f75c7b4f 100644 --- a/lib/blockchain/coins.js +++ b/lib/blockchain/coins.js @@ -90,25 +90,20 @@ Coins.fromOptions = function fromOptions(options) { }; /** - * Add a single coin to the collection. - * @param {Coin} coin + * Add a single output to the collection. + * @param {Number} index + * @param {Output} output */ -Coins.prototype.add = function add(coin) { - if (this.outputs.length === 0) { - this.version = coin.version; - this.hash = coin.hash; - this.height = coin.height; - this.coinbase = coin.coinbase; - } +Coins.prototype.add = function add(index, output) { + assert(!output.script.isUnspendable()); - if (coin.script.isUnspendable()) - return; - - while (this.outputs.length <= coin.index) + while (this.outputs.length <= index) this.outputs.push(null); - this.outputs[coin.index] = CoinEntry.fromCoin(coin); + assert(!this.outputs[index]); + + this.outputs[index] = CoinEntry.fromOutput(output); }; /** @@ -124,42 +119,50 @@ Coins.prototype.has = function has(index) { return this.outputs[index] != null; }; +/** + * Get a coin entry. + * @param {Number} index + * @returns {CoinEntry} + */ + +Coins.prototype.get = function get(index) { + if (index >= this.outputs.length) + return; + + return this.outputs[index]; +}; + /** * Get a coin. * @param {Number} index * @returns {Coin} */ -Coins.prototype.get = function get(index) { - var coin; +Coins.prototype.getCoin = function getCoin(index) { + var entry = this.get(index); - if (index >= this.outputs.length) + if (!entry) return; - coin = this.outputs[index]; - - if (!coin) - return; - - return coin.toCoin(this, index); + return entry.toCoin(this, index); }; /** - * Remove a coin and return it. + * Remove a coin entry and return it. * @param {Number} index - * @returns {Coin} + * @returns {CoinEntry} */ Coins.prototype.spend = function spend(index) { - var coin = this.get(index); + var entry = this.get(index); - if (!coin) + if (!entry) return; this.outputs[index] = null; this.cleanup(); - return coin; + return entry; }; /** @@ -180,7 +183,7 @@ Coins.prototype.cleanup = function cleanup() { * @returns {Boolean} */ -Coins.prototype.isFullySpent = function isFullySpent() { +Coins.prototype.isEmpty = function isEmpty() { return this.outputs.length === 0; }; @@ -409,7 +412,7 @@ Coins.parseCoin = function parseCoin(data, hash, index) { return; // Read compressed output. - decompress.output(coin, br); + decompress.coin(coin, br); break; } @@ -456,7 +459,7 @@ Coins.prototype.fromTX = function fromTX(tx) { continue; } - this.outputs.push(CoinEntry.fromTX(tx, i)); + this.outputs.push(CoinEntry.fromOutput(output)); } this.cleanup(); @@ -500,6 +503,23 @@ function CoinEntry() { this.output = null; } +/** + * Instantiate a reader at the correct offset. + * @private + * @returns {BufferReader} + */ + +CoinEntry.prototype.reader = function reader() { + var br; + + assert(this.raw); + + br = new BufferReader(this.raw); + br.offset = this.offset; + + return br; +}; + /** * Parse the deferred data and return a Coin. * @param {Coins} coins @@ -509,7 +529,6 @@ function CoinEntry() { CoinEntry.prototype.toCoin = function toCoin(coins, index) { var coin = new Coin(); - var br; // Load in all necessary properties // from the parent Coins object. @@ -522,19 +541,24 @@ CoinEntry.prototype.toCoin = function toCoin(coins, index) { if (this.output) { coin.script = this.output.script; coin.value = this.output.value; - return coin; + } else { + decompress.coin(coin, this.reader()); } - br = new BufferReader(this.raw); - - // Seek to the coin's offset. - br.seek(this.offset); - - decompress.output(coin, br); - return coin; }; +/** + * Parse the deferred data and return an Output. + * @returns {Output} + */ + +CoinEntry.prototype.toOutput = function toOutput() { + if (this.output) + return this.output; + return decompress.output(new Output(), this.reader()); +}; + /** * Slice off the part of the buffer * relevant to this particular coin. @@ -543,13 +567,12 @@ CoinEntry.prototype.toCoin = function toCoin(coins, index) { CoinEntry.prototype.toWriter = function toWriter(bw) { var raw; - if (this.output) { + if (!this.raw) { + assert(this.output); compress.output(this.output, bw); return bw; } - assert(this.raw); - // If we read this coin from the db and // didn't use it, it's still in its // compressed form. Just write it back @@ -575,30 +598,15 @@ CoinEntry.fromReader = function fromReader(br) { return entry; }; -/** - * Instantiate compressed coin from tx. - * @param {TX} tx - * @param {Number} index - * @returns {CoinEntry} - */ - -CoinEntry.fromTX = function fromTX(tx, index) { - var entry = new CoinEntry(); - entry.output = tx.outputs[index]; - return entry; -}; - /** * Instantiate compressed coin from coin. - * @param {Coin} coin + * @param {Output} output * @returns {CoinEntry} */ -CoinEntry.fromCoin = function fromCoin(coin) { +CoinEntry.fromOutput = function fromOutput(output) { var entry = new CoinEntry(); - entry.output = new Output(); - entry.output.script = coin.script; - entry.output.value = coin.value; + entry.output = output; return entry; }; diff --git a/lib/blockchain/coinview.js b/lib/blockchain/coinview.js index a7cb371f..b2a6540e 100644 --- a/lib/blockchain/coinview.js +++ b/lib/blockchain/coinview.js @@ -6,8 +6,8 @@ 'use strict'; -var assert = require('assert'); var Coins = require('./coins'); +var UndoCoins = require('./undocoins'); /** * A collections of {@link Coins} objects. @@ -22,6 +22,7 @@ function CoinView(coins) { return new CoinView(coins); this.coins = coins || {}; + this.undo = new UndoCoins(); } /** @@ -33,18 +34,6 @@ CoinView.prototype.add = function add(coins) { this.coins[coins.hash] = coins; }; -/** - * Add a coin to the collection. - * @param {Coin} coin - */ - -CoinView.prototype.addCoin = function addCoin(coin) { - assert(typeof coin.hash === 'string'); - if (!this.coins[coin.hash]) - this.coins[coin.hash] = new Coins(); - this.coins[coin.hash].add(coin); -}; - /** * Add a tx to the collection. * @param {TX} tx @@ -63,11 +52,17 @@ CoinView.prototype.addTX = function addTX(tx) { CoinView.prototype.get = function get(hash, index) { var coins = this.coins[hash]; + var entry; if (!coins) return; - return coins.get(index); + entry = coins.get(index); + + if (!entry) + return; + + return entry.toCoin(coins, index); }; /** @@ -95,11 +90,26 @@ CoinView.prototype.has = function has(hash, index) { CoinView.prototype.spend = function spend(hash, index) { var coins = this.coins[hash]; + var entry, undo; if (!coins) return; - return coins.spend(index); + entry = coins.spend(index); + + if (!entry) + return; + + this.undo.push(entry); + + if (coins.isEmpty()) { + undo = this.undo.top(); + undo.height = coins.height; + undo.coinbase = coins.coinbase; + undo.version = coins.version; + } + + return entry.toCoin(coins, index); }; /** diff --git a/lib/blockchain/compress.js b/lib/blockchain/compress.js index 74252d75..b3fe6eb9 100644 --- a/lib/blockchain/compress.js +++ b/lib/blockchain/compress.js @@ -117,7 +117,7 @@ function decompressScript(script, br) { /** * Compress an output. - * @param {Output|Coin} output + * @param {Output} output * @param {BufferWriter} bw */ @@ -129,7 +129,7 @@ function compressOutput(output, bw) { /** * Decompress a script from buffer reader. - * @param {Output|Coin} output + * @param {Output} output * @param {BufferReader} br */ @@ -139,6 +139,30 @@ function decompressOutput(output, br) { return output; } +/** + * Compress an output. + * @param {Coin} coin + * @param {BufferWriter} bw + */ + +function compressCoin(coin, bw) { + bw.writeVarint(coin.value); + compressScript(coin.script, bw); + return bw; +} + +/** + * Decompress a script from buffer reader. + * @param {Coin} coin + * @param {BufferReader} br + */ + +function decompressCoin(coin, br) { + coin.value = br.readVarint(); + decompressScript(coin.script, br); + return coin; +} + /** * Skip past a compressed output. * @param {BufferWriter} bw @@ -331,6 +355,7 @@ function decompressKey(key) { exports.compress = { output: compressOutput, + coin: compressCoin, script: compressScript, value: compressValue, key: compressKey @@ -338,6 +363,7 @@ exports.compress = { exports.decompress = { output: decompressOutput, + coin: decompressCoin, skip: skipOutput, script: decompressScript, value: decompressValue, diff --git a/lib/blockchain/undocoins.js b/lib/blockchain/undocoins.js new file mode 100644 index 00000000..1a5c1b76 --- /dev/null +++ b/lib/blockchain/undocoins.js @@ -0,0 +1,248 @@ +/*! + * undocoins.js - undocoins object for bcoin + * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +var assert = require('assert'); +var BufferReader = require('../utils/reader'); +var BufferWriter = require('../utils/writer'); +var Output = require('../primitives/output'); +var Coins = require('./coins'); +var compressor = require('./compress'); +var compress = compressor.compress; +var decompress = compressor.decompress; + +/** + * UndoCoins + * Coins need to be resurrected from somewhere + * during a reorg. The undo coins store all + * spent coins in a single record per block + * (in a compressed format). + * @constructor + * @property {UndoCoin[]} items + */ + +function UndoCoins() { + this.items = []; +} + +/** + * Push coin entry onto undo coin array. + * @param {CoinEntry} + */ + +UndoCoins.prototype.push = function push(entry) { + var undo = new UndoCoin(); + undo.entry = entry; + this.items.push(undo); +}; + +/** + * Serialize all undo coins. + * @returns {Buffer} + */ + +UndoCoins.prototype.toRaw = function toRaw() { + var bw = new BufferWriter(); + var i, coin; + + bw.writeU32(this.items.length); + + for (i = 0; i < this.items.length; i++) { + coin = this.items[i]; + coin.toRaw(bw); + } + + return bw.render(); +}; + +/** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + * @returns {UndoCoins} + */ + +UndoCoins.prototype.fromRaw = function fromRaw(data) { + var br = new BufferReader(data); + var count = br.readU32(); + var i; + + for (i = 0; i < count; i++) + this.items.push(UndoCoin.fromRaw(br)); + + return this; +}; + +/** + * Instantiate undo coins from serialized data. + * @param {Buffer} data + * @returns {UndoCoins} + */ + +UndoCoins.fromRaw = function fromRaw(data) { + return new UndoCoins().fromRaw(data); +}; + +/** + * Test whether the undo coins have any members. + * @returns {Boolean} + */ + +UndoCoins.prototype.isEmpty = function isEmpty() { + return this.items.length === 0; +}; + +/** + * Retrieve the last undo coin. + * @returns {UndoCoin} + */ + +UndoCoins.prototype.top = function top() { + return this.items[this.items.length - 1]; +}; + +/** + * Re-apply undo coins to a view, effectively unspending them. + * @param {Number} i + * @param {CoinView} view + * @param {Outpoint} outpoint + * @returns {Coin} + */ + +UndoCoins.prototype.apply = function apply(i, view, outpoint) { + var undo = this.items[i]; + var hash = outpoint.hash; + var index = outpoint.index; + var coins; + + assert(undo); + + if (undo.height !== -1) { + coins = new Coins(); + + assert(!view.coins[hash]); + view.coins[hash] = coins; + + coins.coinbase = undo.coinbase; + coins.height = undo.height; + coins.version = undo.version; + } else { + coins = view.coins[hash]; + assert(coins); + } + + coins.add(index, undo.toOutput()); + + return coins.getCoin(index); +}; + +/** + * UndoCoin + * @constructor + * @property {CoinEntry|null} entry + * @property {Output|null} output + * @property {Number} version + * @property {Number} height + * @property {Boolean} coinbase + */ + +function UndoCoin() { + this.entry = null; + this.output = null; + this.version = -1; + this.height = -1; + this.coinbase = false; +} + +/** + * Convert undo coin to an output. + * @returns {Output} + */ + +UndoCoin.prototype.toOutput = function toOutput() { + if (!this.output) { + assert(this.entry); + return this.entry.toOutput(); + } + return this.output; +}; + +/** + * Serialize the undo coin. + * @returns {Buffer} + */ + +UndoCoin.prototype.toRaw = function toRaw(writer) { + var bw = new BufferWriter(writer); + var height = this.height; + + if (height === -1) + height = 0; + + bw.writeVarint(height * 2 + (this.coinbase ? 1 : 0)); + + if (this.height !== -1) { + assert(this.version !== -1); + bw.writeVarint(this.version); + } + + if (this.entry) { + // Cached from spend. + this.entry.toWriter(bw); + } else { + compress.output(this.output, bw); + } + + if (!writer) + bw = bw.render(); + + return bw; +}; + +/** + * Inject properties from serialized data. + * @private + * @param {Buffer} data + * @returns {UndoCoin} + */ + +UndoCoin.prototype.fromRaw = function fromRaw(data) { + var br = new BufferReader(data); + var code = br.readVarint(); + + this.output = new Output(); + + this.height = code / 2 | 0; + + if (this.height === 0) + this.height = -1; + + this.coinbase = (code & 1) !== 0; + + if (this.height !== -1) + this.version = br.readVarint(); + + decompress.output(this.output, br); + + return this; +}; + +/** + * Instantiate undo coin from serialized data. + * @param {Buffer} data + * @returns {UndoCoin} + */ + +UndoCoin.fromRaw = function fromRaw(data) { + return new UndoCoin().fromRaw(data); +}; + +/* + * Expose + */ + +module.exports = UndoCoins; diff --git a/migrate/chaindb1to2.js b/migrate/chaindb1to2.js index 30ab42de..0a04fa9c 100644 --- a/migrate/chaindb1to2.js +++ b/migrate/chaindb1to2.js @@ -6,11 +6,11 @@ var BufferWriter = require('../lib/utils/writer'); var BufferReader = require('../lib/utils/reader'); var OldCoins = require('../lib/blockchain/coins-old'); var Coins = require('../lib/blockchain/coins'); -var crypto = require('../lib/crypto/crypto'); +var UndoCoins = require('../lib/blockchain/undocoins'); +var Coin = require('../lib/primitives/coin'); +var Output = require('../lib/primitives/output'); var util = require('../lib/utils/util'); var LDB = require('../lib/db/ldb'); -var BN = require('bn.js'); -var DUMMY = new Buffer([0]); var file = process.argv[2]; var options = {}; var db, batch, index; @@ -117,7 +117,7 @@ var updateDeployments = co(function* updateDeployments() { var reserializeCoins = co(function* reserializeCoins() { var total = 0; - var i, iter, item, hash, old, coins, coin; + var i, iter, item, hash, old, coins, coin, output; iter = db.iterator({ gte: pair('c', constants.ZERO_HASH), @@ -142,11 +142,18 @@ var reserializeCoins = co(function* reserializeCoins() { for (i = 0; i < old.outputs.length; i++) { coin = old.get(i); + if (!coin) { coins.outputs.push(null); continue; } - coins.add(coin); + + output = new Output(); + output.script = coin.script; + output.value = coin.value; + + if (!output.script.isUnspendable()) + coins.add(coin.index, output); } coins.cleanup(); @@ -160,6 +167,39 @@ var reserializeCoins = co(function* reserializeCoins() { console.log('Reserialized %d coins.', total); }); +var reserializeUndo = co(function* reserializeUndo() { + var total = 0; + var iter, item, br, undo; + + iter = db.iterator({ + gte: pair('u', constants.ZERO_HASH), + lte: pair('u', constants.MAX_HASH), + values: true + }); + + for (;;) { + item = yield iter.next(); + + if (!item) + break; + + br = new BufferReader(item.value); + undo = new UndoCoins(); + + while (br.left()) { + undo.push(null); + injectCoin(undo.top(), Coin.fromRaw(br)); + } + + batch.write(item.key, undo.toRaw()); + + if (++total % 10000 === 0) + console.log('Reserialized %d undo coins.', total); + } + + console.log('Reserialized %d undo coins.', total); +}); + function write(data, str, off) { if (Buffer.isBuffer(str)) return str.copy(data, off); @@ -184,6 +224,18 @@ function ipair(prefix, num) { return key; } +function injectCoin(undo, coin) { + var output = new Output(); + + output.value = coin.value; + output.script = coin.script; + + undo.output = output; + undo.version = coin.version; + undo.height = coin.height; + undo.coinbase = coin.coinbase; +} + function defaultOptions() { var bw = new BufferWriter(); var flags = 0; @@ -235,6 +287,7 @@ co.spawn(function* () { yield updateOptions(); yield updateDeployments(); yield reserializeCoins(); + yield reserializeUndo(); yield batch.write(); }).then(function() { console.log('Migration complete.');