From 2884a794c3d20cd748feb4e2792c04eb761fe0ab Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Thu, 26 May 2016 17:59:43 -0700 Subject: [PATCH] coins --- lib/bcoin/chaindb.js | 19 +- lib/bcoin/coins.js | 420 +++++++++++++++++++++++++++--------------- lib/bcoin/coinview.js | 86 ++++----- test/bloom-test.js | 4 +- test/chain-test.js | 39 +++- test/protocol-test.js | 2 +- 6 files changed, 354 insertions(+), 216 deletions(-) diff --git a/lib/bcoin/chaindb.js b/lib/bcoin/chaindb.js index 15faed92..2621d24f 100644 --- a/lib/bcoin/chaindb.js +++ b/lib/bcoin/chaindb.js @@ -84,9 +84,20 @@ function ChainDB(chain, options) { // check. this.cacheWindow = (this.network.pow.retargetInterval + 1) * 2 + 100; - this.coinCache = new bcoin.lru(100000); - this.cacheHash = new bcoin.lru(this.cacheWindow); - this.cacheHeight = new bcoin.lru(this.cacheWindow); + // We want to keep the last 5 blocks of unspents in memory. + // Some explanation for the numbers: + // Average block output size: 165kb + // Average number of outputs: 5000 + // Average output size: 33.6b + // Average number of outputs per tx: 2.2 + // Average size of outputs per tx: 74b + // Average number of txs: 2300 + // Key size: 68b (* 2) + this.coinWindow = ((165 * 1024 + 5000 * 9) + (5000 * 68 * 2)) * 5; + + this.coinCache = new bcoin.lru(this.coinWindow); + this.cacheHash = new bcoin.lru(this.cacheWindow, 1); + this.cacheHeight = new bcoin.lru(this.cacheWindow, 1); this._init(); } @@ -915,7 +926,7 @@ ChainDB.prototype.disconnectBlock = function disconnectBlock(block, batch, callb batch.put('c/' + key, coin); - self.coinCache.set(key, coint); + self.coinCache.set(key, coin); } for (j = 0; j < tx.outputs.length; j++) { diff --git a/lib/bcoin/coins.js b/lib/bcoin/coins.js index 75e5433c..409364dc 100644 --- a/lib/bcoin/coins.js +++ b/lib/bcoin/coins.js @@ -7,6 +7,8 @@ var bcoin = require('./env'); var utils = bcoin.utils; +var assert = utils.assert; +var constants = bcoin.protocol.constants; var BufferReader = require('./reader'); var BufferWriter = require('./writer'); @@ -15,7 +17,6 @@ var BufferWriter = require('./writer'); * @exports Coins * @constructor * @param {TX|Object} tx/options - TX or options object. - * @param {Hash|Buffer} hash - Transaction hash. * @property {Hash} hash - Transaction hash. * @property {Number} version - Transaction version. * @property {Number} height - Transaction height (-1 if unconfirmed). @@ -24,53 +25,41 @@ var BufferWriter = require('./writer'); * @property {Coin[]} outputs - Coins. */ -function Coins(options, hash) { +function Coins(options) { + var i, coin; + if (!(this instanceof Coins)) - return new Coins(options, hash); + return new Coins(options); - this.version = options.version; - this.height = options.height; + if (!options) + options = {}; - this.coinbase = options.isCoinbase - ? options.isCoinbase() - : options.coinbase; - - this.hash = hash; - - this.outputs = options.outputs.map(function(coin, i) { - if (!coin) - return null; - - if (coin instanceof bcoin.coin) - return coin; - - coin = utils.merge({}, coin); - coin.version = options.version; - coin.height = options.height; - coin.hash = hash; - coin.index = i; - coin.coinbase = this.coinbase; - - return new bcoin.coin(coin); - }, this); + this.version = options.version != null ? options.version : -1; + this.hash = options.hash || null; + this.height = options.height != null ? options.height : -1; + this.coinbase = options.coinbase || false; + this.outputs = options.outputs || []; } /** - * Add a coin to the collection. - * @param {Coin|TX} tx/coin - * @param {Number?} index + * Add a single coin to the collection. + * @param {Coin} coin */ -Coins.prototype.add = function add(tx, i) { - var coin; +Coins.prototype.add = function add(coin) { + if (this.version === -1) { + this.version = coin.version; + this.hash = coin.hash; + this.height = coin.height; + this.coinbase = coin.coinbase; + } - if (i == null) { - coin = tx; - this.outputs[coin.index] = coin; + if (coin.script.isUnspendable()) { + this.outputs[coin.index] = null; return; } - this.outputs[i] = new bcoin.coin.fromTX(tx, i); + this.outputs[coin.index] = coin; }; /** @@ -94,13 +83,20 @@ Coins.prototype.get = function get(index) { }; /** - * Remove a coin. - * @param {Number} index + * Count unspent coins. + * @returns {Number} */ -Coins.prototype.remove = function remove(index) { - if (index < this.outputs.length) - this.outputs[index] = null; +Coins.prototype.count = function count(index) { + var total = 0; + var i; + + for (i = 0; i < this.outputs.length; i++) { + if (this.outputs[i]) + total++; + } + + return total; }; /** @@ -111,82 +107,249 @@ Coins.prototype.remove = function remove(index) { Coins.prototype.spend = function spend(index) { var coin = this.get(index); - this.remove(index); + this.outputs[index] = null; return coin; }; /** * Fill transaction(s) with coins. - * @param {TX|TX[]} tx + * @param {TX} tx * @param {Boolean?} spend - Whether the coins should * be spent when filling. - * @returns {Boolean} True if any inputs were filled. + * @returns {Boolean} True if all inputs were filled. */ -Coins.prototype.fill = function fill(tx, spend) { +Coins.prototype.fill = function fill(tx) { var res = true; - var i, input; - - if (tx.txs) - tx = tx.txs; - - if (Array.isArray(tx)) { - for (i = 0; i < tx.length; i++) { - if (!this.fill(tx[i])) - res = false; - } - return res; - } + var i, input, prevout; for (i = 0; i < tx.inputs.length; i++) { input = tx.inputs[i]; - - if (input.prevout.hash !== this.hash) + prevout = input.prevout; + if (prevout.hash !== this.hash) continue; - - if (!input.coin) { - if (spend) - input.coin = this.spend(input.prevout.index); - else - input.coin = this.get(input.prevout.index); - - if (!input.coin) - res = false; - } + input.coin = this.spend(prevout.index); + if (!input.coin) + res = false; } return res; }; -/** - * Count number of available coins. - * @returns {Number} Total. - */ - -Coins.prototype.count = function count() { - return this.outputs.reduce(function(total, output) { - if (!output) - return total; - return total + 1; - }, 0); -}; - /** * Convert collection to an array. * @returns {Coin[]} */ Coins.prototype.toArray = function toArray() { - return this.outputs.filter(Boolean); + var out = []; + var i; + + for (i = 0; i < this.outputs.length; i++) { + if (this.outputs[i]) + out.push(this.outputs[i]); + } + + return out; }; /** * Serialize the coins object. + * @param {TX|Coins} tx * @returns {Buffer} */ -Coins.prototype.toRaw = function toRaw() { - return Coins.toRaw(this); +Coins.prototype.toRaw = function toRaw(writer) { + var p = new BufferWriter(writer); + var height = this.height; + var i, output, prefix, hash, coinbase, mask; + + if (height === -1) + height = 0x7fffffff; + + coinbase = this.coinbase; + + mask = (height << 1) | (coinbase ? 1 : 0); + + p.writeVarint(this.version); + p.writeU32(mask >>> 0); + + for (i = 0; i < this.outputs.length; i++) { + output = this.outputs[i]; + + if (!output) { + p.writeU8(0xff); + continue; + } + + prefix = 0; + + // Saves up to 7 bytes. + if (isPubkeyhash(output.script)) { + prefix = 1; + hash = output.script.code[2]; + } else if (isScripthash(output.script)) { + prefix = 2; + hash = output.script.code[1]; + } + + // p.writeU8(((output.spent ? 1 : 0) << 2) | prefix); + p.writeU8(prefix); + + if (prefix) + p.writeBytes(hash); + else + bcoin.protocol.framer.script(output.script, p); + + p.writeVarint(output.value); + } + + if (!writer) + p = p.render(); + + return p; +}; + +/** + * Parse serialized coins. + * @param {Buffer} data + * @param {Hash} hash + * @returns {Object} A "naked" coins object. + */ + +Coins.parseRaw = function parseRaw(data, hash) { + var coins = {}; + var p = new BufferReader(data); + var i = 0; + var coin, mask, prefix; + + coins.version = p.readVarint(); + coins.height = p.readU32(); + coins.hash = hash; + coins.coinbase = (coins.height & 1) !== 0; + coins.height >>>= 1; + coins.outputs = []; + + if (coins.height === 0x7fffffff) + coins.height = -1; + + while (p.left()) { + mask = p.readU8(); + + if (mask === 0xff) { + coins.outputs.push(null); + i++; + continue; + } + + coin = {}; + coin.version = coins.version; + coin.coinbase = coins.coinbase; + coin.height = coins.height; + coin.hash = coins.hash; + coin.index = i++; + + // coin.spent = (mask & 4) !== 0; + prefix = mask & 3; + + if (prefix === 0) + coin.script = new bcoin.script(bcoin.protocol.parser.parseScript(p)); + else if (prefix === 1) + coin.script = bcoin.script.createPubkeyhash(p.readBytes(20)); + else if (prefix === 2) + coin.script = bcoin.script.createScripthash(p.readBytes(20)); + else + assert(false, 'Bad prefix.'); + + coin.value = p.readVarint(); + + coins.outputs.push(new bcoin.coin(coin)); + } + + return coins; +}; + +/** + * Parse a single serialized coin. + * @param {Buffer} data + * @param {Hash} hash + * @param {Number} index + * @returns {Coin} + */ + +Coins.parseCoin = function parseCoin(data, hash, index) { + var p = new BufferReader(data); + var i = 0; + var mask, prefix, version, height, coinbase, spent, script, value; + + version = p.readVarint(); + height = p.readU32(); + coinbase = (height & 1) !== 0; + height >>>= 1; + + if (height === 0x7fffffff) + height = -1; + + while (p.left()) { + mask = p.readU8(); + + if (mask === 0xff) { + if (i === index) + break; + i++; + continue; + } + + // spent = (mask & 4) !== 0; + prefix = mask & 3; + + if (i !== index) { + if (prefix === 0) + p.seek(p.readVarint()); + else if (prefix <= 2) + p.seek(20); + else + assert(false, 'Bad prefix.'); + p.readVarint(); + i++; + continue; + } + + if (prefix === 0) + script = new bcoin.script(bcoin.protocol.parser.parseScript(p)); + else if (prefix === 1) + script = bcoin.script.createPubkeyhash(p.readBytes(20)); + else if (prefix === 2) + script = bcoin.script.createScripthash(p.readBytes(20)); + else + assert(false, 'Bad prefix.'); + + value = p.readVarint(); + + return new bcoin.coin({ + version: version, + coinbase: coinbase, + height: height, + hash: hash, + index: i, + // spent: spent, + script: script, + value: value + }); + } + + assert(false, 'No coin.'); +}; + +/** + * Instantiate coins from a serialized Buffer. + * @param {Buffer} data + * @param {Hash} hash - Transaction hash. + * @returns {Coins} + */ + +Coins.fromRaw = function fromRaw(data, hash) { + return new Coins(Coins.parseRaw(data, hash)); }; /** @@ -196,80 +359,37 @@ Coins.prototype.toRaw = function toRaw() { */ Coins.fromTX = function fromTX(tx) { - return new Coins(tx, tx.hash('hex')); -}; + var outputs = []; + var i; -/** - * Serialize the coins object. - * @param {TX|Coins} tx - * @returns {Buffer} - */ - -Coins.toRaw = function toRaw(tx) { - var p = new BufferWriter(); - var height = tx.height; - - if (height === -1) - height = 0x7fffffff; - - p.writeU32(tx.version); - p.writeU32(height); - p.writeU8(tx.coinbase ? 1 : 0); - p.writeVarint(tx.outputs.length); - - tx.outputs.forEach(function(output) { - if (!output) { - p.writeVarint(0); - return; - } - p.writeVarBytes(bcoin.protocol.framer.output(output)); - }); - - return p.render(); -}; - -/** - * Parse serialized coins. - * @param {Buffer} buf - * @returns {Object} A "naked" coins object. - */ - -Coins.parseRaw = function parseRaw(buf) { - var tx = { outputs: [] }; - var p = new BufferReader(buf); - var coinCount, i, coin; - - tx.version = p.readU32(); - tx.height = p.readU32(); - tx.coinbase = p.readU8() === 1; - - if (tx.height === 0x7fffffff) - tx.height = -1; - - coinCount = p.readVarint(); - for (i = 0; i < coinCount; i++) { - coin = p.readVarBytes(); - if (coin.length === 0) { - tx.outputs.push(null); + for (i = 0; i < tx.outputs.length; i++) { + if (tx.outputs[i].script.isUnspendable()) { + outputs.push(null); continue; } - coin = bcoin.protocol.parser.parseOutput(coin); - tx.outputs.push(coin); + outputs.push(bcoin.coin.fromTX(tx, i)); } - return tx; + return new Coins({ + version: tx.version, + hash: tx.hash('hex'), + height: tx.height, + coinbase: tx.isCoinbase(), + outputs: outputs + }); }; -/** - * Instantiate coins from a serialized Buffer. - * @param {Buffer} data - * @param {Hash|Buffer} hash - Transaction hash. - * @returns {Coins} +/* + * Helpers */ -Coins.fromRaw = function fromRaw(buf, hash) { - return new Coins(Coins.parseRaw(buf), hash); -}; +function isPubkeyhash(script) { + return script.isPubkeyhash() && bcoin.script.checkMinimal(script.code[2]); +} + +function isScripthash(script) { + return script.isScripthash() && bcoin.script.checkMinimal(script.code[1]); +} /* * Expose diff --git a/lib/bcoin/coinview.js b/lib/bcoin/coinview.js index 26df58f3..7948f62b 100644 --- a/lib/bcoin/coinview.js +++ b/lib/bcoin/coinview.js @@ -6,6 +6,9 @@ */ var bcoin = require('./env'); +var utils = bcoin.utils; +var assert = utils.assert; +var constants = bcoin.protocol.constants; /** * A collections of {@link Coins} objects. @@ -25,30 +28,31 @@ function CoinView(coins) { /** * Add a coin to the collection. * @param {Coins|TX} tx/coins - * @param {Number?} index */ -CoinView.prototype.add = function add(tx, i) { - var coin, hash; +CoinView.prototype.add = function add(coins) { + this.coins[coins.hash] = coins; +}; - if (i == null) { - coin = tx; - this.coins[coin.hash] = coin; - return; - } +/** + * Add a coin to the collection. + * @param {Coins|TX} tx/coins + */ - hash = tx.hash('hex'); +CoinView.prototype.addCoin = function addCoin(coin) { + assert(typeof coin.hash === 'string'); + if (!this.coins[coin.hash]) + this.coins[coin.hash] = new bcoin.coins(); + this.coins[coin.hash].add(coin); +}; - if (!this.coins[hash]) { - this.coins[hash] = Object.create(bcoin.coins.prototype); - this.coins[hash].version = tx.version; - this.coins[hash].height = tx.height; - this.coins[hash].coinbase = tx.isCoinbase(); - this.coins[hash].hash = hash; - this.coins[hash].outputs = new Array(tx.outputs.length); - } +/** + * Remove a collection from the view. + * @param {Coins|TX} tx/coins + */ - this.coins[hash].add(tx, i); +CoinView.prototype.remove = function remove(coins) { + delete this.coins[coins.hash]; }; /** @@ -65,19 +69,6 @@ CoinView.prototype.get = function get(hash, index) { return this.coins[hash].get(index); }; -/** - * Count number of available coins. - * @param {Hash} hash - * @returns {Number} Total. - */ - -CoinView.prototype.count = function count(hash) { - if (!this.coins[hash]) - return 0; - - return this.coins[hash].count(); -}; - /** * Test whether the collection has a coin. * @param {Hash} hash @@ -92,19 +83,6 @@ CoinView.prototype.has = function has(hash, index) { return this.coins[hash].has(index); }; -/** - * Remove a coin. - * @param {Hash} hash - * @param {Number} index - */ - -CoinView.prototype.remove = function remove(hash, index) { - if (!this.coins[hash]) - return; - - return this.coins[hash].remove(index); -}; - /** * Remove a coin and return it. * @param {Hash} hash @@ -121,19 +99,19 @@ CoinView.prototype.spend = function spend(hash, index) { /** * Fill transaction(s) with coins. - * @param {TX|TX[]} tx - * @param {Boolean?} spend - Whether the coins should - * be spent when filling. - * @returns {Boolean} True if any inputs were filled. + * @param {TX} tx + * @returns {Boolean} True if all inputs were filled. */ -CoinView.prototype.fill = function fill(obj, spend) { - var keys = Object.keys(this.coins); +CoinView.prototype.fill = function fill(tx) { var res = true; - var i; + var i, input, prevout; - for (i = 0; i < keys.length; i++) { - if (!this.coins[keys[i]].fill(obj, spend)) + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + prevout = input.prevout; + input.coin = this.spend(prevout.hash, prevout.index); + if (!input.coin) res = false; } @@ -152,7 +130,7 @@ CoinView.prototype.toArray = function toArray() { for (i = 0; i < keys.length; i++) { hash = keys[i]; - out = out.concat(this.coins[hash].toArray()); + out.push(this.coins[hash]); } return out; diff --git a/test/bloom-test.js b/test/bloom-test.js index d96378da..6a0b601c 100644 --- a/test/bloom-test.js +++ b/test/bloom-test.js @@ -47,7 +47,7 @@ describe('Bloom', function() { assert.equal(filter.filter.toString('hex'), filterHex); }); - it('should test regular filter', function() { + it('should handle 1m ops with regular filter', function() { var filter = bcoin.bloom.fromRate(210000, 0.00001, -1); filter.tweak = 0xdeadbeef; // ~1m operations @@ -63,7 +63,7 @@ describe('Bloom', function() { } }); - it('should test rolling filter', function() { + it('should handle 1m ops with rolling filter', function() { var filter = new bcoin.bloom.rolling(210000, 0.00001); filter.tweak = 0xdeadbeef; // ~1m operations diff --git a/test/chain-test.js b/test/chain-test.js index 3cb38dbd..d8897059 100644 --- a/test/chain-test.js +++ b/test/chain-test.js @@ -39,6 +39,10 @@ describe('Chain', function() { address: wallet.getAddress(), value: utils.satoshi('25.0') }); + redeemer.addOutput({ + address: wallet.createAddress().getAddress(), + value: utils.satoshi('5.0') + }); redeemer.addInput(tx, 0); redeemer.setLocktime(chain.height); wallet.sign(redeemer); @@ -49,6 +53,11 @@ describe('Chain', function() { } function deleteCoins(tx) { + if (tx.txs) { + delete tx.view; + deleteCoins(tx.txs); + return; + } if (Array.isArray(tx)) { tx.forEach(deleteCoins); return; @@ -78,10 +87,10 @@ describe('Chain', function() { mineBlock(ch2, cb2, function(err, chain2) { assert.ifError(err); cb2 = chain2.txs[0]; - deleteCoins(chain1.txs); + deleteCoins(chain1); chain.add(chain1, function(err) { assert.ifError(err); - deleteCoins(chain2.txs); + deleteCoins(chain2); chain.add(chain2, function(err) { assert.ifError(err); assert(chain.tip.hash === chain1.hash('hex')); @@ -123,7 +132,7 @@ describe('Chain', function() { chain.once('fork', function() { forked = true; }); - deleteCoins(reorg.txs); + deleteCoins(reorg); chain.add(reorg, function(err) { assert.ifError(err); assert(forked); @@ -146,7 +155,7 @@ describe('Chain', function() { it('should mine a block after a reorg', function(cb) { mineBlock(null, cb2, function(err, block) { assert.ifError(err); - deleteCoins(block.txs); + deleteCoins(block); chain.add(block, function(err) { assert.ifError(err); chain.db.get(block.hash('hex'), function(err, entry) { @@ -166,7 +175,7 @@ describe('Chain', function() { it('should fail to mine a block with coins on an alternate chain', function(cb) { mineBlock(null, cb1, function(err, block) { assert.ifError(err); - deleteCoins(block.txs); + deleteCoins(block); chain.add(block, function(err) { assert(err); cb(); @@ -174,6 +183,26 @@ describe('Chain', function() { }); }); + it('should get coin', function(cb) { + mineBlock(null, null, function(err, block) { + assert.ifError(err); + chain.add(block, function(err) { + assert.ifError(err); + mineBlock(null, block.txs[0], function(err, block) { + assert.ifError(err); + chain.add(block, function(err) { + assert.ifError(err); + chain.db.getCoin(block.txs[1].hash('hex'), 1, function(err, coin) { + assert.ifError(err); + assert.deepEqual(coin.toRaw(), bcoin.coin.fromTX(block.txs[1], 1).toRaw()); + cb(); + }); + }); + }); + }); + }); + }); + it('should cleanup', function(cb) { constants.tx.COINBASE_MATURITY = 100; cb(); diff --git a/test/protocol-test.js b/test/protocol-test.js index fcbafb5f..2566e71d 100644 --- a/test/protocol-test.js +++ b/test/protocol-test.js @@ -55,7 +55,7 @@ describe('Protocol', function() { }, { services: constants.LOCAL_SERVICES, - host: 'ffff:0123:4567:89ab:cdef:0123:4567:89ab', + host: '::123:456:789a', port: 18333, ts: Date.now() / 1000 | 0 }