diff --git a/bench/coin.js b/bench/coin.js index 6d44a6d7..b7405476 100644 --- a/bench/coin.js +++ b/bench/coin.js @@ -15,28 +15,30 @@ wtx = TX.fromRaw(wtx.trim(), 'hex'); var coins = Coins.fromTX(wtx); var raw; +//raw = coins.toRaw2(); +//console.log(Coins.fromRaw2(raw)); var end = bench('serialize'); for (var i = 0; i < 10000; i++) - raw = coins.toRaw(); + raw = coins.toRaw2(); end(i); var end = bench('parse'); for (var i = 0; i < 10000; i++) - Coins.fromRaw(raw); + Coins.fromRaw2(raw); end(i); var end = bench('parse-single'); var hash = wtx.hash('hex'); for (var i = 0; i < 10000; i++) - Coins.parseCoin(raw, hash, 5); + Coins.parseCoin2(raw, hash, 5); end(i); -var coins = Coins.fromRaw(raw); +var coins = Coins.fromRaw2(raw); var end = bench('get'); var j; for (var i = 0; i < 10000; i++) for (var j = 0; j < coins.outputs.length; j++) - coins.get(j); + coins.get2(j); end(i * coins.outputs.length); diff --git a/lib/blockchain/chain.js b/lib/blockchain/chain.js index d9cce0fb..8dac98f8 100644 --- a/lib/blockchain/chain.js +++ b/lib/blockchain/chain.js @@ -639,7 +639,7 @@ Chain.prototype.verifyInputs = co(function* verifyInputs(block, prev, state) { // Ensure tx is not double spending an output. if (!tx.isCoinbase()) { - if (!view.fillCoins(tx)) { + if (!view.fillCoins2(tx)) { assert(!historical, 'BUG: Spent inputs in historical data!'); throw new VerifyError(block, 'invalid', diff --git a/lib/blockchain/chaindb.js b/lib/blockchain/chaindb.js index 5608b310..1c818db1 100644 --- a/lib/blockchain/chaindb.js +++ b/lib/blockchain/chaindb.js @@ -633,7 +633,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) { coins = this.coinCache.get(hash); if (coins) - return Coins.parseCoin(coins, hash, index); + return Coins.parseCoin2(coins, hash, index); coins = yield this.db.get(layout.c(hash)); @@ -643,7 +643,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) { if (state === this.state) this.coinCache.set(hash, coins); - return Coins.parseCoin(coins, hash, index); + return Coins.parseCoin2(coins, hash, index); }); /** @@ -662,7 +662,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) { coins = this.coinCache.get(hash); if (coins) - return Coins.fromRaw(coins, hash); + return Coins.fromRaw2(coins, hash); coins = yield this.db.get(layout.c(hash)); @@ -672,7 +672,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) { if (state === this.state) this.coinCache.set(hash, coins); - return Coins.fromRaw(coins, hash); + return Coins.fromRaw2(coins, hash); }); /** @@ -1672,7 +1672,7 @@ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) { for (i = 0; i < view.length; i++) { coins = view[i]; - raw = coins.toRaw(); + raw = coins.toRaw2(); if (!raw) { this.del(layout.c(coins.hash)); this.coinCache.unpush(coins.hash); @@ -1768,7 +1768,7 @@ ChainDB.prototype.disconnectBlock = co(function* disconnectBlock(block) { for (i = 0; i < view.length; i++) { coins = view[i]; - raw = coins.toRaw(); + raw = coins.toRaw2(); if (!raw) { this.del(layout.c(coins.hash)); this.coinCache.unpush(coins.hash); diff --git a/lib/blockchain/coins.js b/lib/blockchain/coins.js index cce0501c..b1e34d2b 100644 --- a/lib/blockchain/coins.js +++ b/lib/blockchain/coins.js @@ -102,14 +102,12 @@ Coins.prototype.add = function add(coin) { this.coinbase = coin.coinbase; } + if (coin.script.isUnspendable()) + return; + while (this.outputs.length <= coin.index) this.outputs.push(null); - if (coin.script.isUnspendable()) { - this.outputs[coin.index] = null; - return; - } - this.outputs[coin.index] = CoinEntry.fromCoin(coin); }; @@ -159,10 +157,62 @@ Coins.prototype.spend = function spend(index) { return; this.outputs[index] = null; + this.cleanup(); return coin; }; +/** + * Get a coin. + * @param {Number} index + * @returns {Coin} + */ + +Coins.prototype.get2 = function get2(index) { + var coin; + + if (index >= this.outputs.length) + return; + + coin = this.outputs[index]; + + if (!coin) + return; + + return coin.toCoin2(this, index); +}; + +/** + * Remove a coin and return it. + * @param {Number} index + * @returns {Coin} + */ + +Coins.prototype.spend2 = function spend2(index) { + var coin = this.get2(index); + + if (!coin) + return; + + this.outputs[index] = null; + this.cleanup(); + + return coin; +}; + +/** + * Cleanup spent outputs. + */ + +Coins.prototype.cleanup = function cleanup() { + var len = this.outputs.length; + + while (len > 0 && !this.outputs[len - 1]) + len--; + + this.outputs.length = len; +}; + /** * Count up to the last available index. * @returns {Number} @@ -294,7 +344,7 @@ Coins.prototype.toRaw = function toRaw() { * @returns {Object} A "naked" coins object. */ -Coins.prototype.fromRaw = function fromRaw(data, hash, index) { +Coins.prototype.fromRaw = function fromRaw(data, hash) { var br = new BufferReader(data); var pos = 0; var bits, len, start, bit, oct, spent, coin; @@ -395,7 +445,7 @@ Coins.parseCoin = function parseCoin(data, hash, index) { } // Skip past the compressed coin. - skipCoin(br); + decompress.skip(br); pos++; } }; @@ -411,6 +461,227 @@ Coins.fromRaw = function fromRaw(data, hash) { return new Coins().fromRaw(data, hash); }; +/** + * Serialize the coins object. + * @param {TX|Coins} tx + * @returns {Buffer} + */ + +Coins.prototype.toRaw2 = function toRaw2() { + var bw = new BufferWriter(); + var len = this.outputs.length; + var first = len > 0 && this.outputs[0]; + var second = len > 1 && this.outputs[1]; + var size = 0; + var nonzero = 0; + var i, j, code, ch, output; + + // Return nothing if we're fully spent. + if (len === 0) + return; + + // Calculate number of unspents and spent field size. + // size = number of bytes required for the bit field. + // nonzero = number of non-zero bytes required. + for (i = 0; 2 + i * 8 < len; i++) { + for (j = 0; j < 8 && 2 + i * 8 + j < len; j++) { + if (this.outputs[2 + i * 8 + j]) { + size = i + 1; + nonzero++; + break; + } + } + } + + if (!first && !second) + nonzero -= 1; + + // Calculate header code. + code = 8 * nonzero; + + if (this.coinbase) + code += 1; + + if (first) + code += 2; + + if (second) + code += 4; + + // Write headers. + bw.writeVarint(this.version); + bw.writeU32(this.height); + bw.writeVarint(code); + + // Write the spent field. + for (i = 0; i < size; i++) { + ch = 0; + for (j = 0; j < 8 && 2 + i * 8 + j < len; j++) { + if (this.outputs[2 + i * 8 + j]) + ch |= 1 << j; + } + bw.writeU8(ch); + } + + // Write the compressed outputs. + for (i = 0; i < len; i++) { + output = this.outputs[i]; + + if (!output) + continue; + + output.toWriter2(bw); + } + + return bw.render(); +}; + +/** + * Parse serialized coins. + * @param {Buffer} data + * @param {Hash} hash + * @returns {Object} A "naked" coins object. + */ + +Coins.prototype.fromRaw2 = function fromRaw2(data, hash) { + var br = new BufferReader(data); + var i, code, field, nonzero, ch, unspent, coin; + + this.hash = hash; + + // Read headers. + this.version = br.readVarint(); + this.height = br.readU32(); + code = br.readVarint(); + this.coinbase = (code & 1) !== 0; + + // Setup spent field. + field = [ + (code & 2) !== 0, + (code & 4) !== 0 + ]; + + // Recalculate number of non-zero bytes. + nonzero = code / 8 | 0; + + if ((code & 6) === 0) + nonzero += 1; + + // Read spent field. + while (nonzero > 0) { + ch = br.readU8(); + for (i = 0; i < 8; i++) { + unspent = (ch & (1 << i)) !== 0; + field.push(unspent); + } + if (ch !== 0) + nonzero--; + } + + // Read outputs. + for (i = 0; i < field.length; i++) { + if (!field[i]) { + this.outputs.push(null); + continue; + } + + // Store the offset and size + // in the compressed coin object. + coin = CoinEntry.fromReader2(br); + + this.outputs.push(coin); + } + + this.cleanup(); + + return this; +}; + +/** + * Parse a single serialized coin. + * @param {Buffer} data + * @param {Hash} hash + * @param {Number} index + * @returns {Coin} + */ + +Coins.parseCoin2 = function parseCoin2(data, hash, index) { + var br = new BufferReader(data); + var coin = new Coin(); + var i, code, field, nonzero, ch, unspent; + + coin.hash = hash; + coin.index = index; + + // Read headers. + coin.version = br.readVarint(); + coin.height = br.readU32(); + code = br.readVarint(); + coin.coinbase = (code & 1) !== 0; + + // Setup spent field. + field = [ + (code & 2) !== 0, + (code & 4) !== 0 + ]; + + // Recalculate number of non-zero bytes. + nonzero = code / 8 | 0; + + if ((code & 6) === 0) + nonzero += 1; + + // Read spent field. + while (nonzero > 0 && field.length <= index) { + ch = br.readU8(); + for (i = 0; i < 8; i++) { + unspent = (ch & (1 << i)) !== 0; + field.push(unspent); + } + if (ch !== 0) + nonzero--; + } + + if (field.length <= index) + return; + + while (nonzero > 0) { + if (br.readU8() !== 0) + nonzero--; + } + + // Read outputs. + for (i = 0; i < field.length; i++) { + if (i === index) { + if (!field[i]) + return; + + // Read compressed output. + decompress.output2(coin, br); + + break; + } + + if (!field[i]) + continue; + + decompress.skip2(br); + } + + return coin; +}; + +/** + * Instantiate coins from a serialized Buffer. + * @param {Buffer} data + * @param {Hash} hash - Transaction hash. + * @returns {Coins} + */ + +Coins.fromRaw2 = function fromRaw2(data, hash) { + return new Coins().fromRaw2(data, hash); +}; + /** * Inject properties from tx. * @private @@ -436,6 +707,8 @@ Coins.prototype.fromTX = function fromTX(tx) { this.outputs.push(CoinEntry.fromTX(tx, i)); } + this.cleanup(); + return this; }; @@ -505,9 +778,42 @@ CoinEntry.prototype.toCoin = function toCoin(coins, index) { // Seek to the coin's offset. br.seek(this.offset); - decompress.script(coin.script, br); + decompress.output(coin, br); - coin.value = br.readVarint(); + return coin; +}; + +/** + * Parse the deferred data and return a Coin. + * @param {Coins} coins + * @param {Number} index + * @returns {Coin} + */ + +CoinEntry.prototype.toCoin2 = function toCoin2(coins, index) { + var coin = new Coin(); + var br; + + // Load in all necessary properties + // from the parent Coins object. + coin.version = coins.version; + coin.coinbase = coins.coinbase; + coin.height = coins.height; + coin.hash = coins.hash; + coin.index = index; + + if (this.output) { + coin.script = this.output.script; + coin.value = this.output.value; + return coin; + } + + br = new BufferReader(this.raw); + + // Seek to the coin's offset. + br.seek(this.offset); + + decompress.output2(coin, br); return coin; }; @@ -521,8 +827,31 @@ CoinEntry.prototype.toWriter = function toWriter(bw) { var raw; if (this.output) { - compress.script(this.output.script, bw); - bw.writeVarint(this.output.value); + compress.output(this.output, bw); + return; + } + + assert(this.raw); + + // If we read this coin from the db and + // didn't use it, it's still in its + // compressed form. Just write it back + // as a buffer for speed. + raw = this.raw.slice(this.offset, this.offset + this.size); + + bw.writeBytes(raw); +}; + +/** + * Slice off the part of the buffer + * relevant to this particular coin. + */ + +CoinEntry.prototype.toWriter2 = function toWriter2(bw) { + var raw; + + if (this.output) { + compress.output2(this.output, bw); return; } @@ -546,7 +875,21 @@ CoinEntry.prototype.toWriter = function toWriter(bw) { CoinEntry.fromReader = function fromReader(br) { var entry = new CoinEntry(); entry.offset = br.offset; - entry.size = skipCoin(br); + entry.size = decompress.skip(br); + entry.raw = br.data; + return entry; +}; + +/** + * Instantiate compressed coin from reader. + * @param {BufferReader} br + * @returns {CoinEntry} + */ + +CoinEntry.fromReader2 = function fromReader2(br) { + var entry = new CoinEntry(); + entry.offset = br.offset; + entry.size = decompress.skip2(br); entry.raw = br.data; return entry; }; @@ -578,35 +921,6 @@ CoinEntry.fromCoin = function fromCoin(coin) { return entry; }; -/* - * Helpers - */ - -function skipCoin(br) { - var start = br.offset; - - // Skip past the compressed scripts. - switch (br.readU8()) { - case 0: - br.seek(br.readVarint()); - break; - case 1: - case 2: - br.seek(20); - break; - case 3: - br.seek(33); - break; - default: - throw new Error('Bad prefix.'); - } - - // Skip past the value. - br.skipVarint(); - - return br.offset - start; -} - /* * Expose */ diff --git a/lib/blockchain/coinview.js b/lib/blockchain/coinview.js index a7cb371f..aa9ecd23 100644 --- a/lib/blockchain/coinview.js +++ b/lib/blockchain/coinview.js @@ -70,6 +70,22 @@ CoinView.prototype.get = function get(hash, index) { return coins.get(index); }; +/** + * Get a coin. + * @param {Hash} hash + * @param {Number} index + * @returns {Coin} + */ + +CoinView.prototype.get2 = function get2(hash, index) { + var coins = this.coins[hash]; + + if (!coins) + return; + + return coins.get2(index); +}; + /** * Test whether the collection has a coin. * @param {Hash} hash @@ -102,6 +118,22 @@ CoinView.prototype.spend = function spend(hash, index) { return coins.spend(index); }; +/** + * Remove a coin and return it. + * @param {Hash} hash + * @param {Number} index + * @returns {Coin} + */ + +CoinView.prototype.spend2 = function spend2(hash, index) { + var coins = this.coins[hash]; + + if (!coins) + return; + + return coins.spend2(index); +}; + /** * Fill transaction(s) with coins. * @param {TX} tx @@ -122,6 +154,26 @@ CoinView.prototype.fillCoins = function fillCoins(tx) { return true; }; +/** + * Fill transaction(s) with coins. + * @param {TX} tx + * @returns {Boolean} True if all inputs were filled. + */ + +CoinView.prototype.fillCoins2 = function fillCoins2(tx) { + var i, input, prevout; + + for (i = 0; i < tx.inputs.length; i++) { + input = tx.inputs[i]; + prevout = input.prevout; + input.coin = this.spend2(prevout.hash, prevout.index); + if (!input.coin) + return false; + } + + return true; +}; + /** * Convert collection to an array. * @returns {Coins[]} diff --git a/lib/blockchain/compress.js b/lib/blockchain/compress.js index 32ff8756..95dc1970 100644 --- a/lib/blockchain/compress.js +++ b/lib/blockchain/compress.js @@ -103,6 +103,218 @@ function decompressScript(script, br) { return script; } +/** + * Compress a script, write directly to the buffer. + * @param {Script} script + * @param {BufferWriter} bw + */ + +function compressScript2(script, bw) { + var data; + + // Attempt to compress the output scripts. + // We can _only_ ever compress them if + // they are serialized as minimaldata, as + // we need to recreate them when we read + // them. + + // P2PKH -> 0 | key-hash + // Saves 5 bytes. + if (script.isPubkeyhash(true)) { + data = script.code[2].data; + bw.writeU8(0); + bw.writeBytes(data); + return bw; + } + + // P2SH -> 1 | script-hash + // Saves 3 bytes. + if (script.isScripthash()) { + data = script.code[1].data; + bw.writeU8(1); + bw.writeBytes(data); + return bw; + } + + // P2PK -> 2-5 | compressed-key + // Only works if the key is valid. + // Saves up to 35 bytes. + if (script.isPubkey(true)) { + data = script.code[0].data; + if (publicKeyVerify(data)) { + data = compressKey2(data); + bw.writeBytes(data); + return bw; + } + } + + // Raw -> varlen + 6 | script + bw.writeVarint(script.raw.length + 6); + bw.writeBytes(script.raw); + + return bw; +} + +/** + * Decompress a script from buffer reader. + * @param {Script} script + * @param {BufferReader} br + */ + +function decompressScript2(script, br) { + var size, data; + + // Decompress the script. + switch (br.readU8()) { + case 0: + data = br.readBytes(20, true); + script.fromPubkeyhash(data); + break; + case 1: + data = br.readBytes(20, true); + script.fromScripthash(data); + break; + case 2: + case 3: + case 4: + case 5: + br.offset -= 1; + data = br.readBytes(33, true); + // Decompress the key. If this fails, + // we have database corruption! + data = decompressKey2(data); + script.fromPubkey(data); + break; + default: + br.offset -= 1; + size = br.readVarint() - 6; + if (size > 10000) { + // This violates consensus rules. + // We don't need to read it. + script.fromUnspendable(); + p.seek(size); + } else { + data = br.readBytes(size); + script.fromRaw(data); + } + break; + } + + return script; +} + +/** + * Compress an output. + * @param {Output|Coin} output + * @param {BufferWriter} bw + */ + +function compressOutput(output, bw) { + compressScript(output.script, bw); + bw.writeVarint(output.value); + return bw; +} + +/** + * Decompress a script from buffer reader. + * @param {Output|Coin} output + * @param {BufferReader} br + */ + +function decompressOutput(output, br) { + decompressScript(output.script, br); + output.value = br.readVarint(); + return output; +} + +/** + * Skip past a compressed output. + * @param {BufferWriter} bw + * @returns {Number} + */ + +function skipOutput(br) { + var start = br.offset; + + // Skip past the compressed scripts. + switch (br.readU8()) { + case 0: + br.seek(br.readVarint()); + break; + case 1: + case 2: + br.seek(20); + break; + case 3: + br.seek(33); + break; + default: + throw new Error('Bad prefix.'); + } + + // Skip past the value. + br.skipVarint(); + + return br.offset - start; +} + +/** + * Compress an output. + * @param {Output|Coin} output + * @param {BufferWriter} bw + */ + +function compressOutput2(output, bw) { + compressScript2(output.script, bw); + bw.writeVarint(output.value); + return bw; +} + +/** + * Decompress a script from buffer reader. + * @param {Output|Coin} output + * @param {BufferReader} br + */ + +function decompressOutput2(output, br) { + decompressScript2(output.script, br); + output.value = br.readVarint(); + return output; +} + +/** + * Skip past a compressed output. + * @param {BufferWriter} bw + * @returns {Number} + */ + +function skipOutput2(br) { + var start = br.offset; + + // Skip past the compressed scripts. + switch (br.readU8()) { + case 0: + case 1: + br.seek(20); + break; + case 2: + case 3: + case 4: + case 5: + br.seek(32); + break; + default: + br.offset -= 1; + br.seek(br.readVarint() - 6); + break; + } + + // Skip past the value. + br.skipVarint(); + + return br.offset - start; +} + /** * Compress value using an exponent. Takes advantage of * the fact that many bitcoin values are divisible by 10. @@ -236,18 +448,120 @@ function decompressKey(key) { return out; } +/** + * Verify a public key (no hybrid keys allowed). + * @param {Buffer} key + * @returns {Boolean} + */ + +function publicKeyVerify(key) { + if (key.length === 0) + return false; + + switch (key[0]) { + case 0x02: + case 0x03: + return key.length === 33; + case 0x04: + if (key.length !== 65) + return false; + + return ec.publicKeyVerify(key); + default: + return false; + } +} + +/** + * Compress a public key to coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +function compressKey2(key) { + var out; + + switch (key[0]) { + case 0x02: + case 0x03: + // Key is already compressed. + out = key; + break; + case 0x04: + // Compress the key normally. + out = ec.publicKeyConvert(key, true); + // Store the oddness. + out[0] = 0x04 | (key[64] & 0x01); + break; + default: + throw new Error('Bad point format.'); + } + + assert(out.length === 33); + + return out; +} + +/** + * Decompress a public key from the coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +function decompressKey2(key) { + var format = key[0]; + var out; + + assert(key.length === 33); + + switch (format) { + case 0x02: + case 0x03: + return key; + case 0x04: + key[0] = 0x02; + break; + case 0x05: + key[0] = 0x03; + break; + default: + throw new Error('Bad point format.'); + } + + // Decompress the key, and off the + // low bits so publicKeyConvert + // actually understands it. + out = ec.publicKeyConvert(key, false); + + // Reset the first byte so as not to + // mutate the original buffer. + key[0] = format; + + return out; +} + /* * Expose */ exports.compress = { + output: compressOutput, + output2: compressOutput2, script: compressScript, + script2: compressScript2, value: compressValue, - key: compressKey + key: compressKey, + key2: compressKey2 }; exports.decompress = { + output: decompressOutput, + output2: decompressOutput2, + skip: skipOutput, + skip2: skipOutput2, script: decompressScript, + script2: decompressScript2, value: decompressValue, - key: decompressKey + key: decompressKey, + key2: decompressKey }; diff --git a/lib/script/script.js b/lib/script/script.js index 1d37ac39..4d785bcc 100644 --- a/lib/script/script.js +++ b/lib/script/script.js @@ -1500,6 +1500,28 @@ Script.isCode = function isCode(raw) { return true; }; +/** + * Inject properties from a unspendable script. + * @private + * @param {Buffer} key + */ + +Script.prototype.fromUnspendable = function fromUnspendable() { + this.raw = new Buffer(1); + this.raw[0] = opcodes.OP_RETURN; + this.code.push(new Opcode(opcodes.OP_RETURN)); + return this; +}; + +/** + * Create an unspendable script. + * @returns {Script} + */ + +Script.fromUnspendable = function fromUnspendable() { + return new Script().fromUnspendable(); +}; + /** * Inject properties from a pay-to-pubkey script. * @private diff --git a/lib/utils/reader.js b/lib/utils/reader.js index 6974aa78..ce2ab4f9 100644 --- a/lib/utils/reader.js +++ b/lib/utils/reader.js @@ -142,7 +142,7 @@ BufferReader.prototype.destroy = function destroy() { BufferReader.prototype.readU8 = function readU8() { var ret; assert(this.offset + 1 <= this.data.length); - ret = this.data.readUInt8(this.offset, true); + ret = this.data[this.offset]; this.offset += 1; return ret; };