From e9bd890c8b8327b7cda69ba72490d36126ccc4ca Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Thu, 7 Jul 2016 01:34:19 -0700 Subject: [PATCH] optimize coins serialization. --- lib/bcoin/coins.js | 274 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 250 insertions(+), 24 deletions(-) diff --git a/lib/bcoin/coins.js b/lib/bcoin/coins.js index 3747bed6..3fac0726 100644 --- a/lib/bcoin/coins.js +++ b/lib/bcoin/coins.js @@ -177,7 +177,7 @@ Coins.prototype.getLength = function getLength() { * Coins serialization: * version: varint * bits: varint ((height << 1) | coinbase-flag) - * spent-field: varint size | bitmap (0=unspent, 1=spent) + * spent-field: varint size | bitfield (0=unspent, 1=spent) * outputs (repeated): * prefix: 0x00 = varint size | raw script * 0x01 = 20 byte pubkey hash @@ -194,16 +194,22 @@ Coins.prototype.getLength = function getLength() { */ Coins.prototype.toRaw = function toRaw(writer) { - var p = new BufferWriter(writer); + var p = new BufferWriter(); var height = this.height; var length = this.getLength(); - var i, output, prefix, data, bits, field, pos, bit, oct; + var i, output, prefix, data, bits, fstart, flen, bit, oct; + + // Varint version: hopefully some smartass + // miner doesn't start mining `-1` versions. + p.writeVarint(this.version); // Unfortunately, we don't have a compact // way to store unconfirmed height. if (height === -1) height = 0x7fffffff; + // Create the `bits` value: + // (height | coinbase-flag). bits = height << 1; if (this.coinbase) @@ -212,22 +218,23 @@ Coins.prototype.toRaw = function toRaw(writer) { if (bits < 0) bits += 0x100000000; - p.writeVarint(this.version); + // Making this a varint would actually + // make 99% of coins bigger. Varints + // are really only useful up until + // 0x10000, but since we're also + // storing the coinbase flag on the + // lo bit, varints are useless (and + // actually harmful) after height + // 32767 (0x7fff). p.writeU32(bits); - field = new Buffer(Math.ceil(length / 8)); - field.fill(0); - pos = 0; - - for (i = 0; i < length; i++) { - output = this.outputs[i] ? 0 : 1; - bit = pos % 8; - oct = (pos - bit) / 8; - field[oct] |= output << (7 - bit); - pos++; - } - - p.writeVarBytes(field); + // Fill the spent field with zeroes to avoid + // allocating a buffer. We mark the spents + // after rendering the final buffer. + fstart = p.written; + flen = Math.ceil(length / 8); + p.writeVarint(flen); + p.fill(0, flen); for (i = 0; i < length; i++) { output = this.outputs[i]; @@ -235,11 +242,17 @@ Coins.prototype.toRaw = function toRaw(writer) { if (!output) continue; + // If we read this coin from the db and + // didn't use it, it's still in its + // compressed form. Just write it back + // as a buffer for speed. if (output instanceof CompressedCoin) { p.writeBytes(output.toRaw()); continue; } + // Prefix byte (default=0 + // for uncompressed script). prefix = 0; // Attempt to compress the output scripts. @@ -276,8 +289,22 @@ Coins.prototype.toRaw = function toRaw(writer) { p.writeVarint(output.value); } - if (!writer) - p = p.render(); + p = p.render(); + + // Mark the spents in the spent field. + // This is essentially a NOP for new coins. + for (i = 0; i < length; i++) { + if (this.outputs[i]) + continue; + bit = i % 8; + oct = (i - bit) / 8; + p[fstart + oct] |= 1 << (7 - bit); + } + + if (writer) { + writer.writeBytes(p); + return writer; + } return p; }; @@ -292,7 +319,7 @@ Coins.prototype.toRaw = function toRaw(writer) { Coins.prototype.fromRaw = function fromRaw(data, hash, index) { var p = new BufferReader(data); var i = 0; - var bits, coin, prefix, offset, size, field, bit, oct, spent; + var bits, coin, prefix, offset, size, fstart, bit, oct, spent; this.version = p.readVarint(); @@ -305,12 +332,16 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { if (this.height === 0x7fffffff) this.height = -1; - field = p.readVarBytes(true); + // Mark the start of the spent field and + // seek past it to avoid reading a buffer. + fstart = p.offset; + p.seek(p.readVarint()); while (p.left()) { + // Read a single bit out of the spent field. bit = i % 8; oct = (i - bit) / 8; - spent = (field[oct] >>> (7 - bit)) & 1; + spent = (p.data[fstart + oct] >>> (7 - bit)) & 1; // Already spent. if (spent) { @@ -328,7 +359,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { continue; } - offset = p.start(); + offset = p.offset; prefix = p.readU8(); // Skip past the compressed scripts. @@ -350,7 +381,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { // Skip past the value. p.readVarint(); - size = p.end(); + size = p.offset - offset; // Keep going if we're seeking // to a specific index. @@ -437,6 +468,197 @@ Coins.fromTX = function fromTX(tx) { return new Coins().fromTX(tx); }; +Coins.compressScript = function compressScript(script, p) { + // Prefix byte (default=0 + // for uncompressed script). + var prefix = 0; + var data; + + // Attempt to compress the output scripts. + // We can _only_ ever compress them if + // they are serialized as minimaldata, as + // we need to recreate them when we read + // them. + if (script.isPubkeyhash(true)) { + prefix = 1; + data = script.code[2].data; + } else if (script.isScripthash()) { + prefix = 2; + data = script.code[1].data; + } else if (script.isPubkey(true)) { + prefix = 3; + data = script.code[0].data; + + // Try to compress the key. + data = Coins.compressKey(data); + + // If we can't compress it, + // just store the script. + if (!data) + prefix = 0; + } + + p.writeU8(prefix); + + if (prefix === 0) + p.writeVarBytes(script.toRaw()); + else + p.writeBytes(data); +}; + +Coins.decompressScript = function decompressScript(p, script) { + var prefix = p.readU8(); + var key; + + // Decompress the script. + switch (prefix) { + case 0: + script.fromRaw(p.readVarBytes()); + break; + case 1: + script.fromPubkeyhash(p.readBytes(20)); + break; + case 2: + script.fromScripthash(p.readBytes(20)); + break; + case 3: + // Decompress the key. If this fails, + // we have database corruption! + key = Coins.decompressKey(p.readBytes(33)); + script.fromPubkey(key); + break; + default: + assert(false, 'Bad prefix.'); + } +}; + +// See: +// https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go + +Coins.compressValue = function compressValue(value) { + var exp, last; + + if (value === 0) + return 0; + + exp = 0; + while (value % 10 === 0 && exp < 9) { + value /= 10; + exp++; + } + + if (exp < 9) { + last = value % 10; + value = (value - last) / 10; + return 1 + 10 * (9 * value + last - 1) + exp; + } + + return 10 + 10 * (value - 1); +}; + +Coins.decompressValue = function decompressValue(value) { + var exp, n, last; + + if (value === 0) + return 0; + + value--; + + exp = value % 10; + value = (value - exp) / 10; + + n = 0; + if (exp < 9) { + last = value % 9; + value = (value - last) / 9; + n = value * 10 + last + 1; + } else { + n = value + 1; + } + + while (exp > 0) { + n *= 10; + exp--; + } + + return n; +}; + +/** + * Compress a public key to coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +Coins.compressKey = function compressKey(key) { + var out; + + // We can't compress it if it's not valid. + if (!bcoin.ec.publicKeyVerify(key)) + return; + + switch (key[0]) { + case 0x02: + case 0x03: + // Key is already compressed. + out = key; + break; + case 0x04: + case 0x06: + case 0x07: + // Compress the key normally. + out = bcoin.ec.publicKeyConvert(key, true); + // Store the original format (which + // may be a hybrid byte) in the hi + // 3 bits so we can restore it later. + // The hi bits being set also lets us + // know that this key was originally + // decompressed. + out[0] |= key[0] << 2; + break; + default: + throw new Error('Bad point format.'); + } + + assert(out.length === 33); + + return out; +}; + +/** + * Decompress a public key from the coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +Coins.decompressKey = function decompressKey(key) { + var format = key[0] >>> 2; + var out; + + assert(key.length === 33); + + // Hi bits are not set. This key + // is not meant to be decompressed. + if (format === 0) + return key; + + // Decompress the key, and off the + // low bits so publicKeyConvert + // actually understands it. + key[0] &= 0x03; + out = bcoin.ec.publicKeyConvert(key, false); + + // Reset the hi bits so as not to + // mutate the original buffer. + key[0] |= format << 2; + + // Set the original format, which + // may have been a hybrid prefix byte. + out[0] = format; + + return out; +}; + /** * A compressed coin is an object which defers * parsing of a coin. Say there is a transaction @@ -526,6 +748,10 @@ CompressedCoin.prototype.toRaw = function toRaw() { return this.raw.slice(this.offset, this.offset + this.size); }; +/* + * Helpers + */ + /* * Expose */