diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js index 20014ee4..e1404f6a 100644 --- a/lib/bcoin/chain.js +++ b/lib/bcoin/chain.js @@ -322,7 +322,7 @@ Chain.prototype.preload = function preload(callback) { size = 0; stream.on('response', function(res) { - var height = Math.floor(res.headers['content-length'] / 80); + var height = Math.floor(+res.headers['content-length'] / 80); if (res.statusCode >= 400) { stream.destroy(); return callback(new Error('Bad response code: ' + res.statusCode)); diff --git a/lib/bcoin/coin.js b/lib/bcoin/coin.js index 2add595f..b1455d85 100644 --- a/lib/bcoin/coin.js +++ b/lib/bcoin/coin.js @@ -262,6 +262,77 @@ Coin.fromRaw = function fromRaw(data, enc) { return new Coin().fromRaw(data); }; +/** + * Serialize the coin to its compressed form. + * @param {String?} enc - Encoding, can be `'hex'` or null. + * @returns {Buffer|String} + */ + +Coin.prototype.toCompressed = function toCompressed(writer) { + var compress = bcoin.coins.compress; + var p = bcoin.writer(writer); + var height = this.height; + var bits; + + if (height === -1) + height = 0x7fffffff; + + bits = height << 1; + + if (this.coinbase) + bits |= 1; + + if (bits < 0) + bits += 0x100000000; + + p.writeVarint(this.version); + p.writeU32(bits); + p.writeVarint(this.value); + compress.script(this.script, p); + + if (!writer) + p = p.render(); + + return p; +}; + +/** + * Inject properties from compressed serialized data. + * @private + * @param {Buffer} data + */ + +Coin.prototype.fromCompressed = function fromCompressed(data) { + var decompress = bcoin.coins.decompress; + var p = bcoin.reader(data); + var bits; + + this.version = p.readVarint(); + bits = p.readU32(); + this.height = bits >>> 1; + this.coinbase = (bits & 1) !== 0; + this.value = p.readVarint(); + decompress.script(p, this.script); + + if (this.height === 0x7fffffff) + this.height = -1; + + return this; +}; + +/** + * Instantiate an coin from a serialized Buffer. + * @param {Buffer} data + * @param {String?} enc - Encoding, can be `'hex'` or null. + * @returns {Coin} + */ + +Coin.fromCompressed = function fromCompressed(data, enc) { + if (typeof data === 'string') + data = new Buffer(data, enc); + return new Coin().fromCompressed(data); +}; + /** * Serialize the coin to an "extended" format, * including both the hash and the index. diff --git a/lib/bcoin/coins.js b/lib/bcoin/coins.js index 3fac0726..aa34e685 100644 --- a/lib/bcoin/coins.js +++ b/lib/bcoin/coins.js @@ -152,6 +152,9 @@ Coins.prototype.count = function count(index) { Coins.prototype.spend = function spend(index) { var coin = this.get(index); + if (!coin) + return; + this.outputs[index] = null; return coin; }; @@ -161,30 +164,42 @@ Coins.prototype.spend = function spend(index) { * @returns {Number} */ -Coins.prototype.getLength = function getLength() { - var last = -1; +Coins.prototype.size = function size() { + var index = -1; var i; - for (i = 0; i < this.outputs.length; i++) { - if (this.outputs[i]) - last = i; + for (i = this.outputs.length - 1; i >= 0; i--) { + if (this.outputs[i]) { + index = i; + break; + } } - return last + 1; + return index + 1; }; /* * Coins serialization: * version: varint - * bits: varint ((height << 1) | coinbase-flag) + * bits: uint32 (31-bit height | 1-bit coinbase-flag) * spent-field: varint size | bitfield (0=unspent, 1=spent) * outputs (repeated): - * prefix: 0x00 = varint size | raw script - * 0x01 = 20 byte pubkey hash - * 0x02 = 20 byte script hash - * 0x03 = 33 byte compressed key - * data: the data mentioned above + * compressed-script: + * prefix: 0x00 = varint size | raw script + * 0x01 = 20 byte pubkey hash + * 0x02 = 20 byte script hash + * 0x03 = 33 byte compressed key + * data: script data, dictated by the prefix * value: varint + * + * The compression below sacrifices some cpu in exchange + * for reduced size, but in some cases the use of varints + * actually increases speed (varint versions and values + * for example). We do as much compression as possible + * without sacrificing too much cpu. Value compression + * is intentionally excluded for now as it seems to be + * too much of a perf hit. Maybe when v8 optimizes + * non-smi arithmetic better we can enable it. */ /** @@ -196,8 +211,8 @@ Coins.prototype.getLength = function getLength() { Coins.prototype.toRaw = function toRaw(writer) { var p = new BufferWriter(); var height = this.height; - var length = this.getLength(); - var i, output, prefix, data, bits, fstart, flen, bit, oct; + var length = this.size(); + var i, output, bits, fstart, flen, bit, oct; // Varint version: hopefully some smartass // miner doesn't start mining `-1` versions. @@ -231,9 +246,9 @@ Coins.prototype.toRaw = function toRaw(writer) { // Fill the spent field with zeroes to avoid // allocating a buffer. We mark the spents // after rendering the final buffer. - fstart = p.written; flen = Math.ceil(length / 8); p.writeVarint(flen); + fstart = p.written; p.fill(0, flen); for (i = 0; i < length; i++) { @@ -251,44 +266,12 @@ Coins.prototype.toRaw = function toRaw(writer) { continue; } - // Prefix byte (default=0 - // for uncompressed script). - prefix = 0; - - // Attempt to compress the output scripts. - // We can _only_ ever compress them if - // they are serialized as minimaldata, as - // we need to recreate them when we read - // them. - if (output.script.isPubkeyhash(true)) { - prefix = 1; - data = output.script.code[2].data; - } else if (output.script.isScripthash()) { - prefix = 2; - data = output.script.code[1].data; - } else if (output.script.isPubkey(true)) { - prefix = 3; - data = output.script.code[0].data; - - // Try to compress the key. - data = bcoin.ec.compress(data); - - // If we can't compress it, - // just store the script. - if (!data) - prefix = 0; - } - - p.writeU8(prefix); - - if (prefix === 0) - p.writeVarBytes(output.script.toRaw()); - else - p.writeBytes(data); - + compressScript(output.script, p); p.writeVarint(output.value); } + // Render the buffer with all + // zeroes in the spent field. p = p.render(); // Mark the spents in the spent field. @@ -319,7 +302,7 @@ Coins.prototype.toRaw = function toRaw(writer) { Coins.prototype.fromRaw = function fromRaw(data, hash, index) { var p = new BufferReader(data); var i = 0; - var bits, coin, prefix, offset, size, fstart, bit, oct, spent; + var bits, coin, offset, size, fstart, flen, bit, oct, spent; this.version = p.readVarint(); @@ -334,8 +317,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { // Mark the start of the spent field and // seek past it to avoid reading a buffer. + flen = p.readVarint(); fstart = p.offset; - p.seek(p.readVarint()); + p.seek(flen); while (p.left()) { // Read a single bit out of the spent field. @@ -360,10 +344,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { } offset = p.offset; - prefix = p.readU8(); // Skip past the compressed scripts. - switch (prefix) { + switch (p.readU8()) { case 0: p.seek(p.readVarint()); break; @@ -375,7 +358,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) { p.seek(33); break; default: - assert(false, 'Bad prefix.'); + throw new Error('Bad prefix.'); } // Skip past the value. @@ -468,197 +451,6 @@ Coins.fromTX = function fromTX(tx) { return new Coins().fromTX(tx); }; -Coins.compressScript = function compressScript(script, p) { - // Prefix byte (default=0 - // for uncompressed script). - var prefix = 0; - var data; - - // Attempt to compress the output scripts. - // We can _only_ ever compress them if - // they are serialized as minimaldata, as - // we need to recreate them when we read - // them. - if (script.isPubkeyhash(true)) { - prefix = 1; - data = script.code[2].data; - } else if (script.isScripthash()) { - prefix = 2; - data = script.code[1].data; - } else if (script.isPubkey(true)) { - prefix = 3; - data = script.code[0].data; - - // Try to compress the key. - data = Coins.compressKey(data); - - // If we can't compress it, - // just store the script. - if (!data) - prefix = 0; - } - - p.writeU8(prefix); - - if (prefix === 0) - p.writeVarBytes(script.toRaw()); - else - p.writeBytes(data); -}; - -Coins.decompressScript = function decompressScript(p, script) { - var prefix = p.readU8(); - var key; - - // Decompress the script. - switch (prefix) { - case 0: - script.fromRaw(p.readVarBytes()); - break; - case 1: - script.fromPubkeyhash(p.readBytes(20)); - break; - case 2: - script.fromScripthash(p.readBytes(20)); - break; - case 3: - // Decompress the key. If this fails, - // we have database corruption! - key = Coins.decompressKey(p.readBytes(33)); - script.fromPubkey(key); - break; - default: - assert(false, 'Bad prefix.'); - } -}; - -// See: -// https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go - -Coins.compressValue = function compressValue(value) { - var exp, last; - - if (value === 0) - return 0; - - exp = 0; - while (value % 10 === 0 && exp < 9) { - value /= 10; - exp++; - } - - if (exp < 9) { - last = value % 10; - value = (value - last) / 10; - return 1 + 10 * (9 * value + last - 1) + exp; - } - - return 10 + 10 * (value - 1); -}; - -Coins.decompressValue = function decompressValue(value) { - var exp, n, last; - - if (value === 0) - return 0; - - value--; - - exp = value % 10; - value = (value - exp) / 10; - - n = 0; - if (exp < 9) { - last = value % 9; - value = (value - last) / 9; - n = value * 10 + last + 1; - } else { - n = value + 1; - } - - while (exp > 0) { - n *= 10; - exp--; - } - - return n; -}; - -/** - * Compress a public key to coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -Coins.compressKey = function compressKey(key) { - var out; - - // We can't compress it if it's not valid. - if (!bcoin.ec.publicKeyVerify(key)) - return; - - switch (key[0]) { - case 0x02: - case 0x03: - // Key is already compressed. - out = key; - break; - case 0x04: - case 0x06: - case 0x07: - // Compress the key normally. - out = bcoin.ec.publicKeyConvert(key, true); - // Store the original format (which - // may be a hybrid byte) in the hi - // 3 bits so we can restore it later. - // The hi bits being set also lets us - // know that this key was originally - // decompressed. - out[0] |= key[0] << 2; - break; - default: - throw new Error('Bad point format.'); - } - - assert(out.length === 33); - - return out; -}; - -/** - * Decompress a public key from the coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -Coins.decompressKey = function decompressKey(key) { - var format = key[0] >>> 2; - var out; - - assert(key.length === 33); - - // Hi bits are not set. This key - // is not meant to be decompressed. - if (format === 0) - return key; - - // Decompress the key, and off the - // low bits so publicKeyConvert - // actually understands it. - key[0] &= 0x03; - out = bcoin.ec.publicKeyConvert(key, false); - - // Reset the hi bits so as not to - // mutate the original buffer. - key[0] |= format << 2; - - // Set the original format, which - // may have been a hybrid prefix byte. - out[0] = format; - - return out; -}; - /** * A compressed coin is an object which defers * parsing of a coin. Say there is a transaction @@ -697,7 +489,6 @@ function CompressedCoin(offset, size, raw) { CompressedCoin.prototype.toCoin = function toCoin(coins, index) { var p = new BufferReader(this.raw); var coin = new bcoin.coin(); - var prefix, key; // Load in all necessary properties // from the parent Coins object. @@ -710,28 +501,7 @@ CompressedCoin.prototype.toCoin = function toCoin(coins, index) { // Seek to the coin's offset. p.seek(this.offset); - prefix = p.readU8(); - - // Decompress the script. - switch (prefix) { - case 0: - coin.script.fromRaw(p.readVarBytes()); - break; - case 1: - coin.script.fromPubkeyhash(p.readBytes(20)); - break; - case 2: - coin.script.fromScripthash(p.readBytes(20)); - break; - case 3: - // Decompress the key. If this fails, - // we have database corruption! - key = bcoin.ec.decompress(p.readBytes(33)); - coin.script.fromPubkey(key); - break; - default: - assert(false, 'Bad prefix.'); - } + decompressScript(p, coin.script); coin.value = p.readVarint(); @@ -749,11 +519,235 @@ CompressedCoin.prototype.toRaw = function toRaw() { }; /* - * Helpers + * Compression */ +/** + * Compress a script, write directly to the buffer. + * @param {Script} script + * @param {BufferWriter} p + */ + +function compressScript(script, p) { + var prefix = 0; + var data; + + // Attempt to compress the output scripts. + // We can _only_ ever compress them if + // they are serialized as minimaldata, as + // we need to recreate them when we read + // them. + if (script.isPubkeyhash(true)) { + prefix = 1; + data = script.code[2].data; + } else if (script.isScripthash()) { + prefix = 2; + data = script.code[1].data; + } else if (script.isPubkey(true)) { + prefix = 3; + data = script.code[0].data; + + // Try to compress the key. + data = compressKey(data); + + // If we can't compress it, + // just store the script. + if (!data) + prefix = 0; + } + + p.writeU8(prefix); + + if (prefix === 0) + p.writeVarBytes(script.toRaw()); + else + p.writeBytes(data); +} + +/** + * Decompress a script from buffer reader. + * @param {BufferReader} p + * @param {Script} script + */ + +function decompressScript(p, script) { + var key; + + // Decompress the script. + switch (p.readU8()) { + case 0: + script.fromRaw(p.readVarBytes()); + break; + case 1: + script.fromPubkeyhash(p.readBytes(20)); + break; + case 2: + script.fromScripthash(p.readBytes(20)); + break; + case 3: + // Decompress the key. If this fails, + // we have database corruption! + key = decompressKey(p.readBytes(33)); + script.fromPubkey(key); + break; + default: + throw new Error('Bad prefix.'); + } +} + +/** + * Compress value using an exponent. Takes advantage of + * the fact that many bitcoin values are divisible by 10. + * @see https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go + * @param {Amount} value + * @returns {Number} + */ + +function compressValue(value) { + var exp, last; + + if (value === 0) + return 0; + + exp = 0; + while (value % 10 === 0 && exp < 9) { + value /= 10; + exp++; + } + + if (exp < 9) { + last = value % 10; + value = (value - last) / 10; + return 1 + 10 * (9 * value + last - 1) + exp; + } + + return 10 + 10 * (value - 1); +} + +/** + * Decompress value. + * @param {Number} value - Compressed value. + * @returns {Amount} value + */ + +function decompressValue(value) { + var exp, n, last; + + if (value === 0) + return 0; + + value--; + + exp = value % 10; + value = (value - exp) / 10; + + if (exp < 9) { + last = value % 9; + value = (value - last) / 9; + n = value * 10 + last + 1; + } else { + n = value + 1; + } + + while (exp > 0) { + n *= 10; + exp--; + } + + return n; +} + +/** + * Compress a public key to coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +function compressKey(key) { + var out; + + // We can't compress it if it's not valid. + if (!bcoin.ec.publicKeyVerify(key)) + return; + + switch (key[0]) { + case 0x02: + case 0x03: + // Key is already compressed. + out = key; + break; + case 0x04: + case 0x06: + case 0x07: + // Compress the key normally. + out = bcoin.ec.publicKeyConvert(key, true); + // Store the original format (which + // may be a hybrid byte) in the hi + // 3 bits so we can restore it later. + // The hi bits being set also lets us + // know that this key was originally + // decompressed. + out[0] |= key[0] << 2; + break; + default: + throw new Error('Bad point format.'); + } + + assert(out.length === 33); + + return out; +} + +/** + * Decompress a public key from the coins compression format. + * @param {Buffer} key + * @returns {Buffer} + */ + +function decompressKey(key) { + var format = key[0] >>> 2; + var out; + + assert(key.length === 33); + + // Hi bits are not set. This key + // is not meant to be decompressed. + if (format === 0) + return key; + + // Decompress the key, and off the + // low bits so publicKeyConvert + // actually understands it. + key[0] &= 0x03; + out = bcoin.ec.publicKeyConvert(key, false); + + // Reset the hi bits so as not to + // mutate the original buffer. + key[0] |= format << 2; + + // Set the original format, which + // may have been a hybrid prefix byte. + out[0] = format; + + return out; +} + /* * Expose */ +exports = Coins; + +exports.compress = { + script: compressScript, + value: compressValue, + key: compressKey +}; + +exports.decompress = { + script: decompressScript, + value: decompressValue, + key: decompressKey +}; + module.exports = Coins; diff --git a/lib/bcoin/ec.js b/lib/bcoin/ec.js index 66732cd0..5e7c4e92 100644 --- a/lib/bcoin/ec.js +++ b/lib/bcoin/ec.js @@ -134,81 +134,6 @@ ec.publicKeyConvert = function publicKeyConvert(key, compressed) { return new Buffer(point.encode('array', compressed)); }; -/** - * Compress a public key to coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -ec.compress = function compress(key) { - var out; - - // We can't compress it if it's not valid. - if (!ec.publicKeyVerify(key)) - return; - - switch (key[0]) { - case 0x02: - case 0x03: - // Key is already compressed. - out = key; - break; - case 0x04: - case 0x06: - case 0x07: - // Compress the key normally. - out = ec.publicKeyConvert(key, true); - // Store the original format (which - // may be a hybrid byte) in the hi - // 3 bits so we can restore it later. - // The hi bits being set also lets us - // know that this key was originally - // decompressed. - out[0] |= key[0] << 2; - break; - default: - throw new Error('Bad point format.'); - } - - assert(out.length === 33); - - return out; -}; - -/** - * Decompress a public key from the coins compression format. - * @param {Buffer} key - * @returns {Buffer} - */ - -ec.decompress = function decompress(key) { - var format = key[0] >>> 2; - var out; - - assert(key.length === 33); - - // Hi bits are not set. This key - // is not meant to be decompressed. - if (format === 0) - return key; - - // Decompress the key, and off the - // low bits so publicKeyConvert - // actually understands it. - key[0] &= 0x03; - out = ec.publicKeyConvert(key, false); - - // Reset the hi bits so as not to - // mutate the original buffer. - key[0] |= format << 2; - - // Set the original format, which - // may have been a hybrid prefix byte. - out[0] = format; - - return out; -}; - /** * Create an ecdh. * @param {Buffer} pub diff --git a/lib/bcoin/protocol/network.js b/lib/bcoin/protocol/network.js index f32db12c..843a86f0 100644 --- a/lib/bcoin/protocol/network.js +++ b/lib/bcoin/protocol/network.js @@ -368,7 +368,7 @@ main.keyPrefix = { /** * {@link Address} prefixes. - * @enum {Object} + * @enum {Number} */ main.addressPrefix = {