more coin serialization.

This commit is contained in:
Christopher Jeffrey 2016-07-07 02:34:17 -07:00
parent e9bd890c8b
commit c9c38e860b
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD
5 changed files with 337 additions and 347 deletions

View File

@ -322,7 +322,7 @@ Chain.prototype.preload = function preload(callback) {
size = 0;
stream.on('response', function(res) {
var height = Math.floor(res.headers['content-length'] / 80);
var height = Math.floor(+res.headers['content-length'] / 80);
if (res.statusCode >= 400) {
stream.destroy();
return callback(new Error('Bad response code: ' + res.statusCode));

View File

@ -262,6 +262,77 @@ Coin.fromRaw = function fromRaw(data, enc) {
return new Coin().fromRaw(data);
};
/**
* Serialize the coin to its compressed form.
* @param {String?} enc - Encoding, can be `'hex'` or null.
* @returns {Buffer|String}
*/
Coin.prototype.toCompressed = function toCompressed(writer) {
var compress = bcoin.coins.compress;
var p = bcoin.writer(writer);
var height = this.height;
var bits;
if (height === -1)
height = 0x7fffffff;
bits = height << 1;
if (this.coinbase)
bits |= 1;
if (bits < 0)
bits += 0x100000000;
p.writeVarint(this.version);
p.writeU32(bits);
p.writeVarint(this.value);
compress.script(this.script, p);
if (!writer)
p = p.render();
return p;
};
/**
* Inject properties from compressed serialized data.
* @private
* @param {Buffer} data
*/
Coin.prototype.fromCompressed = function fromCompressed(data) {
var decompress = bcoin.coins.decompress;
var p = bcoin.reader(data);
var bits;
this.version = p.readVarint();
bits = p.readU32();
this.height = bits >>> 1;
this.coinbase = (bits & 1) !== 0;
this.value = p.readVarint();
decompress.script(p, this.script);
if (this.height === 0x7fffffff)
this.height = -1;
return this;
};
/**
* Instantiate an coin from a serialized Buffer.
* @param {Buffer} data
* @param {String?} enc - Encoding, can be `'hex'` or null.
* @returns {Coin}
*/
Coin.fromCompressed = function fromCompressed(data, enc) {
if (typeof data === 'string')
data = new Buffer(data, enc);
return new Coin().fromCompressed(data);
};
/**
* Serialize the coin to an "extended" format,
* including both the hash and the index.

View File

@ -152,6 +152,9 @@ Coins.prototype.count = function count(index) {
Coins.prototype.spend = function spend(index) {
var coin = this.get(index);
if (!coin)
return;
this.outputs[index] = null;
return coin;
};
@ -161,30 +164,42 @@ Coins.prototype.spend = function spend(index) {
* @returns {Number}
*/
Coins.prototype.getLength = function getLength() {
var last = -1;
Coins.prototype.size = function size() {
var index = -1;
var i;
for (i = 0; i < this.outputs.length; i++) {
if (this.outputs[i])
last = i;
for (i = this.outputs.length - 1; i >= 0; i--) {
if (this.outputs[i]) {
index = i;
break;
}
}
return last + 1;
return index + 1;
};
/*
* Coins serialization:
* version: varint
* bits: varint ((height << 1) | coinbase-flag)
* bits: uint32 (31-bit height | 1-bit coinbase-flag)
* spent-field: varint size | bitfield (0=unspent, 1=spent)
* outputs (repeated):
* prefix: 0x00 = varint size | raw script
* 0x01 = 20 byte pubkey hash
* 0x02 = 20 byte script hash
* 0x03 = 33 byte compressed key
* data: the data mentioned above
* compressed-script:
* prefix: 0x00 = varint size | raw script
* 0x01 = 20 byte pubkey hash
* 0x02 = 20 byte script hash
* 0x03 = 33 byte compressed key
* data: script data, dictated by the prefix
* value: varint
*
* The compression below sacrifices some cpu in exchange
* for reduced size, but in some cases the use of varints
* actually increases speed (varint versions and values
* for example). We do as much compression as possible
* without sacrificing too much cpu. Value compression
* is intentionally excluded for now as it seems to be
* too much of a perf hit. Maybe when v8 optimizes
* non-smi arithmetic better we can enable it.
*/
/**
@ -196,8 +211,8 @@ Coins.prototype.getLength = function getLength() {
Coins.prototype.toRaw = function toRaw(writer) {
var p = new BufferWriter();
var height = this.height;
var length = this.getLength();
var i, output, prefix, data, bits, fstart, flen, bit, oct;
var length = this.size();
var i, output, bits, fstart, flen, bit, oct;
// Varint version: hopefully some smartass
// miner doesn't start mining `-1` versions.
@ -231,9 +246,9 @@ Coins.prototype.toRaw = function toRaw(writer) {
// Fill the spent field with zeroes to avoid
// allocating a buffer. We mark the spents
// after rendering the final buffer.
fstart = p.written;
flen = Math.ceil(length / 8);
p.writeVarint(flen);
fstart = p.written;
p.fill(0, flen);
for (i = 0; i < length; i++) {
@ -251,44 +266,12 @@ Coins.prototype.toRaw = function toRaw(writer) {
continue;
}
// Prefix byte (default=0
// for uncompressed script).
prefix = 0;
// Attempt to compress the output scripts.
// We can _only_ ever compress them if
// they are serialized as minimaldata, as
// we need to recreate them when we read
// them.
if (output.script.isPubkeyhash(true)) {
prefix = 1;
data = output.script.code[2].data;
} else if (output.script.isScripthash()) {
prefix = 2;
data = output.script.code[1].data;
} else if (output.script.isPubkey(true)) {
prefix = 3;
data = output.script.code[0].data;
// Try to compress the key.
data = bcoin.ec.compress(data);
// If we can't compress it,
// just store the script.
if (!data)
prefix = 0;
}
p.writeU8(prefix);
if (prefix === 0)
p.writeVarBytes(output.script.toRaw());
else
p.writeBytes(data);
compressScript(output.script, p);
p.writeVarint(output.value);
}
// Render the buffer with all
// zeroes in the spent field.
p = p.render();
// Mark the spents in the spent field.
@ -319,7 +302,7 @@ Coins.prototype.toRaw = function toRaw(writer) {
Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
var p = new BufferReader(data);
var i = 0;
var bits, coin, prefix, offset, size, fstart, bit, oct, spent;
var bits, coin, offset, size, fstart, flen, bit, oct, spent;
this.version = p.readVarint();
@ -334,8 +317,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
// Mark the start of the spent field and
// seek past it to avoid reading a buffer.
flen = p.readVarint();
fstart = p.offset;
p.seek(p.readVarint());
p.seek(flen);
while (p.left()) {
// Read a single bit out of the spent field.
@ -360,10 +344,9 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
}
offset = p.offset;
prefix = p.readU8();
// Skip past the compressed scripts.
switch (prefix) {
switch (p.readU8()) {
case 0:
p.seek(p.readVarint());
break;
@ -375,7 +358,7 @@ Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
p.seek(33);
break;
default:
assert(false, 'Bad prefix.');
throw new Error('Bad prefix.');
}
// Skip past the value.
@ -468,197 +451,6 @@ Coins.fromTX = function fromTX(tx) {
return new Coins().fromTX(tx);
};
Coins.compressScript = function compressScript(script, p) {
// Prefix byte (default=0
// for uncompressed script).
var prefix = 0;
var data;
// Attempt to compress the output scripts.
// We can _only_ ever compress them if
// they are serialized as minimaldata, as
// we need to recreate them when we read
// them.
if (script.isPubkeyhash(true)) {
prefix = 1;
data = script.code[2].data;
} else if (script.isScripthash()) {
prefix = 2;
data = script.code[1].data;
} else if (script.isPubkey(true)) {
prefix = 3;
data = script.code[0].data;
// Try to compress the key.
data = Coins.compressKey(data);
// If we can't compress it,
// just store the script.
if (!data)
prefix = 0;
}
p.writeU8(prefix);
if (prefix === 0)
p.writeVarBytes(script.toRaw());
else
p.writeBytes(data);
};
Coins.decompressScript = function decompressScript(p, script) {
var prefix = p.readU8();
var key;
// Decompress the script.
switch (prefix) {
case 0:
script.fromRaw(p.readVarBytes());
break;
case 1:
script.fromPubkeyhash(p.readBytes(20));
break;
case 2:
script.fromScripthash(p.readBytes(20));
break;
case 3:
// Decompress the key. If this fails,
// we have database corruption!
key = Coins.decompressKey(p.readBytes(33));
script.fromPubkey(key);
break;
default:
assert(false, 'Bad prefix.');
}
};
// See:
// https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
Coins.compressValue = function compressValue(value) {
var exp, last;
if (value === 0)
return 0;
exp = 0;
while (value % 10 === 0 && exp < 9) {
value /= 10;
exp++;
}
if (exp < 9) {
last = value % 10;
value = (value - last) / 10;
return 1 + 10 * (9 * value + last - 1) + exp;
}
return 10 + 10 * (value - 1);
};
Coins.decompressValue = function decompressValue(value) {
var exp, n, last;
if (value === 0)
return 0;
value--;
exp = value % 10;
value = (value - exp) / 10;
n = 0;
if (exp < 9) {
last = value % 9;
value = (value - last) / 9;
n = value * 10 + last + 1;
} else {
n = value + 1;
}
while (exp > 0) {
n *= 10;
exp--;
}
return n;
};
/**
* Compress a public key to coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
Coins.compressKey = function compressKey(key) {
var out;
// We can't compress it if it's not valid.
if (!bcoin.ec.publicKeyVerify(key))
return;
switch (key[0]) {
case 0x02:
case 0x03:
// Key is already compressed.
out = key;
break;
case 0x04:
case 0x06:
case 0x07:
// Compress the key normally.
out = bcoin.ec.publicKeyConvert(key, true);
// Store the original format (which
// may be a hybrid byte) in the hi
// 3 bits so we can restore it later.
// The hi bits being set also lets us
// know that this key was originally
// decompressed.
out[0] |= key[0] << 2;
break;
default:
throw new Error('Bad point format.');
}
assert(out.length === 33);
return out;
};
/**
* Decompress a public key from the coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
Coins.decompressKey = function decompressKey(key) {
var format = key[0] >>> 2;
var out;
assert(key.length === 33);
// Hi bits are not set. This key
// is not meant to be decompressed.
if (format === 0)
return key;
// Decompress the key, and off the
// low bits so publicKeyConvert
// actually understands it.
key[0] &= 0x03;
out = bcoin.ec.publicKeyConvert(key, false);
// Reset the hi bits so as not to
// mutate the original buffer.
key[0] |= format << 2;
// Set the original format, which
// may have been a hybrid prefix byte.
out[0] = format;
return out;
};
/**
* A compressed coin is an object which defers
* parsing of a coin. Say there is a transaction
@ -697,7 +489,6 @@ function CompressedCoin(offset, size, raw) {
CompressedCoin.prototype.toCoin = function toCoin(coins, index) {
var p = new BufferReader(this.raw);
var coin = new bcoin.coin();
var prefix, key;
// Load in all necessary properties
// from the parent Coins object.
@ -710,28 +501,7 @@ CompressedCoin.prototype.toCoin = function toCoin(coins, index) {
// Seek to the coin's offset.
p.seek(this.offset);
prefix = p.readU8();
// Decompress the script.
switch (prefix) {
case 0:
coin.script.fromRaw(p.readVarBytes());
break;
case 1:
coin.script.fromPubkeyhash(p.readBytes(20));
break;
case 2:
coin.script.fromScripthash(p.readBytes(20));
break;
case 3:
// Decompress the key. If this fails,
// we have database corruption!
key = bcoin.ec.decompress(p.readBytes(33));
coin.script.fromPubkey(key);
break;
default:
assert(false, 'Bad prefix.');
}
decompressScript(p, coin.script);
coin.value = p.readVarint();
@ -749,11 +519,235 @@ CompressedCoin.prototype.toRaw = function toRaw() {
};
/*
* Helpers
* Compression
*/
/**
* Compress a script, write directly to the buffer.
* @param {Script} script
* @param {BufferWriter} p
*/
function compressScript(script, p) {
var prefix = 0;
var data;
// Attempt to compress the output scripts.
// We can _only_ ever compress them if
// they are serialized as minimaldata, as
// we need to recreate them when we read
// them.
if (script.isPubkeyhash(true)) {
prefix = 1;
data = script.code[2].data;
} else if (script.isScripthash()) {
prefix = 2;
data = script.code[1].data;
} else if (script.isPubkey(true)) {
prefix = 3;
data = script.code[0].data;
// Try to compress the key.
data = compressKey(data);
// If we can't compress it,
// just store the script.
if (!data)
prefix = 0;
}
p.writeU8(prefix);
if (prefix === 0)
p.writeVarBytes(script.toRaw());
else
p.writeBytes(data);
}
/**
* Decompress a script from buffer reader.
* @param {BufferReader} p
* @param {Script} script
*/
function decompressScript(p, script) {
var key;
// Decompress the script.
switch (p.readU8()) {
case 0:
script.fromRaw(p.readVarBytes());
break;
case 1:
script.fromPubkeyhash(p.readBytes(20));
break;
case 2:
script.fromScripthash(p.readBytes(20));
break;
case 3:
// Decompress the key. If this fails,
// we have database corruption!
key = decompressKey(p.readBytes(33));
script.fromPubkey(key);
break;
default:
throw new Error('Bad prefix.');
}
}
/**
* Compress value using an exponent. Takes advantage of
* the fact that many bitcoin values are divisible by 10.
* @see https://github.com/btcsuite/btcd/blob/master/blockchain/compress.go
* @param {Amount} value
* @returns {Number}
*/
function compressValue(value) {
var exp, last;
if (value === 0)
return 0;
exp = 0;
while (value % 10 === 0 && exp < 9) {
value /= 10;
exp++;
}
if (exp < 9) {
last = value % 10;
value = (value - last) / 10;
return 1 + 10 * (9 * value + last - 1) + exp;
}
return 10 + 10 * (value - 1);
}
/**
* Decompress value.
* @param {Number} value - Compressed value.
* @returns {Amount} value
*/
function decompressValue(value) {
var exp, n, last;
if (value === 0)
return 0;
value--;
exp = value % 10;
value = (value - exp) / 10;
if (exp < 9) {
last = value % 9;
value = (value - last) / 9;
n = value * 10 + last + 1;
} else {
n = value + 1;
}
while (exp > 0) {
n *= 10;
exp--;
}
return n;
}
/**
* Compress a public key to coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
function compressKey(key) {
var out;
// We can't compress it if it's not valid.
if (!bcoin.ec.publicKeyVerify(key))
return;
switch (key[0]) {
case 0x02:
case 0x03:
// Key is already compressed.
out = key;
break;
case 0x04:
case 0x06:
case 0x07:
// Compress the key normally.
out = bcoin.ec.publicKeyConvert(key, true);
// Store the original format (which
// may be a hybrid byte) in the hi
// 3 bits so we can restore it later.
// The hi bits being set also lets us
// know that this key was originally
// decompressed.
out[0] |= key[0] << 2;
break;
default:
throw new Error('Bad point format.');
}
assert(out.length === 33);
return out;
}
/**
* Decompress a public key from the coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
function decompressKey(key) {
var format = key[0] >>> 2;
var out;
assert(key.length === 33);
// Hi bits are not set. This key
// is not meant to be decompressed.
if (format === 0)
return key;
// Decompress the key, and off the
// low bits so publicKeyConvert
// actually understands it.
key[0] &= 0x03;
out = bcoin.ec.publicKeyConvert(key, false);
// Reset the hi bits so as not to
// mutate the original buffer.
key[0] |= format << 2;
// Set the original format, which
// may have been a hybrid prefix byte.
out[0] = format;
return out;
}
/*
* Expose
*/
exports = Coins;
exports.compress = {
script: compressScript,
value: compressValue,
key: compressKey
};
exports.decompress = {
script: decompressScript,
value: decompressValue,
key: decompressKey
};
module.exports = Coins;

View File

@ -134,81 +134,6 @@ ec.publicKeyConvert = function publicKeyConvert(key, compressed) {
return new Buffer(point.encode('array', compressed));
};
/**
* Compress a public key to coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
ec.compress = function compress(key) {
var out;
// We can't compress it if it's not valid.
if (!ec.publicKeyVerify(key))
return;
switch (key[0]) {
case 0x02:
case 0x03:
// Key is already compressed.
out = key;
break;
case 0x04:
case 0x06:
case 0x07:
// Compress the key normally.
out = ec.publicKeyConvert(key, true);
// Store the original format (which
// may be a hybrid byte) in the hi
// 3 bits so we can restore it later.
// The hi bits being set also lets us
// know that this key was originally
// decompressed.
out[0] |= key[0] << 2;
break;
default:
throw new Error('Bad point format.');
}
assert(out.length === 33);
return out;
};
/**
* Decompress a public key from the coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
ec.decompress = function decompress(key) {
var format = key[0] >>> 2;
var out;
assert(key.length === 33);
// Hi bits are not set. This key
// is not meant to be decompressed.
if (format === 0)
return key;
// Decompress the key, and off the
// low bits so publicKeyConvert
// actually understands it.
key[0] &= 0x03;
out = ec.publicKeyConvert(key, false);
// Reset the hi bits so as not to
// mutate the original buffer.
key[0] |= format << 2;
// Set the original format, which
// may have been a hybrid prefix byte.
out[0] = format;
return out;
};
/**
* Create an ecdh.
* @param {Buffer} pub

View File

@ -368,7 +368,7 @@ main.keyPrefix = {
/**
* {@link Address} prefixes.
* @enum {Object}
* @enum {Number}
*/
main.addressPrefix = {