chain: new coins compression.

This commit is contained in:
Christopher Jeffrey 2016-11-29 14:37:18 -08:00
parent 923364a70a
commit 63c42bf390
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD
8 changed files with 760 additions and 56 deletions

View File

@ -15,28 +15,30 @@ wtx = TX.fromRaw(wtx.trim(), 'hex');
var coins = Coins.fromTX(wtx);
var raw;
//raw = coins.toRaw2();
//console.log(Coins.fromRaw2(raw));
var end = bench('serialize');
for (var i = 0; i < 10000; i++)
raw = coins.toRaw();
raw = coins.toRaw2();
end(i);
var end = bench('parse');
for (var i = 0; i < 10000; i++)
Coins.fromRaw(raw);
Coins.fromRaw2(raw);
end(i);
var end = bench('parse-single');
var hash = wtx.hash('hex');
for (var i = 0; i < 10000; i++)
Coins.parseCoin(raw, hash, 5);
Coins.parseCoin2(raw, hash, 5);
end(i);
var coins = Coins.fromRaw(raw);
var coins = Coins.fromRaw2(raw);
var end = bench('get');
var j;
for (var i = 0; i < 10000; i++)
for (var j = 0; j < coins.outputs.length; j++)
coins.get(j);
coins.get2(j);
end(i * coins.outputs.length);

View File

@ -639,7 +639,7 @@ Chain.prototype.verifyInputs = co(function* verifyInputs(block, prev, state) {
// Ensure tx is not double spending an output.
if (!tx.isCoinbase()) {
if (!view.fillCoins(tx)) {
if (!view.fillCoins2(tx)) {
assert(!historical, 'BUG: Spent inputs in historical data!');
throw new VerifyError(block,
'invalid',

View File

@ -633,7 +633,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) {
coins = this.coinCache.get(hash);
if (coins)
return Coins.parseCoin(coins, hash, index);
return Coins.parseCoin2(coins, hash, index);
coins = yield this.db.get(layout.c(hash));
@ -643,7 +643,7 @@ ChainDB.prototype.getCoin = co(function* getCoin(hash, index) {
if (state === this.state)
this.coinCache.set(hash, coins);
return Coins.parseCoin(coins, hash, index);
return Coins.parseCoin2(coins, hash, index);
});
/**
@ -662,7 +662,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) {
coins = this.coinCache.get(hash);
if (coins)
return Coins.fromRaw(coins, hash);
return Coins.fromRaw2(coins, hash);
coins = yield this.db.get(layout.c(hash));
@ -672,7 +672,7 @@ ChainDB.prototype.getCoins = co(function* getCoins(hash) {
if (state === this.state)
this.coinCache.set(hash, coins);
return Coins.fromRaw(coins, hash);
return Coins.fromRaw2(coins, hash);
});
/**
@ -1672,7 +1672,7 @@ ChainDB.prototype.connectBlock = co(function* connectBlock(block, view) {
for (i = 0; i < view.length; i++) {
coins = view[i];
raw = coins.toRaw();
raw = coins.toRaw2();
if (!raw) {
this.del(layout.c(coins.hash));
this.coinCache.unpush(coins.hash);
@ -1768,7 +1768,7 @@ ChainDB.prototype.disconnectBlock = co(function* disconnectBlock(block) {
for (i = 0; i < view.length; i++) {
coins = view[i];
raw = coins.toRaw();
raw = coins.toRaw2();
if (!raw) {
this.del(layout.c(coins.hash));
this.coinCache.unpush(coins.hash);

View File

@ -102,14 +102,12 @@ Coins.prototype.add = function add(coin) {
this.coinbase = coin.coinbase;
}
if (coin.script.isUnspendable())
return;
while (this.outputs.length <= coin.index)
this.outputs.push(null);
if (coin.script.isUnspendable()) {
this.outputs[coin.index] = null;
return;
}
this.outputs[coin.index] = CoinEntry.fromCoin(coin);
};
@ -159,10 +157,62 @@ Coins.prototype.spend = function spend(index) {
return;
this.outputs[index] = null;
this.cleanup();
return coin;
};
/**
* Get a coin.
* @param {Number} index
* @returns {Coin}
*/
Coins.prototype.get2 = function get2(index) {
var coin;
if (index >= this.outputs.length)
return;
coin = this.outputs[index];
if (!coin)
return;
return coin.toCoin2(this, index);
};
/**
* Remove a coin and return it.
* @param {Number} index
* @returns {Coin}
*/
Coins.prototype.spend2 = function spend2(index) {
var coin = this.get2(index);
if (!coin)
return;
this.outputs[index] = null;
this.cleanup();
return coin;
};
/**
* Cleanup spent outputs.
*/
Coins.prototype.cleanup = function cleanup() {
var len = this.outputs.length;
while (len > 0 && !this.outputs[len - 1])
len--;
this.outputs.length = len;
};
/**
* Count up to the last available index.
* @returns {Number}
@ -294,7 +344,7 @@ Coins.prototype.toRaw = function toRaw() {
* @returns {Object} A "naked" coins object.
*/
Coins.prototype.fromRaw = function fromRaw(data, hash, index) {
Coins.prototype.fromRaw = function fromRaw(data, hash) {
var br = new BufferReader(data);
var pos = 0;
var bits, len, start, bit, oct, spent, coin;
@ -395,7 +445,7 @@ Coins.parseCoin = function parseCoin(data, hash, index) {
}
// Skip past the compressed coin.
skipCoin(br);
decompress.skip(br);
pos++;
}
};
@ -411,6 +461,227 @@ Coins.fromRaw = function fromRaw(data, hash) {
return new Coins().fromRaw(data, hash);
};
/**
* Serialize the coins object.
* @param {TX|Coins} tx
* @returns {Buffer}
*/
Coins.prototype.toRaw2 = function toRaw2() {
var bw = new BufferWriter();
var len = this.outputs.length;
var first = len > 0 && this.outputs[0];
var second = len > 1 && this.outputs[1];
var size = 0;
var nonzero = 0;
var i, j, code, ch, output;
// Return nothing if we're fully spent.
if (len === 0)
return;
// Calculate number of unspents and spent field size.
// size = number of bytes required for the bit field.
// nonzero = number of non-zero bytes required.
for (i = 0; 2 + i * 8 < len; i++) {
for (j = 0; j < 8 && 2 + i * 8 + j < len; j++) {
if (this.outputs[2 + i * 8 + j]) {
size = i + 1;
nonzero++;
break;
}
}
}
if (!first && !second)
nonzero -= 1;
// Calculate header code.
code = 8 * nonzero;
if (this.coinbase)
code += 1;
if (first)
code += 2;
if (second)
code += 4;
// Write headers.
bw.writeVarint(this.version);
bw.writeU32(this.height);
bw.writeVarint(code);
// Write the spent field.
for (i = 0; i < size; i++) {
ch = 0;
for (j = 0; j < 8 && 2 + i * 8 + j < len; j++) {
if (this.outputs[2 + i * 8 + j])
ch |= 1 << j;
}
bw.writeU8(ch);
}
// Write the compressed outputs.
for (i = 0; i < len; i++) {
output = this.outputs[i];
if (!output)
continue;
output.toWriter2(bw);
}
return bw.render();
};
/**
* Parse serialized coins.
* @param {Buffer} data
* @param {Hash} hash
* @returns {Object} A "naked" coins object.
*/
Coins.prototype.fromRaw2 = function fromRaw2(data, hash) {
var br = new BufferReader(data);
var i, code, field, nonzero, ch, unspent, coin;
this.hash = hash;
// Read headers.
this.version = br.readVarint();
this.height = br.readU32();
code = br.readVarint();
this.coinbase = (code & 1) !== 0;
// Setup spent field.
field = [
(code & 2) !== 0,
(code & 4) !== 0
];
// Recalculate number of non-zero bytes.
nonzero = code / 8 | 0;
if ((code & 6) === 0)
nonzero += 1;
// Read spent field.
while (nonzero > 0) {
ch = br.readU8();
for (i = 0; i < 8; i++) {
unspent = (ch & (1 << i)) !== 0;
field.push(unspent);
}
if (ch !== 0)
nonzero--;
}
// Read outputs.
for (i = 0; i < field.length; i++) {
if (!field[i]) {
this.outputs.push(null);
continue;
}
// Store the offset and size
// in the compressed coin object.
coin = CoinEntry.fromReader2(br);
this.outputs.push(coin);
}
this.cleanup();
return this;
};
/**
* Parse a single serialized coin.
* @param {Buffer} data
* @param {Hash} hash
* @param {Number} index
* @returns {Coin}
*/
Coins.parseCoin2 = function parseCoin2(data, hash, index) {
var br = new BufferReader(data);
var coin = new Coin();
var i, code, field, nonzero, ch, unspent;
coin.hash = hash;
coin.index = index;
// Read headers.
coin.version = br.readVarint();
coin.height = br.readU32();
code = br.readVarint();
coin.coinbase = (code & 1) !== 0;
// Setup spent field.
field = [
(code & 2) !== 0,
(code & 4) !== 0
];
// Recalculate number of non-zero bytes.
nonzero = code / 8 | 0;
if ((code & 6) === 0)
nonzero += 1;
// Read spent field.
while (nonzero > 0 && field.length <= index) {
ch = br.readU8();
for (i = 0; i < 8; i++) {
unspent = (ch & (1 << i)) !== 0;
field.push(unspent);
}
if (ch !== 0)
nonzero--;
}
if (field.length <= index)
return;
while (nonzero > 0) {
if (br.readU8() !== 0)
nonzero--;
}
// Read outputs.
for (i = 0; i < field.length; i++) {
if (i === index) {
if (!field[i])
return;
// Read compressed output.
decompress.output2(coin, br);
break;
}
if (!field[i])
continue;
decompress.skip2(br);
}
return coin;
};
/**
* Instantiate coins from a serialized Buffer.
* @param {Buffer} data
* @param {Hash} hash - Transaction hash.
* @returns {Coins}
*/
Coins.fromRaw2 = function fromRaw2(data, hash) {
return new Coins().fromRaw2(data, hash);
};
/**
* Inject properties from tx.
* @private
@ -436,6 +707,8 @@ Coins.prototype.fromTX = function fromTX(tx) {
this.outputs.push(CoinEntry.fromTX(tx, i));
}
this.cleanup();
return this;
};
@ -505,9 +778,42 @@ CoinEntry.prototype.toCoin = function toCoin(coins, index) {
// Seek to the coin's offset.
br.seek(this.offset);
decompress.script(coin.script, br);
decompress.output(coin, br);
coin.value = br.readVarint();
return coin;
};
/**
* Parse the deferred data and return a Coin.
* @param {Coins} coins
* @param {Number} index
* @returns {Coin}
*/
CoinEntry.prototype.toCoin2 = function toCoin2(coins, index) {
var coin = new Coin();
var br;
// Load in all necessary properties
// from the parent Coins object.
coin.version = coins.version;
coin.coinbase = coins.coinbase;
coin.height = coins.height;
coin.hash = coins.hash;
coin.index = index;
if (this.output) {
coin.script = this.output.script;
coin.value = this.output.value;
return coin;
}
br = new BufferReader(this.raw);
// Seek to the coin's offset.
br.seek(this.offset);
decompress.output2(coin, br);
return coin;
};
@ -521,8 +827,31 @@ CoinEntry.prototype.toWriter = function toWriter(bw) {
var raw;
if (this.output) {
compress.script(this.output.script, bw);
bw.writeVarint(this.output.value);
compress.output(this.output, bw);
return;
}
assert(this.raw);
// If we read this coin from the db and
// didn't use it, it's still in its
// compressed form. Just write it back
// as a buffer for speed.
raw = this.raw.slice(this.offset, this.offset + this.size);
bw.writeBytes(raw);
};
/**
* Slice off the part of the buffer
* relevant to this particular coin.
*/
CoinEntry.prototype.toWriter2 = function toWriter2(bw) {
var raw;
if (this.output) {
compress.output2(this.output, bw);
return;
}
@ -546,7 +875,21 @@ CoinEntry.prototype.toWriter = function toWriter(bw) {
CoinEntry.fromReader = function fromReader(br) {
var entry = new CoinEntry();
entry.offset = br.offset;
entry.size = skipCoin(br);
entry.size = decompress.skip(br);
entry.raw = br.data;
return entry;
};
/**
* Instantiate compressed coin from reader.
* @param {BufferReader} br
* @returns {CoinEntry}
*/
CoinEntry.fromReader2 = function fromReader2(br) {
var entry = new CoinEntry();
entry.offset = br.offset;
entry.size = decompress.skip2(br);
entry.raw = br.data;
return entry;
};
@ -578,35 +921,6 @@ CoinEntry.fromCoin = function fromCoin(coin) {
return entry;
};
/*
* Helpers
*/
function skipCoin(br) {
var start = br.offset;
// Skip past the compressed scripts.
switch (br.readU8()) {
case 0:
br.seek(br.readVarint());
break;
case 1:
case 2:
br.seek(20);
break;
case 3:
br.seek(33);
break;
default:
throw new Error('Bad prefix.');
}
// Skip past the value.
br.skipVarint();
return br.offset - start;
}
/*
* Expose
*/

View File

@ -70,6 +70,22 @@ CoinView.prototype.get = function get(hash, index) {
return coins.get(index);
};
/**
* Get a coin.
* @param {Hash} hash
* @param {Number} index
* @returns {Coin}
*/
CoinView.prototype.get2 = function get2(hash, index) {
var coins = this.coins[hash];
if (!coins)
return;
return coins.get2(index);
};
/**
* Test whether the collection has a coin.
* @param {Hash} hash
@ -102,6 +118,22 @@ CoinView.prototype.spend = function spend(hash, index) {
return coins.spend(index);
};
/**
* Remove a coin and return it.
* @param {Hash} hash
* @param {Number} index
* @returns {Coin}
*/
CoinView.prototype.spend2 = function spend2(hash, index) {
var coins = this.coins[hash];
if (!coins)
return;
return coins.spend2(index);
};
/**
* Fill transaction(s) with coins.
* @param {TX} tx
@ -122,6 +154,26 @@ CoinView.prototype.fillCoins = function fillCoins(tx) {
return true;
};
/**
* Fill transaction(s) with coins.
* @param {TX} tx
* @returns {Boolean} True if all inputs were filled.
*/
CoinView.prototype.fillCoins2 = function fillCoins2(tx) {
var i, input, prevout;
for (i = 0; i < tx.inputs.length; i++) {
input = tx.inputs[i];
prevout = input.prevout;
input.coin = this.spend2(prevout.hash, prevout.index);
if (!input.coin)
return false;
}
return true;
};
/**
* Convert collection to an array.
* @returns {Coins[]}

View File

@ -103,6 +103,218 @@ function decompressScript(script, br) {
return script;
}
/**
* Compress a script, write directly to the buffer.
* @param {Script} script
* @param {BufferWriter} bw
*/
function compressScript2(script, bw) {
var data;
// Attempt to compress the output scripts.
// We can _only_ ever compress them if
// they are serialized as minimaldata, as
// we need to recreate them when we read
// them.
// P2PKH -> 0 | key-hash
// Saves 5 bytes.
if (script.isPubkeyhash(true)) {
data = script.code[2].data;
bw.writeU8(0);
bw.writeBytes(data);
return bw;
}
// P2SH -> 1 | script-hash
// Saves 3 bytes.
if (script.isScripthash()) {
data = script.code[1].data;
bw.writeU8(1);
bw.writeBytes(data);
return bw;
}
// P2PK -> 2-5 | compressed-key
// Only works if the key is valid.
// Saves up to 35 bytes.
if (script.isPubkey(true)) {
data = script.code[0].data;
if (publicKeyVerify(data)) {
data = compressKey2(data);
bw.writeBytes(data);
return bw;
}
}
// Raw -> varlen + 6 | script
bw.writeVarint(script.raw.length + 6);
bw.writeBytes(script.raw);
return bw;
}
/**
* Decompress a script from buffer reader.
* @param {Script} script
* @param {BufferReader} br
*/
function decompressScript2(script, br) {
var size, data;
// Decompress the script.
switch (br.readU8()) {
case 0:
data = br.readBytes(20, true);
script.fromPubkeyhash(data);
break;
case 1:
data = br.readBytes(20, true);
script.fromScripthash(data);
break;
case 2:
case 3:
case 4:
case 5:
br.offset -= 1;
data = br.readBytes(33, true);
// Decompress the key. If this fails,
// we have database corruption!
data = decompressKey2(data);
script.fromPubkey(data);
break;
default:
br.offset -= 1;
size = br.readVarint() - 6;
if (size > 10000) {
// This violates consensus rules.
// We don't need to read it.
script.fromUnspendable();
p.seek(size);
} else {
data = br.readBytes(size);
script.fromRaw(data);
}
break;
}
return script;
}
/**
* Compress an output.
* @param {Output|Coin} output
* @param {BufferWriter} bw
*/
function compressOutput(output, bw) {
compressScript(output.script, bw);
bw.writeVarint(output.value);
return bw;
}
/**
* Decompress a script from buffer reader.
* @param {Output|Coin} output
* @param {BufferReader} br
*/
function decompressOutput(output, br) {
decompressScript(output.script, br);
output.value = br.readVarint();
return output;
}
/**
* Skip past a compressed output.
* @param {BufferWriter} bw
* @returns {Number}
*/
function skipOutput(br) {
var start = br.offset;
// Skip past the compressed scripts.
switch (br.readU8()) {
case 0:
br.seek(br.readVarint());
break;
case 1:
case 2:
br.seek(20);
break;
case 3:
br.seek(33);
break;
default:
throw new Error('Bad prefix.');
}
// Skip past the value.
br.skipVarint();
return br.offset - start;
}
/**
* Compress an output.
* @param {Output|Coin} output
* @param {BufferWriter} bw
*/
function compressOutput2(output, bw) {
compressScript2(output.script, bw);
bw.writeVarint(output.value);
return bw;
}
/**
* Decompress a script from buffer reader.
* @param {Output|Coin} output
* @param {BufferReader} br
*/
function decompressOutput2(output, br) {
decompressScript2(output.script, br);
output.value = br.readVarint();
return output;
}
/**
* Skip past a compressed output.
* @param {BufferWriter} bw
* @returns {Number}
*/
function skipOutput2(br) {
var start = br.offset;
// Skip past the compressed scripts.
switch (br.readU8()) {
case 0:
case 1:
br.seek(20);
break;
case 2:
case 3:
case 4:
case 5:
br.seek(32);
break;
default:
br.offset -= 1;
br.seek(br.readVarint() - 6);
break;
}
// Skip past the value.
br.skipVarint();
return br.offset - start;
}
/**
* Compress value using an exponent. Takes advantage of
* the fact that many bitcoin values are divisible by 10.
@ -236,18 +448,120 @@ function decompressKey(key) {
return out;
}
/**
* Verify a public key (no hybrid keys allowed).
* @param {Buffer} key
* @returns {Boolean}
*/
function publicKeyVerify(key) {
if (key.length === 0)
return false;
switch (key[0]) {
case 0x02:
case 0x03:
return key.length === 33;
case 0x04:
if (key.length !== 65)
return false;
return ec.publicKeyVerify(key);
default:
return false;
}
}
/**
* Compress a public key to coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
function compressKey2(key) {
var out;
switch (key[0]) {
case 0x02:
case 0x03:
// Key is already compressed.
out = key;
break;
case 0x04:
// Compress the key normally.
out = ec.publicKeyConvert(key, true);
// Store the oddness.
out[0] = 0x04 | (key[64] & 0x01);
break;
default:
throw new Error('Bad point format.');
}
assert(out.length === 33);
return out;
}
/**
* Decompress a public key from the coins compression format.
* @param {Buffer} key
* @returns {Buffer}
*/
function decompressKey2(key) {
var format = key[0];
var out;
assert(key.length === 33);
switch (format) {
case 0x02:
case 0x03:
return key;
case 0x04:
key[0] = 0x02;
break;
case 0x05:
key[0] = 0x03;
break;
default:
throw new Error('Bad point format.');
}
// Decompress the key, and off the
// low bits so publicKeyConvert
// actually understands it.
out = ec.publicKeyConvert(key, false);
// Reset the first byte so as not to
// mutate the original buffer.
key[0] = format;
return out;
}
/*
* Expose
*/
exports.compress = {
output: compressOutput,
output2: compressOutput2,
script: compressScript,
script2: compressScript2,
value: compressValue,
key: compressKey
key: compressKey,
key2: compressKey2
};
exports.decompress = {
output: decompressOutput,
output2: decompressOutput2,
skip: skipOutput,
skip2: skipOutput2,
script: decompressScript,
script2: decompressScript2,
value: decompressValue,
key: decompressKey
key: decompressKey,
key2: decompressKey
};

View File

@ -1500,6 +1500,28 @@ Script.isCode = function isCode(raw) {
return true;
};
/**
* Inject properties from a unspendable script.
* @private
* @param {Buffer} key
*/
Script.prototype.fromUnspendable = function fromUnspendable() {
this.raw = new Buffer(1);
this.raw[0] = opcodes.OP_RETURN;
this.code.push(new Opcode(opcodes.OP_RETURN));
return this;
};
/**
* Create an unspendable script.
* @returns {Script}
*/
Script.fromUnspendable = function fromUnspendable() {
return new Script().fromUnspendable();
};
/**
* Inject properties from a pay-to-pubkey script.
* @private

View File

@ -142,7 +142,7 @@ BufferReader.prototype.destroy = function destroy() {
BufferReader.prototype.readU8 = function readU8() {
var ret;
assert(this.offset + 1 <= this.data.length);
ret = this.data.readUInt8(this.offset, true);
ret = this.data[this.offset];
this.offset += 1;
return ret;
};