block tests. fix spv and bloom filters.

This commit is contained in:
Christopher Jeffrey 2016-05-23 19:31:15 -07:00
parent 499d0c6a7e
commit 9c89630490
No known key found for this signature in database
GPG Key ID: 8962AB9DE6666BBD
7 changed files with 39144 additions and 115 deletions

View File

@ -567,7 +567,7 @@ Block.prototype.getPrevout = function getPrevout() {
Block.prototype.inspect = function inspect() {
return {
type: 'block',
hash: utils.revHex(this.hash('hex')),
hash: this.rhash,
height: this.height,
size: this.getSize(),
virtualSize: this.getVirtualSize(),

View File

@ -38,8 +38,8 @@ function Bloom(size, n, tweak, update) {
this.filter = size;
this.size = this.filter.length * 8;
} else {
this.filter = new Buffer(Math.ceil(size / 8));
this.size = size;
this.size = size - (size % 8);
this.filter = new Buffer(this.size / 8);
this.reset();
}
@ -83,18 +83,14 @@ Bloom.prototype.reset = function reset() {
*/
Bloom.prototype.add = function add(val, enc) {
var i, bits, pos, bit;
var i, index;
if (typeof val === 'string')
val = new Buffer(val, enc);
for (i = 0; i < this.n; i++) {
bits = this.hash(val, i);
pos = (bits >>> 5) * 4;
bits &= 0x1f;
bit = bits % 8;
pos += (bits - bit) / 8;
this.filter[pos] |= 1 << bit;
index = this.hash(val, i);
this.filter[index >>> 3] |= 1 << (7 & index);
}
};
@ -106,18 +102,14 @@ Bloom.prototype.add = function add(val, enc) {
*/
Bloom.prototype.test = function test(val, enc) {
var i, bits, pos, bit, oct;
var i, index;
if (typeof val === 'string')
val = new Buffer(val, enc);
for (i = 0; i < this.n; i++) {
bits = this.hash(val, i);
pos = (bits >>> 5) * 4;
bits &= 0x1f;
bit = bits % 8;
pos += (bits - bit) / 8;
if ((this.filter[pos] & (1 << bit)) === 0)
index = this.hash(val, i);
if ((this.filter[index >>> 3] & (1 << (7 & index))) === 0)
return false;
}
@ -134,20 +126,16 @@ Bloom.prototype.test = function test(val, enc) {
Bloom.prototype.added = function added(val, enc) {
var ret = false;
var i, bits, pos, bit, oct;
var i, index;
if (typeof val === 'string')
val = new Buffer(val, enc);
for (i = 0; i < this.n; i++) {
bits = this.hash(val, i);
pos = (bits >>> 5) * 4;
bits &= 0x1f;
bit = bits % 8;
pos += (bits - bit) / 8;
if (!ret && (this.filter[pos] & (1 << bit)) === 0)
index = this.hash(val, i);
if (!ret && (this.filter[index >>> 3] & (1 << (7 & index))) === 0)
ret = true;
this.filter[pos] |= 1 << bit;
this.filter[index >>> 3] |= 1 << (7 & index);
}
return ret;

View File

@ -7,6 +7,7 @@
var bcoin = require('./env');
var utils = require('./utils');
var constants = bcoin.protocol.constants;
/**
* Represents a merkle (filtered) block.
@ -51,7 +52,7 @@ function MerkleBlock(data) {
// List of matched TXs
this.txMap = {};
this.tx = [];
this._partialVerified = null;
this._validPartial = null;
// TXs that will be pushed on
this.txs = [];
@ -132,69 +133,121 @@ MerkleBlock.prototype.hasTX = function hasTX(hash) {
*/
MerkleBlock.prototype.verifyPartial = function verifyPartial() {
var height = 0;
var tx = [];
var txMap = {};
var j = 0;
var tree;
if (this._validPartial != null)
return this._validPartial;
tree = this.extractTree();
if (!tree || tree.root !== this.merkleRoot) {
this._validPartial = false;
return false;
}
this.tx = tree.matches;
this.txMap = utils.toMap(tree.matches);
this._validPartial = true;
return true;
};
/**
* Extract the matches from partial merkle
* tree and calculate merkle root.
* @private
* @returns {Object}
*/
MerkleBlock.prototype.extractTree = function extractTree() {
var self = this;
var hashes = this.hashes;
var flags = this.flags;
var i, root;
var bitsUsed = 0;
var hashUsed = 0;
var matches = [];
var indexes = [];
var failed = false;
var bits = new Array(flags.length * 8);
var totalTX = this.totalTX;
var height = 0;
var root;
var p;
if (this._partialVerified != null)
return this._partialVerified;
function width(height) {
return (totalTX + (1 << height) - 1) >>> height;
}
// Count leaves
for (i = this.totalTX; i > 0; i >>= 1)
height++;
function traverse(height, pos) {
var parent, hash, left, right;
if (this.totalTX > (1 << (height - 1)))
height++;
function visit(depth) {
var hash, flag, left, right;
if (i === flags.length * 8 || j === hashes.length)
return null;
flag = (flags[i >> 3] >>> (i & 7)) & 1;
i++;
if (flag === 0 || depth === height) {
if (depth === height) {
hash = hashes[j].toString('hex');
tx.push(hash);
txMap[hash] = true;
}
return hashes[j++];
if (bitsUsed >= bits.length) {
failed = true;
return constants.ZERO_HASH;
}
// Go deeper
left = visit(depth + 1);
if (!left)
return null;
parent = bits[bitsUsed++];
right = visit(depth + 1);
if (right && utils.equal(right, left))
return null;
if (height === 0 || !parent) {
if (hashUsed >= hashes.length) {
failed = true;
return constants.ZERO_HASH;
}
hash = hashes[hashUsed++];
if (height === 0 && parent) {
matches.push(hash.toString('hex'));
indexes.push(pos);
}
return hash;
}
if (!right)
left = traverse(height - 1, pos * 2);
if (pos * 2 + 1 < width(height - 1)) {
right = traverse(height - 1, pos * 2 + 1);
if (utils.equal(right, left))
failed = true;
} else {
right = left;
}
return utils.dsha256(Buffer.concat([left, right]));
}
root = visit(1);
for (p = 0; p < bits.length; p++)
bits[p] = (flags[p / 8 | 0] & (1 << (p % 8))) !== 0;
if (!root || root.toString('hex') !== this.merkleRoot) {
this._partialVerified = false;
return false;
}
if (totalTX == 0)
return;
this.tx = tx;
this.txMap = txMap;
this._partialVerified = true;
if (totalTX > (constants.block.MAX_SIZE / 60 | 0))
return;
return true;
if (hashes.length > totalTX)
return;
if (bits.length < hashes.length)
return;
height = 0;
while (width(height) > 1)
height++;
root = traverse(height, 0);
if (failed)
return;
if (((bitsUsed + 7) / 8 | 0) !== ((bits.length + 7) / 8 | 0))
return;
if (hashUsed != hashes.length)
return;
return {
root: root.toString('hex'),
matches: matches,
indexes: indexes
};
};
/**
@ -239,14 +292,24 @@ MerkleBlock.prototype.getCoinbaseHeight = function getCoinbaseHeight() {
*/
MerkleBlock.prototype.inspect = function inspect() {
var copy = bcoin.merkleblock(this);
copy.__proto__ = null;
delete copy._raw;
delete copy._chain;
copy.hash = this.hash('hex');
copy.rhash = this.rhash;
copy.date = utils.date(copy.ts);
return copy;
return {
type: 'merkleblock',
hash: this.rhash,
height: this.height,
date: utils.date(this.ts),
version: this.version,
prevBlock: utils.revHex(this.prevBlock),
merkleRoot: utils.revHex(this.merkleRoot),
ts: this.ts,
bits: this.bits,
nonce: this.nonce,
totalTX: this.totalTX,
hashes: this.hashes.map(function(hash) {
return hash.toString('hex');
}),
tx: this.tx,
flags: this.flags
};
};
/**
@ -323,18 +386,18 @@ MerkleBlock.fromBlock = function fromBlock(block, bloom) {
totalTX = leaves.length;
function width(height) {
return (totalTX + (1 << height) - 1) >> height;
return (totalTX + (1 << height) - 1) >>> height;
}
function hash(height, pos, leaves) {
var left, right;
if (height === 0)
return leaves[0];
return leaves[pos];
left = hash(height - 1, pos * 2, leaves);
if (pos * 2 + 1 < width(height - 1, pos * 2 + 1, leaves))
if (pos * 2 + 1 < width(height - 1))
right = hash(height - 1, pos * 2 + 1, leaves);
else
right = left;
@ -369,6 +432,8 @@ MerkleBlock.fromBlock = function fromBlock(block, bloom) {
traverse(height, 0, leaves, matches);
flags = new Buffer((bits.length + 7) / 8 | 0);
flags.fill(0);
for (p = 0; p < bits.length; p++)
flags[p / 8 | 0] |= bits[p] << (p % 8);
@ -398,7 +463,7 @@ MerkleBlock.fromBlock = function fromBlock(block, bloom) {
MerkleBlock.isMerkleBlock = function isMerkleBlock(obj) {
return obj
&& Array.isArray(obj.flags)
&& Buffer.isBuffer(obj.flags)
&& typeof obj.verifyPartial === 'function';
};

View File

@ -4,10 +4,11 @@ var utils = bcoin.utils;
var constants = bcoin.protocol.constants;
var network = bcoin.protocol.network;
var assert = require('assert');
var block300025 = require('./data/block300025.json');
describe('Block', function() {
var parser = bcoin.protocol.parser;
var block = bcoin.merkleblock({
var mblock = bcoin.merkleblock({
version: 2,
prevBlock: 'd1831d4411bdfda89d9d8c842b541beafd1437fc560dbe5c0000000000000000',
merkleRoot: '28bec1d35af480ba3884553d72694f6ba6c163a5c081d7e6edaec15f373f19af',
@ -27,24 +28,46 @@ describe('Block', function() {
'1f5e46b9da3a8b1241f4a1501741d3453bafddf6135b600b926e3f4056c6d564',
'33825657ba32afe269819f01993bd77baba86379043168c94845d32370e53562'
],
flags: new Buffer([245, 90, 0])
flags: new Buffer([245, 122, 0])
});
var raw = block.toRaw('hex');
var raw = mblock.toRaw('hex');
var block;
var raw2 = '02000000d1831d4411bdfda89d9d8c842b541beafd1437fc560dbe5c0'
+ '00000000000000028bec1d35af480ba3884553d72694f6ba6c163a5c081d7e6edaec'
+ '15f373f19af62ef6d536c890019d0b4bf46cd0100000a7d22e53bce1bbb3294d1a39'
+ '6c5acc45bdcc8f192cb492f0d9f55421fd4c62de19d6d585fdaf3737b9a54aaee1dd'
+ '003f498328d699b7dfb42dd2b44b6ebde23338b61da3053d6f382f2145bdd856bc5d'
+ 'cf052c3a11c1784d3d51b2cbe0f6d0923d7bbaae4716cb0d329d755b707cee588cdd'
+ 'c68601f99bc05fef1fabeb8dfe4a07393f84cd04ca8931975c66282ebf1847c78d8d'
+ 'e6c2578d4f9bae23bc6f30857ec8c51de3170301430ec56f6703533d9ea5b05c6fa7'
+ '068954bcb90eed8c2ee5cc7c152869db09a5ae2291fa03142912d9d7aba75be7d491'
+ 'a8ac4230ee9a920cb5adbf04583354515a225f2c418de7c5cdac4cef211820c79717'
+ 'cd2c50412153f1f5e46b9da3a8b1241f4a1501741d3453bafddf6135b600b926e3f4'
+ '056c6d56433825657ba32afe269819f01993bd77baba86379043168c94845d32370e'
+ '5356203f57a00';
var mblock = bcoin.merkleblock.fromRaw(raw2, 'hex');
it('should parse partial merkle tree', function() {
assert(block.verify());
assert.equal(block.tx.length, 2);
assert(mblock.verify());
assert.equal(mblock.tx.length, 2);
assert.equal(mblock.hash('hex'),
'8cc72c02a958de5a8b35a23bb7e3bced8bf840cc0a4e1c820000000000000000');
assert.equal(mblock.rhash,
'0000000000000000821c4e0acc40f88bedbce3b73ba2358b5ade58a9022cc78c');
assert.equal(
block.tx[0],
mblock.tx[0],
'7393f84cd04ca8931975c66282ebf1847c78d8de6c2578d4f9bae23bc6f30857');
assert.equal(
block.tx[1],
mblock.tx[1],
'ec8c51de3170301430ec56f6703533d9ea5b05c6fa7068954bcb90eed8c2ee5c');
});
it('should decode/encode with parser/framer', function() {
var b = bcoin.merkleblock.fromRaw(raw, 'hex');
assert.equal(b.render().toString('hex'), raw);
assert.equal(raw, raw2);
});
it('should be verifiable', function() {
@ -53,7 +76,7 @@ describe('Block', function() {
});
it('should be jsonified and unjsonified and still verify', function() {
var raw = block.toRaw();
var raw = mblock.toRaw();
var b = bcoin.merkleblock.fromRaw(raw);
assert.deepEqual(b.render(), raw);
assert(b.verify());
@ -76,4 +99,39 @@ describe('Block', function() {
assert.equal(height, 6930000);
assert.equal(total, 2099999997690000);
});
it('should parse JSON', function() {
block = bcoin.block.fromJSON(block300025);
assert.equal(block.hash('hex'),
'8cc72c02a958de5a8b35a23bb7e3bced8bf840cc0a4e1c820000000000000000');
assert.equal(block.rhash,
'0000000000000000821c4e0acc40f88bedbce3b73ba2358b5ade58a9022cc78c');
});
it('should create a merkle block', function() {
var filter = bcoin.bloom.fromRate(1000, 0.01, constants.filterFlags.NONE);
var item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b';
var item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10'
+ 'e763272c6c9ca052972c69e3884a9022084215e2eef'
+ '0e6f781656b5d5a87231cd4349e534b6dea55ad4ff55e';
filter.add(item1, 'hex');
filter.add(item2, 'hex');
var mblock2 = bcoin.merkleblock.fromBlock(block, filter);
assert(mblock2.verifyPartial());
assert.deepEqual(mblock2.render(), mblock.render());
});
it('should verify a historical block', function() {
assert(block.verify());
assert(block.txs[0].isSane());
var flags = constants.flags.VERIFY_P2SH | constants.flags.VERIFY_DERSIG;
for (var i = 1; i < block.txs.length; i++) {
var tx = block.txs[i];
assert(tx.isSane());
assert(tx.checkInputs(block.height));
assert(tx.verify(null, true, flags));
}
assert.equal(block.getReward(), 2507773345);
});
});

View File

@ -1,9 +1,17 @@
var bcoin = require('../').set('main');
var utils = bcoin.utils;
var constants = bcoin.protocol.constants;
var assert = require('assert');
describe('Bloom', function() {
this.timeout(20000);
var filterHex = ''
+ '000000000000000000000000000000000000000000000000088004000000000000000'
+ '000000000200000000000000000000000000000000800000000000000000002000000'
+ '000000000000002000000000000000000000000000000000000000000040000200000'
+ '0000000001000000800000080000000';
it('should do proper murmur3', function() {
var murmur3 = bcoin.bloom.murmur3;
assert.equal(murmur3(new Buffer('', 'ascii'), 0), 0);
@ -28,6 +36,17 @@ describe('Bloom', function() {
assert(b.test('ping', 'ascii'));
});
it('should serialize to the correct format', function() {
var filter = new bcoin.bloom(952, 6, 3624314491, constants.filterFlags.NONE);
var item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b';
var item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10'
+ 'e763272c6c9ca052972c69e3884a9022084215e2eef'
+ '0e6f781656b5d5a87231cd4349e534b6dea55ad4ff55e';
filter.add(item1, 'hex');
filter.add(item2, 'hex');
assert.equal(filter.filter.toString('hex'), filterHex);
});
it('should test regular filter', function() {
var filter = bcoin.bloom.fromRate(210000, 0.00001, -1);
filter.tweak = 0xdeadbeef;

38912
test/data/block300025.json Normal file

File diff suppressed because it is too large Load Diff

View File

@ -46,47 +46,34 @@ describe('Protocol', function() {
packetTest('verack', {}, function(payload) {
});
var peers = [
var hosts = [
{
ipv6: '0000:0000:0000:0000:0000:ffff:0000:0000',
ipv4: '0.0.0.0',
services: constants.LOCAL_SERVICES,
host: '127.0.0.1',
port: 8333,
ts: Date.now() / 1000 | 0
},
{
ipv6: '0000:0000:0000:0000:0000:ffff:7f00:0001',
ipv4: '127.0.0.1',
services: constants.LOCAL_SERVICES,
host: 'ffff:0123:4567:89ab:cdef:0123:4567:89ab',
port: 18333,
ts: Date.now() / 1000 | 0
}
];
// Convert peers to framer payload format, backup strings.
peers.forEach(function(addr) {
addr._ipv4 = addr.ipv4;
addr.ipv4 = addr.ipv4.split('.').map(function(n) {
return +n;
});
addr._ipv6 = addr.ipv6;
addr.ipv6 = new Buffer(addr.ipv6.replace(/:/g, ''), 'hex');
addr.services = constants.LOCAL_SERVICES;
});
packetTest('addr', peers, function(payload) {
packetTest('addr', hosts, function(payload) {
assert.equal(typeof payload.length, 'number');
assert.equal(payload.length, 2);
assert.equal(typeof payload[0].ts, 'number');
assert.equal(payload[0].services, constants.LOCAL_SERVICES);
assert.equal(payload[0].ipv6, peers[0]._ipv6);
assert.equal(payload[0].ipv4, peers[0]._ipv4);
assert.equal(payload[0].port, peers[0].port);
assert.equal(payload[0].host, hosts[0].host);
assert.equal(payload[0].port, hosts[0].port);
assert.equal(typeof payload[1].ts, 'number');
assert.equal(payload[1].services, constants.LOCAL_SERVICES);
assert.equal(payload[1].ipv6, peers[1]._ipv6);
assert.equal(payload[1].ipv4, peers[1]._ipv4);
assert.equal(payload[1].port, peers[1].port);
assert.equal(payload[1].host, hosts[1].host);
assert.equal(payload[1].port, hosts[1].port);
});
it('should include the raw data of only one transaction in a ' +