utils: start using bfilter.
This commit is contained in:
Christopher Jeffrey
parent
3a722481f9
commit
e92b1f4cec
@ -125,10 +125,9 @@ bcoin.witness = require('./script/witness');
|
||||
bcoin.utils = require('./utils');
|
||||
bcoin.base32 = require('./utils/base32');
|
||||
bcoin.base58 = require('./utils/base58');
|
||||
bcoin.bloom = require('./utils/bloom');
|
||||
bcoin.bloom = require('bfilter/lib/bloom');
|
||||
bcoin.co = require('./utils/co');
|
||||
bcoin.encoding = require('./utils/encoding');
|
||||
bcoin.int64 = require('./utils/int64');
|
||||
bcoin.lock = require('./utils/lock');
|
||||
bcoin.reader = require('./utils/reader');
|
||||
bcoin.staticwriter = require('./utils/staticwriter');
|
||||
|
||||
@ -163,7 +163,6 @@ bcoin.define('base58', './utils/base58');
|
||||
bcoin.define('bloom', './utils/bloom');
|
||||
bcoin.define('co', './utils/co');
|
||||
bcoin.define('encoding', './utils/encoding');
|
||||
bcoin.define('int64', './utils/int64');
|
||||
bcoin.define('lock', './utils/lock');
|
||||
bcoin.define('reader', './utils/reader');
|
||||
bcoin.define('staticwriter', './utils/staticwriter');
|
||||
|
||||
@ -16,7 +16,7 @@ const policy = require('../protocol/policy');
|
||||
const util = require('../utils/util');
|
||||
const random = require('bcrypto/lib/random');
|
||||
const {VerifyError} = require('../protocol/errors');
|
||||
const RollingFilter = require('../utils/rollingfilter');
|
||||
const RollingFilter = require('bfilter/lib/rolling');
|
||||
const Address = require('../primitives/address');
|
||||
const Script = require('../script/script');
|
||||
const Outpoint = require('../primitives/outpoint');
|
||||
|
||||
@ -9,13 +9,13 @@
|
||||
const assert = require('assert');
|
||||
const path = require('path');
|
||||
const fs = require('bfile');
|
||||
const murmur3 = require('bfilter/lib/murmur3');
|
||||
const util = require('../utils/util');
|
||||
const IP = require('../utils/ip');
|
||||
const co = require('../utils/co');
|
||||
const Network = require('../protocol/network');
|
||||
const NetAddress = require('../primitives/netaddress');
|
||||
const List = require('../utils/list');
|
||||
const murmur3 = require('../utils/murmur3');
|
||||
const common = require('./common');
|
||||
const seeds = require('./seeds');
|
||||
const dns = require('./dns');
|
||||
@ -461,7 +461,7 @@ HostList.prototype.freshBucket = function freshBucket(entry) {
|
||||
const addr = entry.addr;
|
||||
const src = entry.src;
|
||||
const data = concat32(addr.raw, src.raw);
|
||||
const hash = murmur3(data, 0xfba4c795);
|
||||
const hash = murmur3.sum(data, 0xfba4c795);
|
||||
const index = hash % this.fresh.length;
|
||||
return this.fresh[index];
|
||||
};
|
||||
@ -475,7 +475,7 @@ HostList.prototype.freshBucket = function freshBucket(entry) {
|
||||
|
||||
HostList.prototype.usedBucket = function usedBucket(entry) {
|
||||
const addr = entry.addr;
|
||||
const hash = murmur3(addr.raw, 0xfba4c795);
|
||||
const hash = murmur3.sum(addr.raw, 0xfba4c795);
|
||||
const index = hash % this.used.length;
|
||||
return this.used[index];
|
||||
};
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
const common = require('./common');
|
||||
const util = require('../utils/util');
|
||||
const assert = require('assert');
|
||||
const Bloom = require('../utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const bip152 = require('./bip152');
|
||||
const NetAddress = require('../primitives/netaddress');
|
||||
const Headers = require('../primitives/headers');
|
||||
@ -1810,7 +1810,7 @@ MempoolPacket.fromRaw = function fromRaw(data, enc) {
|
||||
/**
|
||||
* Represents a `filterload` packet.
|
||||
* @constructor
|
||||
* @param {Bloom|null} filter
|
||||
* @param {BloomFilter|null} filter
|
||||
*/
|
||||
|
||||
function FilterLoadPacket(filter) {
|
||||
@ -1819,7 +1819,7 @@ function FilterLoadPacket(filter) {
|
||||
|
||||
Packet.call(this);
|
||||
|
||||
this.filter = filter || new Bloom();
|
||||
this.filter = filter || new BloomFilter();
|
||||
}
|
||||
|
||||
Object.setPrototypeOf(FilterLoadPacket.prototype, Packet.prototype);
|
||||
|
||||
@ -18,7 +18,7 @@ const consensus = require('../protocol/consensus');
|
||||
const common = require('./common');
|
||||
const InvItem = require('../primitives/invitem');
|
||||
const Lock = require('../utils/lock');
|
||||
const RollingFilter = require('../utils/rollingfilter');
|
||||
const RollingFilter = require('bfilter/lib/rolling');
|
||||
const BIP151 = require('./bip151');
|
||||
const BIP150 = require('./bip150');
|
||||
const BIP152 = require('./bip152');
|
||||
|
||||
@ -19,8 +19,8 @@ const Address = require('../primitives/address');
|
||||
const BIP150 = require('./bip150');
|
||||
const BIP151 = require('./bip151');
|
||||
const BIP152 = require('./bip152');
|
||||
const Bloom = require('../utils/bloom');
|
||||
const RollingFilter = require('../utils/rollingfilter');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const RollingFilter = require('bfilter/lib/rolling');
|
||||
const secp256k1 = require('bcrypto/lib/secp256k1');
|
||||
const Lock = require('../utils/lock');
|
||||
const Network = require('../protocol/network');
|
||||
@ -112,7 +112,7 @@ function Pool(options) {
|
||||
this.id = 0;
|
||||
|
||||
if (this.options.spv)
|
||||
this.spvFilter = Bloom.fromRate(20000, 0.001, Bloom.flags.ALL);
|
||||
this.spvFilter = BloomFilter.fromRate(20000, 0.001, BloomFilter.flags.ALL);
|
||||
|
||||
if (!this.options.mempool)
|
||||
this.txFilter = new RollingFilter(50000, 0.000001);
|
||||
@ -3199,7 +3199,7 @@ Pool.prototype.unban = function unban(addr) {
|
||||
|
||||
/**
|
||||
* Set the spv filter.
|
||||
* @param {Bloom} filter
|
||||
* @param {BloomFilter} filter
|
||||
* @param {String?} enc
|
||||
*/
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ const path = require('path');
|
||||
const {Server} = require('bweb');
|
||||
const util = require('../utils/util');
|
||||
const base58 = require('../utils/base58');
|
||||
const Bloom = require('../utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const TX = require('../primitives/tx');
|
||||
const Outpoint = require('../primitives/outpoint');
|
||||
const digest = require('bcrypto/lib/digest');
|
||||
@ -413,7 +413,7 @@ class HTTP extends Server {
|
||||
if (!data)
|
||||
throw new Error('Invalid parameter.');
|
||||
|
||||
socket.filter = Bloom.fromRaw(data);
|
||||
socket.filter = BloomFilter.fromRaw(data);
|
||||
|
||||
return null;
|
||||
});
|
||||
|
||||
@ -21,7 +21,7 @@ const Input = require('./input');
|
||||
const Output = require('./output');
|
||||
const Outpoint = require('./outpoint');
|
||||
const InvItem = require('./invitem');
|
||||
const Bloom = require('../utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const consensus = require('../protocol/consensus');
|
||||
const policy = require('../protocol/policy');
|
||||
const ScriptError = require('../script/scripterror');
|
||||
@ -1955,7 +1955,7 @@ TX.prototype.getPrevout = function getPrevout() {
|
||||
* value is.
|
||||
* @see "Filter matching algorithm":
|
||||
* @see https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki
|
||||
* @param {Bloom} filter
|
||||
* @param {BloomFilter} filter
|
||||
* @returns {Boolean} True if the transaction matched.
|
||||
*/
|
||||
|
||||
@ -1972,10 +1972,10 @@ TX.prototype.isWatched = function isWatched(filter) {
|
||||
const output = this.outputs[i];
|
||||
// Test the output script
|
||||
if (output.script.test(filter)) {
|
||||
if (filter.update === Bloom.flags.ALL) {
|
||||
if (filter.update === BloomFilter.flags.ALL) {
|
||||
const prevout = Outpoint.fromTX(this, i);
|
||||
filter.add(prevout.toRaw());
|
||||
} else if (filter.update === Bloom.flags.PUBKEY_ONLY) {
|
||||
} else if (filter.update === BloomFilter.flags.PUBKEY_ONLY) {
|
||||
if (output.script.isPubkey() || output.script.isMultisig()) {
|
||||
const prevout = Outpoint.fromTX(this, i);
|
||||
filter.add(prevout.toRaw());
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const {I64} = require('../utils/int64');
|
||||
const {I64} = require('n64');
|
||||
const ScriptError = require('./scripterror');
|
||||
|
||||
/*
|
||||
|
||||
@ -1,383 +0,0 @@
|
||||
/*!
|
||||
* bloom.js - bloom filter for bcoin
|
||||
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
|
||||
* Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const murmur3 = require('./murmur3');
|
||||
const BufferReader = require('./reader');
|
||||
const StaticWriter = require('./staticwriter');
|
||||
const encoding = require('./encoding');
|
||||
const sum32 = murmur3.sum32;
|
||||
const mul32 = murmur3.mul32;
|
||||
const DUMMY = Buffer.alloc(0);
|
||||
|
||||
/*
|
||||
* Constants
|
||||
*/
|
||||
|
||||
const LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455;
|
||||
const LN2 = 0.6931471805599453094172321214581765680755001343602552;
|
||||
|
||||
/**
|
||||
* Bloom Filter
|
||||
* @alias module:utils.Bloom
|
||||
* @constructor
|
||||
* @param {Number} size - Filter size in bits.
|
||||
* @param {Number} n - Number of hash functions.
|
||||
* @param {Number} tweak - Seed value.
|
||||
* @param {Number|String} - Update type.
|
||||
* @property {Buffer} filter
|
||||
* @property {Number} size
|
||||
* @property {Number} n
|
||||
* @property {Number} tweak
|
||||
* @property {Number} update - Update flag (see {@link Bloom.flags}).
|
||||
*/
|
||||
|
||||
function Bloom(size, n, tweak, update) {
|
||||
if (!(this instanceof Bloom))
|
||||
return new Bloom(size, n, tweak, update);
|
||||
|
||||
this.filter = DUMMY;
|
||||
this.size = 0;
|
||||
this.n = 0;
|
||||
this.tweak = 0;
|
||||
this.update = Bloom.flags.NONE;
|
||||
|
||||
if (size != null)
|
||||
this.fromOptions(size, n, tweak, update);
|
||||
}
|
||||
|
||||
/**
|
||||
* Max bloom filter size.
|
||||
* @const {Number}
|
||||
* @default
|
||||
*/
|
||||
|
||||
Bloom.MAX_BLOOM_FILTER_SIZE = 36000;
|
||||
|
||||
/**
|
||||
* Max number of hash functions.
|
||||
* @const {Number}
|
||||
* @default
|
||||
*/
|
||||
|
||||
Bloom.MAX_HASH_FUNCS = 50;
|
||||
|
||||
/**
|
||||
* Bloom filter update flags.
|
||||
* @enum {Number}
|
||||
* @default
|
||||
*/
|
||||
|
||||
Bloom.flags = {
|
||||
/**
|
||||
* Never update the filter with outpoints.
|
||||
*/
|
||||
|
||||
NONE: 0,
|
||||
|
||||
/**
|
||||
* Always update the filter with outpoints.
|
||||
*/
|
||||
|
||||
ALL: 1,
|
||||
|
||||
/**
|
||||
* Only update the filter with outpoints if it is
|
||||
* "asymmetric" in terms of addresses (pubkey/multisig).
|
||||
*/
|
||||
|
||||
PUBKEY_ONLY: 2
|
||||
};
|
||||
|
||||
/**
|
||||
* Bloom filter update flags by value.
|
||||
* @const {RevMap}
|
||||
*/
|
||||
|
||||
Bloom.flagsByVal = {
|
||||
0: 'NONE',
|
||||
1: 'ALL',
|
||||
2: 'PUBKEY_ONLY'
|
||||
};
|
||||
|
||||
/**
|
||||
* Inject properties from options.
|
||||
* @private
|
||||
* @param {Number} size - Filter size in bits.
|
||||
* @param {Number} n - Number of hash functions.
|
||||
* @param {Number} tweak - Seed value.
|
||||
* @param {Number|String} - Update type.
|
||||
* @returns {Bloom}
|
||||
*/
|
||||
|
||||
Bloom.prototype.fromOptions = function fromOptions(size, n, tweak, update) {
|
||||
assert(typeof size === 'number', '`size` must be a number.');
|
||||
assert(size > 0, '`size` must be greater than zero.');
|
||||
assert(Number.isSafeInteger(size), '`size` must be an integer.');
|
||||
|
||||
size -= size % 8;
|
||||
|
||||
const filter = Buffer.allocUnsafe(size / 8);
|
||||
filter.fill(0);
|
||||
|
||||
if (tweak == null || tweak === -1)
|
||||
tweak = (Math.random() * 0x100000000) >>> 0;
|
||||
|
||||
if (update == null || update === -1)
|
||||
update = Bloom.flags.NONE;
|
||||
|
||||
if (typeof update === 'string') {
|
||||
update = Bloom.flags[update.toUpperCase()];
|
||||
assert(update != null, 'Unknown update flag.');
|
||||
}
|
||||
|
||||
assert(size > 0, '`size` must be greater than zero.');
|
||||
assert(n > 0, '`n` must be greater than zero.');
|
||||
assert(Number.isSafeInteger(n), '`n` must be an integer.');
|
||||
assert(typeof tweak === 'number', '`tweak` must be a number.');
|
||||
assert(Number.isSafeInteger(tweak), '`tweak` must be an integer.');
|
||||
assert(Bloom.flagsByVal[update], 'Unknown update flag.');
|
||||
|
||||
this.filter = filter;
|
||||
this.size = size;
|
||||
this.n = n;
|
||||
this.tweak = tweak;
|
||||
this.update = update;
|
||||
|
||||
return this;
|
||||
};
|
||||
|
||||
/**
|
||||
* Instantiate bloom filter from options.
|
||||
* @param {Number} size - Filter size in bits.
|
||||
* @param {Number} n - Number of hash functions.
|
||||
* @param {Number} tweak - Seed value.
|
||||
* @param {Number|String} - Update type.
|
||||
* @returns {Bloom}
|
||||
*/
|
||||
|
||||
Bloom.fromOptions = function fromOptions(size, n, tweak, update) {
|
||||
return new Bloom().fromOptions(size, n, tweak, update);
|
||||
};
|
||||
|
||||
/**
|
||||
* Perform the mumur3 hash on data.
|
||||
* @param {Buffer} val
|
||||
* @param {Number} n
|
||||
* @returns {Number}
|
||||
*/
|
||||
|
||||
Bloom.prototype.hash = function hash(val, n) {
|
||||
return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak)) % this.size;
|
||||
};
|
||||
|
||||
/**
|
||||
* Reset the filter.
|
||||
*/
|
||||
|
||||
Bloom.prototype.reset = function reset() {
|
||||
this.filter.fill(0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Add data to the filter.
|
||||
* @param {Buffer|String}
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
*/
|
||||
|
||||
Bloom.prototype.add = function add(val, enc) {
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
for (let i = 0; i < this.n; i++) {
|
||||
const index = this.hash(val, i);
|
||||
this.filter[index >>> 3] |= 1 << (7 & index);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Test whether data is present in the filter.
|
||||
* @param {Buffer|String} val
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
* @returns {Boolean}
|
||||
*/
|
||||
|
||||
Bloom.prototype.test = function test(val, enc) {
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
for (let i = 0; i < this.n; i++) {
|
||||
const index = this.hash(val, i);
|
||||
if ((this.filter[index >>> 3] & (1 << (7 & index))) === 0)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Test whether data is present in the
|
||||
* filter and potentially add data.
|
||||
* @param {Buffer|String} val
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
* @returns {Boolean} Whether data was added.
|
||||
*/
|
||||
|
||||
Bloom.prototype.added = function added(val, enc) {
|
||||
let ret = false;
|
||||
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
for (let i = 0; i < this.n; i++) {
|
||||
const index = this.hash(val, i);
|
||||
if (!ret && (this.filter[index >>> 3] & (1 << (7 & index))) === 0)
|
||||
ret = true;
|
||||
this.filter[index >>> 3] |= 1 << (7 & index);
|
||||
}
|
||||
|
||||
return ret;
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a filter from a false positive rate.
|
||||
* @param {Number} items - Expected number of items.
|
||||
* @param {Number} rate - False positive rate (0.0-1.0).
|
||||
* @param {Number|String} update
|
||||
* @example
|
||||
* Bloom.fromRate(800000, 0.0001, 'none');
|
||||
* @returns {Boolean}
|
||||
*/
|
||||
|
||||
Bloom.fromRate = function fromRate(items, rate, update) {
|
||||
assert(typeof items === 'number', '`items` must be a number.');
|
||||
assert(items > 0, '`items` must be greater than zero.');
|
||||
assert(Number.isSafeInteger(items), '`items` must be an integer.');
|
||||
assert(typeof rate === 'number', '`rate` must be a number.');
|
||||
assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.');
|
||||
|
||||
const bits = (-1 / LN2SQUARED * items * Math.log(rate)) | 0;
|
||||
const size = Math.max(8, bits);
|
||||
|
||||
if (update !== -1) {
|
||||
assert(size <= Bloom.MAX_BLOOM_FILTER_SIZE * 8,
|
||||
'Bloom filter size violates policy limits!');
|
||||
}
|
||||
|
||||
const n = Math.max(1, (size / items * LN2) | 0);
|
||||
|
||||
if (update !== -1) {
|
||||
assert(n <= Bloom.MAX_HASH_FUNCS,
|
||||
'Bloom filter size violates policy limits!');
|
||||
}
|
||||
|
||||
return new Bloom(size, n, -1, update);
|
||||
};
|
||||
|
||||
/**
|
||||
* Ensure the filter is within the size limits.
|
||||
* @returns {Boolean}
|
||||
*/
|
||||
|
||||
Bloom.prototype.isWithinConstraints = function isWithinConstraints() {
|
||||
if (this.size > Bloom.MAX_BLOOM_FILTER_SIZE * 8)
|
||||
return false;
|
||||
|
||||
if (this.n > Bloom.MAX_HASH_FUNCS)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get serialization size.
|
||||
* @returns {Number}
|
||||
*/
|
||||
|
||||
Bloom.prototype.getSize = function getSize() {
|
||||
return encoding.sizeVarBytes(this.filter) + 9;
|
||||
};
|
||||
|
||||
/**
|
||||
* Write filter to buffer writer.
|
||||
* @param {BufferWriter} bw
|
||||
*/
|
||||
|
||||
Bloom.prototype.toWriter = function toWriter(bw) {
|
||||
bw.writeVarBytes(this.filter);
|
||||
bw.writeU32(this.n);
|
||||
bw.writeU32(this.tweak);
|
||||
bw.writeU8(this.update);
|
||||
return bw;
|
||||
};
|
||||
|
||||
/**
|
||||
* Serialize bloom filter.
|
||||
* @returns {Buffer}
|
||||
*/
|
||||
|
||||
Bloom.prototype.toRaw = function toRaw() {
|
||||
const size = this.getSize();
|
||||
return this.toWriter(new StaticWriter(size)).render();
|
||||
};
|
||||
|
||||
/**
|
||||
* Inject properties from buffer reader.
|
||||
* @private
|
||||
* @param {BufferReader} br
|
||||
*/
|
||||
|
||||
Bloom.prototype.fromReader = function fromReader(br) {
|
||||
this.filter = br.readVarBytes();
|
||||
this.size = this.filter.length * 8;
|
||||
this.n = br.readU32();
|
||||
this.tweak = br.readU32();
|
||||
this.update = br.readU8();
|
||||
assert(Bloom.flagsByVal[this.update] != null, 'Unknown update flag.');
|
||||
return this;
|
||||
};
|
||||
|
||||
/**
|
||||
* Inject properties from serialized data.
|
||||
* @private
|
||||
* @param {Buffer} data
|
||||
*/
|
||||
|
||||
Bloom.prototype.fromRaw = function fromRaw(data) {
|
||||
return this.fromReader(new BufferReader(data));
|
||||
};
|
||||
|
||||
/**
|
||||
* Instantiate bloom filter from buffer reader.
|
||||
* @param {BufferReader} br
|
||||
* @returns {Bloom}
|
||||
*/
|
||||
|
||||
Bloom.fromReader = function fromReader(br) {
|
||||
return new Bloom().fromReader(br);
|
||||
};
|
||||
|
||||
/**
|
||||
* Instantiate bloom filter from serialized data.
|
||||
* @param {Buffer} data
|
||||
* @param {String?} enc
|
||||
* @returns {Bloom}
|
||||
*/
|
||||
|
||||
Bloom.fromRaw = function fromRaw(data, enc) {
|
||||
if (typeof data === 'string')
|
||||
data = Buffer.from(data, enc);
|
||||
return new Bloom().fromRaw(data);
|
||||
};
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = Bloom;
|
||||
@ -12,7 +12,7 @@
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const {U64, I64} = require('./int64');
|
||||
const {U64, I64} = require('n64');
|
||||
const UINT128_MAX = U64.UINT64_MAX.shrn(7);
|
||||
const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER;
|
||||
const encoding = exports;
|
||||
|
||||
542
lib/utils/gcs.js
542
lib/utils/gcs.js
@ -1,542 +0,0 @@
|
||||
/*!
|
||||
* gcs.js - gcs filters for bcoin
|
||||
* Copyright (c) 2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const {U64} = require('./int64');
|
||||
const hash256 = require('bcrypto/lib/hash256');
|
||||
const siphash = require('bcrypto/lib/siphash');
|
||||
const DUMMY = Buffer.alloc(0);
|
||||
const EOF = new U64(-1);
|
||||
|
||||
/**
|
||||
* GCSFilter
|
||||
* @alias module:utils.GCSFilter
|
||||
* @constructor
|
||||
*/
|
||||
|
||||
function GCSFilter() {
|
||||
this.n = 0;
|
||||
this.p = 0;
|
||||
this.m = new U64(0);
|
||||
this.data = DUMMY;
|
||||
}
|
||||
|
||||
GCSFilter.prototype.hash = function hash(enc) {
|
||||
const h = hash256.digest(this.data);
|
||||
return enc === 'hex' ? h.toString('hex') : h;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.header = function header(prev) {
|
||||
return hash256.root(this.hash(), prev);
|
||||
};
|
||||
|
||||
GCSFilter.prototype.match = function match(key, data) {
|
||||
const br = new BitReader(this.data);
|
||||
const term = siphash24(data, key).imod(this.m);
|
||||
|
||||
let last = new U64(0);
|
||||
|
||||
while (last.lt(term)) {
|
||||
const value = this.readU64(br);
|
||||
|
||||
if (value === EOF)
|
||||
return false;
|
||||
|
||||
value.iadd(last);
|
||||
|
||||
if (value.eq(term))
|
||||
return true;
|
||||
|
||||
last = value;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.matchAny = function matchAny(key, items) {
|
||||
assert(items.length > 0);
|
||||
|
||||
const br = new BitReader(this.data);
|
||||
const last1 = new U64(0);
|
||||
const values = [];
|
||||
|
||||
for (const item of items) {
|
||||
const hash = siphash24(item, key).imod(this.m);
|
||||
values.push(hash);
|
||||
}
|
||||
|
||||
values.sort(compare);
|
||||
|
||||
let last2 = values[0];
|
||||
let i = 1;
|
||||
|
||||
for (;;) {
|
||||
const cmp = last1.cmp(last2);
|
||||
|
||||
if (cmp === 0)
|
||||
break;
|
||||
|
||||
if (cmp > 0) {
|
||||
if (i < values.length) {
|
||||
last2 = values[i];
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const value = this.readU64(br);
|
||||
|
||||
if (value === EOF)
|
||||
return false;
|
||||
|
||||
last1.iadd(value);
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.readU64 = function readU64(br) {
|
||||
try {
|
||||
return this._readU64(br);
|
||||
} catch (e) {
|
||||
if (e.message === 'EOF')
|
||||
return EOF;
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
|
||||
GCSFilter.prototype._readU64 = function _readU64(br) {
|
||||
const num = new U64(0);
|
||||
|
||||
// Unary
|
||||
while (br.readBit())
|
||||
num.iaddn(1);
|
||||
|
||||
const rem = br.readBits64(this.p);
|
||||
|
||||
return num.ishln(this.p).ior(rem);
|
||||
};
|
||||
|
||||
GCSFilter.prototype.toBytes = function toBytes() {
|
||||
return this.data;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.toNBytes = function toNBytes() {
|
||||
const data = Buffer.allocUnsafe(4 + this.data.length);
|
||||
data.writeUInt32BE(this.n, 0, true);
|
||||
this.data.copy(data, 4);
|
||||
return data;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.toPBytes = function toPBytes() {
|
||||
const data = Buffer.allocUnsafe(1 + this.data.length);
|
||||
data.writeUInt8(this.p, 0, true);
|
||||
this.data.copy(data, 1);
|
||||
return data;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.toNPBytes = function toNPBytes() {
|
||||
const data = Buffer.allocUnsafe(5 + this.data.length);
|
||||
data.writeUInt32BE(this.n, 0, true);
|
||||
data.writeUInt8(this.p, 4, true);
|
||||
this.data.copy(data, 5);
|
||||
return data;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.toRaw = function toRaw() {
|
||||
assert(this.p === 20);
|
||||
return this.toNBytes();
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromItems = function fromItems(P, key, items) {
|
||||
assert(typeof P === 'number' && isFinite(P));
|
||||
assert(P >= 0 && P <= 32);
|
||||
|
||||
assert(Buffer.isBuffer(key));
|
||||
assert(key.length === 16);
|
||||
|
||||
assert(Array.isArray(items));
|
||||
assert(items.length > 0);
|
||||
assert(items.length <= 0xffffffff);
|
||||
|
||||
this.n = items.length;
|
||||
this.p = P;
|
||||
this.m = U64(this.n).ishln(this.p);
|
||||
|
||||
const values = [];
|
||||
|
||||
for (const item of items) {
|
||||
assert(Buffer.isBuffer(item));
|
||||
const hash = siphash24(item, key).imod(this.m);
|
||||
values.push(hash);
|
||||
}
|
||||
|
||||
values.sort(compare);
|
||||
|
||||
const bw = new BitWriter();
|
||||
|
||||
let last = new U64(0);
|
||||
|
||||
for (const hash of values) {
|
||||
const rem = hash.sub(last).imaskn(this.p);
|
||||
const value = hash.sub(last).isub(rem).ishrn(this.p);
|
||||
|
||||
last = hash;
|
||||
|
||||
// Unary
|
||||
while (!value.isZero()) {
|
||||
bw.writeBit(1);
|
||||
value.isubn(1);
|
||||
}
|
||||
bw.writeBit(0);
|
||||
|
||||
bw.writeBits64(rem, this.p);
|
||||
}
|
||||
|
||||
this.data = bw.render();
|
||||
|
||||
return this;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromBytes = function fromBytes(N, P, data) {
|
||||
assert(typeof N === 'number' && isFinite(N));
|
||||
assert(typeof P === 'number' && isFinite(P));
|
||||
assert(P >= 0 && P <= 32);
|
||||
assert(Buffer.isBuffer(data));
|
||||
|
||||
this.n = N;
|
||||
this.p = P;
|
||||
this.m = U64(this.n).ishln(this.p);
|
||||
this.data = data;
|
||||
|
||||
return this;
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromNBytes = function fromNBytes(P, data) {
|
||||
assert(typeof P === 'number' && isFinite(P));
|
||||
assert(Buffer.isBuffer(data));
|
||||
assert(data.length >= 4);
|
||||
|
||||
const N = data.readUInt32BE(0, true);
|
||||
|
||||
return this.fromBytes(N, P, data.slice(4));
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromPBytes = function fromPBytes(N, data) {
|
||||
assert(typeof N === 'number' && isFinite(N));
|
||||
assert(Buffer.isBuffer(data));
|
||||
assert(data.length >= 1);
|
||||
|
||||
const P = data.readUInt8(0, true);
|
||||
|
||||
return this.fromBytes(N, P, data.slice(1));
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromNPBytes = function fromNPBytes(data) {
|
||||
assert(Buffer.isBuffer(data));
|
||||
assert(data.length >= 5);
|
||||
|
||||
const N = data.readUInt32BE(0, true);
|
||||
const P = data.readUInt8(4, true);
|
||||
|
||||
return this.fromBytes(N, P, data.slice(5));
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromRaw = function fromRaw(data) {
|
||||
return this.fromNBytes(20, data);
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromBlock = function fromBlock(block) {
|
||||
const hash = block.hash();
|
||||
const key = hash.slice(0, 16);
|
||||
const items = [];
|
||||
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
|
||||
if (i > 0) {
|
||||
for (const input of tx.inputs)
|
||||
items.push(input.prevout.toRaw());
|
||||
}
|
||||
|
||||
for (const output of tx.outputs)
|
||||
getPushes(items, output.script);
|
||||
}
|
||||
|
||||
return this.fromItems(20, key, items);
|
||||
};
|
||||
|
||||
GCSFilter.prototype.fromExtended = function fromExtended(block) {
|
||||
const hash = block.hash();
|
||||
const key = hash.slice(0, 16);
|
||||
const items = [];
|
||||
|
||||
for (let i = 0; i < block.txs.length; i++) {
|
||||
const tx = block.txs[i];
|
||||
|
||||
items.push(tx.hash());
|
||||
|
||||
if (i > 0) {
|
||||
for (const input of tx.inputs) {
|
||||
getWitness(items, input.witness);
|
||||
getPushes(items, input.script);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return this.fromItems(20, key, items);
|
||||
};
|
||||
|
||||
GCSFilter.fromItems = function fromItems(P, key, items) {
|
||||
return new GCSFilter().fromItems(P, key, items);
|
||||
};
|
||||
|
||||
GCSFilter.fromBytes = function fromBytes(N, P, data) {
|
||||
return new GCSFilter().fromBytes(N, P, data);
|
||||
};
|
||||
|
||||
GCSFilter.fromNBytes = function fromNBytes(P, data) {
|
||||
return new GCSFilter().fromNBytes(P, data);
|
||||
};
|
||||
|
||||
GCSFilter.fromPBytes = function fromPBytes(N, data) {
|
||||
return new GCSFilter().fromPBytes(N, data);
|
||||
};
|
||||
|
||||
GCSFilter.fromNPBytes = function fromNPBytes(data) {
|
||||
return new GCSFilter().fromNPBytes(data);
|
||||
};
|
||||
|
||||
GCSFilter.fromRaw = function fromRaw(data) {
|
||||
return new GCSFilter().fromRaw(data);
|
||||
};
|
||||
|
||||
GCSFilter.fromBlock = function fromBlock(block) {
|
||||
return new GCSFilter().fromBlock(block);
|
||||
};
|
||||
|
||||
GCSFilter.fromExtended = function fromExtended(block) {
|
||||
return new GCSFilter().fromExtended(block);
|
||||
};
|
||||
|
||||
/**
|
||||
* BitWriter
|
||||
* @constructor
|
||||
* @ignore
|
||||
*/
|
||||
|
||||
function BitWriter() {
|
||||
this.stream = [];
|
||||
this.remain = 0;
|
||||
}
|
||||
|
||||
BitWriter.prototype.writeBit = function writeBit(bit) {
|
||||
if (this.remain === 0) {
|
||||
this.stream.push(0);
|
||||
this.remain = 8;
|
||||
}
|
||||
|
||||
if (bit) {
|
||||
const index = this.stream.length - 1;
|
||||
this.stream[index] |= 1 << (this.remain - 1);
|
||||
}
|
||||
|
||||
this.remain--;
|
||||
};
|
||||
|
||||
BitWriter.prototype.writeByte = function writeByte(ch) {
|
||||
if (this.remain === 0) {
|
||||
this.stream.push(0);
|
||||
this.remain = 8;
|
||||
}
|
||||
|
||||
const index = this.stream.length - 1;
|
||||
|
||||
this.stream[index] |= (ch >> (8 - this.remain)) & 0xff;
|
||||
this.stream.push(0);
|
||||
this.stream[index + 1] = (ch << this.remain) & 0xff;
|
||||
};
|
||||
|
||||
BitWriter.prototype.writeBits = function writeBits(num, count) {
|
||||
assert(count >= 0);
|
||||
assert(count <= 32);
|
||||
|
||||
num <<= 32 - count;
|
||||
|
||||
while (count >= 8) {
|
||||
const ch = num >>> 24;
|
||||
this.writeByte(ch);
|
||||
num <<= 8;
|
||||
count -= 8;
|
||||
}
|
||||
|
||||
while (count > 0) {
|
||||
const bit = num >>> 31;
|
||||
this.writeBit(bit);
|
||||
num <<= 1;
|
||||
count -= 1;
|
||||
}
|
||||
};
|
||||
|
||||
BitWriter.prototype.writeBits64 = function writeBits64(num, count) {
|
||||
assert(count >= 0);
|
||||
assert(count <= 64);
|
||||
|
||||
if (count > 32) {
|
||||
this.writeBits(num.hi, count - 32);
|
||||
this.writeBits(num.lo, 32);
|
||||
} else {
|
||||
this.writeBits(num.lo, count);
|
||||
}
|
||||
};
|
||||
|
||||
BitWriter.prototype.render = function render() {
|
||||
const data = Buffer.allocUnsafe(this.stream.length);
|
||||
|
||||
for (let i = 0; i < this.stream.length; i++)
|
||||
data[i] = this.stream[i];
|
||||
|
||||
return data;
|
||||
};
|
||||
|
||||
/**
|
||||
* BitReader
|
||||
* @constructor
|
||||
* @ignore
|
||||
*/
|
||||
|
||||
function BitReader(data) {
|
||||
this.stream = data;
|
||||
this.pos = 0;
|
||||
this.remain = 8;
|
||||
}
|
||||
|
||||
BitReader.prototype.readBit = function readBit() {
|
||||
if (this.pos >= this.stream.length)
|
||||
throw new Error('EOF');
|
||||
|
||||
if (this.remain === 0) {
|
||||
this.pos += 1;
|
||||
|
||||
if (this.pos >= this.stream.length)
|
||||
throw new Error('EOF');
|
||||
|
||||
this.remain = 8;
|
||||
}
|
||||
|
||||
this.remain -= 1;
|
||||
|
||||
return (this.stream[this.pos] >> this.remain) & 1;
|
||||
};
|
||||
|
||||
BitReader.prototype.readByte = function readByte() {
|
||||
if (this.pos >= this.stream.length)
|
||||
throw new Error('EOF');
|
||||
|
||||
if (this.remain === 0) {
|
||||
this.pos += 1;
|
||||
|
||||
if (this.pos >= this.stream.length)
|
||||
throw new Error('EOF');
|
||||
|
||||
this.remain = 8;
|
||||
}
|
||||
|
||||
if (this.remain === 8) {
|
||||
const ch = this.stream[this.pos];
|
||||
this.pos += 1;
|
||||
return ch;
|
||||
}
|
||||
|
||||
let ch = this.stream[this.pos] & ((1 << this.remain) - 1);
|
||||
ch <<= 8 - this.remain;
|
||||
|
||||
this.pos += 1;
|
||||
|
||||
if (this.pos >= this.stream.length)
|
||||
throw new Error('EOF');
|
||||
|
||||
ch |= this.stream[this.pos] >> this.remain;
|
||||
|
||||
return ch;
|
||||
};
|
||||
|
||||
BitReader.prototype.readBits = function readBits(count) {
|
||||
assert(count >= 0);
|
||||
assert(count <= 32);
|
||||
|
||||
let num = 0;
|
||||
|
||||
while (count >= 8) {
|
||||
num <<= 8;
|
||||
num |= this.readByte();
|
||||
count -= 8;
|
||||
}
|
||||
|
||||
while (count > 0) {
|
||||
num <<= 1;
|
||||
num |= this.readBit();
|
||||
count -= 1;
|
||||
}
|
||||
|
||||
return num;
|
||||
};
|
||||
|
||||
BitReader.prototype.readBits64 = function readBits64(count) {
|
||||
assert(count >= 0);
|
||||
assert(count <= 64);
|
||||
|
||||
const num = new U64();
|
||||
|
||||
if (count > 32) {
|
||||
num.hi = this.readBits(count - 32);
|
||||
num.lo = this.readBits(32);
|
||||
} else {
|
||||
num.lo = this.readBits(count);
|
||||
}
|
||||
|
||||
return num;
|
||||
};
|
||||
|
||||
/*
|
||||
* Helpers
|
||||
*/
|
||||
|
||||
function compare(a, b) {
|
||||
return a.cmp(b);
|
||||
}
|
||||
|
||||
function siphash24(data, key) {
|
||||
const [hi, lo] = siphash(data, key);
|
||||
return U64.fromBits(hi, lo);
|
||||
}
|
||||
|
||||
function getPushes(items, script) {
|
||||
for (const op of script.code) {
|
||||
if (!op.data || op.data.length === 0)
|
||||
continue;
|
||||
|
||||
items.push(op.data);
|
||||
}
|
||||
}
|
||||
|
||||
function getWitness(items, witness) {
|
||||
for (const item of witness.items) {
|
||||
if (item.length === 0)
|
||||
continue;
|
||||
|
||||
items.push(item);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = GCSFilter;
|
||||
@ -10,31 +10,23 @@
|
||||
* @module utils
|
||||
*/
|
||||
|
||||
// exports.AsyncEmitter = require('./asyncemitter');
|
||||
exports.AsyncEmitter = require('./asyncemitter');
|
||||
exports.AsyncObject = require('./asyncobject');
|
||||
exports.base32 = require('./base32');
|
||||
exports.base58 = require('./base58');
|
||||
exports.bech32 = require('./bech32');
|
||||
exports.binary = require('./binary');
|
||||
exports.Bloom = require('./bloom');
|
||||
exports.co = require('./co');
|
||||
exports.encoding = require('./encoding');
|
||||
exports.enforce = require('./enforce');
|
||||
exports.fixed = require('./fixed');
|
||||
exports.GCSFilter = require('./gcs');
|
||||
exports.Heap = require('./heap');
|
||||
exports.Int64 = require('./int64');
|
||||
exports.IP = require('./ip');
|
||||
exports.List = require('./list');
|
||||
exports.Lock = require('./lock');
|
||||
exports.LRU = require('./lru');
|
||||
exports.MappedLock = require('./mappedlock');
|
||||
exports.murmur3 = require('./murmur3');
|
||||
exports.nfkd = require('./nfkd');
|
||||
exports.ProtoWriter = require('./protowriter');
|
||||
exports.ProtoReader = require('./protoreader');
|
||||
exports.BufferReader = require('./reader');
|
||||
exports.RollingFilter = require('./rollingfilter');
|
||||
exports.StaticWriter = require('./staticwriter');
|
||||
exports.util = require('./util');
|
||||
exports.Validator = require('./validator');
|
||||
|
||||
@ -1,112 +0,0 @@
|
||||
/*!
|
||||
* murmur3.js - murmur3 hash for bcoin
|
||||
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
|
||||
* Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const native = require('../native').binding;
|
||||
|
||||
/**
|
||||
* Murmur3 hash.
|
||||
* @alias module:utils.murmur3
|
||||
* @param {Buffer} data
|
||||
* @param {Number} seed
|
||||
* @returns {Number}
|
||||
*/
|
||||
|
||||
function murmur3(data, seed) {
|
||||
const tail = data.length - (data.length % 4);
|
||||
const c1 = 0xcc9e2d51;
|
||||
const c2 = 0x1b873593;
|
||||
let h1 = seed;
|
||||
let k1;
|
||||
|
||||
for (let i = 0; i < tail; i += 4) {
|
||||
k1 = (data[i + 3] << 24)
|
||||
| (data[i + 2] << 16)
|
||||
| (data[i + 1] << 8)
|
||||
| data[i];
|
||||
k1 = mul32(k1, c1);
|
||||
k1 = rotl32(k1, 15);
|
||||
k1 = mul32(k1, c2);
|
||||
h1 ^= k1;
|
||||
h1 = rotl32(h1, 13);
|
||||
h1 = sum32(mul32(h1, 5), 0xe6546b64);
|
||||
}
|
||||
|
||||
k1 = 0;
|
||||
switch (data.length & 3) {
|
||||
case 3:
|
||||
k1 ^= data[tail + 2] << 16;
|
||||
case 2:
|
||||
k1 ^= data[tail + 1] << 8;
|
||||
case 1:
|
||||
k1 ^= data[tail + 0];
|
||||
k1 = mul32(k1, c1);
|
||||
k1 = rotl32(k1, 15);
|
||||
k1 = mul32(k1, c2);
|
||||
h1 ^= k1;
|
||||
}
|
||||
|
||||
h1 ^= data.length;
|
||||
h1 ^= h1 >>> 16;
|
||||
h1 = mul32(h1, 0x85ebca6b);
|
||||
h1 ^= h1 >>> 13;
|
||||
h1 = mul32(h1, 0xc2b2ae35);
|
||||
h1 ^= h1 >>> 16;
|
||||
|
||||
if (h1 < 0)
|
||||
h1 += 0x100000000;
|
||||
|
||||
return h1;
|
||||
}
|
||||
|
||||
if (native)
|
||||
murmur3 = native.murmur3;
|
||||
|
||||
function mul32(a, b) {
|
||||
const alo = a & 0xffff;
|
||||
const blo = b & 0xffff;
|
||||
const ahi = a >>> 16;
|
||||
const bhi = b >>> 16;
|
||||
|
||||
let lo = alo * blo;
|
||||
let hi = (ahi * blo + bhi * alo) & 0xffff;
|
||||
|
||||
hi += lo >>> 16;
|
||||
lo &= 0xffff;
|
||||
|
||||
let r = (hi << 16) | lo;
|
||||
|
||||
if (r < 0)
|
||||
r += 0x100000000;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
function sum32(a, b) {
|
||||
let r = (a + b) & 0xffffffff;
|
||||
|
||||
if (r < 0)
|
||||
r += 0x100000000;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
function rotl32(w, b) {
|
||||
return (w << b) | (w >>> (32 - b));
|
||||
}
|
||||
|
||||
/**
|
||||
* Expose
|
||||
*/
|
||||
|
||||
exports = murmur3;
|
||||
exports.murmur3 = murmur3;
|
||||
exports.mul32 = mul32;
|
||||
exports.sum32 = sum32;
|
||||
exports.rotl32 = rotl32;
|
||||
module.exports = exports;
|
||||
@ -1,221 +0,0 @@
|
||||
/*!
|
||||
* protoreader.js - protobufs for bcoin
|
||||
* Copyright (c) 2016-2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const BufferReader = require('../utils/reader');
|
||||
|
||||
/*
|
||||
* Constants
|
||||
*/
|
||||
|
||||
const wireType = {
|
||||
VARINT: 0,
|
||||
FIXED64: 1,
|
||||
DELIMITED: 2,
|
||||
START_GROUP: 3,
|
||||
END_GROUP: 4,
|
||||
FIXED32: 5
|
||||
};
|
||||
|
||||
/**
|
||||
* ProtoBuf Reader
|
||||
* @alias module:utils.ProtoReader
|
||||
* @constructor
|
||||
*/
|
||||
|
||||
function ProtoReader(data, zeroCopy) {
|
||||
if (!(this instanceof ProtoReader))
|
||||
return new ProtoReader(data, zeroCopy);
|
||||
|
||||
BufferReader.call(this, data, zeroCopy);
|
||||
}
|
||||
|
||||
Object.setPrototypeOf(ProtoReader.prototype, BufferReader.prototype);
|
||||
|
||||
ProtoReader.prototype.readVarint = function readVarint() {
|
||||
const {size, value} = _readVarint(this.data, this.offset);
|
||||
this.offset += size;
|
||||
return value;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readFieldValue = function readFieldValue(tag, opt) {
|
||||
const field = this.readField(tag, opt);
|
||||
|
||||
if (!field)
|
||||
return -1;
|
||||
|
||||
assert(field.value != null);
|
||||
|
||||
return field.value;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readFieldU64 = function readFieldU64(tag, opt) {
|
||||
const field = this.readField(tag, opt);
|
||||
|
||||
if (!field)
|
||||
return -1;
|
||||
|
||||
assert(field.type === wireType.VARINT || field.type === wireType.FIXED64);
|
||||
|
||||
return field.value;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readFieldU32 = function readFieldU32(tag, opt) {
|
||||
const field = this.readField(tag, opt);
|
||||
|
||||
if (!field)
|
||||
return -1;
|
||||
|
||||
assert(field.type === wireType.VARINT || field.type === wireType.FIXED32);
|
||||
|
||||
return field.value;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readFieldBytes = function readFieldBytes(tag, opt) {
|
||||
const field = this.readField(tag, opt);
|
||||
|
||||
if (!field)
|
||||
return null;
|
||||
|
||||
assert(field.data);
|
||||
|
||||
return field.data;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readFieldString = function readFieldString(tag, opt, enc) {
|
||||
const field = this.readField(tag, opt);
|
||||
|
||||
if (!field)
|
||||
return null;
|
||||
|
||||
assert(field.data);
|
||||
|
||||
return field.data.toString(enc || 'utf8');
|
||||
};
|
||||
|
||||
ProtoReader.prototype.nextTag = function nextTag() {
|
||||
if (this.left() === 0)
|
||||
return -1;
|
||||
|
||||
const field = this.readField();
|
||||
|
||||
this.seek(-field.size);
|
||||
|
||||
return field.tag;
|
||||
};
|
||||
|
||||
ProtoReader.prototype.readField = function readField(tag, opt) {
|
||||
const offset = this.offset;
|
||||
const header = this.readVarint();
|
||||
const field = new Field(header);
|
||||
|
||||
if (tag != null && field.tag !== tag) {
|
||||
assert(opt, 'Non-optional field not present.');
|
||||
this.offset = offset;
|
||||
return null;
|
||||
}
|
||||
|
||||
switch (field.type) {
|
||||
case wireType.VARINT:
|
||||
field.value = this.readVarint();
|
||||
break;
|
||||
case wireType.FIXED64:
|
||||
field.value = this.readU64();
|
||||
break;
|
||||
case wireType.DELIMITED:
|
||||
field.data = this.readVarBytes();
|
||||
break;
|
||||
case wireType.START_GROUP:
|
||||
field.group = [];
|
||||
for (;;) {
|
||||
const inner = this.readField();
|
||||
if (inner.type === wireType.END_GROUP)
|
||||
break;
|
||||
field.group.push(inner);
|
||||
}
|
||||
break;
|
||||
case wireType.END_GROUP:
|
||||
assert(false, 'Unexpected end group.');
|
||||
break;
|
||||
case wireType.FIXED32:
|
||||
field.value = this.readU32();
|
||||
break;
|
||||
default:
|
||||
assert(false, 'Bad wire type.');
|
||||
break;
|
||||
}
|
||||
|
||||
field.size = this.offset - offset;
|
||||
|
||||
return field;
|
||||
};
|
||||
|
||||
/*
|
||||
* Encoding
|
||||
*/
|
||||
|
||||
function _readVarint(data, off) {
|
||||
let num = 0;
|
||||
let ch = 0x80;
|
||||
let size = 0;
|
||||
|
||||
while (ch & 0x80) {
|
||||
if (off >= data.length) {
|
||||
num = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
ch = data[off++];
|
||||
|
||||
// Optimization for javascript insanity.
|
||||
switch (size) {
|
||||
case 0:
|
||||
case 1:
|
||||
case 2:
|
||||
case 3:
|
||||
num += (ch & 0x7f) << (7 * size);
|
||||
break;
|
||||
case 4:
|
||||
num += (ch & 0x7f) * (1 << (7 * size));
|
||||
break;
|
||||
default:
|
||||
num += (ch & 0x7f) * Math.pow(2, 7 * size);
|
||||
break;
|
||||
}
|
||||
|
||||
size++;
|
||||
|
||||
assert(size < 7, 'Number exceeds 2^53-1.');
|
||||
}
|
||||
|
||||
return new Varint(size, num);
|
||||
}
|
||||
|
||||
/*
|
||||
* Helpers
|
||||
*/
|
||||
|
||||
function Field(header) {
|
||||
this.tag = header >>> 3;
|
||||
this.type = header & 7;
|
||||
this.size = 0;
|
||||
this.value = 0;
|
||||
this.data = null;
|
||||
this.group = null;
|
||||
}
|
||||
|
||||
function Varint(size, value) {
|
||||
this.size = size;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = ProtoReader;
|
||||
@ -1,182 +0,0 @@
|
||||
/*!
|
||||
* protowriter.js - protobufs for bcoin
|
||||
* Copyright (c) 2016-2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* @module utils/protobuf
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const BufferWriter = require('../utils/writer');
|
||||
|
||||
/*
|
||||
* Constants
|
||||
*/
|
||||
|
||||
const wireType = {
|
||||
VARINT: 0,
|
||||
FIXED64: 1,
|
||||
DELIMITED: 2,
|
||||
START_GROUP: 3,
|
||||
END_GROUP: 4,
|
||||
FIXED32: 5
|
||||
};
|
||||
|
||||
/**
|
||||
* ProtoBuf Writer
|
||||
* @alias module:utils.ProtoWriter
|
||||
* @constructor
|
||||
*/
|
||||
|
||||
function ProtoWriter() {
|
||||
if (!(this instanceof ProtoWriter))
|
||||
return new ProtoWriter();
|
||||
|
||||
BufferWriter.call(this);
|
||||
}
|
||||
|
||||
Object.setPrototypeOf(ProtoWriter.prototype, BufferWriter.prototype);
|
||||
|
||||
ProtoWriter.prototype.writeVarint = function writeVarint(num) {
|
||||
const size = sizeVarint(num);
|
||||
|
||||
// Avoid an extra allocation until
|
||||
// we make bufferwriter more hackable.
|
||||
// More insanity here...
|
||||
switch (size) {
|
||||
case 6: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU32BE(value / 0x10000 | 0);
|
||||
this.writeU16BE(value & 0xffff);
|
||||
break;
|
||||
}
|
||||
case 5: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU32BE(value / 0x100 | 0);
|
||||
this.writeU8(value & 0xff);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU32BE(value);
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU16BE(value >> 8);
|
||||
this.writeU8(value & 0xff);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU16BE(value);
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
const value = slipVarint(num);
|
||||
this.writeU8(value);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const value = Buffer.allocUnsafe(size);
|
||||
_writeVarint(value, num, 0);
|
||||
this.writeBytes(value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
ProtoWriter.prototype.writeFieldVarint = function writeFieldVarint(tag, value) {
|
||||
const header = (tag << 3) | wireType.VARINT;
|
||||
this.writeVarint(header);
|
||||
this.writeVarint(value);
|
||||
};
|
||||
|
||||
ProtoWriter.prototype.writeFieldU64 = function writeFieldU64(tag, value) {
|
||||
assert(Number.isSafeInteger(value));
|
||||
this.writeFieldVarint(tag, value);
|
||||
};
|
||||
|
||||
ProtoWriter.prototype.writeFieldU32 = function writeFieldU32(tag, value) {
|
||||
assert(value <= 0xffffffff);
|
||||
this.writeFieldVarint(tag, value);
|
||||
};
|
||||
|
||||
ProtoWriter.prototype.writeFieldBytes = function writeFieldBytes(tag, data) {
|
||||
const header = (tag << 3) | wireType.DELIMITED;
|
||||
this.writeVarint(header);
|
||||
this.writeVarint(data.length);
|
||||
this.writeBytes(data);
|
||||
};
|
||||
|
||||
ProtoWriter.prototype.writeFieldString = function writeFieldString(tag, data, enc) {
|
||||
if (typeof data === 'string')
|
||||
data = Buffer.from(data, enc || 'utf8');
|
||||
this.writeFieldBytes(tag, data);
|
||||
};
|
||||
|
||||
/*
|
||||
* Encoding
|
||||
*/
|
||||
|
||||
function _writeVarint(data, num, off) {
|
||||
assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.');
|
||||
|
||||
do {
|
||||
assert(off < data.length);
|
||||
let ch = num & 0x7f;
|
||||
num -= num % 0x80;
|
||||
num /= 0x80;
|
||||
if (num !== 0)
|
||||
ch |= 0x80;
|
||||
data[off] = ch;
|
||||
off++;
|
||||
} while (num > 0);
|
||||
|
||||
return off;
|
||||
};
|
||||
|
||||
function slipVarint(num) {
|
||||
assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.');
|
||||
|
||||
let data = 0;
|
||||
let size = 0;
|
||||
|
||||
do {
|
||||
assert(size < 7);
|
||||
let ch = num & 0x7f;
|
||||
num -= num % 0x80;
|
||||
num /= 0x80;
|
||||
if (num !== 0)
|
||||
ch |= 0x80;
|
||||
data *= 256;
|
||||
data += ch;
|
||||
size++;
|
||||
} while (num > 0);
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
function sizeVarint(num) {
|
||||
assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.');
|
||||
|
||||
let size = 0;
|
||||
|
||||
do {
|
||||
num -= num % 0x80;
|
||||
num /= 0x80;
|
||||
size++;
|
||||
} while (num > 0);
|
||||
|
||||
return size;
|
||||
};
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = ProtoWriter;
|
||||
@ -1,255 +0,0 @@
|
||||
/*!
|
||||
* rollingfilter.js - rolling bloom filter for bcoin
|
||||
* Copyright (c) 2014-2015, Fedor Indutny (MIT License)
|
||||
* Copyright (c) 2014-2017, Christopher Jeffrey (MIT License).
|
||||
* https://github.com/bcoin-org/bcoin
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const murmur3 = require('./murmur3');
|
||||
const sum32 = murmur3.sum32;
|
||||
const mul32 = murmur3.mul32;
|
||||
const DUMMY = Buffer.alloc(0);
|
||||
|
||||
/**
|
||||
* A rolling bloom filter used internally
|
||||
* (do not relay this on the p2p network).
|
||||
* @alias module:utils.RollingFilter
|
||||
* @constructor
|
||||
* @param {Number} items - Expected number of items.
|
||||
* @param {Number} rate - False positive rate (0.0-1.0).
|
||||
*/
|
||||
|
||||
function RollingFilter(items, rate) {
|
||||
if (!(this instanceof RollingFilter))
|
||||
return new RollingFilter(items, rate);
|
||||
|
||||
this.entries = 0;
|
||||
this.generation = 1;
|
||||
this.n = 0;
|
||||
this.limit = 0;
|
||||
this.size = 0;
|
||||
this.items = 0;
|
||||
this.tweak = 0;
|
||||
this.filter = DUMMY;
|
||||
|
||||
if (items != null)
|
||||
this.fromRate(items, rate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Inject properties from items and FPR.
|
||||
* @private
|
||||
* @param {Number} items - Expected number of items.
|
||||
* @param {Number} rate - False positive rate (0.0-1.0).
|
||||
* @returns {RollingFilter}
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.fromRate = function fromRate(items, rate) {
|
||||
assert(typeof items === 'number', '`items` must be a number.');
|
||||
assert(items > 0, '`items` must be greater than zero.');
|
||||
assert(Number.isSafeInteger(items), '`items` must be an integer.');
|
||||
assert(typeof rate === 'number', '`rate` must be a number.');
|
||||
assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.');
|
||||
|
||||
const logRate = Math.log(rate);
|
||||
|
||||
const n = Math.max(1, Math.min(Math.round(logRate / Math.log(0.5)), 50));
|
||||
const limit = (items + 1) / 2 | 0;
|
||||
|
||||
const max = limit * 3;
|
||||
|
||||
let size = -1 * n * max / Math.log(1.0 - Math.exp(logRate / n));
|
||||
size = Math.ceil(size);
|
||||
|
||||
items = ((size + 63) / 64 | 0) << 1;
|
||||
items >>>= 0;
|
||||
items = Math.max(1, items);
|
||||
|
||||
const tweak = (Math.random() * 0x100000000) >>> 0;
|
||||
|
||||
const filter = Buffer.allocUnsafe(items * 8);
|
||||
filter.fill(0);
|
||||
|
||||
this.n = n;
|
||||
this.limit = limit;
|
||||
this.size = size;
|
||||
this.items = items;
|
||||
this.tweak = tweak;
|
||||
this.filter = filter;
|
||||
|
||||
return this;
|
||||
};
|
||||
|
||||
/**
|
||||
* Instantiate rolling filter from items and FPR.
|
||||
* @param {Number} items - Expected number of items.
|
||||
* @param {Number} rate - False positive rate (0.0-1.0).
|
||||
* @returns {RollingFilter}
|
||||
*/
|
||||
|
||||
RollingFilter.fromRate = function fromRate(items, rate) {
|
||||
return new RollingFilter().fromRate(items, rate);
|
||||
};
|
||||
|
||||
/**
|
||||
* Perform the mumur3 hash on data.
|
||||
* @param {Buffer} val
|
||||
* @param {Number} seed
|
||||
* @returns {Number}
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.hash = function hash(val, n) {
|
||||
return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak));
|
||||
};
|
||||
|
||||
/**
|
||||
* Reset the filter.
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.reset = function reset() {
|
||||
if (this.entries === 0)
|
||||
return;
|
||||
|
||||
this.entries = 0;
|
||||
this.generation = 1;
|
||||
this.filter.fill(0);
|
||||
};
|
||||
|
||||
/**
|
||||
* Add data to the filter.
|
||||
* @param {Buffer|String}
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.add = function add(val, enc) {
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
if (this.entries === this.limit) {
|
||||
this.entries = 0;
|
||||
this.generation += 1;
|
||||
|
||||
if (this.generation === 4)
|
||||
this.generation = 1;
|
||||
|
||||
const m1 = (this.generation & 1) * 0xffffffff;
|
||||
const m2 = (this.generation >>> 1) * 0xffffffff;
|
||||
|
||||
for (let i = 0; i < this.items; i += 2) {
|
||||
const pos1 = i * 8;
|
||||
const pos2 = (i + 1) * 8;
|
||||
const v1 = read(this.filter, pos1);
|
||||
const v2 = read(this.filter, pos2);
|
||||
const mhi = (v1.hi ^ m1) | (v2.hi ^ m2);
|
||||
const mlo = (v1.lo ^ m1) | (v2.lo ^ m2);
|
||||
|
||||
v1.hi &= mhi;
|
||||
v1.lo &= mlo;
|
||||
v2.hi &= mhi;
|
||||
v2.lo &= mlo;
|
||||
|
||||
write(this.filter, v1, pos1);
|
||||
write(this.filter, v2, pos2);
|
||||
}
|
||||
}
|
||||
|
||||
this.entries += 1;
|
||||
|
||||
for (let i = 0; i < this.n; i++) {
|
||||
const hash = this.hash(val, i);
|
||||
const bits = hash & 0x3f;
|
||||
const pos = (hash >>> 6) % this.items;
|
||||
const pos1 = (pos & ~1) * 8;
|
||||
const pos2 = (pos | 1) * 8;
|
||||
const bit = bits % 8;
|
||||
const oct = (bits - bit) / 8;
|
||||
|
||||
this.filter[pos1 + oct] &= ~(1 << bit);
|
||||
this.filter[pos1 + oct] |= (this.generation & 1) << bit;
|
||||
|
||||
this.filter[pos2 + oct] &= ~(1 << bit);
|
||||
this.filter[pos2 + oct] |= (this.generation >>> 1) << bit;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Test whether data is present in the filter.
|
||||
* @param {Buffer|String} val
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
* @returns {Boolean}
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.test = function test(val, enc) {
|
||||
if (this.entries === 0)
|
||||
return false;
|
||||
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
for (let i = 0; i < this.n; i++) {
|
||||
const hash = this.hash(val, i);
|
||||
const bits = hash & 0x3f;
|
||||
const pos = (hash >>> 6) % this.items;
|
||||
const pos1 = (pos & ~1) * 8;
|
||||
const pos2 = (pos | 1) * 8;
|
||||
const bit = bits % 8;
|
||||
const oct = (bits - bit) / 8;
|
||||
|
||||
const bit1 = (this.filter[pos1 + oct] >>> bit) & 1;
|
||||
const bit2 = (this.filter[pos2 + oct] >>> bit) & 1;
|
||||
|
||||
if ((bit1 | bit2) === 0)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Test whether data is present in the
|
||||
* filter and potentially add data.
|
||||
* @param {Buffer|String} val
|
||||
* @param {String?} enc - Can be any of the Buffer object's encodings.
|
||||
* @returns {Boolean} Whether data was added.
|
||||
*/
|
||||
|
||||
RollingFilter.prototype.added = function added(val, enc) {
|
||||
if (typeof val === 'string')
|
||||
val = Buffer.from(val, enc);
|
||||
|
||||
if (!this.test(val)) {
|
||||
this.add(val);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
/*
|
||||
* Helpers
|
||||
*/
|
||||
|
||||
function U64(hi, lo) {
|
||||
this.hi = hi;
|
||||
this.lo = lo;
|
||||
}
|
||||
|
||||
function read(data, off) {
|
||||
const hi = data.readUInt32LE(off + 4, true);
|
||||
const lo = data.readUInt32LE(off, true);
|
||||
return new U64(hi, lo);
|
||||
}
|
||||
|
||||
function write(data, value, off) {
|
||||
data.writeUInt32LE(value.hi, off + 4, true);
|
||||
data.writeUInt32LE(value.lo, off, true);
|
||||
}
|
||||
|
||||
/*
|
||||
* Expose
|
||||
*/
|
||||
|
||||
module.exports = RollingFilter;
|
||||
@ -21,7 +21,7 @@ const Path = require('./path');
|
||||
const common = require('./common');
|
||||
const Wallet = require('./wallet');
|
||||
const Account = require('./account');
|
||||
const Bloom = require('../utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const Logger = require('../node/logger');
|
||||
const Outpoint = require('../primitives/outpoint');
|
||||
const layouts = require('./layout');
|
||||
@ -74,7 +74,7 @@ function WalletDB(options) {
|
||||
this.txLock = new Lock();
|
||||
|
||||
// Address and outpoint filter.
|
||||
this.filter = new Bloom();
|
||||
this.filter = new BloomFilter();
|
||||
|
||||
this._init();
|
||||
}
|
||||
@ -99,15 +99,15 @@ WalletDB.prototype._init = function _init() {
|
||||
|
||||
// Highest number of items with an
|
||||
// FPR of 0.001. We have to do this
|
||||
// by hand because Bloom.fromRate's
|
||||
// by hand because BloomFilter.fromRate's
|
||||
// policy limit enforcing is fairly
|
||||
// naive.
|
||||
if (this.options.spv) {
|
||||
items = 20000;
|
||||
flag = Bloom.flags.ALL;
|
||||
flag = BloomFilter.flags.ALL;
|
||||
}
|
||||
|
||||
this.filter = Bloom.fromRate(items, 0.001, flag);
|
||||
this.filter = BloomFilter.fromRate(items, 0.001, flag);
|
||||
this.bind();
|
||||
};
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
|
||||
const assert = require('./util/assert');
|
||||
const common = require('./util/common');
|
||||
const Bloom = require('../lib/utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const Block = require('../lib/primitives/block');
|
||||
const MerkleBlock = require('../lib/primitives/merkleblock');
|
||||
const consensus = require('../lib/protocol/consensus');
|
||||
@ -103,7 +103,7 @@ describe('Block', function() {
|
||||
});
|
||||
|
||||
it('should create a merkle block', () => {
|
||||
const filter = Bloom.fromRate(1000, 0.01, Bloom.flags.NONE);
|
||||
const filter = BloomFilter.fromRate(1000, 0.01, BloomFilter.flags.NONE);
|
||||
|
||||
const item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b';
|
||||
const item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10'
|
||||
|
||||
@ -1,183 +0,0 @@
|
||||
/* eslint-env mocha */
|
||||
/* eslint prefer-arrow-callback: "off" */
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('./util/assert');
|
||||
const Bloom = require('../lib/utils/bloom');
|
||||
const RollingFilter = require('../lib/utils/rollingfilter');
|
||||
const murmur3 = require('../lib/utils/murmur3');
|
||||
|
||||
function testMurmur(str, seed, expect, enc) {
|
||||
if (!enc)
|
||||
enc = 'ascii';
|
||||
|
||||
const data = Buffer.from(str, enc);
|
||||
const hash = murmur3(data, seed);
|
||||
|
||||
assert.strictEqual(hash, expect);
|
||||
}
|
||||
|
||||
describe('Bloom', function() {
|
||||
this.timeout(20000);
|
||||
|
||||
it('should do proper murmur3', () => {
|
||||
testMurmur('', 0, 0);
|
||||
testMurmur('', 0xfba4c795, 0x6a396f08);
|
||||
testMurmur('00', 0xfba4c795, 0x2a101837);
|
||||
testMurmur('hello world', 0, 0x5e928f0f);
|
||||
|
||||
testMurmur('', 0x00000000, 0x00000000, 'hex');
|
||||
testMurmur('', 0xfba4c795, 0x6a396f08, 'hex');
|
||||
testMurmur('', 0xffffffff, 0x81f16f39, 'hex');
|
||||
|
||||
testMurmur('00', 0x00000000, 0x514e28b7, 'hex');
|
||||
testMurmur('00', 0xfba4c795, 0xea3f0b17, 'hex');
|
||||
testMurmur('ff', 0x00000000, 0xfd6cf10d, 'hex');
|
||||
|
||||
testMurmur('0011', 0x00000000, 0x16c6b7ab, 'hex');
|
||||
testMurmur('001122', 0x00000000, 0x8eb51c3d, 'hex');
|
||||
testMurmur('00112233', 0x00000000, 0xb4471bf8, 'hex');
|
||||
testMurmur('0011223344', 0x00000000, 0xe2301fa8, 'hex');
|
||||
testMurmur('001122334455', 0x00000000, 0xfc2e4a15, 'hex');
|
||||
testMurmur('00112233445566', 0x00000000, 0xb074502c, 'hex');
|
||||
testMurmur('0011223344556677', 0x00000000, 0x8034d2a0, 'hex');
|
||||
testMurmur('001122334455667788', 0x00000000, 0xb4698def, 'hex');
|
||||
});
|
||||
|
||||
it('should test and add stuff', () => {
|
||||
const filter = new Bloom(512, 10, 156);
|
||||
|
||||
filter.add('hello', 'ascii');
|
||||
assert(filter.test('hello', 'ascii'));
|
||||
assert(!filter.test('hello!', 'ascii'));
|
||||
assert(!filter.test('ping', 'ascii'));
|
||||
|
||||
filter.add('hello!', 'ascii');
|
||||
assert(filter.test('hello!', 'ascii'));
|
||||
assert(!filter.test('ping', 'ascii'));
|
||||
|
||||
filter.add('ping', 'ascii');
|
||||
assert(filter.test('ping', 'ascii'));
|
||||
});
|
||||
|
||||
it('should serialize to the correct format', () => {
|
||||
const filter = new Bloom(952, 6, 3624314491, Bloom.flags.NONE);
|
||||
const item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b';
|
||||
const item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10'
|
||||
+ 'e763272c6c9ca052972c69e3884a9022084215e2eef'
|
||||
+ '0e6f781656b5d5a87231cd4349e534b6dea55ad4ff55e';
|
||||
|
||||
const expected = Buffer.from(''
|
||||
+ '000000000000000000000000000000000000000000000000088004000000000000000'
|
||||
+ '000000000200000000000000000000000000000000800000000000000000002000000'
|
||||
+ '000000000000002000000000000000000000000000000000000000000040000200000'
|
||||
+ '0000000001000000800000080000000',
|
||||
'hex');
|
||||
|
||||
filter.add(item1, 'hex');
|
||||
filter.add(item2, 'hex');
|
||||
|
||||
assert.bufferEqual(filter.filter, expected);
|
||||
});
|
||||
|
||||
it('should handle 1m ops with regular filter', () => {
|
||||
const filter = Bloom.fromRate(210000, 0.00001, -1);
|
||||
|
||||
filter.tweak = 0xdeadbeef;
|
||||
|
||||
// ~1m operations
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j--);
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle 1m ops with rolling filter', () => {
|
||||
const filter = new RollingFilter(210000, 0.00001);
|
||||
|
||||
filter.tweak = 0xdeadbeef;
|
||||
|
||||
// ~1m operations
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j--);
|
||||
}
|
||||
});
|
||||
|
||||
it('should handle rolling generations', () => {
|
||||
const filter = new RollingFilter(50, 0.00001);
|
||||
|
||||
filter.tweak = 0xdeadbeee;
|
||||
|
||||
for (let i = 0; i < 25; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j--);
|
||||
}
|
||||
|
||||
for (let i = 25; i < 50; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j--);
|
||||
}
|
||||
|
||||
for (let i = 50; i < 75; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j--);
|
||||
}
|
||||
|
||||
for (let i = 75; i < 100; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j-- > 25);
|
||||
assert(!filter.test('foobar 24', 'ascii'));
|
||||
}
|
||||
|
||||
for (let i = 100; i < 125; i++) {
|
||||
const str = 'foobar' + i;
|
||||
let j = i;
|
||||
filter.add(str, 'ascii');
|
||||
do {
|
||||
const str = 'foobar' + j;
|
||||
assert(filter.test(str, 'ascii'));
|
||||
assert(!filter.test(str + '-', 'ascii'));
|
||||
} while (j-- > 50);
|
||||
}
|
||||
|
||||
assert(!filter.test('foobar 49', 'ascii'));
|
||||
});
|
||||
});
|
||||
177
test/gcs-test.js
177
test/gcs-test.js
@ -1,177 +0,0 @@
|
||||
/* eslint-env mocha */
|
||||
/* eslint prefer-arrow-callback: "off" */
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('./util/assert');
|
||||
const GCSFilter = require('../lib/utils/gcs');
|
||||
const random = require('bcrypto/lib/random');
|
||||
const Outpoint = require('../lib/primitives/outpoint');
|
||||
const Address = require('../lib/primitives/address');
|
||||
const common = require('./util/common');
|
||||
|
||||
const block928927 = common.readBlock('block928927');
|
||||
const [block] = block928927.getBlock();
|
||||
|
||||
const key = random.randomBytes(16);
|
||||
const P = 20;
|
||||
|
||||
const contents1 = [
|
||||
Buffer.from('Alex', 'ascii'),
|
||||
Buffer.from('Bob', 'ascii'),
|
||||
Buffer.from('Charlie', 'ascii'),
|
||||
Buffer.from('Dick', 'ascii'),
|
||||
Buffer.from('Ed', 'ascii'),
|
||||
Buffer.from('Frank', 'ascii'),
|
||||
Buffer.from('George', 'ascii'),
|
||||
Buffer.from('Harry', 'ascii'),
|
||||
Buffer.from('Ilya', 'ascii'),
|
||||
Buffer.from('John', 'ascii'),
|
||||
Buffer.from('Kevin', 'ascii'),
|
||||
Buffer.from('Larry', 'ascii'),
|
||||
Buffer.from('Michael', 'ascii'),
|
||||
Buffer.from('Nate', 'ascii'),
|
||||
Buffer.from('Owen', 'ascii'),
|
||||
Buffer.from('Paul', 'ascii'),
|
||||
Buffer.from('Quentin', 'ascii')
|
||||
];
|
||||
|
||||
const contents2 = [
|
||||
Buffer.from('Alice', 'ascii'),
|
||||
Buffer.from('Betty', 'ascii'),
|
||||
Buffer.from('Charmaine', 'ascii'),
|
||||
Buffer.from('Donna', 'ascii'),
|
||||
Buffer.from('Edith', 'ascii'),
|
||||
Buffer.from('Faina', 'ascii'),
|
||||
Buffer.from('Georgia', 'ascii'),
|
||||
Buffer.from('Hannah', 'ascii'),
|
||||
Buffer.from('Ilsbeth', 'ascii'),
|
||||
Buffer.from('Jennifer', 'ascii'),
|
||||
Buffer.from('Kayla', 'ascii'),
|
||||
Buffer.from('Lena', 'ascii'),
|
||||
Buffer.from('Michelle', 'ascii'),
|
||||
Buffer.from('Natalie', 'ascii'),
|
||||
Buffer.from('Ophelia', 'ascii'),
|
||||
Buffer.from('Peggy', 'ascii'),
|
||||
Buffer.from('Queenie', 'ascii')
|
||||
];
|
||||
|
||||
const op1 = new Outpoint(
|
||||
'4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c',
|
||||
4);
|
||||
|
||||
const op2 = new Outpoint(
|
||||
'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314',
|
||||
3);
|
||||
|
||||
const op3 = new Outpoint(
|
||||
'4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c',
|
||||
400);
|
||||
|
||||
const op4 = new Outpoint(
|
||||
'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314',
|
||||
300);
|
||||
|
||||
const addr1 = new Address('bc1qmyrddmxglk49ye2wd29wefaavw7es8k5d555lx');
|
||||
const addr2 = new Address('bc1q4645ycu0l9pnvxaxnhemushv0w4cd9flkqh95j');
|
||||
|
||||
let filter1 = null;
|
||||
let filter2 = null;
|
||||
let filter3 = null;
|
||||
let filter4 = null;
|
||||
let filter5 = null;
|
||||
|
||||
describe('GCS', function() {
|
||||
it('should test GCS filter build', () => {
|
||||
filter1 = GCSFilter.fromItems(P, key, contents1);
|
||||
assert(filter1);
|
||||
});
|
||||
|
||||
it('should test GCS filter copy', () => {
|
||||
filter2 = GCSFilter.fromBytes(filter1.n, P, filter1.toBytes());
|
||||
assert(filter2);
|
||||
filter3 = GCSFilter.fromNBytes(P, filter1.toNBytes());
|
||||
assert(filter3);
|
||||
filter4 = GCSFilter.fromPBytes(filter1.n, filter1.toPBytes());
|
||||
assert(filter4);
|
||||
filter5 = GCSFilter.fromNPBytes(filter1.toNPBytes());
|
||||
assert(filter5);
|
||||
});
|
||||
|
||||
it('should test GCS filter metadata', () => {
|
||||
assert.strictEqual(filter1.p, P);
|
||||
assert.strictEqual(filter1.n, contents1.length);
|
||||
assert.strictEqual(filter1.p, filter2.p);
|
||||
assert.strictEqual(filter1.n, filter2.n);
|
||||
assert.bufferEqual(filter1.data, filter2.data);
|
||||
assert.strictEqual(filter1.p, filter3.p);
|
||||
assert.strictEqual(filter1.n, filter3.n);
|
||||
assert.bufferEqual(filter1.data, filter3.data);
|
||||
assert.strictEqual(filter1.p, filter4.p);
|
||||
assert.strictEqual(filter1.n, filter4.n);
|
||||
assert.bufferEqual(filter1.data, filter4.data);
|
||||
assert.strictEqual(filter1.p, filter5.p);
|
||||
assert.strictEqual(filter1.n, filter5.n);
|
||||
assert.bufferEqual(filter1.data, filter5.data);
|
||||
});
|
||||
|
||||
it('should test GCS filter match', () => {
|
||||
let match = filter1.match(key, Buffer.from('Nate'));
|
||||
assert(match);
|
||||
match = filter2.match(key, Buffer.from('Nate'));
|
||||
assert(match);
|
||||
match = filter1.match(key, Buffer.from('Quentin'));
|
||||
assert(match);
|
||||
match = filter2.match(key, Buffer.from('Quentin'));
|
||||
assert(match);
|
||||
|
||||
match = filter1.match(key, Buffer.from('Nates'));
|
||||
assert(!match);
|
||||
match = filter2.match(key, Buffer.from('Nates'));
|
||||
assert(!match);
|
||||
match = filter1.match(key, Buffer.from('Quentins'));
|
||||
assert(!match);
|
||||
match = filter2.match(key, Buffer.from('Quentins'));
|
||||
assert(!match);
|
||||
});
|
||||
|
||||
it('should test GCS filter matchAny', () => {
|
||||
let match = filter1.matchAny(key, contents2);
|
||||
assert(!match);
|
||||
match = filter2.matchAny(key, contents2);
|
||||
assert(!match);
|
||||
|
||||
const contents = contents2.slice();
|
||||
contents.push(Buffer.from('Nate'));
|
||||
|
||||
match = filter1.matchAny(key, contents);
|
||||
assert(match);
|
||||
match = filter2.matchAny(key, contents);
|
||||
assert(match);
|
||||
});
|
||||
|
||||
it('should test GCS filter fromBlock', () => {
|
||||
const key = block.hash().slice(0, 16);
|
||||
const filter = GCSFilter.fromBlock(block);
|
||||
assert(filter.match(key, op1.toRaw()));
|
||||
assert(filter.match(key, op2.toRaw()));
|
||||
assert(!filter.match(key, op3.toRaw()));
|
||||
assert(!filter.match(key, op4.toRaw()));
|
||||
assert(filter.match(key, addr1.hash));
|
||||
assert(filter.match(key, addr2.hash));
|
||||
assert(filter.matchAny(key, [op1.toRaw(), addr1.hash]));
|
||||
assert(filter.matchAny(key, [op1.toRaw(), op3.toRaw()]));
|
||||
assert(!filter.matchAny(key, [op3.toRaw(), op4.toRaw()]));
|
||||
});
|
||||
|
||||
it('should test GCS filter fromExtended', () => {
|
||||
const key = block.hash().slice(0, 16);
|
||||
const filter = GCSFilter.fromExtended(block);
|
||||
assert(!filter.match(key, op1.toRaw()));
|
||||
assert(filter.match(key, block.txs[0].hash()));
|
||||
assert(filter.match(key, block.txs[1].hash()));
|
||||
assert(filter.matchAny(key, [block.txs[0].hash(), block.txs[1].hash()]));
|
||||
assert(filter.matchAny(key, [op1.toRaw(), block.txs[1].hash()]));
|
||||
assert(!filter.matchAny(key, [op1.toRaw(), op2.toRaw()]));
|
||||
});
|
||||
});
|
||||
@ -10,7 +10,7 @@ const assert = require('assert');
|
||||
const Network = require('../../lib/protocol/network');
|
||||
const MTX = require('../../lib/primitives/mtx');
|
||||
const HD = require('../../lib/hd/hd');
|
||||
const Bloom = require('../../lib/utils/bloom');
|
||||
const BloomFilter = require('bfilter/lib/bloom');
|
||||
const KeyRing = require('../../lib/primitives/keyring');
|
||||
const Outpoint = require('../../lib/primitives/outpoint');
|
||||
const Coin = require('../../lib/primitives/coin');
|
||||
@ -34,7 +34,7 @@ function MemWallet(options) {
|
||||
this.paths = new Map();
|
||||
this.balance = 0;
|
||||
this.txs = 0;
|
||||
this.filter = Bloom.fromRate(1000000, 0.001, -1);
|
||||
this.filter = BloomFilter.fromRate(1000000, 0.001, -1);
|
||||
|
||||
if (options)
|
||||
this.fromOptions(options);
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('./util/assert');
|
||||
const {U64, I64} = require('../lib/utils/int64');
|
||||
const {U64, I64} = require('n64');
|
||||
const base58 = require('../lib/utils/base58');
|
||||
const encoding = require('../lib/utils/encoding');
|
||||
const Amount = require('../lib/btc/amount');
|
||||
|
||||
Loading…
Reference in New Issue
Block a user