From e92b1f4cec97ee5adcdbcf1b0c180430dd0acb32 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Mon, 30 Oct 2017 21:22:31 -0700 Subject: [PATCH] utils: start using bfilter. --- lib/bcoin-browser.js | 3 +- lib/bcoin.js | 1 - lib/mempool/mempool.js | 2 +- lib/net/hostlist.js | 6 +- lib/net/packets.js | 6 +- lib/net/peer.js | 2 +- lib/net/pool.js | 8 +- lib/node/http.js | 4 +- lib/primitives/tx.js | 8 +- lib/script/scriptnum.js | 2 +- lib/utils/bloom.js | 383 -------------------------- lib/utils/encoding.js | 2 +- lib/utils/gcs.js | 542 ------------------------------------- lib/utils/index.js | 10 +- lib/utils/murmur3.js | 112 -------- lib/utils/protoreader.js | 221 --------------- lib/utils/protowriter.js | 182 ------------- lib/utils/rollingfilter.js | 255 ----------------- lib/wallet/walletdb.js | 10 +- test/block-test.js | 4 +- test/bloom-test.js | 183 ------------- test/gcs-test.js | 177 ------------ test/util/memwallet.js | 4 +- test/utils-test.js | 2 +- 24 files changed, 32 insertions(+), 2097 deletions(-) delete mode 100644 lib/utils/bloom.js delete mode 100644 lib/utils/gcs.js delete mode 100644 lib/utils/murmur3.js delete mode 100644 lib/utils/protoreader.js delete mode 100644 lib/utils/protowriter.js delete mode 100644 lib/utils/rollingfilter.js delete mode 100644 test/bloom-test.js delete mode 100644 test/gcs-test.js diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 29570261..8fad08f8 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -125,10 +125,9 @@ bcoin.witness = require('./script/witness'); bcoin.utils = require('./utils'); bcoin.base32 = require('./utils/base32'); bcoin.base58 = require('./utils/base58'); -bcoin.bloom = require('./utils/bloom'); +bcoin.bloom = require('bfilter/lib/bloom'); bcoin.co = require('./utils/co'); bcoin.encoding = require('./utils/encoding'); -bcoin.int64 = require('./utils/int64'); bcoin.lock = require('./utils/lock'); bcoin.reader = require('./utils/reader'); bcoin.staticwriter = require('./utils/staticwriter'); diff --git a/lib/bcoin.js b/lib/bcoin.js index b93ebcf2..ce859b78 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -163,7 +163,6 @@ bcoin.define('base58', './utils/base58'); bcoin.define('bloom', './utils/bloom'); bcoin.define('co', './utils/co'); bcoin.define('encoding', './utils/encoding'); -bcoin.define('int64', './utils/int64'); bcoin.define('lock', './utils/lock'); bcoin.define('reader', './utils/reader'); bcoin.define('staticwriter', './utils/staticwriter'); diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index a1c85202..ba45e532 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -16,7 +16,7 @@ const policy = require('../protocol/policy'); const util = require('../utils/util'); const random = require('bcrypto/lib/random'); const {VerifyError} = require('../protocol/errors'); -const RollingFilter = require('../utils/rollingfilter'); +const RollingFilter = require('bfilter/lib/rolling'); const Address = require('../primitives/address'); const Script = require('../script/script'); const Outpoint = require('../primitives/outpoint'); diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index d3973e22..4677bae7 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -9,13 +9,13 @@ const assert = require('assert'); const path = require('path'); const fs = require('bfile'); +const murmur3 = require('bfilter/lib/murmur3'); const util = require('../utils/util'); const IP = require('../utils/ip'); const co = require('../utils/co'); const Network = require('../protocol/network'); const NetAddress = require('../primitives/netaddress'); const List = require('../utils/list'); -const murmur3 = require('../utils/murmur3'); const common = require('./common'); const seeds = require('./seeds'); const dns = require('./dns'); @@ -461,7 +461,7 @@ HostList.prototype.freshBucket = function freshBucket(entry) { const addr = entry.addr; const src = entry.src; const data = concat32(addr.raw, src.raw); - const hash = murmur3(data, 0xfba4c795); + const hash = murmur3.sum(data, 0xfba4c795); const index = hash % this.fresh.length; return this.fresh[index]; }; @@ -475,7 +475,7 @@ HostList.prototype.freshBucket = function freshBucket(entry) { HostList.prototype.usedBucket = function usedBucket(entry) { const addr = entry.addr; - const hash = murmur3(addr.raw, 0xfba4c795); + const hash = murmur3.sum(addr.raw, 0xfba4c795); const index = hash % this.used.length; return this.used[index]; }; diff --git a/lib/net/packets.js b/lib/net/packets.js index d6d1b4b3..c6c661ff 100644 --- a/lib/net/packets.js +++ b/lib/net/packets.js @@ -14,7 +14,7 @@ const common = require('./common'); const util = require('../utils/util'); const assert = require('assert'); -const Bloom = require('../utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const bip152 = require('./bip152'); const NetAddress = require('../primitives/netaddress'); const Headers = require('../primitives/headers'); @@ -1810,7 +1810,7 @@ MempoolPacket.fromRaw = function fromRaw(data, enc) { /** * Represents a `filterload` packet. * @constructor - * @param {Bloom|null} filter + * @param {BloomFilter|null} filter */ function FilterLoadPacket(filter) { @@ -1819,7 +1819,7 @@ function FilterLoadPacket(filter) { Packet.call(this); - this.filter = filter || new Bloom(); + this.filter = filter || new BloomFilter(); } Object.setPrototypeOf(FilterLoadPacket.prototype, Packet.prototype); diff --git a/lib/net/peer.js b/lib/net/peer.js index 6300a9f4..838331a4 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -18,7 +18,7 @@ const consensus = require('../protocol/consensus'); const common = require('./common'); const InvItem = require('../primitives/invitem'); const Lock = require('../utils/lock'); -const RollingFilter = require('../utils/rollingfilter'); +const RollingFilter = require('bfilter/lib/rolling'); const BIP151 = require('./bip151'); const BIP150 = require('./bip150'); const BIP152 = require('./bip152'); diff --git a/lib/net/pool.js b/lib/net/pool.js index 6f8a8502..545d1c22 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -19,8 +19,8 @@ const Address = require('../primitives/address'); const BIP150 = require('./bip150'); const BIP151 = require('./bip151'); const BIP152 = require('./bip152'); -const Bloom = require('../utils/bloom'); -const RollingFilter = require('../utils/rollingfilter'); +const BloomFilter = require('bfilter/lib/bloom'); +const RollingFilter = require('bfilter/lib/rolling'); const secp256k1 = require('bcrypto/lib/secp256k1'); const Lock = require('../utils/lock'); const Network = require('../protocol/network'); @@ -112,7 +112,7 @@ function Pool(options) { this.id = 0; if (this.options.spv) - this.spvFilter = Bloom.fromRate(20000, 0.001, Bloom.flags.ALL); + this.spvFilter = BloomFilter.fromRate(20000, 0.001, BloomFilter.flags.ALL); if (!this.options.mempool) this.txFilter = new RollingFilter(50000, 0.000001); @@ -3199,7 +3199,7 @@ Pool.prototype.unban = function unban(addr) { /** * Set the spv filter. - * @param {Bloom} filter + * @param {BloomFilter} filter * @param {String?} enc */ diff --git a/lib/node/http.js b/lib/node/http.js index 193ff76e..cb5035a6 100644 --- a/lib/node/http.js +++ b/lib/node/http.js @@ -12,7 +12,7 @@ const path = require('path'); const {Server} = require('bweb'); const util = require('../utils/util'); const base58 = require('../utils/base58'); -const Bloom = require('../utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const TX = require('../primitives/tx'); const Outpoint = require('../primitives/outpoint'); const digest = require('bcrypto/lib/digest'); @@ -413,7 +413,7 @@ class HTTP extends Server { if (!data) throw new Error('Invalid parameter.'); - socket.filter = Bloom.fromRaw(data); + socket.filter = BloomFilter.fromRaw(data); return null; }); diff --git a/lib/primitives/tx.js b/lib/primitives/tx.js index afade750..330bce23 100644 --- a/lib/primitives/tx.js +++ b/lib/primitives/tx.js @@ -21,7 +21,7 @@ const Input = require('./input'); const Output = require('./output'); const Outpoint = require('./outpoint'); const InvItem = require('./invitem'); -const Bloom = require('../utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const consensus = require('../protocol/consensus'); const policy = require('../protocol/policy'); const ScriptError = require('../script/scripterror'); @@ -1955,7 +1955,7 @@ TX.prototype.getPrevout = function getPrevout() { * value is. * @see "Filter matching algorithm": * @see https://github.com/bitcoin/bips/blob/master/bip-0037.mediawiki - * @param {Bloom} filter + * @param {BloomFilter} filter * @returns {Boolean} True if the transaction matched. */ @@ -1972,10 +1972,10 @@ TX.prototype.isWatched = function isWatched(filter) { const output = this.outputs[i]; // Test the output script if (output.script.test(filter)) { - if (filter.update === Bloom.flags.ALL) { + if (filter.update === BloomFilter.flags.ALL) { const prevout = Outpoint.fromTX(this, i); filter.add(prevout.toRaw()); - } else if (filter.update === Bloom.flags.PUBKEY_ONLY) { + } else if (filter.update === BloomFilter.flags.PUBKEY_ONLY) { if (output.script.isPubkey() || output.script.isMultisig()) { const prevout = Outpoint.fromTX(this, i); filter.add(prevout.toRaw()); diff --git a/lib/script/scriptnum.js b/lib/script/scriptnum.js index bd2cd0d7..be1fa28c 100644 --- a/lib/script/scriptnum.js +++ b/lib/script/scriptnum.js @@ -7,7 +7,7 @@ 'use strict'; const assert = require('assert'); -const {I64} = require('../utils/int64'); +const {I64} = require('n64'); const ScriptError = require('./scripterror'); /* diff --git a/lib/utils/bloom.js b/lib/utils/bloom.js deleted file mode 100644 index f228e2f1..00000000 --- a/lib/utils/bloom.js +++ /dev/null @@ -1,383 +0,0 @@ -/*! - * bloom.js - bloom filter for bcoin - * Copyright (c) 2014-2015, Fedor Indutny (MIT License) - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); -const murmur3 = require('./murmur3'); -const BufferReader = require('./reader'); -const StaticWriter = require('./staticwriter'); -const encoding = require('./encoding'); -const sum32 = murmur3.sum32; -const mul32 = murmur3.mul32; -const DUMMY = Buffer.alloc(0); - -/* - * Constants - */ - -const LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455; -const LN2 = 0.6931471805599453094172321214581765680755001343602552; - -/** - * Bloom Filter - * @alias module:utils.Bloom - * @constructor - * @param {Number} size - Filter size in bits. - * @param {Number} n - Number of hash functions. - * @param {Number} tweak - Seed value. - * @param {Number|String} - Update type. - * @property {Buffer} filter - * @property {Number} size - * @property {Number} n - * @property {Number} tweak - * @property {Number} update - Update flag (see {@link Bloom.flags}). - */ - -function Bloom(size, n, tweak, update) { - if (!(this instanceof Bloom)) - return new Bloom(size, n, tweak, update); - - this.filter = DUMMY; - this.size = 0; - this.n = 0; - this.tweak = 0; - this.update = Bloom.flags.NONE; - - if (size != null) - this.fromOptions(size, n, tweak, update); -} - -/** - * Max bloom filter size. - * @const {Number} - * @default - */ - -Bloom.MAX_BLOOM_FILTER_SIZE = 36000; - -/** - * Max number of hash functions. - * @const {Number} - * @default - */ - -Bloom.MAX_HASH_FUNCS = 50; - -/** - * Bloom filter update flags. - * @enum {Number} - * @default - */ - -Bloom.flags = { - /** - * Never update the filter with outpoints. - */ - - NONE: 0, - - /** - * Always update the filter with outpoints. - */ - - ALL: 1, - - /** - * Only update the filter with outpoints if it is - * "asymmetric" in terms of addresses (pubkey/multisig). - */ - - PUBKEY_ONLY: 2 -}; - -/** - * Bloom filter update flags by value. - * @const {RevMap} - */ - -Bloom.flagsByVal = { - 0: 'NONE', - 1: 'ALL', - 2: 'PUBKEY_ONLY' -}; - -/** - * Inject properties from options. - * @private - * @param {Number} size - Filter size in bits. - * @param {Number} n - Number of hash functions. - * @param {Number} tweak - Seed value. - * @param {Number|String} - Update type. - * @returns {Bloom} - */ - -Bloom.prototype.fromOptions = function fromOptions(size, n, tweak, update) { - assert(typeof size === 'number', '`size` must be a number.'); - assert(size > 0, '`size` must be greater than zero.'); - assert(Number.isSafeInteger(size), '`size` must be an integer.'); - - size -= size % 8; - - const filter = Buffer.allocUnsafe(size / 8); - filter.fill(0); - - if (tweak == null || tweak === -1) - tweak = (Math.random() * 0x100000000) >>> 0; - - if (update == null || update === -1) - update = Bloom.flags.NONE; - - if (typeof update === 'string') { - update = Bloom.flags[update.toUpperCase()]; - assert(update != null, 'Unknown update flag.'); - } - - assert(size > 0, '`size` must be greater than zero.'); - assert(n > 0, '`n` must be greater than zero.'); - assert(Number.isSafeInteger(n), '`n` must be an integer.'); - assert(typeof tweak === 'number', '`tweak` must be a number.'); - assert(Number.isSafeInteger(tweak), '`tweak` must be an integer.'); - assert(Bloom.flagsByVal[update], 'Unknown update flag.'); - - this.filter = filter; - this.size = size; - this.n = n; - this.tweak = tweak; - this.update = update; - - return this; -}; - -/** - * Instantiate bloom filter from options. - * @param {Number} size - Filter size in bits. - * @param {Number} n - Number of hash functions. - * @param {Number} tweak - Seed value. - * @param {Number|String} - Update type. - * @returns {Bloom} - */ - -Bloom.fromOptions = function fromOptions(size, n, tweak, update) { - return new Bloom().fromOptions(size, n, tweak, update); -}; - -/** - * Perform the mumur3 hash on data. - * @param {Buffer} val - * @param {Number} n - * @returns {Number} - */ - -Bloom.prototype.hash = function hash(val, n) { - return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak)) % this.size; -}; - -/** - * Reset the filter. - */ - -Bloom.prototype.reset = function reset() { - this.filter.fill(0); -}; - -/** - * Add data to the filter. - * @param {Buffer|String} - * @param {String?} enc - Can be any of the Buffer object's encodings. - */ - -Bloom.prototype.add = function add(val, enc) { - if (typeof val === 'string') - val = Buffer.from(val, enc); - - for (let i = 0; i < this.n; i++) { - const index = this.hash(val, i); - this.filter[index >>> 3] |= 1 << (7 & index); - } -}; - -/** - * Test whether data is present in the filter. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} - */ - -Bloom.prototype.test = function test(val, enc) { - if (typeof val === 'string') - val = Buffer.from(val, enc); - - for (let i = 0; i < this.n; i++) { - const index = this.hash(val, i); - if ((this.filter[index >>> 3] & (1 << (7 & index))) === 0) - return false; - } - - return true; -}; - -/** - * Test whether data is present in the - * filter and potentially add data. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} Whether data was added. - */ - -Bloom.prototype.added = function added(val, enc) { - let ret = false; - - if (typeof val === 'string') - val = Buffer.from(val, enc); - - for (let i = 0; i < this.n; i++) { - const index = this.hash(val, i); - if (!ret && (this.filter[index >>> 3] & (1 << (7 & index))) === 0) - ret = true; - this.filter[index >>> 3] |= 1 << (7 & index); - } - - return ret; -}; - -/** - * Create a filter from a false positive rate. - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - * @param {Number|String} update - * @example - * Bloom.fromRate(800000, 0.0001, 'none'); - * @returns {Boolean} - */ - -Bloom.fromRate = function fromRate(items, rate, update) { - assert(typeof items === 'number', '`items` must be a number.'); - assert(items > 0, '`items` must be greater than zero.'); - assert(Number.isSafeInteger(items), '`items` must be an integer.'); - assert(typeof rate === 'number', '`rate` must be a number.'); - assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.'); - - const bits = (-1 / LN2SQUARED * items * Math.log(rate)) | 0; - const size = Math.max(8, bits); - - if (update !== -1) { - assert(size <= Bloom.MAX_BLOOM_FILTER_SIZE * 8, - 'Bloom filter size violates policy limits!'); - } - - const n = Math.max(1, (size / items * LN2) | 0); - - if (update !== -1) { - assert(n <= Bloom.MAX_HASH_FUNCS, - 'Bloom filter size violates policy limits!'); - } - - return new Bloom(size, n, -1, update); -}; - -/** - * Ensure the filter is within the size limits. - * @returns {Boolean} - */ - -Bloom.prototype.isWithinConstraints = function isWithinConstraints() { - if (this.size > Bloom.MAX_BLOOM_FILTER_SIZE * 8) - return false; - - if (this.n > Bloom.MAX_HASH_FUNCS) - return false; - - return true; -}; - -/** - * Get serialization size. - * @returns {Number} - */ - -Bloom.prototype.getSize = function getSize() { - return encoding.sizeVarBytes(this.filter) + 9; -}; - -/** - * Write filter to buffer writer. - * @param {BufferWriter} bw - */ - -Bloom.prototype.toWriter = function toWriter(bw) { - bw.writeVarBytes(this.filter); - bw.writeU32(this.n); - bw.writeU32(this.tweak); - bw.writeU8(this.update); - return bw; -}; - -/** - * Serialize bloom filter. - * @returns {Buffer} - */ - -Bloom.prototype.toRaw = function toRaw() { - const size = this.getSize(); - return this.toWriter(new StaticWriter(size)).render(); -}; - -/** - * Inject properties from buffer reader. - * @private - * @param {BufferReader} br - */ - -Bloom.prototype.fromReader = function fromReader(br) { - this.filter = br.readVarBytes(); - this.size = this.filter.length * 8; - this.n = br.readU32(); - this.tweak = br.readU32(); - this.update = br.readU8(); - assert(Bloom.flagsByVal[this.update] != null, 'Unknown update flag.'); - return this; -}; - -/** - * Inject properties from serialized data. - * @private - * @param {Buffer} data - */ - -Bloom.prototype.fromRaw = function fromRaw(data) { - return this.fromReader(new BufferReader(data)); -}; - -/** - * Instantiate bloom filter from buffer reader. - * @param {BufferReader} br - * @returns {Bloom} - */ - -Bloom.fromReader = function fromReader(br) { - return new Bloom().fromReader(br); -}; - -/** - * Instantiate bloom filter from serialized data. - * @param {Buffer} data - * @param {String?} enc - * @returns {Bloom} - */ - -Bloom.fromRaw = function fromRaw(data, enc) { - if (typeof data === 'string') - data = Buffer.from(data, enc); - return new Bloom().fromRaw(data); -}; - -/* - * Expose - */ - -module.exports = Bloom; diff --git a/lib/utils/encoding.js b/lib/utils/encoding.js index af90305b..0cf7ced7 100644 --- a/lib/utils/encoding.js +++ b/lib/utils/encoding.js @@ -12,7 +12,7 @@ */ const assert = require('assert'); -const {U64, I64} = require('./int64'); +const {U64, I64} = require('n64'); const UINT128_MAX = U64.UINT64_MAX.shrn(7); const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER; const encoding = exports; diff --git a/lib/utils/gcs.js b/lib/utils/gcs.js deleted file mode 100644 index b1d19d84..00000000 --- a/lib/utils/gcs.js +++ /dev/null @@ -1,542 +0,0 @@ -/*! - * gcs.js - gcs filters for bcoin - * Copyright (c) 2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); -const {U64} = require('./int64'); -const hash256 = require('bcrypto/lib/hash256'); -const siphash = require('bcrypto/lib/siphash'); -const DUMMY = Buffer.alloc(0); -const EOF = new U64(-1); - -/** - * GCSFilter - * @alias module:utils.GCSFilter - * @constructor - */ - -function GCSFilter() { - this.n = 0; - this.p = 0; - this.m = new U64(0); - this.data = DUMMY; -} - -GCSFilter.prototype.hash = function hash(enc) { - const h = hash256.digest(this.data); - return enc === 'hex' ? h.toString('hex') : h; -}; - -GCSFilter.prototype.header = function header(prev) { - return hash256.root(this.hash(), prev); -}; - -GCSFilter.prototype.match = function match(key, data) { - const br = new BitReader(this.data); - const term = siphash24(data, key).imod(this.m); - - let last = new U64(0); - - while (last.lt(term)) { - const value = this.readU64(br); - - if (value === EOF) - return false; - - value.iadd(last); - - if (value.eq(term)) - return true; - - last = value; - } - - return false; -}; - -GCSFilter.prototype.matchAny = function matchAny(key, items) { - assert(items.length > 0); - - const br = new BitReader(this.data); - const last1 = new U64(0); - const values = []; - - for (const item of items) { - const hash = siphash24(item, key).imod(this.m); - values.push(hash); - } - - values.sort(compare); - - let last2 = values[0]; - let i = 1; - - for (;;) { - const cmp = last1.cmp(last2); - - if (cmp === 0) - break; - - if (cmp > 0) { - if (i < values.length) { - last2 = values[i]; - i += 1; - continue; - } - return false; - } - - const value = this.readU64(br); - - if (value === EOF) - return false; - - last1.iadd(value); - } - - return true; -}; - -GCSFilter.prototype.readU64 = function readU64(br) { - try { - return this._readU64(br); - } catch (e) { - if (e.message === 'EOF') - return EOF; - throw e; - } -}; - -GCSFilter.prototype._readU64 = function _readU64(br) { - const num = new U64(0); - - // Unary - while (br.readBit()) - num.iaddn(1); - - const rem = br.readBits64(this.p); - - return num.ishln(this.p).ior(rem); -}; - -GCSFilter.prototype.toBytes = function toBytes() { - return this.data; -}; - -GCSFilter.prototype.toNBytes = function toNBytes() { - const data = Buffer.allocUnsafe(4 + this.data.length); - data.writeUInt32BE(this.n, 0, true); - this.data.copy(data, 4); - return data; -}; - -GCSFilter.prototype.toPBytes = function toPBytes() { - const data = Buffer.allocUnsafe(1 + this.data.length); - data.writeUInt8(this.p, 0, true); - this.data.copy(data, 1); - return data; -}; - -GCSFilter.prototype.toNPBytes = function toNPBytes() { - const data = Buffer.allocUnsafe(5 + this.data.length); - data.writeUInt32BE(this.n, 0, true); - data.writeUInt8(this.p, 4, true); - this.data.copy(data, 5); - return data; -}; - -GCSFilter.prototype.toRaw = function toRaw() { - assert(this.p === 20); - return this.toNBytes(); -}; - -GCSFilter.prototype.fromItems = function fromItems(P, key, items) { - assert(typeof P === 'number' && isFinite(P)); - assert(P >= 0 && P <= 32); - - assert(Buffer.isBuffer(key)); - assert(key.length === 16); - - assert(Array.isArray(items)); - assert(items.length > 0); - assert(items.length <= 0xffffffff); - - this.n = items.length; - this.p = P; - this.m = U64(this.n).ishln(this.p); - - const values = []; - - for (const item of items) { - assert(Buffer.isBuffer(item)); - const hash = siphash24(item, key).imod(this.m); - values.push(hash); - } - - values.sort(compare); - - const bw = new BitWriter(); - - let last = new U64(0); - - for (const hash of values) { - const rem = hash.sub(last).imaskn(this.p); - const value = hash.sub(last).isub(rem).ishrn(this.p); - - last = hash; - - // Unary - while (!value.isZero()) { - bw.writeBit(1); - value.isubn(1); - } - bw.writeBit(0); - - bw.writeBits64(rem, this.p); - } - - this.data = bw.render(); - - return this; -}; - -GCSFilter.prototype.fromBytes = function fromBytes(N, P, data) { - assert(typeof N === 'number' && isFinite(N)); - assert(typeof P === 'number' && isFinite(P)); - assert(P >= 0 && P <= 32); - assert(Buffer.isBuffer(data)); - - this.n = N; - this.p = P; - this.m = U64(this.n).ishln(this.p); - this.data = data; - - return this; -}; - -GCSFilter.prototype.fromNBytes = function fromNBytes(P, data) { - assert(typeof P === 'number' && isFinite(P)); - assert(Buffer.isBuffer(data)); - assert(data.length >= 4); - - const N = data.readUInt32BE(0, true); - - return this.fromBytes(N, P, data.slice(4)); -}; - -GCSFilter.prototype.fromPBytes = function fromPBytes(N, data) { - assert(typeof N === 'number' && isFinite(N)); - assert(Buffer.isBuffer(data)); - assert(data.length >= 1); - - const P = data.readUInt8(0, true); - - return this.fromBytes(N, P, data.slice(1)); -}; - -GCSFilter.prototype.fromNPBytes = function fromNPBytes(data) { - assert(Buffer.isBuffer(data)); - assert(data.length >= 5); - - const N = data.readUInt32BE(0, true); - const P = data.readUInt8(4, true); - - return this.fromBytes(N, P, data.slice(5)); -}; - -GCSFilter.prototype.fromRaw = function fromRaw(data) { - return this.fromNBytes(20, data); -}; - -GCSFilter.prototype.fromBlock = function fromBlock(block) { - const hash = block.hash(); - const key = hash.slice(0, 16); - const items = []; - - for (let i = 0; i < block.txs.length; i++) { - const tx = block.txs[i]; - - if (i > 0) { - for (const input of tx.inputs) - items.push(input.prevout.toRaw()); - } - - for (const output of tx.outputs) - getPushes(items, output.script); - } - - return this.fromItems(20, key, items); -}; - -GCSFilter.prototype.fromExtended = function fromExtended(block) { - const hash = block.hash(); - const key = hash.slice(0, 16); - const items = []; - - for (let i = 0; i < block.txs.length; i++) { - const tx = block.txs[i]; - - items.push(tx.hash()); - - if (i > 0) { - for (const input of tx.inputs) { - getWitness(items, input.witness); - getPushes(items, input.script); - } - } - } - - return this.fromItems(20, key, items); -}; - -GCSFilter.fromItems = function fromItems(P, key, items) { - return new GCSFilter().fromItems(P, key, items); -}; - -GCSFilter.fromBytes = function fromBytes(N, P, data) { - return new GCSFilter().fromBytes(N, P, data); -}; - -GCSFilter.fromNBytes = function fromNBytes(P, data) { - return new GCSFilter().fromNBytes(P, data); -}; - -GCSFilter.fromPBytes = function fromPBytes(N, data) { - return new GCSFilter().fromPBytes(N, data); -}; - -GCSFilter.fromNPBytes = function fromNPBytes(data) { - return new GCSFilter().fromNPBytes(data); -}; - -GCSFilter.fromRaw = function fromRaw(data) { - return new GCSFilter().fromRaw(data); -}; - -GCSFilter.fromBlock = function fromBlock(block) { - return new GCSFilter().fromBlock(block); -}; - -GCSFilter.fromExtended = function fromExtended(block) { - return new GCSFilter().fromExtended(block); -}; - -/** - * BitWriter - * @constructor - * @ignore - */ - -function BitWriter() { - this.stream = []; - this.remain = 0; -} - -BitWriter.prototype.writeBit = function writeBit(bit) { - if (this.remain === 0) { - this.stream.push(0); - this.remain = 8; - } - - if (bit) { - const index = this.stream.length - 1; - this.stream[index] |= 1 << (this.remain - 1); - } - - this.remain--; -}; - -BitWriter.prototype.writeByte = function writeByte(ch) { - if (this.remain === 0) { - this.stream.push(0); - this.remain = 8; - } - - const index = this.stream.length - 1; - - this.stream[index] |= (ch >> (8 - this.remain)) & 0xff; - this.stream.push(0); - this.stream[index + 1] = (ch << this.remain) & 0xff; -}; - -BitWriter.prototype.writeBits = function writeBits(num, count) { - assert(count >= 0); - assert(count <= 32); - - num <<= 32 - count; - - while (count >= 8) { - const ch = num >>> 24; - this.writeByte(ch); - num <<= 8; - count -= 8; - } - - while (count > 0) { - const bit = num >>> 31; - this.writeBit(bit); - num <<= 1; - count -= 1; - } -}; - -BitWriter.prototype.writeBits64 = function writeBits64(num, count) { - assert(count >= 0); - assert(count <= 64); - - if (count > 32) { - this.writeBits(num.hi, count - 32); - this.writeBits(num.lo, 32); - } else { - this.writeBits(num.lo, count); - } -}; - -BitWriter.prototype.render = function render() { - const data = Buffer.allocUnsafe(this.stream.length); - - for (let i = 0; i < this.stream.length; i++) - data[i] = this.stream[i]; - - return data; -}; - -/** - * BitReader - * @constructor - * @ignore - */ - -function BitReader(data) { - this.stream = data; - this.pos = 0; - this.remain = 8; -} - -BitReader.prototype.readBit = function readBit() { - if (this.pos >= this.stream.length) - throw new Error('EOF'); - - if (this.remain === 0) { - this.pos += 1; - - if (this.pos >= this.stream.length) - throw new Error('EOF'); - - this.remain = 8; - } - - this.remain -= 1; - - return (this.stream[this.pos] >> this.remain) & 1; -}; - -BitReader.prototype.readByte = function readByte() { - if (this.pos >= this.stream.length) - throw new Error('EOF'); - - if (this.remain === 0) { - this.pos += 1; - - if (this.pos >= this.stream.length) - throw new Error('EOF'); - - this.remain = 8; - } - - if (this.remain === 8) { - const ch = this.stream[this.pos]; - this.pos += 1; - return ch; - } - - let ch = this.stream[this.pos] & ((1 << this.remain) - 1); - ch <<= 8 - this.remain; - - this.pos += 1; - - if (this.pos >= this.stream.length) - throw new Error('EOF'); - - ch |= this.stream[this.pos] >> this.remain; - - return ch; -}; - -BitReader.prototype.readBits = function readBits(count) { - assert(count >= 0); - assert(count <= 32); - - let num = 0; - - while (count >= 8) { - num <<= 8; - num |= this.readByte(); - count -= 8; - } - - while (count > 0) { - num <<= 1; - num |= this.readBit(); - count -= 1; - } - - return num; -}; - -BitReader.prototype.readBits64 = function readBits64(count) { - assert(count >= 0); - assert(count <= 64); - - const num = new U64(); - - if (count > 32) { - num.hi = this.readBits(count - 32); - num.lo = this.readBits(32); - } else { - num.lo = this.readBits(count); - } - - return num; -}; - -/* - * Helpers - */ - -function compare(a, b) { - return a.cmp(b); -} - -function siphash24(data, key) { - const [hi, lo] = siphash(data, key); - return U64.fromBits(hi, lo); -} - -function getPushes(items, script) { - for (const op of script.code) { - if (!op.data || op.data.length === 0) - continue; - - items.push(op.data); - } -} - -function getWitness(items, witness) { - for (const item of witness.items) { - if (item.length === 0) - continue; - - items.push(item); - } -} - -/* - * Expose - */ - -module.exports = GCSFilter; diff --git a/lib/utils/index.js b/lib/utils/index.js index 54f0f748..8c03c2f4 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -10,31 +10,23 @@ * @module utils */ -// exports.AsyncEmitter = require('./asyncemitter'); +exports.AsyncEmitter = require('./asyncemitter'); exports.AsyncObject = require('./asyncobject'); exports.base32 = require('./base32'); exports.base58 = require('./base58'); exports.bech32 = require('./bech32'); exports.binary = require('./binary'); -exports.Bloom = require('./bloom'); exports.co = require('./co'); exports.encoding = require('./encoding'); exports.enforce = require('./enforce'); exports.fixed = require('./fixed'); -exports.GCSFilter = require('./gcs'); exports.Heap = require('./heap'); -exports.Int64 = require('./int64'); exports.IP = require('./ip'); exports.List = require('./list'); exports.Lock = require('./lock'); exports.LRU = require('./lru'); exports.MappedLock = require('./mappedlock'); -exports.murmur3 = require('./murmur3'); -exports.nfkd = require('./nfkd'); -exports.ProtoWriter = require('./protowriter'); -exports.ProtoReader = require('./protoreader'); exports.BufferReader = require('./reader'); -exports.RollingFilter = require('./rollingfilter'); exports.StaticWriter = require('./staticwriter'); exports.util = require('./util'); exports.Validator = require('./validator'); diff --git a/lib/utils/murmur3.js b/lib/utils/murmur3.js deleted file mode 100644 index 57bda5db..00000000 --- a/lib/utils/murmur3.js +++ /dev/null @@ -1,112 +0,0 @@ -/*! - * murmur3.js - murmur3 hash for bcoin - * Copyright (c) 2014-2015, Fedor Indutny (MIT License) - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const native = require('../native').binding; - -/** - * Murmur3 hash. - * @alias module:utils.murmur3 - * @param {Buffer} data - * @param {Number} seed - * @returns {Number} - */ - -function murmur3(data, seed) { - const tail = data.length - (data.length % 4); - const c1 = 0xcc9e2d51; - const c2 = 0x1b873593; - let h1 = seed; - let k1; - - for (let i = 0; i < tail; i += 4) { - k1 = (data[i + 3] << 24) - | (data[i + 2] << 16) - | (data[i + 1] << 8) - | data[i]; - k1 = mul32(k1, c1); - k1 = rotl32(k1, 15); - k1 = mul32(k1, c2); - h1 ^= k1; - h1 = rotl32(h1, 13); - h1 = sum32(mul32(h1, 5), 0xe6546b64); - } - - k1 = 0; - switch (data.length & 3) { - case 3: - k1 ^= data[tail + 2] << 16; - case 2: - k1 ^= data[tail + 1] << 8; - case 1: - k1 ^= data[tail + 0]; - k1 = mul32(k1, c1); - k1 = rotl32(k1, 15); - k1 = mul32(k1, c2); - h1 ^= k1; - } - - h1 ^= data.length; - h1 ^= h1 >>> 16; - h1 = mul32(h1, 0x85ebca6b); - h1 ^= h1 >>> 13; - h1 = mul32(h1, 0xc2b2ae35); - h1 ^= h1 >>> 16; - - if (h1 < 0) - h1 += 0x100000000; - - return h1; -} - -if (native) - murmur3 = native.murmur3; - -function mul32(a, b) { - const alo = a & 0xffff; - const blo = b & 0xffff; - const ahi = a >>> 16; - const bhi = b >>> 16; - - let lo = alo * blo; - let hi = (ahi * blo + bhi * alo) & 0xffff; - - hi += lo >>> 16; - lo &= 0xffff; - - let r = (hi << 16) | lo; - - if (r < 0) - r += 0x100000000; - - return r; -} - -function sum32(a, b) { - let r = (a + b) & 0xffffffff; - - if (r < 0) - r += 0x100000000; - - return r; -} - -function rotl32(w, b) { - return (w << b) | (w >>> (32 - b)); -} - -/** - * Expose - */ - -exports = murmur3; -exports.murmur3 = murmur3; -exports.mul32 = mul32; -exports.sum32 = sum32; -exports.rotl32 = rotl32; -module.exports = exports; diff --git a/lib/utils/protoreader.js b/lib/utils/protoreader.js deleted file mode 100644 index 7a6452ec..00000000 --- a/lib/utils/protoreader.js +++ /dev/null @@ -1,221 +0,0 @@ -/*! - * protoreader.js - protobufs for bcoin - * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); -const BufferReader = require('../utils/reader'); - -/* - * Constants - */ - -const wireType = { - VARINT: 0, - FIXED64: 1, - DELIMITED: 2, - START_GROUP: 3, - END_GROUP: 4, - FIXED32: 5 -}; - -/** - * ProtoBuf Reader - * @alias module:utils.ProtoReader - * @constructor - */ - -function ProtoReader(data, zeroCopy) { - if (!(this instanceof ProtoReader)) - return new ProtoReader(data, zeroCopy); - - BufferReader.call(this, data, zeroCopy); -} - -Object.setPrototypeOf(ProtoReader.prototype, BufferReader.prototype); - -ProtoReader.prototype.readVarint = function readVarint() { - const {size, value} = _readVarint(this.data, this.offset); - this.offset += size; - return value; -}; - -ProtoReader.prototype.readFieldValue = function readFieldValue(tag, opt) { - const field = this.readField(tag, opt); - - if (!field) - return -1; - - assert(field.value != null); - - return field.value; -}; - -ProtoReader.prototype.readFieldU64 = function readFieldU64(tag, opt) { - const field = this.readField(tag, opt); - - if (!field) - return -1; - - assert(field.type === wireType.VARINT || field.type === wireType.FIXED64); - - return field.value; -}; - -ProtoReader.prototype.readFieldU32 = function readFieldU32(tag, opt) { - const field = this.readField(tag, opt); - - if (!field) - return -1; - - assert(field.type === wireType.VARINT || field.type === wireType.FIXED32); - - return field.value; -}; - -ProtoReader.prototype.readFieldBytes = function readFieldBytes(tag, opt) { - const field = this.readField(tag, opt); - - if (!field) - return null; - - assert(field.data); - - return field.data; -}; - -ProtoReader.prototype.readFieldString = function readFieldString(tag, opt, enc) { - const field = this.readField(tag, opt); - - if (!field) - return null; - - assert(field.data); - - return field.data.toString(enc || 'utf8'); -}; - -ProtoReader.prototype.nextTag = function nextTag() { - if (this.left() === 0) - return -1; - - const field = this.readField(); - - this.seek(-field.size); - - return field.tag; -}; - -ProtoReader.prototype.readField = function readField(tag, opt) { - const offset = this.offset; - const header = this.readVarint(); - const field = new Field(header); - - if (tag != null && field.tag !== tag) { - assert(opt, 'Non-optional field not present.'); - this.offset = offset; - return null; - } - - switch (field.type) { - case wireType.VARINT: - field.value = this.readVarint(); - break; - case wireType.FIXED64: - field.value = this.readU64(); - break; - case wireType.DELIMITED: - field.data = this.readVarBytes(); - break; - case wireType.START_GROUP: - field.group = []; - for (;;) { - const inner = this.readField(); - if (inner.type === wireType.END_GROUP) - break; - field.group.push(inner); - } - break; - case wireType.END_GROUP: - assert(false, 'Unexpected end group.'); - break; - case wireType.FIXED32: - field.value = this.readU32(); - break; - default: - assert(false, 'Bad wire type.'); - break; - } - - field.size = this.offset - offset; - - return field; -}; - -/* - * Encoding - */ - -function _readVarint(data, off) { - let num = 0; - let ch = 0x80; - let size = 0; - - while (ch & 0x80) { - if (off >= data.length) { - num = 0; - break; - } - - ch = data[off++]; - - // Optimization for javascript insanity. - switch (size) { - case 0: - case 1: - case 2: - case 3: - num += (ch & 0x7f) << (7 * size); - break; - case 4: - num += (ch & 0x7f) * (1 << (7 * size)); - break; - default: - num += (ch & 0x7f) * Math.pow(2, 7 * size); - break; - } - - size++; - - assert(size < 7, 'Number exceeds 2^53-1.'); - } - - return new Varint(size, num); -} - -/* - * Helpers - */ - -function Field(header) { - this.tag = header >>> 3; - this.type = header & 7; - this.size = 0; - this.value = 0; - this.data = null; - this.group = null; -} - -function Varint(size, value) { - this.size = size; - this.value = value; -} - -/* - * Expose - */ - -module.exports = ProtoReader; diff --git a/lib/utils/protowriter.js b/lib/utils/protowriter.js deleted file mode 100644 index 5f1a7abd..00000000 --- a/lib/utils/protowriter.js +++ /dev/null @@ -1,182 +0,0 @@ -/*! - * protowriter.js - protobufs for bcoin - * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -/** - * @module utils/protobuf - */ - -const assert = require('assert'); -const BufferWriter = require('../utils/writer'); - -/* - * Constants - */ - -const wireType = { - VARINT: 0, - FIXED64: 1, - DELIMITED: 2, - START_GROUP: 3, - END_GROUP: 4, - FIXED32: 5 -}; - -/** - * ProtoBuf Writer - * @alias module:utils.ProtoWriter - * @constructor - */ - -function ProtoWriter() { - if (!(this instanceof ProtoWriter)) - return new ProtoWriter(); - - BufferWriter.call(this); -} - -Object.setPrototypeOf(ProtoWriter.prototype, BufferWriter.prototype); - -ProtoWriter.prototype.writeVarint = function writeVarint(num) { - const size = sizeVarint(num); - - // Avoid an extra allocation until - // we make bufferwriter more hackable. - // More insanity here... - switch (size) { - case 6: { - const value = slipVarint(num); - this.writeU32BE(value / 0x10000 | 0); - this.writeU16BE(value & 0xffff); - break; - } - case 5: { - const value = slipVarint(num); - this.writeU32BE(value / 0x100 | 0); - this.writeU8(value & 0xff); - break; - } - case 4: { - const value = slipVarint(num); - this.writeU32BE(value); - break; - } - case 3: { - const value = slipVarint(num); - this.writeU16BE(value >> 8); - this.writeU8(value & 0xff); - break; - } - case 2: { - const value = slipVarint(num); - this.writeU16BE(value); - break; - } - case 1: { - const value = slipVarint(num); - this.writeU8(value); - break; - } - default: { - const value = Buffer.allocUnsafe(size); - _writeVarint(value, num, 0); - this.writeBytes(value); - break; - } - } -}; - -ProtoWriter.prototype.writeFieldVarint = function writeFieldVarint(tag, value) { - const header = (tag << 3) | wireType.VARINT; - this.writeVarint(header); - this.writeVarint(value); -}; - -ProtoWriter.prototype.writeFieldU64 = function writeFieldU64(tag, value) { - assert(Number.isSafeInteger(value)); - this.writeFieldVarint(tag, value); -}; - -ProtoWriter.prototype.writeFieldU32 = function writeFieldU32(tag, value) { - assert(value <= 0xffffffff); - this.writeFieldVarint(tag, value); -}; - -ProtoWriter.prototype.writeFieldBytes = function writeFieldBytes(tag, data) { - const header = (tag << 3) | wireType.DELIMITED; - this.writeVarint(header); - this.writeVarint(data.length); - this.writeBytes(data); -}; - -ProtoWriter.prototype.writeFieldString = function writeFieldString(tag, data, enc) { - if (typeof data === 'string') - data = Buffer.from(data, enc || 'utf8'); - this.writeFieldBytes(tag, data); -}; - -/* - * Encoding - */ - -function _writeVarint(data, num, off) { - assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - do { - assert(off < data.length); - let ch = num & 0x7f; - num -= num % 0x80; - num /= 0x80; - if (num !== 0) - ch |= 0x80; - data[off] = ch; - off++; - } while (num > 0); - - return off; -}; - -function slipVarint(num) { - assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - let data = 0; - let size = 0; - - do { - assert(size < 7); - let ch = num & 0x7f; - num -= num % 0x80; - num /= 0x80; - if (num !== 0) - ch |= 0x80; - data *= 256; - data += ch; - size++; - } while (num > 0); - - return data; -} - -function sizeVarint(num) { - assert(Number.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - let size = 0; - - do { - num -= num % 0x80; - num /= 0x80; - size++; - } while (num > 0); - - return size; -}; - -/* - * Expose - */ - -module.exports = ProtoWriter; diff --git a/lib/utils/rollingfilter.js b/lib/utils/rollingfilter.js deleted file mode 100644 index df6d8e25..00000000 --- a/lib/utils/rollingfilter.js +++ /dev/null @@ -1,255 +0,0 @@ -/*! - * rollingfilter.js - rolling bloom filter for bcoin - * Copyright (c) 2014-2015, Fedor Indutny (MIT License) - * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const assert = require('assert'); -const murmur3 = require('./murmur3'); -const sum32 = murmur3.sum32; -const mul32 = murmur3.mul32; -const DUMMY = Buffer.alloc(0); - -/** - * A rolling bloom filter used internally - * (do not relay this on the p2p network). - * @alias module:utils.RollingFilter - * @constructor - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - */ - -function RollingFilter(items, rate) { - if (!(this instanceof RollingFilter)) - return new RollingFilter(items, rate); - - this.entries = 0; - this.generation = 1; - this.n = 0; - this.limit = 0; - this.size = 0; - this.items = 0; - this.tweak = 0; - this.filter = DUMMY; - - if (items != null) - this.fromRate(items, rate); -} - -/** - * Inject properties from items and FPR. - * @private - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - * @returns {RollingFilter} - */ - -RollingFilter.prototype.fromRate = function fromRate(items, rate) { - assert(typeof items === 'number', '`items` must be a number.'); - assert(items > 0, '`items` must be greater than zero.'); - assert(Number.isSafeInteger(items), '`items` must be an integer.'); - assert(typeof rate === 'number', '`rate` must be a number.'); - assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.'); - - const logRate = Math.log(rate); - - const n = Math.max(1, Math.min(Math.round(logRate / Math.log(0.5)), 50)); - const limit = (items + 1) / 2 | 0; - - const max = limit * 3; - - let size = -1 * n * max / Math.log(1.0 - Math.exp(logRate / n)); - size = Math.ceil(size); - - items = ((size + 63) / 64 | 0) << 1; - items >>>= 0; - items = Math.max(1, items); - - const tweak = (Math.random() * 0x100000000) >>> 0; - - const filter = Buffer.allocUnsafe(items * 8); - filter.fill(0); - - this.n = n; - this.limit = limit; - this.size = size; - this.items = items; - this.tweak = tweak; - this.filter = filter; - - return this; -}; - -/** - * Instantiate rolling filter from items and FPR. - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - * @returns {RollingFilter} - */ - -RollingFilter.fromRate = function fromRate(items, rate) { - return new RollingFilter().fromRate(items, rate); -}; - -/** - * Perform the mumur3 hash on data. - * @param {Buffer} val - * @param {Number} seed - * @returns {Number} - */ - -RollingFilter.prototype.hash = function hash(val, n) { - return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak)); -}; - -/** - * Reset the filter. - */ - -RollingFilter.prototype.reset = function reset() { - if (this.entries === 0) - return; - - this.entries = 0; - this.generation = 1; - this.filter.fill(0); -}; - -/** - * Add data to the filter. - * @param {Buffer|String} - * @param {String?} enc - Can be any of the Buffer object's encodings. - */ - -RollingFilter.prototype.add = function add(val, enc) { - if (typeof val === 'string') - val = Buffer.from(val, enc); - - if (this.entries === this.limit) { - this.entries = 0; - this.generation += 1; - - if (this.generation === 4) - this.generation = 1; - - const m1 = (this.generation & 1) * 0xffffffff; - const m2 = (this.generation >>> 1) * 0xffffffff; - - for (let i = 0; i < this.items; i += 2) { - const pos1 = i * 8; - const pos2 = (i + 1) * 8; - const v1 = read(this.filter, pos1); - const v2 = read(this.filter, pos2); - const mhi = (v1.hi ^ m1) | (v2.hi ^ m2); - const mlo = (v1.lo ^ m1) | (v2.lo ^ m2); - - v1.hi &= mhi; - v1.lo &= mlo; - v2.hi &= mhi; - v2.lo &= mlo; - - write(this.filter, v1, pos1); - write(this.filter, v2, pos2); - } - } - - this.entries += 1; - - for (let i = 0; i < this.n; i++) { - const hash = this.hash(val, i); - const bits = hash & 0x3f; - const pos = (hash >>> 6) % this.items; - const pos1 = (pos & ~1) * 8; - const pos2 = (pos | 1) * 8; - const bit = bits % 8; - const oct = (bits - bit) / 8; - - this.filter[pos1 + oct] &= ~(1 << bit); - this.filter[pos1 + oct] |= (this.generation & 1) << bit; - - this.filter[pos2 + oct] &= ~(1 << bit); - this.filter[pos2 + oct] |= (this.generation >>> 1) << bit; - } -}; - -/** - * Test whether data is present in the filter. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} - */ - -RollingFilter.prototype.test = function test(val, enc) { - if (this.entries === 0) - return false; - - if (typeof val === 'string') - val = Buffer.from(val, enc); - - for (let i = 0; i < this.n; i++) { - const hash = this.hash(val, i); - const bits = hash & 0x3f; - const pos = (hash >>> 6) % this.items; - const pos1 = (pos & ~1) * 8; - const pos2 = (pos | 1) * 8; - const bit = bits % 8; - const oct = (bits - bit) / 8; - - const bit1 = (this.filter[pos1 + oct] >>> bit) & 1; - const bit2 = (this.filter[pos2 + oct] >>> bit) & 1; - - if ((bit1 | bit2) === 0) - return false; - } - - return true; -}; - -/** - * Test whether data is present in the - * filter and potentially add data. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} Whether data was added. - */ - -RollingFilter.prototype.added = function added(val, enc) { - if (typeof val === 'string') - val = Buffer.from(val, enc); - - if (!this.test(val)) { - this.add(val); - return true; - } - - return false; -}; - -/* - * Helpers - */ - -function U64(hi, lo) { - this.hi = hi; - this.lo = lo; -} - -function read(data, off) { - const hi = data.readUInt32LE(off + 4, true); - const lo = data.readUInt32LE(off, true); - return new U64(hi, lo); -} - -function write(data, value, off) { - data.writeUInt32LE(value.hi, off + 4, true); - data.writeUInt32LE(value.lo, off, true); -} - -/* - * Expose - */ - -module.exports = RollingFilter; diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index d39fcc77..24d8c5c4 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -21,7 +21,7 @@ const Path = require('./path'); const common = require('./common'); const Wallet = require('./wallet'); const Account = require('./account'); -const Bloom = require('../utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const Logger = require('../node/logger'); const Outpoint = require('../primitives/outpoint'); const layouts = require('./layout'); @@ -74,7 +74,7 @@ function WalletDB(options) { this.txLock = new Lock(); // Address and outpoint filter. - this.filter = new Bloom(); + this.filter = new BloomFilter(); this._init(); } @@ -99,15 +99,15 @@ WalletDB.prototype._init = function _init() { // Highest number of items with an // FPR of 0.001. We have to do this - // by hand because Bloom.fromRate's + // by hand because BloomFilter.fromRate's // policy limit enforcing is fairly // naive. if (this.options.spv) { items = 20000; - flag = Bloom.flags.ALL; + flag = BloomFilter.flags.ALL; } - this.filter = Bloom.fromRate(items, 0.001, flag); + this.filter = BloomFilter.fromRate(items, 0.001, flag); this.bind(); }; diff --git a/test/block-test.js b/test/block-test.js index 349514be..2ea570fc 100644 --- a/test/block-test.js +++ b/test/block-test.js @@ -5,7 +5,7 @@ const assert = require('./util/assert'); const common = require('./util/common'); -const Bloom = require('../lib/utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const Block = require('../lib/primitives/block'); const MerkleBlock = require('../lib/primitives/merkleblock'); const consensus = require('../lib/protocol/consensus'); @@ -103,7 +103,7 @@ describe('Block', function() { }); it('should create a merkle block', () => { - const filter = Bloom.fromRate(1000, 0.01, Bloom.flags.NONE); + const filter = BloomFilter.fromRate(1000, 0.01, BloomFilter.flags.NONE); const item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b'; const item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10' diff --git a/test/bloom-test.js b/test/bloom-test.js deleted file mode 100644 index 34b8934d..00000000 --- a/test/bloom-test.js +++ /dev/null @@ -1,183 +0,0 @@ -/* eslint-env mocha */ -/* eslint prefer-arrow-callback: "off" */ - -'use strict'; - -const assert = require('./util/assert'); -const Bloom = require('../lib/utils/bloom'); -const RollingFilter = require('../lib/utils/rollingfilter'); -const murmur3 = require('../lib/utils/murmur3'); - -function testMurmur(str, seed, expect, enc) { - if (!enc) - enc = 'ascii'; - - const data = Buffer.from(str, enc); - const hash = murmur3(data, seed); - - assert.strictEqual(hash, expect); -} - -describe('Bloom', function() { - this.timeout(20000); - - it('should do proper murmur3', () => { - testMurmur('', 0, 0); - testMurmur('', 0xfba4c795, 0x6a396f08); - testMurmur('00', 0xfba4c795, 0x2a101837); - testMurmur('hello world', 0, 0x5e928f0f); - - testMurmur('', 0x00000000, 0x00000000, 'hex'); - testMurmur('', 0xfba4c795, 0x6a396f08, 'hex'); - testMurmur('', 0xffffffff, 0x81f16f39, 'hex'); - - testMurmur('00', 0x00000000, 0x514e28b7, 'hex'); - testMurmur('00', 0xfba4c795, 0xea3f0b17, 'hex'); - testMurmur('ff', 0x00000000, 0xfd6cf10d, 'hex'); - - testMurmur('0011', 0x00000000, 0x16c6b7ab, 'hex'); - testMurmur('001122', 0x00000000, 0x8eb51c3d, 'hex'); - testMurmur('00112233', 0x00000000, 0xb4471bf8, 'hex'); - testMurmur('0011223344', 0x00000000, 0xe2301fa8, 'hex'); - testMurmur('001122334455', 0x00000000, 0xfc2e4a15, 'hex'); - testMurmur('00112233445566', 0x00000000, 0xb074502c, 'hex'); - testMurmur('0011223344556677', 0x00000000, 0x8034d2a0, 'hex'); - testMurmur('001122334455667788', 0x00000000, 0xb4698def, 'hex'); - }); - - it('should test and add stuff', () => { - const filter = new Bloom(512, 10, 156); - - filter.add('hello', 'ascii'); - assert(filter.test('hello', 'ascii')); - assert(!filter.test('hello!', 'ascii')); - assert(!filter.test('ping', 'ascii')); - - filter.add('hello!', 'ascii'); - assert(filter.test('hello!', 'ascii')); - assert(!filter.test('ping', 'ascii')); - - filter.add('ping', 'ascii'); - assert(filter.test('ping', 'ascii')); - }); - - it('should serialize to the correct format', () => { - const filter = new Bloom(952, 6, 3624314491, Bloom.flags.NONE); - const item1 = '8e7445bbb8abd4b3174d80fa4c409fea6b94d96b'; - const item2 = '047b00000078da0dca3b0ec2300c00d0ab4466ed10' - + 'e763272c6c9ca052972c69e3884a9022084215e2eef' - + '0e6f781656b5d5a87231cd4349e534b6dea55ad4ff55e'; - - const expected = Buffer.from('' - + '000000000000000000000000000000000000000000000000088004000000000000000' - + '000000000200000000000000000000000000000000800000000000000000002000000' - + '000000000000002000000000000000000000000000000000000000000040000200000' - + '0000000001000000800000080000000', - 'hex'); - - filter.add(item1, 'hex'); - filter.add(item2, 'hex'); - - assert.bufferEqual(filter.filter, expected); - }); - - it('should handle 1m ops with regular filter', () => { - const filter = Bloom.fromRate(210000, 0.00001, -1); - - filter.tweak = 0xdeadbeef; - - // ~1m operations - for (let i = 0; i < 1000; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j--); - } - }); - - it('should handle 1m ops with rolling filter', () => { - const filter = new RollingFilter(210000, 0.00001); - - filter.tweak = 0xdeadbeef; - - // ~1m operations - for (let i = 0; i < 1000; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j--); - } - }); - - it('should handle rolling generations', () => { - const filter = new RollingFilter(50, 0.00001); - - filter.tweak = 0xdeadbeee; - - for (let i = 0; i < 25; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j--); - } - - for (let i = 25; i < 50; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j--); - } - - for (let i = 50; i < 75; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j--); - } - - for (let i = 75; i < 100; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j-- > 25); - assert(!filter.test('foobar 24', 'ascii')); - } - - for (let i = 100; i < 125; i++) { - const str = 'foobar' + i; - let j = i; - filter.add(str, 'ascii'); - do { - const str = 'foobar' + j; - assert(filter.test(str, 'ascii')); - assert(!filter.test(str + '-', 'ascii')); - } while (j-- > 50); - } - - assert(!filter.test('foobar 49', 'ascii')); - }); -}); diff --git a/test/gcs-test.js b/test/gcs-test.js deleted file mode 100644 index 4d34e13d..00000000 --- a/test/gcs-test.js +++ /dev/null @@ -1,177 +0,0 @@ -/* eslint-env mocha */ -/* eslint prefer-arrow-callback: "off" */ - -'use strict'; - -const assert = require('./util/assert'); -const GCSFilter = require('../lib/utils/gcs'); -const random = require('bcrypto/lib/random'); -const Outpoint = require('../lib/primitives/outpoint'); -const Address = require('../lib/primitives/address'); -const common = require('./util/common'); - -const block928927 = common.readBlock('block928927'); -const [block] = block928927.getBlock(); - -const key = random.randomBytes(16); -const P = 20; - -const contents1 = [ - Buffer.from('Alex', 'ascii'), - Buffer.from('Bob', 'ascii'), - Buffer.from('Charlie', 'ascii'), - Buffer.from('Dick', 'ascii'), - Buffer.from('Ed', 'ascii'), - Buffer.from('Frank', 'ascii'), - Buffer.from('George', 'ascii'), - Buffer.from('Harry', 'ascii'), - Buffer.from('Ilya', 'ascii'), - Buffer.from('John', 'ascii'), - Buffer.from('Kevin', 'ascii'), - Buffer.from('Larry', 'ascii'), - Buffer.from('Michael', 'ascii'), - Buffer.from('Nate', 'ascii'), - Buffer.from('Owen', 'ascii'), - Buffer.from('Paul', 'ascii'), - Buffer.from('Quentin', 'ascii') -]; - -const contents2 = [ - Buffer.from('Alice', 'ascii'), - Buffer.from('Betty', 'ascii'), - Buffer.from('Charmaine', 'ascii'), - Buffer.from('Donna', 'ascii'), - Buffer.from('Edith', 'ascii'), - Buffer.from('Faina', 'ascii'), - Buffer.from('Georgia', 'ascii'), - Buffer.from('Hannah', 'ascii'), - Buffer.from('Ilsbeth', 'ascii'), - Buffer.from('Jennifer', 'ascii'), - Buffer.from('Kayla', 'ascii'), - Buffer.from('Lena', 'ascii'), - Buffer.from('Michelle', 'ascii'), - Buffer.from('Natalie', 'ascii'), - Buffer.from('Ophelia', 'ascii'), - Buffer.from('Peggy', 'ascii'), - Buffer.from('Queenie', 'ascii') -]; - -const op1 = new Outpoint( - '4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c', - 4); - -const op2 = new Outpoint( - 'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314', - 3); - -const op3 = new Outpoint( - '4cba1d1753ed19dbeafffb1a6c805d20e4af00b194a8f85353163cef83319c2c', - 400); - -const op4 = new Outpoint( - 'b7c3c4bce1a23baef2da05f9b7e4bff813449ec7e80f980ec7e4cacfadcd3314', - 300); - -const addr1 = new Address('bc1qmyrddmxglk49ye2wd29wefaavw7es8k5d555lx'); -const addr2 = new Address('bc1q4645ycu0l9pnvxaxnhemushv0w4cd9flkqh95j'); - -let filter1 = null; -let filter2 = null; -let filter3 = null; -let filter4 = null; -let filter5 = null; - -describe('GCS', function() { - it('should test GCS filter build', () => { - filter1 = GCSFilter.fromItems(P, key, contents1); - assert(filter1); - }); - - it('should test GCS filter copy', () => { - filter2 = GCSFilter.fromBytes(filter1.n, P, filter1.toBytes()); - assert(filter2); - filter3 = GCSFilter.fromNBytes(P, filter1.toNBytes()); - assert(filter3); - filter4 = GCSFilter.fromPBytes(filter1.n, filter1.toPBytes()); - assert(filter4); - filter5 = GCSFilter.fromNPBytes(filter1.toNPBytes()); - assert(filter5); - }); - - it('should test GCS filter metadata', () => { - assert.strictEqual(filter1.p, P); - assert.strictEqual(filter1.n, contents1.length); - assert.strictEqual(filter1.p, filter2.p); - assert.strictEqual(filter1.n, filter2.n); - assert.bufferEqual(filter1.data, filter2.data); - assert.strictEqual(filter1.p, filter3.p); - assert.strictEqual(filter1.n, filter3.n); - assert.bufferEqual(filter1.data, filter3.data); - assert.strictEqual(filter1.p, filter4.p); - assert.strictEqual(filter1.n, filter4.n); - assert.bufferEqual(filter1.data, filter4.data); - assert.strictEqual(filter1.p, filter5.p); - assert.strictEqual(filter1.n, filter5.n); - assert.bufferEqual(filter1.data, filter5.data); - }); - - it('should test GCS filter match', () => { - let match = filter1.match(key, Buffer.from('Nate')); - assert(match); - match = filter2.match(key, Buffer.from('Nate')); - assert(match); - match = filter1.match(key, Buffer.from('Quentin')); - assert(match); - match = filter2.match(key, Buffer.from('Quentin')); - assert(match); - - match = filter1.match(key, Buffer.from('Nates')); - assert(!match); - match = filter2.match(key, Buffer.from('Nates')); - assert(!match); - match = filter1.match(key, Buffer.from('Quentins')); - assert(!match); - match = filter2.match(key, Buffer.from('Quentins')); - assert(!match); - }); - - it('should test GCS filter matchAny', () => { - let match = filter1.matchAny(key, contents2); - assert(!match); - match = filter2.matchAny(key, contents2); - assert(!match); - - const contents = contents2.slice(); - contents.push(Buffer.from('Nate')); - - match = filter1.matchAny(key, contents); - assert(match); - match = filter2.matchAny(key, contents); - assert(match); - }); - - it('should test GCS filter fromBlock', () => { - const key = block.hash().slice(0, 16); - const filter = GCSFilter.fromBlock(block); - assert(filter.match(key, op1.toRaw())); - assert(filter.match(key, op2.toRaw())); - assert(!filter.match(key, op3.toRaw())); - assert(!filter.match(key, op4.toRaw())); - assert(filter.match(key, addr1.hash)); - assert(filter.match(key, addr2.hash)); - assert(filter.matchAny(key, [op1.toRaw(), addr1.hash])); - assert(filter.matchAny(key, [op1.toRaw(), op3.toRaw()])); - assert(!filter.matchAny(key, [op3.toRaw(), op4.toRaw()])); - }); - - it('should test GCS filter fromExtended', () => { - const key = block.hash().slice(0, 16); - const filter = GCSFilter.fromExtended(block); - assert(!filter.match(key, op1.toRaw())); - assert(filter.match(key, block.txs[0].hash())); - assert(filter.match(key, block.txs[1].hash())); - assert(filter.matchAny(key, [block.txs[0].hash(), block.txs[1].hash()])); - assert(filter.matchAny(key, [op1.toRaw(), block.txs[1].hash()])); - assert(!filter.matchAny(key, [op1.toRaw(), op2.toRaw()])); - }); -}); diff --git a/test/util/memwallet.js b/test/util/memwallet.js index 917abf73..aa95bac4 100644 --- a/test/util/memwallet.js +++ b/test/util/memwallet.js @@ -10,7 +10,7 @@ const assert = require('assert'); const Network = require('../../lib/protocol/network'); const MTX = require('../../lib/primitives/mtx'); const HD = require('../../lib/hd/hd'); -const Bloom = require('../../lib/utils/bloom'); +const BloomFilter = require('bfilter/lib/bloom'); const KeyRing = require('../../lib/primitives/keyring'); const Outpoint = require('../../lib/primitives/outpoint'); const Coin = require('../../lib/primitives/coin'); @@ -34,7 +34,7 @@ function MemWallet(options) { this.paths = new Map(); this.balance = 0; this.txs = 0; - this.filter = Bloom.fromRate(1000000, 0.001, -1); + this.filter = BloomFilter.fromRate(1000000, 0.001, -1); if (options) this.fromOptions(options); diff --git a/test/utils-test.js b/test/utils-test.js index 70f94ce7..94962484 100644 --- a/test/utils-test.js +++ b/test/utils-test.js @@ -4,7 +4,7 @@ 'use strict'; const assert = require('./util/assert'); -const {U64, I64} = require('../lib/utils/int64'); +const {U64, I64} = require('n64'); const base58 = require('../lib/utils/base58'); const encoding = require('../lib/utils/encoding'); const Amount = require('../lib/btc/amount');