From be1ec1c22d5129e350ccc6656830a15dc5fe795a Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 27 Jun 2017 09:28:37 -0700 Subject: [PATCH] utils: refactor. --- lib/bip70/payment.js | 5 +- lib/bip70/paymentack.js | 5 +- lib/bip70/paymentdetails.js | 5 +- lib/bip70/paymentrequest.js | 5 +- lib/mempool/mempool.js | 4 +- lib/net/peer.js | 6 +- lib/net/pool.js | 3 +- lib/utils/bloom.js | 251 +------------------- lib/utils/index.js | 9 +- lib/utils/lock.js | 153 +------------ lib/utils/mappedlock.js | 184 +++++++++++++++ lib/utils/{protobuf.js => protoreader.js} | 157 +------------ lib/utils/protowriter.js | 179 +++++++++++++++ lib/utils/rollingfilter.js | 265 ++++++++++++++++++++++ lib/wallet/wallet.js | 3 +- lib/wallet/walletdb.js | 3 +- test/bloom-test.js | 5 +- 17 files changed, 663 insertions(+), 579 deletions(-) create mode 100644 lib/utils/mappedlock.js rename lib/utils/{protobuf.js => protoreader.js} (53%) create mode 100644 lib/utils/protowriter.js create mode 100644 lib/utils/rollingfilter.js diff --git a/lib/bip70/payment.js b/lib/bip70/payment.js index 82a84c29..e7ec56d5 100644 --- a/lib/bip70/payment.js +++ b/lib/bip70/payment.js @@ -10,10 +10,9 @@ var assert = require('assert'); var Output = require('../primitives/output'); var TX = require('../primitives/tx'); var Script = require('../script/script'); -var protobuf = require('../utils/protobuf'); +var ProtoReader = require('../utils/protoreader'); +var ProtoWriter = require('../utils/protowriter'); var PaymentDetails = require('./paymentdetails'); -var ProtoReader = protobuf.ProtoReader; -var ProtoWriter = protobuf.ProtoWriter; /** * Represents a BIP70 payment. diff --git a/lib/bip70/paymentack.js b/lib/bip70/paymentack.js index 12ce792e..cc501cf3 100644 --- a/lib/bip70/paymentack.js +++ b/lib/bip70/paymentack.js @@ -7,10 +7,9 @@ 'use strict'; var assert = require('assert'); -var protobuf = require('../utils/protobuf'); +var ProtoReader = require('../utils/protoreader'); +var ProtoWriter = require('../utils/protowriter'); var Payment = require('./payment'); -var ProtoReader = protobuf.ProtoReader; -var ProtoWriter = protobuf.ProtoWriter; /** * Represents a BIP70 payment ack. diff --git a/lib/bip70/paymentdetails.js b/lib/bip70/paymentdetails.js index e947f1fc..13dab8d5 100644 --- a/lib/bip70/paymentdetails.js +++ b/lib/bip70/paymentdetails.js @@ -9,9 +9,8 @@ var assert = require('assert'); var util = require('../utils/util'); var Output = require('../primitives/output'); -var protobuf = require('../utils/protobuf'); -var ProtoReader = protobuf.ProtoReader; -var ProtoWriter = protobuf.ProtoWriter; +var ProtoReader = require('../utils/protoreader'); +var ProtoWriter = require('../utils/protowriter'); /** * Represents BIP70 payment details. diff --git a/lib/bip70/paymentrequest.js b/lib/bip70/paymentrequest.js index da510ab0..5ebc8791 100644 --- a/lib/bip70/paymentrequest.js +++ b/lib/bip70/paymentrequest.js @@ -11,10 +11,9 @@ var util = require('../utils/util'); var digest = require('../crypto/digest'); var x509 = require('./x509'); var PEM = require('../utils/pem'); -var protobuf = require('../utils/protobuf'); +var ProtoReader = require('../utils/protoreader'); +var ProtoWriter = require('../utils/protowriter'); var PaymentDetails = require('./paymentdetails'); -var ProtoReader = protobuf.ProtoReader; -var ProtoWriter = protobuf.ProtoWriter; /** * Represents a BIP70 payment request. diff --git a/lib/mempool/mempool.js b/lib/mempool/mempool.js index 59fbf0b4..eb00cd2e 100644 --- a/lib/mempool/mempool.js +++ b/lib/mempool/mempool.js @@ -13,7 +13,7 @@ var policy = require('../protocol/policy'); var util = require('../utils/util'); var random = require('../crypto/random'); var errors = require('../protocol/errors'); -var Bloom = require('../utils/bloom'); +var RollingFilter = require('../utils/rollingfilter'); var Address = require('../primitives/address'); var Script = require('../script/script'); var Outpoint = require('../primitives/outpoint'); @@ -93,7 +93,7 @@ function Mempool(options) { this.orphans = {}; this.map = {}; this.spents = {}; - this.rejects = new Bloom.Rolling(120000, 0.000001); + this.rejects = new RollingFilter(120000, 0.000001); this.coinIndex = new CoinIndex(); this.txIndex = new TXIndex(); diff --git a/lib/net/peer.js b/lib/net/peer.js index 97e9f0d7..0325735c 100644 --- a/lib/net/peer.js +++ b/lib/net/peer.js @@ -19,7 +19,7 @@ var consensus = require('../protocol/consensus'); var common = require('./common'); var InvItem = require('../primitives/invitem'); var Lock = require('../utils/lock'); -var Bloom = require('../utils/bloom'); +var RollingFilter = require('../utils/rollingfilter'); var BIP151 = require('./bip151'); var BIP150 = require('./bip150'); var BIP152 = require('./bip152'); @@ -136,8 +136,8 @@ function Peer(options) { this.invTimer = null; this.stallTimer = null; - this.addrFilter = new Bloom.Rolling(5000, 0.001); - this.invFilter = new Bloom.Rolling(50000, 0.000001); + this.addrFilter = new RollingFilter(5000, 0.001); + this.invFilter = new RollingFilter(50000, 0.000001); this.blockMap = new Map(); this.txMap = new Map(); diff --git a/lib/net/pool.js b/lib/net/pool.js index 79d6dedb..eb419364 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -20,6 +20,7 @@ var BIP150 = require('./bip150'); var BIP151 = require('./bip151'); var BIP152 = require('./bip152'); var Bloom = require('../utils/bloom'); +var RollingFilter = require('../utils/rollingfilter'); var secp256k1 = require('../crypto/secp256k1'); var Lock = require('../utils/lock'); var Network = require('../protocol/network'); @@ -115,7 +116,7 @@ function Pool(options) { this.spvFilter = Bloom.fromRate(20000, 0.001, Bloom.flags.ALL); if (!this.options.mempool) - this.txFilter = new Bloom.Rolling(50000, 0.000001); + this.txFilter = new RollingFilter(50000, 0.000001); this._init(); }; diff --git a/lib/utils/bloom.js b/lib/utils/bloom.js index 37268d1c..65c7ff9c 100644 --- a/lib/utils/bloom.js +++ b/lib/utils/bloom.js @@ -384,257 +384,8 @@ Bloom.fromRaw = function fromRaw(data, enc) { return new Bloom().fromRaw(data); }; -/** - * A rolling bloom filter used internally - * (do not relay this on the p2p network). - * @alias module:utils.RollingFilter - * @constructor - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - */ - -function RollingFilter(items, rate) { - if (!(this instanceof RollingFilter)) - return new RollingFilter(items, rate); - - this.entries = 0; - this.generation = 1; - this.n = 0; - this.limit = 0; - this.size = 0; - this.items = 0; - this.tweak = 0; - this.filter = DUMMY; - - if (items != null) - this.fromRate(items, rate); -} - -/** - * Inject properties from items and FPR. - * @private - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - * @returns {RollingFilter} - */ - -RollingFilter.prototype.fromRate = function fromRate(items, rate) { - var logRate, max, n, limit, size, tweak, filter; - - assert(typeof items === 'number', '`items` must be a number.'); - assert(items > 0, '`items` must be greater than zero.'); - assert(items % 1 === 0, '`items` must be an integer.'); - assert(typeof rate === 'number', '`rate` must be a number.'); - assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.'); - - logRate = Math.log(rate); - - n = Math.max(1, Math.min(Math.round(logRate / Math.log(0.5)), 50)); - limit = (items + 1) / 2 | 0; - - max = limit * 3; - size = -1 * n * max / Math.log(1.0 - Math.exp(logRate / n)); - size = Math.ceil(size); - - items = ((size + 63) / 64 | 0) << 1; - items >>>= 0; - items = Math.max(1, items); - - tweak = (Math.random() * 0x100000000) >>> 0; - - filter = Buffer.allocUnsafe(items * 8); - filter.fill(0); - - this.n = n; - this.limit = limit; - this.size = size; - this.items = items; - this.tweak = tweak; - this.filter = filter; - - return this; -}; - -/** - * Instantiate rolling filter from items and FPR. - * @param {Number} items - Expected number of items. - * @param {Number} rate - False positive rate (0.0-1.0). - * @returns {RollingFilter} - */ - -RollingFilter.fromRate = function fromRate(items, rate) { - return new RollingFilter().fromRate(items, rate); -}; - -/** - * Perform the mumur3 hash on data. - * @param {Buffer} val - * @param {Number} seed - * @returns {Number} - */ - -RollingFilter.prototype.hash = function hash(val, n) { - return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak)); -}; - -/** - * Reset the filter. - */ - -RollingFilter.prototype.reset = function reset() { - if (this.entries === 0) - return; - - this.entries = 0; - this.generation = 1; - this.filter.fill(0); -}; - -/** - * Add data to the filter. - * @param {Buffer|String} - * @param {String?} enc - Can be any of the Buffer object's encodings. - */ - -RollingFilter.prototype.add = function add(val, enc) { - var i, hash, bits, pos, pos1, pos2, bit, oct; - var m1, m2, v1, v2, mhi, mlo; - - if (typeof val === 'string') - val = Buffer.from(val, enc); - - if (this.entries === this.limit) { - this.entries = 0; - this.generation += 1; - - if (this.generation === 4) - this.generation = 1; - - m1 = (this.generation & 1) * 0xffffffff; - m2 = (this.generation >>> 1) * 0xffffffff; - - for (i = 0; i < this.items; i += 2) { - pos1 = i * 8; - pos2 = (i + 1) * 8; - v1 = read(this.filter, pos1); - v2 = read(this.filter, pos2); - mhi = (v1.hi ^ m1) | (v2.hi ^ m2); - mlo = (v1.lo ^ m1) | (v2.lo ^ m2); - v1.hi &= mhi; - v1.lo &= mlo; - v2.hi &= mhi; - v2.lo &= mlo; - write(this.filter, v1, pos1); - write(this.filter, v2, pos2); - } - } - - this.entries += 1; - - for (i = 0; i < this.n; i++) { - hash = this.hash(val, i); - bits = hash & 0x3f; - pos = (hash >>> 6) % this.items; - pos1 = (pos & ~1) * 8; - pos2 = (pos | 1) * 8; - - bit = bits % 8; - oct = (bits - bit) / 8; - pos1 += oct; - pos2 += oct; - - this.filter[pos1] &= ~(1 << bit); - this.filter[pos1] |= (this.generation & 1) << bit; - - this.filter[pos2] &= ~(1 << bit); - this.filter[pos2] |= (this.generation >>> 1) << bit; - } -}; - -/** - * Test whether data is present in the filter. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} - */ - -RollingFilter.prototype.test = function test(val, enc) { - var i, hash, bits, pos, pos1, pos2, bit, oct; - - if (this.entries === 0) - return false; - - if (typeof val === 'string') - val = Buffer.from(val, enc); - - for (i = 0; i < this.n; i++) { - hash = this.hash(val, i); - bits = hash & 0x3f; - pos = (hash >>> 6) % this.items; - pos1 = (pos & ~1) * 8; - pos2 = (pos | 1) * 8; - - bit = bits % 8; - oct = (bits - bit) / 8; - pos1 += oct; - pos2 += oct; - - bits = (this.filter[pos1] >>> bit) & 1; - bits |= (this.filter[pos2] >>> bit) & 1; - - if (bits === 0) - return false; - } - - return true; -}; - -/** - * Test whether data is present in the - * filter and potentially add data. - * @param {Buffer|String} val - * @param {String?} enc - Can be any of the Buffer object's encodings. - * @returns {Boolean} Whether data was added. - */ - -RollingFilter.prototype.added = function added(val, enc) { - if (typeof val === 'string') - val = Buffer.from(val, enc); - - if (!this.test(val)) { - this.add(val); - return true; - } - - return false; -}; - -/* - * Helpers - */ - -function U64(hi, lo) { - this.hi = hi; - this.lo = lo; -} - -function read(data, off) { - var hi = data.readUInt32LE(off + 4, true); - var lo = data.readUInt32LE(off, true); - return new U64(hi, lo); -} - -function write(data, value, off) { - data.writeUInt32LE(value.hi, off + 4, true); - data.writeUInt32LE(value.lo, off, true); -} - /* * Expose */ -exports = Bloom; -exports.murmur3 = murmur3; -exports.Rolling = RollingFilter; - -module.exports = exports; +module.exports = Bloom; diff --git a/lib/utils/index.js b/lib/utils/index.js index c9e99fbd..e60e2abd 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -17,7 +17,6 @@ exports.base32 = require('./base32'); exports.base58 = require('./base58'); exports.bech32 = require('./bech32'); exports.Bloom = require('./bloom'); -exports.RollingFilter = exports.Bloom.Rolling; exports.co = require('./co'); exports.encoding = require('./encoding'); exports.fs = require('./fs'); @@ -27,17 +26,17 @@ exports.Int64 = require('./int64'); exports.IP = require('./ip'); exports.List = require('./list'); exports.Lock = require('./lock'); -exports.MappedLock = exports.Lock.Mapped; exports.LRU = require('./lru'); exports.Map = require('./map'); +exports.MappedLock = require('./mappedlock'); exports.murmur3 = require('./murmur3'); exports.nfkd = require('./nfkd'); exports.PEM = require('./pem'); -exports.protobuf = require('./protobuf'); -exports.ProtoWriter = exports.protobuf.ProtoWriter; -exports.ProtoReader = exports.protobuf.ProtoReader; +exports.ProtoWriter = require('./protowriter'); +exports.ProtoReader = require('./protoreader'); exports.RBT = require('./rbt'); exports.BufferReader = require('./reader'); +exports.RollingFilter = require('./rollingfilter'); exports.StaticWriter = require('./staticwriter'); exports.util = require('./util'); exports.Validator = require('./validator'); diff --git a/lib/utils/lock.js b/lib/utils/lock.js index f0a14ca0..58931658 100644 --- a/lib/utils/lock.js +++ b/lib/utils/lock.js @@ -178,154 +178,6 @@ Lock.prototype.destroy = function destroy() { } }; -/** - * Represents a mutex lock for locking asynchronous object methods. - * Locks methods according to passed-in key. - * @alias module:utils.MappedLock - * @constructor - */ - -function MappedLock() { - if (!(this instanceof MappedLock)) - return MappedLock.create(); - - this.jobs = Object.create(null); - this.busy = Object.create(null); - this.destroyed = false; -} - -/** - * Create a closure scoped lock. - * @returns {Function} Lock method. - */ - -MappedLock.create = function create() { - var lock = new MappedLock(); - return function _lock(key, force) { - return lock.lock(key, force); - }; -}; - -/** - * Test whether the lock has a pending - * job or a job in progress (by name). - * @param {String} name - * @returns {Boolean} - */ - -MappedLock.prototype.has = function has(name) { - return this.busy[name] === true; -}; - -/** - * Test whether the lock has - * a pending job by name. - * @param {String} name - * @returns {Boolean} - */ - -MappedLock.prototype.hasPending = function hasPending(name) { - return this.jobs[name] != null; -}; - -/** - * Lock the parent object and all its methods - * which use the lock with a specified key. - * Begin to queue calls. - * @param {String|Number} key - * @param {Boolean?} force - Force a call. - * @returns {Promise} - Returns {Function}, must be - * called once the method finishes executing in order - * to resolve the queue. - */ - -MappedLock.prototype.lock = function lock(key, force) { - var self = this; - - if (this.destroyed) - return Promise.reject(new Error('Lock is destroyed.')); - - if (key == null) - return Promise.resolve(nop); - - if (force) { - assert(this.busy[key]); - return Promise.resolve(nop); - } - - if (this.busy[key]) { - return new Promise(function(resolve, reject) { - if (!self.jobs[key]) - self.jobs[key] = []; - self.jobs[key].push(new Job(resolve, reject, null)); - }); - } - - this.busy[key] = true; - - return Promise.resolve(this.unlock(key)); -}; - -/** - * Create an unlock callback. - * @private - * @param {String} key - * @returns {Function} Unlocker. - */ - -MappedLock.prototype.unlock = function unlock(key) { - var self = this; - return function unlocker() { - var jobs = self.jobs[key]; - var job; - - assert(self.destroyed || self.busy[key]); - delete self.busy[key]; - - if (!jobs) - return; - - assert(!self.destroyed); - - job = jobs.shift(); - assert(job); - - if (jobs.length === 0) - delete self.jobs[key]; - - self.busy[key] = true; - - job.resolve(unlocker); - }; -}; - -/** - * Destroy the lock. Purge all pending calls. - */ - -MappedLock.prototype.destroy = function destroy() { - var err = new Error('Lock was destroyed.'); - var map = this.jobs; - var keys = Object.keys(map); - var i, j, key, jobs, job; - - assert(!this.destroyed, 'Lock is already destroyed.'); - - this.destroyed = true; - - this.jobs = Object.create(null); - this.busy = Object.create(null); - - for (i = 0; i < keys.length; i++) { - key = keys[i]; - jobs = map[key]; - for (j = 0; j < jobs.length; j++) { - job = jobs[j]; - job.reject(err); - } - } -}; - /** * Lock Job * @constructor @@ -351,7 +203,4 @@ function nop() {} * Expose */ -exports = Lock; -exports.Mapped = MappedLock; - -module.exports = exports; +module.exports = Lock; diff --git a/lib/utils/mappedlock.js b/lib/utils/mappedlock.js new file mode 100644 index 00000000..f41735d0 --- /dev/null +++ b/lib/utils/mappedlock.js @@ -0,0 +1,184 @@ +/*! + * mappedlock.js - lock and queue for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +var assert = require('assert'); + +/** + * Represents a mutex lock for locking asynchronous object methods. + * Locks methods according to passed-in key. + * @alias module:utils.MappedLock + * @constructor + */ + +function MappedLock() { + if (!(this instanceof MappedLock)) + return MappedLock.create(); + + this.jobs = Object.create(null); + this.busy = Object.create(null); + this.destroyed = false; +} + +/** + * Create a closure scoped lock. + * @returns {Function} Lock method. + */ + +MappedLock.create = function create() { + var lock = new MappedLock(); + return function _lock(key, force) { + return lock.lock(key, force); + }; +}; + +/** + * Test whether the lock has a pending + * job or a job in progress (by name). + * @param {String} name + * @returns {Boolean} + */ + +MappedLock.prototype.has = function has(name) { + return this.busy[name] === true; +}; + +/** + * Test whether the lock has + * a pending job by name. + * @param {String} name + * @returns {Boolean} + */ + +MappedLock.prototype.hasPending = function hasPending(name) { + return this.jobs[name] != null; +}; + +/** + * Lock the parent object and all its methods + * which use the lock with a specified key. + * Begin to queue calls. + * @param {String|Number} key + * @param {Boolean?} force - Force a call. + * @returns {Promise} - Returns {Function}, must be + * called once the method finishes executing in order + * to resolve the queue. + */ + +MappedLock.prototype.lock = function lock(key, force) { + var self = this; + + if (this.destroyed) + return Promise.reject(new Error('Lock is destroyed.')); + + if (key == null) + return Promise.resolve(nop); + + if (force) { + assert(this.busy[key]); + return Promise.resolve(nop); + } + + if (this.busy[key]) { + return new Promise(function(resolve, reject) { + if (!self.jobs[key]) + self.jobs[key] = []; + self.jobs[key].push(new Job(resolve, reject)); + }); + } + + this.busy[key] = true; + + return Promise.resolve(this.unlock(key)); +}; + +/** + * Create an unlock callback. + * @private + * @param {String} key + * @returns {Function} Unlocker. + */ + +MappedLock.prototype.unlock = function unlock(key) { + var self = this; + return function unlocker() { + var jobs = self.jobs[key]; + var job; + + assert(self.destroyed || self.busy[key]); + delete self.busy[key]; + + if (!jobs) + return; + + assert(!self.destroyed); + + job = jobs.shift(); + assert(job); + + if (jobs.length === 0) + delete self.jobs[key]; + + self.busy[key] = true; + + job.resolve(unlocker); + }; +}; + +/** + * Destroy the lock. Purge all pending calls. + */ + +MappedLock.prototype.destroy = function destroy() { + var err = new Error('Lock was destroyed.'); + var map = this.jobs; + var keys = Object.keys(map); + var i, j, key, jobs, job; + + assert(!this.destroyed, 'Lock is already destroyed.'); + + this.destroyed = true; + + this.jobs = Object.create(null); + this.busy = Object.create(null); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + jobs = map[key]; + for (j = 0; j < jobs.length; j++) { + job = jobs[j]; + job.reject(err); + } + } +}; + +/** + * Lock Job + * @constructor + * @ignore + * @param {Function} resolve + * @param {Function} reject + * @param {String?} name + */ + +function Job(resolve, reject) { + this.resolve = resolve; + this.reject = reject; +} + +/* + * Helpers + */ + +function nop() {} + +/* + * Expose + */ + +module.exports = MappedLock; diff --git a/lib/utils/protobuf.js b/lib/utils/protoreader.js similarity index 53% rename from lib/utils/protobuf.js rename to lib/utils/protoreader.js index 4565eb57..feea3705 100644 --- a/lib/utils/protobuf.js +++ b/lib/utils/protoreader.js @@ -1,5 +1,5 @@ /*! - * protobuf.js - protobufs for bcoin + * protoreader.js - protobufs for bcoin * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License). * https://github.com/bcoin-org/bcoin */ @@ -10,10 +10,9 @@ * @module utils/protobuf */ -var util = require('../utils/util'); var assert = require('assert'); +var util = require('../utils/util'); var BufferReader = require('../utils/reader'); -var BufferWriter = require('../utils/writer'); /* * Constants @@ -42,8 +41,8 @@ function ProtoReader(data, zeroCopy) { util.inherits(ProtoReader, BufferReader); -ProtoReader.prototype.readVarint = function readVarint() { - var result = exports.readVarint(this.data, this.offset); +ProtoReader.prototype.readVarint = function _readVarint() { + var result = readVarint(this.data, this.offset); this.offset += result.size; return result.value; }; @@ -148,97 +147,11 @@ ProtoReader.prototype.readField = function readField(tag, opt) { return field; }; -/** - * ProtoWriter - * @constructor - */ - -function ProtoWriter() { - if (!(this instanceof ProtoWriter)) - return new ProtoWriter(); - - BufferWriter.call(this); -} - -util.inherits(ProtoWriter, BufferWriter); - -ProtoWriter.prototype.writeVarint = function writeVarint(num) { - var size = exports.sizeVarint(num); - var value; - - // Avoid an extra allocation until - // we make bufferwriter more hackable. - // More insanity here... - switch (size) { - case 6: - value = exports.slipVarint(num); - this.writeU32BE(value / 0x10000 | 0); - this.writeU16BE(value & 0xffff); - break; - case 5: - value = exports.slipVarint(num); - this.writeU32BE(value / 0x100 | 0); - this.writeU8(value & 0xff); - break; - case 4: - value = exports.slipVarint(num); - this.writeU32BE(value); - break; - case 3: - value = exports.slipVarint(num); - this.writeU16BE(value >> 8); - this.writeU8(value & 0xff); - break; - case 2: - value = exports.slipVarint(num); - this.writeU16BE(value); - break; - case 1: - value = exports.slipVarint(num); - this.writeU8(value); - break; - default: - value = Buffer.allocUnsafe(size); - exports.writeVarint(value, num, 0); - this.writeBytes(value); - break; - } -}; - -ProtoWriter.prototype.writeFieldVarint = function writeFieldVarint(tag, value) { - var header = (tag << 3) | wireType.VARINT; - this.writeVarint(header); - this.writeVarint(value); -}; - -ProtoWriter.prototype.writeFieldU64 = function writeFieldU64(tag, value) { - assert(util.isSafeInteger(value)); - this.writeFieldVarint(tag, value); -}; - -ProtoWriter.prototype.writeFieldU32 = function writeFieldU32(tag, value) { - assert(value <= 0xffffffff); - this.writeFieldVarint(tag, value); -}; - -ProtoWriter.prototype.writeFieldBytes = function writeFieldBytes(tag, data) { - var header = (tag << 3) | wireType.DELIMITED; - this.writeVarint(header); - this.writeVarint(data.length); - this.writeBytes(data); -}; - -ProtoWriter.prototype.writeFieldString = function writeFieldString(tag, data, enc) { - if (typeof data === 'string') - data = Buffer.from(data, enc || 'utf8'); - this.writeFieldBytes(tag, data); -}; - /* * Encoding */ -exports.readVarint = function readVarint(data, off) { +function readVarint(data, off) { var num = 0; var ch = 0x80; var size = 0; @@ -273,62 +186,7 @@ exports.readVarint = function readVarint(data, off) { } return new Varint(size, num); -}; - -exports.writeVarint = function writeVarint(data, num, off) { - var ch; - - assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - do { - assert(off < data.length); - ch = num & 0x7f; - num -= num % 0x80; - num /= 0x80; - if (num !== 0) - ch |= 0x80; - data[off] = ch; - off++; - } while (num > 0); - - return off; -}; - -exports.slipVarint = function slipVarint(num) { - var data = 0; - var size = 0; - var ch; - - assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - do { - assert(size < 7); - ch = num & 0x7f; - num -= num % 0x80; - num /= 0x80; - if (num !== 0) - ch |= 0x80; - data *= 256; - data += ch; - size++; - } while (num > 0); - - return data; -}; - -exports.sizeVarint = function sizeVarint(num) { - var size = 0; - - assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); - - do { - num -= num % 0x80; - num /= 0x80; - size++; - } while (num > 0); - - return size; -}; +} /* * Helpers @@ -352,5 +210,4 @@ function Varint(size, value) { * Expose */ -exports.ProtoReader = ProtoReader; -exports.ProtoWriter = ProtoWriter; +module.exports = ProtoReader; diff --git a/lib/utils/protowriter.js b/lib/utils/protowriter.js new file mode 100644 index 00000000..60144760 --- /dev/null +++ b/lib/utils/protowriter.js @@ -0,0 +1,179 @@ +/*! + * protowriter.js - protobufs for bcoin + * Copyright (c) 2016-2017, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +/** + * @module utils/protobuf + */ + +var assert = require('assert'); +var util = require('../utils/util'); +var BufferWriter = require('../utils/writer'); + +/* + * Constants + */ + +var wireType = { + VARINT: 0, + FIXED64: 1, + DELIMITED: 2, + START_GROUP: 3, + END_GROUP: 4, + FIXED32: 5 +}; + +/** + * ProtoWriter + * @constructor + */ + +function ProtoWriter() { + if (!(this instanceof ProtoWriter)) + return new ProtoWriter(); + + BufferWriter.call(this); +} + +util.inherits(ProtoWriter, BufferWriter); + +ProtoWriter.prototype.writeVarint = function _writeVarint(num) { + var size = sizeVarint(num); + var value; + + // Avoid an extra allocation until + // we make bufferwriter more hackable. + // More insanity here... + switch (size) { + case 6: + value = slipVarint(num); + this.writeU32BE(value / 0x10000 | 0); + this.writeU16BE(value & 0xffff); + break; + case 5: + value = slipVarint(num); + this.writeU32BE(value / 0x100 | 0); + this.writeU8(value & 0xff); + break; + case 4: + value = slipVarint(num); + this.writeU32BE(value); + break; + case 3: + value = slipVarint(num); + this.writeU16BE(value >> 8); + this.writeU8(value & 0xff); + break; + case 2: + value = slipVarint(num); + this.writeU16BE(value); + break; + case 1: + value = slipVarint(num); + this.writeU8(value); + break; + default: + value = Buffer.allocUnsafe(size); + writeVarint(value, num, 0); + this.writeBytes(value); + break; + } +}; + +ProtoWriter.prototype.writeFieldVarint = function writeFieldVarint(tag, value) { + var header = (tag << 3) | wireType.VARINT; + this.writeVarint(header); + this.writeVarint(value); +}; + +ProtoWriter.prototype.writeFieldU64 = function writeFieldU64(tag, value) { + assert(util.isSafeInteger(value)); + this.writeFieldVarint(tag, value); +}; + +ProtoWriter.prototype.writeFieldU32 = function writeFieldU32(tag, value) { + assert(value <= 0xffffffff); + this.writeFieldVarint(tag, value); +}; + +ProtoWriter.prototype.writeFieldBytes = function writeFieldBytes(tag, data) { + var header = (tag << 3) | wireType.DELIMITED; + this.writeVarint(header); + this.writeVarint(data.length); + this.writeBytes(data); +}; + +ProtoWriter.prototype.writeFieldString = function writeFieldString(tag, data, enc) { + if (typeof data === 'string') + data = Buffer.from(data, enc || 'utf8'); + this.writeFieldBytes(tag, data); +}; + +/* + * Encoding + */ + +function writeVarint(data, num, off) { + var ch; + + assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); + + do { + assert(off < data.length); + ch = num & 0x7f; + num -= num % 0x80; + num /= 0x80; + if (num !== 0) + ch |= 0x80; + data[off] = ch; + off++; + } while (num > 0); + + return off; +}; + +function slipVarint(num) { + var data = 0; + var size = 0; + var ch; + + assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); + + do { + assert(size < 7); + ch = num & 0x7f; + num -= num % 0x80; + num /= 0x80; + if (num !== 0) + ch |= 0x80; + data *= 256; + data += ch; + size++; + } while (num > 0); + + return data; +} + +function sizeVarint(num) { + var size = 0; + + assert(util.isSafeInteger(num), 'Number exceeds 2^53-1.'); + + do { + num -= num % 0x80; + num /= 0x80; + size++; + } while (num > 0); + + return size; +}; + +/* + * Expose + */ + +module.exports = ProtoWriter; diff --git a/lib/utils/rollingfilter.js b/lib/utils/rollingfilter.js new file mode 100644 index 00000000..75856fa0 --- /dev/null +++ b/lib/utils/rollingfilter.js @@ -0,0 +1,265 @@ +/*! + * rollingfilter.js - rolling bloom filter for bcoin + * Copyright (c) 2014-2015, Fedor Indutny (MIT License) + * Copyright (c) 2014-2017, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +var assert = require('assert'); +var murmur3 = require('./murmur3'); +var sum32 = murmur3.sum32; +var mul32 = murmur3.mul32; +var DUMMY = Buffer.alloc(0); + +/** + * A rolling bloom filter used internally + * (do not relay this on the p2p network). + * @alias module:utils.RollingFilter + * @constructor + * @param {Number} items - Expected number of items. + * @param {Number} rate - False positive rate (0.0-1.0). + */ + +function RollingFilter(items, rate) { + if (!(this instanceof RollingFilter)) + return new RollingFilter(items, rate); + + this.entries = 0; + this.generation = 1; + this.n = 0; + this.limit = 0; + this.size = 0; + this.items = 0; + this.tweak = 0; + this.filter = DUMMY; + + if (items != null) + this.fromRate(items, rate); +} + +/** + * Inject properties from items and FPR. + * @private + * @param {Number} items - Expected number of items. + * @param {Number} rate - False positive rate (0.0-1.0). + * @returns {RollingFilter} + */ + +RollingFilter.prototype.fromRate = function fromRate(items, rate) { + var logRate, max, n, limit, size, tweak, filter; + + assert(typeof items === 'number', '`items` must be a number.'); + assert(items > 0, '`items` must be greater than zero.'); + assert(items % 1 === 0, '`items` must be an integer.'); + assert(typeof rate === 'number', '`rate` must be a number.'); + assert(rate >= 0 && rate <= 1, '`rate` must be between 0.0 and 1.0.'); + + logRate = Math.log(rate); + + n = Math.max(1, Math.min(Math.round(logRate / Math.log(0.5)), 50)); + limit = (items + 1) / 2 | 0; + + max = limit * 3; + size = -1 * n * max / Math.log(1.0 - Math.exp(logRate / n)); + size = Math.ceil(size); + + items = ((size + 63) / 64 | 0) << 1; + items >>>= 0; + items = Math.max(1, items); + + tweak = (Math.random() * 0x100000000) >>> 0; + + filter = Buffer.allocUnsafe(items * 8); + filter.fill(0); + + this.n = n; + this.limit = limit; + this.size = size; + this.items = items; + this.tweak = tweak; + this.filter = filter; + + return this; +}; + +/** + * Instantiate rolling filter from items and FPR. + * @param {Number} items - Expected number of items. + * @param {Number} rate - False positive rate (0.0-1.0). + * @returns {RollingFilter} + */ + +RollingFilter.fromRate = function fromRate(items, rate) { + return new RollingFilter().fromRate(items, rate); +}; + +/** + * Perform the mumur3 hash on data. + * @param {Buffer} val + * @param {Number} seed + * @returns {Number} + */ + +RollingFilter.prototype.hash = function hash(val, n) { + return murmur3(val, sum32(mul32(n, 0xfba4c795), this.tweak)); +}; + +/** + * Reset the filter. + */ + +RollingFilter.prototype.reset = function reset() { + if (this.entries === 0) + return; + + this.entries = 0; + this.generation = 1; + this.filter.fill(0); +}; + +/** + * Add data to the filter. + * @param {Buffer|String} + * @param {String?} enc - Can be any of the Buffer object's encodings. + */ + +RollingFilter.prototype.add = function add(val, enc) { + var i, hash, bits, pos, pos1, pos2, bit, oct; + var m1, m2, v1, v2, mhi, mlo; + + if (typeof val === 'string') + val = Buffer.from(val, enc); + + if (this.entries === this.limit) { + this.entries = 0; + this.generation += 1; + + if (this.generation === 4) + this.generation = 1; + + m1 = (this.generation & 1) * 0xffffffff; + m2 = (this.generation >>> 1) * 0xffffffff; + + for (i = 0; i < this.items; i += 2) { + pos1 = i * 8; + pos2 = (i + 1) * 8; + v1 = read(this.filter, pos1); + v2 = read(this.filter, pos2); + mhi = (v1.hi ^ m1) | (v2.hi ^ m2); + mlo = (v1.lo ^ m1) | (v2.lo ^ m2); + v1.hi &= mhi; + v1.lo &= mlo; + v2.hi &= mhi; + v2.lo &= mlo; + write(this.filter, v1, pos1); + write(this.filter, v2, pos2); + } + } + + this.entries += 1; + + for (i = 0; i < this.n; i++) { + hash = this.hash(val, i); + bits = hash & 0x3f; + pos = (hash >>> 6) % this.items; + pos1 = (pos & ~1) * 8; + pos2 = (pos | 1) * 8; + + bit = bits % 8; + oct = (bits - bit) / 8; + pos1 += oct; + pos2 += oct; + + this.filter[pos1] &= ~(1 << bit); + this.filter[pos1] |= (this.generation & 1) << bit; + + this.filter[pos2] &= ~(1 << bit); + this.filter[pos2] |= (this.generation >>> 1) << bit; + } +}; + +/** + * Test whether data is present in the filter. + * @param {Buffer|String} val + * @param {String?} enc - Can be any of the Buffer object's encodings. + * @returns {Boolean} + */ + +RollingFilter.prototype.test = function test(val, enc) { + var i, hash, bits, pos, pos1, pos2, bit, oct; + + if (this.entries === 0) + return false; + + if (typeof val === 'string') + val = Buffer.from(val, enc); + + for (i = 0; i < this.n; i++) { + hash = this.hash(val, i); + bits = hash & 0x3f; + pos = (hash >>> 6) % this.items; + pos1 = (pos & ~1) * 8; + pos2 = (pos | 1) * 8; + + bit = bits % 8; + oct = (bits - bit) / 8; + pos1 += oct; + pos2 += oct; + + bits = (this.filter[pos1] >>> bit) & 1; + bits |= (this.filter[pos2] >>> bit) & 1; + + if (bits === 0) + return false; + } + + return true; +}; + +/** + * Test whether data is present in the + * filter and potentially add data. + * @param {Buffer|String} val + * @param {String?} enc - Can be any of the Buffer object's encodings. + * @returns {Boolean} Whether data was added. + */ + +RollingFilter.prototype.added = function added(val, enc) { + if (typeof val === 'string') + val = Buffer.from(val, enc); + + if (!this.test(val)) { + this.add(val); + return true; + } + + return false; +}; + +/* + * Helpers + */ + +function U64(hi, lo) { + this.hi = hi; + this.lo = lo; +} + +function read(data, off) { + var hi = data.readUInt32LE(off + 4, true); + var lo = data.readUInt32LE(off, true); + return new U64(hi, lo); +} + +function write(data, value, off) { + data.writeUInt32LE(value.hi, off + 4, true); + data.writeUInt32LE(value.lo, off, true); +} + +/* + * Expose + */ + +module.exports = RollingFilter; diff --git a/lib/wallet/wallet.js b/lib/wallet/wallet.js index ae7a42df..b2a32480 100644 --- a/lib/wallet/wallet.js +++ b/lib/wallet/wallet.js @@ -13,6 +13,7 @@ var Network = require('../protocol/network'); var util = require('../utils/util'); var encoding = require('../utils/encoding'); var Lock = require('../utils/lock'); +var MappedLock = require('../utils/mappedlock'); var digest = require('../crypto/digest'); var cleanse = require('../crypto/cleanse'); var BufferReader = require('../utils/reader'); @@ -69,7 +70,7 @@ function Wallet(db, options) { this.db = db; this.network = db.network; this.logger = db.logger; - this.readLock = new Lock.Mapped(); + this.readLock = new MappedLock(); this.writeLock = new Lock(); this.fundLock = new Lock(); this.indexCache = new LRU(10000); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index f1fdba3d..4c0c61be 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -11,6 +11,7 @@ var assert = require('assert'); var AsyncObject = require('../utils/asyncobject'); var util = require('../utils/util'); var Lock = require('../utils/lock'); +var MappedLock = require('../utils/mappedlock'); var LRU = require('../utils/lru'); var encoding = require('../utils/encoding'); var ccmp = require('../crypto/ccmp'); @@ -88,7 +89,7 @@ function WalletDB(options) { this.rescanning = false; this.bound = false; - this.readLock = new Lock.Mapped(); + this.readLock = new MappedLock(); this.writeLock = new Lock(); this.txLock = new Lock(); diff --git a/test/bloom-test.js b/test/bloom-test.js index c6980899..a418fd30 100644 --- a/test/bloom-test.js +++ b/test/bloom-test.js @@ -2,6 +2,7 @@ var assert = require('assert'); var Bloom = require('../lib/utils/bloom'); +var RollingFilter = require('../lib/utils/rollingfilter'); var murmur3 = require('../lib/utils/murmur3'); describe('Bloom', function() { @@ -88,7 +89,7 @@ describe('Bloom', function() { }); it('should handle 1m ops with rolling filter', function() { - var filter = new Bloom.Rolling(210000, 0.00001); + var filter = new RollingFilter(210000, 0.00001); var i, j, str; filter.tweak = 0xdeadbeef; @@ -107,7 +108,7 @@ describe('Bloom', function() { }); it('should handle rolling generations', function() { - var filter = new Bloom.Rolling(50, 0.00001); + var filter = new RollingFilter(50, 0.00001); var i, j, str; filter.tweak = 0xdeadbeee;