From e2817436debe7319423c44e54dc911489e9203b4 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 25 May 2016 17:38:42 -0700 Subject: [PATCH] improve address management. --- lib/bcoin/ip.js | 4 + lib/bcoin/peer.js | 207 ++++++++++++++++++++--------- lib/bcoin/pool.js | 247 ++++++++++++++++++----------------- lib/bcoin/protocol/framer.js | 4 +- lib/bcoin/protocol/parser.js | 4 +- lib/bcoin/types.js | 13 +- 6 files changed, 282 insertions(+), 197 deletions(-) diff --git a/lib/bcoin/ip.js b/lib/bcoin/ip.js index 0c49a9c9..d0b0425c 100644 --- a/lib/bcoin/ip.js +++ b/lib/bcoin/ip.js @@ -171,6 +171,10 @@ exports.toString = function toString(ip) { */ exports.normalize = function normalize(ip) { + if (Buffer.isBuffer(ip)) { + assert(ip.length === 16); + return exports.toString(ip); + } return exports.toString(exports.toBuffer(ip)); }; diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js index 5429f0d8..5d4f7f28 100644 --- a/lib/bcoin/peer.js +++ b/lib/bcoin/peer.js @@ -20,18 +20,15 @@ var constants = bcoin.protocol.constants; * @param {Object} options * @param {Function?} options.createSocket - Callback which returns a * node.js-like socket object. Necessary for browser. - * @param {Boolean} priority - Whether this peer is high - * priority (i.e. a loader). * @param {Chain} options.chain * @param {Mempool} options.mempool * @param {Number?} options.ts - Time at which peer was discovered (unix time). * @param {net.Socket?} options.socket - * @param {Seed?} options.seed - Host to connect to. + * @param {NetworkAddress?} options.host - Host to connect to. * @property {Pool} pool * @property {net.Socket?} socket * @property {String?} host * @property {Number} port - * @property {Boolean} priority * @property {Parser} parser * @property {Framer} framer * @property {Chain} chain @@ -62,8 +59,6 @@ var constants = bcoin.protocol.constants; */ function Peer(pool, options) { - var seed; - if (!(this instanceof Peer)) return new Peer(pool, options); @@ -79,7 +74,6 @@ function Peer(pool, options) { this.port = 0; this.hostname = null; this._createSocket = this.options.createSocket; - this.priority = this.options.priority; this.chain = this.pool.chain; this.mempool = this.pool.mempool; this.network = this.chain.network; @@ -109,12 +103,11 @@ function Peer(pool, options) { if (options.socket) { this.socket = options.socket; - this.host = this.socket.remoteAddress; + this.host = IP.normalize(this.socket.remoteAddress); this.port = this.socket.remotePort; - } else if (options.seed) { - seed = IP.parseHost(options.seed); - this.host = seed.host; - this.port = seed.port || this.network.port; + } else if (options.host) { + this.host = options.host.host; + this.port = options.host.port; this.socket = this.createSocket(this.port, this.host); } else { assert(false, 'No seed or socket.'); @@ -122,12 +115,10 @@ function Peer(pool, options) { assert(typeof this.host === 'string'); assert(typeof this.port === 'number'); + assert(this.socket, 'No socket.'); this.hostname = IP.hostname(this.host, this.port); - if (!this.socket) - throw new Error('No socket'); - this.requests = { timeout: this.options.requestTimeout || 10000, skip: {}, @@ -295,14 +286,10 @@ Peer.prototype.createSocket = function createSocket(port, host) { socket = net.connect(port, host); } - bcoin.debug( - 'Connecting to %s (priority=%s).', - hostname, this.priority); + bcoin.debug('Connecting to %s.', hostname); socket.once('connect', function() { - bcoin.debug( - 'Connected to %s (priority=%s).', - hostname, self.priority); + bcoin.debug('Connected to %s.', hostname); }); return socket; @@ -1380,34 +1367,24 @@ Peer.prototype._handleGetData = function _handleGetData(items) { }; Peer.prototype._handleAddr = function _handleAddr(addrs) { + var hosts = []; var now = utils.now(); var i, addr, ts; for (i = 0; i < addrs.length; i++) { - addr = addrs[i]; - - ts = addr.ts; - - if (ts <= 100000000 || ts > now + 10 * 60) - ts = now - 5 * 24 * 60 * 60; - + addr = new NetworkAddress(addrs[i]); this.addrFilter.add(addr.host, 'ascii'); - - this.emit('addr', { - version: addr.version, - ts: ts, - services: addr.services, - host: addr.host, - port: addr.port || this.network.port - }); + hosts.push(addr); } bcoin.debug( - 'Received %d addrs (seeds=%d, peers=%d) (%s).', - addrs.length, - this.pool.seeds.length, + 'Received %d addrs (hosts=%d, peers=%d) (%s).', + hosts.length, + this.pool.hosts.length, this.pool.peers.all.length, this.hostname); + + this.fire('addr', hosts); }; Peer.prototype._handlePing = function _handlePing(data) { @@ -1448,38 +1425,22 @@ Peer.prototype._handlePong = function _handlePong(data) { }; Peer.prototype._handleGetAddr = function _handleGetAddr() { - var hosts = {}; var items = []; - var ts = utils.now() - (process.uptime() | 0); - var i, seed; + var i, host; if (this.pool.options.selfish) return; - for (i = 0; i < this.pool.seeds.length; i++) { - seed = this.pool.seeds[i]; + for (i = 0; i < this.pool.hosts.length; i++) { + host = this.pool.hosts[i]; - assert(typeof seed === 'object'); - - seed = this.pool.getPeer(seed.host) || seed; - - if (IP.version(seed.host) === -1) + if (!host.isIP()) continue; - if (hosts[seed.host]) + if (!this.addrFilter.added(host.host, 'ascii')) continue; - hosts[seed.host] = true; - - if (!this.addrFilter.added(seed.host, 'ascii')) - continue; - - items.push({ - ts: seed.ts || ts, - services: seed.version ? seed.version.services : 0, - host: seed.host, - port: seed.port || this.network.port - }); + items.push(host); if (items.length === 1000) break; @@ -1675,6 +1636,125 @@ Peer.prototype.inspect = function inspect() { + '>'; }; +/** + * Represents a network address. + * @exports NetworkAddress + * @constructor + * @private + * @param {NakedNetworkAddress} options + */ + +function NetworkAddress(options) { + var host, ts, now; + + if (!(this instanceof NetworkAddress)) + return new NetworkAddress(options); + + now = utils.now(); + host = options.host; + ts = options.ts; + + if (ts <= 100000000 || ts > now + 10 * 60) + ts = now - 5 * 24 * 60 * 60; + + if (IP.version(host) !== -1) + host = IP.normalize(host); + + assert(typeof host === 'string'); + assert(typeof options.port === 'number'); + assert(typeof options.services === 'number'); + assert(typeof options.ts === 'number'); + + this.id = NetworkAddress.uid++; + this.host = host; + this.port = options.port; + this.services = options.services; + this.ts = ts; +} + +NetworkAddress.uid = 0; + +/** + * Test whether the `host` field is an ip address. + * @returns {Boolean} + */ + +NetworkAddress.prototype.isIP = function isIP() { + return IP.version(this.host) !== -1; +}; + +/** + * Test whether the NETWORK service bit is set. + * @returns {Boolean} + */ + +NetworkAddress.prototype.hasNetwork = function hasNetwork() { + return (this.services & constants.services.NETWORK) !== 0; +}; + +/** + * Test whether the BLOOM service bit is set. + * @returns {Boolean} + */ + +NetworkAddress.prototype.hasBloom = function hasBloom() { + return (this.services & constants.services.BLOOM) !== 0; +}; + +/** + * Test whether the GETUTXO service bit is set. + * @returns {Boolean} + */ + +NetworkAddress.prototype.hasUTXO = function hasUTXO() { + return (this.services & constants.services.GETUTXO) !== 0; +}; + +/** + * Test whether the WITNESS service bit is set. + * @returns {Boolean} + */ + +NetworkAddress.prototype.hasWitness = function hasWitness() { + return (this.services & constants.services.WITNESS) !== 0; +}; + +/** + * Inspect the network address. + * @returns {Object} + */ + +NetworkAddress.prototype.inspect = function inspect() { + return ''; +}; + +/** + * Instantiate a network address + * from a hostname (i.e. 127.0.0.1:8333). + * @param {String} hostname + * @param {(Network|NetworkType)?} network + * @returns {NetworkAddress} + */ + +NetworkAddress.fromHostname = function fromHostname(hostname, network) { + var address = IP.parseHost(hostname); + network = bcoin.network.get(network); + return new NetworkAddress({ + host: address.host, + port: address.port || network.port, + version: constants.VERSION, + services: constants.services.NETWORK + | constants.services.BLOOM + | constants.services.WITNESS, + ts: utils.now() + }); +}; + /* * Helpers */ @@ -1687,4 +1767,7 @@ function compare(a, b) { * Expose */ -module.exports = Peer; +exports = Peer; +exports.NetworkAddress = NetworkAddress; + +module.exports = exports; diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js index 52c81141..c7bb10b1 100644 --- a/lib/bcoin/pool.js +++ b/lib/bcoin/pool.js @@ -12,6 +12,7 @@ var IP = require('./ip'); var assert = utils.assert; var constants = bcoin.protocol.constants; var VerifyError = bcoin.errors.VerifyError; +var NetworkAddress = bcoin.peer.NetworkAddress; /** * A pool of peers for handling all network activity. @@ -45,7 +46,7 @@ var VerifyError = bcoin.errors.VerifyError; * Only deal with witness peers. * @param {Boolean} [options.discoverPeers=true] Automatically discover new * peers. - * @param {(String[]|Seed[])?} options.seeds + * @param {String[]} options.seeds * @param {Function?} options.createSocket - Custom function to create a socket. * Must accept (port, host) and return a node-like socket. * @param {Function?} options.createServer - Custom function to create a server. @@ -75,6 +76,7 @@ var VerifyError = bcoin.errors.VerifyError; */ function Pool(options) { + var self = this; var seeds; if (!(this instanceof Pool)) @@ -104,10 +106,12 @@ function Pool(options) { if (process.env.BCOIN_SEED) seeds.unshift(process.env.BCOIN_SEED); - this.originalSeeds = seeds.map(IP.parseHost); - this.seeds = []; - this.hosts = {}; - this.setSeeds([]); + this.seeds = seeds.map(function(hostname) { + return NetworkAddress.fromHostname(hostname, self.network); + }); + + this.hosts = []; + this.hostMap = {}; this.host = '0.0.0.0'; this.port = this.network.port; @@ -246,7 +250,7 @@ Pool.prototype.connect = function connect() { }); } - if (this.originalSeeds.length > 0) { + if (this.seeds.length > 0) { this._addLoader(); for (i = 0; i < this.size - 1; i++) @@ -555,8 +559,7 @@ Pool.prototype._addLoader = function _addLoader() { return; peer = this._createPeer({ - seed: this.getSeed(true), - priority: true, + host: this.getLoaderHost(), network: true, spv: this.options.spv, witness: this.options.witness @@ -966,10 +969,9 @@ Pool.prototype._createPeer = function _createPeer(options) { var self = this; var peer = new bcoin.peer(this, { - seed: options.seed, + host: options.host, createSocket: this.options.createSocket, relay: this.options.relay, - priority: options.priority, socket: options.socket, network: options.network, spv: options.spv, @@ -1021,34 +1023,38 @@ Pool.prototype._createPeer = function _createPeer(options) { }); }); - peer.on('addr', function(data) { + peer.on('addr', function(hosts) { + var i, host; + if (self.options.discoverPeers === false) return; - if (!(data.services & constants.services.NETWORK)) - return; + for (i = 0; i < hosts.length; i++) { + host = hosts[i]; - if (self.options.headers) { - if (data.version < 31800) - return; + if (!host.hasNetwork()) + continue; + + if (self.options.headers) { + if (!host.hasHeaders()) + continue; + } + + if (self.options.spv) { + if (!host.hasBloom()) + continue; + } + + if (self.options.witness) { + if (!host.hasWitness()) + continue; + } + + if (self.addHost(host)) + self.emit('host', host, peer); } - if (self.options.spv) { - if (data.version < 70011 || !(data.services & constants.services.BLOOM)) - return; - } - - if (self.options.witness) { - if (!(data.services & constants.services.WITNESS)) - return; - } - - if (self.seeds.length > 300) - self.setSeeds(self.seeds.slice(-150)); - - self.addSeed(data); - - self.emit('addr', data, peer); + self.emit('addr', hosts, peer); }); peer.on('txs', function(txs) { @@ -1148,7 +1154,6 @@ Pool.prototype._addLeech = function _addLeech(socket) { peer = this._createPeer({ socket: socket, - priority: false, network: false, spv: false, witness: false, @@ -1195,7 +1200,7 @@ Pool.prototype._addLeech = function _addLeech(socket) { Pool.prototype._addPeer = function _addPeer() { var self = this; - var peer, seed; + var peer, host; if (this.destroyed) return; @@ -1203,16 +1208,15 @@ Pool.prototype._addPeer = function _addPeer() { if (this.peers.regular.length + this.peers.pending.length >= this.size - 1) return; - seed = this.getSeed(false); + host = this.getHost(); - if (!seed) { + if (!host) { setTimeout(this._addPeer.bind(this), 5000); return; } peer = this._createPeer({ - seed: seed, - priority: false, + host: host, network: true, spv: this.options.spv, witness: this.options.witness @@ -1838,12 +1842,12 @@ Pool.prototype.destroy = function destroy(callback) { /** * Get peer by host. - * @param {Seed|String} addr + * @param {String} addr * @returns {Peer?} */ Pool.prototype.getPeer = function getPeer(host) { - return this.peers.map[host.host || host]; + return this.peers.map[host]; }; /** @@ -1853,7 +1857,23 @@ Pool.prototype.getPeer = function getPeer(host) { */ Pool.prototype.getUTXOs = function getUTXOs(utxos, callback) { - var peer = this.peers.load || this.peers.regular[0]; + var i, peer; + + if (this.peers.load && this.peers.load.version) { + if (this.peers.load.version.services & constants.services.BLOOM) + peer = this.peers.load; + } + + if (!peer) { + for (i = 0; i < this.peers.regular.length; i++) { + peer = this.peers.regular[i]; + if (peer.version.services & constants.services.BLOOM) + break; + + } + if (i === this.peers.regular.length) + peer = null; + } if (!peer) return utils.asyncify(callback)(new Error('No peer available.')); @@ -1867,7 +1887,7 @@ Pool.prototype.getUTXOs = function getUTXOs(utxos, callback) { * @param {Function} callback - Returns [Error, {@link TX}]. */ -Pool.prototype.fillHistory = function fillHistory(tx, callback) { +Pool.prototype.fillCoins = function fillCoins(tx, callback) { var utxos = []; var i, input; @@ -1891,135 +1911,119 @@ Pool.prototype.fillHistory = function fillHistory(tx, callback) { }; /** - * Allocate a new seed which is not currently being used. - * @param {Boolean?} priority - If true, the peer that - * is going to use this seed is high-priority. - * @returns {Seed} + * Allocate a new loader host. + * @returns {NetworkAddress} */ -Pool.prototype.getSeed = function getSeed(priority) { - var addr; +Pool.prototype.getLoaderHost = function getLoaderHost() { + var host; - if (priority) { - if (!this.connected) - return this.originalSeeds[0]; + if (!this.connected) + return this.seeds[0]; - addr = this._getRandom(this.originalSeeds); - if (addr) - return addr; + host = this.getRandom(this.seeds); + if (host) + return host; - addr = this._getRandom(this.seeds); - if (addr) - return addr; + return this.getRandom(this.hosts); +}; - addr = this.seeds[Math.random() * this.seeds.length | 0]; - if (addr) - return addr; +/** + * Allocate a new host which is not currently being used. + * @returns {NetworkAddress} + */ - return this.originalSeeds[Math.random() * this.originalSeeds.length | 0]; - } +Pool.prototype.getHost = function getHost() { + var host; // Hang back if we don't have a loader peer yet. if (!this.peers.load) return; - addr = this._getRandom(this.originalSeeds, true); - if (addr) - return addr; + host = this.getRandom(this.seeds, true); + if (host) + return host; - addr = this._getRandom(this.seeds, true); - if (addr) - return addr; + return this.getRandom(this.hosts, true); }; -Pool.prototype._getRandom = function _getRandom(seeds, uniq) { +/** + * Get a random host from collection of hosts. + * @param {NetworkAddress[]} hosts + * @param {Boolean} unique + * @returns {NetworkAddress} + */ + +Pool.prototype.getRandom = function getRandom(hosts, unique) { var tried = {}; var tries = 0; - var index, addr; + var index, host; + + if (!unique) + return hosts[Math.random() * hosts.length | 0]; for (;;) { - if (tries === seeds.length) + if (tries === hosts.length) return; - index = Math.random() * seeds.length | 0; - addr = seeds[index]; + index = Math.random() * hosts.length | 0; + host = hosts[index]; if (!tried[index]) { tried[index] = true; tries++; } - if (this.isMisbehaving(addr.host)) + if (this.getPeer(host.host)) continue; - if (uniq && this.getPeer(addr.host)) - continue; - - return addr; + return host; } }; /** - * Reset seeds list. - * @param {String[]|Seed[]} seeds - */ - -Pool.prototype.setSeeds = function setSeeds(seeds) { - var i; - - this.seeds = []; - this.hosts = {}; - - for (i = 0; i < seeds.length; i++) - this.addSeed(seeds[i]); -}; - -/** - * Add seed to seed list. - * @param {String|Seed} seed + * Add host to host list. + * @param {String|NetworkAddress} host * @returns {Boolean} */ -Pool.prototype.addSeed = function addSeed(seed) { - seed = IP.parseHost(seed); +Pool.prototype.addHost = function addHost(host) { + if (typeof host === 'string') + host = NetworkAddress.fromHostname(host); - if (this.hosts[seed.host] != null) - return false; + if (this.hosts.length > 500) + return; - this.seeds.push({ - host: seed.host, - port: seed.port || this.network.port - }); + if (this.hostMap[host.host]) + return; - this.hosts[seed.host] = true; + utils.binaryInsert(this.hosts, host, compare); - return true; + this.hostMap[host.host] = host; + + return host; }; /** - * Remove seed from seed list. - * @param {String|Seed} seed + * Remove host from host list. + * @param {String|NetworkAddress} host * @returns {Boolean} */ -Pool.prototype.removeSeed = function removeSeed(seed) { - var i; +Pool.prototype.removeHost = function removeHost(host) { + if (host.host) + host = host.host; - seed = IP.parseHost(seed); + host = this.hostMap[host]; - if (this.hosts[seed.host] == null) - return false; + if (!host) + return; - for (i = 0; i < this.seeds.length; i++) { - if (this.seeds[i].host === seed.host) { - this.seeds.splice(i, 1); - break; - } - } + utils.binaryRemove(this.hosts, host, compare); - delete this.hosts[seed.host]; + delete this.hostMap[host]; - return true; + return host; }; /** @@ -2034,6 +2038,7 @@ Pool.prototype.setMisbehavior = function setMisbehavior(peer, score) { if (peer.banScore >= constants.BAN_SCORE) { this.peers.misbehaving[peer.host] = utils.now(); + this.removeHost(peer.host); bcoin.debug('Ban threshold exceeded (%s).', peer.host); peer.destroy(); return true; @@ -2056,7 +2061,7 @@ Pool.prototype.isMisbehaving = function isMisbehaving(host) { time = this.peers.misbehaving[host]; - if (time) { + if (time != null) { if (utils.now() > time + constants.BAN_TIME) { delete this.peers.misbehaving[host]; peer = this.getPeer(host); diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js index 45ac540d..2b6688be 100644 --- a/lib/bcoin/protocol/framer.js +++ b/lib/bcoin/protocol/framer.js @@ -416,7 +416,7 @@ Framer.prototype.feeFilter = function feeFilter(options) { /** * Serialize an address. - * @param {NetworkAddress} data + * @param {NakedNetworkAddress} data * @param {Boolean?} full - Whether to include the timestamp. * @param {BufferWriter?} writer - A buffer writer to continue writing from. * @returns {Buffer} Returns a BufferWriter if `writer` was passed in. @@ -1163,7 +1163,7 @@ Framer.reject = function reject(details, writer) { /** * Create an addr packet (without a header). - * @param {NetworkAddress[]} hosts + * @param {NakedNetworkAddress[]} hosts * @param {BufferWriter?} writer - A buffer writer to continue writing from. * @returns {Buffer} Returns a BufferWriter if `writer` was passed in. */ diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js index aab4f17b..e903d55a 100644 --- a/lib/bcoin/protocol/parser.js +++ b/lib/bcoin/protocol/parser.js @@ -1176,7 +1176,7 @@ Parser.parseReject = function parseReject(p) { /** * Parse serialized network address. * @param {Buffer|BufferReader} p - * @returns {NetworkAddress} + * @returns {NakedNetworkAddress} */ Parser.parseAddress = function parseAddress(p, full) { @@ -1206,7 +1206,7 @@ Parser.parseAddress = function parseAddress(p, full) { /** * Parse addr packet. * @param {Buffer|BufferReader} p - * @returns {NetworkAddress[]} + * @returns {NakedNetworkAddress[]} */ Parser.parseAddr = function parseAddr(p) { diff --git a/lib/bcoin/types.js b/lib/bcoin/types.js index bb76417f..244d3d6e 100644 --- a/lib/bcoin/types.js +++ b/lib/bcoin/types.js @@ -46,13 +46,6 @@ * @global */ -/** - * @typedef {Object} Seed - * @property {String} host - * @property {Number} port - * @global - */ - /** * A map of addresses ({@link Base58Address} -> value). * @typedef {Object} AddressMap @@ -255,7 +248,7 @@ */ /** - * @typedef {Object} NetworkAddress + * @typedef {Object} NakedNetworkAddress * @property {Number?} ts - Timestamp. * @property {Number?} services - Service bits. * @property {String?} host - IP address (IPv6 or IPv4). @@ -268,8 +261,8 @@ * @property {Number} version - Protocol version. * @property {Number} services - Service bits. * @property {Number} ts - Timestamp of discovery. - * @property {NetworkAddress} local - Our address. - * @property {NetworkAddress} remote - Their address. + * @property {NakedNetworkAddress} local - Our address. + * @property {NakedNetworkAddress} remote - Their address. * @property {BN} nonce * @property {String} agent - User agent string. * @property {Number} height - Chain height.