From dd6bd8db4a8a35a142040345e9f703ad275793dd Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Tue, 27 Dec 2016 14:38:32 -0800 Subject: [PATCH] net: move hostlist. --- lib/net/hostlist.js | 1075 +++++++++++++++++++++++++++++++++++++++++++ lib/net/index.js | 1 + lib/net/pool.js | 1043 +---------------------------------------- 3 files changed, 1077 insertions(+), 1042 deletions(-) create mode 100644 lib/net/hostlist.js diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js new file mode 100644 index 00000000..8e035e16 --- /dev/null +++ b/lib/net/hostlist.js @@ -0,0 +1,1075 @@ +/*! + * hostlist.js - address management for bcoin + * Copyright (c) 2014-2016, Christopher Jeffrey (MIT License). + * https://github.com/bcoin-org/bcoin + */ + +'use strict'; + +var assert = require('assert'); +var util = require('../utils/util'); +var IP = require('../utils/ip'); +var co = require('../utils/co'); +var NetAddress = require('../primitives/netaddress'); +var List = require('../utils/list'); +var murmur3 = require('../utils/murmur3'); +var StaticWriter = require('../utils/staticwriter'); + +/** + * Host List + * @constructor + * @param {Object} options + */ + +function HostList(options) { + if (!(this instanceof HostList)) + return new HostList(options); + + assert(options, 'Options are required.'); + assert(options.address); + assert(options.network); + assert(options.logger); + assert(typeof options.resolve === 'function'); + assert(options.banTime >= 0); + + this.address = options.address; + this.network = options.network; + this.logger = options.logger; + this.proxyServer = options.proxyServer; + this.resolve = options.resolve; + this.banTime = options.banTime; + + this.seeds = []; + this.banned = {}; + + this.map = {}; + this.fresh = []; + this.used = []; + + this.totalFresh = 0; + this.totalUsed = 0; + + this.maxBuckets = 20; + this.maxEntries = 50; + this.maxAddresses = this.maxBuckets * this.maxEntries; + + this.horizonDays = 30; + this.retries = 3; + this.minFailDays = 7; + this.maxFailures = 10; + this.maxRefs = 8; + + this._init(); +} + +/** + * Initialize list. + * @private + */ + +HostList.prototype._init = function init() { + var i; + + for (i = 0; i < this.maxBuckets; i++) + this.fresh.push(new MapBucket()); + + for (i = 0; i < this.maxBuckets; i++) + this.used.push(new List()); + + this.setSeeds(this.network.seeds); +}; + +/** + * Get list size. + * @returns {Number} + */ + +HostList.prototype.size = function size() { + return this.totalFresh + this.totalUsed; +}; + +/** + * Test whether the host list is full. + * @returns {Boolean} + */ + +HostList.prototype.isFull = function isFull() { + return this.size() >= this.maxAddresses; +}; + +/** + * Reset host list. + */ + +HostList.prototype.reset = function reset() { + var i, bucket; + + this.map = {}; + + for (i = 0; i < this.fresh.length; i++) { + bucket = this.fresh[i]; + bucket.reset(); + } + + for (i = 0; i < this.used.length; i++) { + bucket = this.used[i]; + bucket.reset(); + } + + this.totalFresh = 0; + this.totalUsed = 0; +}; + +/** + * Mark a peer as banned. + * @param {String} host + */ + +HostList.prototype.ban = function ban(host) { + this.banned[host] = util.now(); +}; + +/** + * Unban host. + * @param {String} host + */ + +HostList.prototype.unban = function unban(host) { + delete this.banned[host]; +}; + +/** + * Clear banned hosts. + */ + +HostList.prototype.clearBanned = function clearBanned() { + this.banned = {}; +}; + +/** + * Test whether the host is banned. + * @param {String} host + * @returns {Boolean} + */ + +HostList.prototype.isBanned = function isBanned(host) { + var time = this.banned[host]; + + if (time == null) + return false; + + if (util.now() > time + this.banTime) { + delete this.banned[host]; + return false; + } + + return true; +}; + +/** + * Allocate a new host. + * @returns {HostEntry} + */ + +HostList.prototype.getHost = function getHost() { + var now = this.network.now(); + var buckets = null; + var factor = 1; + var index, key, bucket, entry, num; + + if (this.totalFresh > 0) + buckets = this.fresh; + + if (this.totalUsed > 0) { + if (this.totalFresh === 0 || util.random(0, 2) === 0) + buckets = this.used; + } + + if (!buckets) + return; + + for (;;) { + index = util.random(0, buckets.length); + bucket = buckets[index]; + + if (bucket.size === 0) + continue; + + index = util.random(0, bucket.size); + + if (buckets === this.used) { + entry = bucket.head; + while (index--) + entry = entry.next; + } else { + key = bucket.keys()[index]; + entry = bucket.get(key); + } + + num = util.random(0, 1 << 30); + + if (num < factor * entry.chance(now) * (1 << 30)) + return entry; + + factor *= 1.2; + } +}; + +/** + * Get fresh bucket for host. + * @private + * @param {HostEntry} entry + * @returns {MapBucket} + */ + +HostList.prototype.freshBucket = function freshBucket(entry) { + var size = 0; + var bw, hash, index; + + size += entry.addr.host.length; + size += entry.src.host.length; + + bw = new StaticWriter(size); + bw.writeString(entry.addr.host, 'ascii'); + bw.writeString(entry.src.host, 'ascii'); + + hash = murmur3(bw.render(), 0xfba4c795); + index = hash % this.fresh.length; + + return this.fresh[index]; +}; + +/** + * Get used bucket for host. + * @private + * @param {HostEntry} entry + * @returns {List} + */ + +HostList.prototype.usedBucket = function usedBucket(entry) { + var data = new Buffer(entry.addr.host, 'ascii'); + var hash = murmur3(data, 0xfba4c795); + var index = hash % this.used.length; + return this.used[index]; +}; + +/** + * Add host to host list. + * @param {NetAddress} addr + * @param {NetAddress?} src + * @returns {Boolean} + */ + +HostList.prototype.add = function add(addr, src) { + var now = this.network.now(); + var penalty = 2 * 60 * 60; + var interval = 24 * 60 * 60; + var factor = 1; + var i, entry, bucket; + + if (this.isFull()) + return false; + + entry = this.map[addr.hostname]; + + if (entry) { + // No source means we're inserting + // this ourselves. No penalty. + if (!src) + penalty = 0; + + // Update services. + entry.addr.services |= addr.services; + + // Online? + if (now - addr.ts < 24 * 60 * 60) + interval = 60 * 60; + + // Periodically update time. + if (entry.addr.ts < addr.ts - interval - penalty) + entry.addr.ts = addr.ts; + + // Do not update if no new + // information is present. + if (entry.addr.ts && addr.ts <= entry.addr.ts) + return false; + + // Do not update if the entry was + // already in the "used" table. + if (entry.used) + return false; + + assert(entry.refCount > 0); + + // Do not update if the max + // reference count is reached. + if (entry.refCount === this.maxRefs) + return false; + + assert(entry.refCount < this.maxRefs); + + // Stochastic test: previous refCount + // N: 2^N times harder to increase it. + for (i = 0; i < entry.refCount; i++) + factor *= 2; + + if (util.random(0, factor) !== 0) + return false; + } else { + if (!src) + src = this.address; + + entry = new HostEntry(addr, src); + + this.totalFresh++; + } + + bucket = this.freshBucket(entry); + + if (bucket.has(entry.key())) + return false; + + if (bucket.size >= this.maxEntries) + this.evictFresh(bucket); + + bucket.set(entry.key(), entry); + entry.refCount++; + + this.map[entry.key()] = entry; + + return true; +}; + +/** + * Evict a host from fresh bucket. + * @param {MapBucket} bucket + */ + +HostList.prototype.evictFresh = function evictFresh(bucket) { + var keys = bucket.keys(); + var i, key, entry, old; + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + entry = bucket.get(key); + + if (this.isStale(entry)) { + bucket.remove(entry.key()); + + if (--entry.refCount === 0) { + delete this.map[entry.key()]; + this.totalFresh--; + } + + continue; + } + + if (!old) { + old = entry; + continue; + } + + if (entry.addr.ts < old.addr.ts) + old = entry; + } + + if (!old) + return; + + bucket.remove(old.key()); + + if (--old.refCount === 0) { + delete this.map[old.key()]; + this.totalFresh--; + } +}; + +/** + * Test whether a host is evictable. + * @param {HostEntry} entry + * @returns {Boolean} + */ + +HostList.prototype.isStale = function isStale(entry) { + var now = this.network.now(); + + if (entry.lastAttempt && entry.lastAttempt >= now - 60) + return false; + + if (entry.addr.ts > now + 10 * 60) + return true; + + if (entry.addr.ts === 0) + return true; + + if (now - entry.addr.ts > this.horizonDays * 24 * 60 * 60) + return true; + + if (entry.lastSuccess === 0 && entry.attempts >= this.retries) + return true; + + if (now - entry.lastSuccess > this.minFailDays * 24 * 60 * 60) { + if (entry.attempts >= this.maxFailures) + return true; + } + + return false; +}; + +/** + * Remove host from host list. + * @param {String} hostname + * @returns {NetAddress} + */ + +HostList.prototype.remove = function remove(hostname) { + var entry = this.map[hostname]; + var i, head, bucket; + + if (!entry) + return; + + if (entry.used) { + assert(entry.refCount === 0); + + head = entry; + while (head.prev) + head = head.prev; + + for (i = 0; i < this.used.length; i++) { + bucket = this.used[i]; + if (bucket.head === head) { + bucket.remove(entry); + this.totalUsed--; + break; + } + } + + assert(i < this.used.length); + } else { + for (i = 0; i < this.fresh.length; i++) { + bucket = this.fresh[i]; + if (bucket.remove(entry.key())) + entry.refCount--; + } + + this.totalFresh--; + assert(entry.refCount === 0); + } + + delete this.map[entry.key()]; + + return entry.addr; +}; + +/** + * Mark host as failed. + * @param {String} hostname + */ + +HostList.prototype.markAttempt = function markAttempt(hostname) { + var entry = this.map[hostname]; + var now = this.network.now(); + + if (!entry) + return; + + entry.attempts++; + entry.lastAttempt = now; +}; + +/** + * Mark host as successfully connected. + * @param {String} hostname + */ + +HostList.prototype.markSuccess = function markSuccess(hostname) { + var entry = this.map[hostname]; + var now = this.network.now(); + + if (!entry) + return; + + if (now - entry.addr.ts > 20 * 60) + entry.addr.ts = now; +}; + +/** + * Mark host as successfully connected. + * @param {String} hostname + * @param {Number} services + */ + +HostList.prototype.markAck = function markAck(hostname, services) { + var entry = this.map[hostname]; + var now = this.network.now(); + var i, bucket, evicted, old, fresh; + + if (!entry) + return; + + entry.addr.services |= services; + entry.lastSuccess = now; + entry.lastAttempt = now; + entry.attempts = 0; + + if (entry.used) + return; + + assert(entry.refCount > 0); + + // Remove from fresh. + for (i = 0; i < this.fresh.length; i++) { + bucket = this.fresh[i]; + if (bucket.remove(entry.key())) { + entry.refCount--; + old = bucket; + } + } + + assert(old); + assert(entry.refCount === 0); + this.totalFresh--; + + // Find room in used bucket. + bucket = this.usedBucket(entry); + + if (bucket.size < this.maxEntries) { + entry.used = true; + bucket.push(entry); + this.totalUsed++; + return; + } + + // No room. Evict. + evicted = this.evictUsed(bucket); + fresh = this.freshBucket(evicted); + + // Move to entry's old bucket if no room. + if (fresh.size >= this.maxEntries) + fresh = old; + + // Swap to evicted's used bucket. + entry.used = true; + bucket.replace(evicted, entry); + + // Move evicted to fresh bucket. + evicted.used = false; + fresh.set(evicted.key(), evicted); + assert(evicted.refCount === 0); + evicted.refCount++; + this.totalFresh++; +}; + +/** + * Pick used for eviction. + * @param {List} bucket + */ + +HostList.prototype.evictUsed = function evictUsed(bucket) { + var old = bucket.head; + var entry; + + for (entry = bucket.head; entry; entry = entry.next) { + if (entry.addr.ts < old.addr.ts) + old = entry; + } + + return old; +}; + +/** + * Convert address list to array. + * @returns {NetAddress[]} + */ + +HostList.prototype.toArray = function toArray() { + var keys = Object.keys(this.map); + var out = []; + var i, key, entry; + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + entry = this.map[key]; + out.push(entry.addr); + } + + assert.equal(out.length, this.size()); + + return out; +}; + +/** + * Set initial seeds. + * @param {String[]} seeds + */ + +HostList.prototype.setSeeds = function setSeeds(seeds) { + var i, seed; + + this.seeds.length = 0; + + for (i = 0; i < seeds.length; i++) { + seed = seeds[i]; + this.addSeed(seed); + } +}; + +/** + * Add a preferred seed. + * @param {String} hostname + */ + +HostList.prototype.addSeed = function addSeed(host) { + var addr = IP.parseHost(host, this.network.port); + return this.seeds.push(addr); +}; + +/** + * Discover hosts from seeds. + * @returns {Promise} + */ + +HostList.prototype.discover = co(function* discover() { + var jobs = []; + var i, seed; + + for (i = 0; i < this.seeds.length; i++) { + seed = this.seeds[i]; + jobs.push(this.populate(seed)); + } + + yield Promise.all(jobs); +}); + +/** + * Populate from seed. + * @param {Object} seed + * @returns {Promise} + */ + +HostList.prototype.populate = co(function* populate(seed) { + var i, addr, hosts, host; + + if (seed.version !== -1) { + addr = NetAddress.fromHost(seed.host, seed.port, this.network); + this.add(addr); + return; + } + + this.logger.info('Resolving hosts from seed: %s.', seed.host); + + try { + hosts = yield this.resolve(seed.host, this.proxyServer); + } catch (e) { + this.logger.error(e); + return; + } + + for (i = 0; i < hosts.length; i++) { + host = hosts[i]; + addr = NetAddress.fromHost(host, seed.port, this.network); + this.add(addr); + } +}); + +/** + * Convert host list to json-friendly object. + * @returns {Object} + */ + +HostList.prototype.toJSON = function toJSON() { + var addrs = []; + var fresh = []; + var used = []; + var i, keys, key, bucket, entry; + + keys = Object.keys(this.map); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + entry = this.map[key]; + addrs.push(entry.toJSON()); + } + + for (i = 0; i < this.fresh.length; i++) { + bucket = this.fresh[i]; + keys = bucket.keys(); + fresh.push(keys); + } + + for (i = 0; i < this.used.length; i++) { + bucket = this.used[i]; + keys = []; + for (entry = bucket.head; entry; entry = entry.next) + keys.push(entry.key()); + used.push(keys); + } + + return { + version: 1, + addrs: addrs, + fresh: fresh, + used: used + }; +}; + +/** + * Inject properties from json object. + * @private + * @param {Object} json + * @returns {HostList} + */ + +HostList.prototype.fromJSON = function fromJSON(json) { + var sources = {}; + var i, j, bucket, keys, key, addr, entry, src; + + assert(json && typeof json === 'object'); + assert(json.version === 1, 'Bad address serialization version.'); + + assert(Array.isArray(json.addrs)); + + for (i = 0; i < json.addrs.length; i++) { + addr = json.addrs[i]; + entry = HostEntry.fromJSON(addr, this.network); + src = sources[entry.src.hostname]; + + // Save some memory. + if (!src) { + src = entry.src; + sources[src.hostname] = src; + } + + entry.src = src; + + this.map[entry.key()] = entry; + } + + assert(Array.isArray(json.fresh)); + + for (i = 0; i < json.fresh.length; i++) { + keys = json.fresh[i]; + bucket = this.fresh[i]; + assert(bucket, 'No bucket available.'); + for (j = 0; j < keys.length; j++) { + key = keys[j]; + entry = this.map[key]; + assert(entry); + if (entry.refCount === 0) + this.totalFresh++; + entry.refCount++; + bucket.set(key, entry); + } + } + + assert(Array.isArray(json.used)); + + for (i = 0; i < json.used.length; i++) { + keys = json.used[i]; + bucket = this.used[i]; + assert(bucket, 'No bucket available.'); + for (j = 0; j < keys.length; j++) { + key = keys[j]; + entry = this.map[key]; + assert(entry); + assert(entry.refCount === 0); + assert(!entry.used); + entry.used = true; + this.totalUsed++; + bucket.push(entry); + } + } + + keys = Object.keys(this.map); + + for (i = 0; i < keys.length; i++) { + key = keys[i]; + entry = this.map[key]; + assert(entry.used || entry.refCount > 0); + } + + return this; +}; + +/** + * Instantiate host list from json object. + * @param {Object} options + * @param {Object} json + * @returns {HostList} + */ + +HostList.fromJSON = function fromJSON(options, json) { + return new HostEntry(options).fromJSON(json); +}; + +/** + * MapBucket + * @constructor + */ + +function MapBucket() { + if (!(this instanceof MapBucket)) + return new MapBucket(); + + this.map = {}; + this.size = 0; +} + +/** + * Get map keys. + * @returns {String[]} + */ + +MapBucket.prototype.keys = function keys() { + return Object.keys(this.map); +}; + +/** + * Get item from map. + * @param {String} key + * @returns {Object|null} + */ + +MapBucket.prototype.get = function get(key) { + return this.map[key]; +}; + +/** + * Test whether map has an item. + * @param {String} key + * @returns {Boolean} + */ + +MapBucket.prototype.has = function has(key) { + return this.map[key] !== undefined; +}; + +/** + * Set a key to value in map. + * @param {String} key + * @param {Object} value + * @returns {Boolean} + */ + +MapBucket.prototype.set = function set(key, value) { + var item = this.map[key]; + + assert(value !== undefined); + + this.map[key] = value; + + if (item === undefined) { + this.size++; + return true; + } + + return false; +}; + +/** + * Remove an item from map. + * @param {String} key + * @returns {Object|null} + */ + +MapBucket.prototype.remove = function remove(key) { + var item = this.map[key]; + + if (item === undefined) + return; + + delete this.map[key]; + this.size--; + + return item; +}; + +/** + * Reset the map. + */ + +MapBucket.prototype.reset = function reset() { + this.map = {}; + this.size = 0; +}; + +/** + * HostEntry + * @constructor + * @param {NetAddress} addr + * @param {NetAddress} src + */ + +function HostEntry(addr, src) { + if (!(this instanceof HostEntry)) + return new HostEntry(addr, src); + + this.addr = addr || new NetAddress(); + this.src = src || new NetAddress(); + this.prev = null; + this.next = null; + this.used = false; + this.refCount = 0; + this.attempts = 0; + this.lastSuccess = 0; + this.lastAttempt = 0; + + if (addr) + this.fromOptions(addr, src); +} + +/** + * Inject properties from options. + * @private + * @param {NetAddress} addr + * @param {NetAddress} src + * @returns {HostEntry} + */ + +HostEntry.prototype.fromOptions = function fromOptions(addr, src) { + assert(addr instanceof NetAddress); + assert(src instanceof NetAddress); + this.addr = addr; + this.src = src; + return this; +}; + +/** + * Instantiate host entry from options. + * @param {NetAddress} addr + * @param {NetAddress} src + * @returns {HostEntry} + */ + +HostEntry.fromOptions = function fromOptions(addr, src) { + return new HostEntry().fromOptions(addr, src); +}; + +/** + * Get key suitable for a hash table (hostname). + * @returns {String} + */ + +HostEntry.prototype.key = function key() { + return this.addr.hostname; +}; + +/** + * Get host priority. + * @param {Number} now + * @returns {Number} + */ + +HostEntry.prototype.chance = function _chance(now) { + var attempts = this.attempts; + var chance = 1; + + if (now - this.lastAttempt < 60 * 10) + chance *= 0.01; + + chance *= Math.pow(0.66, Math.min(attempts, 8)); + + return chance; +}; + +/** + * Inspect host address. + * @returns {Object} + */ + +HostEntry.prototype.inspect = function inspect() { + return { + addr: this.addr, + src: this.src, + used: this.used, + refCount: this.refCount, + attempts: this.attempts, + lastSuccess: util.date(this.lastSuccess), + lastAttempt: util.date(this.lastAttempt) + }; +}; + +/** + * Convert host entry to json-friendly object. + * @returns {Object} + */ + +HostEntry.prototype.toJSON = function toJSON() { + return { + addr: this.addr.hostname, + src: this.src.hostname, + services: this.addr.services.toString(2), + ts: this.addr.ts, + attempts: this.attempts, + lastSuccess: this.lastSuccess, + lastAttempt: this.lastAttempt + }; +}; + +/** + * Inject properties from json object. + * @private + * @param {Object} json + * @param {Network} network + * @returns {HostEntry} + */ + +HostEntry.prototype.fromJSON = function fromJSON(json, network) { + assert(json && typeof json === 'object'); + assert(typeof json.addr === 'string'); + assert(typeof json.src === 'string'); + + this.addr.fromHostname(json.addr, network); + + if (json.services != null) { + assert(typeof json.services === 'string'); + assert(json.services.length > 0); + assert(json.services.length < 64); + this.addr.services = parseInt(json.services, 2); + } + + if (json.ts != null) { + assert(util.isNumber(json.ts)); + this.addr.ts = json.ts; + } + + if (json.src != null) { + assert(typeof json.src === 'string'); + this.src.fromHostname(json.src, network); + } + + if (json.attempts != null) { + assert(util.isNumber(json.attempts)); + this.attempts = json.attempts; + } + + if (json.lastSuccess != null) { + assert(util.isNumber(json.lastSuccess)); + this.lastSuccess = json.lastSuccess; + } + + if (json.lastAttempt != null) { + assert(util.isNumber(json.lastAttempt)); + this.lastAttempt = json.lastAttempt; + } + + return this; +}; + +/** + * Instantiate host entry from json object. + * @param {Object} json + * @param {Network} network + * @returns {HostEntry} + */ + +HostEntry.fromJSON = function fromJSON(json, network) { + return new HostEntry().fromJSON(json, network); +}; + +/* + * Expose + */ + +module.exports = HostList; diff --git a/lib/net/index.js b/lib/net/index.js index dbede865..6822f967 100644 --- a/lib/net/index.js +++ b/lib/net/index.js @@ -5,6 +5,7 @@ exports.BIP151 = require('./bip151'); exports.bip152 = require('./bip152'); exports.packets = require('./packets'); exports.Framer = require('./framer'); +exports.HostList = require('./hostlist'); exports.Parser = require('./parser'); exports.Peer = require('./peer'); exports.Pool = require('./pool'); diff --git a/lib/net/pool.js b/lib/net/pool.js index 5d370595..54746339 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -28,8 +28,7 @@ var request = require('../http/request'); var List = require('../utils/list'); var tcp = require('./tcp'); var dns = require('./dns'); -var murmur3 = require('../utils/murmur3'); -var StaticWriter = require('../utils/staticwriter'); +var HostList = require('./hostlist'); var invTypes = constants.inv; var VerifyError = errors.VerifyError; var VerifyResult = errors.VerifyResult; @@ -2240,1046 +2239,6 @@ PeerList.prototype.destroy = function destroy() { } }; -/** - * Host List - * @constructor - * @param {Object} options - */ - -function HostList(options) { - this.address = options.address; - this.network = options.network; - this.logger = options.logger; - this.proxyServer = options.proxyServer; - this.resolve = options.resolve; - this.banTime = options.banTime; - - this.seeds = []; - this.banned = {}; - - this.map = {}; - this.fresh = []; - this.used = []; - - this.totalFresh = 0; - this.totalUsed = 0; - - this.maxBuckets = 20; - this.maxEntries = 50; - this.maxAddresses = this.maxBuckets * this.maxEntries; - - this.horizonDays = 30; - this.retries = 3; - this.minFailDays = 7; - this.maxFailures = 10; - this.maxRefs = 8; - - this._init(); -} - -/** - * Initialize list. - * @private - */ - -HostList.prototype._init = function init() { - var i; - - for (i = 0; i < this.maxBuckets; i++) - this.fresh.push(new MapBucket()); - - for (i = 0; i < this.maxBuckets; i++) - this.used.push(new List()); - - this.setSeeds(this.network.seeds); -}; - -/** - * Get list size. - * @returns {Number} - */ - -HostList.prototype.size = function size() { - return this.totalFresh + this.totalUsed; -}; - -/** - * Test whether the host list is full. - * @returns {Boolean} - */ - -HostList.prototype.isFull = function isFull() { - return this.size() >= this.maxAddresses; -}; - -/** - * Reset host list. - */ - -HostList.prototype.reset = function reset() { - var i, bucket; - - this.map = {}; - - for (i = 0; i < this.fresh.length; i++) { - bucket = this.fresh[i]; - bucket.reset(); - } - - for (i = 0; i < this.used.length; i++) { - bucket = this.used[i]; - bucket.reset(); - } - - this.totalFresh = 0; - this.totalUsed = 0; -}; - -/** - * Mark a peer as banned. - * @param {String} host - */ - -HostList.prototype.ban = function ban(host) { - this.banned[host] = util.now(); -}; - -/** - * Unban host. - * @param {String} host - */ - -HostList.prototype.unban = function unban(host) { - delete this.banned[host]; -}; - -/** - * Clear banned hosts. - */ - -HostList.prototype.clearBanned = function clearBanned() { - this.banned = {}; -}; - -/** - * Test whether the host is banned. - * @param {String} host - * @returns {Boolean} - */ - -HostList.prototype.isBanned = function isBanned(host) { - var time = this.banned[host]; - - if (time == null) - return false; - - if (util.now() > time + this.banTime) { - delete this.banned[host]; - return false; - } - - return true; -}; - -/** - * Allocate a new host. - * @returns {HostEntry} - */ - -HostList.prototype.getHost = function getHost() { - var now = this.network.now(); - var buckets = null; - var factor = 1; - var index, key, bucket, entry, num; - - if (this.totalFresh > 0) - buckets = this.fresh; - - if (this.totalUsed > 0) { - if (this.totalFresh === 0 || util.random(0, 2) === 0) - buckets = this.used; - } - - if (!buckets) - return; - - for (;;) { - index = util.random(0, buckets.length); - bucket = buckets[index]; - - if (bucket.size === 0) - continue; - - index = util.random(0, bucket.size); - - if (buckets === this.used) { - entry = bucket.head; - while (index--) - entry = entry.next; - } else { - key = bucket.keys()[index]; - entry = bucket.get(key); - } - - num = util.random(0, 1 << 30); - - if (num < factor * entry.chance(now) * (1 << 30)) - return entry; - - factor *= 1.2; - } -}; - -/** - * Get fresh bucket for host. - * @private - * @param {HostEntry} entry - * @returns {MapBucket} - */ - -HostList.prototype.freshBucket = function freshBucket(entry) { - var size = 0; - var bw, hash, index; - - size += entry.addr.host.length; - size += entry.src.host.length; - - bw = new StaticWriter(size); - bw.writeString(entry.addr.host, 'ascii'); - bw.writeString(entry.src.host, 'ascii'); - - hash = murmur3(bw.render(), 0xfba4c795); - index = hash % this.fresh.length; - - return this.fresh[index]; -}; - -/** - * Get used bucket for host. - * @private - * @param {HostEntry} entry - * @returns {List} - */ - -HostList.prototype.usedBucket = function usedBucket(entry) { - var data = new Buffer(entry.addr.host, 'ascii'); - var hash = murmur3(data, 0xfba4c795); - var index = hash % this.used.length; - return this.used[index]; -}; - -/** - * Add host to host list. - * @param {NetAddress} addr - * @param {NetAddress?} src - * @returns {Boolean} - */ - -HostList.prototype.add = function add(addr, src) { - var now = this.network.now(); - var penalty = 2 * 60 * 60; - var interval = 24 * 60 * 60; - var factor = 1; - var i, entry, bucket; - - if (this.isFull()) - return false; - - entry = this.map[addr.hostname]; - - if (entry) { - // No source means we're inserting - // this ourselves. No penalty. - if (!src) - penalty = 0; - - // Update services. - entry.addr.services |= addr.services; - - // Online? - if (now - addr.ts < 24 * 60 * 60) - interval = 60 * 60; - - // Periodically update time. - if (entry.addr.ts < addr.ts - interval - penalty) - entry.addr.ts = addr.ts; - - // Do not update if no new - // information is present. - if (entry.addr.ts && addr.ts <= entry.addr.ts) - return false; - - // Do not update if the entry was - // already in the "used" table. - if (entry.used) - return false; - - assert(entry.refCount > 0); - - // Do not update if the max - // reference count is reached. - if (entry.refCount === this.maxRefs) - return false; - - assert(entry.refCount < this.maxRefs); - - // Stochastic test: previous refCount - // N: 2^N times harder to increase it. - for (i = 0; i < entry.refCount; i++) - factor *= 2; - - if (util.random(0, factor) !== 0) - return false; - } else { - if (!src) - src = this.address; - - entry = new HostEntry(addr, src); - - this.totalFresh++; - } - - bucket = this.freshBucket(entry); - - if (bucket.has(entry.key())) - return false; - - if (bucket.size >= this.maxEntries) - this.evictFresh(bucket); - - bucket.set(entry.key(), entry); - entry.refCount++; - - this.map[entry.key()] = entry; - - return true; -}; - -/** - * Evict a host from fresh bucket. - * @param {MapBucket} bucket - */ - -HostList.prototype.evictFresh = function evictFresh(bucket) { - var keys = bucket.keys(); - var i, key, entry, old; - - for (i = 0; i < keys.length; i++) { - key = keys[i]; - entry = bucket.get(key); - - if (this.isStale(entry)) { - bucket.remove(entry.key()); - - if (--entry.refCount === 0) { - delete this.map[entry.key()]; - this.totalFresh--; - } - - continue; - } - - if (!old) { - old = entry; - continue; - } - - if (entry.addr.ts < old.addr.ts) - old = entry; - } - - if (!old) - return; - - bucket.remove(old.key()); - - if (--old.refCount === 0) { - delete this.map[old.key()]; - this.totalFresh--; - } -}; - -/** - * Test whether a host is evictable. - * @param {HostEntry} entry - * @returns {Boolean} - */ - -HostList.prototype.isStale = function isStale(entry) { - var now = this.network.now(); - - if (entry.lastAttempt && entry.lastAttempt >= now - 60) - return false; - - if (entry.addr.ts > now + 10 * 60) - return true; - - if (entry.addr.ts === 0) - return true; - - if (now - entry.addr.ts > this.horizonDays * 24 * 60 * 60) - return true; - - if (entry.lastSuccess === 0 && entry.attempts >= this.retries) - return true; - - if (now - entry.lastSuccess > this.minFailDays * 24 * 60 * 60) { - if (entry.attempts >= this.maxFailures) - return true; - } - - return false; -}; - -/** - * Remove host from host list. - * @param {String} hostname - * @returns {NetAddress} - */ - -HostList.prototype.remove = function remove(hostname) { - var entry = this.map[hostname]; - var i, head, bucket; - - if (!entry) - return; - - if (entry.used) { - assert(entry.refCount === 0); - - head = entry; - while (head.prev) - head = head.prev; - - for (i = 0; i < this.used.length; i++) { - bucket = this.used[i]; - if (bucket.head === head) { - bucket.remove(entry); - this.totalUsed--; - break; - } - } - - assert(i < this.used.length); - } else { - for (i = 0; i < this.fresh.length; i++) { - bucket = this.fresh[i]; - if (bucket.remove(entry.key())) - entry.refCount--; - } - - this.totalFresh--; - assert(entry.refCount === 0); - } - - delete this.map[entry.key()]; - - return entry.addr; -}; - -/** - * Mark host as failed. - * @param {String} hostname - */ - -HostList.prototype.markAttempt = function markAttempt(hostname) { - var entry = this.map[hostname]; - var now = this.network.now(); - - if (!entry) - return; - - entry.attempts++; - entry.lastAttempt = now; -}; - -/** - * Mark host as successfully connected. - * @param {String} hostname - */ - -HostList.prototype.markSuccess = function markSuccess(hostname) { - var entry = this.map[hostname]; - var now = this.network.now(); - - if (!entry) - return; - - if (now - entry.addr.ts > 20 * 60) - entry.addr.ts = now; -}; - -/** - * Mark host as successfully connected. - * @param {String} hostname - * @param {Number} services - */ - -HostList.prototype.markAck = function markAck(hostname, services) { - var entry = this.map[hostname]; - var now = this.network.now(); - var i, bucket, evicted, old, fresh; - - if (!entry) - return; - - entry.addr.services |= services; - entry.lastSuccess = now; - entry.lastAttempt = now; - entry.attempts = 0; - - if (entry.used) - return; - - assert(entry.refCount > 0); - - // Remove from fresh. - for (i = 0; i < this.fresh.length; i++) { - bucket = this.fresh[i]; - if (bucket.remove(entry.key())) { - entry.refCount--; - old = bucket; - } - } - - assert(old); - assert(entry.refCount === 0); - this.totalFresh--; - - // Find room in used bucket. - bucket = this.usedBucket(entry); - - if (bucket.size < this.maxEntries) { - entry.used = true; - bucket.push(entry); - this.totalUsed++; - return; - } - - // No room. Evict. - evicted = this.evictUsed(bucket); - fresh = this.freshBucket(evicted); - - // Move to entry's old bucket if no room. - if (fresh.size >= this.maxEntries) - fresh = old; - - // Swap to evicted's used bucket. - entry.used = true; - bucket.replace(evicted, entry); - - // Move evicted to fresh bucket. - evicted.used = false; - fresh.set(evicted.key(), evicted); - assert(evicted.refCount === 0); - evicted.refCount++; - this.totalFresh++; -}; - -/** - * Pick used for eviction. - * @param {List} bucket - */ - -HostList.prototype.evictUsed = function evictUsed(bucket) { - var old = bucket.head; - var entry; - - for (entry = bucket.head; entry; entry = entry.next) { - if (entry.addr.ts < old.addr.ts) - old = entry; - } - - return old; -}; - -/** - * Convert address list to array. - * @returns {NetAddress[]} - */ - -HostList.prototype.toArray = function toArray() { - var keys = Object.keys(this.map); - var out = []; - var i, key, entry; - - for (i = 0; i < keys.length; i++) { - key = keys[i]; - entry = this.map[key]; - out.push(entry.addr); - } - - assert.equal(out.length, this.size()); - - return out; -}; - -/** - * Set initial seeds. - * @param {String[]} seeds - */ - -HostList.prototype.setSeeds = function setSeeds(seeds) { - var i, seed; - - this.seeds.length = 0; - - for (i = 0; i < seeds.length; i++) { - seed = seeds[i]; - this.addSeed(seed); - } -}; - -/** - * Add a preferred seed. - * @param {String} hostname - */ - -HostList.prototype.addSeed = function addSeed(host) { - var addr = IP.parseHost(host, this.network.port); - return this.seeds.push(addr); -}; - -/** - * Discover hosts from seeds. - * @returns {Promise} - */ - -HostList.prototype.discover = co(function* discover() { - var jobs = []; - var i, seed; - - for (i = 0; i < this.seeds.length; i++) { - seed = this.seeds[i]; - jobs.push(this.populate(seed)); - } - - yield Promise.all(jobs); -}); - -/** - * Populate from seed. - * @param {Object} seed - * @returns {Promise} - */ - -HostList.prototype.populate = co(function* populate(seed) { - var i, addr, hosts, host; - - if (seed.version !== -1) { - addr = NetAddress.fromHost(seed.host, seed.port, this.network); - this.add(addr); - return; - } - - this.logger.info('Resolving hosts from seed: %s.', seed.host); - - try { - hosts = yield this.resolve(seed.host, this.proxyServer); - } catch (e) { - this.logger.error(e); - return; - } - - for (i = 0; i < hosts.length; i++) { - host = hosts[i]; - addr = NetAddress.fromHost(host, seed.port, this.network); - this.add(addr); - } -}); - -/** - * Convert host list to json-friendly object. - * @returns {Object} - */ - -HostList.prototype.toJSON = function toJSON() { - var addrs = []; - var fresh = []; - var used = []; - var i, keys, key, bucket, entry; - - keys = Object.keys(this.map); - - for (i = 0; i < keys.length; i++) { - key = keys[i]; - entry = this.map[key]; - addrs.push(entry.toJSON()); - } - - for (i = 0; i < this.fresh.length; i++) { - bucket = this.fresh[i]; - keys = bucket.keys(); - fresh.push(keys); - } - - for (i = 0; i < this.used.length; i++) { - bucket = this.used[i]; - keys = []; - for (entry = bucket.head; entry; entry = entry.next) - keys.push(entry.key()); - used.push(keys); - } - - return { - version: 1, - addrs: addrs, - fresh: fresh, - used: used - }; -}; - -/** - * Inject properties from json object. - * @private - * @param {Object} json - * @returns {HostList} - */ - -HostList.prototype.fromJSON = function fromJSON(json) { - var sources = {}; - var i, j, bucket, keys, key, addr, entry, src; - - assert(json && typeof json === 'object'); - assert(json.version === 1, 'Bad address serialization version.'); - - assert(Array.isArray(json.addrs)); - - for (i = 0; i < json.addrs.length; i++) { - addr = json.addrs[i]; - entry = HostEntry.fromJSON(addr, this.network); - src = sources[entry.src.hostname]; - - // Save some memory. - if (!src) { - src = entry.src; - sources[src.hostname] = src; - } - - entry.src = src; - - this.map[entry.key()] = entry; - } - - assert(Array.isArray(json.fresh)); - - for (i = 0; i < json.fresh.length; i++) { - keys = json.fresh[i]; - bucket = this.fresh[i]; - assert(bucket, 'No bucket available.'); - for (j = 0; j < keys.length; j++) { - key = keys[j]; - entry = this.map[key]; - assert(entry); - if (entry.refCount === 0) - this.totalFresh++; - entry.refCount++; - bucket.set(key, entry); - } - } - - assert(Array.isArray(json.used)); - - for (i = 0; i < json.used.length; i++) { - keys = json.used[i]; - bucket = this.used[i]; - assert(bucket, 'No bucket available.'); - for (j = 0; j < keys.length; j++) { - key = keys[j]; - entry = this.map[key]; - assert(entry); - assert(entry.refCount === 0); - assert(!entry.used); - entry.used = true; - this.totalUsed++; - bucket.push(entry); - } - } - - keys = Object.keys(this.map); - - for (i = 0; i < keys.length; i++) { - key = keys[i]; - entry = this.map[key]; - assert(entry.used || entry.refCount > 0); - } - - return this; -}; - -/** - * Instantiate host list from json object. - * @param {Object} options - * @param {Object} json - * @returns {HostList} - */ - -HostList.fromJSON = function fromJSON(options, json) { - return new HostEntry(options).fromJSON(json); -}; - -/** - * MapBucket - * @constructor - */ - -function MapBucket() { - this.map = {}; - this.size = 0; -} - -/** - * Get map keys. - * @returns {String[]} - */ - -MapBucket.prototype.keys = function keys() { - return Object.keys(this.map); -}; - -/** - * Get item from map. - * @param {String} key - * @returns {Object|null} - */ - -MapBucket.prototype.get = function get(key) { - return this.map[key]; -}; - -/** - * Test whether map has an item. - * @param {String} key - * @returns {Boolean} - */ - -MapBucket.prototype.has = function has(key) { - return this.map[key] !== undefined; -}; - -/** - * Set a key to value in map. - * @param {String} key - * @param {Object} value - * @returns {Boolean} - */ - -MapBucket.prototype.set = function set(key, value) { - var item = this.map[key]; - - assert(value !== undefined); - - this.map[key] = value; - - if (item === undefined) { - this.size++; - return true; - } - - return false; -}; - -/** - * Remove an item from map. - * @param {String} key - * @returns {Object|null} - */ - -MapBucket.prototype.remove = function remove(key) { - var item = this.map[key]; - - if (item === undefined) - return; - - delete this.map[key]; - this.size--; - - return item; -}; - -/** - * Reset the map. - */ - -MapBucket.prototype.reset = function reset() { - this.map = {}; - this.size = 0; -}; - -/** - * HostEntry - * @constructor - * @param {NetAddress} addr - * @param {NetAddress} src - */ - -function HostEntry(addr, src) { - if (!(this instanceof HostEntry)) - return new HostEntry(addr, src); - - this.addr = addr || new NetAddress(); - this.src = src || new NetAddress(); - this.prev = null; - this.next = null; - this.used = false; - this.refCount = 0; - this.attempts = 0; - this.lastSuccess = 0; - this.lastAttempt = 0; - - if (addr) - this.fromOptions(addr, src); -} - -/** - * Inject properties from options. - * @private - * @param {NetAddress} addr - * @param {NetAddress} src - * @returns {HostEntry} - */ - -HostEntry.prototype.fromOptions = function fromOptions(addr, src) { - assert(addr instanceof NetAddress); - assert(src instanceof NetAddress); - this.addr = addr; - this.src = src; - return this; -}; - -/** - * Instantiate host entry from options. - * @param {NetAddress} addr - * @param {NetAddress} src - * @returns {HostEntry} - */ - -HostEntry.fromOptions = function fromOptions(addr, src) { - return new HostEntry().fromOptions(addr, src); -}; - -/** - * Get key suitable for a hash table (hostname). - * @returns {String} - */ - -HostEntry.prototype.key = function key() { - return this.addr.hostname; -}; - -/** - * Get host priority. - * @param {Number} now - * @returns {Number} - */ - -HostEntry.prototype.chance = function _chance(now) { - var attempts = this.attempts; - var chance = 1; - - if (now - this.lastAttempt < 60 * 10) - chance *= 0.01; - - chance *= Math.pow(0.66, Math.min(attempts, 8)); - - return chance; -}; - -/** - * Inspect host address. - * @returns {Object} - */ - -HostEntry.prototype.inspect = function inspect() { - return { - addr: this.addr, - src: this.src, - used: this.used, - refCount: this.refCount, - attempts: this.attempts, - lastSuccess: util.date(this.lastSuccess), - lastAttempt: util.date(this.lastAttempt) - }; -}; - -/** - * Convert host entry to json-friendly object. - * @returns {Object} - */ - -HostEntry.prototype.toJSON = function toJSON() { - return { - addr: this.addr.hostname, - src: this.src.hostname, - services: this.addr.services.toString(2), - ts: this.addr.ts, - attempts: this.attempts, - lastSuccess: this.lastSuccess, - lastAttempt: this.lastAttempt - }; -}; - -/** - * Inject properties from json object. - * @private - * @param {Object} json - * @param {Network} network - * @returns {HostEntry} - */ - -HostEntry.prototype.fromJSON = function fromJSON(json, network) { - assert(json && typeof json === 'object'); - assert(typeof json.addr === 'string'); - assert(typeof json.src === 'string'); - - this.addr.fromHostname(json.addr, network); - - if (json.services != null) { - assert(typeof json.services === 'string'); - assert(json.services.length > 0); - assert(json.services.length < 64); - this.addr.services = parseInt(json.services, 2); - } - - if (json.ts != null) { - assert(util.isNumber(json.ts)); - this.addr.ts = json.ts; - } - - if (json.src != null) { - assert(typeof json.src === 'string'); - this.src.fromHostname(json.src, network); - } - - if (json.attempts != null) { - assert(util.isNumber(json.attempts)); - this.attempts = json.attempts; - } - - if (json.lastSuccess != null) { - assert(util.isNumber(json.lastSuccess)); - this.lastSuccess = json.lastSuccess; - } - - if (json.lastAttempt != null) { - assert(util.isNumber(json.lastAttempt)); - this.lastAttempt = json.lastAttempt; - } - - return this; -}; - -/** - * Instantiate host entry from json object. - * @param {Object} json - * @param {Network} network - * @returns {HostEntry} - */ - -HostEntry.fromJSON = function fromJSON(json, network) { - return new HostEntry().fromJSON(json, network); -}; - /** * Represents an in-flight block or transaction. * @exports LoadRequest