From da69c5d88817a9a4c70b0545c7eed98fcb8d38f6 Mon Sep 17 00:00:00 2001 From: Christopher Jeffrey Date: Wed, 1 Mar 2017 11:27:44 -0800 Subject: [PATCH] hostlist: persist to file. --- lib/net/hostlist.js | 201 +++++++++++++++++++++++++++++++++++++++++--- lib/net/pool.js | 7 +- 2 files changed, 196 insertions(+), 12 deletions(-) diff --git a/lib/net/hostlist.js b/lib/net/hostlist.js index cfb0a87a..e8937503 100644 --- a/lib/net/hostlist.js +++ b/lib/net/hostlist.js @@ -17,6 +17,7 @@ var murmur3 = require('../utils/murmur3'); var Map = require('../utils/map'); var common = require('./common'); var dns = require('./dns'); +var fs = require('fs'); /** * Host List @@ -60,6 +61,10 @@ function HostList(options) { this.maxFailures = 10; this.maxRefs = 8; + this.filename = null; + this.flushInterval = 120000; + this.timer = null; + this._initOptions(options); this._init(); } @@ -107,6 +112,16 @@ HostList.prototype._initOptions = function initOptions(options) { assert(Array.isArray(options.nodes)); this.rawNodes = options.nodes; } + + if (options.hostLocation != null) { + assert(typeof options.hostLocation === 'string'); + this.filename = options.hostLocation; + } + + if (options.flushInterval != null) { + assert(options.flushInterval >= 0); + this.flushInterval = options.flushInterval; + } }; /** @@ -127,6 +142,114 @@ HostList.prototype._init = function init() { this.setNodes(this.rawNodes); }; +/** + * Open hostlist and read hosts file. + * @method + * @returns {Promise} + */ + +HostList.prototype.open = co(function* open() { + try { + yield this.read(); + } catch (e) { + if (this.logger) { + this.logger.warning('Hosts deserialization failed.'); + this.logger.error(e); + } + } + + this.start(); +}); + +/** + * Close hostlist. + * @method + * @returns {Promise} + */ + +HostList.prototype.close = co(function* close() { + this.stop(); + yield this.flush(); +}); + +/** + * Start flush interval. + */ + +HostList.prototype.start = function start() { + if (!this.filename) + return; + + assert(!this.timer); + this.timer = setInterval(this.flush.bind(this), this.flushInterval); +}; + +/** + * Stop flush interval. + */ + +HostList.prototype.stop = function stop() { + if (!this.filename) + return; + + assert(this.timer != null); + clearInterval(this.timer); + this.timer = null; +}; + +/** + * Read and initialie from hosts file. + * @method + * @returns {Promise} + */ + +HostList.prototype.read = co(function* read() { + var data, json; + + if (!this.filename) + return; + + try { + data = yield readFile(this.filename, 'utf8'); + } catch (e) { + if (e.code === 'ENOENT') + return; + throw e; + } + + json = JSON.parse(data); + + this.fromJSON(json); +}); + +/** + * Flush addrs to hosts file. + * @method + * @returns {Promise} + */ + +HostList.prototype.flush = co(function* flush() { + var json, data; + + if (!this.filename) + return; + + if (this.logger) + this.logger.debug('Writing hosts to %s.', this.filename); + + json = this.toJSON(); + data = JSON.stringify(json); + + try { + yield writeFile(this.filename, data, 'utf8'); + } catch (e) { + if (this.logger) { + this.logger.warning('Writing hosts failed.'); + this.logger.error(e); + } + } +}); + /** * Get list size. * @returns {Number} @@ -310,6 +433,8 @@ HostList.prototype.add = function add(addr, src) { var factor = 1; var i, entry, bucket; + assert(addr.port !== 0); + entry = this.map[addr.hostname]; if (entry) { @@ -863,6 +988,11 @@ HostList.prototype.toJSON = function toJSON() { HostList.prototype.fromJSON = function fromJSON(json) { var sources = {}; + var map = {}; + var fresh = []; + var totalFresh = 0; + var used = []; + var totalUsed = 0; var i, j, bucket, keys, key, addr, entry, src; assert(json && typeof json === 'object'); @@ -883,52 +1013,74 @@ HostList.prototype.fromJSON = function fromJSON(json) { entry.src = src; - this.map[entry.key()] = entry; + map[entry.key()] = entry; } assert(Array.isArray(json.fresh)); for (i = 0; i < json.fresh.length; i++) { keys = json.fresh[i]; - bucket = this.fresh[i]; - assert(bucket, 'No bucket available.'); + bucket = new Map(); + for (j = 0; j < keys.length; j++) { key = keys[j]; - entry = this.map[key]; + entry = map[key]; assert(entry); if (entry.refCount === 0) - this.totalFresh++; + totalFresh++; entry.refCount++; bucket.set(key, entry); } + + assert(bucket.size <= this.maxEntries, + 'Bucket size mismatch.'); + + fresh.push(bucket); } + assert(fresh.length === this.fresh.length, + 'Buckets mismatch.'); + assert(Array.isArray(json.used)); for (i = 0; i < json.used.length; i++) { keys = json.used[i]; - bucket = this.used[i]; - assert(bucket, 'No bucket available.'); + bucket = new List(); + for (j = 0; j < keys.length; j++) { key = keys[j]; - entry = this.map[key]; + entry = map[key]; assert(entry); assert(entry.refCount === 0); assert(!entry.used); entry.used = true; - this.totalUsed++; + totalUsed++; bucket.push(entry); } + + assert(bucket.size <= this.maxEntries, + 'Bucket size mismatch.'); + + used.push(bucket); } - keys = Object.keys(this.map); + assert(used.length === this.used.length, + 'Buckets mismatch.'); + + keys = Object.keys(map); for (i = 0; i < keys.length; i++) { key = keys[i]; - entry = this.map[key]; + entry = map[key]; assert(entry.used || entry.refCount > 0); } + this.map = map; + this.fresh = fresh; + this.totalFresh = totalFresh; + this.used = used; + this.totalUsed = totalUsed; + return this; }; @@ -1123,6 +1275,33 @@ HostEntry.fromJSON = function fromJSON(json, network) { return new HostEntry().fromJSON(json, network); }; +/* + * Helpers + */ + +function readFile(filename, enc) { + return new Promise(function(resolve, reject) { + var err; + + if (fs.unsupported) { + err = new Error('File not found.'); + err.code = 'ENOENT'; + return reject(err); + } + + fs.readFile(filename, enc, co.wrap(resolve, reject)); + }); +} + +function writeFile(filename, data, enc) { + return new Promise(function(resolve, reject) { + if (fs.unsupported) + return resolve(); + + fs.writeFile(filename, data, enc, co.wrap(resolve, reject)); + }); +} + /* * Expose */ diff --git a/lib/net/pool.js b/lib/net/pool.js index 942b47eb..c044f539 100644 --- a/lib/net/pool.js +++ b/lib/net/pool.js @@ -212,6 +212,8 @@ Pool.prototype._open = co(function* _open() { else yield this.chain.open(); + yield this.hosts.open(); + this.logger.info('Pool loaded (maxpeers=%d).', this.options.maxOutbound); if (this.options.bip150) { @@ -259,7 +261,7 @@ Pool.prototype.resetChain = function resetChain() { Pool.prototype._close = co(function* close() { yield this.disconnect(); - this.hosts.reset(); + yield this.hosts.close(); }); /** @@ -1328,6 +1330,9 @@ Pool.prototype.handleAddr = co(function* handleAddr(peer, packet) { if (addr.ts <= 100000000 || addr.ts > now + 10 * 60) addr.ts = now - 5 * 24 * 60 * 60; + if (addr.port === 0) + continue; + this.hosts.add(addr, peer.address); }