hostlist: persist to file.

lib/net/hostlist.js

@@ -17,6 +17,7 @@ var murmur3 = require('../utils/murmur3');
 var Map = require('../utils/map');
 var common = require('./common');
 var dns = require('./dns');
+var fs = require('fs');
 
 /**
  * Host List
@@ -60,6 +61,10 @@ function HostList(options) {
   this.maxFailures = 10;
   this.maxRefs = 8;
 
+  this.filename = null;
+  this.flushInterval = 120000;
+  this.timer = null;
+
   this._initOptions(options);
   this._init();
 }
@@ -107,6 +112,16 @@ HostList.prototype._initOptions = function initOptions(options) {
     assert(Array.isArray(options.nodes));
     this.rawNodes = options.nodes;
   }
+
+  if (options.hostLocation != null) {
+    assert(typeof options.hostLocation === 'string');
+    this.filename = options.hostLocation;
+  }
+
+  if (options.flushInterval != null) {
+    assert(options.flushInterval >= 0);
+    this.flushInterval = options.flushInterval;
+  }
 };
 
 /**
@@ -127,6 +142,114 @@ HostList.prototype._init = function init() {
   this.setNodes(this.rawNodes);
 };
 
+/**
+ * Open hostlist and read hosts file.
+ * @method
+ * @returns {Promise}
+ */
+
+HostList.prototype.open = co(function* open() {
+  try {
+    yield this.read();
+  } catch (e) {
+    if (this.logger) {
+      this.logger.warning('Hosts deserialization failed.');
+      this.logger.error(e);
+    }
+  }
+
+  this.start();
+});
+
+/**
+ * Close hostlist.
+ * @method
+ * @returns {Promise}
+ */
+
+HostList.prototype.close = co(function* close() {
+  this.stop();
+  yield this.flush();
+});
+
+/**
+ * Start flush interval.
+ */
+
+HostList.prototype.start = function start() {
+  if (!this.filename)
+    return;
+
+  assert(!this.timer);
+  this.timer = setInterval(this.flush.bind(this), this.flushInterval);
+};
+
+/**
+ * Stop flush interval.
+ */
+
+HostList.prototype.stop = function stop() {
+  if (!this.filename)
+    return;
+
+  assert(this.timer != null);
+  clearInterval(this.timer);
+  this.timer = null;
+};
+
+/**
+ * Read and initialie from hosts file.
+ * @method
+ * @returns {Promise}
+ */
+
+HostList.prototype.read = co(function* read() {
+  var data, json;
+
+  if (!this.filename)
+    return;
+
+  try {
+    data = yield readFile(this.filename, 'utf8');
+  } catch (e) {
+    if (e.code === 'ENOENT')
+      return;
+    throw e;
+  }
+
+  json = JSON.parse(data);
+
+  this.fromJSON(json);
+});
+
+/**
+ * Flush addrs to hosts file.
+ * @method
+ * @returns {Promise}
+ */
+
+HostList.prototype.flush = co(function* flush() {
+  var json, data;
+
+  if (!this.filename)
+    return;
+
+  if (this.logger)
+    this.logger.debug('Writing hosts to %s.', this.filename);
+
+  json = this.toJSON();
+  data = JSON.stringify(json);
+
+  try {
+    yield writeFile(this.filename, data, 'utf8');
+  } catch (e) {
+    if (this.logger) {
+      this.logger.warning('Writing hosts failed.');
+      this.logger.error(e);
+    }
+  }
+});
+
 /**
  * Get list size.
  * @returns {Number}
@@ -310,6 +433,8 @@ HostList.prototype.add = function add(addr, src) {
   var factor = 1;
   var i, entry, bucket;
 
+  assert(addr.port !== 0);
+
   entry = this.map[addr.hostname];
 
   if (entry) {
@@ -863,6 +988,11 @@ HostList.prototype.toJSON = function toJSON() {
 
 HostList.prototype.fromJSON = function fromJSON(json) {
   var sources = {};
+  var map = {};
+  var fresh = [];
+  var totalFresh = 0;
+  var used = [];
+  var totalUsed = 0;
   var i, j, bucket, keys, key, addr, entry, src;
 
   assert(json && typeof json === 'object');
@@ -883,52 +1013,74 @@ HostList.prototype.fromJSON = function fromJSON(json) {
 
     entry.src = src;
 
-    this.map[entry.key()] = entry;
+    map[entry.key()] = entry;
   }
 
   assert(Array.isArray(json.fresh));
 
   for (i = 0; i < json.fresh.length; i++) {
     keys = json.fresh[i];
-    bucket = this.fresh[i];
+    bucket = new Map();
-    assert(bucket, 'No bucket available.');
+
     for (j = 0; j < keys.length; j++) {
       key = keys[j];
-      entry = this.map[key];
+      entry = map[key];
       assert(entry);
       if (entry.refCount === 0)
-        this.totalFresh++;
+        totalFresh++;
       entry.refCount++;
       bucket.set(key, entry);
     }
+
+    assert(bucket.size <= this.maxEntries,
+      'Bucket size mismatch.');
+
+    fresh.push(bucket);
   }
 
+  assert(fresh.length === this.fresh.length,
+    'Buckets mismatch.');
+
   assert(Array.isArray(json.used));
 
   for (i = 0; i < json.used.length; i++) {
     keys = json.used[i];
-    bucket = this.used[i];
+    bucket = new List();
-    assert(bucket, 'No bucket available.');
+
     for (j = 0; j < keys.length; j++) {
       key = keys[j];
-      entry = this.map[key];
+      entry = map[key];
       assert(entry);
       assert(entry.refCount === 0);
       assert(!entry.used);
       entry.used = true;
-      this.totalUsed++;
+      totalUsed++;
       bucket.push(entry);
     }
+
+    assert(bucket.size <= this.maxEntries,
+      'Bucket size mismatch.');
+
+    used.push(bucket);
   }
 
-  keys = Object.keys(this.map);
+  assert(used.length === this.used.length,
+    'Buckets mismatch.');
+
+  keys = Object.keys(map);
 
   for (i = 0; i < keys.length; i++) {
     key = keys[i];
-    entry = this.map[key];
+    entry = map[key];
     assert(entry.used || entry.refCount > 0);
   }
 
+  this.map = map;
+  this.fresh = fresh;
+  this.totalFresh = totalFresh;
+  this.used = used;
+  this.totalUsed = totalUsed;
+
   return this;
 };
 
@@ -1123,6 +1275,33 @@ HostEntry.fromJSON = function fromJSON(json, network) {
   return new HostEntry().fromJSON(json, network);
 };
 
+/*
+ * Helpers
+ */
+
+function readFile(filename, enc) {
+  return new Promise(function(resolve, reject) {
+    var err;
+
+    if (fs.unsupported) {
+      err = new Error('File not found.');
+      err.code = 'ENOENT';
+      return reject(err);
+    }
+
+    fs.readFile(filename, enc, co.wrap(resolve, reject));
+  });
+}
+
+function writeFile(filename, data, enc) {
+  return new Promise(function(resolve, reject) {
+    if (fs.unsupported)
+      return resolve();
+
+    fs.writeFile(filename, data, enc, co.wrap(resolve, reject));
+  });
+}
+
 /*
  * Expose
  */

lib/net/pool.js

@@ -212,6 +212,8 @@ Pool.prototype._open = co(function* _open() {
   else
     yield this.chain.open();
 
+  yield this.hosts.open();
+
   this.logger.info('Pool loaded (maxpeers=%d).', this.options.maxOutbound);
 
   if (this.options.bip150) {
@@ -259,7 +261,7 @@ Pool.prototype.resetChain = function resetChain() {
 
 Pool.prototype._close = co(function* close() {
   yield this.disconnect();
-  this.hosts.reset();
+  yield this.hosts.close();
 });
 
 /**
@@ -1328,6 +1330,9 @@ Pool.prototype.handleAddr = co(function* handleAddr(peer, packet) {
     if (addr.ts <= 100000000 || addr.ts > now + 10 * 60)
       addr.ts = now - 5 * 24 * 60 * 60;
 
+    if (addr.port === 0)
+      continue;
+
     this.hosts.add(addr, peer.address);
   }