chain: better preload generation algorithm

lib/bcoin/chain.js

@@ -28,8 +28,8 @@ function Chain(options) {
     count: 0
   };
   this.index = {
-    bloom: null,
     initialSize: 0,
+    bloom: null,
     hashes: [],
     ts: []
   };
@@ -44,6 +44,17 @@ function compareTs(a, b) {
   return a -b;
 }
 
+Chain.prototype._getRange = function _getRange(ts) {
+  if (this.index.ts[this.index.ts.length - 1] < ts)
+    ts = this.index.ts[this.index.ts.length - 1];
+
+  var start = utils.binaryInsert(this.index.ts, ts - 2 * 3600, compareTs, true);
+  start = Math.max(0, start - 2);
+  var end = utils.binaryInsert(this.index.ts, ts + 2 * 3600, compareTs, true);
+
+  return { start: start, end: end };
+};
+
 Chain.prototype.probeIndex = function probeIndex(hash, ts) {
   if (!this.index.bloom.test(hash, 'hex'))
     return false;
@@ -51,9 +62,9 @@ Chain.prototype.probeIndex = function probeIndex(hash, ts) {
   var start = 0;
   var end = this.index.ts.length;
   if (ts) {
-    start = utils.binaryInsert(this.index.ts, ts - 2 * 3600, compareTs, true);
-    start = Math.max(0, start - 1);
-    end = utils.binaryInsert(this.index.ts, ts + 2 * 3600, compareTs, true);
+    var range = this._getRange(ts);
+    start = range.start;
+    end = range.end;
   }
 
   for (var i = start; i < end; i++)
@@ -77,15 +88,10 @@ Chain.prototype.addIndex = function addIndex(hash, ts) {
 };
 
 Chain.prototype.getRange = function getRange(ts) {
-  var start = utils.binaryInsert(this.index.ts, ts - 2 * 3600, compareTs, true);
-  var end = utils.binaryInsert(this.index.ts, ts + 2 * 3600, compareTs, true);
-
-  if (start > 0)
-    start--;
-  if (end > 0)
-    end--;
-
-  return { start: this.index.hashes[start], end: this.index.hashes[end] };
+  var range = this._getRange(ts);
+  if (range.end > 0)
+    range.end--;
+  return range;
 };
 
 Chain.prototype.add = function add(block) {
@@ -209,7 +215,7 @@ Chain.prototype.hashesInRange = function hashesInRange(start, end) {
   var ts = this.index.ts;
 
   var pos = utils.binaryInsert(ts, start - 2 * 3600, compareTs, true);
-  start = Math.max(0, pos - 1);
+  start = Math.max(0, pos - 2);
   var pos = utils.binaryInsert(ts, end + 2 * 3600, compareTs, true);
   end = pos;
   return this.index.hashes.slice(start, end);
@@ -220,21 +226,38 @@ Chain.prototype.getLast = function getLast() {
 };
 
 Chain.prototype.toJSON = function toJSON() {
-  var r = this.index.hashes.length % 50;
-  var keep = 1000 + r;
+  var keep = 1000;
 
-  // Keep only last 1000 consequent blocks, use every 50th for older
+  // Keep only last 1000 consequent blocks, dilate others at:
+  // 7 day range for blocks before 2013
+  // 12 hour for blocks before 2014
+  // 6 hour for blocks in 2014 and after it
+  // (or at maximum 250 block range)
   var last = {
     hashes: this.index.hashes.slice(-keep),
     ts: this.index.ts.slice(-keep)
   };
 
+  var start = Math.max(0, this.index.initialSize - keep);
   var first = {
-    hashes: this.index.hashes.slice(0, this.index.initialSize - 1000),
-    ts: this.index.ts.slice(0, this.index.initialSize - 1000)
+    hashes: this.index.hashes.slice(0, start),
+    ts: this.index.ts.slice(0, start)
   };
-  var len = this.index.hashes.length - keep;
-  for (var i = this.index.initialSize - 1000; i < len; i += 50) {
+  var lastTs = this.index.ts[start] || 0;
+  var lastI = start;
+  var delta1 = 7 * 24 * 3600;
+  var delta2 = 12 * 3600;
+  var delta3 = 6 * 3600;
+
+  for (var i = this.index.initialSize; i < this.index.ts.length - keep; i++) {
+    var ts = this.index.ts[i];
+    var delta = ts < 1356984000 ? delta1 :
+                ts < 1388520000 ? delta2 : delta3;
+    if (ts - lastTs < delta && i - lastI < 250)
+      continue;
+
+    lastTs = ts;
+    lastI = i;
     first.hashes.push(this.index.hashes[i]);
     first.ts.push(this.index.ts[i]);
   }

lib/bcoin/peer.js

@@ -318,7 +318,6 @@ Peer.prototype._handleInv = function handleInv(items) {
   this.getData(txs);
 };
 
-Peer.prototype.loadBlocks = function loadBlocks(hash, stop) {
-  this._write(this.framer.getBlocks(Array.isArray(hash) ? hash : [ hash ],
-                                    stop));
+Peer.prototype.loadBlocks = function loadBlocks(hashes, stop) {
+  this._write(this.framer.getBlocks(hashes, stop));
 };

lib/bcoin/pool.js

@@ -16,7 +16,7 @@ function Pool(options) {
   this.destroyed = false;
   this.size = options.size || 32;
   this.parallel = options.parallel || 2000;
-  this.redundancy = 2;
+  this.redundancy = options.redundancy || 2;
   this.load = {
     timeout: options.loadTimeout || 5000,
     interval: options.loadInterval || 5000,
@@ -26,8 +26,8 @@ function Pool(options) {
     hwm: options.hwm || this.parallel * 8,
     hiReached: false
   };
-  this.maxRetries = options.maxRetries || 10;
-  this.requestTimeout = options.requestTimeout || 5000;
+  this.maxRetries = options.maxRetries || 42;
+  this.requestTimeout = options.requestTimeout || 10000;
   this.chain = new bcoin.chain();
   this.watchMap = {};
   this.bloom = new bcoin.bloom(8 * 1024,
@@ -150,7 +150,7 @@ Pool.prototype._addLoader = function _addLoader() {
     self._scheduleRequests();
 
     // Store last hash to continue global load
-    self._lastHash = hashes[hashes.length - 1];
+    self.block.lastHash = hashes[hashes.length - 1];
 
     clearTimeout(timer);
 
@@ -170,7 +170,9 @@ Pool.prototype._loadRange = function _loadRange(range) {
 
   if (!this.peers.load)
     this._addLoader();
-  this.peers.load.loadBlocks(range.start, range.end);
+
+  var hashes = this.chain.hashesInRange(range.start, range.end);
+  this.peers.load.loadBlocks(hashes);
 };
 
 Pool.prototype._load = function _load() {
@@ -190,7 +192,7 @@ Pool.prototype._load = function _load() {
   if (!this.peers.load)
     this._addLoader();
   else
-    this.peers.load.loadBlocks(hash);
+    this.peers.load.loadBlocks([ hash ]);
 
   return true;
 };

scripts/update.js

@@ -15,8 +15,12 @@ var addrs = [
 ];
 
 var pool = bcoin.pool({
-  count: 16,
+  size: 32,
+  redundancy: 1,
+  parallel: 4000,
+  loadWindow: 750,
   createConnection: function() {
+    console.log('connecting...');
     return net.connect(8333, addrs[(Math.random() * addrs.length) | 0]);
   }
 });
@@ -27,7 +31,12 @@ var last = 0;
 pool.on('block', function(block) {
   if (block.ts <= last)
     return;
-  console.log('Got: ' + block.hash('hex') + ' ' + new Date(block.ts * 1000));
+  console.log('Got: %s from %s chain len %d act %d queue %d',
+              block.hash('hex'),
+              new Date(block.ts * 1000).toString(),
+              pool.chain.index.hashes.length,
+              pool.request.active,
+              pool.request.queue.length);
 });
 
 pool.once('full', finish);