chain: improvements

lib/bcoin/chain.js

@@ -14,6 +14,7 @@ function Chain(options) {
   EventEmitter.call(this);
 
   this.options = options || {};
+  this.strict = this.options.strict || false;
   this.block = {
     list: [],
 
@@ -28,10 +29,10 @@ function Chain(options) {
     count: 0
   };
   this.index = {
-    initialSize: 0,
     bloom: null,
     hashes: [],
-    ts: []
+    ts: [],
+    heights: []
   };
   this.request = new utils.RequestCache();
 
@@ -44,13 +45,23 @@ function compareTs(a, b) {
   return a -b;
 }
 
-Chain.prototype._getRange = function _getRange(ts) {
+Chain.prototype._getRange = function _getRange(hash, ts, futureOnly) {
-  if (this.index.ts[this.index.ts.length - 1] < ts)
+  var pos = utils.binaryInsert(this.index.ts, ts, compareTs, true);
-    ts = this.index.ts[this.index.ts.length - 1];
+  var start = Math.min(Math.max(0, pos), this.index.ts.length - 1);
 
-  var start = utils.binaryInsert(this.index.ts, ts - 2 * 3600, compareTs, true);
+  while (start > 0 && this.index.ts[start] > ts)
-  start = Math.max(0, start - 2);
+    start--;
-  var end = utils.binaryInsert(this.index.ts, ts + 2 * 3600, compareTs, true);
+
+  var curr = this.index.ts[start];
+  var wnd = 2 * 3600;
+
+  if (!futureOnly)
+    while (start > 0 && this.index.ts[start] + wnd > curr)
+      start--;
+
+  var end = Math.min(Math.max(0, pos), this.index.ts.length - 1);
+  while (end < this.index.ts.length - 1 && this.index.ts[end] - wnd < ts)
+    end++;
 
   return { start: start, end: end };
 };
@@ -58,16 +69,18 @@ Chain.prototype._getRange = function _getRange(ts) {
 Chain.prototype.probeIndex = function probeIndex(hash, ts) {
   if (!this.index.bloom.test(hash, 'hex'))
     return false;
+  if (!this.strict)
+    return true;
 
   var start = 0;
   var end = this.index.ts.length;
   if (ts) {
-    var range = this._getRange(ts);
+    var range = this._getRange(hash, ts);
     start = range.start;
     end = range.end;
   }
 
-  for (var i = start; i < end; i++)
+  for (var i = start; i <= end; i++)
     if (this.index.hashes[i] === hash)
       return true;
 
@@ -79,19 +92,17 @@ Chain.prototype.addIndex = function addIndex(hash, ts) {
     return;
 
   var pos = utils.binaryInsert(this.index.ts, ts, compareTs, true);
-  if (pos <= this.index.ts.length - 1000)
-    return;
 
-  this.index.ts.splice(pos, 0, ts);
+  // Avoid duplicates
-  this.index.hashes.splice(pos, 0, hash);
+  if (this.index.hashes[pos] !== hash &&
-  this.index.bloom.add(hash, 'hex');
+      this.index.hashes[pos - 1] !== hash &&
-};
+      this.index.hashes[pos + 1] !== hash) {
-
+    this.index.ts.splice(pos, 0, ts);
-Chain.prototype.getRange = function getRange(ts) {
+    this.index.hashes.splice(pos, 0, hash);
-  var range = this._getRange(ts);
+    this.index.bloom.add(hash, 'hex');
-  if (range.end > 0)
+    assert(pos > 0);
-    range.end--;
+    this.index.heights.splice(pos, 0, this.index.heights[pos - 1] + 1);
-  return range;
+  }
 };
 
 Chain.prototype.add = function add(block) {
@@ -114,7 +125,9 @@ Chain.prototype.add = function add(block) {
       this.orphan.count++;
       this.orphan.map[prev] = block;
 
-      this.emit('missing', prev, this.getRange(block.ts));
+      var range = this._getRange(hash, block.ts, true);
+      var hashes = this.index.hashes.slice(range.start, range.end + 1);
+      this.emit('missing', prev, hashes);
       break;
     }
 
@@ -164,13 +177,25 @@ Chain.prototype._bloomBlock = function _bloomBlock(block) {
     this.block.merkleBloom.add(block.hashes[i], 'hex');
 };
 
-Chain.prototype.has = function has(hash) {
+Chain.prototype.has = function has(hash, noProbe) {
-  return this.probeIndex(hash) || !!this.orphan.map[hash];
+  if (noProbe && this.block.bloom.test(hash, 'hex')) {
+    if (this.strict) {
+      for (var i = 0; i < this.block.list.length; i++)
+        if (this.block.list[i].hash('hex') === hash)
+          return true;
+    } else {
+      return true;
+    }
+  }
+
+  return !noProbe && this.probeIndex(hash) && !!this.orphan.map[hash];
 };
 
 Chain.prototype.hasMerkle = function hasMerkle(hash) {
   if (!this.block.merkleBloom.test(hash, 'hex'))
     return false;
+  if (!this.strict)
+    return true;
 
   hash = utils.toHex(hash);
   for (var i = 0; i < this.block.list.length; i++)
@@ -207,17 +232,14 @@ Chain.prototype.isFull = function isFull() {
          (+new Date / 1000) - this.index.ts[this.index.ts.length - 1] < 10 * 60;
 };
 
-Chain.prototype.covers = function covers(ts) {
-  return ts >= this.index.ts[0];
-};
-
 Chain.prototype.hashesInRange = function hashesInRange(start, end) {
   var ts = this.index.ts;
 
-  var pos = utils.binaryInsert(ts, start - 2 * 3600, compareTs, true);
+  start = utils.binaryInsert(ts, start, compareTs, true);
-  start = Math.max(0, pos - 2);
+  if (start > 0 && ts[start - 1] >= start)
-  var pos = utils.binaryInsert(ts, end + 2 * 3600, compareTs, true);
+    start--;
-  end = pos;
+  end = utils.binaryInsert(ts, end, compareTs, true);
+
   return this.index.hashes.slice(start, end);
 };
 
@@ -235,31 +257,35 @@ Chain.prototype.toJSON = function toJSON() {
   // (or at maximum 250 block range)
   var last = {
     hashes: this.index.hashes.slice(-keep),
-    ts: this.index.ts.slice(-keep)
+    ts: this.index.ts.slice(-keep),
+    heights: this.index.heights.slice(-keep)
   };
 
-  var start = Math.max(0, this.index.initialSize - keep);
   var first = {
-    hashes: this.index.hashes.slice(0, start),
+    hashes: [],
-    ts: this.index.ts.slice(0, start)
+    ts: [],
+    heights: []
   };
-  var lastTs = this.index.ts[start] || 0;
+
-  var lastI = start;
   var delta1 = 7 * 24 * 3600;
   var delta2 = 12 * 3600;
   var delta3 = 6 * 3600;
 
-  for (var i = this.index.initialSize; i < this.index.ts.length - keep; i++) {
+  var lastTs = 0;
+  var lastHeight = -1000;
+  for (var i = 0; i < this.index.ts.length - keep; i++) {
     var ts = this.index.ts[i];
     var delta = ts < 1356984000 ? delta1 :
                 ts < 1388520000 ? delta2 : delta3;
-    if (ts - lastTs < delta && i - lastI < 250)
+    var hdelta = this.index.heights[i] - lastHeight;
+    if (ts - lastTs < delta && hdelta < 250)
       continue;
 
     lastTs = ts;
-    lastI = i;
+    lastHeight = this.index.heights[i];
     first.hashes.push(this.index.hashes[i]);
     first.ts.push(this.index.ts[i]);
+    first.heights.push(this.index.heights[i]);
   }
 
   return {
@@ -267,19 +293,20 @@ Chain.prototype.toJSON = function toJSON() {
     type: 'chain',
     hashes: first.hashes.concat(last.hashes),
     ts: first.ts.concat(last.ts),
+    heights: first.heights.concat(last.heights)
   };
 };
 
 Chain.prototype.fromJSON = function fromJSON(json) {
   assert.equal(json.v, 1);
   assert.equal(json.type, 'chain');
-  this.index.initialSize = json.hashes.length;
   this.index.hashes = json.hashes.slice();
   this.index.ts = json.ts.slice();
+  this.index.heights = json.heights.slice();
   if (this.index.bloom)
     this.index.bloom.reset();
   else
-    this.index.bloom = new bcoin.bloom(28 * 1024 * 1024, 33, 0xdeadbee0);
+    this.index.bloom = new bcoin.bloom(28 * 1024 * 1024, 16, 0xdeadbee0);
 
   if (this.index.hashes.length === 0)
     this.add(new bcoin.block(constants.genesis));

lib/bcoin/pool.js

@@ -21,6 +21,8 @@ function Pool(options) {
     timeout: options.loadTimeout || 5000,
     interval: options.loadInterval || 5000,
     window: options.loadWindow || 250,
+    lastRange: null,
+    rangeWindow: options.rangeWindow || 1000,
     timer: null,
     lwm: options.lwm || this.parallel * 2,
     hwm: options.hwm || this.parallel * 8,
@@ -80,10 +82,10 @@ Pool.prototype._init = function _init() {
     this._addPeer();
 
   var self = this;
-  this.chain.on('missing', function(hash, range) {
+  this.chain.on('missing', function(hash, preload) {
     self._request('block', hash, { force: true });
     self._scheduleRequests();
-    self._loadRange(range);
+    self._loadRange(preload);
   });
 };
 
@@ -149,30 +151,43 @@ Pool.prototype._addLoader = function _addLoader() {
 
     self._scheduleRequests();
 
+    // The part of the response is in chain, no need to escalate requests
+    var allNew = hashes.every(function(hash) {
+      return !self.chain.has(utils.toHex(hash));
+    });
+    if (!allNew)
+      return;
+
     // Store last hash to continue global load
     self.block.lastHash = hashes[hashes.length - 1];
 
     clearTimeout(timer);
 
     // Reinstantiate timeout
-    if (self._load())
+    if (hashes.length === 500 && self._load())
       timer = setTimeout(destroy, self.load.timeout);
   });
 };
 
-Pool.prototype._loadRange = function _loadRange(range) {
+Pool.prototype._loadRange = function _loadRange(hashes) {
-  if (!range)
+  if (!hashes)
     return;
 
-  // We will be requesting block anyway
+  if (hashes.length <= 1)
-  if (range.start === range.end)
     return;
 
+  // Limit number of requests
+  var now = +new Date;
+  if (now - this.load.lastRange < this.load.rangeWindow)
+    return;
+  this.load.lastRange = now;
+
   if (!this.peers.load)
     this._addLoader();
 
-  var hashes = this.chain.hashesInRange(range.start, range.end);
+  var last = hashes[hashes.length - 1];
-  this.peers.load.loadBlocks(hashes);
+  function rev(s) { var r = ''; for (var i = 0; i < s.length; i += 2) r = s.slice(i, i + 2) + r; return r}
+  this.peers.load.loadBlocks([ hashes[0] ], last);
 };
 
 Pool.prototype._load = function _load() {
@@ -240,6 +255,8 @@ Pool.prototype._addPeer = function _addPeer() {
         entry.e.emit('ack');
       });
     });
+
+    self._scheduleRequests();
   });
 
   peer.on('merkleblock', function(block) {
@@ -351,6 +368,7 @@ Pool.prototype.search = function search(id, range) {
   if (id)
     this.watch(id);
 
+  this._loadRange(hashes);
   hashes.slice().reverse().forEach(function(hash) {
     // Get the block that is in index
     this.chain.get(hash, function(block) {
@@ -395,7 +413,7 @@ Pool.prototype._request = function _request(type, hash, options, cb) {
     return this.request.map[hex].addCallback(cb);
 
   // Block should be not in chain, or be requested
-  if (!options.force && type === 'block' && this.chain.has(hash))
+  if (type === 'block' && this.chain.has(hash, options.force))
     return;
 
   var req = new LoadRequest(this, type, hex, cb);

scripts/update.js

@@ -27,14 +27,12 @@ var pool = bcoin.pool({
 
 console.log('Updating bcoin preloaded chain...');
 
-var last = 0;
 pool.on('block', function(block) {
-  if (block.ts <= last)
+  console.log('Got: %s from %s chain len %d orp %d act %d queue %d',
-    return;
-  console.log('Got: %s from %s chain len %d act %d queue %d',
               block.hash('hex'),
               new Date(block.ts * 1000).toString(),
               pool.chain.index.hashes.length,
+              pool.chain.orphan.count,
               pool.request.active,
               pool.request.queue.length);
 });