From 0a97cebf25c6e869d78a12ad2d8381f5c743958b Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Mon, 4 Jan 2016 18:22:42 -0800
Subject: [PATCH] improve getblocks download, spv chain, and events.

---
 lib/bcoin/block.js     |  9 +++++
 lib/bcoin/chain.js     | 62 +++++++++++++++++---------------
 lib/bcoin/fullchain.js |  9 ++---
 lib/bcoin/pool.js      | 82 ++++++++++++++++++++++--------------------
 4 files changed, 90 insertions(+), 72 deletions(-)

diff --git a/lib/bcoin/block.js b/lib/bcoin/block.js
index 24d883e8..08950072 100644
--- a/lib/bcoin/block.js
+++ b/lib/bcoin/block.js
@@ -78,6 +78,15 @@ function Block(data, subtype) {
   }
 
   this.verify();
+
+  if (this.subtype === 'block' && !this.valid) {
+    this.txs = this.txs.map(function(tx) {
+      tx.block = null;
+      if (tx.ts === self.ts)
+        tx.ts = 0;
+      return tx;
+    });
+  }
 }
 
 Block.prototype.hash = function hash(enc) {
diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js
index 199a47d8..4ceff1e8 100644
--- a/lib/bcoin/chain.js
+++ b/lib/bcoin/chain.js
@@ -46,10 +46,10 @@ function Chain(options) {
   };
 
   this.index = {
-    bloom: null,
     hashes: [],
     ts: [],
     heights: [],
+    lookup: {},
     lastTs: 0
   };
 
@@ -153,7 +153,7 @@ Chain.prototype._getRange = function _getRange(hash, ts, futureOnly) {
 };
 
 Chain.prototype._probeIndex = function _probeIndex(hash, ts) {
-  if (!this.index.bloom.test(hash, 'hex'))
+  if (this.index.lookup[hash] == null)
     return false;
 
   var start = 0;
@@ -195,7 +195,7 @@ Chain.prototype._addIndex = function _addIndex(hash, ts, height) {
   if (checkpoint) {
     this.emit('checkpoint', height, hash, checkpoint);
     if (hash !== checkpoint) {
-      this.resetLastCheckpoint(height);
+      // this.resetLastCheckpoint(height);
       this.emit('fork', height, hash, checkpoint);
       return Chain.codes.badCheckpoint;
     }
@@ -204,7 +204,9 @@ Chain.prototype._addIndex = function _addIndex(hash, ts, height) {
   this.index.ts.splice(pos, 0, ts);
   this.index.hashes.splice(pos, 0, hash);
   this.index.heights.splice(pos, 0, height);
-  this.index.bloom.add(hash, 'hex');
+
+  this.index.lookup[hash] = pos;
+  this.index.lookup[height] = pos;
 
   this.tip = this.getTip();
   this.emit('tip', this.tip);
@@ -228,10 +230,10 @@ Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
 
 Chain.prototype.resetHeight = function resetHeight(height) {
   var self = this;
-  var index = this.index.heights.indexOf(height);
+  var index = this.index.lookup[height];
   var ahead = this.index.hashes.slice(index + 1);
 
-  assert(index >= 0);
+  assert(index != null);
 
   this.block.list.length = 0;
   this.block.bloom.reset();
@@ -241,9 +243,10 @@ Chain.prototype.resetHeight = function resetHeight(height) {
   this.index.ts.length = index + 1;
   this.index.hashes.length = index + 1;
   this.index.heights.length = index + 1;
-  this.index.bloom.reset();
-  this.index.hashes.forEach(function(hash) {
-    self.index.bloom.add(hash, 'hex');
+  this.index.lookup = {};
+  this.index.hashes.forEach(function(hash, i) {
+    self.index.lookup[self.index.hashes[i]] = i;
+    self.index.lookup[self.index.heights[i]] = i;
   });
 
   this.index.lastTs = Math.min(
@@ -301,6 +304,9 @@ Chain.prototype._killFork = function _killFork(probe) {
   this.index.ts.splice(index, 1);
   this.index.heights.splice(index, 1);
 
+  delete this.index.lookup[this.index.hashes[index]];
+  delete this.index.lookup[this.index.heights[index]];
+
   this.tip = this.getTip();
   this.emit('tip', this.tip);
 
@@ -310,7 +316,7 @@ Chain.prototype._killFork = function _killFork(probe) {
   return true;
 };
 
-Chain.prototype.add = function add(block) {
+Chain.prototype.add = function add(block, peer) {
   if (this.loading) {
     this.once('load', function() {
       this.add(block);
@@ -361,8 +367,10 @@ Chain.prototype.add = function add(block) {
     }
 
     // Validated known block at this point - add it to index
-    if (prevProbe)
+    if (prevProbe) {
       code = this._addIndex(hash, block.ts, prevProbe.height + 1);
+      this.emit('block', block, peer);
+    }
 
     // At least one block was added
     this.block.list.push(block);
@@ -450,9 +458,9 @@ Chain.prototype.byHash = function byHash(hash) {
   else if (hash.hash)
     hash = hash.hash('hex');
 
-  var index = this.index.hashes.indexOf(hash);
+  var index = this.index.lookup[hash];
 
-  if (index === -1)
+  if (index == null)
     return null;
 
   return {
@@ -464,9 +472,9 @@ Chain.prototype.byHash = function byHash(hash) {
 };
 
 Chain.prototype.byHeight = function byHeight(height) {
-  var index = this.index.heights.indexOf(height);
+  var index = this.index.lookup[height];
 
-  if (index === -1)
+  if (index == null)
     return null;
 
   return {
@@ -491,8 +499,7 @@ Chain.prototype.hasBlock = function hasBlock(hash) {
   else if (hash.hash)
     hash = hash.hash('hex');
 
-  // return this.byHash(hash);
-  return this.index.bloom.test(hash, 'hex');
+  return this.index.lookup[hash] != null;
 };
 
 Chain.prototype.hasOrphan = function hasOrphan(hash) {
@@ -708,9 +715,10 @@ Chain.prototype.compact = function compact(keep) {
   this.index.hashes = index.hashes;
   this.index.ts = index.ts;
   this.index.heights = index.heights;
-  this.index.bloom.reset();
-  this.index.hashes.forEach(function(hash) {
-    this.index.bloom.add(hash, 'hex');
+  this.index.lookup = {};
+  this.index.hashes.forEach(function(hash, i) {
+    this.index.lookup[this.index.hashes[i]] = i;
+    this.index.lookup[this.index.heights[i]] = i;
   }, this);
 };
 
@@ -817,16 +825,12 @@ Chain.prototype.fromJSON = function fromJSON(json) {
   this.index.ts = json.ts.slice();
   this.index.heights = json.heights.slice();
 
-  if (this.index.bloom)
-    this.index.bloom.reset();
-  else
-    this.index.bloom = new bcoin.bloom(28 * 1024 * 1024, 16, 0xdeadbeef);
+  assert(this.index.hashes.length > 0);
 
-  if (this.index.hashes.length === 0)
-    this.add(new bcoin.block(network.genesis, 'block'));
-
-  for (i = 0; i < this.index.hashes.length; i++)
-    this.index.bloom.add(this.index.hashes[i], 'hex');
+  for (i = 0; i < this.index.hashes.length; i++) {
+    this.index.lookup[this.index.hashes[i]] = i;
+    this.index.lookup[this.index.heights[i]] = i;
+  }
 };
 
 /**
diff --git a/lib/bcoin/fullchain.js b/lib/bcoin/fullchain.js
index 8350010e..f6f03014 100644
--- a/lib/bcoin/fullchain.js
+++ b/lib/bcoin/fullchain.js
@@ -153,7 +153,7 @@ Chain.prototype._addIndex = function _addIndex(entry) {
   if (checkpoint) {
     this.emit('checkpoint', entry.height, entry.hash, checkpoint);
     if (hash !== checkpoint) {
-      this.resetLastCheckpoint(entry.height);
+      // this.resetLastCheckpoint(entry.height);
       this.emit('fork', entry.height, entry.hash, checkpoint);
       return Chain.codes.badCheckpoint;
     }
@@ -216,7 +216,7 @@ Chain.prototype.resetTime = function resetTime(ts) {
   return this.resetHeight(entry.height);
 };
 
-Chain.prototype.add = function add(block) {
+Chain.prototype.add = function add(block, peer) {
   if (this.loading) {
     this.once('load', function() {
       this.add(block);
@@ -277,12 +277,14 @@ Chain.prototype.add = function add(block) {
           code = Chain.codes.forked;
           // Breaking here only works because
           // we deleted the orphan map in resetHeight.
+          this.emit('block', block, peer);
           break;
         }
       }
 
       // Validated known block at this point - add it to index
       code = this._addIndex(entry);
+      this.emit('block', block, peer);
     }
 
     // Fullfill request
@@ -613,8 +615,7 @@ Chain.prototype.fromJSON = function fromJSON(json) {
     this._addIndex(ChainBlock.fromJSON(this, entry));
   }, this);
 
-  if (this.index.entries.length === 0)
-    this.add(new bcoin.block(network.genesis, 'block'));
+  assert(this.index.entries.length > 0);
 };
 
 /**
diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js
index 67f5882b..6560ad91 100644
--- a/lib/bcoin/pool.js
+++ b/lib/bcoin/pool.js
@@ -51,7 +51,7 @@ function Pool(options) {
       this.options.multiplePeers = true;
   } else {
     if (this.options.headers == null)
-      this.options.headers = true;
+      this.options.headers = false;
     if (this.options.multiplePeers == null)
       this.options.multiplePeers = false;
   }
@@ -170,6 +170,10 @@ Pool.prototype._init = function _init() {
     }
   });
 
+  this.chain.on('block', function(block, peer) {
+    self.emit('block', block, peer);
+  });
+
   this.chain.on('fork', function(height, hash, checkpoint) {
     var peer = self.peers.load;
 
@@ -346,9 +350,6 @@ Pool.prototype._handleHeaders = function _handleHeaders(headers, peer) {
     if (!this.chain.has(block))
       this._request(this.block.type, block.hash('hex'));
 
-    // For headers-first:
-    // this._addIndex(block, peer);
-
     last = block;
   }
 
@@ -401,6 +402,12 @@ Pool.prototype._handleBlocks = function _handleBlocks(hashes, peer) {
   for (i = 0; i < hashes.length; i++) {
     hash = hashes[i];
 
+    // Request block if we don't have it
+    if (!this.chain.has(hash)) {
+      this._request(this.block.type, hash);
+      continue;
+    }
+
     // Resolve orphan chain
     if (this.chain.hasOrphan(hash)) {
       peer.loadBlocks(
@@ -409,15 +416,8 @@ Pool.prototype._handleBlocks = function _handleBlocks(hashes, peer) {
       );
       continue;
     }
-
-    // Request block if we don't have it
-    if (!this.chain.has(hash))
-      this._request(this.block.type, hash);
   }
 
-  // Restart the entire getblocks process
-  peer.loadBlocks(this.chain.locatorHashes(), null);
-
   // Push our getdata packet
   this._scheduleRequests();
 
@@ -461,52 +461,56 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer) {
     return;
 
   // Emulate bip37 - emit all the "watched" txs
-  if (this.options.fullNode
-      && this.listeners('watched').length > 0
-      && block.verify()) {
+  if (this.options.fullNode && this.listeners('watched').length > 0) {
     block.txs.forEach(function(tx) {
       if (self.isWatched(tx))
         self.emit('watched', tx, peer);
     });
   }
 
-  // Do not use with headers-first:
-  if (!this._addIndex(block, peer))
+  // Ignore if we already have
+  if (this.chain.has(block))
     return;
 
-  this.emit('block', block, peer);
-};
+  // Make sure the block is valid
+  if (!block.verify())
+    return;
 
-Pool.prototype._addIndex = function _addIndex(block, peer) {
-  var self = this;
-  var hash, size, orphan, res;
-
-  hash = block.hash('hex');
-  size = this.chain.size();
-  orphan = this.chain.hasOrphan(block);
-
-  res = this.chain.add(block);
-  if (res)
-    this.emit('chain-error', bcoin.chain.msg(res), peer);
-
-  if (this.chain.hasOrphan(block)) {
-    // Resolve orphan chain
-    if (!this.options.headers) {
+  // Resolve orphan chain
+  if (!this.options.headers) {
+    if (!this.chain.hasBlock(block.prevBlock)) {
+      this._addIndex(block, peer);
       peer.loadBlocks(
         this.chain.locatorHashes(),
         this.chain.getOrphanRoot(block)
       );
+      return;
     }
-    // Emit our orphan if it is new
-    if (!orphan)
+  }
+
+  // Add to index and emit/save
+  this._addIndex(block, peer);
+};
+
+Pool.prototype._addIndex = function _addIndex(block, peer) {
+  var self = this;
+  var hash, size, orphans, res;
+
+  hash = block.hash('hex');
+  size = this.chain.size();
+  orphans = this.chain.orphan.count;
+
+  res = this.chain.add(block, peer);
+  if (res)
+    this.emit('chain-error', bcoin.chain.msg(res), peer);
+
+  // Do not emit if nothing was added to the chain
+  if (this.chain.size() === size) {
+    if (this.chain.orphan.count > orphans)
       return true;
     return false;
   }
 
-  // Do not emit if nothing was added to the chain
-  if (this.chain.size() === size)
-    return false;
-
   this.emit('chain-progress', this.chain.fillPercent(), peer);
 
   return true;