From fcc9d661c1e3b823d579ad499cab2579ae0733a9 Mon Sep 17 00:00:00 2001
From: Christopher Jeffrey <chjjeffrey@gmail.com>
Date: Mon, 30 Nov 2015 18:15:57 -0800
Subject: [PATCH] satoshi: fix blockchain download.

---
 lib/bcoin/chain.js           | 31 ++-----------
 lib/bcoin/peer.js            | 60 +++++++++----------------
 lib/bcoin/pool.js            | 85 +++++++++++++-----------------------
 lib/bcoin/protocol/framer.js | 10 ++++-
 lib/bcoin/protocol/parser.js | 41 +++++++++++++++--
 5 files changed, 100 insertions(+), 127 deletions(-)

diff --git a/lib/bcoin/chain.js b/lib/bcoin/chain.js
index 26cd3776..37cb4df3 100644
--- a/lib/bcoin/chain.js
+++ b/lib/bcoin/chain.js
@@ -38,8 +38,8 @@ function Chain(options) {
   this.request = new utils.RequestCache();
 
   // Start from the genesis block
-  // if we're using the original protocol.
-  if (!this.options.relay) {
+  // if we're a full node.
+  if (this.options.fullNode) {
     preload = {
       v: preload.v,
       type: preload.type,
@@ -210,31 +210,6 @@ Chain.prototype.add = function add(block) {
     return;
   }
 
-  if (!this.options.relay) {
-    var hash = block.hash('hex');
-    var prev = block.prevBlock;
-    var prevProbe = this._probeIndex(prev, block.ts);
-    if (prevProbe) {
-      this._addIndex(hash, block.ts, prevProbe.height + 1);
-      if (this.orphan.map[hash]) {
-        delete this.orphan.map[hash];
-        this.orphan.count--;
-      }
-      this.block.list.push(block);
-      this._bloomBlock(block);
-      this.request.fullfill(hash, block);
-      this._compress();
-      return true;
-    } else {
-      if (!this.orphan.map[prev]) {
-        this.orphan.map[prev] = block;
-        this.orphan.count++;
-      }
-      this._compress();
-      return false;
-    }
-  }
-
   var res = false;
   var initial = block;
   do {
@@ -426,7 +401,7 @@ Chain.prototype.getLast = function getLast(cb) {
 };
 
 Chain.prototype.getStartHeight = function getLast(cb) {
-  if (this.options.relay) {
+  if (!this.options.fullNode) {
     if (this.options.startHeight != null) {
       return this.options.startHeight;
     }
diff --git a/lib/bcoin/peer.js b/lib/bcoin/peer.js
index 1af4efeb..66d8fa76 100644
--- a/lib/bcoin/peer.js
+++ b/lib/bcoin/peer.js
@@ -89,7 +89,7 @@ Peer.prototype._init = function init() {
     self._error(err);
   });
 
-  if (!this.pool.options.relay) {
+  if (this.pool.options.fullNode) {
     this.once('version', function() {
       var ip = self.socket && self.socket.remoteAddress || '0.0.0.0';
       self.pool.emit('debug', 'version (%s): loading locator hashes for getblocks', ip);
@@ -173,7 +173,7 @@ Peer.prototype.broadcast = function broadcast(items) {
 };
 
 Peer.prototype.updateWatch = function updateWatch() {
-  if (!this.pool.options.relay)
+  if (this.pool.options.fullNode)
     return;
 
   if (this.ack)
@@ -299,6 +299,12 @@ Peer.prototype._onPacket = function onPacket(packet) {
   else if (cmd === 'getaddr')
     return this._handleGetAddr();
 
+  if (cmd === 'headers') {
+    payload = bcoin.block(payload, 'block');
+    this.emit(cmd, payload);
+    return;
+  }
+
   if (cmd === 'merkleblock' || cmd === 'block') {
     payload = bcoin.block(payload, cmd);
     this.lastBlock = payload;
@@ -433,58 +439,34 @@ Peer.prototype._handleInv = function handleInv(items) {
   });
   this.emit('blocks', blocks);
 
-  if (!this.pool.options.relay) {
+  if (this.pool.options.fullNode) {
     var self = this;
 
     if (txs.length) {
       this.emit('txs', txs.map(function(tx) {
         return tx.hash;
       }));
-      // this.getData(txs);
+      this.getData(txs);
     }
 
-    var hashes = blocks.map(utils.toHex);
-
     var orphans = Object.keys(this.chain.orphan.map).reduce(function(out, prev) {
       var orphan = self.chain.orphan.map[prev];
-      out[orphan.hash('hex')] = true;
+      out[orphan.hash('hex')] = orphan;
       return out;
     }, {});
 
-    hashes = hashes.filter(function(hash) {
-      return !orphans[hash];
-    });
-
-    // hashes = hashes.filter(function(hash) {
-    //   return !self.chain.index.bloom.test(hash, 'hex')
-    //       && !self.chain.block.bloom.test(hash, 'hex');
-    // });
-
-    if (blocks.length === 1) {
-      var ip = this.socket && this.socket.remoteAddress || '0.0.0.0';
-      this.pool.emit('debug', 'inv (%s): %s', ip, utils.revHex(utils.toHex(blocks[0])));
-    }
-
-    if (!hashes.length) {
-      var ip = this.socket && this.socket.remoteAddress || '0.0.0.0';
-      //if (blocks.length === 1 && this._latestBlock) {
-      if (blocks.length === 1) {
-        // Already have the latest orphan, another getblocks:
-        this.pool.emit('debug', 'inv (%s): loading locator hashes for getblocks', ip);
-        this.loadBlocks(this.pool.locatorHashes(), 0);
-      } else {
-        this.pool.emit('debug', 'inv (%s): no hashes to getdata', ip);
+    for (var i = 0; i < blocks.length; i++) {
+      var hash = utils.toHex(blocks[i]);
+      if (orphans[hash]) {
+        this.loadBlocks(this.pool.locatorHashes(), orphans[hash].prevBlock);
+        continue;
+      }
+      if (!this.chain.index.hashes[hash]) {
+        this.getData([{ type: 'block', hash: hash }]);
+      } else if (i === blocks.length - 1) {
+        this.getData([{ type: 'block', hash: hash }]);
       }
-      return;
     }
-
-    this.getData(hashes.map(function(hash) {
-      return {
-        type: 'block',
-        hash: hash
-      };
-    }));
-
     return;
   }
 
diff --git a/lib/bcoin/pool.js b/lib/bcoin/pool.js
index c6f8422a..fc34bd2f 100644
--- a/lib/bcoin/pool.js
+++ b/lib/bcoin/pool.js
@@ -13,7 +13,10 @@ function Pool(options) {
   EventEmitter.call(this);
 
   this.options = options || {};
-  this.options.relay = this.options.relay !== false;
+  this.options.fullNode = !!this.options.fullNode;
+  this.options.relay == null
+    ? (this.options.fullNode ? false : true)
+    : this.options.relay;
   this.storage = this.options.storage;
   this.destroyed = false;
   this.size = options.size || 32;
@@ -40,8 +43,8 @@ function Pool(options) {
     storage: this.storage,
     // Since regular blocks contain transactions and full merkle
     // trees, it's risky to cache 2000 blocks. Let's do 100.
-    cacheLimit: !this.options.relay ? 100 : null,
-    relay: this.options.relay,
+    cacheLimit: this.options.fullNode ? 100 : null,
+    fullNode: this.options.fullNode,
     startHeight: this.options.startHeight
   });
   this.watchMap = {};
@@ -100,41 +103,11 @@ Pool.prototype._init = function _init() {
 
   var self = this;
   this.chain.on('missing', function(hash, preload, parent) {
-    if (!self.options.relay) return;
+    if (self.options.fullNode) return;
     self._request('block', hash, { force: true });
     self._scheduleRequests();
     self._loadRange(preload);
   });
-
-  if (!this.options.relay) {
-    // Non-standard stall recovery,
-    // this shouldn't be necessary.
-    this._lastLocator = null;
-    setInterval(function() {
-      if (!self._lastLocator) return;
-      if (Date.now() - 60 * 1000 > self._lastLocator) {
-        self.emit('debug', 'invoking stall recovery!');
-        [].concat(
-          self.peers.pending,
-          self.peers.block,
-          self.peers.load
-        ).reduce(function(out, peer) {
-          if (peer && !~out.indexOf(peer)) {
-            out.push(peer);
-          }
-          return out;
-        }, []).forEach(function(peer) {
-          var ip = peer.socket && peer.socket.remoteAddress || '0.0.0.0';
-          self.emit('debug', 'invoking stall recovery on peer (%s)!', ip);
-          peer.loadBlocks(self.locatorHashes(), 0);
-        });
-      } else {
-        self.emit('debug', 'stall recovery diff: %d/%d',
-          self._lastLocator / 1000 | 0,
-          Date.now() / 1000 | 0);
-      }
-    }, 10 * 1000);
-  }
 };
 
 Pool.prototype._addLoader = function _addLoader() {
@@ -190,7 +163,7 @@ Pool.prototype._addLoader = function _addLoader() {
 
   // Split blocks and request them using multiple peers
   peer.on('blocks', function(hashes) {
-    if (!self.options.relay) return;
+    if (self.options.fullNode) return;
 
     if (hashes.length === 0) {
       // Reset global load
@@ -233,7 +206,7 @@ Pool.prototype.isFull = function isFull() {
 };
 
 Pool.prototype._loadRange = function _loadRange(hashes, force) {
-  if (!this.options.relay) return;
+  if (this.options.fullNode) return;
 
   if (!hashes)
     return;
@@ -257,7 +230,7 @@ Pool.prototype._loadRange = function _loadRange(hashes, force) {
 };
 
 Pool.prototype._load = function _load() {
-  if (!this.options.relay) return;
+  if (this.options.fullNode) return;
 
   if (this.request.queue.length >= this.load.hwm) {
     this.load.hiReached = true;
@@ -334,7 +307,7 @@ Pool.prototype._addPeer = function _addPeer(backoff) {
     self._scheduleRequests();
   });
 
-  if (this.options.relay) {
+  if (!this.options.fullNode) {
     peer.on('merkleblock', function(block) {
       // Reset backoff, peer seems to be responsive
       backoff = 0;
@@ -346,22 +319,22 @@ Pool.prototype._addPeer = function _addPeer(backoff) {
     });
   } else {
     peer.on('block', function(block) {
-      var index = self.chain.index;
       backoff = 0;
+
+      // Ignore if we already have.
+      if (self.chain.index.hashes[block.hash('hex')]
+          || self.chain.orphan.map[block.prevBlock])
+        return;
+
+      // Store as orphan if prevBlock not in main chain
+      if (self.chain.index.hashes[block.prevBlock])
+        peer.loadBlocks(self.locatorHashes(), block.prevBlock);
+
+      // Otherwise, accept block
       self._response(block);
-      var ret = self.chain.add(block);
+      self.chain.add(block);
       self.emit('chain-progress', self.chain.fillPercent(), peer);
       self.emit('block', block, peer);
-      if (!ret) {
-        var ip = peer.socket && peer.socket.remoteAddress || '0.0.0.0';
-        self.emit('debug', 'block (%s): %s', ip, utils.revHex(block.hash('hex')));
-        self.emit('debug', 'block (%s): loading locator hashes for getblocks', ip);
-        peer._latestBlock = { hash: block.hash('hex'), ts: block.ts };
-        if (!self._latestBlock || peer._latestBlock.ts > self._latestBlock.ts) {
-          self._latestBlock = peer._latestBlock;
-        }
-        peer.loadBlocks(self.locatorHashes(), 0);
-      }
     });
   }
 
@@ -409,13 +382,17 @@ Pool.prototype.locatorHashes = function() {
   var hashes = this.chain.index.hashes;
   var indicies = [];
   var top = hashes.length - 1;
-  var step = 1, start = 0;
+  var step = 1;
 
-  for (var i = top; i > 0; i -= step, ++start) {
-    if (start >= 10) step *= 2;
+  for (var j = 0, i = top - 1; j < 10 && i > 0; j++, i--) {
     indicies.push(hashes[i]);
   }
 
+  for (; i > 0; i -= step) {
+    indicies.push(hashes[i]);
+    step *= 2;
+  }
+
   indicies.push(hashes[0]);
 
   indicies = indicies.reduce(function(out, hash) {
@@ -425,8 +402,6 @@ Pool.prototype.locatorHashes = function() {
     return out;
   }, []);
 
-  this._lastLocator = Date.now();
-
   return indicies;
 };
 
diff --git a/lib/bcoin/protocol/framer.js b/lib/bcoin/protocol/framer.js
index 64e5b9da..ce6ea7c0 100644
--- a/lib/bcoin/protocol/framer.js
+++ b/lib/bcoin/protocol/framer.js
@@ -201,7 +201,15 @@ Framer.prototype.filterClear = function filterClear() {
   return this.packet('filterclear', []);
 };
 
+Framer.prototype.getHeaders = function getBlocks(hashes, stop) {
+  return this._getBlocks('getheaders', hashes, stop);
+};
+
 Framer.prototype.getBlocks = function getBlocks(hashes, stop) {
+  return this._getBlocks('getblocks', hashes, stop);
+};
+
+Framer.prototype._getBlocks = function _getBlocks(cmd, hashes, stop) {
   var p = [];
   writeU32(p, constants.version, 0);
   var off = 4 + varint(p, hashes.length, 4);
@@ -227,7 +235,7 @@ Framer.prototype.getBlocks = function getBlocks(hashes, stop) {
     p[off + len] = 0;
   assert.equal(off + len, p.length);
 
-  return this.packet('getblocks', p);
+  return this.packet(cmd, p);
 };
 
 Framer.tx = function tx(tx) {
diff --git a/lib/bcoin/protocol/parser.js b/lib/bcoin/protocol/parser.js
index 7fc165ba..3a2b1da0 100644
--- a/lib/bcoin/protocol/parser.js
+++ b/lib/bcoin/protocol/parser.js
@@ -97,6 +97,8 @@ Parser.prototype.parsePayload = function parsePayload(cmd, p) {
     return this.parseInvList(p);
   else if (cmd === 'merkleblock')
     return this.parseMerkleBlock(p);
+  else if (cmd === 'headers')
+    return this.parseHeaders(p);
   else if (cmd === 'block')
     return this.parseBlock(p);
   else if (cmd === 'tx')
@@ -135,10 +137,6 @@ Parser.prototype.parseVersion = function parseVersion(p) {
   // Relay
   var relay = p.length > off ? p[off] === 1 : true;
 
-  // NOTE: Could just do this to make relay
-  // `false` when it's not included:
-  // var relay = p[off] === 1;
-
   return {
     v: v,
     services: services,
@@ -226,6 +224,41 @@ Parser.prototype.parseMerkleBlock = function parseMerkleBlock(p) {
   };
 };
 
+Parser.prototype.parseHeaders = function parseHeaders(p) {
+  if (p.length < 81)
+    return this._error('Invalid headers size');
+
+  var result = readIntv(p, 0);
+  var off = result.off;
+  var count = result.r;
+
+  var headers = [];
+
+  if (p.length >= off + 81) {
+    for (var i = 0; i < count; i++) {
+      var header = {};
+      header.version = readU32(p, off);
+      off += 4;
+      header.prevBlock = p.slice(off, off + 32);
+      off += 32;
+      header.merkleRoot = p.slice(off, off + 32);
+      off += 32;
+      header.ts = readU32(p, off);
+      off += 4;
+      header.bits = readU32(p, off);
+      off += 4;
+      header.nonce = readU32(p, off);
+      off += 4;
+      var r = readIntv(p, off);
+      header.totalTX = r.r;
+      off += r.off;
+      headers.push(header);
+    }
+  }
+
+  return headers;
+};
+
 Parser.prototype.parseBlock = function parseBlock(p) {
   if (p.length < 81)
     return this._error('Invalid block size');