get blockchain sync working. fix pushdata ops.

README.md

@@ -69,6 +69,10 @@ var pool = new bcoin.pool({
   })
 });
 
+pool.on('error', function(err) {
+  console.log('Error: %s', err.message);
+});
+
 // Receive the address of another peer.
 pool.on('addr', function(data, peer) {
   var host = data.ipv4 + ':' + data.port;

lib/bcoin/block.js

@@ -1,5 +1,6 @@
 var bcoin = require('../bcoin');
 var utils = bcoin.utils;
+var constants = bcoin.protocol.constants;
 
 function Block(data, subtype) {
   if (!(this instanceof Block))
@@ -36,7 +37,7 @@ function Block(data, subtype) {
       return tx.hash('hex');
     });
     this.invalid = !this._checkBlock();
-    this.hashes = this.merkleTree;
+    this.hashes = [];
   }
 
   this._hash = null;
@@ -127,21 +128,31 @@ Block.prototype._verifyMerkle = function verifyMerkle() {
   }
 };
 
-Block.prototype._buildMerkle = function buildMerkle() {
+Block.prototype.getMerkleRoot = function getMerkleRoot() {
   var merkleTree = [];
+
   for (var i = 0; i < this.txs.length; i++) {
     merkleTree.push(this.txs[i].hash('hex'));
   }
+
   var j = 0;
   for (var size = this.txs.length; size > 1; size = ((size + 1) / 2) | 0) {
     for (var i = 0; i < size; i += 2) {
       var i2 = Math.min(i + 1, size - 1);
-      var hash = utils.dsha256(merkleTree[j+i] + merkleTree[j+i2], 'hex');
+      if (i2 === i + 1 && i2 + 1 === size
+          && merkleTree[j + i] === merkleTree[j + i2]) {
+        return utils.toHex(constants.zeroHash);
+      }
+      var hash = utils.dsha256(merkleTree[j + i] + merkleTree[j + i2], 'hex');
       merkleTree.push(utils.toHex(hash));
     }
     j += size;
   }
-  return merkleTree;
+
+  if (!merkleTree.length)
+    return utils.toHex(constants.zeroHash);
+
+  return merkleTree[merkleTree.length - 1];
 };
 
 // This mimics the behavior of CheckBlockHeader()
@@ -172,9 +183,6 @@ Block.prototype._checkBlock = function checkBlock() {
       return false;
   }
 
-  // Build MerkleTree
-  this.merkleTree = this._buildMerkle();
-
   // Check for duplicate tx ids
   var unique = {};
   for (var i = 0; i < this.txs.length; i++) {
@@ -184,8 +192,14 @@ Block.prototype._checkBlock = function checkBlock() {
     unique[hash] = true;
   }
 
+  // Build MerkleTree
+  var merkleRoot = this.getMerkleRoot();
+
   // Check merkle root
-  return this.merkleTree[this.merkleTree.length - 1] === this.merkleRoot;
+  if (merkleRoot !== this.merkleRoot)
+    return false;
+
+  return true;
 };
 
 Block.prototype.toJSON = function toJSON() {
@@ -212,7 +226,7 @@ Block.fromJSON = function fromJSON(json) {
     parser.parseMerkleBlock(raw) :
     parser.parseBlock(raw);
 
-  var block = new bcoin.block(data, json.subtype);
+  var block = new Block(data, json.subtype);
 
   block._hash = json.hash;
 

lib/bcoin/chain.js

@@ -153,6 +153,14 @@ Chain.prototype._addIndex = function _addIndex(hash, ts, height) {
     return;
   }
 
+  // var checkpoint = network.checkpoints[height];
+  // if (checkpoint) {
+  //   this.emit('checkpoint', height, hash, checkpoint);
+  //   if (hash !== checkpoint) {
+  //     this.resetLastCheckpoint(height);
+  //   }
+  // }
+
   this.index.ts.splice(pos, 0, ts);
   this.index.hashes.splice(pos, 0, hash);
   this.index.heights.splice(pos, 0, height);
@@ -169,6 +177,33 @@ Chain.prototype._addIndex = function _addIndex(hash, ts, height) {
   });
 };
 
+Chain.prototype.resetLastCheckpoint = function resetLastCheckpoint(height) {
+  var lastHeight = Object.keys(network.checkpoints).sort().indexOf(height) - 1;
+
+  if (lastHeight < 0)
+    i = 0;
+
+  this.resetHeight(lastHeight);
+};
+
+Chain.prototype.resetHeight = function resetHeight(height) {
+  var index = this.index.heights.indexOf(height);
+
+  if (index < 0)
+    throw new Error('Cannot reset to height of ' + height);
+
+  this.block.list = [];
+  this.block.bloom.reset();
+  this.orphan.map = {};
+  this.orphan.bmap = {};
+  this.orphan.count = 0;
+  this.index.ts = this.index.ts.slice(0, index + 1);
+  this.index.hashes = this.index.hashes.slice(0, index + 1);
+  this.index.heights = this.index.heights.slice(0, index + 1);
+  this.index.bloom.reset();
+  this.index.lastTs = this.index.ts[this.index.ts.length - 1];
+};
+
 Chain.prototype._killFork = function _killFork(probe) {
   var delta = 2 * 3600;
   var upper = probe.ts + delta;
@@ -224,30 +259,37 @@ Chain.prototype.add = function add(block) {
   }
 
   var res = false;
+  var err = null;
   var initial = block;
   do {
     // No need to revalidate orphans
-    if (!res && !block.verify())
+    if (!res && !block.verify()) {
+      err = new Error('Block verification failed.');
       break;
+    }
 
     var hash = block.hash('hex');
     var prev = block.prevBlock;
 
     // If the block is already known to be an orphan
-    if (this.orphan.map[prev])
+    if (this.orphan.map[prev]) {
+      err = new Error('Block is a known orphan.');
       break;
+    }
 
     var prevProbe = this._probeIndex(prev, block.ts);
 
     // Remove forked nodes from storage, if shorter chain is detected
-    if (this._killFork(prevProbe))
+    if (this._killFork(prevProbe)) {
+      err = new Error('Fork found.');
       break;
+    }
 
     // If previous block wasn't ever seen - add current to orphans
     if (!this._probeIndex(hash, block.ts) && !prevProbe) {
       this.orphan.count++;
       this.orphan.map[prev] = block;
-      this.orphan.bmap[block.hash('hex')] = block;
+      this.orphan.bmap[hash] = block;
 
       var range = this._getRange(hash, block.ts, true);
       var hashes = this.index.hashes.slice(range.start, range.end + 1);
@@ -272,15 +314,15 @@ Chain.prototype.add = function add(block) {
 
     // We have orphan child for this block - add it to chain
     block = this.orphan.map[hash];
-    delete this.orphan.map[hash];
     delete this.orphan.bmap[block.hash('hex')];
+    delete this.orphan.map[hash];
     this.orphan.count--;
   } while (true);
 
   // Compress old blocks
   this._compress();
 
-  return res;
+  return err;
 };
 
 Chain.prototype._compress = function compress() {
@@ -331,6 +373,13 @@ Chain.prototype.has = function has(hash, noProbe, cb) {
   return cb(!!this.orphan.map[hash]);
 };
 
+Chain.prototype.hasBlock = function hasBlock(hash, noProbe, strict) {
+  if (this.loading)
+    return false;
+
+  return this.index.bloom.test(hash, 'hex');
+};
+
 Chain.prototype.get = function get(hash, force, cb) {
   if (typeof force === 'function') {
     cb = force;

lib/bcoin/peer.js

@@ -96,7 +96,7 @@ Peer.prototype._init = function init() {
         'Sent version (%s): height=%s',
         ip, this.pool.chain.getStartHeight());
       self.pool.emit('debug', 'version (%s): sending locator hashes', ip);
-      self.loadHeaders(self.chain.locatorHashes(), 0);
+      self.loadBlocks(self.chain.locatorHashes(), 0);
     });
   }
 
@@ -436,14 +436,37 @@ Peer.prototype._handleInv = function handleInv(items) {
   }, this).map(function(item) {
     return item.hash;
   });
+
+  if (blocks.length === 1)
+    this.bestBlock = utils.toHex(blocks[0]);
+
   this.emit('blocks', blocks);
 
+  if (this.pool.options.fullNode) {
+    for (var i = 0; i < blocks.length; i++) {
+      var hash = utils.toHex(blocks[i]);
+      if (this.chain.orphan.bmap[hash]) {
+        this.loadBlocks(this.chain.locatorHashes(), this.chain.getOrphanRoot(hash));
+        continue;
+      }
+      if (!this.chain.hasBlock(hash)) {
+        this.getData([{ type: 'block', hash: hash }]);
+        continue;
+      }
+      if (i === blocks.length - 1) {
+        this.loadBlocks(this.chain.locatorHashes(), 0);
+        continue;
+      }
+    }
+  }
+
   if (txs.length === 0)
     return;
 
   this.emit('txs', txs.map(function(tx) {
     return tx.hash;
   }));
+
   this.getData(txs);
 };
 
@@ -457,24 +480,9 @@ Peer.prototype._handleHeaders = function handleHeaders(headers) {
     return header;
   });
 
-  if (this.pool.options.fullNode) {
-    for (var i = 0; i < headers.length; i++) {
-      var header = headers[i];
-      var hash = header.hash;
-      // if (this.chain.orphan.bmap[hash]) {
-      if (this.chain.orphan.map[header.prevBlock]) {
-        this.loadHeaders(this.chain.locatorHashes(), this.chain.getOrphanRoot(hash));
-        continue;
-      }
-      if (!this.chain.index.bloom.test(hash, 'hex') || i === headers.length - 1)
-        this.getData([{ type: 'block', hash: hash }]);
-    }
-  }
-
   this.emit('headers', headers);
 };
 
-
 Peer.prototype.loadHeaders = function loadHeaders(hashes, stop) {
   this._write(this.framer.getHeaders(hashes, stop));
 };

lib/bcoin/pool.js

@@ -5,6 +5,7 @@ var EventEmitter = require('events').EventEmitter;
 var bcoin = require('../bcoin');
 var utils = bcoin.utils;
 var assert = utils.assert;
+var network = bcoin.protocol.network;
 
 function Pool(options) {
   var self = this;
@@ -89,8 +90,8 @@ function Pool(options) {
   // Added and watched wallets
   this.wallets = [];
 
-  this.createConnection = options.createConnection;
+  this.createSocket = options.createConnection || options.createSocket;
-  assert(this.createConnection);
+  assert(this.createSocket);
 
   this.chain.on('debug', function() {
     var args = Array.prototype.slice.call(arguments);
@@ -130,7 +131,7 @@ Pool.prototype._addLoader = function _addLoader() {
   if (this.peers.load !== null)
     return;
 
-  var peer = new bcoin.peer(this, this.createConnection, {
+  var peer = new bcoin.peer(this, this.createSocket, {
     backoff: 750 * Math.random(),
     startHeight: this.options.startHeight,
     relay: this.options.relay
@@ -275,7 +276,7 @@ Pool.prototype._addPeer = function _addPeer(backoff) {
   if (this.peers.block.length + this.peers.pending.length >= this.size)
     return;
 
-  var peer = new bcoin.peer(this, this.createConnection, {
+  var peer = new bcoin.peer(this, this.createSocket, {
     backoff: backoff,
     startHeight: this.options.startHeight,
     relay: this.options.relay
@@ -335,21 +336,30 @@ Pool.prototype._addPeer = function _addPeer(backoff) {
     peer.on('block', function(block) {
       backoff = 0;
 
-      var height = self.chain.index.hashes.length;
+      var len = self.chain.index.hashes.length;
       var hash = block.hash('hex');
 
-      if (!self.chain.index.bloom.test(block.prevBlock, 'hex')) {
+      if (!self.chain.hasBlock(block.prevBlock))
-        peer.loadHeaders(self.chain.locatorHashes(), self.chain.getOrphanRoot(block));
+        peer.loadBlocks(self.chain.locatorHashes(), self.chain.getOrphanRoot(block));
-      }
 
       self._response(block);
-      self.chain.add(block);
+      var err = self.chain.add(block);
+      if (err)
+        self.emit('chain-error', err, peer);
 
-      // if (!self.chain.index.bloom.test(block.prevBlock, 'hex')) {
+      self.emit('_block', block, peer);
-      //   peer.loadHeaders(self.chain.locatorHashes(), self.chain.getOrphanRoot(block));
+
-      // } else if (self.chain.index.hashes.length === height) {
+      if (self.chain.index.hashes.length === len)
-      //   return;
+        return;
-      // }
+
+      var height = self.chain.index.heights.length[self.chain.index.heights.length - 1];
+
+      var checkpoint = network.checkpoints[height];
+      if (checkpoint) {
+        self.emit('checkpoint', block, height, checkpoint, peer);
+        // if (checkpoint !== hash)
+        //   self.chain.resetLastCheckpoint(height);
+      }
 
       self.emit('chain-progress', self.chain.fillPercent(), peer);
       self.emit('block', block, peer);
@@ -379,6 +389,8 @@ Pool.prototype._addPeer = function _addPeer(backoff) {
   });
 
   peer.on('blocks', function(blocks) {
+    if (blocks.length === 1)
+      self.bestBlock = peer.bestBlock;
     self.emit('blocks', blocks, peer);
   });
 
@@ -410,11 +422,7 @@ Pool.prototype._removePeer = function _removePeer(peer) {
 
 Pool.prototype.watch = function watch(id) {
   if (id instanceof bcoin.wallet) {
-    // this.watch(id.getAddress());
+    this.watchWallet(id);
-    this.watch(id.getFullHash());
-    this.watch(id.getFullPublicKey());
-    this.watch(id.getOwnHash());
-    this.watch(id.getOwnPublicKey());
     return;
   }
 
@@ -463,10 +471,7 @@ Pool.prototype.unwatch = function unwatch(id) {
 Pool.prototype.addWallet = function addWallet(w, defaultTs) {
   if (this.wallets.indexOf(w) !== -1)
     return false;
-  this.watch(w.getFullHash());
+  this.watchWallet(w);
-  this.watch(w.getFullPublicKey());
-  this.watch(w.getOwnHash());
-  this.watch(w.getOwnPublicKey());
 
   var self = this;
   var e = new EventEmitter();
@@ -490,18 +495,41 @@ Pool.prototype.addWallet = function addWallet(w, defaultTs) {
   }
 
   return e;
-}
+};
 
 Pool.prototype.removeWallet = function removeWallet(w) {
   var i = this.wallets.indexOf(w);
   if (i == -1)
     return;
   this.wallets.splice(i, 1);
-  this.unwatch(w.getFullHash());
+  this.unwatchWallet(w);
-  this.unwatch(w.getFullPublicKey());
+};
+
+Pool.prototype.watchWallet = function watchWallet(w) {
+  if (w.type === 'script') {
+    // For the redeem script hash in outputs:
+    this.watch(w.getFullHash());
+    // For the redeem script in inputs:
+    this.watch(w.getFullPublicKey());
+  }
+  // For the pubkey hash in outputs:
+  this.watch(w.getOwnHash());
+  // For the pubkey in inputs:
+  this.watch(w.getOwnPublicKey());
+};
+
+Pool.prototype.unwatchWallet = function unwatchWallet(w) {
+  if (w.type === 'script') {
+    // For the redeem script hash in p2sh outputs:
+    this.unwatch(w.getFullHash());
+    // For the redeem script in p2sh inputs:
+    this.unwatch(w.getFullPublicKey());
+  }
+  // For the pubkey hash in p2pk/multisig outputs:
   this.unwatch(w.getOwnHash());
+  // For the pubkey in p2pkh inputs:
   this.unwatch(w.getOwnPublicKey());
-}
+};
 
 Pool.prototype.search = function search(id, range, e) {
   e = e || new EventEmitter();

lib/bcoin/protocol/constants.js

@@ -155,3 +155,8 @@ exports.oneHash = utils.toArray(
   '0000000000000000000000000000000000000000000000000000000000000001',
   'hex'
 );
+
+exports.zeroHash = utils.toArray(
+  '0000000000000000000000000000000000000000000000000000000000000000',
+  'hex'
+);

lib/bcoin/protocol/network.js

@@ -62,6 +62,11 @@ main.checkpoints = [
   { height: 295000, hash: '00000000000000004d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983' }
 ];
 
+main.checkpoints = main.checkpoints.reduce(function(out, block) {
+  out[block.height] = utils.revHex(block.hash);
+  return block.height;
+}, {});
+
 main.checkpoints.tsLastCheckpoint = 1397080064;
 main.checkpoints.txsLastCheckpoint = 36544669;
 main.checkpoints.txsPerDay = 60000.0;
@@ -128,6 +133,11 @@ testnet.checkpoints = [
   { height: 546, hash: '000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70' }
 ];
 
+testnet.checkpoints = testnet.checkpoints.reduce(function(out, block) {
+  out[block.height] = utils.revHex(block.hash);
+  return block.height;
+}, {});
+
 testnet.checkpoints.tsLastCheckpoint = 1338180505;
 testnet.checkpoints.txsLastCheckpoint = 16341;
 testnet.checkpoints.txsPerDay = 300;

lib/bcoin/script.js

@@ -16,6 +16,10 @@ script.decode = function decode(s) {
     if (b >= 0x01 && b <= 0x4b) {
       opcodes.push(s.slice(i, i + b));
       i += b;
+      utils.hidden(opcodes[opcodes.length - 1], 'pushdata', {
+        opcode: null,
+        len: b
+      });
       continue;
     }
 
@@ -33,29 +37,51 @@ script.decode = function decode(s) {
 
     var opcode = constants.opcodesByVal[b];
     if (opcode === 'pushdata1') {
-      var len = s[i++];
+      // NOTE: pushdata1 could be substituted with 0x01-0x4b
+      // later if less than or equal to 0x4b.
+      // bad when passed back into encode().
+      var len = s[i];
+      i += 1;
       opcodes.push(s.slice(i, i + len));
-      i += 2 + len;
+      i += len;
+      utils.hidden(opcodes[opcodes.length - 1], 'pushdata', {
+        opcode: opcode,
+        len: len
+      });
     } else if (opcode === 'pushdata2') {
+      // NOTE: len could theoretically be less than 0xffff
+      // here. bad when passed back into encode().
       var len = utils.readU16(s, i);
       i += 2;
       opcodes.push(s.slice(i, i + len));
       i += len;
+      utils.hidden(opcodes[opcodes.length - 1], 'pushdata', {
+        opcode: opcode,
+        len: len
+      });
     } else if (opcode === 'pushdata4') {
+      // NOTE: len could theoretically be less than 0xffffffff
+      // here. bad when passed back into encode().
       var len = utils.readU32(s, i);
       i += 4;
       opcodes.push(s.slice(i, i + len));
       i += len;
+      utils.hidden(opcodes[opcodes.length - 1], 'pushdata', {
+        opcode: opcode,
+        len: len
+      });
     } else {
       opcodes.push(opcode || b);
     }
   }
+
   return opcodes;
 };
 
 script.encode = function encode(s) {
   if (!s)
     return [];
+
   var opcodes = constants.opcodes;
   var res = [];
   for (var i = 0; i < s.length; i++) {
@@ -63,6 +89,24 @@ script.encode = function encode(s) {
 
     // Push value to stack
     if (Array.isArray(instr)) {
+      // Check for nonstandard pushdatas that
+      // may have been decoded from before.
+      if (instr.pushdata) {
+        if (instr.pushdata.opcode === null) {
+          res = res.concat(instr.pushdata.len, instr);
+        } else if (instr.pushdata.opcode === 'pushdata1') {
+          res = res.concat(opcodes.pushdata1, instr.pushdata.len, instr);
+        } else if (instr.pushdata.opcode === 'pushdata2') {
+          res.push(opcodes.pushdata2);
+          utils.writeU16(res, instr.pushdata.len, res.length);
+          res = res.concat(instr);
+        } else if (instr.pushdata.opcode === 'pushdata4') {
+          res.push(opcodes.pushdata4);
+          utils.writeU32(res, instr.pushdata.len, res.length);
+          res = res.concat(instr);
+        }
+        continue;
+      }
       if (instr.length === 0) {
         // OP_FALSE
         res.push(0);

lib/bcoin/utils.js

@@ -447,7 +447,7 @@ RequestCache.prototype.fullfill = function fullfill(id, err, data) {
 utils.asyncify = function asyncify(fn) {
   return function _asynicifedFn(err, data1, data2) {
     if (!fn)
-      return;
+      return err || data1;
     utils.nextTick(function() {
       fn(err, data1, data2);
     });
@@ -576,3 +576,13 @@ utils.merge = function(target) {
   });
   return target;
 };
+
+utils.hidden = function(obj, prop, value) {
+  Object.defineProperty(obj, prop, {
+    value: value,
+    enumerable: false,
+    configurable: true,
+    writable: true
+  });
+  return obj;
+};