queue. fixes.

lib/bcoin/block.js

@@ -70,7 +70,11 @@ function Block(data, subtype) {
 
   if (this.subtype === 'block') {
     this.txs = this.txs.map(function(data) {
-      assert(!(data instanceof bcoin.tx));
+      // assert(!(data instanceof bcoin.tx));
+      if (data instanceof bcoin.tx) {
+        // assert(data.ts === self.ts);
+        return data;
+      }
       return bcoin.tx(data, self);
     });
   }
@@ -527,7 +531,10 @@ Block.fromRaw = function fromRaw(data, enc) {
   var parser = new bcoin.protocol.parser();
 
   if (enc === 'hex')
-    data = utils.toArray(data, 'hex');
+    data = new Buffer(data, 'hex');
+
+  if (Array.isArray(data))
+    data = new Buffer(data);
 
   return new Block(parser.parseBlock(data), 'block');
 };

lib/bcoin/blockdb.js

@@ -185,6 +185,9 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) {
         var address = input.getAddress();
         var uaddr;
 
+        if (input.isCoinbase())
+          return;
+
         if (type === 'pubkey' || type === 'multisig')
           address = null;
 
@@ -213,7 +216,7 @@ BlockDB.prototype.saveBlock = function saveBlock(block, callback) {
           self.cache.unspent.remove(input.prevout.hash + '/' + input.prevout.index);
       });
 
-      tx.outputs.forEach(function(output) {
+      tx.outputs.forEach(function(output, i) {
         var type = output.getType();
         var address = output.getAddress();
         var uaddr, coinOffset;
@@ -304,6 +307,9 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
           var address = input.getAddress();
           var uaddr, coinOffset;
 
+          if (input.isCoinbase())
+            return;
+
           if (type === 'pubkey' || type === 'multisig')
             address = null;
 
@@ -338,7 +344,7 @@ BlockDB.prototype.removeBlock = function removeBlock(hash, callback) {
             coinOffset);
         });
 
-        tx.outputs.forEach(function(output) {
+        tx.outputs.forEach(function(output, i) {
           var type = output.getType();
           var address = output.getAddress();
           var uaddr;
@@ -420,6 +426,9 @@ BlockDB.prototype.fillCoin = function fillCoin(tx, callback) {
   if (!pending)
     return callback();
 
+  if (tx.isCoinbase())
+    return callback();
+
   tx.inputs.forEach(function(input) {
     if (input.output) {
       if (!--pending)
@@ -448,6 +457,9 @@ BlockDB.prototype.fillTX = function fillTX(tx, callback) {
   if (!pending)
     return callback();
 
+  if (tx.isCoinbase())
+    return callback();
+
   tx.inputs.forEach(function(input) {
     if (input.output) {
       if (!--pending)
@@ -789,10 +801,10 @@ BlockDB.prototype.getTX = function getTX(hash, callback) {
 
 BlockDB.prototype.getBlock = function getBlock(hash, callback) {
   var self = this;
-  var id = 'b/b/' + value;
+  var id = 'b/b/' + hash;
 
   if (typeof hash === 'number')
-    id = 'b/h/' + value;
+    id = 'b/h/' + hash;
 
   this.index.get(id, function(err, record) {
     if (err) {
@@ -836,10 +848,10 @@ BlockDB.prototype.getBlock = function getBlock(hash, callback) {
 
 BlockDB.prototype.hasBlock = function hasBlock(hash, callback) {
   var self = this;
-  var id = 'b/b/' + value;
+  var id = 'b/b/' + hash;
 
   if (typeof hash === 'number')
-    id = 'b/h/' + value;
+    id = 'b/h/' + hash;
 
   this.index.get(id, function(err, record) {
     if (err && err.type !== 'NotFoundError')
@@ -916,8 +928,8 @@ BlockDB.prototype.getHeight = function getHeight(callback) {
     end: 'b/h~'
   });
 
-  stream.on('data', function(data) {
+  stream.on('data', function(key) {
-    var parts = data.key.split('/').slice(2);
+    var parts = key.split('/').slice(2);
     var height = +parts[0];
     if (height > maxHeight)
       maxHeight = height;
@@ -942,13 +954,13 @@ BlockDB.prototype.resetHeight = function resetHeight(height, callback) {
       return callback(new Error('Cannot reset to height ' + height));
 
     (function next() {
-      if (height === currentHeight)
+      if (currentHeight === height)
         return callback();
 
-      self.removeBlock(height, function(err, block) {
+      self.removeBlock(currentHeight, function(err, block) {
         if (err)
           return callback(err);
-        height--;
+        currentHeight--;
         next();
       });
     })();

lib/bcoin/chain.js

@@ -91,6 +91,37 @@ Chain.msg = function msg(code) {
   return Chain.messages[code] || 'Unknown';
 };
 
+Chain.prototype._ensureGenesis = function _ensureGenesis(callback) {
+  var self = this;
+
+  callback = utils.asyncify(callback);
+
+  if (!this.blockdb)
+    return callback();
+
+  self.blockdb.hasBlock(network.genesis.hash, function(err, result) {
+    var genesis;
+
+    if (err)
+      return callback(err);
+
+    if (result)
+      return callback();
+
+    utils.debug('BlockDB does not have genesis block. Adding.');
+
+    genesis = bcoin.block.fromRaw(network.genesisBlock, 'hex');
+    genesis.height = 0;
+
+    self.blockdb.saveBlock(genesis, function(err) {
+      if (err)
+        return callback(err);
+
+      return callback();
+    });
+  });
+};
+
 Chain.prototype._init = function _init() {
   var self = this;
 
@@ -98,16 +129,20 @@ Chain.prototype._init = function _init() {
 
   utils.debug('Chain is loading.');
 
-  this._preload(function(err, start) {
+  this._ensureGenesis(function(err) {
-    if (err) {
+    if (err)
-      utils.debug('Preloading chain failed.');
+      throw err;
-      utils.debug('Reason: %s', err.message);
+
-    }
+    self._preload(function(err, start) {
-    utils.nextTick(function() {
       var count = self.db.count();
       var i = start || 1;
       var lastEntry;
 
+      if (err) {
+        utils.debug('Preloading chain failed.');
+        utils.debug('Reason: %s', err.message);
+      }
+
       utils.debug('Starting chain load at height: %s', i);
 
       function doneForReal() {
@@ -125,11 +160,11 @@ Chain.prototype._init = function _init() {
           utils.debug('Chain successfully loaded.');
         }
 
+        // self.resetHeight(self.tip.height - 1);
+
         if (!self.blockdb)
           return doneForReal();
 
-        return doneForReal();
-
         self.blockdb.getHeight(function(err, height) {
           if (err)
             throw err;
@@ -563,10 +598,10 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, flags, callbac
 
   // If we are an ancestor of a checkpoint, we can
   // skip the input verification.
-  if (height < network.checkpoints.lastHeight && !network.checkpoints[height])
+  // if (height < network.checkpoints.lastHeight && !network.checkpoints[height])
-    return callback(null, true);
+  //   return callback(null, true);
 
-  this.blockdb.fillCoins(block.txs, function(err) {
+  this._fillBlock(block, function(err) {
     var i, j, input, hash;
 
     if (err)
@@ -604,6 +639,38 @@ Chain.prototype._checkInputs = function _checkInputs(block, prev, flags, callbac
   });
 };
 
+Chain.prototype._fillBlock = function _fillBlock(block, callback) {
+  var self = this;
+
+  return this.blockdb.fillCoins(block.txs, function(err) {
+    var coins, i, tx, hash, j, input, id;
+
+    if (err)
+      return callback(err);
+
+    coins = {};
+
+    for (i = 0; i < block.txs.length; i++) {
+      tx = block.txs[i];
+      hash = tx.hash('hex');
+
+      for (j = 0; j < tx.inputs.length; j++) {
+        input = tx.inputs[j];
+        id = input.prevout.hash + '/' + input.prevout.index;
+        if (!input.output && coins[id]) {
+          input.output = coins[id];
+          delete coins[id];
+        }
+      }
+
+      for (j = 0; j < tx.outputs.length; j++)
+        coins[hash + '/' + j] = bcoin.coin(tx, j);
+    }
+
+    return callback();
+  });
+};
+
 Chain.prototype._addEntry = function _addEntry(entry, block, callback) {
   var self = this;
   var existing;

lib/bcoin/input.js

@@ -229,11 +229,15 @@ Input.prototype.getData = function getData() {
 
 Input.prototype.getType = function getType() {
   var prev = this.output ? this.output.script : null;
+  if (this.isCoinbase())
+    return 'coinbase';
   return bcoin.script.getInputType(this.script, prev);
 };
 
 Input.prototype.getAddress = function getAddress() {
   var prev = this.output ? this.output.script : null;
+  if (this.isCoinbase())
+    return;
   return bcoin.script.getInputAddress(this.script, prev);
 };
 

lib/bcoin/peer.js

@@ -85,7 +85,8 @@ function Peer(pool, options) {
     interval: this.options.pingInterval || 30000
   };
 
-  this._queue = [];
+  this._blockQueue = [];
+  this._txQueue = [];
 
   Peer.uid.iaddn(1);
 

lib/bcoin/pool.js

@@ -102,6 +102,8 @@ function Pool(options) {
     (Math.random() * 0xffffffff) | 0
   );
 
+  this._handlingBlock = 0;
+
   this.peers = {
     // Peers that are loading blocks themselves
     regular: [],
@@ -121,7 +123,8 @@ function Pool(options) {
     bestHeight: 0,
     bestHash: null,
     type: !options.spv ? 'block' : 'filtered',
-    invalid: {}
+    invalid: {},
+    total: 0
   };
 
   this.tx = {
@@ -514,6 +517,10 @@ Pool.prototype._handleHeaders = function _handleHeaders(headers, peer) {
   if (last && headers.length === 2000)
     peer.loadHeaders(this.chain.getLocator(last), null);
 
+  // If we're not currently handling a
+  // block, start the request cycle.
+  this._nextBlock(peer);
+
   // Reset interval to avoid calling getheaders unnecessarily
   this._startInterval();
 };
@@ -568,6 +575,10 @@ Pool.prototype._handleBlocks = function _handleBlocks(hashes, peer) {
     }
   }
 
+  // If we're not currently handling a
+  // block, start the request cycle.
+  this._nextBlock(peer);
+
   // Reset interval to avoid calling getblocks unnecessarily
   this._startInterval();
 
@@ -606,11 +617,19 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
 
   callback = utils.asyncify(callback);
 
+  function done(err, result) {
+    if (!--self._handlingBlock)
+      self.emit('flush');
+    callback(err, result);
+  }
+
+  this._handlingBlock++;
+
   this._prehandleBlock(block, peer, function(err) {
     var requested;
 
     if (err)
-      return callback(err);
+      return done(err);
 
     // Fulfill our request.
     requested = self._response(block);
@@ -628,7 +647,7 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
     if (self.block.invalid[block.hash('hex')]) {
       utils.debug('Peer is sending an invalid chain (%s)', peer.host);
       self.setMisbehavior(peer, 100);
-      return callback(null, false);
+      return done(null, false);
     }
 
     // Ensure this is not a continuation
@@ -638,14 +657,14 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
         'Peer is sending an invalid continuation chain (%s)',
         peer.host);
       self.setMisbehavior(peer, 100);
-      return callback(null, false);
+      return done(null, false);
     }
 
     // Ignore if we already have.
     if (self.chain.has(block)) {
       utils.debug('Already have block %s (%s)', block.height, peer.host);
       self.setMisbehavior(peer, 1);
-      return callback(null, false);
+      return done(null, false);
     }
 
     // Make sure the block is valid.
@@ -655,7 +674,7 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
         block.rhash, peer.host);
       self.block.invalid[block.hash('hex')] = true;
       self.setMisbehavior(peer, 100);
-      return callback(null, false);
+      return done(null, false);
     }
 
     // Someone is sending us blocks without
@@ -671,14 +690,14 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
       if (!self.chain.hasBlock(block.prevBlock)) {
         // Special case for genesis block.
         if (block.isGenesis())
-          return callback(null, false);
+          return done(null, false);
 
         // Make sure the peer doesn't send us
         // more than 200 orphans every 3 minutes.
         if (self.isOrphaning(peer)) {
           utils.debug('Peer is orphaning (%s)', peer.host);
           self.setMisbehavior(peer, 100);
-          return callback(null, false);
+          return done(null, false);
         }
 
         // NOTE: If we were to emit new orphans here, we
@@ -687,7 +706,7 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
         // the height until later.
         self._addIndex(block, peer, function(err, added) {
           if (err)
-            return callback(err);
+            return done(err);
 
           if (added)
             self.emit('pool block', block, peer);
@@ -700,7 +719,7 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
 
           utils.debug('Handled orphan %s (%s)', block.rhash, peer.host);
 
-          return callback(null, false);
+          return done(null, false);
         });
 
         return;
@@ -709,12 +728,12 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
       if (!self.chain.hasBlock(block.prevBlock)) {
         // Special case for genesis block.
         if (block.isGenesis())
-          return callback(null, false);
+          return done(null, false);
 
         // Increase banscore by 10 if we're using getheaders.
         if (!self.options.multiplePeers) {
           if (self.setMisbehavior(peer, 10))
-            return callback(null, false);
+            return done(null, false);
         }
       }
     }
@@ -722,14 +741,14 @@ Pool.prototype._handleBlock = function _handleBlock(block, peer, callback) {
     // Add to index and emit/save
     self._addIndex(block, peer, function(err, added) {
       if (err)
-        return callback(err);
+        return done(err);
 
       if (added) {
         self.emit('pool block', block, peer);
-        return callback(null, true);
+        return done(null, true);
       }
 
-      return callback(null, false);
+      return done(null, false);
     });
   });
 };
@@ -745,6 +764,32 @@ Pool.prototype._addIndex = function _addIndex(block, peer, callback) {
 
     self.emit('chain-progress', self.chain.fillPercent(), peer);
 
+    // if (peer._requesting && peer._requesting.hash === block.hash('hex')) {
+    //   delete peer._requesting;
+    //   self._nextBlock(peer);
+    // }
+
+    if (peer._requesting)
+      delete peer._requesting;
+
+    self._nextBlock(peer);
+
+    self.block.total += added;
+
+    if (self.chain.height() % 100 === 0) {
+      utils.debug(
+        'Got: %s from %s chain len %d blocks %d orp %d act %d queue %d target %s peers %d',
+        block.rhash,
+        new Date(block.ts * 1000).toString(),
+        self.chain.height(),
+        self.block.total,
+        self.chain.orphan.count,
+        self.request.active,
+        self.request.queue.length,
+        self.chain.currentTarget(),
+        self.peers.all.length);
+    }
+
     return callback(null, true);
   });
 };
@@ -873,7 +918,7 @@ Pool.prototype._handleTX = function _handleTX(tx, peer, callback) {
     addMempool(tx, peer, function(err) {
       var requested, added;
 
-      if (err)
+      if (err && self.synced)
         utils.debug('Mempool error: %s', err.message);
 
       requested = self._response(tx);
@@ -1524,25 +1569,74 @@ Pool.prototype._request = function _request(peer, type, hash, options, cb) {
 
   item = new LoadRequest(this, peer, type, hash, cb);
 
-  if (peer._queue.length === 0) {
+  if (item.type === 'tx') {
-    utils.nextTick(function() {
+    if (peer._txQueue.length === 0) {
-      utils.debug(
+      utils.nextTick(function() {
-        'Requesting %d/%d items from %s with getdata',
+        utils.debug(
-        peer._queue.length,
+          'Requesting %d/%d items from %s with getdata',
-        self.request.active,
+          peer._txQueue.length,
-        peer.host);
+          self.request.active,
+          peer.host);
 
-      peer.getData(peer._queue);
+        peer.getData(peer._txQueue);
-      peer._queue.length = 0;
+        peer._txQueue.length = 0;
+      });
+    }
+
+    peer._txQueue.push({
+      type: type,
+      hash: hash
     });
+
+    return;
   }
 
-  peer._queue.push({
+  peer._blockQueue.push({
     type: type,
     hash: hash
   });
 };
 
+Pool.prototype._startRequests = function _startRequests(peer) {
+  var self = this;
+
+  if (!this._handlingBlock)
+    return this._nextBlock(peer);
+
+  this.on('flush', function() {
+    self._nextBlock(peer);
+  });
+};
+
+Pool.prototype._nextBlock = function _nextBlock(peer) {
+  var item;
+
+  if (peer._requesting)
+    return;
+
+  item = peer._blockQueue.shift();
+
+  if (peer.destroyed)
+    return;
+
+  if (!item)
+    return;
+
+  // XXX MAYBE ONLY CREATE LOAD REQUEST HERE
+
+  utils.debug(
+    'Requesting block %s (%d/%d) from %s with getdata (height %d)',
+    utils.revHex(item.hash),
+    peer._blockQueue.length,
+    this.request.active,
+    peer.host,
+    this.chain.height());
+
+  peer._requesting = item;
+
+  peer.getData([item]);
+};
+
 Pool.prototype._response = function _response(hash) {
   var hash;
 
@@ -1940,9 +2034,15 @@ LoadRequest.prototype.finish = function finish() {
     this.pool.request.active--;
   }
 
-  index = this.peer._queue.indexOf(this);
+  if (this.type === 'tx') {
-  if (index !== -1)
+    index = this.peer._txQueue.indexOf(this);
-    this.peer._queue.splice(index, 1);
+    if (index !== -1)
+      this.peer._txQueue.splice(index, 1);
+  } else {
+    index = this.peer._blockQueue.indexOf(this);
+    if (index !== -1)
+      this.peer._blockQueue.splice(index, 1);
+  }
 
   this.peer.removeListener('close', this._finish);
 

lib/bcoin/protocol/network.js

@@ -7,6 +7,7 @@
 var bcoin = require('../../bcoin');
 var bn = require('bn.js');
 var utils = bcoin.utils;
+var assert = utils.assert;
 
 /**
  * Network
@@ -78,7 +79,7 @@ main.checkpoints = main.checkpoints.reduce(function(out, block) {
 main.checkpoints.tsLastCheckpoint = 1397080064;
 main.checkpoints.txsLastCheckpoint = 36544669;
 main.checkpoints.txsPerDay = 60000.0;
-main.checkpoints.lastHeight = Object.keys(main.checkpoints).sort().pop();
+main.checkpoints.lastHeight = 295000;
 
 main.halvingInterval = 210000;
 
@@ -120,6 +121,8 @@ main.block = {
   majorityWindow: 1000
 };
 
+main.genesisBlock = '0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000';
+
 /**
  * Testnet (v3)
  * https://en.bitcoin.it/wiki/Testnet
@@ -165,7 +168,7 @@ testnet.checkpoints = testnet.checkpoints.reduce(function(out, block) {
 testnet.checkpoints.tsLastCheckpoint = 1338180505;
 testnet.checkpoints.txsLastCheckpoint = 16341;
 testnet.checkpoints.txsPerDay = 300;
-testnet.checkpoints.lastHeight = Object.keys(testnet.checkpoints).sort().pop();
+testnet.checkpoints.lastHeight = 546;
 
 testnet.halvingInterval = 210000;
 
@@ -207,6 +210,8 @@ testnet.block = {
   majorityWindow: 100
 };
 
+testnet.genesisBlock = '0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000';
+
 /**
  * Regtest
  */
@@ -275,3 +280,5 @@ regtest.block = {
   majorityRejectOutdated: 950,
   majorityWindow: 1000
 };
+
+regtest.genesisBlock = '0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff7f20020000000101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000';

lib/bcoin/tx.js

@@ -1856,7 +1856,10 @@ TX.fromRaw = function fromRaw(data, enc) {
   var parser = new bcoin.protocol.parser();
 
   if (enc === 'hex')
-    data = utils.toArray(data, 'hex');
+    data = new Buffer(data, 'hex');
+
+  if (Array.isArray(data))
+    data = new Buffer(data);
 
   return new bcoin.tx(parser.parseTX(data));
 };