wip

lib/logger.js

@@ -13,7 +13,6 @@ function Logger(options) {
     options = {};
   }
   this.formatting = _.isUndefined(options.formatting) ? Logger.DEFAULT_FORMATTING : options.formatting;
-  this.level = options.logLevel || process.env.LOG_LEVEL || 0; // lower numbers means less logs
 }
 
 Logger.DEFAULT_FORMATTING = true;
@@ -39,31 +38,11 @@ Logger.prototype.error = function() {
  * #debug
  */
 Logger.prototype.debug = function() {
-  if (this.level >= 1) {
+  if (process.env.BITCORE_ENV === 'debug') {
-    this._log.apply(this, ['magenta', 'debug'].concat(Array.prototype.slice.call(arguments)));
-  }
-};
-
-/**
- * Prints an debug2 message
- * #debug2
- */
-Logger.prototype.debug2 = function() {
-  if (this.level >= 2) {
     this._log.apply(this, ['green', 'debug'].concat(Array.prototype.slice.call(arguments)));
   }
 };
 
-/**
- * Prints an debug3 message
- * #debug3
- */
-Logger.prototype.debug3 = function() {
-  if (this.level >= 3) {
-    this._log.apply(this, ['cyan', 'debug'].concat(Array.prototype.slice.call(arguments)));
-  }
-};
-
 /**
  * Prints an warn message
  * #warn
@@ -86,7 +65,7 @@ Logger.prototype._log = function(color) {
     var typeString = colors[color].italic(level + ':');
     args[0] = '[' + date.toISOString() + ']' + ' ' + typeString + ' ' + args[0];
   }
-  var fn = console[level] || console.log;
+  var fn = console.log;
   fn.apply(console, args);
 };
 

lib/services/block/encoding.js

@@ -1,6 +1,6 @@
 'use strict';
 
-var Block = require('bitcore-lib').Block;
+var Block = require('bcoin').block;
 // stores -- block header as key, block itself as value (optionally)
 
 function Encoding(servicePrefix) {

lib/services/block/index.js

@@ -48,7 +48,7 @@ inherits(BlockService, BaseService);
 
 BlockService.dependencies = [ 'p2p', 'db', 'header' ];
 
-BlockService.MAX_BLOCKS = 500;
+BlockService.MAX_BLOCKS = 250;
 
 // --- public prototype functions
 BlockService.prototype.getAPIMethods = function() {
@@ -86,18 +86,18 @@ BlockService.prototype.getBlockOverview = function(hash, callback) {
       return callback(err);
     }
 
-    var header = block.toHeaders();
+    var header = block.toHeaders().toJSON();
 
     var blockOverview = {
       hash: block.hash,
       version: header.version,
       confirmations: null,
-      height: null,
+      height: header.height,
-      chainWork: null,
+      chainWork: header.chainwork,
       prevHash: header.prevBlock,
       nextHash: null,
       merkleRoot: block.merkleroot,
-      time: null,
+      time: header.timestamp,
       medianTime: null,
       nonce: block.nonce,
       bits: block.bits,
@@ -134,7 +134,7 @@ BlockService.prototype.getRawBlock = function(hash, callback) {
     if(err) {
       return callback(err);
     }
-    callback(null, block.toString());
+    callback(null, block.toRaw().toString('hex'));
   });
 };
 
@@ -181,7 +181,7 @@ BlockService.prototype._primeBlockQueue = function(callback) {
 
   async.timesSeries(50, function(index, next) {
 
-    self._db.get(self._encoding.encodeBlockKey(hash), function(err, block) {
+    self._db.get(self._encoding.encodeBlockKey(hash), function(err, data) {
 
       if(err) {
         return next(err);
@@ -191,8 +191,9 @@ BlockService.prototype._primeBlockQueue = function(callback) {
         return next();
       }
 
-      hash = block.toHeaders().prevBlock;
+      var block = self._encoding.decodeBlockValue(data);
-      self._blockQueue.set(block.hash, block);
+      hash = block.toHeaders().toJSON().prevBlock;
+      self._blockQueue.set(block.rhash(), block);
       next();
 
     });
@@ -254,7 +255,7 @@ BlockService.prototype.unsubscribe = function(name, emitter) {
 
 BlockService.prototype._blockAlreadyProcessed = function(block) {
 
-  return this._blockQueue.get(block.hash) ? true : false;
+  return this._blockQueue.get(block.rhash()) ? true : false;
 
 };
 
@@ -274,17 +275,7 @@ BlockService.prototype._cacheBlock = function(block) {
   // 2. store the block in the database
   var operations = this._getBlockOperations(block);
 
-  this._db.batch(operations, function(err) {
+  this._db.batch(operations);
-
-    if(err) {
-      log.error('There was an error attempting to save block for hash: ' + block.hash);
-      this._db.emit('error', err);
-      return;
-    }
-
-    log.debug('Success saving block hash ' + block.hash);
-  });
-
 };
 
 BlockService.prototype._determineBlockState = function(block) {
@@ -325,13 +316,14 @@ BlockService.prototype._findCommonAncestor = function(hash) {
     function(next) {
 
       // old tip has to be in database
-      self._db.get(self._encoding.encodeBlockKey(_oldTip), function(err, block) {
+      self._db.get(self._encoding.encodeBlockKey(_oldTip), function(err, data) {
 
         if (err || !block) {
           return next(err || new Error('missing block'));
         }
 
-        _oldTip = block.toHeaders().prevBlock;
+        var block = self._encoding.decodeBlockValue(data);
+        _oldTip = block.toHeaders().toJSON().prevBlock;
         var header = headers.get(_newTip);
 
         if (!header) {
@@ -352,11 +344,29 @@ BlockService.prototype._findCommonAncestor = function(hash) {
 };
 
 BlockService.prototype._getBlock = function(hash, callback) {
+
+  var self = this;
+
   var block = this._blockQueue(hash);
+
   if (block) {
     return callback(null, block);
   }
-  this._db.get(this._encoding.encodeBlockKey(hash), callback);
+
+  this._db.get(this._encoding.encodeBlockKey(hash), function(err, data) {
+
+    if(err) {
+      return callback(err);
+    }
+
+    if (!block) {
+      return callback();
+    }
+
+    var block = self._encoding.decodeBlockValue(data);
+    callback(null, block);
+
+  });
 };
 
 BlockService.prototype._getBlockOperations = function(block) {
@@ -392,7 +402,7 @@ BlockService.prototype._getDelta = function(tip) {
 
   while (_tip !== this._tip.hash) {
     var blk = this._blockQueue.get(_tip);
-    _tip = blk.toHeaders().prevBlock;
+    _tip = blk.toHeaders().toJSON().prevBlock;
     blocks.push(blk);
   }
 
@@ -415,7 +425,7 @@ BlockService.prototype._getIncompleteChainIndexes = function(block) {
   var ret = [];
   for(var i = 0; i < this._incompleteChains.length; i++) {
     var chain = this._incompleteChains[i];
-    var lastEntry = chain[chain.length - 1].toHeaders();
+    var lastEntry = chain[chain.length - 1].toHeaders().toJSON();
     if (lastEntry.prevHash === block.rhash()) {
       ret.push(i);
     }
@@ -511,7 +521,7 @@ BlockService.prototype._onReorg = function(oldBlockList, commonAncestorHeader) {
 BlockService.prototype._isChainReorganizing = function(block) {
 
   // if we aren't an out of order block, then is this block's prev hash our tip?
-  var prevHash = block.toHeaders().prevBlock;
+  var prevHash = block.toHeaders().toJSON().prevBlock;
 
   return prevHash !== this._tip.hash;
 
@@ -520,7 +530,7 @@ BlockService.prototype._isChainReorganizing = function(block) {
 BlockService.prototype._isOutOfOrder = function(block) {
 
   // is this block the direct child of one of our chain tips? If so, not an out of order block
-  var prevHash = block.toHeaders().prevBlock;
+  var prevHash = block.toHeaders().toJSON().prevBlock;
 
   for(var i = 0; i < this._chainTips.length; i++) {
     var chainTip = this._chainTips[i];
@@ -547,7 +557,7 @@ BlockService.prototype._onBlock = function(block) {
   }
 
   // 2. log the reception
-  log.info('New block received: ' + block.rhash());
+  log.debug('New block received: ' + block.rhash());
 
   // 3. store the block for safe keeping
   this._cacheBlock(block);
@@ -572,6 +582,7 @@ BlockService.prototype._onBlock = function(block) {
       break;
     default:
       // send all unsent blocks now that we have a complete chain
+
       this._sendDelta();
       break;
   }
@@ -581,12 +592,19 @@ BlockService.prototype._selectActiveChain = function() {
 
   var chainTip;
   var mostChainWork = new BN(0);
+  var headers = this._header.getAllHeaders();
 
   var self = this;
 
+  if (this._chainTips.length === 1) {
+    return this._chainTips[0];
+  }
+
   for(var i = 0; i < this._chainTips.length; i++) {
 
-    var work = self._getChainwork(this._chainTips[i]);
+    var header = headers.get(this._chainTips[i]);
+    assert(header, 'we must have a header for chain tip.');
+    var work = new BN(new Buffer(header.chainwork, 'hex'));
 
     if (work.gt(mostChainWork)) {
       mostChainWork = work;
@@ -611,51 +629,28 @@ BlockService.prototype._sendDelta = function() {
     this._broadcast(this._subscriptions.block, 'block/block', blocks[i]);
   }
 
-  var headers = this._header.getAllHeaders();
+  var newTipHeight = this._tip.height + i;
-  var len = headers.length - 1;
+  var newTipHash = blocks[i - 1].rhash();
-  this._setTip({ height: len, hash: headers[len].hash });
+
+  this._setTip({ height: newTipHeight, hash: newTipHash });
+  var tipOps = utils.encodeTip(this._tip, this.name);
+
+  var tipOps = [{
+    type: 'put',
+    key: tipOps.key,
+    value: tipOps.value
+  }];
+
+  this._db.batch(tipOps);
 
   if (++this._blockCount >= BlockService.MAX_BLOCKS) {
-    this._latestBlockHash = this._tip.hash;
     this._blockCount = 0;
-    this._continueSync();
+    log.debug('Block Service: calling sync again for more blocks.');
+    this._sync();
   }
 
 };
 
-BlockService.prototype._continueSync = function() {
-  /*
-     Essentially, we have three distinct tasks taking place during a sync operation
-
-     1. receiving network data
-     2. light processing of that data (data manipulation)
-     3. writing that data to disk
-
-     Our goal: Sync as quickly as our least performant task will allow using a modest amount of memory.
-     If our network is very fast and our disk is very slow and we have very little memory to work with,
-     this is the worst case scenario. We want to defer asking the p2p service for more blocks until the
-     disk has time to process all the queued everts.
-
-     Our way of dealing with this is to review the resident set size and defer the next call to sync if
-     we are in danger of a 'JS Object Allocation Failed - Out of Memory' fault.
-
-     You can use the "maxMemory" config setting for this service to limit the amount of memory considered.
-     The default is 1.5GB of memory, max.
-
-     If we are over 95% of max memory usage, we will defer the sync until we are 95% or less.
-  */
-
-  var self = this;
-  var totalRss = process.memoryUsage().rss;
-  if ((totalRss / 1E6) / self._maxMem > 0.95) {
-    self._deferTimeout = setTimeout(function() {
-      self._continueSync();
-    }, 1000);
-    return;
-  }
-  self._sync();
-};
-
 BlockService.prototype._setListeners = function() {
 
   this._header.once('headers', this._onAllHeaders.bind(this));
@@ -677,8 +672,6 @@ BlockService.prototype._startSync = function() {
 
   log.info('Gathering: ' + this._numNeeded + ' ' + 'block(s) from the peer-to-peer network.');
 
-  this._p2pBlockCallsNeeded = Math.ceil(this._numNeeded / 500);
-  this._latestBlockHash = this._tip.hash || this.GENESIS_HASH;
   this._sync();
 };
 
@@ -699,20 +692,25 @@ BlockService.prototype._startSubscriptions = function() {
 
 BlockService.prototype._sync = function() {
 
-  if (--this._p2pBlockCallsNeeded > 0) {
+  var headers = this._header.getAllHeaders();
+  var size = headers.size - 1;
+
+  if (this._tip.height < size) {
 
     log.info('Blocks download progress: ' + this._tip.height + '/' +
       this._numNeeded + '  (' + (this._tip.height / this._numNeeded*100).toFixed(2) + '%)');
-    this._p2p.getBlocks({ startHash: this._latestBlockHash });
-    return;
 
+    var endHash = Array.from(headers)[ Math.min(this._tip.height + BlockService.MAX_BLOCKS, size) ][0];
+
+    this._p2p.getBlocks({ startHash: this._tip.hash, endHash: endHash });
+    return;
   }
 
 };
 
 BlockService.prototype._updateChainInfo = function(block, state) {
 
-  var prevHash = block.toHeaders().prevBlock;
+  var prevHash = block.toHeaders().toJSON().prevBlock;
 
   if (state === 'normal') {
     this._updateNormalStateChainInfo(block, prevHash);
@@ -740,13 +738,13 @@ BlockService.prototype._updateNormalStateChainInfo = function(block, prevHash) {
   var incompleteChainIndexes = this._getIncompleteChainIndexes(block);
   //retrieving more than one incomplete chain for a given block means there is a future reorg in the incomplete chain
   if (incompleteChainIndexes.length < 1) {
-    this._chainTips.push(block.hash);
+    this._chainTips.push(block.rhash());
   } else {
 
     incompleteChainIndexes.sort().reverse();
     for(var i = 0; i < incompleteChainIndexes.length; i++) {
       var incompleteIndex = incompleteChainIndexes[i];
-      this._chainTips.push(this._incompleteChains[incompleteIndex][0].hash);
+      this._chainTips.push(this._incompleteChains[incompleteIndex][0].rhash());
       this._incompleteChains.splice(incompleteIndex, 1);
     }
 
@@ -771,7 +769,7 @@ BlockService.prototype._updateOutOfOrderStateChainInfo = function(block) {
 
   */
 
-  var prevHash = block.toHeaders().prevBlock;
+  var prevHash = block.toHeaders().toJSON().prevBlock;
   var possibleJoins = { tip: [], genesis: [] };
   var newChains = [];
   var joinedChains = false;
@@ -788,7 +786,7 @@ BlockService.prototype._updateOutOfOrderStateChainInfo = function(block) {
     var chains;
 
     // criteria 1
-    var lastEntryPrevHash = lastEntry.toHeaders().prevBlock;
+    var lastEntryPrevHash = lastEntry.toHeaders().toJSON().prevBlock;
     if (lastEntryPrevHash === block.hash) {
 
       joinedChains = true;

lib/services/header/index.js

@@ -108,22 +108,15 @@ HeaderService.prototype._startSubscriptions = function() {
 
 HeaderService.prototype._onBlock = function(block) {
   // we just want the header to keep a running list
-  log.debug('Header Service: new block: ' + block.hash);
+  log.debug('Header Service: new block: ' + block.rhash());
 
-  var header = {
+  var header = block.toHeaders().toJSON();
-    hash: block.hash,
+  header.timestamp = header.ts;
-    version: block.version,
+  header.prevHash = header.prevBlock;
-    timestamp: block.ts,
+  this._onHeaders([header], 1);
-    bits: block.bits,
-    nonce: block.nonce,
-    merkleRoot: block.merkleRoot,
-    prevHash: block.prevBlock
-  };
-
-  this._onHeaders([header]);
 };
 
-HeaderService.prototype._onHeaders = function(headers) {
+HeaderService.prototype._onHeaders = function(headers, convert) {
 
   if (!headers || headers.length < 1) {
     return;
@@ -131,10 +124,18 @@ HeaderService.prototype._onHeaders = function(headers) {
 
   log.debug('Header Service: Received: ' + headers.length + ' header(s).');
 
-  var operations = this._getHeaderOperations(headers);
+  var newHeaders = headers;
+  if (!convert) {
+    newHeaders = headers.map(function(header) {
+      header = header.toObject();
+      return header;
+    });
+  }
 
-  this._tip.hash = headers[headers.length - 1].hash;
+  var operations = this._getHeaderOperations(newHeaders);
-  this._tip.height = this._tip.height + headers.length;
+
+  this._tip.hash = newHeaders[newHeaders.length - 1].hash;
+  this._tip.height = this._tip.height + newHeaders.length;
 
   this._db.batch(operations);
 
@@ -154,7 +155,7 @@ HeaderService.prototype._getHeaderOperations = function(headers) {
     header.chainwork = self._getChainwork(header, prevHeader).toString(16, 32);
     prevHeader = header;
     // set the header in the in-memory map
-    self._headers.set(header.hash, utils.convertLEHeaderStrings(header));
+    self._headers.set(header.hash, header);
     return {
       type: 'put',
       key: self._encoding.encodeHeaderKey(header.height, header.hash),
@@ -191,7 +192,7 @@ HeaderService.prototype._sync = function() {
 
   if (this._tip.height < this._bestHeight) {
 
-    log.info('Header Service: download progress: ' + this._tip.height + '/' +
+    log.debug('Header Service: download progress: ' + this._tip.height + '/' +
       this._numNeeded + '  (' + (this._tip.height / this._numNeeded*100).toFixed(2) + '%)');
 
 

lib/services/p2p/index.js

@@ -172,7 +172,7 @@ P2P.prototype._connect = function() {
 };
 
 P2P.prototype._getBestHeight = function() {
-  if (this._peers < this._minPeers) {
+  if (this._peers === 0) {
     return 0;
   }
 
@@ -184,18 +184,12 @@ P2P.prototype._getBestHeight = function() {
     }
   }
   return maxHeight;
-
 };
 
 // we should only choose from a list of peers that sync'ed
 P2P.prototype._getPeer = function() {
 
   return this._peer;
-  //if (this._peers.length === 0) {
-  //  return;
-  //}
-  //var index = this._peerIndex++ % this._peers.length;
-  //return this._peers[index];
 };
 
 P2P.prototype._hasPeers = function() {
@@ -211,7 +205,7 @@ P2P.prototype._initCache = function() {
 
 P2P.prototype._initP2P = function() {
   this._maxPeers = this._options.maxPeers || 60;
-  this._minPeers = this._options.minPeers || 1;
+  this._minPeers = this._options.minPeers || 0;
   this._configPeers = this._options.peers;
   this.messages = new p2p.Messages({ network: this.node.network });
   this._peerHeights = [];
@@ -226,7 +220,7 @@ P2P.prototype._initPool = function() {
     opts.addrs = this._configPeers;
   }
   // TODO: bcoin stuff goes here
-  opts.dnsSeed = false; // do not set to true unless you want to be pwned
+  opts.dnsSeed = false;
   opts.maxPeers = this._maxPeers;
   opts.network = this.node.getNetworkName();
   this._pool = new p2p.Pool(opts);

lib/utils.js

@@ -56,6 +56,9 @@ utils.diffTime = function(time) {
 };
 
 utils.reverseBufferToString = function(buf) {
+  if (_.isString(buf)) {
+    buf = new Buffer(buf, 'hex');
+  }
   return BufferUtil.reverse(buf).toString('hex');
 };
 
@@ -213,11 +216,5 @@ utils.isHeight = function(arg) {
   return _.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))
 };
 
-utils.convertLEHeaderStrings = function(header) {
-  return Object.assign(header, {
-    prevHash: utils.reverseBufferToString(new Buffer(header.prevHash, 'hex')),
-    merkleRoot: utils.reverseBufferToString(new Buffer(header.merkleRoot, 'hex'))
-  });
-};
 
 module.exports = utils;