wip

lib/scaffold/start.js

@@ -7,6 +7,7 @@ var bitcore = require('bitcore-lib');
 var _ = bitcore.deps._;
 var log = index.log;
 var shuttingDown = false;
+var fs = require('fs');
 
 log.debug = function() {};
 
@@ -126,6 +127,22 @@ function checkService(service) {
   }
 }
 
+function lookInRequirePathConfig(req, service) {
+  if (!service.config.requirePath) {
+    return;
+  }
+  //if a file, then remove any possible '.js' extension
+  //if a directory, then leave alone
+  try {
+    if (fs.statSync(service.config.requirePath).isDirectory()) {
+      return req(service.config.requirePath);
+    }
+    var serviceFile = service.config.requirePath.replace(/.js$/, '');
+    return req(serviceFile);
+  } catch(e) {
+  }
+}
+
 function lookInCwd(req, service) {
   try {
     return req(process.cwd + '/' + service);
@@ -165,15 +182,20 @@ function lookInModuleManifest(req, service) {
 function loadModule(req, service) {
   var serviceCode;
 
-  //first look in the current working directory (of the controlling terminal, if there is one) for the service code
+  //first, if we have explicitly set the require path for our service, use this.
-  serviceCode = lookInCwd(req, service);
+  serviceCode = lookInRequirePathConfig(req, service);
 
-  //second try the built-in services
+  //second, look in the current working directory (of the controlling terminal, if there is one) for the service code
+  if(!serviceCode) {
+    serviceCode = lookInCwd(req, service);
+  }
+
+  //third, try the built-in services
   if(!serviceCode) {
     serviceCode = lookInBuiltInPath(req, service);
   }
 
-  //third see if there is directory in our module search path that has a
+  //fourth, see if there is directory in our module search path that has a
   //package.json file, if so, then see if there is a bitcoreNode field, if so
   //use this as the path to the service module
   if(!serviceCode) {
@@ -181,7 +203,8 @@ function loadModule(req, service) {
   }
 
   if (!serviceCode) {
-    throw new Error('Attempted to load the ' + service.name + ' service from: "' +
+    throw new Error('Attempted to load the ' + service.name + ' service from: ' +
+      'the requirePath in the services\' config, then "' +
       process.cwd() + '" then from: "' + __dirname + '/../lib/services' + '" finally from: "' +
       process.cwd() + '/package.json" - bitcoreNode field. All paths failed to find valid nodeJS code.');
   }
@@ -213,6 +236,7 @@ function setupServices(req, servicesPath, config) {
   if (config.services) {
     for (var i = 0; i < config.services.length; i++) {
       var service = {};
+
       service.name = config.services[i];
 
       var hasConfig = config.servicesConfig && config.servicesConfig[service.name];

lib/services/block/block_handler.js

@@ -174,6 +174,10 @@ BlockStream.prototype._getBlocks = function(heights, callback) {
 
     self.block.getBlock(height, function(err, block) {
 
+      if (err === 'reorg') {
+        return self.push(null);
+      }
+
       if(err) {
         return next(err);
       }
@@ -273,7 +277,7 @@ ProcessConcurrent.prototype._transform = function(block, enc, callback) {
     self.operations = self.operations.concat(operations);
 
     if(self.blockCount >= 1) {
-      self.operations.push(self.block.getConcurrentTipOperation(block, true));
+      self.operations.push(self.block.getTipOperation(block, true, 'concurrentTip'));
       var obj = {
         concurrentTip: block,
         operations: self.operations
@@ -290,7 +294,7 @@ ProcessConcurrent.prototype._transform = function(block, enc, callback) {
 
 ProcessConcurrent.prototype._flush = function(callback) {
   if(this.operations.length) {
-    this.operations.push(this.block.getConcurrentTipOperation(this.lastBlock, true));
+    this.operations.push(this.block.getTipOperation(this.lastBlock, true));
     this.operations = [];
     return callback(null, this.operations);
   }
@@ -323,7 +327,7 @@ ProcessBoth.prototype._write = function(block, encoding, callback) {
       if(err) {
         return callback(err);
       }
-      operations.push(self.block.getConcurrentTipOperation(block, true));
+      operations.push(self.block.getTipOperation(block, true, 'concurrentTip'));
       next(null, operations);
   });
   }, function(next) {

lib/services/block/index.js

@@ -50,7 +50,7 @@ BlockService.prototype._startSubscriptions = function() {
     self.bus = self.node.openBus({remoteAddress: 'localhost'});
 
     self.bus.on('bitcoind/hashblock', function() {
-      self.sync();
+      self._blockHandler.sync();
     });
 
     self.bus.subscribe('bitcoind/hashblock');
@@ -102,7 +102,7 @@ BlockService.prototype._sync = function() {
 BlockService.prototype._getBlocks = function(callback) {
 
   var self = this;
-  var blocksDiff = self.bitcoind.height - self.tip.__height - 1;
+  var blocksDiff = self.bitcoind.height - self.tip.__height;
 
   if (blocksDiff < 0) {
       self._log('Peer\'s height is less than our own. The peer may be syncing.' +
@@ -118,7 +118,7 @@ BlockService.prototype._getBlocks = function(callback) {
 
   async.timesLimit(blocksDiff, 8, function(n, next) {
 
-    var blockNumber = n + self.tip.__height + 2;
+    var blockNumber = n + self.tip.__height + 1;
     self.bitcoind.getBlockHeader(blockNumber, function(err, header) {
 
       if(err) {
@@ -201,28 +201,31 @@ BlockService.prototype._detectReorg = function(callback) {
     return callback();
   }
 
+  // all synced
   if (self.tip.hash === self.bitcoind.tiphash && self.tip.__height === self.bitcoind.height) {
     return callback();
   }
 
+  // check if our tip height has the same hash as the network's
   self.bitcoind.getBlockHeader(self.tip.__height, function(err, header) {
 
     if(err) {
       return callback(err);
     }
 
-    //we still might have a reorg, but later
+    // we still might have a reorg if our tip is greater than the network's
+    // we won't know about this until we start syncing
     if (header.hash === self.tip.hash) {
       return callback();
     }
 
-    //our hash isn't in the network anymore, we have definitely reorg'ed
+    //our hash isn't in the network chain anymore, we have reorg'ed
-    callback(header, callback);
+    self._handleReorg(header.hash, callback);
 
   });
 };
 
-BlockService.prototype._handleReorg = function(header, callback) {
+BlockService.prototype._handleReorg = function(hash, callback) {
 
   var self = this;
   self.printTipInfo('Reorg detected!');
@@ -231,7 +234,7 @@ BlockService.prototype._handleReorg = function(header, callback) {
 
   var reorg = new Reorg(self.node, self);
 
-  reorg.handleReorg(header.hash, function(err) {
+  reorg.handleReorg(hash, function(err) {
 
     if(err) {
       self._log('Reorg failed! ' + err, log.error);
@@ -378,11 +381,12 @@ BlockService.prototype.getSerialBlockOperations = function(block, add, callback)
   );
 };
 
-BlockService.prototype.getTipOperation = function(block, add) {
+BlockService.prototype.getTipOperation = function(block, add, tipType) {
+
   var heightBuffer = new Buffer(4);
   var tipData;
 
-  if(add) {
+  if (add) {
     heightBuffer.writeUInt32BE(block.__height);
     tipData = Buffer.concat([new Buffer(block.hash, 'hex'), heightBuffer]);
   } else {
@@ -390,35 +394,54 @@ BlockService.prototype.getTipOperation = function(block, add) {
     tipData = Buffer.concat([BufferUtil.reverse(block.header.prevHash), heightBuffer]);
   }
 
-  return {
+  var type = tipType || 'tip';
-    type: 'put',
-    key: this.dbPrefix + 'tip',
-    value: tipData
-  };
-};
-
-BlockService.prototype.getConcurrentTipOperation = function(block, add) {
-  var heightBuffer = new Buffer(4);
-  var tipData;
-  if(add) {
-    heightBuffer.writeUInt32BE(block.__height);
-    tipData = Buffer.concat([new Buffer(block.hash, 'hex'), heightBuffer]);
-  } else {
-    heightBuffer.writeUInt32BE(block.__height - 1);
-    tipData = Buffer.concat([BufferUtil.reverse(block.header.prevHash), heightBuffer]);
-  }
 
   return {
     type: 'put',
-    key: this.dbPrefix + 'concurrentTip',
+    key: this.dbPrefix + type,
     value: tipData
   };
 };
 
 BlockService.prototype.getBlock = function(height, callback) {
-  //if our block service's tip is ahead of the network tip, then we need to
+
-  //watch for a reorg
+  var self = this;
-  this.bitcoind.getBlock(height, callback);
+  if (self.tip.__height >= self.bitcoind.height) {
+
+    // if our block service's tip is ahead of the network tip, then we need to
+    // watch for a reorg by getting what we have for the tip hash and comparing it to
+    // what the network has.
+    return self.db.get(self.encoding.encodeBlockHeightKey(height), function(err, hash) {
+
+      if(err) {
+        return callback(err);
+      }
+
+      self.bitcoind.getBlock(height, function(res, block) {
+
+        if(err) {
+          return callback(err);
+        }
+
+        //oh noes! reorg sitch
+        if (hash !== block.hash) {
+
+          callback('reorg');
+          return self._handleReorg(block.hash, function() {
+            self._blockHandler.sync();
+          });
+
+        }
+
+        callback(null, block);
+      });
+
+
+    });
+  }
+
+  self.bitcoind.getBlock(height, callback);
+
 };
 
 
@@ -435,4 +458,17 @@ BlockService.prototype.getBlockHash = function(height, callback) {
   });
 };
 
+BlockService.prototype.getBlockHeight = function(hash, callback) {
+  var self = this;
+  self.db.get(this.encoding.encodeBlockHashKey(hash), function(err, heightBuf) {
+    if (err instanceof levelup.errors.NotFoundError) {
+      return callback();
+    }
+    if (err) {
+      return callback(err);
+    }
+    callback(null, self.encoding.decodeBlockHeightValue(heightBuf));
+  });
+};
+
 module.exports = BlockService;

lib/services/block/reorg.js

@@ -68,7 +68,7 @@ Reorg.prototype.rewindConcurrentTip = function(commonAncestor, callback) {
           return next(err);
         }
 
-        operations.push(self.block.getConcurrentTipOperation(self.block.concurrentTip, false));
+        operations.push(self.block.getTipOperation(self.block.concurrentTip, false, 'concurrentTip'));
         self.block.batch(operations, function(err) {
           if(err) {
             return next(err);
@@ -107,7 +107,7 @@ Reorg.prototype.fastForwardConcurrentTip = function(newHashes, callback) {
           return next(err);
         }
 
-        operations.push(self.block.getConcurrentTipOperation(block, true));
+        operations.push(self.block.getTipOperation(block, true, 'concurrentTip'));
         self.block.batch(operations, function(err) {
           if(err) {
             return next(err);
@@ -136,7 +136,7 @@ Reorg.prototype.rewindBothTips = function(commonAncestor, callback) {
               if(err) {
                 return next(err);
               }
-              operations.push(self.block.getConcurrentTipOperation(self.block.concurrentTip, false));
+              operations.push(self.block.getTipOperation(self.block.concurrentTip, false, 'concurrentTip'));
               next(null, operations);
             });
           },
@@ -198,7 +198,7 @@ Reorg.prototype.fastForwardBothTips = function(newHashes, callback) {
                 return next(err);
               }
 
-              operations.push(self.block.getConcurrentTipOperation(block, true));
+              operations.push(self.block.getTipOperation(block, true, 'concurrentTip'));
               next(null, operations);
             });
           },

regtest/block.js

@@ -47,9 +47,9 @@ var bitcore = {
       services: [
         'bitcoind',
         'db',
-        'block',
         'web',
-        'block.regtest',
+        'block',
+        'block-test'
       ],
       servicesConfig: {
         bitcoind: {
@@ -62,6 +62,9 @@ var bitcore = {
               zmqpubrawtx: bitcoin.args.zmqpubrawtx
             }
           ]
+        },
+        'block-test': {
+          requirePath: path.resolve(__dirname + '/test_web.js')
         }
       }
     }
@@ -111,8 +114,55 @@ describe('Block Operations', function() {
       ], done);
     });
 
-    it('should sync block headers', function(done) {
+    it('should sync block hashes as keys and heights as values', function(done) {
-      done();
+
+      async.timesLimit(opts.initialHeight + 1, 12, function(n, next) {
+        utils.queryBitcoreNode(Object.assign({
+          path: '/test/hash/' + n
+        }, bitcore.httpOpts), function(err, res) {
+
+          if(err) {
+            return done(err);
+          }
+          res = JSON.parse(res);
+          expect(res.height).to.equal(n);
+          expect(res.hash.length).to.equal(64);
+          next(null, res.hash);
+        });
+      }, function(err, hashes) {
+
+        if(err) {
+          return done(err);
+        }
+        self.hashes = hashes;
+        done();
+
+      });
+    });
+
+    it('should sync block heights as keys and hashes as values', function(done) {
+      async.timesLimit(opts.initialHeight + 1, 12, function(n, next) {
+        utils.queryBitcoreNode(Object.assign({
+          path: '/test/height/' + self.hashes[n]
+        }, bitcore.httpOpts), function(err, res) {
+
+          if(err) {
+            return done(err);
+          }
+          res = JSON.parse(res);
+          expect(res.height).to.equal(n);
+          expect(res.hash).to.equal(self.hashes[n]);
+          next();
+        });
+      }, function(err) {
+
+        if(err) {
+          return done(err);
+        }
+        done();
+
+      });
+
     });
   });
 

regtest/db.js

@@ -16,7 +16,7 @@ var BufferUtil = bitcore.util.buffer;
    Bitcoind does not need to be started or run
 */
 
-var debug = false;
+var debug = true;
 var bitcoreDataDir = '/tmp/bitcore';
 var pubSocket;
 var rpcServer;
@@ -41,12 +41,9 @@ var bitcore = {
       services: [
         'bitcoind',
         'db',
-        'transaction',
+        'web',
-        'timestamp',
+        'block',
-        'address',
+        'reorg-test'
-        'mempool',
-        'wallet-api',
-        'web'
       ],
       servicesConfig: {
         bitcoind: {
@@ -59,7 +56,8 @@ var bitcore = {
               zmqpubrawtx: 'tcp://127.0.0.1:38332'
             }
           ]
-        }
+        },
+        'reorg-test': { requirePath: path.resolve(__dirname + '/test_web.js') }
       }
     }
   },
@@ -96,9 +94,11 @@ describe('DB Operations', function() {
 
     var responses = [
       genesis.hash,
-      { hash: genesis.hash, height: 0 },
+      { height: 0, hash: genesis.hash },
       genesis.hash,
-      blocks.genesis, //end initChain
+      blocks.genesis,
+      genesis.hash,
+      { height: 0, hash: genesis.hash, previousblockhash: new Array(65).join('0') },
       block1.hash,
       blocks.block1a,
       { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block1.header.prevHash).toString('hex') },
@@ -108,7 +108,6 @@ describe('DB Operations', function() {
       { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block1.header.prevHash).toString('hex') },
       { height: 1, hash: block2.header.hash, previousblockhash: BufferUtil.reverse(block2.header.prevHash).toString('hex') },
       blocks.genesis,
-      { height: 0, hash: genesis.hash },
       blocks.block1b,
       { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block2.header.prevHash).toString('hex') },
     ];
@@ -136,9 +135,9 @@ describe('DB Operations', function() {
 
         req.on('end', function() {
           var body = JSON.parse(data);
-          //console.log('request', body);
+          console.log('request', body);
-          var response = JSON.stringify({ result: responses[responseCount++] });
+          var response = JSON.stringify({ result: responses[responseCount++], count: responseCount });
-          //console.log('response', response, 'id: ', body.id);
+          console.log('response', response, 'id: ', body.id);
           res.write(response);
           res.end();
         });

regtest/test_web.js

@@ -0,0 +1,42 @@
+var BaseService = require('../lib/service');
+var inherits = require('util').inherits;
+
+var TestWebService = function(options) {
+  BaseService.call(this, options);
+};
+
+inherits(TestWebService, BaseService);
+
+TestWebService.dependencies = ['web', 'block'];
+
+TestWebService.prototype.start = function(callback) {
+  callback();
+};
+
+TestWebService.prototype.stop = function(callback) {
+  callback();
+};
+
+TestWebService.prototype.setupRoutes = function(app) {
+
+  var self = this;
+
+  app.get('/hash/:height', function(req, res) {
+    self.node.services.block.getBlockHash(req.params.height, function(err, hash) {
+      res.status(200).jsonp({ hash: hash, height: parseInt(req.params.height) });
+    });
+  });
+
+  app.get('/height/:hash', function(req, res) {
+    self.node.services.block.getBlockHeight(req.params.hash, function(err, height) {
+      res.status(200).jsonp({ hash: req.params.hash, height: height });
+    });
+  });
+
+};
+
+TestWebService.prototype.getRoutePrefix = function() {
+  return 'test';
+};
+
+module.exports = TestWebService;