diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 21e93f47..3f683608 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -7,6 +7,7 @@ var bitcore = require('bitcore-lib'); var _ = bitcore.deps._; var log = index.log; var shuttingDown = false; +var fs = require('fs'); log.debug = function() {}; @@ -126,6 +127,22 @@ function checkService(service) { } } +function lookInRequirePathConfig(req, service) { + if (!service.config.requirePath) { + return; + } + //if a file, then remove any possible '.js' extension + //if a directory, then leave alone + try { + if (fs.statSync(service.config.requirePath).isDirectory()) { + return req(service.config.requirePath); + } + var serviceFile = service.config.requirePath.replace(/.js$/, ''); + return req(serviceFile); + } catch(e) { + } +} + function lookInCwd(req, service) { try { return req(process.cwd + '/' + service); @@ -165,15 +182,20 @@ function lookInModuleManifest(req, service) { function loadModule(req, service) { var serviceCode; - //first look in the current working directory (of the controlling terminal, if there is one) for the service code - serviceCode = lookInCwd(req, service); + //first, if we have explicitly set the require path for our service, use this. + serviceCode = lookInRequirePathConfig(req, service); - //second try the built-in services + //second, look in the current working directory (of the controlling terminal, if there is one) for the service code + if(!serviceCode) { + serviceCode = lookInCwd(req, service); + } + + //third, try the built-in services if(!serviceCode) { serviceCode = lookInBuiltInPath(req, service); } - //third see if there is directory in our module search path that has a + //fourth, see if there is directory in our module search path that has a //package.json file, if so, then see if there is a bitcoreNode field, if so //use this as the path to the service module if(!serviceCode) { @@ -181,7 +203,8 @@ function loadModule(req, service) { } if (!serviceCode) { - throw new Error('Attempted to load the ' + service.name + ' service from: "' + + throw new Error('Attempted to load the ' + service.name + ' service from: ' + + 'the requirePath in the services\' config, then "' + process.cwd() + '" then from: "' + __dirname + '/../lib/services' + '" finally from: "' + process.cwd() + '/package.json" - bitcoreNode field. All paths failed to find valid nodeJS code.'); } @@ -213,6 +236,7 @@ function setupServices(req, servicesPath, config) { if (config.services) { for (var i = 0; i < config.services.length; i++) { var service = {}; + service.name = config.services[i]; var hasConfig = config.servicesConfig && config.servicesConfig[service.name]; diff --git a/lib/services/block/block_handler.js b/lib/services/block/block_handler.js index d5f121ea..fd9b3198 100644 --- a/lib/services/block/block_handler.js +++ b/lib/services/block/block_handler.js @@ -174,6 +174,10 @@ BlockStream.prototype._getBlocks = function(heights, callback) { self.block.getBlock(height, function(err, block) { + if (err === 'reorg') { + return self.push(null); + } + if(err) { return next(err); } @@ -273,7 +277,7 @@ ProcessConcurrent.prototype._transform = function(block, enc, callback) { self.operations = self.operations.concat(operations); if(self.blockCount >= 1) { - self.operations.push(self.block.getConcurrentTipOperation(block, true)); + self.operations.push(self.block.getTipOperation(block, true, 'concurrentTip')); var obj = { concurrentTip: block, operations: self.operations @@ -290,7 +294,7 @@ ProcessConcurrent.prototype._transform = function(block, enc, callback) { ProcessConcurrent.prototype._flush = function(callback) { if(this.operations.length) { - this.operations.push(this.block.getConcurrentTipOperation(this.lastBlock, true)); + this.operations.push(this.block.getTipOperation(this.lastBlock, true)); this.operations = []; return callback(null, this.operations); } @@ -323,7 +327,7 @@ ProcessBoth.prototype._write = function(block, encoding, callback) { if(err) { return callback(err); } - operations.push(self.block.getConcurrentTipOperation(block, true)); + operations.push(self.block.getTipOperation(block, true, 'concurrentTip')); next(null, operations); }); }, function(next) { diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 2bdd31ad..c1ab2f6a 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -50,7 +50,7 @@ BlockService.prototype._startSubscriptions = function() { self.bus = self.node.openBus({remoteAddress: 'localhost'}); self.bus.on('bitcoind/hashblock', function() { - self.sync(); + self._blockHandler.sync(); }); self.bus.subscribe('bitcoind/hashblock'); @@ -102,7 +102,7 @@ BlockService.prototype._sync = function() { BlockService.prototype._getBlocks = function(callback) { var self = this; - var blocksDiff = self.bitcoind.height - self.tip.__height - 1; + var blocksDiff = self.bitcoind.height - self.tip.__height; if (blocksDiff < 0) { self._log('Peer\'s height is less than our own. The peer may be syncing.' + @@ -118,7 +118,7 @@ BlockService.prototype._getBlocks = function(callback) { async.timesLimit(blocksDiff, 8, function(n, next) { - var blockNumber = n + self.tip.__height + 2; + var blockNumber = n + self.tip.__height + 1; self.bitcoind.getBlockHeader(blockNumber, function(err, header) { if(err) { @@ -201,28 +201,31 @@ BlockService.prototype._detectReorg = function(callback) { return callback(); } + // all synced if (self.tip.hash === self.bitcoind.tiphash && self.tip.__height === self.bitcoind.height) { return callback(); } + // check if our tip height has the same hash as the network's self.bitcoind.getBlockHeader(self.tip.__height, function(err, header) { if(err) { return callback(err); } - //we still might have a reorg, but later + // we still might have a reorg if our tip is greater than the network's + // we won't know about this until we start syncing if (header.hash === self.tip.hash) { return callback(); } - //our hash isn't in the network anymore, we have definitely reorg'ed - callback(header, callback); + //our hash isn't in the network chain anymore, we have reorg'ed + self._handleReorg(header.hash, callback); }); }; -BlockService.prototype._handleReorg = function(header, callback) { +BlockService.prototype._handleReorg = function(hash, callback) { var self = this; self.printTipInfo('Reorg detected!'); @@ -231,7 +234,7 @@ BlockService.prototype._handleReorg = function(header, callback) { var reorg = new Reorg(self.node, self); - reorg.handleReorg(header.hash, function(err) { + reorg.handleReorg(hash, function(err) { if(err) { self._log('Reorg failed! ' + err, log.error); @@ -378,11 +381,12 @@ BlockService.prototype.getSerialBlockOperations = function(block, add, callback) ); }; -BlockService.prototype.getTipOperation = function(block, add) { +BlockService.prototype.getTipOperation = function(block, add, tipType) { + var heightBuffer = new Buffer(4); var tipData; - if(add) { + if (add) { heightBuffer.writeUInt32BE(block.__height); tipData = Buffer.concat([new Buffer(block.hash, 'hex'), heightBuffer]); } else { @@ -390,35 +394,54 @@ BlockService.prototype.getTipOperation = function(block, add) { tipData = Buffer.concat([BufferUtil.reverse(block.header.prevHash), heightBuffer]); } - return { - type: 'put', - key: this.dbPrefix + 'tip', - value: tipData - }; -}; - -BlockService.prototype.getConcurrentTipOperation = function(block, add) { - var heightBuffer = new Buffer(4); - var tipData; - if(add) { - heightBuffer.writeUInt32BE(block.__height); - tipData = Buffer.concat([new Buffer(block.hash, 'hex'), heightBuffer]); - } else { - heightBuffer.writeUInt32BE(block.__height - 1); - tipData = Buffer.concat([BufferUtil.reverse(block.header.prevHash), heightBuffer]); - } + var type = tipType || 'tip'; return { type: 'put', - key: this.dbPrefix + 'concurrentTip', + key: this.dbPrefix + type, value: tipData }; }; BlockService.prototype.getBlock = function(height, callback) { - //if our block service's tip is ahead of the network tip, then we need to - //watch for a reorg - this.bitcoind.getBlock(height, callback); + + var self = this; + if (self.tip.__height >= self.bitcoind.height) { + + // if our block service's tip is ahead of the network tip, then we need to + // watch for a reorg by getting what we have for the tip hash and comparing it to + // what the network has. + return self.db.get(self.encoding.encodeBlockHeightKey(height), function(err, hash) { + + if(err) { + return callback(err); + } + + self.bitcoind.getBlock(height, function(res, block) { + + if(err) { + return callback(err); + } + + //oh noes! reorg sitch + if (hash !== block.hash) { + + callback('reorg'); + return self._handleReorg(block.hash, function() { + self._blockHandler.sync(); + }); + + } + + callback(null, block); + }); + + + }); + } + + self.bitcoind.getBlock(height, callback); + }; @@ -435,4 +458,17 @@ BlockService.prototype.getBlockHash = function(height, callback) { }); }; +BlockService.prototype.getBlockHeight = function(hash, callback) { + var self = this; + self.db.get(this.encoding.encodeBlockHashKey(hash), function(err, heightBuf) { + if (err instanceof levelup.errors.NotFoundError) { + return callback(); + } + if (err) { + return callback(err); + } + callback(null, self.encoding.decodeBlockHeightValue(heightBuf)); + }); +}; + module.exports = BlockService; diff --git a/lib/services/block/reorg.js b/lib/services/block/reorg.js index d981ba0e..da864d61 100644 --- a/lib/services/block/reorg.js +++ b/lib/services/block/reorg.js @@ -68,7 +68,7 @@ Reorg.prototype.rewindConcurrentTip = function(commonAncestor, callback) { return next(err); } - operations.push(self.block.getConcurrentTipOperation(self.block.concurrentTip, false)); + operations.push(self.block.getTipOperation(self.block.concurrentTip, false, 'concurrentTip')); self.block.batch(operations, function(err) { if(err) { return next(err); @@ -107,7 +107,7 @@ Reorg.prototype.fastForwardConcurrentTip = function(newHashes, callback) { return next(err); } - operations.push(self.block.getConcurrentTipOperation(block, true)); + operations.push(self.block.getTipOperation(block, true, 'concurrentTip')); self.block.batch(operations, function(err) { if(err) { return next(err); @@ -136,7 +136,7 @@ Reorg.prototype.rewindBothTips = function(commonAncestor, callback) { if(err) { return next(err); } - operations.push(self.block.getConcurrentTipOperation(self.block.concurrentTip, false)); + operations.push(self.block.getTipOperation(self.block.concurrentTip, false, 'concurrentTip')); next(null, operations); }); }, @@ -198,7 +198,7 @@ Reorg.prototype.fastForwardBothTips = function(newHashes, callback) { return next(err); } - operations.push(self.block.getConcurrentTipOperation(block, true)); + operations.push(self.block.getTipOperation(block, true, 'concurrentTip')); next(null, operations); }); }, diff --git a/regtest/block.regtest.js b/regtest/block.js similarity index 63% rename from regtest/block.regtest.js rename to regtest/block.js index 6db0b2a2..0f088181 100644 --- a/regtest/block.regtest.js +++ b/regtest/block.js @@ -47,9 +47,9 @@ var bitcore = { services: [ 'bitcoind', 'db', - 'block', 'web', - 'block.regtest', + 'block', + 'block-test' ], servicesConfig: { bitcoind: { @@ -62,6 +62,9 @@ var bitcore = { zmqpubrawtx: bitcoin.args.zmqpubrawtx } ] + }, + 'block-test': { + requirePath: path.resolve(__dirname + '/test_web.js') } } } @@ -111,8 +114,55 @@ describe('Block Operations', function() { ], done); }); - it('should sync block headers', function(done) { - done(); + it('should sync block hashes as keys and heights as values', function(done) { + + async.timesLimit(opts.initialHeight + 1, 12, function(n, next) { + utils.queryBitcoreNode(Object.assign({ + path: '/test/hash/' + n + }, bitcore.httpOpts), function(err, res) { + + if(err) { + return done(err); + } + res = JSON.parse(res); + expect(res.height).to.equal(n); + expect(res.hash.length).to.equal(64); + next(null, res.hash); + }); + }, function(err, hashes) { + + if(err) { + return done(err); + } + self.hashes = hashes; + done(); + + }); + }); + + it('should sync block heights as keys and hashes as values', function(done) { + async.timesLimit(opts.initialHeight + 1, 12, function(n, next) { + utils.queryBitcoreNode(Object.assign({ + path: '/test/height/' + self.hashes[n] + }, bitcore.httpOpts), function(err, res) { + + if(err) { + return done(err); + } + res = JSON.parse(res); + expect(res.height).to.equal(n); + expect(res.hash).to.equal(self.hashes[n]); + next(); + }); + }, function(err) { + + if(err) { + return done(err); + } + done(); + + }); + }); }); diff --git a/regtest/db.js b/regtest/db.js index 8f3e7181..400fff4d 100644 --- a/regtest/db.js +++ b/regtest/db.js @@ -16,7 +16,7 @@ var BufferUtil = bitcore.util.buffer; Bitcoind does not need to be started or run */ -var debug = false; +var debug = true; var bitcoreDataDir = '/tmp/bitcore'; var pubSocket; var rpcServer; @@ -41,12 +41,9 @@ var bitcore = { services: [ 'bitcoind', 'db', - 'transaction', - 'timestamp', - 'address', - 'mempool', - 'wallet-api', - 'web' + 'web', + 'block', + 'reorg-test' ], servicesConfig: { bitcoind: { @@ -59,7 +56,8 @@ var bitcore = { zmqpubrawtx: 'tcp://127.0.0.1:38332' } ] - } + }, + 'reorg-test': { requirePath: path.resolve(__dirname + '/test_web.js') } } } }, @@ -96,9 +94,11 @@ describe('DB Operations', function() { var responses = [ genesis.hash, - { hash: genesis.hash, height: 0 }, + { height: 0, hash: genesis.hash }, genesis.hash, - blocks.genesis, //end initChain + blocks.genesis, + genesis.hash, + { height: 0, hash: genesis.hash, previousblockhash: new Array(65).join('0') }, block1.hash, blocks.block1a, { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block1.header.prevHash).toString('hex') }, @@ -108,7 +108,6 @@ describe('DB Operations', function() { { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block1.header.prevHash).toString('hex') }, { height: 1, hash: block2.header.hash, previousblockhash: BufferUtil.reverse(block2.header.prevHash).toString('hex') }, blocks.genesis, - { height: 0, hash: genesis.hash }, blocks.block1b, { height: 1, hash: block1.header.hash, previousblockhash: BufferUtil.reverse(block2.header.prevHash).toString('hex') }, ]; @@ -136,9 +135,9 @@ describe('DB Operations', function() { req.on('end', function() { var body = JSON.parse(data); - //console.log('request', body); - var response = JSON.stringify({ result: responses[responseCount++] }); - //console.log('response', response, 'id: ', body.id); + console.log('request', body); + var response = JSON.stringify({ result: responses[responseCount++], count: responseCount }); + console.log('response', response, 'id: ', body.id); res.write(response); res.end(); }); diff --git a/regtest/test_web.js b/regtest/test_web.js new file mode 100644 index 00000000..5d1acf55 --- /dev/null +++ b/regtest/test_web.js @@ -0,0 +1,42 @@ +var BaseService = require('../lib/service'); +var inherits = require('util').inherits; + +var TestWebService = function(options) { + BaseService.call(this, options); +}; + +inherits(TestWebService, BaseService); + +TestWebService.dependencies = ['web', 'block']; + +TestWebService.prototype.start = function(callback) { + callback(); +}; + +TestWebService.prototype.stop = function(callback) { + callback(); +}; + +TestWebService.prototype.setupRoutes = function(app) { + + var self = this; + + app.get('/hash/:height', function(req, res) { + self.node.services.block.getBlockHash(req.params.height, function(err, hash) { + res.status(200).jsonp({ hash: hash, height: parseInt(req.params.height) }); + }); + }); + + app.get('/height/:hash', function(req, res) { + self.node.services.block.getBlockHeight(req.params.hash, function(err, height) { + res.status(200).jsonp({ hash: req.params.hash, height: height }); + }); + }); + +}; + +TestWebService.prototype.getRoutePrefix = function() { + return 'test'; +}; + +module.exports = TestWebService;