From cd043df59a52e1f3810bd015ea42926ce64833ae Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Mon, 27 Apr 2015 16:15:18 -0300 Subject: [PATCH 1/4] add network reporting to /v1/node --- api/config/default.yml | 8 ++++---- api/config/livenet.yml | 10 +++++++++- api/config/testnet.yml | 12 ++++++++++-- api/index.js | 4 ++++ api/test/v1/node.js | 3 ++- lib/node.js | 1 + 6 files changed, 30 insertions(+), 8 deletions(-) diff --git a/api/config/default.yml b/api/config/default.yml index 7d5332ac..35e47ea2 100644 --- a/api/config/default.yml +++ b/api/config/default.yml @@ -2,15 +2,15 @@ BitcoreHTTP: port: 8080 logging: true BitcoreNode: - LevelUp: ./testnet-db - network: testnet + LevelUp: ./db + network: livenet NetworkMonitor: host: localhost - port: 18333 + port: 8333 Reporter: none # none, simple, matrix RPC: user: user pass: password protocol: http host: 127.0.0.1 - port: 18332 + port: 8332 diff --git a/api/config/livenet.yml b/api/config/livenet.yml index 1ba65a51..35e47ea2 100644 --- a/api/config/livenet.yml +++ b/api/config/livenet.yml @@ -2,7 +2,15 @@ BitcoreHTTP: port: 8080 logging: true BitcoreNode: + LevelUp: ./db + network: livenet NetworkMonitor: - network: livenet host: localhost port: 8333 + Reporter: none # none, simple, matrix + RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 8332 diff --git a/api/config/testnet.yml b/api/config/testnet.yml index 1ba65a51..7d5332ac 100644 --- a/api/config/testnet.yml +++ b/api/config/testnet.yml @@ -2,7 +2,15 @@ BitcoreHTTP: port: 8080 logging: true BitcoreNode: + LevelUp: ./testnet-db + network: testnet NetworkMonitor: - network: livenet host: localhost - port: 8333 + port: 18333 + Reporter: none # none, simple, matrix + RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 18332 diff --git a/api/index.js b/api/index.js index d0a21153..e6759cc5 100755 --- a/api/index.js +++ b/api/index.js @@ -1,9 +1,13 @@ 'use strict'; var BitcoreHTTP = require('./lib/http'); +var bitcore = require('bitcore'); if (require.main === module) { var config = require('config'); + var network = config.get('BitcoreHTTP.BitcoreNode').network; + console.log('Starting bitcore-node-http', network, 'network'); + bitcore.Networks.defaultNetwork = bitcore.Networks.get(network); var http = BitcoreHTTP.create(config.get('BitcoreHTTP')); http.start(); } diff --git a/api/test/v1/node.js b/api/test/v1/node.js index ff45b99c..16ebcad6 100644 --- a/api/test/v1/node.js +++ b/api/test/v1/node.js @@ -19,7 +19,8 @@ describe('BitcoreHTTP v1 node routes', function() { nodeMock.status = { sync: 0.75, peerCount: 8, - version: 'test' + version: 'test', + network: 'test', }; nodeMock.getStatus = function() { return Promise.resolve(nodeMock.status); diff --git a/lib/node.js b/lib/node.js index 14ae583d..b0965207 100644 --- a/lib/node.js +++ b/lib/node.js @@ -167,6 +167,7 @@ BitcoreNode.prototype.getStatus = function() { sync: this.getSyncProgress(), peerCount: this.networkMonitor.getConnectedPeers(), version: pjson.version, + network: bitcore.Networks.defaultNetwork.name, }); }; From 68a9e929558297e4dcdab7fb2ba3c021146c7cb4 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 28 Apr 2015 12:33:46 -0300 Subject: [PATCH 2/4] add rpc probing --- api/config/livenet.yml | 14 ++++---- api/config/testnet.yml | 14 ++++---- api/controllers/blocks.js | 2 +- api/index.js | 6 +++- api/lib/http.js | 6 +++- api/test/v1/blocks.js | 4 +-- config/default.yml | 14 ++++---- index.js | 4 ++- lib/networkmonitor.js | 5 ++- lib/node.js | 43 +++++++++++++++++------- lib/services/block.js | 69 +++++++++++++++++++++++++++++++-------- 11 files changed, 127 insertions(+), 54 deletions(-) diff --git a/api/config/livenet.yml b/api/config/livenet.yml index 35e47ea2..9caa99c9 100644 --- a/api/config/livenet.yml +++ b/api/config/livenet.yml @@ -7,10 +7,10 @@ BitcoreHTTP: NetworkMonitor: host: localhost port: 8333 - Reporter: none # none, simple, matrix - RPC: - user: user - pass: password - protocol: http - host: 127.0.0.1 - port: 8332 + Reporter: none # none, simple, matrix + RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 8332 diff --git a/api/config/testnet.yml b/api/config/testnet.yml index 7d5332ac..9b91773b 100644 --- a/api/config/testnet.yml +++ b/api/config/testnet.yml @@ -7,10 +7,10 @@ BitcoreHTTP: NetworkMonitor: host: localhost port: 18333 - Reporter: none # none, simple, matrix - RPC: - user: user - pass: password - protocol: http - host: 127.0.0.1 - port: 18332 + Reporter: none # none, simple, matrix + RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 18332 diff --git a/api/controllers/blocks.js b/api/controllers/blocks.js index 01aab381..8eafd3ea 100644 --- a/api/controllers/blocks.js +++ b/api/controllers/blocks.js @@ -76,7 +76,7 @@ Blocks.list = function(req, res) { // TODO: add more parameter validation // TODO: return block_summary instead of block_full - node.listBlocks(from, to, offset, limit) + node.blockService.listBlocks(from, to, offset, limit) .then(function(blocks) { res.send(blocks); }); diff --git a/api/index.js b/api/index.js index e6759cc5..0d542cc6 100755 --- a/api/index.js +++ b/api/index.js @@ -9,7 +9,11 @@ if (require.main === module) { console.log('Starting bitcore-node-http', network, 'network'); bitcore.Networks.defaultNetwork = bitcore.Networks.get(network); var http = BitcoreHTTP.create(config.get('BitcoreHTTP')); - http.start(); + http.start() + .catch(function(err) { + http.stop(); + throw err; + }); } module.exports = BitcoreHTTP; diff --git a/api/lib/http.js b/api/lib/http.js index e56b61f8..c946c43a 100644 --- a/api/lib/http.js +++ b/api/lib/http.js @@ -98,8 +98,12 @@ BitcoreHTTP.prototype.onListening = function() { BitcoreHTTP.prototype.start = function() { - this.node.start(); this.server.listen(this.port); + return this.node.start(); +}; + +BitcoreHTTP.prototype.stop = function() { + return this.node.stop(); }; module.exports = BitcoreHTTP; diff --git a/api/test/v1/blocks.js b/api/test/v1/blocks.js index 5cf29082..18a541f0 100644 --- a/api/test/v1/blocks.js +++ b/api/test/v1/blocks.js @@ -49,13 +49,13 @@ describe('BitcoreHTTP v1 blocks routes', function() { nodeMock.blockService.getLatest = function() { return Promise.resolve(lastBlock); }; - nodeMock.listBlocks = function(from, to, offset, limit) { + nodeMock.blockService.listBlocks = function(from, to, offset, limit) { var start = from - 1e5; var end = to - 1e5; var section = blockList.slice(start, end); return Promise.resolve(section.slice(offset, offset + limit)); }; - app = new BitcoreHTTP(nodeMock).app; + app = require('../app')(nodeMock); agent = request(app); }); diff --git a/config/default.yml b/config/default.yml index 564f1073..d0285e53 100644 --- a/config/default.yml +++ b/config/default.yml @@ -4,10 +4,10 @@ BitcoreNode: NetworkMonitor: host: localhost port: 8333 -Reporter: none # none, simple, matrix -RPC: - user: user - pass: password - protocol: http - host: 127.0.0.1 - port: 8332 + Reporter: none # none, simple, matrix + RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 8332 diff --git a/index.js b/index.js index 8ffbaab9..974a31c0 100644 --- a/index.js +++ b/index.js @@ -10,7 +10,9 @@ BitcoreNode.errors = require('./lib/errors'); if (require.main === module) { var config = require('config'); - bitcore.Networks.defaultNetwork = bitcore.Networks.get(config.get('BitcoreNode').network); + var network = config.get('BitcoreHTTP.BitcoreNode').network; + console.log('Starting bitcore-node', network, 'network'); + bitcore.Networks.defaultNetwork = bitcore.Networks.get(network); var node = BitcoreNode.create(config.get('BitcoreNode')); node.start(); diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index fa2bff76..46f471fd 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -95,7 +95,10 @@ NetworkMonitor.prototype.getConnectedPeers = function() { }; NetworkMonitor.prototype.abort = function(reason) { - this.peer.disconnect(); + // TODO: improve Peer interface to know if it's connected + if (this.peer.socket) { + this.peer.disconnect(); + } if (reason) { throw reason; } diff --git a/lib/node.js b/lib/node.js index b0965207..b24f9790 100644 --- a/lib/node.js +++ b/lib/node.js @@ -78,7 +78,7 @@ BitcoreNode.prototype.initialize = function() { var prevHeight = 0; var statTimer = 5 * 1000; - setInterval(function() { + this.interval = setInterval(function() { console.log('MB used:', process.memoryUsage().heapTotal / 1024 / 1024, 100 * self.getSyncProgress() + '% synced'); if (!self.blockchain) { @@ -146,20 +146,40 @@ BitcoreNode.prototype.start = function() { var self = this; var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); - this.blockService.getBlockchain().then(function(blockchain) { - if (!blockchain) { - self.blockchain = new BlockChain(); - self.bus.process(genesis); - } else { - self.blockchain = blockchain; - } - self.sync(); - self.networkMonitor.start(); - }); + return this.probeRPC() + .catch(function(err) { + console.log('RPC connection unsuccessful. Please check your configuration'); + throw err; + }) + .then(function() { + return self.blockService.getBlockchain(); + }) + .then(function(blockchain) { + if (!blockchain) { + self.blockchain = new BlockChain(); + self.bus.process(genesis); + } else { + self.blockchain = blockchain; + } + self.sync(); + return self.networkMonitor.start(); + }); }; BitcoreNode.prototype.stop = function(reason) { + clearInterval(this.interval); this.networkMonitor.abort(reason); + return this.blockService.database.closeAsync(); +}; + + +BitcoreNode.prototype.probeRPC = function() { + // TODO: nicer way to do this? + console.log('Probing RPC connection to check health...'); + return this.blockService.rpc.getBlockHashAsync(1) + .then(function() { + return true; + }); }; BitcoreNode.prototype.getStatus = function() { @@ -178,7 +198,6 @@ BitcoreNode.prototype.getSyncProgress = function() { BitcoreNode.prototype._requestFromTip = function() { var locator = this.blockchain.getBlockLocator(); - //console.log('requesting blocks, locator size:', locator.length); this.networkMonitor.requestBlocks(locator); }; diff --git a/lib/services/block.js b/lib/services/block.js index d6b966e4..1f503177 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -4,24 +4,19 @@ var config = require('config'); var LevelUp = require('levelup'); var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); -var TransactionService = require('./transaction'); var bitcore = require('bitcore'); -var Transaction = bitcore.Transaction; var BufferUtil = bitcore.util.buffer; +var Block = bitcore.Block; var errors = require('../errors'); var BlockChain = require('../blockchain'); +var genesisBlocks = require('../data/genesis'); +var TransactionService = require('./transaction'); var $ = bitcore.util.preconditions; var JSUtil = bitcore.util.js; var _ = bitcore.deps._; -var NULLBLOCKHASH = bitcore.util.buffer.emptyBuffer(32).toString('hex'); -var GENESISPARENT = { - height: -1, - prevBlockHash: NULLBLOCKHASH -}; - var helper = function(index) { return function(maybeHash) { if (_.isString(maybeHash)) { @@ -110,11 +105,15 @@ BlockService.blockRPCtoBitcore = function(blockData) { /** * A helper function to return an error when a block couldn't be found * - * @param {*} err + * @param {*} err an error message or the block hash * @return {Promise} a promise that will always be rejected */ var blockNotFound = function(err) { - throw new errors.Blocks.NotFound(err); + if (err) { + throw err; + } + var hash = err; + throw new errors.Blocks.NotFound(hash); }; /** @@ -167,21 +166,63 @@ BlockService.prototype.getBlock = function(blockHash, opts) { * @return {Promise} */ BlockService.prototype.getBlockByHeight = function(height) { - $.checkArgument(_.isNumber(height), 'Block height must be a number'); var self = this; + if (height === 0) { + return Promise.resolve( + Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork]) + ); + } return Promise.try(function() { - return self.rpc.getBlockHashAsync(height); - }) .catch(blockNotFound) .then(function(result) { - var blockHash = result.result; return self.getBlock(blockHash); + }); +}; + +/** + * Returns a list of blocks given certain required query options. + * + * @param {Number} from block height as lower limit + * @param {Number} to ditto, but for the upper limit, non inclusive + * @param {Number} offset skip the first offset blocks + * @param {Number} limit max amount of blocks returned + * + */ +BlockService.prototype.listBlocks = function(from, to, offset, limit) { + $.checkArgument(_.isNumber(from), 'from is required, and must be a number'); + $.checkArgument(_.isNumber(to), 'to is required, and must be a number'); + $.checkArgument(_.isNumber(offset), 'offset is required, and must be a number'); + $.checkArgument(_.isNumber(limit), 'limit is required, and must be a number'); + $.checkArgument(from <= to, 'from must be <= to'); + + var self = this; + var start = from + offset; + var end = Math.min(to, start + limit - 1); + var blocks = []; + var fetchBlock = function(height) { + if (height > end) { + return; + } + console.log('fetching block', height); + return self.getBlockByHeight(height) + .then(function(block) { + if (!block) { + // TODO: report? + return; + } + blocks.push(block); + return fetchBlock(height + 1); + }); + }; + return fetchBlock(start) + .then(function() { + return blocks; }); }; From 50413c8204c404c77f301bf7e9a1ee72dea9bd1d Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 28 Apr 2015 19:39:29 -0300 Subject: [PATCH 3/4] fix edge case for listBlocks --- api/test/app.js | 7 +++++++ lib/services/block.js | 10 +++++----- 2 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 api/test/app.js diff --git a/api/test/app.js b/api/test/app.js new file mode 100644 index 00000000..b0443d8e --- /dev/null +++ b/api/test/app.js @@ -0,0 +1,7 @@ +'use strict'; + +var BitcoreHTTP = require('../lib/http'); + +module.exports = function(nodeMock) { + return process.env.INTEGRATION === 'true' ? BitcoreHTTP.create().app : new BitcoreHTTP(nodeMock).app; +}; diff --git a/lib/services/block.js b/lib/services/block.js index 1f503177..d52f1f42 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -205,6 +205,7 @@ BlockService.prototype.listBlocks = function(from, to, offset, limit) { var start = from + offset; var end = Math.min(to, start + limit - 1); var blocks = []; + // TODO: optimize: precompute heights and fetch all blocks in parallel? var fetchBlock = function(height) { if (height > end) { return; @@ -212,12 +213,11 @@ BlockService.prototype.listBlocks = function(from, to, offset, limit) { console.log('fetching block', height); return self.getBlockByHeight(height) .then(function(block) { - if (!block) { - // TODO: report? - return; - } - blocks.push(block); + blocks.push(block.toObject()); return fetchBlock(height + 1); + }) + .catch(function(err) { + console.log(err); }); }; return fetchBlock(start) From d2e25db29e1de3ee1a0194cfa8d0a6c167540db7 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 28 Apr 2015 19:43:42 -0300 Subject: [PATCH 4/4] fix tests --- test/node.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/node.js b/test/node.js index 8170c1f7..01075a34 100644 --- a/test/node.js +++ b/test/node.js @@ -23,6 +23,10 @@ describe('BitcoreNode', function() { bsMock.getBlockchain = function() { return Promise.resolve(chainMock); }; + bsMock.rpc = {}; + bsMock.rpc.getBlockHashAsync = function() { + return Promise.resolve(true); + }; tsMock = {}; asMock = {}; node = new BitcoreNode(busMock, nmMock, bsMock, tsMock, asMock);