diff --git a/README.md b/README.md index f4feeca..b733135 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ ## Requirements -Insight requires [Node.js](https://nodejs.org) and [MongoDB](https://www.mongodb.com/). Consider using [n](https://github.com/tj/n) and [m](https://github.com/aheckmann/m) to install the latest versions. +Insight requires [Node.js](https://nodejs.org) 8.2 and [MongoDB](https://www.mongodb.com/). Consider using [n](https://github.com/tj/n) and [m](https://github.com/aheckmann/m) to install the latest versions. ## Quick Start To get started, clone this repository, then – with `mongod` running – install and run insight: ```bash -git clone -b next https://github.com/bitpay/insight.git && cd insight +git clone -b next https://github.com/bitpay/insight.git && cd insight/server npm install npm start ``` diff --git a/server/config/index.js b/server/config/index.js index 2dd1526..4c45558 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -29,6 +29,7 @@ const config = { ticker_prop: 'bitstamp', max_blocks: 72, max_txs: 10, + request_ttl: 100000, }, }; diff --git a/server/lib/api/address.js b/server/lib/api/address.js index ed0cab9..722adb2 100644 --- a/server/lib/api/address.js +++ b/server/lib/api/address.js @@ -3,12 +3,16 @@ const request = require('request'); const config = require('../../config'); const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`; +const TTL = config.api.request_ttl; module.exports = function AddressAPI(router) { router.get('/addr/:addr', (req, res) => { const addr = req.params.addr || ''; + logger.log('debug', + 'Warning: Requesting data from Bcoin by address, may take some time'); // Get Bcoin data return request(`${API_URL}/tx/address/${addr}`, + { timeout: TTL }, (error, bcoinRes, bcoinTxs) => { if (error) { logger.log('error', diff --git a/server/lib/api/transaction.js b/server/lib/api/transaction.js index af5b7bd..d0f742b 100644 --- a/server/lib/api/transaction.js +++ b/server/lib/api/transaction.js @@ -6,6 +6,7 @@ const util = require('../util'); const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`; const MAX_TXS = config.api.max_txs; +const TTL = config.api.request_ttl; module.exports = function transactionAPI(router) { // Txs by txid @@ -17,57 +18,56 @@ module.exports = function transactionAPI(router) { } // Get max block height for calculating confirmations - db.blocks.getBestHeight( - (err, blockHeight) => { - if (err) { - logger.log('err', err); + const height = db.blocks.bestHeight(); + // Bcoin transaction data + return request(`${API_URL}/tx/${req.params.txid}`, + { timeout: TTL }, + (error, localRes, tx) => { + if (error) { + logger.log('error', + `${error}`); + return res.status(404).send(); + } + // Catch JSON errors + try { + tx = JSON.parse(tx); + } catch (e) { + logger.log('error', + `${e}`); + return res.status(404).send(); + } + if (!tx || !tx.hash) { + logger.log('error', + 'No results found'); return res.status(404).send(); } - const height = blockHeight; - // Bcoin transaction data - return request(`${API_URL}/tx/${req.params.txid}`, (error, localRes, tx) => { - if (error) { - logger.log('error', - `${error}`); - return res.status(404).send(); - } - // Catch JSON errors - try { - tx = JSON.parse(tx); - } catch (e) { - logger.log('error', - `${e}`); - return res.status(404).send(); - } - if (!tx || !tx.hash) { - logger.log('error', - 'No results found'); - return res.status(404).send(); - } - // Return UI JSON - return res.send({ - txid: tx.hash, - version: tx.version, - time: tx.ps, - blocktime: tx.ps, - locktime: tx.locktime, - blockhash: tx.block, - fees: tx.fee / 1e8, - confirmations: (height - tx.height) + 1, - valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, - vin: tx.inputs.map(input => ({ - addr: input.coin ? input.coin.address : '', - value: input.coin ? input.coin.value / 1e8 : 0, - })), - vout: tx.outputs.map(output => ({ - scriptPubKey: { - addresses: [output.address], - }, - value: output.value / 1e8, - })), - isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', - }); + // Return UI JSON + return res.send({ + txid: tx.hash, + version: tx.version, + time: tx.ps, + blocktime: tx.ps, + locktime: tx.locktime, + blockhash: tx.block, + fees: tx.fee / 1e8, + confirmations: (height - tx.height) + 1, + valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, + vin: tx.inputs.map(input => ({ + addr: input.coin ? input.coin.address : '', + value: input.coin ? input.coin.value / 1e8 : 0, + scriptSig: { + asm: input.script, + }, + })), + vout: tx.outputs.map(output => ({ + scriptPubKey: { + asm: output.script, + addresses: [output.address], + }, + value: output.value / 1e8, + })), + isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', }); }); }); @@ -86,58 +86,58 @@ module.exports = function transactionAPI(router) { return res.status(400).send({ error: 'Invalid block hash', }); - } - - db.blocks.getBestHeight( - (err, blockHeight) => { - if (err) { - logger.log('err', err); + } + const height = db.blocks.bestHeight(); + // Get Bcoin data + return request(`${API_URL}/block/${req.query.block}`, + { timeout: TTL }, + (error, localRes, block) => { + if (error) { + logger.log('error', + `${error}`); + return res.status(404).send(); + } + // Catch JSON errors + try { + block = JSON.parse(block); + } catch (e) { + logger.log('error', + `${e}`); return res.status(404).send(); } - const height = blockHeight; - // Get Bcoin data - return request(`${API_URL}/block/${req.query.block}`, (error, localRes, block) => { - if (error) { - logger.log('error', - `${error}`); - } - // Catch JSON errors - try { - block = JSON.parse(block); - } catch (e) { - logger.log('error', - `${e}`); - return res.status(404).send(); - } - if (!block.txs.length) { - logger.log('error', - `${'No tx results'}`); - res.status(404).send(); - } - // Setup UI JSON - const totalPages = Math.ceil(block.txs.length / MAX_TXS); - block.txs = block.txs.slice(rangeStart, rangeEnd); - return res.send({ - pagesTotal: totalPages, - txs: block.txs.map(tx => ({ - txid: tx.hash, - fees: tx.fee / 1e8, - confirmations: (height - block.height) + 1, - valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, - vin: tx.inputs.map(input => ({ - addr: input.coin ? input.coin.address : '', - value: input.coin ? input.coin.value / 1e8 : 0, - })), - vout: tx.outputs.map(output => ({ - scriptPubKey: { - addresses: [output.address], - }, - value: output.value / 1e8, - })), - isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', + if (block.error) { + logger.log('error', + `${'No tx results'}`); + return res.status(404).send(); + } + // Setup UI JSON + const totalPages = Math.ceil(block.txs.length / MAX_TXS); + block.txs = block.txs.slice(rangeStart, rangeEnd); + + return res.send({ + pagesTotal: totalPages, + txs: block.txs.map(tx => ({ + txid: tx.hash, + fees: tx.fee / 1e8, + confirmations: (height - block.height) + 1, + valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, + vin: tx.inputs.map(input => ({ + addr: input.coin ? input.coin.address : '', + value: input.coin ? input.coin.value / 1e8 : 0, + scriptSig: { + asm: input.script, + }, })), - }); + vout: tx.outputs.map(output => ({ + scriptPubKey: { + asm: output.script, + addresses: [output.address], + }, + value: output.value / 1e8, + })), + isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', + })), }); }); } else if (req.query.address) { @@ -148,94 +148,63 @@ module.exports = function transactionAPI(router) { } // Get txs by address, start with best height to calc confirmations - db.blocks.getBestHeight( - (err, blockHeight) => { - if (err) { - logger.log('err', err); + const height = db.blocks.bestHeight(); + const addr = req.query.address || ''; + + logger.log('debug', + 'Warning: Requesting data from Bcoin by address, may take some time'); + + return request(`${API_URL}/tx/address/${addr}`, + { timeout: TTL }, + (error, localRes, txs) => { + if (error) { + logger.log('error', + `${error}`); return res.status(404).send(); } - - const height = blockHeight; - const addr = req.query.address || ''; - - return request(`${API_URL}/tx/address/${addr}`, (error, localRes, txs) => { - if (error) { - logger.log('error', - `${error}`); - return res.status(404).send(); - } - // Catch JSON errors - try { - txs = JSON.parse(txs); - } catch (e) { - logger.log('error', - `${e}`); - return res.status(404).send(); - } - // Setup UI JSON - return res.send({ - pagesTotal: 1, - txs: txs.map(tx => ({ - txid: tx.hash, - fees: tx.fee / 1e8, - confirmations: (height - tx.height) + 1, - valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, - vin: tx.inputs.map(input => ({ - addr: input.coin ? input.coin.address : '', - value: input.coin ? input.coin.value / 1e8 : 0, - })), - vout: tx.outputs.map(output => ({ - scriptPubKey: { - addresses: [output.address], - }, - value: output.value / 1e8, - })), - isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', - })), - }); - }); - }); - } else { - // Get last n txs - db.txs.getTransactions( - {}, - {}, - MAX_TXS, - (err, txs) => { - if (err) { - logger.log('err', - `getTransactions: ${err}`); - res.status(404).send(); + // Catch JSON errors + try { + txs = JSON.parse(txs); + } catch (e) { + logger.log('error', + `${e}`); + return res.status(404).send(); } - return res.json({ + // Bcoin returns error as part of data object + if (txs.error) { + logger.log('error', + `${'No tx results'}`); + return res.status(404).send(); + } + // Setup UI JSON + return res.send({ pagesTotal: 1, txs: txs.map(tx => ({ txid: tx.hash, - version: tx.version, - locktime: tx.locktime, + fees: tx.fee / 1e8, + confirmations: (height - tx.height) + 1, + valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, vin: tx.inputs.map(input => ({ - coinbase: input.script, - sequence: input.sequence, - n: 0, + addr: input.coin ? input.coin.address : '', + value: input.coin ? input.coin.value / 1e8 : 0, + scriptSig: { + asm: input.script, + }, })), vout: tx.outputs.map(output => ({ - value: output.value, - n: 0, scriptPubKey: { - hex: '', - asm: '', + asm: output.script, addresses: [output.address], - type: output.type, }, - spentTxid: '', - spentIndex: 0, - spentHeight: 0, + value: output.value / 1e8, })), + isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', })), }); - }, - ); + }); } + // Get last n txs + return res.status(404).send({ error: 'Block hash or address expected' }); }); router.get('/rawtx/:txid', (req, res) => { diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index d0c9a53..5659d34 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -4,6 +4,8 @@ const config = require('../../config'); const MAX_BLOCKS = config.api.max_blocks; // ~ 12 hours +let bestBlockHeight = 0; + function getBlocks(params, options, limit, cb) { // Do not return mongo ids const defaultOptions = { _id: 0 }; @@ -21,6 +23,7 @@ function getBlocks(params, options, limit, cb) { if (limit < 1) { limit = 1; } + // Query mongo Block.find( params, @@ -53,20 +56,31 @@ function getBlock(params, options, limit, cb) { return cb(null, blocks[0]); }); } -// Highest known height -function getBestHeight(cb) { +// Highest known height in mongo +function getBestHeight() { getBlock({}, {}, 1, (err, block) => { if (err) { logger.log('error', - `getBlock: ${err.err}`); - return cb(err); + `getBestHeight: ${err.err}`); + return; } - return cb(null, block.height); + bestBlockHeight = block.height; }); } +// 1e9 limit = ~2M years from now +// Mostly for sync to set height +function bestHeight(height) { + if (Number.isInteger(height) && + height > 0 && + height < 1 * 1e9) { + bestBlockHeight = height; + return bestBlockHeight; + } + return bestBlockHeight; +} module.exports = { getBlock, getBlocks, - getBestHeight, + bestHeight, }; diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 455727d..bfd90c3 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -2,6 +2,9 @@ const Transactions = require('../../models/transaction.js'); const logger = require('../logger'); const config = require('../../config'); +// For now, blocks handles these calls. +// These will be replaced with more advanced mongo + const MAX_TXS = config.api.max_txs; function getTransactions(params, options, limit, cb) { diff --git a/server/lib/node/index.js b/server/lib/node/index.js index 266f5b1..bea48c0 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -3,6 +3,7 @@ const logger = require('../../lib/logger'); const BlockParser = require('../parser').Block; const config = require('../../config'); const socket = require('../../lib/api/socket'); +const db = require('../../lib/db'); const node = new FullNode(config.bcoin); @@ -18,6 +19,7 @@ function start() { node.chain.on('connect', (entry, block) => { BlockParser.parse(entry, block); socket.processBlock(entry, block); + db.blocks.bestHeight(entry.height); }); node.on('error', (err) => { diff --git a/server/models/input.js b/server/models/input.js index 9f2c067..a6ebc1b 100644 --- a/server/models/input.js +++ b/server/models/input.js @@ -10,6 +10,8 @@ const InputSchema = new Schema({ address: { type: String, default: '' }, }); +InputSchema.index({ address: 1 }); + const Input = mongoose.model('Input', InputSchema); module.exports = Input; diff --git a/server/models/output.js b/server/models/output.js index 65f2194..56694eb 100644 --- a/server/models/output.js +++ b/server/models/output.js @@ -9,6 +9,8 @@ const OutputSchema = new Schema({ type: { type: String, default: '' }, }); +OutputSchema.index({ address: 1 }); + const Output = mongoose.model('Output', OutputSchema); module.exports = Output; diff --git a/server/models/transaction.js b/server/models/transaction.js index 0c5974b..8d30aba 100644 --- a/server/models/transaction.js +++ b/server/models/transaction.js @@ -22,6 +22,8 @@ const TransactionSchema = new Schema({ network: { type: String, default: '' }, }); +TransactionSchema.index({ hash: 1 }); + const Transaction = mongoose.model('Transaction', TransactionSchema); module.exports = Transaction;