Merge branch 'next-merge' into feature/txEndpointValidation

README.md

@@ -4,14 +4,14 @@
 
 ## Requirements
 
-Insight requires [Node.js](https://nodejs.org) and [MongoDB](https://www.mongodb.com/). Consider using [n](https://github.com/tj/n) and [m](https://github.com/aheckmann/m) to install the latest versions. 
+Insight requires [Node.js](https://nodejs.org) 8.2 and [MongoDB](https://www.mongodb.com/). Consider using [n](https://github.com/tj/n) and [m](https://github.com/aheckmann/m) to install the latest versions.
 
 ## Quick Start
 
 To get started, clone this repository, then – with `mongod` running – install and run insight:
 
 ```bash
-git clone -b next https://github.com/bitpay/insight.git && cd insight
+git clone -b next https://github.com/bitpay/insight.git && cd insight/server
 npm install
 npm start
 ```

server/config/index.js

@@ -29,6 +29,7 @@ const config = {
     ticker_prop: 'bitstamp',
     max_blocks: 72,
     max_txs: 10,
+    request_ttl: 100000,
   },
 };
 

server/lib/api/address.js

@@ -3,12 +3,16 @@ const request = require('request');
 const config  = require('../../config');
 
 const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`;
+const TTL = config.api.request_ttl;
 
 module.exports = function AddressAPI(router) {
   router.get('/addr/:addr', (req, res) => {
     const addr = req.params.addr || '';
+    logger.log('debug',
+      'Warning: Requesting data from Bcoin by address, may take some time');
     // Get Bcoin data
     return request(`${API_URL}/tx/address/${addr}`,
+      { timeout: TTL },
       (error, bcoinRes, bcoinTxs) => {
         if (error) {
           logger.log('error',

server/lib/api/transaction.js

@@ -6,6 +6,7 @@ const util = require('../util');
 
 const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`;
 const MAX_TXS = config.api.max_txs;
+const TTL     = config.api.request_ttl;
 
 module.exports = function transactionAPI(router) {
   // Txs by txid
@@ -17,57 +18,56 @@ module.exports = function transactionAPI(router) {
     }
 
     // Get max block height for calculating confirmations
-    db.blocks.getBestHeight(
+    const height = db.blocks.bestHeight();
-      (err, blockHeight) => {
+    // Bcoin transaction data
-        if (err) {
+    return request(`${API_URL}/tx/${req.params.txid}`,
-          logger.log('err', err);
+      { timeout: TTL },
+      (error, localRes, tx) => {
+        if (error) {
+          logger.log('error',
+            `${error}`);
+          return res.status(404).send();
+        }
+        // Catch JSON errors
+        try {
+          tx = JSON.parse(tx);
+        } catch (e) {
+          logger.log('error',
+            `${e}`);
+          return res.status(404).send();
+        }
+        if (!tx || !tx.hash) {
+          logger.log('error',
+            'No results found');
           return res.status(404).send();
         }
-        const height = blockHeight;
-        // Bcoin transaction data
-        return request(`${API_URL}/tx/${req.params.txid}`, (error, localRes, tx) => {
-          if (error) {
-            logger.log('error',
-              `${error}`);
-            return res.status(404).send();
-          }
-          // Catch JSON errors
-          try {
-            tx = JSON.parse(tx);
-          } catch (e) {
-            logger.log('error',
-              `${e}`);
-            return res.status(404).send();
-          }
-          if (!tx || !tx.hash) {
-            logger.log('error',
-              'No results found');
-            return res.status(404).send();
-          }
 
-          // Return UI JSON
+        // Return UI JSON
-          return res.send({
+        return res.send({
-            txid: tx.hash,
+          txid: tx.hash,
-            version: tx.version,
+          version: tx.version,
-            time: tx.ps,
+          time: tx.ps,
-            blocktime: tx.ps,
+          blocktime: tx.ps,
-            locktime: tx.locktime,
+          locktime: tx.locktime,
-            blockhash: tx.block,
+          blockhash: tx.block,
-            fees: tx.fee / 1e8,
+          fees: tx.fee / 1e8,
-            confirmations: (height - tx.height) + 1,
+          confirmations: (height - tx.height) + 1,
-            valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
+          valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
-            vin: tx.inputs.map(input => ({
+          vin: tx.inputs.map(input => ({
-              addr: input.coin ? input.coin.address : '',
+            addr: input.coin ? input.coin.address : '',
-              value: input.coin ? input.coin.value / 1e8 : 0,
+            value: input.coin ? input.coin.value / 1e8 : 0,
-            })),
+            scriptSig: {
-            vout: tx.outputs.map(output => ({
+              asm: input.script,
-              scriptPubKey: {
+            },
-                addresses: [output.address],
+          })),
-              },
+          vout: tx.outputs.map(output => ({
-              value: output.value / 1e8,
+            scriptPubKey: {
-            })),
+              asm: output.script,
-            isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
+              addresses: [output.address],
-          });
+            },
+            value: output.value / 1e8,
+          })),
+          isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
         });
       });
   });
@@ -86,58 +86,58 @@ module.exports = function transactionAPI(router) {
         return res.status(400).send({
           error: 'Invalid block hash',
         });
-      }
+      }      
-
+      const height = db.blocks.bestHeight();
-      db.blocks.getBestHeight(
+      // Get Bcoin data
-        (err, blockHeight) => {
+      return request(`${API_URL}/block/${req.query.block}`,
-          if (err) {
+        { timeout: TTL },
-            logger.log('err', err);
+        (error, localRes, block) => {
+          if (error) {
+            logger.log('error',
+              `${error}`);
+            return res.status(404).send();
+          }
+          // Catch JSON errors
+          try {
+            block = JSON.parse(block);
+          } catch (e) {
+            logger.log('error',
+              `${e}`);
             return res.status(404).send();
           }
-          const height = blockHeight;
-          // Get Bcoin data
-          return request(`${API_URL}/block/${req.query.block}`, (error, localRes, block) => {
-            if (error) {
-              logger.log('error',
-                `${error}`);
-            }
-            // Catch JSON errors
-            try {
-              block = JSON.parse(block);
-            } catch (e) {
-              logger.log('error',
-                `${e}`);
-              return res.status(404).send();
-            }
-            if (!block.txs.length) {
-              logger.log('error',
-                `${'No tx results'}`);
-              res.status(404).send();
-            }
-            //  Setup UI JSON
-            const totalPages = Math.ceil(block.txs.length / MAX_TXS);
-            block.txs = block.txs.slice(rangeStart, rangeEnd);
 
-            return res.send({
+          if (block.error) {
-              pagesTotal: totalPages,
+            logger.log('error',
-              txs: block.txs.map(tx => ({
+              `${'No tx results'}`);
-                txid: tx.hash,
+            return res.status(404).send();
-                fees: tx.fee / 1e8,
+          }
-                confirmations: (height - block.height) + 1,
+          //  Setup UI JSON
-                valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
+          const totalPages = Math.ceil(block.txs.length / MAX_TXS);
-                vin: tx.inputs.map(input => ({
+          block.txs = block.txs.slice(rangeStart, rangeEnd);
-                  addr: input.coin ? input.coin.address : '',
+
-                  value: input.coin ? input.coin.value / 1e8 : 0,
+          return res.send({
-                })),
+            pagesTotal: totalPages,
-                vout: tx.outputs.map(output => ({
+            txs: block.txs.map(tx => ({
-                  scriptPubKey: {
+              txid: tx.hash,
-                    addresses: [output.address],
+              fees: tx.fee / 1e8,
-                  },
+              confirmations: (height - block.height) + 1,
-                  value: output.value / 1e8,
+              valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
-                })),
+              vin: tx.inputs.map(input => ({
-                isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
+                addr: input.coin ? input.coin.address : '',
+                value: input.coin ? input.coin.value / 1e8 : 0,
+                scriptSig: {
+                  asm: input.script,
+                },
               })),
-            });
+              vout: tx.outputs.map(output => ({
+                scriptPubKey: {
+                  asm: output.script,
+                  addresses: [output.address],
+                },
+                value: output.value / 1e8,
+              })),
+              isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
+            })),
           });
         });
     } else if (req.query.address) {
@@ -148,94 +148,63 @@ module.exports = function transactionAPI(router) {
       }
 
       // Get txs by address, start with best height to calc confirmations
-      db.blocks.getBestHeight(
+      const height = db.blocks.bestHeight();
-        (err, blockHeight) => {
+      const addr = req.query.address || '';
-          if (err) {
+
-            logger.log('err', err);
+      logger.log('debug',
+        'Warning: Requesting data from Bcoin by address, may take some time');
+
+      return request(`${API_URL}/tx/address/${addr}`,
+        { timeout: TTL },
+        (error, localRes, txs) => {
+          if (error) {
+            logger.log('error',
+              `${error}`);
             return res.status(404).send();
           }
-
+          // Catch JSON errors
-          const height = blockHeight;
+          try {
-          const addr = req.query.address || '';
+            txs = JSON.parse(txs);
-
+          } catch (e) {
-          return request(`${API_URL}/tx/address/${addr}`, (error, localRes, txs) => {
+            logger.log('error',
-            if (error) {
+              `${e}`);
-              logger.log('error',
+            return res.status(404).send();
-                `${error}`);
-              return res.status(404).send();
-            }
-            // Catch JSON errors
-            try {
-              txs = JSON.parse(txs);
-            } catch (e) {
-              logger.log('error',
-                `${e}`);
-              return res.status(404).send();
-            }
-            // Setup UI JSON
-            return res.send({
-              pagesTotal: 1,
-              txs: txs.map(tx => ({
-                txid: tx.hash,
-                fees: tx.fee / 1e8,
-                confirmations: (height - tx.height) +  1,
-                valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
-                vin: tx.inputs.map(input => ({
-                  addr: input.coin ? input.coin.address : '',
-                  value: input.coin ? input.coin.value / 1e8 : 0,
-                })),
-                vout: tx.outputs.map(output => ({
-                  scriptPubKey: {
-                    addresses: [output.address],
-                  },
-                  value: output.value / 1e8,
-                })),
-                isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
-              })),
-            });
-          });
-        });
-    } else {
-      // Get last n txs
-      db.txs.getTransactions(
-        {},
-        {},
-        MAX_TXS,
-        (err, txs) => {
-          if (err) {
-            logger.log('err',
-              `getTransactions: ${err}`);
-            res.status(404).send();
           }
-          return res.json({
+          // Bcoin returns error as part of data object
+          if (txs.error) {
+            logger.log('error',
+              `${'No tx results'}`);
+            return res.status(404).send();
+          }
+          // Setup UI JSON
+          return res.send({
             pagesTotal: 1,
             txs: txs.map(tx => ({
               txid: tx.hash,
-              version: tx.version,
+              fees: tx.fee / 1e8,
-              locktime: tx.locktime,
+              confirmations: (height - tx.height) +  1,
+              valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
               vin: tx.inputs.map(input => ({
-                coinbase: input.script,
+                addr: input.coin ? input.coin.address : '',
-                sequence: input.sequence,
+                value: input.coin ? input.coin.value / 1e8 : 0,
-                n: 0,
+                scriptSig: {
+                  asm: input.script,
+                },
               })),
               vout: tx.outputs.map(output => ({
-                value: output.value,
-                n: 0,
                 scriptPubKey: {
-                  hex: '',
+                  asm: output.script,
-                  asm: '',
                   addresses: [output.address],
-                  type: output.type,
                 },
-                spentTxid: '',
+                value: output.value / 1e8,
-                spentIndex: 0,
-                spentHeight: 0,
               })),
+              isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
             })),
           });
-        },
+        });
-      );
     }
+    // Get last n txs
+    return res.status(404).send({ error: 'Block hash or address expected' });
   });
 
   router.get('/rawtx/:txid', (req, res) => {

server/lib/db/blocks.js

@@ -4,6 +4,8 @@ const config = require('../../config');
 
 const MAX_BLOCKS = config.api.max_blocks; // ~ 12 hours
 
+let bestBlockHeight = 0;
+
 function getBlocks(params, options, limit, cb) {
   // Do not return mongo ids
   const defaultOptions = { _id: 0 };
@@ -21,6 +23,7 @@ function getBlocks(params, options, limit, cb) {
   if (limit < 1) {
     limit = 1;
   }
+
   // Query mongo
   Block.find(
     params,
@@ -53,20 +56,31 @@ function getBlock(params, options, limit, cb) {
     return cb(null, blocks[0]);
   });
 }
-// Highest known height
+// Highest known height in mongo
-function getBestHeight(cb) {
+function getBestHeight() {
   getBlock({}, {}, 1, (err, block) => {
     if (err) {
       logger.log('error',
-        `getBlock: ${err.err}`);
+        `getBestHeight: ${err.err}`);
-      return cb(err);
+      return;
     }
-    return cb(null, block.height);
+    bestBlockHeight = block.height;
   });
 }
+// 1e9 limit = ~2M years from now
+// Mostly for sync to set height
+function bestHeight(height) {
+  if (Number.isInteger(height) &&
+    height > 0 &&
+    height < 1 * 1e9) {
+    bestBlockHeight = height;
+    return bestBlockHeight;
+  }
+  return bestBlockHeight;
+}
 
 module.exports = {
   getBlock,
   getBlocks,
-  getBestHeight,
+  bestHeight,
 };

server/lib/db/transactions.js

@@ -2,6 +2,9 @@ const Transactions = require('../../models/transaction.js');
 const logger       = require('../logger');
 const config       = require('../../config');
 
+// For now, blocks handles these calls.
+// These will be replaced with more advanced mongo
+
 const MAX_TXS = config.api.max_txs;
 
 function getTransactions(params, options, limit, cb) {

server/lib/node/index.js

@@ -3,6 +3,7 @@ const logger      = require('../../lib/logger');
 const BlockParser = require('../parser').Block;
 const config      = require('../../config');
 const socket      = require('../../lib/api/socket');
+const db          = require('../../lib/db');
 
 const node = new FullNode(config.bcoin);
 
@@ -18,6 +19,7 @@ function start() {
   node.chain.on('connect', (entry, block) => {
     BlockParser.parse(entry, block);
     socket.processBlock(entry, block);
+    db.blocks.bestHeight(entry.height);
   });
 
   node.on('error', (err) => {

server/models/input.js

@@ -10,6 +10,8 @@ const InputSchema = new Schema({
   address:  { type: String, default: '' },
 });
 
+InputSchema.index({ address: 1 });
+
 const Input = mongoose.model('Input', InputSchema);
 
 module.exports = Input;

server/models/output.js

@@ -9,6 +9,8 @@ const OutputSchema = new Schema({
   type:    { type: String, default: '' },
 });
 
+OutputSchema.index({ address: 1 });
+
 const Output = mongoose.model('Output', OutputSchema);
 
 module.exports = Output;

server/models/transaction.js

@@ -22,6 +22,8 @@ const TransactionSchema = new Schema({
   network:     { type: String, default: '' },
 });
 
+TransactionSchema.index({ hash: 1 });
+
 const Transaction = mongoose.model('Transaction', TransactionSchema);
 
 module.exports = Transaction;