Merge branch 'next-merge' of https://github.com/tenthirtyone/insight into next

app/src/providers/api/api.ts

@@ -11,7 +11,7 @@ import 'rxjs/add/operator/map';
 @Injectable()
 export class ApiProvider {
 
-  public apiPrefix: string = 'https://insight.bitpay.com/api/';
+  public apiPrefix: string = '/api/';
 
   constructor(public http: Http) {
   }

app/src/services/blocksService.ts

@@ -11,7 +11,7 @@ export class BlocksService {
   constructor(private http: Http) {}
 
   public getLatestBlocks(): void {
-    this.http.request('https://insight.bitpay.com/api/blocks').subscribe((res: Response) => {
+    this.http.request('/api/blocks').subscribe((res: Response) => {
       const data: {
         blocks: InsightBlockObject[],
         length: number,

server/config/index.js

@@ -28,7 +28,7 @@ const config = {
     ticker_url: 'https://www.bitstamp.net/api/ticker/',
     ticker_prop: 'bitstamp',
     max_blocks: 72,
-    max_txs: 50,
+    max_txs: 10,
     max_page_txs: 10,
     request_ttl: 100000,
   },

server/lib/api/address.js

@@ -1,72 +1,55 @@
 const logger  = require('../logger');
-const request = require('request');
-const config  = require('../../config');
 const util    = require('../util');
-
-const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`;
-const TTL = config.api.request_ttl;
+const db      = require('../db');
 
 module.exports = function AddressAPI(router) {
   router.get('/addr/:addr', (req, res) => {
     const addr = req.params.addr || '';
 
     if (!util.isBitcoinAddress(addr)) {
-      return res.status(400).send({
+      return res.status(404).send({
         error: 'Invalid bitcoin address',
       });
     }
 
-    logger.log('debug',
-      'Warning: Requesting data from Bcoin by address, may take some time');
-    // Get Bcoin data
-    return request(`${API_URL}/tx/address/${addr}`,
-      { timeout: TTL },
-      (error, bcoinRes, bcoinTxs) => {
-        if (error) {
-          logger.log('error',
-            `${error}`);
-          return res.status(404).send({});
+    return db.txs.getTxByAddress(addr, 0, 999999999, (error, txs) => {
+      if (error) {
+        logger.log('error',
+          `getTxByBlock ${error}`);
+        return res.status(404).send();
+      }
+
+      // Sum the matching outputs for every tx
+      const totalReceived = txs.reduce((total, tx) => total + tx.outputs.reduce((sum, output) => {
+        if (output.address === req.params.addr) {
+          return sum + output.value;
         }
-        let txs = {};
-        try {
-          txs = JSON.parse(bcoinTxs);
-        } catch (e) {
-          logger.log('error',
-            `${e}`);
-          return res.status(404).send({});
+        return sum;
+      }, 0), 0) || 0;
+
+      // Sum the matching inputs for every tx
+      const totalSpent = txs.reduce((total, tx) => total + tx.inputs.reduce((sum, input) => {
+        if (input.coin && input.coin.address === req.params.addr) {
+          return sum + input.coin.value;
         }
+        return sum;
+      }, 0), 0) || 0;
 
-        // Sum the matching outputs for every tx
-        const totalReceived = txs.reduce((total, tx) => total + tx.outputs.reduce((sum, output) => {
-          if (output.address === req.params.addr) {
-            return sum + output.value;
-          }
-          return sum;
-        }, 0), 0) || 0;
-
-        // Sum the matching inputs for every tx
-        const totalSpent = txs.reduce((total, tx) => total + tx.inputs.reduce((sum, input) => {
-          if (input.coin && input.coin.address === req.params.addr) {
-            return sum + input.coin.value;
-          }
-          return sum;
-        }, 0), 0) || 0;
-
-        // Match Insight API
-        return res.json({
-          addrStr:                 req.params.addr,
-          balance:                 (totalReceived - totalSpent) / 1e8,
-          balanceSat:              totalReceived - totalSpent,
-          totalReceived:           totalReceived / 1e8,
-          totalReceivedSat:        totalReceived,
-          totalSent:               totalSpent / 1e8,
-          totalSentSat:            totalSpent,
-          unconfirmedBalance:      0,
-          unconfirmedBalanceSat:   0,
-          unconfirmedTxApperances: 0,
-          txApperances:            txs.length,
-        });
+      // Match Insight API
+      return res.json({
+        addrStr: req.params.addr,
+        balance: (totalReceived - totalSpent) / 1e8,
+        balanceSat: totalReceived - totalSpent,
+        totalReceived: totalReceived / 1e8,
+        totalReceivedSat: totalReceived,
+        totalSent: totalSpent / 1e8,
+        totalSentSat: totalSpent,
+        unconfirmedBalance: 0,
+        unconfirmedBalanceSat: 0,
+        unconfirmedTxApperances: 0,
+        txApperances: txs.length,
       });
+    });
   });
 
   // Stubbed by # to help with tasking

server/lib/api/block.js

@@ -7,18 +7,15 @@ module.exports = function BlockAPI(router) {
     const blockHash = req.params.blockHash;
 
     if (!util.isBlockHash(blockHash)) {
-      return res.status(400).send({
+      return res.status(404).send({
         error: 'Invalid bitcoin address',
       });
     }
 
     // Pass Mongo params, fields and limit to db api.
-    db.blocks.getBlock(
-      { hash: blockHash },
-      { rawBlock: 0 },
-      1,
+    return db.blocks.getByHash(blockHash,
       (err, block) => {
-        if (err) {
+        if (err || !block) {
           logger.log('err', err);
           return res.status(404).send();
         }
@@ -46,21 +43,11 @@ module.exports = function BlockAPI(router) {
   });
 
   router.get('/blocks', (req, res) => {
-    const limit = parseInt(req.query.limit, 10) || 100;
     // Pass Mongo params, fields and limit to db api.
-    db.blocks.getBlocks(
-      {},
-      { height: 1,
-        size: 1,
-        hash: 1,
-        ts: 1,
-        txs: 1,
-        poolInfo: 1,
-      },
-      limit,
+    db.blocks.getTopBlocks(
       (err, blocks) => {
         if (err) {
-          logger.log('err',
+          logger.log('error',
             `/blocks: ${err}`);
           return res.status(404).send();
         }
@@ -90,13 +77,10 @@ module.exports = function BlockAPI(router) {
     }
 
     // Pass Mongo params, fields and limit to db api.
-    db.blocks.getBlock(
-      { hash: blockHash },
-      { rawBlock: 1 },
-      1,
+    return db.blocks.getRawBlock(blockHash,
       (err, block) => {
-        if (err) {
-          logger.log('err',
+        if (err || !block) {
+          logger.log('error',
             `/rawblock/:blockHash: ${err}`);
           return res.status(404).send();
         }
@@ -105,15 +89,12 @@ module.exports = function BlockAPI(router) {
   });
 
   router.get('/block-index/:height', (req, res) => {
-    const blockHeight = parseInt(req.params.height, 10) || 1;
+    const height = parseInt(req.params.height, 10) || 1;
     // Pass Mongo params, fields and limit to db api.
-    db.blocks.getBlock(
-      { height: blockHeight },
-      { hash: 1 },
-      1,
+    return db.blocks.byHeight(height,
       (err, block) => {
-        if (err) {
-          logger.log('err',
+        if (err || !block) {
+          logger.log('error',
             `/block-index/:height: ${err}`);
           return res.status(404).send();
         }

server/lib/api/index.js

@@ -1,28 +1,19 @@
 const express    = require('express');
 const config     = require('../../config');
 const bodyParser = require('body-parser');
+const helmet     = require('helmet');
 
 const app = express();
 const api = express.Router();
 const cors = require('./cors');
 
 app.use(cors);
+app.use(helmet());
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
 
 // Serve insight ui front end from root dir public folder
-app.use(express.static('./public'));
-app.use('/:stuff', express.static('./public'));
-app.use('/blocks', express.static('./public'));
-app.use('/blocks/:blockhash', express.static('./public'));
-app.use('/block-index', express.static('./public'));
-app.use('/block-index/:height', express.static('./public'));
-app.use('/blocks-date/:date', express.static('./public'));
-app.use('/block/:blockhash', express.static('./public'));
-app.use('/tx/:txid', express.static('./public'));
-app.use('/address/:addr', express.static('./public'));
-app.use('/status', express.static('./public'));
-app.use('/status/:stuff', express.static('./public'));
+app.use(express.static('../app/www', { maxage: '1w' }));
 
 app.set('json spaces', config.api.json_spaces);
 
@@ -34,16 +25,14 @@ const StatusAPI      = require('./status')(api);
 const TransactionAPI = require('./transaction')(api);
 const MessageAPI     = require('./message')(api);
 
-app.use('/insight-api', api);
+app.use('/api', api);
 
 // 404
-app.use((req, res) => {
-  res.status(404).send({
-    status: 404,
-    url: req.originalUrl,
-    error: 'Not found',
-  });
-});
+app.use((req, res) => res.status(404).send({
+  status: 404,
+  url: req.originalUrl,
+  error: 'Not found',
+}));
 
 // Socket server
 const server  = require('http').Server(app);

server/lib/api/message.js

@@ -27,7 +27,7 @@ function verifyMessage(req, res) {
       code: 1,
     });
   }
-  res.json({ result: valid });
+  return res.json({ result: valid });
 }
 
 module.exports = function messageAPI(router) {
@@ -39,7 +39,5 @@ module.exports = function messageAPI(router) {
     verifyMessage(req, res);
   });
 
-  router.get('/utils/estimatefee', (req, res) => {
-    res.send('estimate fees');
-  });
+  router.get('/utils/estimatefee', (req, res) => res.send('estimate fees'));
 };

server/lib/api/status.js

@@ -30,10 +30,7 @@ module.exports = function statusAPI(router) {
   // Get last block hash or node status
   router.get('/status', (req, res) => {
     if (req.query.q === 'getLastBlockHash') {
-      db.blocks.getBlock(
-        {},
-        { hash: 1 },
-        1,
+      db.blocks.getLastBlock(
         (err, block) => {
           if (err) {
             logger.log('error',
@@ -48,12 +45,12 @@ module.exports = function statusAPI(router) {
     } else {
       getStatus((err, status) => {
         if (err) {
-          logger.log('err',
+          logger.log('error',
             `/status getStatus: ${err}`);
           return res.status(404).send(err);
         }
         if (!status) {
-          logger.log('err',
+          logger.log('error',
             '/status getStatus: no Status');
           return  res.status(404).send();
         }
@@ -79,12 +76,12 @@ module.exports = function statusAPI(router) {
   router.get('/sync', (req, res) => {
     getStatus((err, status) => {
       if (err) {
-        logger.log('err',
+        logger.log('error',
           `/sync: ${err}`);
         return res.status(404).send(err);
       }
       if (!status) {
-        logger.log('err',
+        logger.log('error',
           '/sync: no status');
         return res.status(404).send();
       }
@@ -99,17 +96,13 @@ module.exports = function statusAPI(router) {
     });
   });
   // Copied from previous source
-  router.get('/peer', (req, res) => {
-    res.json({
-      connected: true,
-      host: '127.0.0.1',
-      port: null,
-    });
-  });
+  router.get('/peer', (req, res) => res.json({
+    connected: true,
+    host: '127.0.0.1',
+    port: null,
+  }));
 
-  router.get('/version', (req, res) => {
-    res.json({
-      version: pkg.version,
-    });
-  });
+  router.get('/version', (req, res) => res.json({
+    version: pkg.version,
+  }));
 };

server/lib/api/transaction.js

@@ -12,7 +12,7 @@ module.exports = function transactionAPI(router) {
   // Txs by txid
   router.get('/tx/:txid', (req, res) => {
     if (!util.isTxid(req.params.txid)) {
-      return res.status(400).send({
+      return res.status(404).send({
         error: 'Invalid transaction id',
       });
     }
@@ -20,62 +20,46 @@ module.exports = function transactionAPI(router) {
     // Get max block height for calculating confirmations
     const height = db.blocks.bestHeight();
     // Bcoin transaction data
-    return request(`${API_URL}/tx/${req.params.txid}`,
-      { timeout: TTL },
-      (error, localRes, tx) => {
-        if (error) {
-          logger.log('error',
-            `${error}`);
-          return res.status(404).send();
-        }
-        // Catch JSON errors
-        try {
-          tx = JSON.parse(tx);
-        } catch (e) {
-          logger.log('error',
-            `${e}`);
-          return res.status(404).send();
-        }
-        if (!tx || !tx.hash) {
-          logger.log('error',
-            'No results found');
-          return res.status(404).send();
-        }
+    const txid = req.params.txid || '';
 
-        // Return UI JSON
-        return res.send({
-          txid: tx.hash,
-          version: tx.version,
-          time: tx.ps,
-          blocktime: tx.ps,
-          locktime: tx.locktime,
-          blockhash: tx.block,
-          fees: tx.fee / 1e8,
-          confirmations: (height - tx.height) + 1,
-          valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
-          vin: tx.inputs.map(input => ({
-            addr: input.coin ? input.coin.address : '',
-            value: input.coin ? input.coin.value / 1e8 : 0,
-            scriptSig: {
-              asm: input.script,
-            },
-          })),
-          vout: tx.outputs.map(output => ({
-            scriptPubKey: {
-              asm: output.script,
-              addresses: [output.address],
-            },
-            value: output.value / 1e8,
-          })),
-          isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
-        });
+    db.txs.getTxById(txid, (err, transaction) => {
+      if (err) {
+        logger.log('error',
+          `/tx/:tid getTxById: ${err.err}`);
+        return res.status(404).send();
+      }
+
+      const tx = transaction;
+      return res.send({
+        txid: tx.hash,
+        version: tx.version,
+        time: tx.ps,
+        blocktime: tx.ps,
+        locktime: tx.locktime,
+        blockhash: tx.block,
+        fees: tx.fee / 1e8,
+        size: tx.size,
+        confirmations: (height - tx.height) + 1,
+        valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
+        vin: tx.inputs.map(input => ({
+          addr: input.address,
+          value: input.value / 1e8,
+        })),
+        vout: tx.outputs.map(output => ({
+          scriptPubKey: {
+            addresses: [output.address],
+          },
+          value: output.value / 1e8,
+        })),
+        isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
       });
+    });
   });
 
   // /txs is overloaded. Next ver separate concerns
   // query by block
   // query by address
-  // last n txs
+  // last n txs - haha jk YOU 404
   router.get('/txs', (req, res) => {
     const pageNum    = parseInt(req.query.pageNum, 10)  || 0;
     const rangeStart = pageNum * MAX_TXS;
@@ -88,46 +72,35 @@ module.exports = function transactionAPI(router) {
         });
       }
       const height = db.blocks.bestHeight();
-      // Get Bcoin data
-      return request(`${API_URL}/block/${req.query.block}`,
-        { timeout: TTL },
-        (error, localRes, block) => {
+
+      return db.txs.getTxCountByBlock(req.query.block, (err, count) => {
+        if (err) {
+          logger.log('error',
+            `getTxByBlock ${err}`);
+          return res.status(404).send();
+        }
+        const totalPages = Math.ceil(count / MAX_TXS);
+
+        return db.txs.getTxByBlock(req.query.block, pageNum, MAX_TXS, (error, txs) => {
           if (error) {
             logger.log('error',
-              `${error}`);
+              `getTxByBlock ${error}`);
             return res.status(404).send();
           }
-          // Catch JSON errors
-          try {
-            block = JSON.parse(block);
-          } catch (e) {
-            logger.log('error',
-              `${e}`);
-            return res.status(404).send();
-          }
-
-          if (block.error) {
-            logger.log('error',
-              `${'No tx results'}`);
-            return res.status(404).send();
-          }
-          //  Setup UI JSON
-          const totalPages = Math.ceil(block.txs.length / MAX_TXS);
-          block.txs = block.txs.slice(rangeStart, rangeEnd);
-
           return res.send({
             pagesTotal: totalPages,
-            txs: block.txs.map(tx => ({
+            txs: txs.map(tx => ({
               txid: tx.hash,
               fees: tx.fee / 1e8,
-              confirmations: (height - block.height) + 1,
+              size: tx.size,
+              confirmations: (height - tx.height) + 1,
               valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
               vin: tx.inputs.map(input => ({
-                addr: input.coin ? input.coin.address : '',
-                value: input.coin ? input.coin.value / 1e8 : 0,
                 scriptSig: {
                   asm: input.script,
                 },
+                addr: input.address,
+                value: input.value / 1e8,
               })),
               vout: tx.outputs.map(output => ({
                 scriptPubKey: {
@@ -140,6 +113,7 @@ module.exports = function transactionAPI(router) {
             })),
           });
         });
+      });
     } else if (req.query.address) {
       if (!util.isBitcoinAddress(req.query.address)) {
         return res.status(400).send({
@@ -151,45 +125,34 @@ module.exports = function transactionAPI(router) {
       const height = db.blocks.bestHeight();
       const addr = req.query.address || '';
 
-      logger.log('debug',
-        'Warning: Requesting data from Bcoin by address, may take some time');
+      db.txs.getTxCountByAddress(req.query.address, (err, count) => {
+        if (err) {
+          logger.log('error',
+            `getTxByBlock ${err}`);
+          return res.status(404).send();
+        }
+        const totalPages = Math.ceil(count / MAX_TXS);
 
-      return request(`${API_URL}/tx/address/${addr}`,
-        { timeout: TTL },
-        (error, localRes, txs) => {
+        return db.txs.getTxByAddress(req.query.address, pageNum, MAX_TXS, (error, txs) => {
           if (error) {
             logger.log('error',
-              `${error}`);
+              `getTxByBlock ${error}`);
             return res.status(404).send();
           }
-          // Catch JSON errors
-          try {
-            txs = JSON.parse(txs);
-          } catch (e) {
-            logger.log('error',
-              `${e}`);
-            return res.status(404).send();
-          }
-          // Bcoin returns error as part of data object
-          if (txs.error) {
-            logger.log('error',
-              `${'No tx results'}`);
-            return res.status(404).send();
-          }
-          // Setup UI JSON
           return res.send({
-            pagesTotal: 1,
+            pagesTotal: totalPages,
             txs: txs.map(tx => ({
               txid: tx.hash,
               fees: tx.fee / 1e8,
-              confirmations: (height - tx.height) +  1,
+              size: tx.size,
+              confirmations: (height - tx.height) + 1,
               valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
               vin: tx.inputs.map(input => ({
-                addr: input.coin ? input.coin.address : '',
-                value: input.coin ? input.coin.value / 1e8 : 0,
                 scriptSig: {
                   asm: input.script,
                 },
+                addr: input.address,
+                value: input.value / 1e8,
               })),
               vout: tx.outputs.map(output => ({
                 scriptPubKey: {
@@ -202,21 +165,21 @@ module.exports = function transactionAPI(router) {
             })),
           });
         });
+      });
+    } else {
+      // Get last n txs
+      db.txs.getTopTransactions((err, txs) => {
+        if (err) {
+          logger.log('err',
+            `/txs getTopTransactions ${err}`);
+          return res.status(404).send(err);
+        }
+        return res.json(txs);
+      });
     }
-    // Get last n txs
-    db.txs.getTopTransactions((err, txs) => {
-      if (err) {
-        logger.log('err',
-          `/txs getTopTransactions ${err}`);
-        return res.status(404).send(err);
-      }
-      return res.json(txs);
-    });
   });
 
-  router.get('/rawtx/:txid', (req, res) => {
-    res.send(req.params.txid);
-  });
+  router.get('/rawtx/:txid', (req, res) => res.send(req.params.txid));
 
   router.post('/tx/send', (req, res) => {
     const rawtx = req.body.rawtx || '';
@@ -228,10 +191,10 @@ module.exports = function transactionAPI(router) {
       if (err) {
         logger.log('error',
           `${err}`);
-        return res.status(400).send(err);
+        return res.status(404).send(err);
       }
 
-      res.json(true);
+      return res.json(true);
     });
   });
 };

server/lib/db/blocks.js

@@ -2,70 +2,10 @@ const Block = require('../../models/block.js');
 const logger = require('../logger');
 const config = require('../../config');
 
-const MAX_BLOCKS = config.api.max_blocks; // ~ 12 hours
+const block = new Block();
 
 let bestBlockHeight = 0;
 
-function getBlocks(params, options, limit, cb) {
-  // Do not return mongo ids
-  const defaultOptions = { _id: 0 };
-  // Copy over mongo options
-  Object.assign(defaultOptions, options);
-  // Simple sanitizing
-  if (!Number.isInteger(limit)) {
-    limit = 1;
-  }
-
-  if (limit > MAX_BLOCKS) {
-    limit = MAX_BLOCKS;
-  }
-
-  if (limit < 1) {
-    limit = 1;
-  }
-
-  // Query mongo
-  Block.find(
-    params,
-    defaultOptions,
-    (err, blocks) => {
-      if (err) {
-        logger.log('error',
-          `getBlocks: ${err}`);
-        return cb(err);
-      }
-      if (!blocks.length > 0) {
-        return cb({ err: 'Block not found' });
-      }
-      return cb(null, blocks);
-    })
-    .sort({ height: -1 })
-    .limit(limit);
-}
-// Retrieve a single block. For convenience mostly
-function getBlock(params, options, limit, cb) {
-  getBlocks(params, options, limit, (err, blocks) => {
-    if (err) {
-      logger.log('error',
-        `getBlock: ${err.err}`);
-      return cb(err);
-    }
-    if (!blocks.length > 0) {
-      return cb({ err: 'Block not found' });
-    }
-    return cb(null, blocks[0]);
-  });
-}
-// Highest known height in mongo
-function getBestHeight() {
-  getBlock({}, {}, 1, (err, block) => {
-    if (err) {
-      return logger.log('error',
-        `getBestHeight: ${err.err}`);
-    }
-    bestBlockHeight = block.height;
-  });
-}
 // 1e9 limit = ~2M years from now
 // Mostly for sync to set height
 function bestHeight(height) {
@@ -78,8 +18,32 @@ function bestHeight(height) {
   return bestBlockHeight;
 }
 
+function getRawBlock(hash, cb) {
+  return block.getRawBlock(hash, cb);
+}
+
+function byHeight(height, cb) {
+  return block.byHeight(height, cb);
+}
+
+function getTopBlocks(cb) {
+  return block.last(cb);
+}
+
+function getByHash(hash, cb) {
+  return block.byHash(hash, cb);
+}
+
+function getLastBlock(cb) {
+  return block.last(cb)
+    .limit(1);
+}
+
 module.exports = {
-  getBlock,
-  getBlocks,
+  getRawBlock,
+  getTopBlocks,
+  getLastBlock,
+  getByHash,
+  byHeight,
   bestHeight,
 };

server/lib/db/transactions.js

@@ -1,89 +1,51 @@
 const Transactions = require('../../models/transaction.js');
-const logger       = require('../logger');
 const config       = require('../../config');
 
-// For now, blocks handles these calls.
-// These will be replaced with more advanced mongo
-// No optimization yet.
-
-const MAX_TXS = config.api.max_txs;
+const Txs = new Transactions();
 const MAX_PAGE_TXS = config.api.max_page_txs;
 
-// For Paging
-function getTransactions(params, options, limit, cb) {
-  // Do not return mongo ids
-  const defaultOptions = { _id: 0 };
-  // Copy over mongo options
-  Object.assign(defaultOptions, options);
-  // Simple sanitizing
-  if (!Number.isInteger(limit)) {
-    limit = 1;
-  }
-
-  if (limit > MAX_PAGE_TXS) {
-    limit = MAX_PAGE_TXS;
-  }
-
-  if (limit < 1) {
-    limit = 1;
-  }
-  // Query mongo
-  Transactions.find(
-    params,
-    defaultOptions,
-    (err, txs) => {
-      if (err) {
-        logger.log('error',
-          `getTransactions: ${err}`);
-        return cb(err);
-      }
-      if (!txs.length > 0) {
-        return cb({ err: 'Tx not found' });
-      }
-      return cb(null, txs);
-    })
-    .sort({ height: -1 })
-    .limit(limit);
+function getEmptyInputs(cb) {
+  return Txs.getEmptyInputs(cb);
 }
 
-function getTransaction(params, options, limit, cb) {
-  getTransactions(params, options, limit, (err, tx) => {
-    if (err) {
-      logger.log('error',
-        `getTransaction: ${err.err}`);
-      return cb(err);
-    }
-    if (!tx.length > 0) {
-      return cb({ err: 'Tx not found' });
-    }
-    return cb(null, tx[0]);
-  });
-}
-
-// Req Change, refactor above
 function getTopTransactions(cb) {
-  // Do not return mongo ids
-  const defaultOptions = { _id: 0 };
-  // Query mongo
-  Transactions.find(
-    {},
-    (err, txs) => {
-      if (err) {
-        logger.log('error',
-          `getTransactions: ${err}`);
-        return cb(err);
-      }
-      if (!txs.length > 0) {
-        return cb({ err: 'Tx not found' });
-      }
-      return cb(null, txs);
-    })
-    .sort({ height: -1 })
-    .limit(MAX_TXS);
+  return Txs.last(cb);
+}
+
+function getTxById(txid, cb) {
+  return Txs.byId(txid, cb);
+}
+
+function getTxByBlock(blockHash, page, limit, cb) {
+  return Txs.byBlockHash(blockHash, cb)
+    .skip(limit * page);
+}
+
+function getTxByAddress(address, page, limit, cb) {
+  return Txs.byAddress(address, cb)
+    .limit(limit)
+    .skip(limit * page);
+}
+
+function getTxCountByBlock(blockHash, cb) {
+  return Txs.countByBlock(blockHash, cb);
+}
+
+function getTxCountByAddress(address, cb) {
+  return Txs.countByAddress(address, cb);
+}
+
+function updateInput(txid, inputid, value, address) {
+  return Txs.updateInput(txid, inputid, value, address);
 }
 
 module.exports = {
-  getTransaction,
-  getTransactions,
+  getEmptyInputs,
   getTopTransactions,
+  getTxById,
+  getTxByBlock,
+  getTxCountByBlock,
+  getTxByAddress,
+  getTxCountByAddress,
+  updateInput,
 };

server/lib/node/index.js

@@ -1,7 +1,7 @@
 const FullNode    = require('bcoin/lib/node/fullnode');
 const logger      = require('../../lib/logger');
 const BlockParser = require('../parser').Block;
-const TxParser = require('../parser').Transaction;
+const TxParser    = require('../parser').Transaction;
 const config      = require('../../config');
 const socket      = require('../../lib/api/socket');
 const db          = require('../../lib/db');
@@ -28,10 +28,6 @@ function start() {
     logger.log('error',
       `${err}`);
   });
-
-  node.mempool.on('tx', (tx) => {
-    socket.emitTx(tx);
-  });
 }
 
 module.exports = {

server/lib/parser/index.js

@@ -1,5 +1,5 @@
 const Block       = require('./block');
-const Transaction       = require('./transaction');
+const Transaction = require('./transaction');
 
 module.exports = {
   Block,

server/lib/parser/transaction.js

@@ -1,16 +1,10 @@
-const TxModel = require('../../models/transaction');
-const InputModel = require('../../models/input');
+const TxModel     = require('../../models/transaction');
+const InputModel  = require('../../models/input');
 const OutputModel = require('../../models/output');
-const config = require('../../config');
-const util = require('../../lib/util');
-const logger = require('../logger');
-const db = require('../db');
-
-// Bleh, Bcoin pulls in blocks 20 at a time
-// Crappy delay for now otherwise async saves
-// could miss a tx if an input refs a block within
-// the last 20 that hasn't saved.
-// Aggregate stuff will replace all of this.
+const config      = require('../../config');
+const util        = require('../../lib/util');
+const logger      = require('../logger');
+const db          = require('../db');
 
 function parse(entry, txs) {
   txs.forEach((tx) => {
@@ -58,10 +52,37 @@ function parse(entry, txs) {
       if (err) {
         logger.log('error', err.message);
       }
+
+      findEmptyInputs();
     });
   });
 }
 
+function findEmptyInputs() {
+  db.txs.getEmptyInputs(
+    (err, txs) => {
+      if (err) {
+        return logger.log('error',
+          `No Empty Inputs found: ${err.err}`);
+      }
+      // For each tx with unmarked inputs
+      return txs.forEach((inputTx) => {
+        inputTx.inputs.forEach((input) => {
+          const txHash = input.prevout.hash;
+          const outIdx = input.prevout.index;
+
+          return db.txs.getTxById(txHash, (error, tx) => {
+            if (error || !tx) {
+              return logger.log('error',
+                `No Tx found: ${txHash} ${error}`);
+            }
+            return db.txs.updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address);
+          });
+        });
+      });
+    });
+}
+
 module.exports = {
   parse,
 };

server/models/block.js

@@ -1,24 +1,26 @@
 const mongoose = require('mongoose');
-const Transaction = require('./transaction');
+const config = require('../config');
 
 const Schema = mongoose.Schema;
+// These limits can be overriden higher up the stack
+const MAX_BLOCKS = config.api.max_blocks;
 
 const BlockSchema = new Schema({
-  hash:       { type:  String, default: '' },
-  height:     { type:  Number, default: 0 },
-  size:       { type:  Number, default: 0 },
-  version:    { type:  Number, default: 0 },
-  prevBlock:  { type:  String, default: '' },
-  merkleRoot: { type:  String, default: '' },
-  ts:         { type:  Number, default: 0 },
-  bits:       { type:  Number, default: 0 },
-  nonce:      { type:  Number, default: 0 },
+  hash:       { type: String, default: '' },
+  height:     { type: Number, default: 0 },
+  size:       { type: Number, default: 0 },
+  version:    { type: Number, default: 0 },
+  prevBlock:  { type: String, default: '' },
+  merkleRoot: { type: String, default: '' },
+  ts:         { type: Number, default: 0 },
+  bits:       { type: Number, default: 0 },
+  nonce:      { type: Number, default: 0 },
   txs:        [{ type: String, default: '' }],
-  chainwork:  { type:  Number, default: 0 },
-  reward:     { type:  Number, default: 0 },
-  network:    { type:  String, default: '' },
-  poolInfo:   { type:  Object, default: {} },
-  rawBlock:   { type:  String, default: '' },
+  chainwork:  { type: Number, default: 0 },
+  reward:     { type: Number, default: 0 },
+  network:    { type: String, default: '' },
+  poolInfo:   { type: Object, default: {} },
+  rawBlock:   { type: String, default: '' },
 }, {
   toJSON: {
     virtuals: true,
@@ -29,6 +31,31 @@ const BlockSchema = new Schema({
 BlockSchema.index({ hash: 1 });
 BlockSchema.index({ height: 1 });
 
-const Block = mongoose.model('Block', BlockSchema);
+BlockSchema.methods.byHeight = function blockByHeight(height, cb) {
+  return this.model('Block').findOne(
+    { height },
+    cb);
+};
 
-module.exports = Block;
+BlockSchema.methods.byHash = function byHash(hash, cb) {
+  return this.model('Block').findOne(
+    { hash },
+    cb);
+};
+
+BlockSchema.methods.getRawBlock = function getRawBlock(hash, cb) {
+  return this.model('Block').findOne(
+    { hash },
+    { rawBlock: 1 },
+    cb);
+};
+
+BlockSchema.methods.last = function lastBlocks(cb) {
+  return this.model('Block').find(
+    {},
+    cb)
+    .limit(MAX_BLOCKS)
+    .sort({ height: -1 });
+};
+
+module.exports = mongoose.model('Block', BlockSchema);

server/models/input.js

@@ -3,6 +3,7 @@ const mongoose = require('mongoose');
 const Schema   = mongoose.Schema;
 
 const InputSchema = new Schema({
+  value:    { type: Number, default: 0 },
   prevout:  { type: Object, default: {} },
   script:   { type: String, default: '' },
   witness:  { type: String, default: '' },
@@ -10,8 +11,6 @@ const InputSchema = new Schema({
   address:  { type: String, default: '' },
 });
 
-InputSchema.index({ address: 1 });
-
 const Input = mongoose.model('Input', InputSchema);
 
 module.exports = Input;

server/models/output.js

@@ -9,8 +9,6 @@ const OutputSchema = new Schema({
   type:    { type: String, default: '' },
 });
 
-OutputSchema.index({ address: 1 });
-
 const Output = mongoose.model('Output', OutputSchema);
 
 module.exports = Output;

server/models/transaction.js

@@ -1,29 +1,114 @@
 const mongoose = require('mongoose');
 const Input = require('./input');
 const Output = require('./output');
+const logger = require('../lib/logger');
+const config = require('../config');
 
 const Schema = mongoose.Schema;
+// These limits can be overriden higher up the stack
+const MAX_TXS = config.api.max_txs;
+const MAX_PAGE_TXS = config.api.max_page_txs;
 
 const TransactionSchema = new Schema({
-  hash: { type: String, default: '' },
+  hash:        { type: String, default: '' },
   witnessHash: { type: String, default: '' },
-  fee: { type: Number, default: 0 },
-  rate: { type: Number, default: 0 },
-  ps: { type: Number, default: 0 },
-  height: { type: Number, default: 0 },
-  block: { type: String, default: '' },
-  index: { type: Number, default: 0 },
-  version: { type: Number, default: 0 },
-  flag: { type: Number, default: 0 },
-  lockTime: { type: Number, default: 0 },
-  inputs: [Input.schema],
-  outputs: [Output.schema],
-  size: { type: Number, default: 0 },
-  network: { type: String, default: '' },
+  fee:         { type: Number, default: 0 },
+  rate:        { type: Number, default: 0 },
+  ps:          { type: Number, default: 0 },
+  height:      { type: Number, default: 0 },
+  block:       { type: String, default: '' },
+  index:       { type: Number, default: 0 },
+  version:     { type: Number, default: 0 },
+  flag:        { type: Number, default: 0 },
+  lockTime:    { type: Number, default: 0 },
+  inputs:      [Input.schema],
+  outputs:     [Output.schema],
+  size:        { type: Number, default: 0 },
+  network:     { type: String, default: '' },
 });
 
 TransactionSchema.index({ hash: 1 });
+TransactionSchema.index({ 'outputs.address': 1 });
+TransactionSchema.index({ 'inputs.address': 1 });
 
-const Transaction = mongoose.model('Transaction', TransactionSchema);
 
-module.exports = Transaction;
+TransactionSchema.methods.byId = function txById(txid, cb) {
+  return this.model('Transaction').findOne(
+    { hash: txid },
+    cb);
+};
+
+TransactionSchema.methods.byHash = function txByHash(hash, cb) {
+  return this.byId(hash, cb);
+};
+
+TransactionSchema.methods.byBlockHash = function txByBlockHash(hash, cb) {
+  return this.model('Transaction').find(
+    { block: hash },
+    cb)
+    .limit(MAX_TXS);
+};
+
+TransactionSchema.methods.byAddress = function txByAddress(address, cb) {
+  return this.model('Transaction').find(
+    {
+      $or: [
+        { 'inputs.address': address },
+        { 'outputs.address': address }],
+    },
+    cb)
+    .limit(MAX_TXS);
+};
+
+TransactionSchema.methods.countByBlock = function txByAddress(hash, cb) {
+  return this.model('Transaction').count(
+    { block: hash },
+    cb);
+};
+
+TransactionSchema.methods.countByAddress = function txByAddress(address, cb) {
+  return this.model('Transaction').count(
+    {
+      $or: [
+        { 'inputs.address': address },
+        { 'outputs.address': address }],
+    },
+    cb);
+};
+
+TransactionSchema.methods.last = function lastTx(cb) {
+  return this.model('Transaction').find(
+    {},
+    cb)
+    .limit(MAX_TXS)
+    .sort({ height: -1 });
+};
+
+TransactionSchema.methods.getEmptyInputs = function getEmptyInputs(cb) {
+  return this.model('Transaction').find({
+    'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' },
+    'inputs.address': '',
+  },
+  cb)
+    .limit(MAX_TXS);
+};
+
+TransactionSchema.methods.updateInput = function updateInput(txid, inputid, value, address) {
+  return this.model('Transaction').findOneAndUpdate(
+    { _id: txid, 'inputs._id': inputid },
+    {
+      $set: {
+        'inputs.$.value': value,
+        'inputs.$.address': address,
+      },
+    },
+    (err, tx) => {
+      if (err) {
+        logger.log('error',
+          `updateInput: ${err}`);
+      }
+    },
+  );
+};
+
+module.exports = mongoose.model('Transaction', TransactionSchema);

server/package.json

@@ -17,6 +17,7 @@
     "bitcore-message": "^1.0.4",
     "body-parser": "^1.17.2",
     "express": "^4.15.3",
+    "helmet": "^3.8.1",
     "mongoose": "^4.11.5",
     "request": "^2.81.0",
     "socket.io": "^2.0.3",