Merge pull request #805 from tenthirtyone/next-merge

Next-merge API updates

server/index.js

@@ -10,10 +10,16 @@ logger.log('debug',
 db.connect(config.mongodb.uri, config.mongodb.options);
 
 db.connection.once('open', () => {
-  if (config.start_node) Bcoin.start();
-
-  Api.listen(config.api.port, () => {
+  db.blocks.getBestBlockHeight((err, bestBlockHeight) => {
+    // Pass height to node to start Sync
     logger.log('debug',
-      'listening on port 3000');
+      `Starting Bcoin from best height: ${bestBlockHeight}`);
+
+    if (config.start_node) Bcoin.start(bestBlockHeight);
+
+    Api.listen(config.api.port, () => {
+      logger.log('debug',
+        'listening on port 3000');
+    });
   });
 });

server/lib/api/address.js

@@ -1,21 +1,14 @@
 const logger  = require('../logger');
-const util    = require('../util');
 const db      = require('../db');
 
 module.exports = function AddressAPI(router) {
   router.get('/addr/:addr', (req, res) => {
     const addr = req.params.addr || '';
 
-    if (!util.isBitcoinAddress(addr)) {
-      return res.status(404).send({
-        error: 'Invalid bitcoin address',
-      });
-    }
-
-    return db.txs.getTxByAddress(addr, 0, 999999999, (error, txs) => {
-      if (error) {
+    return db.txs.getTxByAddress(addr, 0, 999999999, (err, txs) => {
+      if (err || txs.length === 0) {
         logger.log('error',
-          `getTxByBlock ${error}`);
+          `getTxByBlock ${err}`);
         return res.status(404).send();
       }
 

server/lib/api/block.js

@@ -1,17 +1,10 @@
 const logger = require('../logger');
 const db     = require('../db');
-const util   = require('../util');
 
 module.exports = function BlockAPI(router) {
   router.get('/block/:blockHash', (req, res) => {
     const blockHash = req.params.blockHash;
 
-    if (!util.isBlockHash(blockHash)) {
-      return res.status(404).send({
-        error: 'Invalid bitcoin address',
-      });
-    }
-
     // Pass Mongo params, fields and limit to db api.
     return db.blocks.getByHash(blockHash,
       (err, block) => {
@@ -70,12 +63,6 @@ module.exports = function BlockAPI(router) {
   router.get('/rawblock/:blockHash', (req, res) => {
     const blockHash = req.params.blockHash || '';
 
-    if (!util.isBlockHash(blockHash)) {
-      return res.status(400).send({
-        error: 'Invalid bitcoin address',
-      });
-    }
-
     // Pass Mongo params, fields and limit to db api.
     return db.blocks.getRawBlock(blockHash,
       (err, block) => {

server/lib/api/index.js

@@ -2,20 +2,19 @@ const express    = require('express');
 const config     = require('../../config');
 const bodyParser = require('body-parser');
 const helmet     = require('helmet');
+const sanitizer  = require('./middleware/sanitizer');
 
 const app = express();
 const api = express.Router();
-const cors = require('./cors');
+const cors = require('./middleware/cors');
 
 app.use(cors);
 app.use(helmet());
 app.use(bodyParser.urlencoded({ extended: false }));
 app.use(bodyParser.json());
-
+app.use(sanitizer);
 // Serve insight ui front end from root dir public folder
 app.use(express.static('../app/www', { maxage: '1w' }));
-// Legacy UI - useful for 1:1 compares
-// app.use(express.static('./public', { maxage: '1w' }));
 
 app.set('json spaces', config.api.json_spaces);
 
@@ -36,7 +35,6 @@ app.use((req, res) => res.status(404).send({
   error: 'Not found',
 }));
 
-// Socket server
 const server  = require('http').Server(app);
 
 module.exports = {

server/lib/api/message.js

@@ -1,17 +1,10 @@
 const Message = require('bitcore-message');
-const util    = require('../util');
 // Copied from previous source
 function verifyMessage(req, res) {
   const address   = req.body.address || req.query.address;
   const signature = req.body.signature || req.query.signature;
   const message   = req.body.message || req.query.message;
 
-  if (!util.isBitcoinAddress(address)) {
-    return res.status(400).send({
-      error: 'Invalid bitcoin address',
-    });
-  }
-
   if (!address || !signature || !message) {
     return res.json({
       message: 'Missing parameters (expected "address", "signature" and "message")',

server/lib/api/middleware/sanitizer.js

@@ -0,0 +1,140 @@
+const util = require('../../util');
+
+// Strip the request, sanitize inputs, rebuild
+module.exports = function sanitize(req, res, next) {
+  const params = req.params || null;
+  const body   = req.body || null;
+  const query  = req.query || null;
+
+  let cleanParams = null;
+  let cleanBody   = null;
+  let cleanQuery  = null;
+
+  // req.params
+  if (params) {
+    // Transaction Id
+    if (params.txid  && !util.isTxid(params.txid)) {
+      return res.status(404).send({
+        error: 'Invalid Transaction Id',
+      });
+    }
+    // Address
+    if (params.addr && typeof (params.addr) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Bitcoin Address',
+      });
+    }
+    // Block Hash
+    if (params.blockHash && typeof (params.blockHash) !== 'string')  {
+      return res.status(404).send({
+        error: 'Invalid Block Hash',
+      });
+    }
+    // Height
+    if (params.height) {
+      if (typeof (params.height) !== 'number')  {
+        return res.status(404).send({
+          error: 'Invalid Block Hash',
+        });
+      }
+      params.height = parseInt(params.height, 10);
+    }
+
+    cleanParams = {
+      txid:      params.txid      || null,
+      addr:      params.addr      || null,
+      blockHash: params.blockHash || null,
+      height:    params.height    || null,
+    };
+  }
+
+  // req.body
+  if (body) {
+    // Signature
+    if (body.signature && typeof (body.signature) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Signature',
+      });
+    }
+    // Message
+    if (body.message && typeof (body.message) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Message',
+      });
+    }
+    // Address
+    if (body.address && !util.isBitcoinAddress(body.address)) {
+      return res.status(404).send({
+        error: 'Invalid Bitcoin Address',
+      });
+    }
+    cleanBody = {
+      signature: body.signature || null,
+      message:   body.message   || null,
+      address:   body.address   || null,
+    };
+  }
+
+  if (query) {
+    // Address
+    if (query.address && !util.isBitcoinAddress(query.address)) {
+      return res.status(404).send({
+        error: 'Invalid Bitcoin Address',
+      });
+    }
+    // Signature
+    if (query.signature && typeof (query.signature) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Signature',
+      });
+    }
+    // Message
+    if (query.message && typeof (query.message) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Message',
+      });
+    }
+    // q
+    if (query.q && typeof (query.q) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Q',
+      });
+    }
+    // Page Number
+    if (query.pageNum && typeof (query.pageNum) !== 'number') {
+      return res.status(404).send({
+        error: 'Invalid Page Number',
+      });
+    }
+    // Block (hash - implicit)
+    if (query.block && typeof (query.block) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Block',
+      });
+    }
+    // Raw Tx
+    if (query.rawtx && typeof (query.rawtx) !== 'string') {
+      return res.status(404).send({
+        error: 'Invalid Bitcoin Address',
+      });
+    }
+
+    cleanQuery = {
+      address:   query.address   || null,
+      signature: query.signature || null,
+      message:   query.message   || null,
+      q:         query.q         || null,
+      pageNum:   query.pageNum   || null,
+      block:     query.block     || null,
+      rawtx:     query.rawtx     || null,
+    };
+  }
+
+  // Strip off unexpected params
+  req.params = cleanParams;
+  req.body   = cleanBody;
+  req.query  = cleanQuery;
+
+  return next();
+};
+

server/lib/api/socket.js

@@ -1,68 +0,0 @@
-const server = require('.');
-const io = require('socket.io')(server);
-
-let refreshBlocks = false;
-const txInterval = 200;
-let txCounter = 0;
-
-// Not quite debouncing
-setInterval(() => {
-  refreshBlocks = true;
-}, 10000);
-
-
-io.on('connection', (socket) => {
-  socket.on('subscribe', (data) => {
-  });
-
-  socket.on('message', (data) => {
-  });
-
-  socket.on('unsubscribe', (data) => {
-  });
-
-  socket.on('disconnect', (data) => {
-  });
-});
-
-// Emit block refresh and txs
-function processBlock(entry, block) {
-  if (refreshBlocks) {
-    refreshBlocks = false;
-    emitBlock(entry);
-  }
-  block.txs.forEach((tx) => {
-    txCounter++;
-    if (txCounter % txInterval === 0) {
-      txCounter = 0;
-      emitTx(tx);
-    }
-  });
-}
-
-function emitBlock(block) {
-  io.sockets.emit('block', {
-    hash: block.toJSON().hash,
-  });
-}
-
-function emitTx(transaction) {
-  const txJSON = transaction.toJSON();
-  io.sockets.emit('tx', {
-    txid: txJSON.hash,
-    valueOut: transaction.outputs.reduce((sum, output) => {
-      output = output.toJSON();
-
-      const valB = (output.value || output.valueOut.value || 0) / 1e8;
-
-      return sum + valB;
-    }, 0),
-  });
-}
-
-module.exports = {
-  io,
-  processBlock,
-  emitBlock,
-  emitTx,
-};

server/lib/api/transaction.js

@@ -2,34 +2,24 @@ const logger = require('../logger');
 const request = require('request');
 const config = require('../../config');
 const db = require('../db');
-const util = require('../util');
 
 const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`;
 const MAX_TXS = config.api.max_txs;
-const TTL     = config.api.request_ttl;
 
 module.exports = function transactionAPI(router) {
   // Txs by txid
   router.get('/tx/:txid', (req, res) => {
-    if (!util.isTxid(req.params.txid)) {
-      return res.status(404).send({
-        error: 'Invalid transaction id',
-      });
-    }
-
     // Get max block height for calculating confirmations
     const height = db.blocks.bestHeight();
-    // Bcoin transaction data
     const txid = req.params.txid || '';
 
-    db.txs.getTxById(txid, (err, transaction) => {
-      if (err) {
+    return db.txs.getTxById(txid, (err, tx) => {
+      if (err || !tx) {
         logger.log('error',
-          `/tx/:tid getTxById: ${err.err}`);
+          `/tx/:tid getTxById: ${err ? err.err : ''}`);
         return res.status(404).send();
       }
 
-      const tx = transaction;
       return res.send({
         txid: tx.hash,
         version: tx.version,
@@ -61,17 +51,9 @@ module.exports = function transactionAPI(router) {
   // query by address
   router.get('/txs', (req, res) => {
     const pageNum    = parseInt(req.query.pageNum, 10)  || 0;
-    const rangeStart = pageNum * MAX_TXS;
-    const rangeEnd   = rangeStart + MAX_TXS;
     const height     = db.blocks.bestHeight();
     // get txs for blockhash, start with best height to calc confirmations
     if (req.query.block) {
-      if (!util.isBlockHash(req.query.block)) {
-        return res.status(400).send({
-          error: 'Invalid block hash',
-        });
-      }
-
       return db.txs.getTxCountByBlock(req.query.block, (err, count) => {
         if (err) {
           logger.log('error',
@@ -115,16 +97,10 @@ module.exports = function transactionAPI(router) {
         });
       });
     } else if (req.query.address) {
-      if (!util.isBitcoinAddress(req.query.address)) {
-        return res.status(400).send({
-          error: 'Invalid bitcoin address',
-        });
-      }
-
       // Get txs by address, start with best height to calc confirmations
       const addr = req.query.address || '';
 
-      db.txs.getTxCountByAddress(req.query.address, (err, count) => {
+      return db.txs.getTxCountByAddress(addr, (err, count) => {
         if (err) {
           logger.log('error',
             `getTxByBlock ${err}`);
@@ -132,7 +108,7 @@ module.exports = function transactionAPI(router) {
         }
         const totalPages = Math.ceil(count / MAX_TXS);
 
-        return db.txs.getTxByAddress(req.query.address, pageNum, MAX_TXS, (error, txs) => {
+        return db.txs.getTxByAddress(addr, pageNum, MAX_TXS, (error, txs) => {
           if (error) {
             logger.log('error',
               `getTxByBlock ${error}`);
@@ -165,39 +141,38 @@ module.exports = function transactionAPI(router) {
           });
         });
       });
-    } else {
-      // Get last n txs
-      db.txs.getTopTransactions((err, txs) => {
-        if (err) {
-          logger.log('err',
-            `/txs getTopTransactions ${err}`);
-          return res.status(404).send(err);
-        }
-        return res.send(txs.map(tx => ({
-          txid: tx.hash,
-          fees: tx.fee / 1e8,
-          size: tx.size,
-          confirmations: (height - tx.height) + 1,
-          valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
-          vin: tx.inputs.map(input => ({
-            scriptSig: {
-              asm: input.script,
-            },
-            addr: input.address,
-            value: input.value / 1e8,
-          })),
-          vout: tx.outputs.map(output => ({
-            scriptPubKey: {
-              asm: output.script,
-              addresses: [output.address],
-            },
-            value: output.value / 1e8,
-          })),
-          isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
-        })),
-        );
-      });
     }
+    // Get last n txs
+    return db.txs.getTopTransactions((err, txs) => {
+      if (err) {
+        logger.log('err',
+          `/txs getTopTransactions ${err}`);
+        return res.status(404).send(err);
+      }
+      return res.send(txs.map(tx => ({
+        txid: tx.hash,
+        fees: tx.fee / 1e8,
+        size: tx.size,
+        confirmations: (height - tx.height) + 1,
+        valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
+        vin: tx.inputs.map(input => ({
+          scriptSig: {
+            asm: input.script,
+          },
+          addr: input.address,
+          value: input.value / 1e8,
+        })),
+        vout: tx.outputs.map(output => ({
+          scriptPubKey: {
+            asm: output.script,
+            addresses: [output.address],
+          },
+          value: output.value / 1e8,
+        })),
+        isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
+      })),
+      );
+    });
   });
 
   router.get('/rawtx/:txid', (req, res) => res.send(req.params.txid));

server/lib/db/blocks.js

@@ -1,14 +1,14 @@
-const Block = require('../../models/block.js');
+const Block = require('../models/block.js');
 const logger = require('../logger');
 const config = require('../../config');
 
-const block = new Block();
 
 let bestBlockHeight = 0;
 
 // 1e9 limit = ~2M years from now
 // Mostly for sync to set height
 function bestHeight(height) {
+  height = parseInt(height, 10) || 0;
   if (Number.isInteger(height) &&
     height > 0 &&
     height < 1 * 1e9) {
@@ -19,31 +19,56 @@ function bestHeight(height) {
 }
 
 function getRawBlock(hash, cb) {
-  return block.getRawBlock(hash, cb);
+  return Block.getRawBlock(hash, cb);
 }
 
 function byHeight(height, cb) {
-  return block.byHeight(height, cb);
+  return Block.byHeight(height, cb);
 }
 
 function getTopBlocks(cb) {
-  return block.last(cb);
+  return Block.last(cb);
 }
 
 function getByHash(hash, cb) {
-  return block.byHash(hash, cb);
+  return Block.byHash(hash, cb);
 }
 
 function getLastBlock(cb) {
-  return block.last(cb)
+  return Block.last(cb)
     .limit(1);
 }
 
+function saveBcoinBlock(entry, block, cb) {
+  return Block.saveBcoinBlock(entry, block, cb);
+}
+
+// Returns highest consecutive block height
+function getBestBlockHeight(cb) {
+  logger.log('debug',
+    'Verifying Mongo Blockchain');
+  return Block.getHeights((err, blocks) => {
+    if (err) {
+      return cb(err);
+    }
+    // Blocks are in ascending order
+    let lastGoodHeight = 0;
+    blocks.forEach((block) => {
+      if (lastGoodHeight === block.height - 1) {
+        lastGoodHeight = block.height;
+      }
+    });
+    return cb(null, lastGoodHeight);
+  });
+}
+
 module.exports = {
+  getBestBlockHeight,
   getRawBlock,
   getTopBlocks,
   getLastBlock,
   getByHash,
   byHeight,
   bestHeight,
+  saveBcoinBlock,
 };

server/lib/db/index.js

@@ -9,6 +9,19 @@ mongoose.connection.on('error', (err) => {
     ${err}`);
 });
 
+process.on('SIGINT', gracefulExit).on('SIGTERM', gracefulExit);
+
+// Catastrophic Fails can still result in data loss
+function gracefulExit() {
+  logger.log('debug',
+    'Graceful Shutdown Starting...');
+  mongoose.connection.close(() => {
+    logger.log('debug',
+      'Mongoose connection with DB disconnected through app termination');
+    process.exit(0);
+  });
+}
+
 module.exports = {
   connect:    mongoose.connect,
   connection: mongoose.connection,

server/lib/db/transactions.js

@@ -1,51 +1,47 @@
-const Transactions = require('../../models/transaction.js');
+const Transactions = require('../models/transaction.js');
 const config       = require('../../config');
+const logger       = require('../logger');
 
-const Txs = new Transactions();
 const MAX_PAGE_TXS = config.api.max_page_txs;
 
-function getEmptyInputs(cb) {
-  return Txs.getEmptyInputs(cb);
-}
-
 function getTopTransactions(cb) {
-  return Txs.last(cb);
+  return Transactions.last(cb);
 }
 
 function getTxById(txid, cb) {
-  return Txs.byId(txid, cb);
+  return Transactions.byId(txid, cb);
 }
 
 function getTxByBlock(blockHash, page, limit, cb) {
-  return Txs.byBlockHash(blockHash, cb)
+  return Transactions.byBlockHash(blockHash, cb)
     .skip(limit * page);
 }
 
 function getTxByAddress(address, page, limit, cb) {
-  return Txs.byAddress(address, cb)
+  return Transactions.byAddress(address, cb)
     .limit(limit)
     .skip(limit * page);
 }
 
 function getTxCountByBlock(blockHash, cb) {
-  return Txs.countByBlock(blockHash, cb);
+  return Transactions.countByBlock(blockHash, cb);
 }
 
 function getTxCountByAddress(address, cb) {
-  return Txs.countByAddress(address, cb);
+  return Transactions.countByAddress(address, cb);
 }
 
-function updateInput(txid, inputid, value, address) {
-  return Txs.updateInput(txid, inputid, value, address);
+function saveBcoinTransactions(entry, txs, cb) {
+  return Transactions.saveBcoinTransactions(entry, txs, cb);
 }
 
+
 module.exports = {
-  getEmptyInputs,
   getTopTransactions,
   getTxById,
   getTxByBlock,
   getTxCountByBlock,
   getTxByAddress,
   getTxCountByAddress,
-  updateInput,
+  saveBcoinTransactions,
 };

server/lib/models/block.js

@@ -1,5 +1,6 @@
 const mongoose = require('mongoose');
-const config = require('../config');
+const config = require('../../config');
+const util = require('../util');
 
 const Schema = mongoose.Schema;
 // These limits can be overriden higher up the stack
@@ -31,26 +32,26 @@ const BlockSchema = new Schema({
 BlockSchema.index({ hash: 1 });
 BlockSchema.index({ height: 1 });
 
-BlockSchema.methods.byHeight = function blockByHeight(height, cb) {
+BlockSchema.statics.byHeight = function blockByHeight(height, cb) {
   return this.model('Block').findOne(
     { height },
     cb);
 };
 
-BlockSchema.methods.byHash = function byHash(hash, cb) {
+BlockSchema.statics.byHash = function byHash(hash, cb) {
   return this.model('Block').findOne(
     { hash },
     cb);
 };
 
-BlockSchema.methods.getRawBlock = function getRawBlock(hash, cb) {
+BlockSchema.statics.getRawBlock = function getRawBlock(hash, cb) {
   return this.model('Block').findOne(
     { hash },
     { rawBlock: 1 },
     cb);
 };
 
-BlockSchema.methods.last = function lastBlocks(cb) {
+BlockSchema.statics.last = function lastBlocks(cb) {
   return this.model('Block').find(
     {},
     cb)
@@ -58,4 +59,40 @@ BlockSchema.methods.last = function lastBlocks(cb) {
     .sort({ height: -1 });
 };
 
+BlockSchema.statics.getHeights = function findMissing(cb) {
+  return this.model('Block').find(
+    {},
+    { height: 1 },
+    cb)
+    .sort({ height: 1 });
+};
+
+BlockSchema.statics.saveBcoinBlock = function saveBcoinBlock(entry, block, cb) {
+  const Block = this.model('Block');
+  const rawBlock = block.toRaw().toString('hex');
+  const blockJSON = block.toJSON();
+  const reward = util.calcBlockReward(entry.height);
+
+  return new Block({
+    hash: blockJSON.hash,
+    height: entry.height,
+    size: block.getSize(),
+    version: blockJSON.version,
+    prevBlock: blockJSON.prevBlock,
+    merkleRoot: blockJSON.merkleRoot,
+    ts: blockJSON.ts,
+    bits: blockJSON.bits,
+    nonce: blockJSON.nonce,
+    txs: block.txs.map((tx) => {
+      const txJSON = tx.toJSON();
+      return txJSON.hash;
+    }),
+    chainwork: entry.chainwork,
+    reward,
+    network: config.bcoin.network,
+    poolInfo: {},
+    rawBlock,
+  }).save(cb);
+};
+
 module.exports = mongoose.model('Block', BlockSchema);

server/lib/models/transaction.js

@@ -1,13 +1,13 @@
 const mongoose = require('mongoose');
 const Input = require('./input');
 const Output = require('./output');
-const logger = require('../lib/logger');
-const config = require('../config');
+const logger = require('../logger');
+const config = require('../../config');
+const util = require('../util');
 
 const Schema = mongoose.Schema;
 // These limits can be overriden higher up the stack
 const MAX_TXS = config.api.max_txs;
-const MAX_PAGE_TXS = config.api.max_page_txs;
 
 const TransactionSchema = new Schema({
   hash:        { type: String, default: '' },
@@ -28,28 +28,29 @@ const TransactionSchema = new Schema({
 });
 
 TransactionSchema.index({ hash: 1 });
+TransactionSchema.index({ block: 1 });
 TransactionSchema.index({ 'outputs.address': 1 });
 TransactionSchema.index({ 'inputs.address': 1 });
 
 
-TransactionSchema.methods.byId = function txById(txid, cb) {
+TransactionSchema.statics.byId = function txById(txid, cb) {
   return this.model('Transaction').findOne(
     { hash: txid },
     cb);
 };
 
-TransactionSchema.methods.byHash = function txByHash(hash, cb) {
+TransactionSchema.statics.byHash = function txByHash(hash, cb) {
   return this.byId(hash, cb);
 };
 
-TransactionSchema.methods.byBlockHash = function txByBlockHash(hash, cb) {
+TransactionSchema.statics.byBlockHash = function txByBlockHash(hash, cb) {
   return this.model('Transaction').find(
     { block: hash },
     cb)
     .limit(MAX_TXS);
 };
 
-TransactionSchema.methods.byAddress = function txByAddress(address, cb) {
+TransactionSchema.statics.byAddress = function txByAddress(address, cb) {
   return this.model('Transaction').find(
     {
       $or: [
@@ -60,13 +61,13 @@ TransactionSchema.methods.byAddress = function txByAddress(address, cb) {
     .limit(MAX_TXS);
 };
 
-TransactionSchema.methods.countByBlock = function txByAddress(hash, cb) {
+TransactionSchema.statics.countByBlock = function txByAddress(hash, cb) {
   return this.model('Transaction').count(
     { block: hash },
     cb);
 };
 
-TransactionSchema.methods.countByAddress = function txByAddress(address, cb) {
+TransactionSchema.statics.countByAddress = function txByAddress(address, cb) {
   return this.model('Transaction').count(
     {
       $or: [
@@ -76,7 +77,7 @@ TransactionSchema.methods.countByAddress = function txByAddress(address, cb) {
     cb);
 };
 
-TransactionSchema.methods.last = function lastTx(cb) {
+TransactionSchema.statics.last = function lastTx(cb) {
   return this.model('Transaction').find(
     {},
     cb)
@@ -84,31 +85,44 @@ TransactionSchema.methods.last = function lastTx(cb) {
     .sort({ height: -1 });
 };
 
-TransactionSchema.methods.getEmptyInputs = function getEmptyInputs(cb) {
-  return this.model('Transaction').find({
-    'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' },
-    'inputs.address': '',
-  },
-  cb)
-    .limit(MAX_TXS);
+TransactionSchema.statics.saveBcoinTransactions = function saveBcoinTransactions(entry, txs, cb) {
+  txs.forEach((tx) => {
+    this.saveBcoinTransaction(entry, tx, cb);
+  });
 };
 
-TransactionSchema.methods.updateInput = function updateInput(txid, inputid, value, address) {
-  return this.model('Transaction').findOneAndUpdate(
-    { _id: txid, 'inputs._id': inputid },
-    {
-      $set: {
-        'inputs.$.value': value,
-        'inputs.$.address': address,
-      },
-    },
-    (err, tx) => {
-      if (err) {
-        logger.log('error',
-          `updateInput: ${err}`);
-      }
-    },
-  );
+TransactionSchema.statics.saveBcoinTransaction = function saveBcoinTransaction(entry, tx, cb) {
+  const Transaction = this.model('Transaction');
+  return new Transaction({
+    hash: tx.hash,
+    witnessHash: tx.witnessHash,
+    fee: tx.fee,
+    rate: tx.rate,
+    ps: tx.ps,
+    height: entry.height,
+    block: util.revHex(entry.hash),
+    ts: entry.ts,
+    date: entry.tx,
+    index: tx.index,
+    version: tx.version,
+    flag: tx.flag,
+    inputs: tx.inputs.map(input => new Input({
+      value: input.coin ? input.coin.value : 0,
+      prevout: input.prevout,
+      script: input.script,
+      witness: input.witness,
+      sequence: input.sequence,
+      address: input.coin ? input.coin.address : '',
+    })),
+    outputs: tx.outputs.map(output => new Output({
+      address: output.address,
+      script: output.script,
+      value: output.value,
+    })),
+    lockTime: tx.locktime,
+    chain: config.bcoin.network,
+  })
+    .save(cb);
 };
 
 module.exports = mongoose.model('Transaction', TransactionSchema);

server/lib/node/index.js

@@ -1,27 +1,45 @@
 const FullNode    = require('bcoin/lib/node/fullnode');
 const logger      = require('../../lib/logger');
-const BlockParser = require('../parser').Block;
-const TxParser    = require('../parser').Transaction;
 const config      = require('../../config');
-const socket      = require('../../lib/api/socket');
 const db          = require('../../lib/db');
 
-const node = new FullNode(config.bcoin);
+const node      = new FullNode(config.bcoin);
 
-function start() {
+function start(bestBlockHeight) {
   node.open()
     .then(() => {
       node.connect()
         .then(() => {
+          node.chain.reset(bestBlockHeight);
           node.startSync();
         });
     });
 
   node.chain.on('connect', (entry, block) => {
-    BlockParser.parse(entry, block);
-    TxParser.parse(entry, block.txs);
-    socket.processBlock(entry, block);
     db.blocks.bestHeight(entry.height);
+    // Assemble Bcoin block data
+    node.chain.db.getBlockView(block)
+      .then((view) => {
+        const fullBlock = block.getJSON(node.network, view, entry.height);
+        // Save the block
+        db.blocks.saveBcoinBlock(entry, block, (err) => {
+          if (err) {
+            logger.log('error',
+              `Error saving block ${err}`);
+          }
+        });
+        // Save the Txs
+        db.txs.saveBcoinTransactions(entry, fullBlock.txs, (err) => {
+          if (err) {
+            logger.log('error',
+              `Error saving txs ${err}`);
+          }
+        });
+      });
+  });
+
+  node.chain.on('full', () => {
+
   });
 
   node.on('error', (err) => {

server/lib/parser/block.js

@@ -1,42 +0,0 @@
-const BlockModel = require('../../models/block');
-const config = require('../../config');
-const util = require('../../lib/util');
-const logger = require('../logger');
-
-function parse(entry, block) {
-  const rawBlock = block.toRaw().toString('hex');
-  const blockJSON = block.toJSON();
-  const reward = util.calcBlockReward(entry.height);
-
-  // Can probably use destructuring to build something nicer
-  const newBlock = new BlockModel({
-    hash: blockJSON.hash,
-    height: entry.height,
-    size: block.getSize(),
-    version: blockJSON.version,
-    prevBlock: blockJSON.prevBlock,
-    merkleRoot: blockJSON.merkleRoot,
-    ts: blockJSON.ts,
-    bits: blockJSON.bits,
-    nonce: blockJSON.nonce,
-    txs: block.txs.map((tx) => {
-      const txJSON = tx.toJSON();
-      return txJSON.hash;
-    }),
-    chainwork: entry.chainwork,
-    reward,
-    network: config.bcoin.network,
-    poolInfo: {},
-    rawBlock,
-  });
-
-  newBlock.save((err) => {
-    if (err) {
-      logger.log('error', err.message);
-    }
-  });
-}
-
-module.exports = {
-  parse,
-};

server/lib/parser/index.js

@@ -1,7 +0,0 @@
-const Block       = require('./block');
-const Transaction = require('./transaction');
-
-module.exports = {
-  Block,
-  Transaction,
-};

server/lib/parser/transaction.js

@@ -1,88 +0,0 @@
-const TxModel     = require('../../models/transaction');
-const InputModel  = require('../../models/input');
-const OutputModel = require('../../models/output');
-const config      = require('../../config');
-const util        = require('../../lib/util');
-const logger      = require('../logger');
-const db          = require('../db');
-
-function parse(entry, txs) {
-  txs.forEach((tx) => {
-    const txJSON = tx.toJSON();
-    const txRAW = tx.toRaw();
-
-    const t = new TxModel({
-      hash: txJSON.hash,
-      witnessHash: txJSON.witnessHash,
-      fee: txJSON.fee,
-      rate: txJSON.rate,
-      size: txRAW.length,
-      ps: txJSON.ps,
-      height: entry.height,
-      block: util.revHex(entry.hash),
-      ts: entry.ts,
-      date: txJSON.date,
-      index: txJSON.index,
-      version: txJSON.version,
-      flag: txJSON.flag,
-      inputs: tx.inputs.map((input) => {
-        const inputJSON = input.toJSON();
-        return new InputModel({
-          prevout: inputJSON.prevout,
-          script: inputJSON.script,
-          witness: inputJSON.witness,
-          sequence: inputJSON.sequence,
-          address: inputJSON.address,
-        });
-      }),
-      outputs: tx.outputs.map((output) => {
-        const outputJSON = output.toJSON();
-        return new OutputModel({
-          address: outputJSON.address,
-          script: outputJSON.script,
-          value: outputJSON.value,
-        });
-      }),
-      lockTime: txJSON.locktime,
-      chain: config.bcoin.network,
-    });
-
-
-    t.save((err) => {
-      if (err) {
-        logger.log('error', err.message);
-      }
-
-      findEmptyInputs();
-    });
-  });
-}
-
-function findEmptyInputs() {
-  db.txs.getEmptyInputs(
-    (err, txs) => {
-      if (err) {
-        return logger.log('error',
-          `No Empty Inputs found: ${err.err}`);
-      }
-      // For each tx with unmarked inputs
-      return txs.forEach((inputTx) => {
-        inputTx.inputs.forEach((input) => {
-          const txHash = input.prevout.hash;
-          const outIdx = input.prevout.index;
-
-          return db.txs.getTxById(txHash, (error, tx) => {
-            if (error || !tx) {
-              return logger.log('error',
-                `No Tx found: ${txHash} ${error}`);
-            }
-            return db.txs.updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address);
-          });
-        });
-      });
-    });
-}
-
-module.exports = {
-  parse,
-};

server/package.json

@@ -20,7 +20,6 @@
     "helmet": "^3.8.1",
     "mongoose": "^4.11.5",
     "request": "^2.81.0",
-    "socket.io": "^2.0.3",
     "winston": "^2.3.1"
   },
   "devDependencies": {

server/test/model.test.js

@@ -1,5 +1,5 @@
 const db = require('../lib/db');
-const Block = require('../models/block.js');
+const Block = require('../lib/models/block.js');
 
 Block.findOne({}, (err, block) => {
   console.log(err);