From c560ae3cff9733aef18a8a6d2f6c41f32adbb95d Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Wed, 23 Aug 2017 11:17:49 -0400 Subject: [PATCH 01/15] fix bestHeight bug --- server/lib/db/blocks.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index 623dcde..e4536bd 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -9,6 +9,7 @@ let bestBlockHeight = 0; // 1e9 limit = ~2M years from now // Mostly for sync to set height function bestHeight(height) { + height = parseInt(height, 10) || 0; if (Number.isInteger(height) && height > 0 && height < 1 * 1e9) { From a3941f75efa7dfcca64714db4f992f9f197155d8 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Wed, 23 Aug 2017 23:25:36 -0400 Subject: [PATCH 02/15] Update inputs --- server/lib/db/transactions.js | 30 ++++++++++++++++++++++++++++++ server/lib/node/index.js | 8 ++++++++ server/lib/parser/transaction.js | 28 +--------------------------- server/models/transaction.js | 7 ++++--- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 9c7b724..98d7f0c 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -1,5 +1,6 @@ const Transactions = require('../../models/transaction.js'); const config = require('../../config'); +const logger = require('../logger'); const Txs = new Transactions(); const MAX_PAGE_TXS = config.api.max_page_txs; @@ -39,7 +40,36 @@ function updateInput(txid, inputid, value, address) { return Txs.updateInput(txid, inputid, value, address); } +function auditInputs() { + getEmptyInputs( + (err, txs) => { + if (err) { + return logger.log('error', + `No Empty Inputs found: ${err.err}`); + } + // For each tx with unmarked inputs + logger.log('debug', + `Found ${txs.length} txs with inputs to update`); + + return txs.forEach((inputTx) => { + inputTx.inputs.forEach((input) => { + const txHash = input.prevout.hash; + const outIdx = input.prevout.index; + + return getTxById(txHash, (error, tx) => { + if (error || !tx) { + return logger.log('error', + `No Tx found: ${txHash} ${error}`); + } + return updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address); + }); + }); + }); + }); +} + module.exports = { + auditInputs, getEmptyInputs, getTopTransactions, getTxById, diff --git a/server/lib/node/index.js b/server/lib/node/index.js index 2f73f29..8bb864e 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -7,6 +7,7 @@ const socket = require('../../lib/api/socket'); const db = require('../../lib/db'); const node = new FullNode(config.bcoin); +let doneSyncing = false; function start() { node.open() @@ -22,6 +23,13 @@ function start() { TxParser.parse(entry, block.txs); socket.processBlock(entry, block); db.blocks.bestHeight(entry.height); + if (entry.height % 20 === 0 || doneSyncing) { + db.txs.auditInputs(); + } + }); + + node.chain.on('full', () => { + doneSyncing = true; }); node.on('error', (err) => { diff --git a/server/lib/parser/transaction.js b/server/lib/parser/transaction.js index 73c559e..c6bf908 100644 --- a/server/lib/parser/transaction.js +++ b/server/lib/parser/transaction.js @@ -7,6 +7,7 @@ const logger = require('../logger'); const db = require('../db'); function parse(entry, txs) { + // findEmptyInputs(); txs.forEach((tx) => { const txJSON = tx.toJSON(); const txRAW = tx.toRaw(); @@ -52,37 +53,10 @@ function parse(entry, txs) { if (err) { logger.log('error', err.message); } - - findEmptyInputs(); }); }); } -function findEmptyInputs() { - db.txs.getEmptyInputs( - (err, txs) => { - if (err) { - return logger.log('error', - `No Empty Inputs found: ${err.err}`); - } - // For each tx with unmarked inputs - return txs.forEach((inputTx) => { - inputTx.inputs.forEach((input) => { - const txHash = input.prevout.hash; - const outIdx = input.prevout.index; - - return db.txs.getTxById(txHash, (error, tx) => { - if (error || !tx) { - return logger.log('error', - `No Tx found: ${txHash} ${error}`); - } - return db.txs.updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address); - }); - }); - }); - }); -} - module.exports = { parse, }; diff --git a/server/models/transaction.js b/server/models/transaction.js index 6a03caf..4c7140c 100644 --- a/server/models/transaction.js +++ b/server/models/transaction.js @@ -87,13 +87,14 @@ TransactionSchema.methods.last = function lastTx(cb) { TransactionSchema.methods.getEmptyInputs = function getEmptyInputs(cb) { return this.model('Transaction').find({ 'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' }, - 'inputs.address': '', + 'inputs.value': 0, }, - cb) - .limit(MAX_TXS); + cb); }; TransactionSchema.methods.updateInput = function updateInput(txid, inputid, value, address) { + logger.log('debug', + `${txid} ${address}value is ${value}`); return this.model('Transaction').findOneAndUpdate( { _id: txid, 'inputs._id': inputid }, { From 2ccaf2fce0f17a0975646224e68fd2945dbe7d8d Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Thu, 24 Aug 2017 14:28:21 -0400 Subject: [PATCH 03/15] Reliably update inputs. Self-healing. Audit Mongo blockheights --- server/index.js | 15 ++++++++++----- server/lib/db/blocks.js | 32 ++++++++++++++++++++++++++------ server/lib/db/transactions.js | 22 ++++++++++------------ server/lib/node/index.js | 3 ++- server/models/block.js | 16 ++++++++++++---- server/models/transaction.js | 20 +++++++++----------- 6 files changed, 69 insertions(+), 39 deletions(-) diff --git a/server/index.js b/server/index.js index 6cf93ad..00ec06c 100644 --- a/server/index.js +++ b/server/index.js @@ -9,11 +9,16 @@ logger.log('debug', db.connect(config.mongodb.uri, config.mongodb.options); -db.connection.once('open', () => { - if (config.start_node) Bcoin.start(); - Api.listen(config.api.port, () => { - logger.log('debug', - 'listening on port 3000'); +db.connection.once('open', () => { + // DB Audit returns best height to node + db.blocks.findMissingBlocks((err, lastBestHeight) => { + // Pass height to node to start Sync + if (config.start_node) Bcoin.start(lastBestHeight); + + Api.listen(config.api.port, () => { + logger.log('debug', + 'listening on port 3000'); + }); }); }); diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index e4536bd..547319b 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -2,7 +2,6 @@ const Block = require('../../models/block.js'); const logger = require('../logger'); const config = require('../../config'); -const block = new Block(); let bestBlockHeight = 0; @@ -20,27 +19,48 @@ function bestHeight(height) { } function getRawBlock(hash, cb) { - return block.getRawBlock(hash, cb); + return Block.getRawBlock(hash, cb); } function byHeight(height, cb) { - return block.byHeight(height, cb); + return Block.byHeight(height, cb); } function getTopBlocks(cb) { - return block.last(cb); + return Block.last(cb); } function getByHash(hash, cb) { - return block.byHash(hash, cb); + return Block.byHash(hash, cb); } function getLastBlock(cb) { - return block.last(cb) + return Block.last(cb) .limit(1); } +// Returns the missing block if it exists. Otherwise, return tip. +function findMissingBlocks(cb) { + logger.log('debug', + 'Verifying Mongo Blockchain'); + return Block.getHeights((err, blocks) => { + if (err) { + return cb(err); + } + // Blocks are in ascending order + let lastGoodHeight = 0; + blocks.forEach((block) => { + if (lastGoodHeight !== block.height - 1) { + return lastGoodHeight; + } + lastGoodHeight = block.height; + }); + return lastGoodHeight; + }); +} + module.exports = { + findMissingBlocks, getRawBlock, getTopBlocks, getLastBlock, diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 98d7f0c..6f84645 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -2,44 +2,45 @@ const Transactions = require('../../models/transaction.js'); const config = require('../../config'); const logger = require('../logger'); -const Txs = new Transactions(); + const MAX_PAGE_TXS = config.api.max_page_txs; function getEmptyInputs(cb) { - return Txs.getEmptyInputs(cb); + return Transactions.getEmptyInputs(cb); } function getTopTransactions(cb) { - return Txs.last(cb); + return Transactions.last(cb); } function getTxById(txid, cb) { - return Txs.byId(txid, cb); + return Transactions.byId(txid, cb); } function getTxByBlock(blockHash, page, limit, cb) { - return Txs.byBlockHash(blockHash, cb) + return Transactions.byBlockHash(blockHash, cb) .skip(limit * page); } function getTxByAddress(address, page, limit, cb) { - return Txs.byAddress(address, cb) + return Transactions.byAddress(address, cb) .limit(limit) .skip(limit * page); } function getTxCountByBlock(blockHash, cb) { - return Txs.countByBlock(blockHash, cb); + return Transactions.countByBlock(blockHash, cb); } function getTxCountByAddress(address, cb) { - return Txs.countByAddress(address, cb); + return Transactions.countByAddress(address, cb); } function updateInput(txid, inputid, value, address) { - return Txs.updateInput(txid, inputid, value, address); + return Transactions.updateInput(txid, inputid, value, address); } +// Updates empty inputs with prevout addr & value function auditInputs() { getEmptyInputs( (err, txs) => { @@ -48,9 +49,6 @@ function auditInputs() { `No Empty Inputs found: ${err.err}`); } // For each tx with unmarked inputs - logger.log('debug', - `Found ${txs.length} txs with inputs to update`); - return txs.forEach((inputTx) => { inputTx.inputs.forEach((input) => { const txHash = input.prevout.hash; diff --git a/server/lib/node/index.js b/server/lib/node/index.js index 8bb864e..ee4171a 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -9,11 +9,12 @@ const db = require('../../lib/db'); const node = new FullNode(config.bcoin); let doneSyncing = false; -function start() { +function start(bestBlockHeight) { node.open() .then(() => { node.connect() .then(() => { + node.reset(bestBlockHeight); node.startSync(); }); }); diff --git a/server/models/block.js b/server/models/block.js index 9c18bc7..445e119 100644 --- a/server/models/block.js +++ b/server/models/block.js @@ -31,26 +31,26 @@ const BlockSchema = new Schema({ BlockSchema.index({ hash: 1 }); BlockSchema.index({ height: 1 }); -BlockSchema.methods.byHeight = function blockByHeight(height, cb) { +BlockSchema.statics.byHeight = function blockByHeight(height, cb) { return this.model('Block').findOne( { height }, cb); }; -BlockSchema.methods.byHash = function byHash(hash, cb) { +BlockSchema.statics.byHash = function byHash(hash, cb) { return this.model('Block').findOne( { hash }, cb); }; -BlockSchema.methods.getRawBlock = function getRawBlock(hash, cb) { +BlockSchema.statics.getRawBlock = function getRawBlock(hash, cb) { return this.model('Block').findOne( { hash }, { rawBlock: 1 }, cb); }; -BlockSchema.methods.last = function lastBlocks(cb) { +BlockSchema.statics.last = function lastBlocks(cb) { return this.model('Block').find( {}, cb) @@ -58,4 +58,12 @@ BlockSchema.methods.last = function lastBlocks(cb) { .sort({ height: -1 }); }; +BlockSchema.statics.getHeights = function findMissing(cb) { + return this.model('Block').find( + {}, + { height: 1 }, + cb) + .sort({ height: 1 }); +}; + module.exports = mongoose.model('Block', BlockSchema); diff --git a/server/models/transaction.js b/server/models/transaction.js index 4c7140c..2376357 100644 --- a/server/models/transaction.js +++ b/server/models/transaction.js @@ -32,24 +32,24 @@ TransactionSchema.index({ 'outputs.address': 1 }); TransactionSchema.index({ 'inputs.address': 1 }); -TransactionSchema.methods.byId = function txById(txid, cb) { +TransactionSchema.statics.byId = function txById(txid, cb) { return this.model('Transaction').findOne( { hash: txid }, cb); }; -TransactionSchema.methods.byHash = function txByHash(hash, cb) { +TransactionSchema.statics.byHash = function txByHash(hash, cb) { return this.byId(hash, cb); }; -TransactionSchema.methods.byBlockHash = function txByBlockHash(hash, cb) { +TransactionSchema.statics.byBlockHash = function txByBlockHash(hash, cb) { return this.model('Transaction').find( { block: hash }, cb) .limit(MAX_TXS); }; -TransactionSchema.methods.byAddress = function txByAddress(address, cb) { +TransactionSchema.statics.byAddress = function txByAddress(address, cb) { return this.model('Transaction').find( { $or: [ @@ -60,13 +60,13 @@ TransactionSchema.methods.byAddress = function txByAddress(address, cb) { .limit(MAX_TXS); }; -TransactionSchema.methods.countByBlock = function txByAddress(hash, cb) { +TransactionSchema.statics.countByBlock = function txByAddress(hash, cb) { return this.model('Transaction').count( { block: hash }, cb); }; -TransactionSchema.methods.countByAddress = function txByAddress(address, cb) { +TransactionSchema.statics.countByAddress = function txByAddress(address, cb) { return this.model('Transaction').count( { $or: [ @@ -76,7 +76,7 @@ TransactionSchema.methods.countByAddress = function txByAddress(address, cb) { cb); }; -TransactionSchema.methods.last = function lastTx(cb) { +TransactionSchema.statics.last = function lastTx(cb) { return this.model('Transaction').find( {}, cb) @@ -84,7 +84,7 @@ TransactionSchema.methods.last = function lastTx(cb) { .sort({ height: -1 }); }; -TransactionSchema.methods.getEmptyInputs = function getEmptyInputs(cb) { +TransactionSchema.statics.getEmptyInputs = function getEmptyInputs(cb) { return this.model('Transaction').find({ 'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' }, 'inputs.value': 0, @@ -92,9 +92,7 @@ TransactionSchema.methods.getEmptyInputs = function getEmptyInputs(cb) { cb); }; -TransactionSchema.methods.updateInput = function updateInput(txid, inputid, value, address) { - logger.log('debug', - `${txid} ${address}value is ${value}`); +TransactionSchema.statics.updateInput = function updateInput(txid, inputid, value, address) { return this.model('Transaction').findOneAndUpdate( { _id: txid, 'inputs._id': inputid }, { From 333eec3dd8c61841b322fd37b4206694f20d5154 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Thu, 24 Aug 2017 14:42:50 -0400 Subject: [PATCH 04/15] Reset Bcoin chain if blocks are missing from mongo. --- server/index.js | 7 +++++-- server/lib/db/blocks.js | 7 +++---- server/lib/node/index.js | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/server/index.js b/server/index.js index 00ec06c..5e164fb 100644 --- a/server/index.js +++ b/server/index.js @@ -12,9 +12,12 @@ db.connect(config.mongodb.uri, config.mongodb.options); db.connection.once('open', () => { // DB Audit returns best height to node - db.blocks.findMissingBlocks((err, lastBestHeight) => { + db.blocks.findMissingBlocks((err, bestBlockHeight) => { // Pass height to node to start Sync - if (config.start_node) Bcoin.start(lastBestHeight); + logger.log('debug', + `Starting Bcoin from best height: ${bestBlockHeight}`); + + if (config.start_node) Bcoin.start(bestBlockHeight); Api.listen(config.api.port, () => { logger.log('debug', diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index 547319b..78eba2f 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -50,12 +50,11 @@ function findMissingBlocks(cb) { // Blocks are in ascending order let lastGoodHeight = 0; blocks.forEach((block) => { - if (lastGoodHeight !== block.height - 1) { - return lastGoodHeight; + if (lastGoodHeight === block.height - 1) { + lastGoodHeight = block.height; } - lastGoodHeight = block.height; }); - return lastGoodHeight; + return cb(null, lastGoodHeight); }); } diff --git a/server/lib/node/index.js b/server/lib/node/index.js index ee4171a..a3b4b00 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -14,7 +14,7 @@ function start(bestBlockHeight) { .then(() => { node.connect() .then(() => { - node.reset(bestBlockHeight); + node.chain.reset(bestBlockHeight); node.startSync(); }); }); From 12cb8a0c299c34153470385c3395ff5eb3c930fe Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Thu, 24 Aug 2017 15:01:43 -0400 Subject: [PATCH 05/15] Graceful Shutdown for mongodb --- server/lib/db/index.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/lib/db/index.js b/server/lib/db/index.js index 32e5be1..974ca5b 100644 --- a/server/lib/db/index.js +++ b/server/lib/db/index.js @@ -9,6 +9,15 @@ mongoose.connection.on('error', (err) => { ${err}`); }); +process.on('SIGINT', gracefulExit).on('SIGTERM', gracefulExit); + +function gracefulExit() { + mongoose.connection.close(() => { + console.log('Mongoose connection with DB disconnected through app termination'); + process.exit(0); + }); +} + module.exports = { connect: mongoose.connect, connection: mongoose.connection, From afc063fb97e6d76d6306eb6118f994ae133ac4a1 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Fri, 25 Aug 2017 10:47:03 -0400 Subject: [PATCH 06/15] sync & async saving. Bleh --- server/config/index.js | 2 +- server/lib/api/transaction.js | 2 +- server/lib/db/index.js | 6 +++++- server/lib/db/transactions.js | 5 +++-- server/lib/node/index.js | 12 ++++++++---- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/server/config/index.js b/server/config/index.js index 011f3d8..79c87f3 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -4,7 +4,7 @@ const config = { bcoin_http: 'localhost', bcoin: { network: 'main', - db: 'leveldb', + db: 'mem', prefix: '.', checkpoints: true, workers: false, diff --git a/server/lib/api/transaction.js b/server/lib/api/transaction.js index a882ced..1552e07 100644 --- a/server/lib/api/transaction.js +++ b/server/lib/api/transaction.js @@ -23,7 +23,7 @@ module.exports = function transactionAPI(router) { const txid = req.params.txid || ''; db.txs.getTxById(txid, (err, transaction) => { - if (err) { + if (err || !transaction) { logger.log('error', `/tx/:tid getTxById: ${err.err}`); return res.status(404).send(); diff --git a/server/lib/db/index.js b/server/lib/db/index.js index 974ca5b..65af60d 100644 --- a/server/lib/db/index.js +++ b/server/lib/db/index.js @@ -11,9 +11,13 @@ mongoose.connection.on('error', (err) => { process.on('SIGINT', gracefulExit).on('SIGTERM', gracefulExit); +// Catastrophic Fails can still result in data loss function gracefulExit() { + logger.log('debug', + 'Graceful Shutdown Starting...'); mongoose.connection.close(() => { - console.log('Mongoose connection with DB disconnected through app termination'); + logger.log('debug', + 'Mongoose connection with DB disconnected through app termination'); process.exit(0); }); } diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 6f84645..35dde60 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -45,7 +45,7 @@ function auditInputs() { getEmptyInputs( (err, txs) => { if (err) { - return logger.log('error', + return logger.log('warn', `No Empty Inputs found: ${err.err}`); } // For each tx with unmarked inputs @@ -56,7 +56,8 @@ function auditInputs() { return getTxById(txHash, (error, tx) => { if (error || !tx) { - return logger.log('error', + // Mongo save is async. Bcoin is kinda sync... Does not mean the tx will not be found + return logger.log('warn', `No Tx found: ${txHash} ${error}`); } return updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address); diff --git a/server/lib/node/index.js b/server/lib/node/index.js index a3b4b00..458c872 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -6,9 +6,10 @@ const config = require('../../config'); const socket = require('../../lib/api/socket'); const db = require('../../lib/db'); -const node = new FullNode(config.bcoin); +const node = new FullNode(config.bcoin); let doneSyncing = false; + function start(bestBlockHeight) { node.open() .then(() => { @@ -20,13 +21,16 @@ function start(bestBlockHeight) { }); node.chain.on('connect', (entry, block) => { + // Saved block acts like a journal BlockParser.parse(entry, block); TxParser.parse(entry, block.txs); socket.processBlock(entry, block); db.blocks.bestHeight(entry.height); - if (entry.height % 20 === 0 || doneSyncing) { - db.txs.auditInputs(); - } + + node.chain.db.getBlockView(block) + .then((view) => { + console.log(view); + }); }); node.chain.on('full', () => { From df2beddf16b278db5bc403385bd81a0c8a672b0c Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Fri, 25 Aug 2017 17:25:26 -0400 Subject: [PATCH 07/15] Better input handling. Huge increase in performance and accuracy. --- server/lib/node/index.js | 12 +++---- server/lib/parser/transaction.js | 57 ++++++++++++++------------------ 2 files changed, 28 insertions(+), 41 deletions(-) diff --git a/server/lib/node/index.js b/server/lib/node/index.js index 458c872..cd99eb7 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -7,8 +7,6 @@ const socket = require('../../lib/api/socket'); const db = require('../../lib/db'); const node = new FullNode(config.bcoin); -let doneSyncing = false; - function start(bestBlockHeight) { node.open() @@ -21,20 +19,18 @@ function start(bestBlockHeight) { }); node.chain.on('connect', (entry, block) => { - // Saved block acts like a journal - BlockParser.parse(entry, block); - TxParser.parse(entry, block.txs); - socket.processBlock(entry, block); db.blocks.bestHeight(entry.height); node.chain.db.getBlockView(block) .then((view) => { - console.log(view); + const fullBlock = block.getJSON(node.network, view, entry.height); + BlockParser.parse(entry, block); + TxParser.parse(entry, fullBlock.txs); }); }); node.chain.on('full', () => { - doneSyncing = true; + }); node.on('error', (err) => { diff --git a/server/lib/parser/transaction.js b/server/lib/parser/transaction.js index c6bf908..6cc1482 100644 --- a/server/lib/parser/transaction.js +++ b/server/lib/parser/transaction.js @@ -7,44 +7,35 @@ const logger = require('../logger'); const db = require('../db'); function parse(entry, txs) { - // findEmptyInputs(); txs.forEach((tx) => { - const txJSON = tx.toJSON(); - const txRAW = tx.toRaw(); - const t = new TxModel({ - hash: txJSON.hash, - witnessHash: txJSON.witnessHash, - fee: txJSON.fee, - rate: txJSON.rate, - size: txRAW.length, - ps: txJSON.ps, + hash: tx.hash, + witnessHash: tx.witnessHash, + fee: tx.fee, + rate: tx.rate, + size: tx.size, + ps: tx.ps, height: entry.height, block: util.revHex(entry.hash), ts: entry.ts, - date: txJSON.date, - index: txJSON.index, - version: txJSON.version, - flag: txJSON.flag, - inputs: tx.inputs.map((input) => { - const inputJSON = input.toJSON(); - return new InputModel({ - prevout: inputJSON.prevout, - script: inputJSON.script, - witness: inputJSON.witness, - sequence: inputJSON.sequence, - address: inputJSON.address, - }); - }), - outputs: tx.outputs.map((output) => { - const outputJSON = output.toJSON(); - return new OutputModel({ - address: outputJSON.address, - script: outputJSON.script, - value: outputJSON.value, - }); - }), - lockTime: txJSON.locktime, + date: entry.tx, + index: tx.index, + version: tx.version, + flag: tx.flag, + inputs: tx.inputs.map(input => new InputModel({ + value: input.coin ? input.coin.value : 0, + prevout: input.prevout, + script: input.script, + witness: input.witness, + sequence: input.sequence, + address: input.coin ? input.coin.address : '', + })), + outputs: tx.outputs.map(output => new OutputModel({ + address: output.address, + script: output.script, + value: output.value, + })), + lockTime: tx.locktime, chain: config.bcoin.network, }); From cbcd42c0966da6388fa14d75fd5955f289272658 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Sun, 27 Aug 2017 21:53:42 -0400 Subject: [PATCH 08/15] remove tx.size until bcoin pr is merged --- server/config/index.js | 2 +- server/lib/parser/transaction.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/config/index.js b/server/config/index.js index 79c87f3..011f3d8 100644 --- a/server/config/index.js +++ b/server/config/index.js @@ -4,7 +4,7 @@ const config = { bcoin_http: 'localhost', bcoin: { network: 'main', - db: 'mem', + db: 'leveldb', prefix: '.', checkpoints: true, workers: false, diff --git a/server/lib/parser/transaction.js b/server/lib/parser/transaction.js index 6cc1482..43f634f 100644 --- a/server/lib/parser/transaction.js +++ b/server/lib/parser/transaction.js @@ -13,7 +13,6 @@ function parse(entry, txs) { witnessHash: tx.witnessHash, fee: tx.fee, rate: tx.rate, - size: tx.size, ps: tx.ps, height: entry.height, block: util.revHex(entry.hash), From 4ba24bba25d4ba31651d4b65729ffa6e355ff6bc Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Tue, 22 Aug 2017 17:04:46 -0400 Subject: [PATCH 09/15] input sanitization moved to middleware --- server/lib/api/address.js | 13 +-- server/lib/api/block.js | 13 --- server/lib/api/index.js | 5 +- server/lib/api/message.js | 7 -- server/lib/api/middleware/sanitizer.js | 140 +++++++++++++++++++++++++ server/lib/api/transaction.js | 97 +++++++---------- server/models/transaction.js | 1 - 7 files changed, 181 insertions(+), 95 deletions(-) create mode 100644 server/lib/api/middleware/sanitizer.js diff --git a/server/lib/api/address.js b/server/lib/api/address.js index cc9c077..1472f1b 100644 --- a/server/lib/api/address.js +++ b/server/lib/api/address.js @@ -1,21 +1,14 @@ const logger = require('../logger'); -const util = require('../util'); const db = require('../db'); module.exports = function AddressAPI(router) { router.get('/addr/:addr', (req, res) => { const addr = req.params.addr || ''; - if (!util.isBitcoinAddress(addr)) { - return res.status(404).send({ - error: 'Invalid bitcoin address', - }); - } - - return db.txs.getTxByAddress(addr, 0, 999999999, (error, txs) => { - if (error) { + return db.txs.getTxByAddress(addr, 0, 999999999, (err, txs) => { + if (err || txs.length === 0) { logger.log('error', - `getTxByBlock ${error}`); + `getTxByBlock ${err}`); return res.status(404).send(); } diff --git a/server/lib/api/block.js b/server/lib/api/block.js index 3041c2c..9ca7636 100644 --- a/server/lib/api/block.js +++ b/server/lib/api/block.js @@ -1,17 +1,10 @@ const logger = require('../logger'); const db = require('../db'); -const util = require('../util'); module.exports = function BlockAPI(router) { router.get('/block/:blockHash', (req, res) => { const blockHash = req.params.blockHash; - if (!util.isBlockHash(blockHash)) { - return res.status(404).send({ - error: 'Invalid bitcoin address', - }); - } - // Pass Mongo params, fields and limit to db api. return db.blocks.getByHash(blockHash, (err, block) => { @@ -70,12 +63,6 @@ module.exports = function BlockAPI(router) { router.get('/rawblock/:blockHash', (req, res) => { const blockHash = req.params.blockHash || ''; - if (!util.isBlockHash(blockHash)) { - return res.status(400).send({ - error: 'Invalid bitcoin address', - }); - } - // Pass Mongo params, fields and limit to db api. return db.blocks.getRawBlock(blockHash, (err, block) => { diff --git a/server/lib/api/index.js b/server/lib/api/index.js index c8aced1..db8693a 100644 --- a/server/lib/api/index.js +++ b/server/lib/api/index.js @@ -2,6 +2,7 @@ const express = require('express'); const config = require('../../config'); const bodyParser = require('body-parser'); const helmet = require('helmet'); +const sanitizer = require('./middleware/sanitizer'); const app = express(); const api = express.Router(); @@ -11,11 +12,9 @@ app.use(cors); app.use(helmet()); app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); - +app.use(sanitizer); // Serve insight ui front end from root dir public folder app.use(express.static('../app/www', { maxage: '1w' })); -// Legacy UI - useful for 1:1 compares -// app.use(express.static('./public', { maxage: '1w' })); app.set('json spaces', config.api.json_spaces); diff --git a/server/lib/api/message.js b/server/lib/api/message.js index e6e01b3..2ec2d13 100644 --- a/server/lib/api/message.js +++ b/server/lib/api/message.js @@ -1,17 +1,10 @@ const Message = require('bitcore-message'); -const util = require('../util'); // Copied from previous source function verifyMessage(req, res) { const address = req.body.address || req.query.address; const signature = req.body.signature || req.query.signature; const message = req.body.message || req.query.message; - if (!util.isBitcoinAddress(address)) { - return res.status(400).send({ - error: 'Invalid bitcoin address', - }); - } - if (!address || !signature || !message) { return res.json({ message: 'Missing parameters (expected "address", "signature" and "message")', diff --git a/server/lib/api/middleware/sanitizer.js b/server/lib/api/middleware/sanitizer.js new file mode 100644 index 0000000..a8fb1de --- /dev/null +++ b/server/lib/api/middleware/sanitizer.js @@ -0,0 +1,140 @@ +const util = require('../../util'); + +// Strip the request, sanitize inputs, rebuild +module.exports = function sanitize(req, res, next) { + const params = req.params || null; + const body = req.body || null; + const query = req.query || null; + + let cleanParams = null; + let cleanBody = null; + let cleanQuery = null; + + // req.params + if (params) { + // Transaction Id + if (params.txid && !util.isTxid(params.txid)) { + return res.status(404).send({ + error: 'Invalid Transaction Id', + }); + } + // Address + if (params.addr && typeof (params.addr) !== 'string') { + return res.status(404).send({ + error: 'Invalid Bitcoin Address', + }); + } + // Block Hash + if (params.blockHash && typeof (params.blockHash) !== 'string') { + return res.status(404).send({ + error: 'Invalid Block Hash', + }); + } + // Height + if (params.height) { + if (typeof (params.height) !== 'number') { + return res.status(404).send({ + error: 'Invalid Block Hash', + }); + } + params.height = parseInt(params.height, 10); + } + + cleanParams = { + txid: params.txid || null, + addr: params.addr || null, + blockHash: params.blockHash || null, + height: params.height || null, + }; + } + + // req.body + if (body) { + // Signature + if (body.signature && typeof (body.signature) !== 'string') { + return res.status(404).send({ + error: 'Invalid Signature', + }); + } + // Message + if (body.message && typeof (body.message) !== 'string') { + return res.status(404).send({ + error: 'Invalid Message', + }); + } + // Address + if (body.address && !util.isBitcoinAddress(body.address)) { + return res.status(404).send({ + error: 'Invalid Bitcoin Address', + }); + } + cleanBody = { + signature: body.signature || null, + message: body.message || null, + address: body.address || null, + }; + } + + if (query) { + // Address + if (query.address && !util.isBitcoinAddress(query.address)) { + return res.status(404).send({ + error: 'Invalid Bitcoin Address', + }); + } + // Signature + if (query.signature && typeof (query.signature) !== 'string') { + return res.status(404).send({ + error: 'Invalid Signature', + }); + } + // Message + if (query.message && typeof (query.message) !== 'string') { + return res.status(404).send({ + error: 'Invalid Message', + }); + } + // q + if (query.q && typeof (query.q) !== 'string') { + return res.status(404).send({ + error: 'Invalid Q', + }); + } + // Page Number + if (query.pageNum && typeof (query.pageNum) !== 'number') { + return res.status(404).send({ + error: 'Invalid Page Number', + }); + } + // Block (hash - implicit) + if (query.block && typeof (query.block) !== 'string') { + return res.status(404).send({ + error: 'Invalid Block', + }); + } + // Raw Tx + if (query.rawtx && typeof (query.rawtx) !== 'string') { + return res.status(404).send({ + error: 'Invalid Bitcoin Address', + }); + } + + cleanQuery = { + address: query.address || null, + signature: query.signature || null, + message: query.message || null, + q: query.q || null, + pageNum: query.pageNum || null, + block: query.block || null, + rawtx: query.rawtx || null, + }; + } + + // Strip off unexpected params + req.params = cleanParams; + req.body = cleanBody; + req.query = cleanQuery; + + return next(); +}; + diff --git a/server/lib/api/transaction.js b/server/lib/api/transaction.js index 1552e07..00b29c6 100644 --- a/server/lib/api/transaction.js +++ b/server/lib/api/transaction.js @@ -2,34 +2,24 @@ const logger = require('../logger'); const request = require('request'); const config = require('../../config'); const db = require('../db'); -const util = require('../util'); const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`; const MAX_TXS = config.api.max_txs; -const TTL = config.api.request_ttl; module.exports = function transactionAPI(router) { // Txs by txid router.get('/tx/:txid', (req, res) => { - if (!util.isTxid(req.params.txid)) { - return res.status(404).send({ - error: 'Invalid transaction id', - }); - } - // Get max block height for calculating confirmations const height = db.blocks.bestHeight(); - // Bcoin transaction data const txid = req.params.txid || ''; - db.txs.getTxById(txid, (err, transaction) => { - if (err || !transaction) { + return db.txs.getTxById(txid, (err, tx) => { + if (err || !tx) { logger.log('error', - `/tx/:tid getTxById: ${err.err}`); + `/tx/:tid getTxById: ${err ? err.err : ''}`); return res.status(404).send(); } - const tx = transaction; return res.send({ txid: tx.hash, version: tx.version, @@ -61,17 +51,9 @@ module.exports = function transactionAPI(router) { // query by address router.get('/txs', (req, res) => { const pageNum = parseInt(req.query.pageNum, 10) || 0; - const rangeStart = pageNum * MAX_TXS; - const rangeEnd = rangeStart + MAX_TXS; const height = db.blocks.bestHeight(); // get txs for blockhash, start with best height to calc confirmations if (req.query.block) { - if (!util.isBlockHash(req.query.block)) { - return res.status(400).send({ - error: 'Invalid block hash', - }); - } - return db.txs.getTxCountByBlock(req.query.block, (err, count) => { if (err) { logger.log('error', @@ -115,16 +97,10 @@ module.exports = function transactionAPI(router) { }); }); } else if (req.query.address) { - if (!util.isBitcoinAddress(req.query.address)) { - return res.status(400).send({ - error: 'Invalid bitcoin address', - }); - } - // Get txs by address, start with best height to calc confirmations const addr = req.query.address || ''; - db.txs.getTxCountByAddress(req.query.address, (err, count) => { + return db.txs.getTxCountByAddress(addr, (err, count) => { if (err) { logger.log('error', `getTxByBlock ${err}`); @@ -132,7 +108,7 @@ module.exports = function transactionAPI(router) { } const totalPages = Math.ceil(count / MAX_TXS); - return db.txs.getTxByAddress(req.query.address, pageNum, MAX_TXS, (error, txs) => { + return db.txs.getTxByAddress(addr, pageNum, MAX_TXS, (error, txs) => { if (error) { logger.log('error', `getTxByBlock ${error}`); @@ -165,39 +141,38 @@ module.exports = function transactionAPI(router) { }); }); }); - } else { - // Get last n txs - db.txs.getTopTransactions((err, txs) => { - if (err) { - logger.log('err', - `/txs getTopTransactions ${err}`); - return res.status(404).send(err); - } - return res.send(txs.map(tx => ({ - txid: tx.hash, - fees: tx.fee / 1e8, - size: tx.size, - confirmations: (height - tx.height) + 1, - valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, - vin: tx.inputs.map(input => ({ - scriptSig: { - asm: input.script, - }, - addr: input.address, - value: input.value / 1e8, - })), - vout: tx.outputs.map(output => ({ - scriptPubKey: { - asm: output.script, - addresses: [output.address], - }, - value: output.value / 1e8, - })), - isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', - })), - ); - }); } + // Get last n txs + return db.txs.getTopTransactions((err, txs) => { + if (err) { + logger.log('err', + `/txs getTopTransactions ${err}`); + return res.status(404).send(err); + } + return res.send(txs.map(tx => ({ + txid: tx.hash, + fees: tx.fee / 1e8, + size: tx.size, + confirmations: (height - tx.height) + 1, + valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8, + vin: tx.inputs.map(input => ({ + scriptSig: { + asm: input.script, + }, + addr: input.address, + value: input.value / 1e8, + })), + vout: tx.outputs.map(output => ({ + scriptPubKey: { + asm: output.script, + addresses: [output.address], + }, + value: output.value / 1e8, + })), + isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000', + })), + ); + }); }); router.get('/rawtx/:txid', (req, res) => res.send(req.params.txid)); diff --git a/server/models/transaction.js b/server/models/transaction.js index 2376357..f6f5c0e 100644 --- a/server/models/transaction.js +++ b/server/models/transaction.js @@ -7,7 +7,6 @@ const config = require('../config'); const Schema = mongoose.Schema; // These limits can be overriden higher up the stack const MAX_TXS = config.api.max_txs; -const MAX_PAGE_TXS = config.api.max_page_txs; const TransactionSchema = new Schema({ hash: { type: String, default: '' }, From 5f7e2b761396c9b3e8e28f91fb8ef9857d21e145 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Sun, 27 Aug 2017 22:30:41 -0400 Subject: [PATCH 10/15] removed old audit code --- server/index.js | 4 +--- server/lib/db/blocks.js | 6 +++--- server/lib/db/transactions.js | 32 -------------------------------- 3 files changed, 4 insertions(+), 38 deletions(-) diff --git a/server/index.js b/server/index.js index 5e164fb..a799335 100644 --- a/server/index.js +++ b/server/index.js @@ -9,10 +9,8 @@ logger.log('debug', db.connect(config.mongodb.uri, config.mongodb.options); - db.connection.once('open', () => { - // DB Audit returns best height to node - db.blocks.findMissingBlocks((err, bestBlockHeight) => { + db.blocks.getBestBlockHeight((err, bestBlockHeight) => { // Pass height to node to start Sync logger.log('debug', `Starting Bcoin from best height: ${bestBlockHeight}`); diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index 78eba2f..198f99c 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -39,8 +39,8 @@ function getLastBlock(cb) { .limit(1); } -// Returns the missing block if it exists. Otherwise, return tip. -function findMissingBlocks(cb) { +// Returns highest consecutive block height +function getBestBlockHeight(cb) { logger.log('debug', 'Verifying Mongo Blockchain'); return Block.getHeights((err, blocks) => { @@ -59,7 +59,7 @@ function findMissingBlocks(cb) { } module.exports = { - findMissingBlocks, + getBestBlockHeight, getRawBlock, getTopBlocks, getLastBlock, diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 35dde60..340116a 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -36,39 +36,8 @@ function getTxCountByAddress(address, cb) { return Transactions.countByAddress(address, cb); } -function updateInput(txid, inputid, value, address) { - return Transactions.updateInput(txid, inputid, value, address); -} - -// Updates empty inputs with prevout addr & value -function auditInputs() { - getEmptyInputs( - (err, txs) => { - if (err) { - return logger.log('warn', - `No Empty Inputs found: ${err.err}`); - } - // For each tx with unmarked inputs - return txs.forEach((inputTx) => { - inputTx.inputs.forEach((input) => { - const txHash = input.prevout.hash; - const outIdx = input.prevout.index; - - return getTxById(txHash, (error, tx) => { - if (error || !tx) { - // Mongo save is async. Bcoin is kinda sync... Does not mean the tx will not be found - return logger.log('warn', - `No Tx found: ${txHash} ${error}`); - } - return updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address); - }); - }); - }); - }); -} module.exports = { - auditInputs, getEmptyInputs, getTopTransactions, getTxById, @@ -76,5 +45,4 @@ module.exports = { getTxCountByBlock, getTxByAddress, getTxCountByAddress, - updateInput, }; From 56078e24874aa1155add363cd20881fd6549b9ef Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Sun, 27 Aug 2017 22:38:40 -0400 Subject: [PATCH 11/15] move cors to middleware --- server/lib/api/index.js | 2 +- server/lib/api/{ => middleware}/cors.js | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename server/lib/api/{ => middleware}/cors.js (100%) diff --git a/server/lib/api/index.js b/server/lib/api/index.js index db8693a..11d1994 100644 --- a/server/lib/api/index.js +++ b/server/lib/api/index.js @@ -6,7 +6,7 @@ const sanitizer = require('./middleware/sanitizer'); const app = express(); const api = express.Router(); -const cors = require('./cors'); +const cors = require('./middleware/cors'); app.use(cors); app.use(helmet()); diff --git a/server/lib/api/cors.js b/server/lib/api/middleware/cors.js similarity index 100% rename from server/lib/api/cors.js rename to server/lib/api/middleware/cors.js From b836390b22fb9ce256c5f22032ec2eb22ebbb22a Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Sun, 27 Aug 2017 22:53:39 -0400 Subject: [PATCH 12/15] remove sockets --- server/lib/api/index.js | 1 - server/lib/api/socket.js | 68 ---------------------------------------- server/package.json | 1 - 3 files changed, 70 deletions(-) delete mode 100644 server/lib/api/socket.js diff --git a/server/lib/api/index.js b/server/lib/api/index.js index 11d1994..7811a31 100644 --- a/server/lib/api/index.js +++ b/server/lib/api/index.js @@ -35,7 +35,6 @@ app.use((req, res) => res.status(404).send({ error: 'Not found', })); -// Socket server const server = require('http').Server(app); module.exports = { diff --git a/server/lib/api/socket.js b/server/lib/api/socket.js deleted file mode 100644 index fef4ee8..0000000 --- a/server/lib/api/socket.js +++ /dev/null @@ -1,68 +0,0 @@ -const server = require('.'); -const io = require('socket.io')(server); - -let refreshBlocks = false; -const txInterval = 200; -let txCounter = 0; - -// Not quite debouncing -setInterval(() => { - refreshBlocks = true; -}, 10000); - - -io.on('connection', (socket) => { - socket.on('subscribe', (data) => { - }); - - socket.on('message', (data) => { - }); - - socket.on('unsubscribe', (data) => { - }); - - socket.on('disconnect', (data) => { - }); -}); - -// Emit block refresh and txs -function processBlock(entry, block) { - if (refreshBlocks) { - refreshBlocks = false; - emitBlock(entry); - } - block.txs.forEach((tx) => { - txCounter++; - if (txCounter % txInterval === 0) { - txCounter = 0; - emitTx(tx); - } - }); -} - -function emitBlock(block) { - io.sockets.emit('block', { - hash: block.toJSON().hash, - }); -} - -function emitTx(transaction) { - const txJSON = transaction.toJSON(); - io.sockets.emit('tx', { - txid: txJSON.hash, - valueOut: transaction.outputs.reduce((sum, output) => { - output = output.toJSON(); - - const valB = (output.value || output.valueOut.value || 0) / 1e8; - - return sum + valB; - }, 0), - }); -} - -module.exports = { - io, - processBlock, - emitBlock, - emitTx, -}; diff --git a/server/package.json b/server/package.json index acdc50d..af0b5d1 100644 --- a/server/package.json +++ b/server/package.json @@ -20,7 +20,6 @@ "helmet": "^3.8.1", "mongoose": "^4.11.5", "request": "^2.81.0", - "socket.io": "^2.0.3", "winston": "^2.3.1" }, "devDependencies": { From 1a18a369c12855837c4c704196bf6da828cdf1ac Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Sun, 27 Aug 2017 23:44:08 -0400 Subject: [PATCH 13/15] Save logic moved to models. Parsers removed --- server/lib/db/blocks.js | 5 +++ server/lib/db/transactions.js | 11 +++--- server/lib/node/index.js | 21 ++++++++---- server/lib/parser/block.js | 42 ----------------------- server/lib/parser/index.js | 7 ---- server/lib/parser/transaction.js | 52 ---------------------------- server/models/block.js | 29 ++++++++++++++++ server/models/transaction.js | 59 ++++++++++++++++++++------------ 8 files changed, 91 insertions(+), 135 deletions(-) delete mode 100644 server/lib/parser/block.js delete mode 100644 server/lib/parser/index.js delete mode 100644 server/lib/parser/transaction.js diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index 198f99c..c4b6956 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -39,6 +39,10 @@ function getLastBlock(cb) { .limit(1); } +function saveBcoinBlock(entry, block, cb) { + return Block.saveBcoinBlock(entry, block, cb); +} + // Returns highest consecutive block height function getBestBlockHeight(cb) { logger.log('debug', @@ -66,4 +70,5 @@ module.exports = { getByHash, byHeight, bestHeight, + saveBcoinBlock, }; diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 340116a..7fa16a9 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -2,13 +2,8 @@ const Transactions = require('../../models/transaction.js'); const config = require('../../config'); const logger = require('../logger'); - const MAX_PAGE_TXS = config.api.max_page_txs; -function getEmptyInputs(cb) { - return Transactions.getEmptyInputs(cb); -} - function getTopTransactions(cb) { return Transactions.last(cb); } @@ -36,13 +31,17 @@ function getTxCountByAddress(address, cb) { return Transactions.countByAddress(address, cb); } +function saveBcoinTransactions(entry, txs, cb) { + return Transactions.saveBcoinTransactions(entry, txs, cb); +} + module.exports = { - getEmptyInputs, getTopTransactions, getTxById, getTxByBlock, getTxCountByBlock, getTxByAddress, getTxCountByAddress, + saveBcoinTransactions, }; diff --git a/server/lib/node/index.js b/server/lib/node/index.js index cd99eb7..fe5d7c8 100644 --- a/server/lib/node/index.js +++ b/server/lib/node/index.js @@ -1,9 +1,6 @@ const FullNode = require('bcoin/lib/node/fullnode'); const logger = require('../../lib/logger'); -const BlockParser = require('../parser').Block; -const TxParser = require('../parser').Transaction; const config = require('../../config'); -const socket = require('../../lib/api/socket'); const db = require('../../lib/db'); const node = new FullNode(config.bcoin); @@ -20,12 +17,24 @@ function start(bestBlockHeight) { node.chain.on('connect', (entry, block) => { db.blocks.bestHeight(entry.height); - + // Assemble Bcoin block data node.chain.db.getBlockView(block) .then((view) => { const fullBlock = block.getJSON(node.network, view, entry.height); - BlockParser.parse(entry, block); - TxParser.parse(entry, fullBlock.txs); + // Save the block + db.blocks.saveBcoinBlock(entry, block, (err) => { + if (err) { + logger.log('error', + `Error saving block ${err}`); + } + }); + // Save the Txs + db.txs.saveBcoinTransactions(entry, fullBlock.txs, (err) => { + if (err) { + logger.log('error', + `Error saving txs ${err}`); + } + }); }); }); diff --git a/server/lib/parser/block.js b/server/lib/parser/block.js deleted file mode 100644 index fb0000d..0000000 --- a/server/lib/parser/block.js +++ /dev/null @@ -1,42 +0,0 @@ -const BlockModel = require('../../models/block'); -const config = require('../../config'); -const util = require('../../lib/util'); -const logger = require('../logger'); - -function parse(entry, block) { - const rawBlock = block.toRaw().toString('hex'); - const blockJSON = block.toJSON(); - const reward = util.calcBlockReward(entry.height); - - // Can probably use destructuring to build something nicer - const newBlock = new BlockModel({ - hash: blockJSON.hash, - height: entry.height, - size: block.getSize(), - version: blockJSON.version, - prevBlock: blockJSON.prevBlock, - merkleRoot: blockJSON.merkleRoot, - ts: blockJSON.ts, - bits: blockJSON.bits, - nonce: blockJSON.nonce, - txs: block.txs.map((tx) => { - const txJSON = tx.toJSON(); - return txJSON.hash; - }), - chainwork: entry.chainwork, - reward, - network: config.bcoin.network, - poolInfo: {}, - rawBlock, - }); - - newBlock.save((err) => { - if (err) { - logger.log('error', err.message); - } - }); -} - -module.exports = { - parse, -}; diff --git a/server/lib/parser/index.js b/server/lib/parser/index.js deleted file mode 100644 index 8989195..0000000 --- a/server/lib/parser/index.js +++ /dev/null @@ -1,7 +0,0 @@ -const Block = require('./block'); -const Transaction = require('./transaction'); - -module.exports = { - Block, - Transaction, -}; diff --git a/server/lib/parser/transaction.js b/server/lib/parser/transaction.js deleted file mode 100644 index 43f634f..0000000 --- a/server/lib/parser/transaction.js +++ /dev/null @@ -1,52 +0,0 @@ -const TxModel = require('../../models/transaction'); -const InputModel = require('../../models/input'); -const OutputModel = require('../../models/output'); -const config = require('../../config'); -const util = require('../../lib/util'); -const logger = require('../logger'); -const db = require('../db'); - -function parse(entry, txs) { - txs.forEach((tx) => { - const t = new TxModel({ - hash: tx.hash, - witnessHash: tx.witnessHash, - fee: tx.fee, - rate: tx.rate, - ps: tx.ps, - height: entry.height, - block: util.revHex(entry.hash), - ts: entry.ts, - date: entry.tx, - index: tx.index, - version: tx.version, - flag: tx.flag, - inputs: tx.inputs.map(input => new InputModel({ - value: input.coin ? input.coin.value : 0, - prevout: input.prevout, - script: input.script, - witness: input.witness, - sequence: input.sequence, - address: input.coin ? input.coin.address : '', - })), - outputs: tx.outputs.map(output => new OutputModel({ - address: output.address, - script: output.script, - value: output.value, - })), - lockTime: tx.locktime, - chain: config.bcoin.network, - }); - - - t.save((err) => { - if (err) { - logger.log('error', err.message); - } - }); - }); -} - -module.exports = { - parse, -}; diff --git a/server/models/block.js b/server/models/block.js index 445e119..d02f4e1 100644 --- a/server/models/block.js +++ b/server/models/block.js @@ -1,5 +1,6 @@ const mongoose = require('mongoose'); const config = require('../config'); +const util = require('../lib/util'); const Schema = mongoose.Schema; // These limits can be overriden higher up the stack @@ -66,4 +67,32 @@ BlockSchema.statics.getHeights = function findMissing(cb) { .sort({ height: 1 }); }; +BlockSchema.statics.saveBcoinBlock = function saveBcoinBlock(entry, block, cb) { + const Block = this.model('Block'); + const rawBlock = block.toRaw().toString('hex'); + const blockJSON = block.toJSON(); + const reward = util.calcBlockReward(entry.height); + + return new Block({ + hash: blockJSON.hash, + height: entry.height, + size: block.getSize(), + version: blockJSON.version, + prevBlock: blockJSON.prevBlock, + merkleRoot: blockJSON.merkleRoot, + ts: blockJSON.ts, + bits: blockJSON.bits, + nonce: blockJSON.nonce, + txs: block.txs.map((tx) => { + const txJSON = tx.toJSON(); + return txJSON.hash; + }), + chainwork: entry.chainwork, + reward, + network: config.bcoin.network, + poolInfo: {}, + rawBlock, + }).save(cb); +}; + module.exports = mongoose.model('Block', BlockSchema); diff --git a/server/models/transaction.js b/server/models/transaction.js index f6f5c0e..bcce44b 100644 --- a/server/models/transaction.js +++ b/server/models/transaction.js @@ -3,6 +3,7 @@ const Input = require('./input'); const Output = require('./output'); const logger = require('../lib/logger'); const config = require('../config'); +const util = require('../lib/util'); const Schema = mongoose.Schema; // These limits can be overriden higher up the stack @@ -83,30 +84,44 @@ TransactionSchema.statics.last = function lastTx(cb) { .sort({ height: -1 }); }; -TransactionSchema.statics.getEmptyInputs = function getEmptyInputs(cb) { - return this.model('Transaction').find({ - 'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' }, - 'inputs.value': 0, - }, - cb); +TransactionSchema.statics.saveBcoinTransactions = function saveBcoinTransactions(entry, txs, cb) { + txs.forEach((tx) => { + this.saveBcoinTransaction(entry, tx, cb); + }); }; -TransactionSchema.statics.updateInput = function updateInput(txid, inputid, value, address) { - return this.model('Transaction').findOneAndUpdate( - { _id: txid, 'inputs._id': inputid }, - { - $set: { - 'inputs.$.value': value, - 'inputs.$.address': address, - }, - }, - (err, tx) => { - if (err) { - logger.log('error', - `updateInput: ${err}`); - } - }, - ); +TransactionSchema.statics.saveBcoinTransaction = function saveBcoinTransaction(entry, tx, cb) { + const Transaction = this.model('Transaction'); + return new Transaction({ + hash: tx.hash, + witnessHash: tx.witnessHash, + fee: tx.fee, + rate: tx.rate, + ps: tx.ps, + height: entry.height, + block: util.revHex(entry.hash), + ts: entry.ts, + date: entry.tx, + index: tx.index, + version: tx.version, + flag: tx.flag, + inputs: tx.inputs.map(input => new Input({ + value: input.coin ? input.coin.value : 0, + prevout: input.prevout, + script: input.script, + witness: input.witness, + sequence: input.sequence, + address: input.coin ? input.coin.address : '', + })), + outputs: tx.outputs.map(output => new Output({ + address: output.address, + script: output.script, + value: output.value, + })), + lockTime: tx.locktime, + chain: config.bcoin.network, + }) + .save(cb); }; module.exports = mongoose.model('Transaction', TransactionSchema); From ee0d5595b3b470548ba5a64e64ed78b6566ec5ac Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Mon, 28 Aug 2017 00:38:48 -0400 Subject: [PATCH 14/15] move models into lib --- server/lib/db/blocks.js | 2 +- server/lib/db/transactions.js | 2 +- server/{ => lib}/models/address.js | 0 server/{ => lib}/models/block.js | 4 ++-- server/{ => lib}/models/input.js | 0 server/{ => lib}/models/output.js | 0 server/{ => lib}/models/transaction.js | 6 +++--- server/test/model.test.js | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename server/{ => lib}/models/address.js (100%) rename server/{ => lib}/models/block.js (97%) rename server/{ => lib}/models/input.js (100%) rename server/{ => lib}/models/output.js (100%) rename server/{ => lib}/models/transaction.js (96%) diff --git a/server/lib/db/blocks.js b/server/lib/db/blocks.js index c4b6956..684fe15 100644 --- a/server/lib/db/blocks.js +++ b/server/lib/db/blocks.js @@ -1,4 +1,4 @@ -const Block = require('../../models/block.js'); +const Block = require('../models/block.js'); const logger = require('../logger'); const config = require('../../config'); diff --git a/server/lib/db/transactions.js b/server/lib/db/transactions.js index 7fa16a9..b9bad80 100644 --- a/server/lib/db/transactions.js +++ b/server/lib/db/transactions.js @@ -1,4 +1,4 @@ -const Transactions = require('../../models/transaction.js'); +const Transactions = require('../models/transaction.js'); const config = require('../../config'); const logger = require('../logger'); diff --git a/server/models/address.js b/server/lib/models/address.js similarity index 100% rename from server/models/address.js rename to server/lib/models/address.js diff --git a/server/models/block.js b/server/lib/models/block.js similarity index 97% rename from server/models/block.js rename to server/lib/models/block.js index d02f4e1..b5e0e91 100644 --- a/server/models/block.js +++ b/server/lib/models/block.js @@ -1,6 +1,6 @@ const mongoose = require('mongoose'); -const config = require('../config'); -const util = require('../lib/util'); +const config = require('../../config'); +const util = require('../util'); const Schema = mongoose.Schema; // These limits can be overriden higher up the stack diff --git a/server/models/input.js b/server/lib/models/input.js similarity index 100% rename from server/models/input.js rename to server/lib/models/input.js diff --git a/server/models/output.js b/server/lib/models/output.js similarity index 100% rename from server/models/output.js rename to server/lib/models/output.js diff --git a/server/models/transaction.js b/server/lib/models/transaction.js similarity index 96% rename from server/models/transaction.js rename to server/lib/models/transaction.js index bcce44b..2c85473 100644 --- a/server/models/transaction.js +++ b/server/lib/models/transaction.js @@ -1,9 +1,9 @@ const mongoose = require('mongoose'); const Input = require('./input'); const Output = require('./output'); -const logger = require('../lib/logger'); -const config = require('../config'); -const util = require('../lib/util'); +const logger = require('../logger'); +const config = require('../../config'); +const util = require('../util'); const Schema = mongoose.Schema; // These limits can be overriden higher up the stack diff --git a/server/test/model.test.js b/server/test/model.test.js index 2e2cfde..73a892a 100644 --- a/server/test/model.test.js +++ b/server/test/model.test.js @@ -1,5 +1,5 @@ const db = require('../lib/db'); -const Block = require('../models/block.js'); +const Block = require('../lib/models/block.js'); Block.findOne({}, (err, block) => { console.log(err); From 2146ff290b038864d00a2918d20a3d90d6a54761 Mon Sep 17 00:00:00 2001 From: tenthirtyone Date: Mon, 28 Aug 2017 12:49:46 -0400 Subject: [PATCH 15/15] index block on tx model --- server/lib/models/transaction.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/lib/models/transaction.js b/server/lib/models/transaction.js index 2c85473..ae93b20 100644 --- a/server/lib/models/transaction.js +++ b/server/lib/models/transaction.js @@ -28,6 +28,7 @@ const TransactionSchema = new Schema({ }); TransactionSchema.index({ hash: 1 }); +TransactionSchema.index({ block: 1 }); TransactionSchema.index({ 'outputs.address': 1 }); TransactionSchema.index({ 'inputs.address': 1 });