RanchiMall/flosight-api
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge mongo changes

Browse Source
This commit is contained in:
tenthirtyone 2017-08-19 20:59:37 -04:00
commit 968ab3c103
16 changed files with 317 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/config/index.js
View File

@ -1,5 +1,5 @@
const config = {
start_node: true,
start_node: false,
logging: 'debug',
bcoin_http: 'localhost',
bcoin: {

86
server/lib/api/address.js
View File

@ -2,6 +2,7 @@ const logger = require('../logger');
const request = require('request');
const config = require('../../config');
const util = require('../util');
const db = require('../db');
const API_URL = `http://${config.bcoin_http}:${config.bcoin['http-port']}`;
const TTL = config.api.request_ttl;
@ -10,63 +11,44 @@ module.exports = function AddressAPI(router) {
router.get('/addr/:addr', (req, res) => {
const addr = req.params.addr || '';
if (!util.isBitcoinAddress(addr)) {
return res.status(400).send({
error: 'Invalid bitcoin address',
});
}
db.txs.getTxByAddress(addr, 0, 999999999, (error, txs) => {
if (error) {
logger.log('error',
`getTxByBlock ${error}`);
return res.status(404).send();
}
logger.log('debug',
'Warning: Requesting data from Bcoin by address, may take some time');
// Get Bcoin data
return request(`${API_URL}/tx/address/${addr}`,
{ timeout: TTL },
(error, bcoinRes, bcoinTxs) => {
if (error) {
logger.log('error',
`${error}`);
return res.status(404).send({});
// Sum the matching outputs for every tx
const totalReceived = txs.reduce((total, tx) => total + tx.outputs.reduce((sum, output) => {
if (output.address === req.params.addr) {
return sum + output.value;
}
let txs = {};
try {
txs = JSON.parse(bcoinTxs);
} catch (e) {
logger.log('error',
`${e}`);
return res.status(404).send({});
return sum;
}, 0), 0) || 0;
// Sum the matching inputs for every tx
const totalSpent = txs.reduce((total, tx) => total + tx.inputs.reduce((sum, input) => {
if (input.coin && input.coin.address === req.params.addr) {
return sum + input.coin.value;
}
return sum;
}, 0), 0) || 0;
// Sum the matching outputs for every tx
const totalReceived = txs.reduce((total, tx) => total + tx.outputs.reduce((sum, output) => {
if (output.address === req.params.addr) {
return sum + output.value;
}
return sum;
}, 0), 0) || 0;
// Sum the matching inputs for every tx
const totalSpent = txs.reduce((total, tx) => total + tx.inputs.reduce((sum, input) => {
if (input.coin && input.coin.address === req.params.addr) {
return sum + input.coin.value;
}
return sum;
}, 0), 0) || 0;
// Match Insight API
return res.json({
addrStr: req.params.addr,
balance: (totalReceived - totalSpent) / 1e8,
balanceSat: totalReceived - totalSpent,
totalReceived: totalReceived / 1e8,
totalReceivedSat: totalReceived,
totalSent: totalSpent / 1e8,
totalSentSat: totalSpent,
unconfirmedBalance: 0,
unconfirmedBalanceSat: 0,
unconfirmedTxApperances: 0,
txApperances: txs.length,
});
// Match Insight API
return res.json({
addrStr: req.params.addr,
balance: (totalReceived - totalSpent) / 1e8,
balanceSat: totalReceived - totalSpent,
totalReceived: totalReceived / 1e8,
totalReceivedSat: totalReceived,
totalSent: totalSpent / 1e8,
totalSentSat: totalSpent,
unconfirmedBalance: 0,
unconfirmedBalanceSat: 0,
unconfirmedTxApperances: 0,
txApperances: txs.length,
});
});
});
// Stubbed by # to help with tasking

6
server/lib/api/block.js
View File

@ -60,7 +60,7 @@ module.exports = function BlockAPI(router) {
limit,
(err, blocks) => {
if (err) {
logger.log('err',
logger.log('error',
`/blocks: ${err}`);
return res.status(404).send();
}
@ -96,7 +96,7 @@ module.exports = function BlockAPI(router) {
1,
(err, block) => {
if (err) {
logger.log('err',
logger.log('error',
`/rawblock/:blockHash: ${err}`);
return res.status(404).send();
}
@ -113,7 +113,7 @@ module.exports = function BlockAPI(router) {
1,
(err, block) => {
if (err) {
logger.log('err',
logger.log('error',
`/block-index/:height: ${err}`);
return res.status(404).send();
}

6
server/lib/api/index.js
View File

@ -37,13 +37,11 @@ const MessageAPI = require('./message')(api);
app.use('/insight-api', api);
// 404
app.use((req, res) => {
res.status(404).send({
app.use((req, res) => res.status(404).send({
status: 404,
url: req.originalUrl,
error: 'Not found',
});
});
}));
// Socket server
const server = require('http').Server(app);

6
server/lib/api/message.js
View File

@ -27,7 +27,7 @@ function verifyMessage(req, res) {
code: 1,
});
}
res.json({ result: valid });
return res.json({ result: valid });
}
module.exports = function messageAPI(router) {
@ -39,7 +39,5 @@ module.exports = function messageAPI(router) {
verifyMessage(req, res);
});
router.get('/utils/estimatefee', (req, res) => {
res.send('estimate fees');
});
router.get('/utils/estimatefee', (req, res) => res.send('estimate fees'));
};

12
server/lib/api/status.js
View File

@ -48,12 +48,12 @@ module.exports = function statusAPI(router) {
} else {
getStatus((err, status) => {
if (err) {
logger.log('err',
logger.log('error',
`/status getStatus: ${err}`);
return res.status(404).send(err);
}
if (!status) {
logger.log('err',
logger.log('error',
'/status getStatus: no Status');
return res.status(404).send();
}
@ -79,12 +79,12 @@ module.exports = function statusAPI(router) {
router.get('/sync', (req, res) => {
getStatus((err, status) => {
if (err) {
logger.log('err',
logger.log('error',
`/sync: ${err}`);
return res.status(404).send(err);
}
if (!status) {
logger.log('err',
logger.log('error',
'/sync: no status');
return res.status(404).send();
}
@ -100,7 +100,7 @@ module.exports = function statusAPI(router) {
});
// Copied from previous source
router.get('/peer', (req, res) => {
res.json({
return res.json({
connected: true,
host: '127.0.0.1',
port: null,
@ -108,7 +108,7 @@ module.exports = function statusAPI(router) {
});
router.get('/version', (req, res) => {
res.json({
return res.json({
version: pkg.version,
});
});

195
server/lib/api/transaction.js
View File

@ -12,7 +12,7 @@ module.exports = function transactionAPI(router) {
// Txs by txid
router.get('/tx/:txid', (req, res) => {
if (!util.isTxid(req.params.txid)) {
return res.status(400).send({
return res.status(404).send({
error: 'Invalid transaction id',
});
}
@ -20,62 +20,46 @@ module.exports = function transactionAPI(router) {
// Get max block height for calculating confirmations
const height = db.blocks.bestHeight();
// Bcoin transaction data
return request(`${API_URL}/tx/${req.params.txid}`,
{ timeout: TTL },
(error, localRes, tx) => {
if (error) {
logger.log('error',
`${error}`);
return res.status(404).send();
}
// Catch JSON errors
try {
tx = JSON.parse(tx);
} catch (e) {
logger.log('error',
`${e}`);
return res.status(404).send();
}
if (!tx || !tx.hash) {
logger.log('error',
'No results found');
return res.status(404).send();
}
const txid = req.params.txid || '';
// Return UI JSON
return res.send({
txid: tx.hash,
version: tx.version,
time: tx.ps,
blocktime: tx.ps,
locktime: tx.locktime,
blockhash: tx.block,
fees: tx.fee / 1e8,
confirmations: (height - tx.height) + 1,
valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
vin: tx.inputs.map(input => ({
addr: input.coin ? input.coin.address : '',
value: input.coin ? input.coin.value / 1e8 : 0,
scriptSig: {
asm: input.script,
},
})),
vout: tx.outputs.map(output => ({
scriptPubKey: {
asm: output.script,
addresses: [output.address],
},
value: output.value / 1e8,
})),
isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
});
db.txs.getTxById(txid, (err, transaction) => {
if (err) {
logger.log('error',
`/tx/:tid getTxById: ${err.err}`);
return res.status(404).send();
}
const tx = transaction;
return res.send({
txid: tx.hash,
version: tx.version,
time: tx.ps,
blocktime: tx.ps,
locktime: tx.locktime,
blockhash: tx.block,
fees: tx.fee / 1e8,
size: tx.size,
confirmations: (height - tx.height) + 1,
valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
vin: tx.inputs.map(input => ({
addr: input.address,
value: input.value / 1e8,
})),
vout: tx.outputs.map(output => ({
scriptPubKey: {
addresses: [output.address],
},
value: output.value / 1e8,
})),
isCoinBase: tx.inputs[0].prevout.hash === '0000000000000000000000000000000000000000000000000000000000000000',
});
});
});
// /txs is overloaded. Next ver separate concerns
// query by block
// query by address
// last n txs
// last n txs - haha jk YOU 404
router.get('/txs', (req, res) => {
const pageNum = parseInt(req.query.pageNum, 10) || 0;
const rangeStart = pageNum * MAX_TXS;
@ -88,46 +72,35 @@ module.exports = function transactionAPI(router) {
});
}
const height = db.blocks.bestHeight();
// Get Bcoin data
return request(`${API_URL}/block/${req.query.block}`,
{ timeout: TTL },
(error, localRes, block) => {
return db.txs.getTxCountByBlock(req.query.block, (err, count) => {
if (err) {
logger.log('error',
`getTxByBlock ${err}`);
return res.status(404).send();
}
const totalPages = Math.ceil(count / MAX_TXS);
return db.txs.getTxByBlock(req.query.block, pageNum, MAX_TXS, (error, txs) => {
if (error) {
logger.log('error',
`${error}`);
`getTxByBlock ${error}`);
return res.status(404).send();
}
// Catch JSON errors
try {
block = JSON.parse(block);
} catch (e) {
logger.log('error',
`${e}`);
return res.status(404).send();
}
if (block.error) {
logger.log('error',
`${'No tx results'}`);
return res.status(404).send();
}
// Setup UI JSON
const totalPages = Math.ceil(block.txs.length / MAX_TXS);
block.txs = block.txs.slice(rangeStart, rangeEnd);
return res.send({
pagesTotal: totalPages,
txs: block.txs.map(tx => ({
txs: txs.map(tx => ({
txid: tx.hash,
fees: tx.fee / 1e8,
confirmations: (height - block.height) + 1,
size: tx.size,
confirmations: (height - tx.height) + 1,
valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
vin: tx.inputs.map(input => ({
addr: input.coin ? input.coin.address : '',
value: input.coin ? input.coin.value / 1e8 : 0,
scriptSig: {
asm: input.script,
},
addr: input.address,
value: input.value / 1e8,
})),
vout: tx.outputs.map(output => ({
scriptPubKey: {
@ -140,6 +113,7 @@ module.exports = function transactionAPI(router) {
})),
});
});
});
} else if (req.query.address) {
if (!util.isBitcoinAddress(req.query.address)) {
return res.status(400).send({
@ -151,45 +125,34 @@ module.exports = function transactionAPI(router) {
const height = db.blocks.bestHeight();
const addr = req.query.address || '';
logger.log('debug',
'Warning: Requesting data from Bcoin by address, may take some time');
db.txs.getTxCountByAddress(req.query.address, (err, count) => {
if (err) {
logger.log('error',
`getTxByBlock ${err}`);
return res.status(404).send();
}
const totalPages = Math.ceil(count / MAX_TXS);
return request(`${API_URL}/tx/address/${addr}`,
{ timeout: TTL },
(error, localRes, txs) => {
return db.txs.getTxByAddress(req.query.address, pageNum, MAX_TXS, (error, txs) => {
if (error) {
logger.log('error',
`${error}`);
`getTxByBlock ${error}`);
return res.status(404).send();
}
// Catch JSON errors
try {
txs = JSON.parse(txs);
} catch (e) {
logger.log('error',
`${e}`);
return res.status(404).send();
}
// Bcoin returns error as part of data object
if (txs.error) {
logger.log('error',
`${'No tx results'}`);
return res.status(404).send();
}
// Setup UI JSON
return res.send({
pagesTotal: 1,
pagesTotal: totalPages,
txs: txs.map(tx => ({
txid: tx.hash,
fees: tx.fee / 1e8,
confirmations: (height - tx.height) + 1,
size: tx.size,
confirmations: (height - tx.height) + 1,
valueOut: tx.outputs.reduce((sum, output) => sum + output.value, 0) / 1e8,
vin: tx.inputs.map(input => ({
addr: input.coin ? input.coin.address : '',
value: input.coin ? input.coin.value / 1e8 : 0,
scriptSig: {
asm: input.script,
},
addr: input.address,
value: input.value / 1e8,
})),
vout: tx.outputs.map(output => ({
scriptPubKey: {
@ -202,21 +165,21 @@ module.exports = function transactionAPI(router) {
})),
});
});
});
} else {
// Get last n txs
db.txs.getTopTransactions((err, txs) => {
if (err) {
logger.log('err',
`/txs getTopTransactions ${err}`);
return res.status(404).send(err);
}
return res.json(txs);
});
}
// Get last n txs
db.txs.getTopTransactions((err, txs) => {
if (err) {
logger.log('err',
`/txs getTopTransactions ${err}`);
return res.status(404).send(err);
}
return res.json(txs);
});
});
router.get('/rawtx/:txid', (req, res) => {
res.send(req.params.txid);
});
router.get('/rawtx/:txid', (req, res) => res.send(req.params.txid));
router.post('/tx/send', (req, res) => {
const rawtx = req.body.rawtx || '';
@ -228,10 +191,10 @@ module.exports = function transactionAPI(router) {
if (err) {
logger.log('error',
`${err}`);
return res.status(400).send(err);
return res.status(404).send(err);
}
res.json(true);
return res.json(true);
});
});
};

4
server/lib/db/blocks.js
View File

@ -6,6 +6,8 @@ const MAX_BLOCKS = config.api.max_blocks; // ~ 12 hours
let bestBlockHeight = 0;
// This naive querying will be replaced by more advanced mongo
function getBlocks(params, options, limit, cb) {
// Do not return mongo ids
const defaultOptions = { _id: 0 };
@ -56,7 +58,7 @@ function getBlock(params, options, limit, cb) {
return cb(null, blocks[0]);
});
}
// Highest known height in mongo
// Highest known height in mongo - Not Used
function getBestHeight() {
getBlock({}, {}, 1, (err, block) => {
if (err) {

125
server/lib/db/transactions.js
View File

@ -5,14 +5,14 @@ const config = require('../../config');
// For now, blocks handles these calls.
// These will be replaced with more advanced mongo
// No optimization yet.
// Will be replaced with a more sophisticated api soon
const MAX_TXS = config.api.max_txs;
const MAX_PAGE_TXS = config.api.max_page_txs;
// For Paging
function getTransactions(params, options, limit, cb) {
function getTransactions(params, options, limit, skip, cb) {
// Do not return mongo ids
const defaultOptions = { _id: 0 };
const defaultOptions = { };
// Copy over mongo options
Object.assign(defaultOptions, options);
// Simple sanitizing
@ -42,12 +42,13 @@ function getTransactions(params, options, limit, cb) {
}
return cb(null, txs);
})
.sort({ height: -1 })
.sort({ height: 1 })
.skip()
.limit(limit);
}
function getTransaction(params, options, limit, cb) {
getTransactions(params, options, limit, (err, tx) => {
function getTransaction(params, options, limit, skip, cb) {
getTransactions(params, options, limit, skip, (err, tx) => {
if (err) {
logger.log('error',
`getTransaction: ${err.err}`);
@ -82,8 +83,120 @@ function getTopTransactions(cb) {
.limit(MAX_TXS);
}
function getTxById(txid, cb) {
getTransaction(
{ hash: txid },
{ },
1,
0,
(err, transaction) => {
if (err) {
logger.log('error',
`getTxById: ${txid} ${err.err}`);
return cb(err);
}
return cb(null, transaction);
});
}
function getTxByBlock(blockHash, page, limit, cb) {
getTransactions(
{ block: blockHash },
{},
limit,
page * limit,
(err, tx) => {
if (err) {
logger.log('error',
`getTxByBlock: ${err.err}`);
return cb(err);
}
if (!tx.length > 0) {
return cb({ err: 'Tx not found' });
}
return cb(null, tx);
});
}
function getTxByAddress(address, page, limit, cb) {
getTransactions(
{
$or: [
{ 'inputs.address': address },
{ 'outputs.address': address }],
},
{},
limit,
page * limit,
(err, tx) => {
if (err) {
logger.log('error',
`getTxByAddress: ${err.err}`);
return cb(err);
}
if (!tx.length > 0) {
return cb({ err: 'Tx not found' });
}
return cb(null, tx);
});
}
function getTxCountByBlock(blockHash, cb) {
Transactions.count(
{ block: blockHash },
(err, count) => {
if (err) {
logger.log('error',
`getTxCountByBlock ${err}`);
return cb(err);
}
return cb(null, count);
});
}
function getTxCountByAddress(address, cb) {
Transactions.count(
{ $or: [
{ 'inputs.address': address },
{ 'outputs.address': address }],
},
(err, count) => {
if (err) {
logger.log('error',
`getTxCountByAddress ${err}`);
return cb(err);
}
return cb(null, count);
});
}
function updateInput(txid, inputid, value, address) {
Transactions.findOneAndUpdate(
{ _id: txid, 'inputs._id': inputid },
{
$set: {
'inputs.$.value': value,
'inputs.$.address': address,
},
},
(err, tx) => {
if (err) {
logger.log('error',
`updateInput: ${err}`);
}
},
);
}
module.exports = {
getTransaction,
getTransactions,
getTopTransactions,
getTxById,
getTxByBlock,
getTxCountByBlock,
getTxByAddress,
getTxCountByAddress,
updateInput,
};

6
server/lib/node/index.js
View File

@ -1,7 +1,7 @@
const FullNode = require('bcoin/lib/node/fullnode');
const logger = require('../../lib/logger');
const BlockParser = require('../parser').Block;
const TxParser = require('../parser').Transaction;
const TxParser = require('../parser').Transaction;
const config = require('../../config');
const socket = require('../../lib/api/socket');
const db = require('../../lib/db');
@ -24,6 +24,10 @@ function start() {
db.blocks.bestHeight(entry.height);
});
node.chain.on('full', (block) => {
});
node.on('error', (err) => {
logger.log('error',
`${err}`);

2
server/lib/parser/index.js
View File

@ -1,5 +1,5 @@
const Block = require('./block');
const Transaction = require('./transaction');
const Transaction = require('./transaction');
module.exports = {
Block,

53
server/lib/parser/transaction.js
View File

@ -1,10 +1,10 @@
const TxModel = require('../../models/transaction');
const InputModel = require('../../models/input');
const TxModel = require('../../models/transaction');
const InputModel = require('../../models/input');
const OutputModel = require('../../models/output');
const config = require('../../config');
const util = require('../../lib/util');
const logger = require('../logger');
const db = require('../db');
const config = require('../../config');
const util = require('../../lib/util');
const logger = require('../logger');
const db = require('../db');
// Bleh, Bcoin pulls in blocks 20 at a time
// Crappy delay for now otherwise async saves
@ -12,7 +12,10 @@ const db = require('../db');
// the last 20 that hasn't saved.
// Aggregate stuff will replace all of this.
let counter = 0;
function parse(entry, txs) {
counter++;
txs.forEach((tx) => {
const txJSON = tx.toJSON();
const txRAW = tx.toRaw();
@ -58,10 +61,48 @@ function parse(entry, txs) {
if (err) {
logger.log('error', err.message);
}
// As long as this modulo is divisible by 20 we should be OK for now.
// Closer to 20 = chattier at start but ideal later on
if (counter % 20 === 0) {
findEmptyInputs();
counter = 0;
}
});
});
}
function findEmptyInputs() {
db.txs.getTransactions(
{
'inputs.prevout.hash': { $ne: '0000000000000000000000000000000000000000000000000000000000000000' },
'inputs.address': '',
},
{},
100,
0,
(err, txs) => {
if (err) {
return logger.log('error',
`No Empty Inputs found: ${err.err}`);
}
// For each tx with unmarked inputs
return txs.forEach((inputTx) => {
inputTx.inputs.forEach((input) => {
const txHash = input.prevout.hash;
const outIdx = input.prevout.index;
return db.txs.getTxById(txHash, (err, tx) => {
if (err) {
return logger.log('error',
`No Tx found: ${txHash} ${err.err}`);
}
return db.txs.updateInput(inputTx._id, input._id, tx.outputs[outIdx].value, tx.outputs[outIdx].address);
});
});
});
});
}
module.exports = {
parse,
};

28
server/models/block.js
View File

@ -4,21 +4,21 @@ const Transaction = require('./transaction');
const Schema = mongoose.Schema;
const BlockSchema = new Schema({
hash: { type: String, default: '' },
height: { type: Number, default: 0 },
size: { type: Number, default: 0 },
version: { type: Number, default: 0 },
prevBlock: { type: String, default: '' },
merkleRoot: { type: String, default: '' },
ts: { type: Number, default: 0 },
bits: { type: Number, default: 0 },
nonce: { type: Number, default: 0 },
hash: { type: String, default: '' },
height: { type: Number, default: 0 },
size: { type: Number, default: 0 },
version: { type: Number, default: 0 },
prevBlock: { type: String, default: '' },
merkleRoot: { type: String, default: '' },
ts: { type: Number, default: 0 },
bits: { type: Number, default: 0 },
nonce: { type: Number, default: 0 },
txs: [{ type: String, default: '' }],
chainwork: { type: Number, default: 0 },
reward: { type: Number, default: 0 },
network: { type: String, default: '' },
poolInfo: { type: Object, default: {} },
rawBlock: { type: String, default: '' },
chainwork: { type: Number, default: 0 },
reward: { type: Number, default: 0 },
network: { type: String, default: '' },
poolInfo: { type: Object, default: {} },
rawBlock: { type: String, default: '' },
}, {
toJSON: {
virtuals: true,

1
server/models/input.js
View File

@ -3,6 +3,7 @@ const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const InputSchema = new Schema({
value: { type: Number, default: 0 },
prevout: { type: Object, default: {} },
script: { type: String, default: '' },
witness: { type: String, default: '' },

BIN
server/public/img/bcoin.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.6 KiB

BIN
server/public/img/leveldb.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB