RanchiMall/flocore-node
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pagination Optimization

Browse Source 
- Sort and paginate before getting full transaction details.
- Only get detailed transaction information for items within the current page.
- Improves the performance with large sets of transactions.
This commit is contained in:
Braydon Fuller 2015-09-14 09:00:18 -04:00
parent 1cf34f2dd8
commit d3641f3b0a
2 changed files with 161 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
lib/services/address/history.js
View File

@ -21,12 +21,11 @@ function AddressHistory(args) {
this.addresses = [args.addresses]; this.addresses = [args.addresses];
} }
this.transactionInfo = []; this.transactionInfo = [];
this.transactions = {}; this.combinedArray = [];
this.sortedArray = []; this.detailedArray = [];
} }
AddressHistory.MAX_ADDRESS_QUERIES = 20; AddressHistory.MAX_ADDRESS_QUERIES = 20;
AddressHistory.MAX_TX_QUERIES = 10;
AddressHistory.prototype.get = function(callback) { AddressHistory.prototype.get = function(callback) {
var self = this; var self = this;
@ -45,8 +44,11 @@ AddressHistory.prototype.get = function(callback) {
return callback(err); return callback(err);
} }
self.combineTransactionInfo();
self.sortAndPaginateCombinedArray();
async.eachSeries( async.eachSeries(
self.transactionInfo, self.combinedArray,
function(txInfo, next) { function(txInfo, next) {
self.getDetailedInfo(txInfo, next); self.getDetailedInfo(txInfo, next);
}, },
@ -54,9 +56,7 @@ AddressHistory.prototype.get = function(callback) {
if (err) { if (err) {
return callback(err); return callback(err);
} }
self.sortTransactionsIntoArray(); callback(null, self.detailedArray);
self.paginateSortedArray();
callback(null, self.sortedArray);
} }
); );
} }
@ -103,86 +103,107 @@ AddressHistory.prototype.getTransactionInfo = function(address, next) {
}); });
}; };
/**
* This function combines results from getInputs and getOutputs by
* combining inputIndexes and outputIndexes for address transaction
* matching combinations.
*/
AddressHistory.prototype.combineTransactionInfo = function() {
var combinedArrayMap = {};
this.combinedArray = [];
var l = this.transactionInfo.length;
for(var i = 0; i < l; i++) {
var item = this.transactionInfo[i];
var mapKey = item.address + item.txid;
if (combinedArrayMap[mapKey] >= 0) {
var combined = this.combinedArray[combinedArrayMap[mapKey]];
if (item.outputIndex >= 0) {
combined.satoshis += item.satoshis;
combined.outputIndexes.push(item.outputIndex);
} else if (item.inputIndex >= 0) {
combined.inputIndexes.push(item.inputIndex);
}
} else {
item.outputIndexes = [];
item.inputIndexes = [];
if (item.outputIndex >= 0) {
item.outputIndexes.push(item.outputIndex);
delete item.outputIndex;
} else if (item.inputIndex >= 0) {
item.inputIndexes.push(item.inputIndex);
delete item.inputIndex;
}
this.combinedArray.push(item);
combinedArrayMap[mapKey] = this.combinedArray.length - 1;
}
}
};
AddressHistory.prototype.sortAndPaginateCombinedArray = function() {
this.combinedArray.sort(AddressHistory.sortByHeight);
if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) {
this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to);
}
};
AddressHistory.sortByHeight = function(a, b) { AddressHistory.sortByHeight = function(a, b) {
// TODO consider timestamp for mempool transactions // TODO consider timestamp for mempool transactions
return a.height < b.height; return a.height < b.height;
}; };
AddressHistory.prototype.paginateSortedArray = function() {
if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) {
this.sortedArray = this.sortedArray.slice(this.options.from, this.options.to);
}
};
AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { AddressHistory.prototype.getDetailedInfo = function(txInfo, next) {
var self = this; var self = this;
var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool;
if (self.transactions[txInfo.address] && self.transactions[txInfo.address][txInfo.txid]) { self.node.services.db.getTransactionWithBlockInfo(
self.amendDetailedInfoWithSatoshis(txInfo); txInfo.txid,
setImmediate(next); queryMempool,
} else { function(err, transaction) {
self.node.services.db.getTransactionWithBlockInfo( if (err) {
txInfo.txid, return next(err);
queryMempool, }
function(err, transaction) {
if (err) { transaction.populateInputs(self.node.services.db, [], function(err) {
if(err) {
return next(err); return next(err);
} }
transaction.populateInputs(self.node.services.db, [], function(err) { self.detailedArray.push({
if(err) { address: txInfo.address,
return next(err); satoshis: self.getSatoshisDetail(transaction, txInfo),
} height: transaction.__height,
var confirmations = 0; confirmations: self.getConfirmationsDetail(transaction),
if (transaction.__height >= 0) { timestamp: transaction.__timestamp,
confirmations = self.node.services.db.tip.__height - transaction.__height + 1; // TODO bitcore should return null instead of throwing error on coinbase
} fees: !transaction.isCoinbase() ? transaction.getFee() : null,
outputIndexes: txInfo.outputIndexes,
if (!self.transactions[txInfo.address]) { inputIndexes: txInfo.inputIndexes,
self.transactions[txInfo.address] = {}; tx: transaction
}
self.transactions[txInfo.address][txInfo.txid] = {
address: txInfo.address,
satoshis: 0,
height: transaction.__height,
confirmations: confirmations,
timestamp: transaction.__timestamp,
// TODO bitcore should return null instead of throwing error on coinbase
fees: !transaction.isCoinbase() ? transaction.getFee() : null,
outputIndexes: [],
inputIndexes: [],
tx: transaction
};
self.amendDetailedInfoWithSatoshis(txInfo);
next();
}); });
}
); next();
} });
}
);
}; };
AddressHistory.prototype.amendDetailedInfoWithSatoshis = function(txInfo) { AddressHistory.prototype.getConfirmationsDetail = function(transaction) {
var historyItem = this.transactions[txInfo.address][txInfo.txid]; var confirmations = 0;
if (txInfo.outputIndex >= 0) { if (transaction.__height >= 0) {
historyItem.outputIndexes.push(txInfo.outputIndex); confirmations = this.node.services.db.tip.__height - transaction.__height + 1;
historyItem.satoshis += txInfo.satoshis;
} else if (txInfo.inputIndex >= 0){
historyItem.inputIndexes.push(txInfo.inputIndex);
historyItem.satoshis -= historyItem.tx.inputs[txInfo.inputIndex].output.satoshis;
} }
return confirmations;
}; };
AddressHistory.prototype.sortTransactionsIntoArray = function() { AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) {
this.sortedArray = []; var satoshis = txInfo.satoshis || 0;
for(var address in this.transactions) {
for(var txid in this.transactions[address]) { if (txInfo.inputIndexes.length >= 0) {
this.sortedArray.push(this.transactions[address][txid]); for(var j = 0; j < txInfo.inputIndexes.length; j++) {
satoshis -= transaction.inputs[txInfo.inputIndexes[j]].output.satoshis;
} }
} }
this.sortedArray.sort(AddressHistory.sortByHeight); return satoshis;
}; };
module.exports = AddressHistory; module.exports = AddressHistory;

194
test/services/address/history.unit.js
View File

@ -23,9 +23,9 @@ describe('Address Service History', function() {
history.node.should.equal(node); history.node.should.equal(node);
history.options.should.equal(options); history.options.should.equal(options);
history.addresses.should.equal(addresses); history.addresses.should.equal(addresses);
history.transactions.should.deep.equal({});
history.transactionInfo.should.deep.equal([]); history.transactionInfo.should.deep.equal([]);
history.sortedArray.should.deep.equal([]); history.combinedArray.should.deep.equal([]);
history.detailedArray.should.deep.equal([]);
}); });
it('will set addresses an array if only sent a string', function() { it('will set addresses an array if only sent a string', function() {
var history = new AddressHistory({ var history = new AddressHistory({
@ -46,10 +46,11 @@ describe('Address Service History', function() {
addresses: addresses addresses: addresses
}); });
var expected = [{}]; var expected = [{}];
history.sortedArray = expected; history.detailedArray = expected;
history.transactionInfo = [{}]; history.combinedArray = [{}];
history.getTransactionInfo = sinon.stub().callsArg(1); history.getTransactionInfo = sinon.stub().callsArg(1);
history.paginateSortedArray = sinon.stub(); history.combineTransactionInfo = sinon.stub();
history.sortAndPaginateCombinedArray = sinon.stub();
history.getDetailedInfo = sinon.stub().callsArg(1); history.getDetailedInfo = sinon.stub().callsArg(1);
history.sortTransactionsIntoArray = sinon.stub(); history.sortTransactionsIntoArray = sinon.stub();
history.get(function(err, results) { history.get(function(err, results) {
@ -58,8 +59,8 @@ describe('Address Service History', function() {
} }
history.getTransactionInfo.callCount.should.equal(1); history.getTransactionInfo.callCount.should.equal(1);
history.getDetailedInfo.callCount.should.equal(1); history.getDetailedInfo.callCount.should.equal(1);
history.sortTransactionsIntoArray.callCount.should.equal(1); history.combineTransactionInfo.callCount.should.equal(1);
history.paginateSortedArray.callCount.should.equal(1); history.sortAndPaginateCombinedArray.callCount.should.equal(1);
results.should.equal(expected); results.should.equal(expected);
done(); done();
}); });
@ -236,7 +237,7 @@ describe('Address Service History', function() {
}); });
}); });
describe('#paginateSortedArray', function() { describe('#sortAndPaginateCombinedArray', function() {
it('from 0 to 2', function() { it('from 0 to 2', function() {
var history = new AddressHistory({ var history = new AddressHistory({
node: {}, node: {},
@ -246,21 +247,21 @@ describe('Address Service History', function() {
}, },
addresses: [] addresses: []
}); });
history.sortedArray = [ history.combinedArray = [
{ {
height: 14 height: 13
}, },
{ {
height: 13, height: 14,
}, },
{ {
height: 12 height: 12
} }
]; ];
history.paginateSortedArray(); history.sortAndPaginateCombinedArray();
history.sortedArray.length.should.equal(2); history.combinedArray.length.should.equal(2);
history.sortedArray[0].height.should.equal(14); history.combinedArray[0].height.should.equal(14);
history.sortedArray[1].height.should.equal(13); history.combinedArray[1].height.should.equal(13);
}); });
it('from 0 to 4 (exceeds length)', function() { it('from 0 to 4 (exceeds length)', function() {
var history = new AddressHistory({ var history = new AddressHistory({
@ -271,22 +272,22 @@ describe('Address Service History', function() {
}, },
addresses: [] addresses: []
}); });
history.sortedArray = [ history.combinedArray = [
{ {
height: 14 height: 13
}, },
{ {
height: 13, height: 14,
}, },
{ {
height: 12 height: 12
} }
]; ];
history.paginateSortedArray(); history.sortAndPaginateCombinedArray();
history.sortedArray.length.should.equal(3); history.combinedArray.length.should.equal(3);
history.sortedArray[0].height.should.equal(14); history.combinedArray[0].height.should.equal(14);
history.sortedArray[1].height.should.equal(13); history.combinedArray[1].height.should.equal(13);
history.sortedArray[2].height.should.equal(12); history.combinedArray[2].height.should.equal(12);
}); });
it('from 0 to 1', function() { it('from 0 to 1', function() {
var history = new AddressHistory({ var history = new AddressHistory({
@ -297,20 +298,20 @@ describe('Address Service History', function() {
}, },
addresses: [] addresses: []
}); });
history.sortedArray = [ history.combinedArray = [
{ {
height: 14 height: 13
}, },
{ {
height: 13, height: 14,
}, },
{ {
height: 12 height: 12
} }
]; ];
history.paginateSortedArray(); history.sortAndPaginateCombinedArray();
history.sortedArray.length.should.equal(1); history.combinedArray.length.should.equal(1);
history.sortedArray[0].height.should.equal(14); history.combinedArray[0].height.should.equal(14);
}); });
it('from 2 to 3', function() { it('from 2 to 3', function() {
var history = new AddressHistory({ var history = new AddressHistory({
@ -321,20 +322,20 @@ describe('Address Service History', function() {
}, },
addresses: [] addresses: []
}); });
history.sortedArray = [ history.combinedArray = [
{ {
height: 14 height: 13
}, },
{ {
height: 13, height: 14,
}, },
{ {
height: 12 height: 12
} }
]; ];
history.paginateSortedArray(); history.sortAndPaginateCombinedArray();
history.sortedArray.length.should.equal(1); history.combinedArray.length.should.equal(1);
history.sortedArray[0].height.should.equal(12); history.combinedArray[0].height.should.equal(12);
}); });
it('from 10 to 20 (out of range)', function() { it('from 10 to 20 (out of range)', function() {
var history = new AddressHistory({ var history = new AddressHistory({
@ -345,19 +346,19 @@ describe('Address Service History', function() {
}, },
addresses: [] addresses: []
}); });
history.sortedArray = [ history.combinedArray = [
{ {
height: 14 height: 13
}, },
{ {
height: 13, height: 14,
}, },
{ {
height: 12 height: 12
} }
]; ];
history.paginateSortedArray(); history.sortAndPaginateCombinedArray();
history.sortedArray.length.should.equal(0); history.combinedArray.length.should.equal(0);
}); });
}); });
@ -375,13 +376,10 @@ describe('Address Service History', function() {
options: {}, options: {},
addresses: [] addresses: []
}); });
history.transactions[txid] = {};
history.amendDetailedInfoWithSatoshis = sinon.stub();
history.getDetailedInfo(txid, function(err) { history.getDetailedInfo(txid, function(err) {
if (err) { if (err) {
throw err; throw err;
} }
history.amendDetailedInfoWithSatoshis.callCount.should.equal(1);
history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0);
}); });
}); });
@ -460,7 +458,8 @@ describe('Address Service History', function() {
var transactionInfo = { var transactionInfo = {
txid: txid, txid: txid,
timestamp: 1407292005, timestamp: 1407292005,
outputIndex: 1, outputIndexes: [1],
inputIndexes: [],
satoshis: 48020000, satoshis: 48020000,
address: txAddress address: txAddress
}; };
@ -468,7 +467,7 @@ describe('Address Service History', function() {
if (err) { if (err) {
throw err; throw err;
} }
var info = history.transactions[txAddress][txid]; var info = history.detailedArray[0];
info.address.should.equal(txAddress); info.address.should.equal(txAddress);
info.satoshis.should.equal(48020000); info.satoshis.should.equal(48020000);
info.height.should.equal(314159); info.height.should.equal(314159);
@ -481,94 +480,47 @@ describe('Address Service History', function() {
}); });
}); });
}); });
describe('#getConfirmationsDetail', function() {
describe('#amendDetailedInfoWithSatoshis', function() { it('the correct confirmations when included in the tip', function() {
it('will amend info with inputIndex and subtract satoshis', function() {
var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0';
var history = new AddressHistory({ var history = new AddressHistory({
node: {}, node: {
options: {}, services: {
addresses: [] db: {
}); tip: {
history.transactions[address] = {}; __height: 100
history.transactions[address][txid] = {
inputIndexes: [],
satoshis: 10,
tx: {
inputs: [
{
output: {
satoshis: 3000
} }
} }
] }
} },
}; options: {},
history.amendDetailedInfoWithSatoshis({ addresses: []
address: address,
txid: txid,
inputIndex: 0
}); });
history.transactions[address][txid].inputIndexes.should.deep.equal([0]); var transaction = {
history.transactions[address][txid].satoshis.should.equal(-2990); __height: 100
};
history.getConfirmationsDetail(transaction).should.equal(1);
}); });
it('will amend info with outputIndex and add satoshis', function() { });
var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; describe('#getSatoshisDetail', function() {
it('subtract inputIndexes satoshis without outputIndexes', function() {
var history = new AddressHistory({ var history = new AddressHistory({
node: {}, node: {},
options: {}, options: {},
addresses: [] addresses: []
}); });
history.transactions[address] = {}; var transaction = {
history.transactions[address][txid] = { inputs: [
outputIndexes: [], {
satoshis: 10 output: {
}; satoshis: 10000
history.amendDetailedInfoWithSatoshis({ }
address: address,
txid: txid,
outputIndex: 10,
satoshis: 2000
});
history.transactions[address][txid].outputIndexes.should.deep.equal([10]);
history.transactions[address][txid].satoshis.should.equal(2010);
});
});
describe('#sortTransactionIntoArray', function() {
it('will convert this.transactions into an array and sort by height', function() {
var history = new AddressHistory({
node: {},
options: {
from: 10,
to: 20
},
addresses: []
});
history.transactions = {
address1: {
txid1: {
height: 12
},
txid2: {
height: 14,
},
txid3: {
height: 13
} }
}, ]
address2: {
txid4: {
height: 15
}
}
}; };
history.sortTransactionsIntoArray(); var txInfo = {
history.sortedArray.length.should.equal(4); inputIndexes: [0]
history.sortedArray[0].height.should.equal(15); };
history.sortedArray[1].height.should.equal(14); history.getSatoshisDetail(transaction, txInfo).should.equal(-10000);
history.sortedArray[2].height.should.equal(13);
history.sortedArray[3].height.should.equal(12);
}); });
}); });
}); });