From d3641f3b0a09b2576efc8ad12673265aed53ec46 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 09:00:18 -0400 Subject: [PATCH] Pagination Optimization - Sort and paginate before getting full transaction details. - Only get detailed transaction information for items within the current page. - Improves the performance with large sets of transactions. --- lib/services/address/history.js | 155 +++++++++++--------- test/services/address/history.unit.js | 194 ++++++++++---------------- 2 files changed, 161 insertions(+), 188 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index ad36b2d6..c20bc964 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -21,12 +21,11 @@ function AddressHistory(args) { this.addresses = [args.addresses]; } this.transactionInfo = []; - this.transactions = {}; - this.sortedArray = []; + this.combinedArray = []; + this.detailedArray = []; } AddressHistory.MAX_ADDRESS_QUERIES = 20; -AddressHistory.MAX_TX_QUERIES = 10; AddressHistory.prototype.get = function(callback) { var self = this; @@ -45,8 +44,11 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } + self.combineTransactionInfo(); + self.sortAndPaginateCombinedArray(); + async.eachSeries( - self.transactionInfo, + self.combinedArray, function(txInfo, next) { self.getDetailedInfo(txInfo, next); }, @@ -54,9 +56,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - self.sortTransactionsIntoArray(); - self.paginateSortedArray(); - callback(null, self.sortedArray); + callback(null, self.detailedArray); } ); } @@ -103,86 +103,107 @@ AddressHistory.prototype.getTransactionInfo = function(address, next) { }); }; +/** + * This function combines results from getInputs and getOutputs by + * combining inputIndexes and outputIndexes for address transaction + * matching combinations. + */ +AddressHistory.prototype.combineTransactionInfo = function() { + var combinedArrayMap = {}; + this.combinedArray = []; + var l = this.transactionInfo.length; + for(var i = 0; i < l; i++) { + var item = this.transactionInfo[i]; + var mapKey = item.address + item.txid; + if (combinedArrayMap[mapKey] >= 0) { + var combined = this.combinedArray[combinedArrayMap[mapKey]]; + if (item.outputIndex >= 0) { + combined.satoshis += item.satoshis; + combined.outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + combined.inputIndexes.push(item.inputIndex); + } + } else { + item.outputIndexes = []; + item.inputIndexes = []; + if (item.outputIndex >= 0) { + item.outputIndexes.push(item.outputIndex); + delete item.outputIndex; + } else if (item.inputIndex >= 0) { + item.inputIndexes.push(item.inputIndex); + delete item.inputIndex; + } + this.combinedArray.push(item); + combinedArrayMap[mapKey] = this.combinedArray.length - 1; + } + } +}; + +AddressHistory.prototype.sortAndPaginateCombinedArray = function() { + this.combinedArray.sort(AddressHistory.sortByHeight); + if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { + this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); + } +}; + AddressHistory.sortByHeight = function(a, b) { // TODO consider timestamp for mempool transactions return a.height < b.height; }; -AddressHistory.prototype.paginateSortedArray = function() { - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.sortedArray = this.sortedArray.slice(this.options.from, this.options.to); - } -}; - AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { var self = this; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; - if (self.transactions[txInfo.address] && self.transactions[txInfo.address][txInfo.txid]) { - self.amendDetailedInfoWithSatoshis(txInfo); - setImmediate(next); - } else { - self.node.services.db.getTransactionWithBlockInfo( - txInfo.txid, - queryMempool, - function(err, transaction) { - if (err) { + self.node.services.db.getTransactionWithBlockInfo( + txInfo.txid, + queryMempool, + function(err, transaction) { + if (err) { + return next(err); + } + + transaction.populateInputs(self.node.services.db, [], function(err) { + if(err) { return next(err); } - transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { - return next(err); - } - var confirmations = 0; - if (transaction.__height >= 0) { - confirmations = self.node.services.db.tip.__height - transaction.__height + 1; - } - - if (!self.transactions[txInfo.address]) { - self.transactions[txInfo.address] = {}; - } - - self.transactions[txInfo.address][txInfo.txid] = { - address: txInfo.address, - satoshis: 0, - height: transaction.__height, - confirmations: confirmations, - timestamp: transaction.__timestamp, - // TODO bitcore should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - outputIndexes: [], - inputIndexes: [], - tx: transaction - }; - - self.amendDetailedInfoWithSatoshis(txInfo); - next(); + self.detailedArray.push({ + address: txInfo.address, + satoshis: self.getSatoshisDetail(transaction, txInfo), + height: transaction.__height, + confirmations: self.getConfirmationsDetail(transaction), + timestamp: transaction.__timestamp, + // TODO bitcore should return null instead of throwing error on coinbase + fees: !transaction.isCoinbase() ? transaction.getFee() : null, + outputIndexes: txInfo.outputIndexes, + inputIndexes: txInfo.inputIndexes, + tx: transaction }); - } - ); - } + + next(); + }); + } + ); }; -AddressHistory.prototype.amendDetailedInfoWithSatoshis = function(txInfo) { - var historyItem = this.transactions[txInfo.address][txInfo.txid]; - if (txInfo.outputIndex >= 0) { - historyItem.outputIndexes.push(txInfo.outputIndex); - historyItem.satoshis += txInfo.satoshis; - } else if (txInfo.inputIndex >= 0){ - historyItem.inputIndexes.push(txInfo.inputIndex); - historyItem.satoshis -= historyItem.tx.inputs[txInfo.inputIndex].output.satoshis; +AddressHistory.prototype.getConfirmationsDetail = function(transaction) { + var confirmations = 0; + if (transaction.__height >= 0) { + confirmations = this.node.services.db.tip.__height - transaction.__height + 1; } + return confirmations; }; -AddressHistory.prototype.sortTransactionsIntoArray = function() { - this.sortedArray = []; - for(var address in this.transactions) { - for(var txid in this.transactions[address]) { - this.sortedArray.push(this.transactions[address][txid]); +AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { + var satoshis = txInfo.satoshis || 0; + + if (txInfo.inputIndexes.length >= 0) { + for(var j = 0; j < txInfo.inputIndexes.length; j++) { + satoshis -= transaction.inputs[txInfo.inputIndexes[j]].output.satoshis; } } - this.sortedArray.sort(AddressHistory.sortByHeight); + return satoshis; }; module.exports = AddressHistory; diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 06970b4b..61925bfa 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -23,9 +23,9 @@ describe('Address Service History', function() { history.node.should.equal(node); history.options.should.equal(options); history.addresses.should.equal(addresses); - history.transactions.should.deep.equal({}); history.transactionInfo.should.deep.equal([]); - history.sortedArray.should.deep.equal([]); + history.combinedArray.should.deep.equal([]); + history.detailedArray.should.deep.equal([]); }); it('will set addresses an array if only sent a string', function() { var history = new AddressHistory({ @@ -46,10 +46,11 @@ describe('Address Service History', function() { addresses: addresses }); var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; + history.detailedArray = expected; + history.combinedArray = [{}]; history.getTransactionInfo = sinon.stub().callsArg(1); - history.paginateSortedArray = sinon.stub(); + history.combineTransactionInfo = sinon.stub(); + history.sortAndPaginateCombinedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArg(1); history.sortTransactionsIntoArray = sinon.stub(); history.get(function(err, results) { @@ -58,8 +59,8 @@ describe('Address Service History', function() { } history.getTransactionInfo.callCount.should.equal(1); history.getDetailedInfo.callCount.should.equal(1); - history.sortTransactionsIntoArray.callCount.should.equal(1); - history.paginateSortedArray.callCount.should.equal(1); + history.combineTransactionInfo.callCount.should.equal(1); + history.sortAndPaginateCombinedArray.callCount.should.equal(1); results.should.equal(expected); done(); }); @@ -236,7 +237,7 @@ describe('Address Service History', function() { }); }); - describe('#paginateSortedArray', function() { + describe('#sortAndPaginateCombinedArray', function() { it('from 0 to 2', function() { var history = new AddressHistory({ node: {}, @@ -246,21 +247,21 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(2); - history.sortedArray[0].height.should.equal(14); - history.sortedArray[1].height.should.equal(13); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(2); + history.combinedArray[0].height.should.equal(14); + history.combinedArray[1].height.should.equal(13); }); it('from 0 to 4 (exceeds length)', function() { var history = new AddressHistory({ @@ -271,22 +272,22 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(3); - history.sortedArray[0].height.should.equal(14); - history.sortedArray[1].height.should.equal(13); - history.sortedArray[2].height.should.equal(12); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(3); + history.combinedArray[0].height.should.equal(14); + history.combinedArray[1].height.should.equal(13); + history.combinedArray[2].height.should.equal(12); }); it('from 0 to 1', function() { var history = new AddressHistory({ @@ -297,20 +298,20 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(1); - history.sortedArray[0].height.should.equal(14); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(1); + history.combinedArray[0].height.should.equal(14); }); it('from 2 to 3', function() { var history = new AddressHistory({ @@ -321,20 +322,20 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(1); - history.sortedArray[0].height.should.equal(12); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(1); + history.combinedArray[0].height.should.equal(12); }); it('from 10 to 20 (out of range)', function() { var history = new AddressHistory({ @@ -345,19 +346,19 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(0); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(0); }); }); @@ -375,13 +376,10 @@ describe('Address Service History', function() { options: {}, addresses: [] }); - history.transactions[txid] = {}; - history.amendDetailedInfoWithSatoshis = sinon.stub(); history.getDetailedInfo(txid, function(err) { if (err) { throw err; } - history.amendDetailedInfoWithSatoshis.callCount.should.equal(1); history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); }); }); @@ -460,7 +458,8 @@ describe('Address Service History', function() { var transactionInfo = { txid: txid, timestamp: 1407292005, - outputIndex: 1, + outputIndexes: [1], + inputIndexes: [], satoshis: 48020000, address: txAddress }; @@ -468,7 +467,7 @@ describe('Address Service History', function() { if (err) { throw err; } - var info = history.transactions[txAddress][txid]; + var info = history.detailedArray[0]; info.address.should.equal(txAddress); info.satoshis.should.equal(48020000); info.height.should.equal(314159); @@ -481,94 +480,47 @@ describe('Address Service History', function() { }); }); }); - - describe('#amendDetailedInfoWithSatoshis', function() { - it('will amend info with inputIndex and subtract satoshis', function() { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + describe('#getConfirmationsDetail', function() { + it('the correct confirmations when included in the tip', function() { var history = new AddressHistory({ - node: {}, - options: {}, - addresses: [] - }); - history.transactions[address] = {}; - history.transactions[address][txid] = { - inputIndexes: [], - satoshis: 10, - tx: { - inputs: [ - { - output: { - satoshis: 3000 + node: { + services: { + db: { + tip: { + __height: 100 } } - ] - } - }; - history.amendDetailedInfoWithSatoshis({ - address: address, - txid: txid, - inputIndex: 0 + } + }, + options: {}, + addresses: [] }); - history.transactions[address][txid].inputIndexes.should.deep.equal([0]); - history.transactions[address][txid].satoshis.should.equal(-2990); + var transaction = { + __height: 100 + }; + history.getConfirmationsDetail(transaction).should.equal(1); }); - it('will amend info with outputIndex and add satoshis', function() { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + }); + describe('#getSatoshisDetail', function() { + it('subtract inputIndexes satoshis without outputIndexes', function() { var history = new AddressHistory({ node: {}, options: {}, addresses: [] }); - history.transactions[address] = {}; - history.transactions[address][txid] = { - outputIndexes: [], - satoshis: 10 - }; - history.amendDetailedInfoWithSatoshis({ - address: address, - txid: txid, - outputIndex: 10, - satoshis: 2000 - }); - history.transactions[address][txid].outputIndexes.should.deep.equal([10]); - history.transactions[address][txid].satoshis.should.equal(2010); - }); - }); - - describe('#sortTransactionIntoArray', function() { - it('will convert this.transactions into an array and sort by height', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 10, - to: 20 - }, - addresses: [] - }); - history.transactions = { - address1: { - txid1: { - height: 12 - }, - txid2: { - height: 14, - }, - txid3: { - height: 13 + var transaction = { + inputs: [ + { + output: { + satoshis: 10000 + } } - }, - address2: { - txid4: { - height: 15 - } - } + ] }; - history.sortTransactionsIntoArray(); - history.sortedArray.length.should.equal(4); - history.sortedArray[0].height.should.equal(15); - history.sortedArray[1].height.should.equal(14); - history.sortedArray[2].height.should.equal(13); - history.sortedArray[3].height.should.equal(12); + var txInfo = { + inputIndexes: [0] + }; + history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); }); }); });