From 8d2f69c5fddd0cbf3504e2833b8c771995c261dc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 11 Jan 2016 14:34:04 -0500 Subject: [PATCH] Address Service: Restored multi-address history queries - Restored functionality to be able to query the history of multiple addresses in one query - Sorted mempool transactions by timestamp in txid lists --- lib/services/address/constants.js | 11 ++- lib/services/address/history.js | 121 ++++++++++++++++-------------- lib/services/address/index.js | 35 ++++++--- 3 files changed, 94 insertions(+), 73 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 6ab3aba1..00fbafbe 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -36,16 +36,21 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); +exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); // The total number of transactions that an address can receive before it will start // to cache the summary to disk. exports.SUMMARY_CACHE_THRESHOLD = 10000; - -// The default maximum length queries +// The maximum number of inputs that can be queried at once exports.MAX_INPUTS_QUERY_LENGTH = 50000; +// The maximum number of outputs that can be queried at once exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; -exports.MAX_HISTORY_QUERY_LENGTH = 1000; +// The maximum number of transactions that can be queried at once +exports.MAX_HISTORY_QUERY_LENGTH = 100; +// The maximum number of addresses that can be queried at once +exports.MAX_ADDRESSES_QUERY = 100; module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js index dc15766c..f1b42cd5 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -22,7 +22,8 @@ function AddressHistory(args) { this.addresses = [args.addresses]; } - this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH; + this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; + this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; this.addressStrings = []; for (var i = 0; i < this.addresses.length; i++) { @@ -39,7 +40,33 @@ function AddressHistory(args) { this.detailedArray = []; } -AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.prototype._mergeAndSortTxids = function(summaries) { + var appearanceIds = {}; + var unconfirmedAppearanceIds = {}; + for (var i = 0; i < summaries.length; i++) { + var summary = summaries[i]; + for (var key in summary.appearanceIds) { + appearanceIds[key] = summary.appearanceIds[key]; + delete summary.appearanceIds[key]; + } + for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key]; + delete summary.unconfirmedAppearanceIds[key]; + } + } + var confirmedTxids = Object.keys(appearanceIds); + confirmedTxids.sort(function(a, b) { + // Confirmed are sorted by height + return appearanceIds[a] - appearanceIds[b]; + }); + var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); + unconfirmedTxids.sort(function(a, b) { + // Unconfirmed are sorted by timestamp + return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; + }); + var txids = confirmedTxids.concat(unconfirmedTxids); + return txids; +}; /** * This function will give detailed history for the configured @@ -50,30 +77,49 @@ AddressHistory.prototype.get = function(callback) { var self = this; var totalCount; - // TODO: handle multiple addresses (restore previous functionality) - if (self.addresses.length > 1) { - return callback('Only single address queries supported currently'); + if (this.addresses.length > this.maxAddressesQuery) { + return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded')); } - var address = self.addresses[0]; + if (this.addresses.length === 0) { + var address = this.addresses[0]; + self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } + return finish(summary.txids); + }); + } else { + var opts = _.clone(this.options); + opts.fullTxList = true; + async.map( + self.addresses, + function(address, next) { + self.node.services.address.getAddressSummary(address, opts, next); + }, + function(err, summaries) { + if (err) { + return callback(err); + } + var txids = self._mergeAndSortTxids(summaries); + return finish(txids); + } + ); + } - this.node.services.address.getAddressSummary(address, this.options, function(err, summary) { - if (err) { - return callback(err); - } + function finish(allTxids) { + totalCount = allTxids.length; - totalCount = summary.txids.length; - - // TODO: Make sure txids are sorted by height and time - var fromOffset = summary.txids.length - self.options.from; - var toOffset = summary.txids.length - self.options.to; - var txids = summary.txids.slice(toOffset, fromOffset); + // Slice the page starting with the most recent + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + var txids = allTxids.slice(toOffset, fromOffset); // Verify that this query isn't too long if (txids.length > self.maxHistoryQueryLength) { return callback(new Error( 'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' + - this.address.join(',') + self.address.join(',') )); } @@ -96,49 +142,8 @@ AddressHistory.prototype.get = function(callback) { } ); - }); - -}; - -/** - * A helper function to sort and slice/paginate the `combinedArray` - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.prototype.sortAndPaginateCombinedArray = function() { - this.combinedArray.sort(AddressHistory.sortByHeight); - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); } -}; -/** - * A helper sort function to order by height and then by date - * for transactions that are in the mempool. - * @param {Object} a - An item from the `combinedArray` - * @param {Object} b - */ -// TODO: Remove once txids summary results are verified to be sorted -AddressHistory.sortByHeight = function(a, b) { - if (a.height < 0 && b.height < 0) { - // Both are from the mempool, compare timestamps - if (a.timestamp === b.timestamp) { - return 0; - } else { - return a.timestamp < b.timestamp ? 1 : -1; - } - } else if (a.height < 0 && b.height > 0) { - // A is from the mempool and B is in a block - return -1; - } else if (a.height > 0 && b.height < 0) { - // A is in a block and B is in the mempool - return 1; - } else if (a.height === b.height) { - // The heights are equal - return 0; - } else { - // Otherwise compare heights - return a.height < b.height ? 1 : -1; - } }; /** diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 0d3a5576..0a0a8bac 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -298,6 +298,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { /* jshint maxstatements: 100 */ var operations = []; + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(new Date().getTime()); var action = 'put'; if (!add) { @@ -326,6 +328,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + timestampBuffer, txidBuffer, outputIndexBuffer ]); @@ -392,6 +395,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, + timestampBuffer, input.prevTxId, inputOutputIndexBuffer ]); @@ -899,22 +903,23 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { + var timestamp = data.key.readDoubleBE(22); var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); - var output = { + var input = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer @@ -922,7 +927,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha height: -1, confirmations: 0 }; - mempoolInputs.push(output); + mempoolInputs.push(input); }); var error; @@ -1108,22 +1113,24 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MIN + constants.TIMESTAMP_MIN ]), lte: Buffer.concat([ constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - constants.SPACER_MAX + constants.TIMESTAMP_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 - var txid = data.key.slice(22, 54); - var outputIndex = data.key.readUInt32BE(54); + // Format of data: + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4 + var timestamp = data.key.readDoubleBE(22); + var txid = data.key.slice(30, 62); + var outputIndex = data.key.readUInt32BE(62); var value = encoding.decodeOutputValue(data.value); var output = { address: addressStr, @@ -1131,6 +1138,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, + timestamp: timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1362,7 +1370,10 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb log.warn('Address Summary:', summary); } - if (!options.noTxList) { + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { summary.txids = confirmedTxids.concat(unconfirmedTxids); } @@ -1465,7 +1476,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - cache.result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp; } callback(error, cache); }); @@ -1537,7 +1548,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - cache.result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly