Address Service: Restored multi-address history queries

- Restored functionality to be able to query the history of multiple addresses in one query
- Sorted mempool transactions by timestamp in txid lists

lib/services/address/constants.js

@@ -36,16 +36,21 @@ exports.HASH_TYPES_MAP = {
 
 exports.SPACER_MIN = new Buffer('00', 'hex');
 exports.SPACER_MAX = new Buffer('ff', 'hex');
+exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex');
+exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex');
 
 // The total number of transactions that an address can receive before it will start
 // to cache the summary to disk.
 exports.SUMMARY_CACHE_THRESHOLD = 10000;
 
-
+// The maximum number of inputs that can be queried at once
-// The default maximum length queries
 exports.MAX_INPUTS_QUERY_LENGTH = 50000;
+// The maximum number of outputs that can be queried at once
 exports.MAX_OUTPUTS_QUERY_LENGTH = 50000;
-exports.MAX_HISTORY_QUERY_LENGTH = 1000;
+// The maximum number of transactions that can be queried at once
+exports.MAX_HISTORY_QUERY_LENGTH = 100;
+// The maximum number of addresses that can be queried at once
+exports.MAX_ADDRESSES_QUERY = 100;
 
 module.exports = exports;
 

lib/services/address/history.js

@@ -22,7 +22,8 @@ function AddressHistory(args) {
     this.addresses = [args.addresses];
   }
 
-  this.maxHistoryQueryLength = constants.MAX_HISTORY_QUERY_LENGTH;
+  this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH;
+  this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY;
 
   this.addressStrings = [];
   for (var i = 0; i < this.addresses.length; i++) {
@@ -39,7 +40,33 @@ function AddressHistory(args) {
   this.detailedArray = [];
 }
 
-AddressHistory.MAX_ADDRESS_QUERIES = 20;
+AddressHistory.prototype._mergeAndSortTxids = function(summaries) {
+  var appearanceIds = {};
+  var unconfirmedAppearanceIds = {};
+  for (var i = 0; i < summaries.length; i++) {
+    var summary = summaries[i];
+    for (var key in summary.appearanceIds) {
+      appearanceIds[key] = summary.appearanceIds[key];
+      delete summary.appearanceIds[key];
+    }
+    for (var unconfirmedKey in summary.unconfirmedAppearanceIds) {
+      unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[key];
+      delete summary.unconfirmedAppearanceIds[key];
+    }
+  }
+  var confirmedTxids = Object.keys(appearanceIds);
+  confirmedTxids.sort(function(a, b) {
+    // Confirmed are sorted by height
+    return appearanceIds[a] - appearanceIds[b];
+  });
+  var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds);
+  unconfirmedTxids.sort(function(a, b) {
+    // Unconfirmed are sorted by timestamp
+    return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b];
+  });
+  var txids = confirmedTxids.concat(unconfirmedTxids);
+  return txids;
+};
 
 /**
  * This function will give detailed history for the configured
@@ -50,30 +77,49 @@ AddressHistory.prototype.get = function(callback) {
   var self = this;
   var totalCount;
 
-  // TODO: handle multiple addresses (restore previous functionality)
+  if (this.addresses.length > this.maxAddressesQuery) {
-  if (self.addresses.length > 1) {
+    return callback(new Error('Maximum number of addresses (' + this.maxAddressQuery + ') exceeded'));
-    return callback('Only single address queries supported currently');
   }
 
-  var address = self.addresses[0];
+  if (this.addresses.length === 0) {
+    var address = this.addresses[0];
+    self.node.services.address.getAddressSummary(address, this.options, function(err, summary) {
+      if (err) {
+        return callback(err);
+      }
+      return finish(summary.txids);
+    });
+  } else {
+    var opts = _.clone(this.options);
+    opts.fullTxList = true;
+    async.map(
+      self.addresses,
+      function(address, next) {
+        self.node.services.address.getAddressSummary(address, opts, next);
+      },
+      function(err, summaries) {
+        if (err) {
+          return callback(err);
+        }
+        var txids = self._mergeAndSortTxids(summaries);
+        return finish(txids);
+      }
+    );
+  }
 
-  this.node.services.address.getAddressSummary(address, this.options, function(err, summary) {
+  function finish(allTxids) {
-    if (err) {
+    totalCount = allTxids.length;
-      return callback(err);
-    }
 
-    totalCount = summary.txids.length;
+    // Slice the page starting with the most recent
-
+    var fromOffset = totalCount - self.options.from;
-    // TODO: Make sure txids are sorted by height and time
+    var toOffset = totalCount - self.options.to;
-    var fromOffset = summary.txids.length - self.options.from;
+    var txids = allTxids.slice(toOffset, fromOffset);
-    var toOffset = summary.txids.length - self.options.to;
-    var txids = summary.txids.slice(toOffset, fromOffset);
 
     // Verify that this query isn't too long
     if (txids.length > self.maxHistoryQueryLength) {
       return callback(new Error(
         'Maximum length query (' + self.maxAddressQueryLength + ') exceeded for addresses:' +
-          this.address.join(',')
+          self.address.join(',')
       ));
     }
 
@@ -96,49 +142,8 @@ AddressHistory.prototype.get = function(callback) {
       }
     );
 
-  });
-
-};
-
-/**
- * A helper function to sort and slice/paginate the `combinedArray`
- */
-// TODO: Remove once txids summary results are verified to be sorted
-AddressHistory.prototype.sortAndPaginateCombinedArray = function() {
-  this.combinedArray.sort(AddressHistory.sortByHeight);
-  if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) {
-    this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to);
   }
-};
 
-/**
- * A helper sort function to order by height and then by date
- * for transactions that are in the mempool.
- * @param {Object} a - An item from the `combinedArray`
- * @param {Object} b
- */
-// TODO: Remove once txids summary results are verified to be sorted
-AddressHistory.sortByHeight = function(a, b) {
-  if (a.height < 0 && b.height < 0) {
-    // Both are from the mempool, compare timestamps
-    if (a.timestamp === b.timestamp) {
-      return 0;
-    } else {
-      return a.timestamp < b.timestamp ? 1 : -1;
-    }
-  } else if (a.height < 0 && b.height > 0) {
-    // A is from the mempool and B is in a block
-    return -1;
-  } else if (a.height > 0 && b.height < 0) {
-    // A is in a block and B is in the mempool
-    return 1;
-  } else if (a.height === b.height) {
-    // The heights are equal
-    return 0;
-  } else {
-    // Otherwise compare heights
-    return a.height < b.height ? 1 : -1;
-  }
 };
 
 /**

lib/services/address/index.js

@@ -298,6 +298,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) {
   /* jshint maxstatements: 100 */
 
   var operations = [];
+  var timestampBuffer = new Buffer(new Array(8));
+  timestampBuffer.writeDoubleBE(new Date().getTime());
 
   var action = 'put';
   if (!add) {
@@ -326,6 +328,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) {
       constants.MEMPREFIXES.OUTPUTS,
       addressInfo.hashBuffer,
       addressInfo.hashTypeBuffer,
+      timestampBuffer,
       txidBuffer,
       outputIndexBuffer
     ]);
@@ -392,6 +395,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) {
       constants.MEMPREFIXES.SPENTS,
       inputHashBuffer,
       inputHashType,
+      timestampBuffer,
       input.prevTxId,
       inputOutputIndexBuffer
     ]);
@@ -899,22 +903,23 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha
       constants.MEMPREFIXES.SPENTS,
       hashBuffer,
       hashTypeBuffer,
-      constants.SPACER_MIN
+      constants.TIMESTAMP_MIN
     ]),
     lte: Buffer.concat([
       constants.MEMPREFIXES.SPENTS,
       hashBuffer,
       hashTypeBuffer,
-      constants.SPACER_MAX
+      constants.TIMESTAMP_MAX
     ]),
     valueEncoding: 'binary',
     keyEncoding: 'binary'
   });
 
   stream.on('data', function(data) {
+    var timestamp = data.key.readDoubleBE(22);
     var txid = data.value.slice(0, 32);
     var inputIndex = data.value.readUInt32BE(32);
-    var output = {
+    var input = {
       address: addressStr,
       hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')],
       txid: txid.toString('hex'), //TODO use a buffer
@@ -922,7 +927,7 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha
       height: -1,
       confirmations: 0
     };
-    mempoolInputs.push(output);
+    mempoolInputs.push(input);
   });
 
   var error;
@@ -1108,22 +1113,24 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h
       constants.MEMPREFIXES.OUTPUTS,
       hashBuffer,
       hashTypeBuffer,
-      constants.SPACER_MIN
+      constants.TIMESTAMP_MIN
     ]),
     lte: Buffer.concat([
       constants.MEMPREFIXES.OUTPUTS,
       hashBuffer,
       hashTypeBuffer,
-      constants.SPACER_MAX
+      constants.TIMESTAMP_MAX
     ]),
     valueEncoding: 'binary',
     keyEncoding: 'binary'
   });
 
   stream.on('data', function(data) {
-    // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4
+    // Format of data:
-    var txid = data.key.slice(22, 54);
+    // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, timestamp: 8, txid: 32, outputIndex: 4
-    var outputIndex = data.key.readUInt32BE(54);
+    var timestamp = data.key.readDoubleBE(22);
+    var txid = data.key.slice(30, 62);
+    var outputIndex = data.key.readUInt32BE(62);
     var value = encoding.decodeOutputValue(data.value);
     var output = {
       address: addressStr,
@@ -1131,6 +1138,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h
       txid: txid.toString('hex'), //TODO use a buffer
       outputIndex: outputIndex,
       height: -1,
+      timestamp: timestamp,
       satoshis: value.satoshis,
       script: value.scriptBuffer.toString('hex'), //TODO use a buffer
       confirmations: 0
@@ -1362,7 +1370,10 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb
       log.warn('Address Summary:', summary);
     }
 
-    if (!options.noTxList) {
+    if (options.fullTxList) {
+      summary.appearanceIds = result.appearanceIds;
+      summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds;
+    } else if (!options.noTxList) {
       summary.txids = confirmedTxids.concat(unconfirmedTxids);
     }
 
@@ -1465,7 +1476,7 @@ AddressService.prototype._getAddressInputsSummary = function(address, cache, tip
       }
       for(var i = 0; i < mempoolInputs.length; i++) {
         var input = mempoolInputs[i];
-        cache.result.unconfirmedAppearanceIds[input.txid] = true;
+        cache.result.unconfirmedAppearanceIds[input.txid] = input.timestamp;
       }
       callback(error, cache);
     });
@@ -1537,7 +1548,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, cache, ti
       for(var i = 0; i < mempoolOutputs.length; i++) {
         var output = mempoolOutputs[i];
 
-        cache.result.unconfirmedAppearanceIds[output.txid] = true;
+        cache.result.unconfirmedAppearanceIds[output.txid] = output.timestamp;
 
         var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(
           new Buffer(output.txid, 'hex'), // TODO: get buffer directly