Merge pull request #392 from braydonf/large-queries

Memory optimizations for large address queries

integration/regtest-node.js

@@ -28,6 +28,7 @@ var Transaction = index.Transaction;
 var BitcoreNode = index.Node;
 var AddressService = index.services.Address;
 var BitcoinService = index.services.Bitcoin;
+var encoding = require('../lib/services/address/encoding');
 var DBService = index.services.DB;
 var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG';
 var testKey;
@@ -43,22 +44,6 @@ describe('Node Functionality', function() {
   before(function(done) {
     this.timeout(30000);
 
-    // Add the regtest network
-    bitcore.Networks.remove(bitcore.Networks.testnet);
-    bitcore.Networks.add({
-      name: 'regtest',
-      alias: 'regtest',
-      pubkeyhash: 0x6f,
-      privatekey: 0xef,
-      scripthash: 0xc4,
-      xpubkey: 0x043587cf,
-      xprivkey: 0x04358394,
-      networkMagic: 0xfabfb5da,
-      port: 18444,
-      dnsSeeds: [ ]
-    });
-    regtest = bitcore.Networks.get('regtest');
-
     var datadir = __dirname + '/data';
 
     testKey = bitcore.PrivateKey(testWIF);
@@ -93,6 +78,9 @@ describe('Node Functionality', function() {
 
       node = new BitcoreNode(configuration);
 
+      regtest = bitcore.Networks.get('regtest');
+      should.exist(regtest);
+
       node.on('error', function(err) {
         log.error(err);
       });
@@ -208,7 +196,7 @@ describe('Node Functionality', function() {
 
       // We need to add a transaction to the mempool so that the next block will
       // have a different hash as the hash has been invalidated.
-      client.sendToAddress(testKey.toAddress().toString(), 10, function(err) {
+      client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) {
         if (err) {
           throw err;
         }
@@ -250,7 +238,7 @@ describe('Node Functionality', function() {
     var address;
     var unspentOutput;
     before(function() {
-      address = testKey.toAddress().toString();
+      address = testKey.toAddress(regtest).toString();
     });
     it('should be able to get the balance of the test address', function(done) {
       node.services.address.getBalance(address, false, function(err, balance) {
@@ -333,19 +321,19 @@ describe('Node Functionality', function() {
         /* jshint maxstatements: 50 */
 
         testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x');
-        address2 = testKey2.toAddress().toString();
+        address2 = testKey2.toAddress(regtest).toString();
 
         testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt');
-        address3 = testKey3.toAddress().toString();
+        address3 = testKey3.toAddress(regtest).toString();
 
         testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw');
-        address4 = testKey4.toAddress().toString();
+        address4 = testKey4.toAddress(regtest).toString();
 
         testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF');
-        address5 = testKey5.toAddress().toString();
+        address5 = testKey5.toAddress(regtest).toString();
 
         testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF');
-        address6 = testKey6.toAddress().toString();
+        address6 = testKey6.toAddress(regtest).toString();
 
         var tx = new Transaction();
         tx.from(unspentOutput);
@@ -726,7 +714,7 @@ describe('Node Functionality', function() {
         node.services.bitcoind.sendTransaction(tx.serialize());
 
         setImmediate(function() {
-          var addrObj = node.services.address._getAddressInfo(address);
+          var addrObj = encoding.getAddressInfo(address);
           node.services.address._getOutputsMempool(address, addrObj.hashBuffer,
                                                    addrObj.hashTypeBuffer, function(err, outs) {
             if (err) {

lib/services/address/constants.js

@@ -0,0 +1,54 @@
+'use strict';
+
+var exports = {};
+
+exports.PREFIXES = {
+  OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+  SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+  SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+};
+
+exports.MEMPREFIXES = {
+  OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address
+  SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address
+  SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output
+};
+
+// To save space, we're only storing the PubKeyHash or ScriptHash in our index.
+// To avoid intentional unspendable collisions, which have been seen on the blockchain,
+// we must store the hash type (PK or Script) as well.
+exports.HASH_TYPES = {
+  PUBKEY: new Buffer('01', 'hex'),
+  REDEEMSCRIPT: new Buffer('02', 'hex')
+};
+
+// Translates from our enum type back into the hash types returned by
+// bitcore-lib/address.
+exports.HASH_TYPES_READABLE = {
+  '01': 'pubkeyhash',
+  '02': 'scripthash'
+};
+
+exports.HASH_TYPES_MAP = {
+  'pubkeyhash': exports.HASH_TYPES.PUBKEY,
+  'scripthash': exports.HASH_TYPES.REDEEMSCRIPT
+};
+
+exports.SPACER_MIN = new Buffer('00', 'hex');
+exports.SPACER_MAX = new Buffer('ff', 'hex');
+exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex');
+exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex');
+exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex');
+exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex');
+
+// The maximum number of inputs that can be queried at once
+exports.MAX_INPUTS_QUERY_LENGTH = 50000;
+// The maximum number of outputs that can be queried at once
+exports.MAX_OUTPUTS_QUERY_LENGTH = 50000;
+// The maximum number of transactions that can be queried at once
+exports.MAX_HISTORY_QUERY_LENGTH = 100;
+// The maximum number of addresses that can be queried at once
+exports.MAX_ADDRESSES_QUERY = 10000;
+
+module.exports = exports;
+

lib/services/address/encoding.js

@@ -0,0 +1,298 @@
+'use strict';
+
+var bitcore = require('bitcore-lib');
+var BufferReader = bitcore.encoding.BufferReader;
+var Address = bitcore.Address;
+var PublicKey = bitcore.PublicKey;
+var constants = require('./constants');
+var $ = bitcore.util.preconditions;
+
+var exports = {};
+
+exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  var key = Buffer.concat([
+    txidBuffer,
+    outputIndexBuffer
+  ]);
+  return key.toString('binary');
+};
+
+exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) {
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  var key = Buffer.concat([
+    constants.PREFIXES.OUTPUTS,
+    hashBuffer,
+    hashTypeBuffer,
+    constants.SPACER_MIN,
+    heightBuffer,
+    txidBuffer,
+    outputIndexBuffer
+  ]);
+  return key;
+};
+
+exports.decodeOutputKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashBuffer = reader.read(20);
+  var hashTypeBuffer = reader.read(1);
+  var spacer = reader.read(1);
+  var height = reader.readUInt32BE();
+  var txid = reader.read(32);
+  var outputIndex = reader.readUInt32BE();
+  return {
+    prefix: prefix,
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    height: height,
+    txid: txid,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeOutputValue = function(satoshis, scriptBuffer) {
+  var satoshisBuffer = new Buffer(8);
+  satoshisBuffer.writeDoubleBE(satoshis);
+  return Buffer.concat([satoshisBuffer, scriptBuffer]);
+};
+
+exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) {
+  var satoshisBuffer = new Buffer(8);
+  satoshisBuffer.writeDoubleBE(satoshis);
+  return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]);
+};
+
+exports.decodeOutputValue = function(buffer) {
+  var satoshis = buffer.readDoubleBE(0);
+  var scriptBuffer = buffer.slice(8, buffer.length);
+  return {
+    satoshis: satoshis,
+    scriptBuffer: scriptBuffer
+  };
+};
+
+exports.decodeOutputMempoolValue = function(buffer) {
+  var satoshis = buffer.readDoubleBE(0);
+  var timestamp = buffer.readDoubleBE(8);
+  var scriptBuffer = buffer.slice(16, buffer.length);
+  return {
+    satoshis: satoshis,
+    timestamp: timestamp,
+    scriptBuffer: scriptBuffer
+  };
+};
+
+exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) {
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  return Buffer.concat([
+    constants.PREFIXES.SPENTS,
+    hashBuffer,
+    hashTypeBuffer,
+    constants.SPACER_MIN,
+    heightBuffer,
+    prevTxIdBuffer,
+    outputIndexBuffer
+  ]);
+};
+
+exports.decodeInputKey = function(buffer) {
+  var reader = new BufferReader(buffer);
+  var prefix = reader.read(1);
+  var hashBuffer = reader.read(20);
+  var hashTypeBuffer = reader.read(1);
+  var spacer = reader.read(1);
+  var height = reader.readUInt32BE();
+  var prevTxId = reader.read(32);
+  var outputIndex = reader.readUInt32BE();
+  return {
+    prefix: prefix,
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    height: height,
+    prevTxId: prevTxId,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeInputValue = function(txidBuffer, inputIndex) {
+  var inputIndexBuffer = new Buffer(4);
+  inputIndexBuffer.writeUInt32BE(inputIndex);
+  return Buffer.concat([
+    txidBuffer,
+    inputIndexBuffer
+  ]);
+};
+
+exports.decodeInputValue = function(buffer) {
+  var txid = buffer.slice(0, 32);
+  var inputIndex = buffer.readUInt32BE(32);
+  return {
+    txid: txid,
+    inputIndex: inputIndex
+  };
+};
+
+exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  return Buffer.concat([
+    constants.PREFIXES.SPENTSMAP,
+    outputTxIdBuffer,
+    outputIndexBuffer
+  ]);
+};
+
+exports.decodeInputKeyMap = function(buffer) {
+  var txid = buffer.slice(1, 33);
+  var outputIndex = buffer.readUInt32BE(33);
+  return {
+    outputTxId: txid,
+    outputIndex: outputIndex
+  };
+};
+
+exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) {
+  var inputIndexBuffer = new Buffer(4);
+  inputIndexBuffer.writeUInt32BE(inputIndex);
+  return Buffer.concat([
+    inputTxIdBuffer,
+    inputIndexBuffer
+  ]);
+};
+
+exports.decodeInputValueMap = function(buffer) {
+  var txid = buffer.slice(0, 32);
+  var inputIndex = buffer.readUInt32BE(32);
+  return {
+    inputTxId: txid,
+    inputIndex: inputIndex
+  };
+};
+
+exports.encodeSummaryCacheKey = function(address) {
+  return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_MAP[address.type]]);
+};
+
+exports.decodeSummaryCacheKey = function(buffer, network) {
+  var hashBuffer = buffer.read(20);
+  var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')];
+  var address = new Address({
+    hashBuffer: hashBuffer,
+    type: type,
+    network: network
+  });
+  return address;
+};
+
+exports.encodeSummaryCacheValue = function(cache, tipHeight, tipHash) {
+  var tipHashBuffer = new Buffer(tipHash, 'hex');
+  var buffer = new Buffer(new Array(20));
+  buffer.writeUInt32BE(tipHeight);
+  buffer.writeDoubleBE(cache.result.totalReceived, 4);
+  buffer.writeDoubleBE(cache.result.balance, 12);
+  var txidBuffers = [];
+  for (var i = 0; i < cache.result.txids.length; i++) {
+    var buf = new Buffer(new Array(36));
+    var txid = cache.result.txids[i];
+    buf.write(txid, 'hex');
+    buf.writeUInt32BE(cache.result.appearanceIds[txid], 32);
+    txidBuffers.push(buf);
+  }
+  var txidsBuffer = Buffer.concat(txidBuffers);
+  var value = Buffer.concat([tipHashBuffer, buffer, txidsBuffer]);
+
+  return value;
+};
+
+exports.decodeSummaryCacheValue = function(buffer) {
+
+  var hash = buffer.slice(0, 32).toString('hex');
+  var height = buffer.readUInt32BE(32);
+  var totalReceived = buffer.readDoubleBE(36);
+  var balance = buffer.readDoubleBE(44);
+
+  // read 32 byte chunks until exhausted
+  var appearanceIds = {};
+  var txids = [];
+  var pos = 52;
+  while(pos < buffer.length) {
+    var txid = buffer.slice(pos, pos + 32).toString('hex');
+    var txidHeight = buffer.readUInt32BE(pos + 32);
+    txids.push(txid);
+    appearanceIds[txid] = txidHeight;
+    pos += 36;
+  }
+
+  var cache = {
+    height: height,
+    hash: hash,
+    result: {
+      appearanceIds: appearanceIds,
+      txids: txids,
+      totalReceived: totalReceived,
+      balance: balance,
+      unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache
+      unconfirmedBalance: 0
+    }
+  };
+
+  return cache;
+};
+
+exports.getAddressInfo = function(addressStr) {
+  var addrObj = bitcore.Address(addressStr);
+  var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type];
+  
+  return {
+    hashBuffer: addrObj.hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    hashTypeReadable: addrObj.type
+  };
+};
+
+/**
+ * This function is optimized to return address information about an output script
+ * without constructing a Bitcore Address instance.
+ * @param {Script} - An instance of a Bitcore Script
+ * @param {Network|String} - The network for the address
+ */
+exports.extractAddressInfoFromScript = function(script, network) {
+  $.checkArgument(network, 'Second argument is expected to be a network');
+  var hashBuffer;
+  var addressType;
+  var hashTypeBuffer;
+  if (script.isPublicKeyHashOut()) {
+    hashBuffer = script.chunks[2].buf;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    addressType = Address.PayToPublicKeyHash;
+  } else if (script.isScriptHashOut()) {
+    hashBuffer = script.chunks[1].buf;
+    hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT;
+    addressType = Address.PayToScriptHash;
+  } else if (script.isPublicKeyOut()) {
+    var pubkey = script.chunks[0].buf;
+    var address = Address.fromPublicKey(new PublicKey(pubkey), network);
+    hashBuffer = address.hashBuffer;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    // pay-to-publickey doesn't have an address, however for compatibility
+    // purposes, we can create an address
+    addressType = Address.PayToPublicKeyHash;
+  } else {
+    return false;
+  }
+  return {
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    addressType: addressType
+  };
+};
+
+module.exports = exports;

lib/services/address/history.js

@@ -4,6 +4,8 @@ var bitcore = require('bitcore-lib');
 var async = require('async');
 var _ = bitcore.deps._;
 
+var constants = require('./constants');
+
 /**
  * This represents an instance that keeps track of data over a series of
  * asynchronous I/O calls to get the transaction history for a group of
@@ -19,12 +21,51 @@ function AddressHistory(args) {
   } else {
     this.addresses = [args.addresses];
   }
-  this.transactionInfo = [];
+
-  this.combinedArray = [];
+  this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH;
+  this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY;
+
+  this.addressStrings = [];
+  for (var i = 0; i < this.addresses.length; i++) {
+    var address = this.addresses[i];
+    if (address instanceof bitcore.Address) {
+      this.addressStrings.push(address.toString());
+    } else if (_.isString(address)) {
+      this.addressStrings.push(address);
+    } else {
+      throw new TypeError('Addresses are expected to be strings');
+    }
+  }
+
   this.detailedArray = [];
 }
 
-AddressHistory.MAX_ADDRESS_QUERIES = 20;
+AddressHistory.prototype._mergeAndSortTxids = function(summaries) {
+  var appearanceIds = {};
+  var unconfirmedAppearanceIds = {};
+  for (var i = 0; i < summaries.length; i++) {
+    var summary = summaries[i];
+    for (var key in summary.appearanceIds) {
+      appearanceIds[key] = summary.appearanceIds[key];
+      delete summary.appearanceIds[key];
+    }
+    for (var unconfirmedKey in summary.unconfirmedAppearanceIds) {
+      unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey];
+      delete summary.unconfirmedAppearanceIds[key];
+    }
+  }
+  var confirmedTxids = Object.keys(appearanceIds);
+  confirmedTxids.sort(function(a, b) {
+    // Confirmed are sorted by height
+    return appearanceIds[a] - appearanceIds[b];
+  });
+  var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds);
+  unconfirmedTxids.sort(function(a, b) {
+    // Unconfirmed are sorted by timestamp
+    return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b];
+  });
+  return confirmedTxids.concat(unconfirmedTxids);
+};
 
 /**
  * This function will give detailed history for the configured
@@ -33,170 +74,79 @@ AddressHistory.MAX_ADDRESS_QUERIES = 20;
  */
 AddressHistory.prototype.get = function(callback) {
   var self = this;
-  var totalCount;
+  if (this.addresses.length > this.maxAddressesQuery) {
+    return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded'));
+  }
 
-  async.eachLimit(
+  if (this.addresses.length === 1) {
-    self.addresses,
+    var address = this.addresses[0];
-    AddressHistory.MAX_ADDRESS_QUERIES,
+    self.node.services.address.getAddressSummary(address, this.options, function(err, summary) {
-    function(address, next) {
+      if (err) {
-      self.getTransactionInfo(address, next);
+        return callback(err);
+      }
+      return self._paginateWithDetails.call(self, summary.txids, callback);
+    });
+  } else {
+    var opts = _.clone(this.options);
+    opts.fullTxList = true;
+    async.map(
+      self.addresses,
+      function(address, next) {
+        self.node.services.address.getAddressSummary(address, opts, next);
+      },
+      function(err, summaries) {
+        if (err) {
+          return callback(err);
+        }
+        var txids = self._mergeAndSortTxids(summaries);
+        return self._paginateWithDetails.call(self, txids, callback);
+      }
+    );
+  }
+
+};
+
+AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) {
+  var self = this;
+  var totalCount = allTxids.length;
+
+  // Slice the page starting with the most recent
+  var txids;
+  if (self.options.from >= 0 && self.options.to >= 0) {
+    var fromOffset = totalCount - self.options.from;
+    var toOffset = totalCount - self.options.to;
+    txids = allTxids.slice(toOffset, fromOffset);
+  } else {
+    txids = allTxids;
+  }
+
+  // Verify that this query isn't too long
+  if (txids.length > self.maxHistoryQueryLength) {
+    return callback(new Error(
+      'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' +
+        self.addresses.join(',')
+    ));
+  }
+
+  // Reverse to include most recent at the top
+  txids.reverse();
+
+  async.eachSeries(
+    txids,
+    function(txid, next) {
+      self.getDetailedInfo(txid, next);
     },
     function(err) {
       if (err) {
         return callback(err);
       }
-
+      callback(null, {
-      self.combineTransactionInfo();
+        totalCount: totalCount,
-      totalCount = Number(self.combinedArray.length);
+        items: self.detailedArray
-      self.sortAndPaginateCombinedArray();
+      });
-
-      async.eachSeries(
-        self.combinedArray,
-        function(txInfo, next) {
-          self.getDetailedInfo(txInfo, next);
-        },
-        function(err) {
-          if (err) {
-            return callback(err);
-          }
-          callback(null, {
-            totalCount: totalCount,
-            items: self.detailedArray
-          });
-        }
-      );
     }
   );
-};
 
-/**
- * This function will retrieve input and output information for an address
- * and set the property `this.transactionInfo`.
- * @param {String} address - A base58check encoded address
- * @param {Function} next
- */
-AddressHistory.prototype.getTransactionInfo = function(address, next) {
-  var self = this;
-
-  var args = {
-    start: self.options.start,
-    end: self.options.end,
-    queryMempool: _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool
-  };
-
-  var outputs;
-  var inputs;
-
-  async.parallel([
-    function(done) {
-      self.node.services.address.getOutputs(address, args, function(err, result) {
-        if (err) {
-          return done(err);
-        }
-        outputs = result;
-        done();
-      });
-    },
-    function(done) {
-      self.node.services.address.getInputs(address, args, function(err, result) {
-        if (err) {
-          return done(err);
-        }
-        inputs = result;
-        done();
-      });
-    }
-  ], function(err) {
-    if (err) {
-      return next(err);
-    }
-    self.transactionInfo = self.transactionInfo.concat(outputs, inputs);
-    next();
-  });
-};
-
-/**
- * This function combines results from getInputs and getOutputs at
- * `this.transactionInfo` to be "txid" unique at `this.combinedArray`.
- */
-AddressHistory.prototype.combineTransactionInfo = function() {
-  var combinedArrayMap = {};
-  this.combinedArray = [];
-  var l = this.transactionInfo.length;
-  for(var i = 0; i < l; i++) {
-    var item = this.transactionInfo[i];
-    var mapKey = item.txid;
-    if (combinedArrayMap[mapKey] >= 0) {
-      var combined = this.combinedArray[combinedArrayMap[mapKey]];
-      if (!combined.addresses[item.address]) {
-        combined.addresses[item.address] = {
-          outputIndexes: [],
-          inputIndexes: []
-        };
-      }
-      if (item.outputIndex >= 0) {
-        combined.satoshis += item.satoshis;
-        combined.addresses[item.address].outputIndexes.push(item.outputIndex);
-      } else if (item.inputIndex >= 0) {
-        combined.addresses[item.address].inputIndexes.push(item.inputIndex);
-      }
-    } else {
-      item.addresses = {};
-      item.addresses[item.address] = {
-        outputIndexes: [],
-        inputIndexes: []
-      };
-      if (item.outputIndex >= 0) {
-        item.addresses[item.address].outputIndexes.push(item.outputIndex);
-      } else if (item.inputIndex >= 0) {
-        item.addresses[item.address].inputIndexes.push(item.inputIndex);
-      }
-      delete item.outputIndex;
-      delete item.inputIndex;
-      delete item.address;
-      this.combinedArray.push(item);
-      combinedArrayMap[mapKey] = this.combinedArray.length - 1;
-    }
-  }
-};
-
-/**
- * A helper function to sort and slice/paginate the `combinedArray`
- */
-AddressHistory.prototype.sortAndPaginateCombinedArray = function() {
-  this.combinedArray.sort(AddressHistory.sortByHeight);
-  if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) {
-    this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to);
-  }
-};
-
-/**
- * A helper sort function to order by height and then by date
- * for transactions that are in the mempool.
- * @param {Object} a - An item from the `combinedArray`
- * @param {Object} b
- */
-AddressHistory.sortByHeight = function(a, b) {
-  if (a.height < 0 && b.height < 0) {
-    // Both are from the mempool, compare timestamps
-    if (a.timestamp === b.timestamp) {
-      return 0;
-    } else {
-      return a.timestamp < b.timestamp ? 1 : -1;
-    }
-  } else if (a.height < 0 && b.height > 0) {
-    // A is from the mempool and B is in a block
-    return -1;
-  } else if (a.height > 0 && b.height < 0) {
-    // A is in a block and B is in the mempool
-    return 1;
-  } else if (a.height === b.height) {
-    // The heights are equal
-    return 0;
-  } else {
-    // Otherwise compare heights
-    return a.height < b.height ? 1 : -1;
-  }
 };
 
 /**
@@ -205,12 +155,12 @@ AddressHistory.sortByHeight = function(a, b) {
  * @param {Object} txInfo - An item from the `combinedArray`
  * @param {Function} next
  */
-AddressHistory.prototype.getDetailedInfo = function(txInfo, next) {
+AddressHistory.prototype.getDetailedInfo = function(txid, next) {
   var self = this;
   var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool;
 
   self.node.services.db.getTransactionWithBlockInfo(
-    txInfo.txid,
+    txid,
     queryMempool,
     function(err, transaction) {
       if (err) {
@@ -218,13 +168,15 @@ AddressHistory.prototype.getDetailedInfo = function(txInfo, next) {
       }
 
       transaction.populateInputs(self.node.services.db, [], function(err) {
-        if(err) {
+        if (err) {
           return next(err);
         }
 
+        var addressDetails = self.getAddressDetailsForTransaction(transaction);
+
         self.detailedArray.push({
-          addresses: txInfo.addresses,
+          addresses: addressDetails.addresses,
-          satoshis: self.getSatoshisDetail(transaction, txInfo),
+          satoshis: addressDetails.satoshis,
           height: transaction.__height,
           confirmations: self.getConfirmationsDetail(transaction),
           timestamp: transaction.__timestamp,
@@ -251,23 +203,58 @@ AddressHistory.prototype.getConfirmationsDetail = function(transaction) {
   return confirmations;
 };
 
-/**
+AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) {
- * A helper function for `getDetailedInfo` for getting the satoshis.
+  var result = {
- * @param {Transaction} transaction - A transaction populated with previous outputs
+    addresses: {},
- * @param {Object} txInfo - An item from `combinedArray`
+    satoshis: 0
- */
+  };
-AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) {
-  var satoshis = txInfo.satoshis || 0;
 
-  for(var address in txInfo.addresses) {
+  for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) {
-    if (txInfo.addresses[address].inputIndexes.length >= 0) {
+    var input = transaction.inputs[inputIndex];
-      for(var j = 0; j < txInfo.addresses[address].inputIndexes.length; j++) {
+    if (!input.script) {
-        satoshis -= transaction.inputs[txInfo.addresses[address].inputIndexes[j]].output.satoshis;
+      continue;
+    }
+    var inputAddress = input.script.toAddress(this.node.network);
+    if (inputAddress) {
+      var inputAddressString = inputAddress.toString();
+      if (this.addressStrings.indexOf(inputAddressString) >= 0) {
+        if (!result.addresses[inputAddressString]) {
+          result.addresses[inputAddressString] = {
+            inputIndexes: [inputIndex],
+            outputIndexes: []
+          };
+        } else {
+          result.addresses[inputAddressString].inputIndexes.push(inputIndex);
+        }
+        result.satoshis -= input.output.satoshis;
       }
     }
   }
 
-  return satoshis;
+  for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) {
+    var output = transaction.outputs[outputIndex];
+    if (!output.script) {
+      continue;
+    }
+    var outputAddress = output.script.toAddress(this.node.network);
+    if (outputAddress) {
+      var outputAddressString = outputAddress.toString();
+      if (this.addressStrings.indexOf(outputAddressString) >= 0) {
+        if (!result.addresses[outputAddressString]) {
+          result.addresses[outputAddressString] = {
+            inputIndexes: [],
+            outputIndexes: [outputIndex]
+          };
+        } else {
+          result.addresses[outputAddressString].outputIndexes.push(outputIndex);
+        }
+        result.satoshis += output.satoshis;
+      }
+    }
+  }
+
+  return result;
+
 };
 
 module.exports = AddressHistory;

lib/services/address/streams/inputs-transform.js

@@ -0,0 +1,40 @@
+'use strict';
+
+var Transform = require('stream').Transform;
+var inherits = require('util').inherits;
+var bitcore = require('bitcore-lib');
+var encodingUtil = require('../encoding');
+var $ = bitcore.util.preconditions;
+
+function InputsTransformStream(options) {
+  $.checkArgument(options.address instanceof bitcore.Address);
+  Transform.call(this, {
+    objectMode: true
+  });
+  this._address = options.address;
+  this._addressStr = this._address.toString();
+  this._tipHeight = options.tipHeight;
+}
+inherits(InputsTransformStream, Transform);
+
+InputsTransformStream.prototype._transform = function(chunk, encoding, callback) {
+  var self = this;
+
+  var key = encodingUtil.decodeInputKey(chunk.key);
+  var value = encodingUtil.decodeInputValue(chunk.value);
+
+  var input = {
+    address: this._addressStr,
+    hashType: this._address.type,
+    txid: value.txid.toString('hex'),
+    inputIndex: value.inputIndex,
+    height: key.height,
+    confirmations: this._tipHeight - key.height + 1
+  };
+
+  self.push(input);
+  callback();
+
+};
+
+module.exports = InputsTransformStream;

lib/services/address/streams/outputs-transform.js

@@ -0,0 +1,42 @@
+'use strict';
+
+var Transform = require('stream').Transform;
+var inherits = require('util').inherits;
+var bitcore = require('bitcore-lib');
+var encodingUtil = require('../encoding');
+var $ = bitcore.util.preconditions;
+
+function OutputsTransformStream(options) {
+  Transform.call(this, {
+    objectMode: true
+  });
+  $.checkArgument(options.address instanceof bitcore.Address);
+  this._address = options.address;
+  this._addressStr = this._address.toString();
+  this._tipHeight = options.tipHeight;
+}
+inherits(OutputsTransformStream, Transform);
+
+OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) {
+  var self = this;
+
+  var key = encodingUtil.decodeOutputKey(chunk.key);
+  var value = encodingUtil.decodeOutputValue(chunk.value);
+
+  var output = {
+    address: this._addressStr,
+    hashType: this._address.type,
+    txid: key.txid.toString('hex'), //TODO use a buffer
+    outputIndex: key.outputIndex,
+    height: key.height,
+    satoshis: value.satoshis,
+    script: value.scriptBuffer.toString('hex'), //TODO use a buffer
+    confirmations: this._tipHeight - key.height + 1
+  };
+
+  self.push(output);
+  callback();
+
+};
+
+module.exports = OutputsTransformStream;

lib/services/db.js

@@ -46,6 +46,8 @@ function DB(options) {
 
   this._setDataPath();
 
+  this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES;
+
   this.levelupStore = leveldown;
   if (options.store) {
     this.levelupStore = options.store;
@@ -68,6 +70,8 @@ DB.PREFIXES = {
   TIP: new Buffer('04', 'hex')
 };
 
+DB.DEFAULT_MAX_OPEN_FILES = 200;
+
 /**
  * This function will set `this.dataPath` based on `this.node.network`.
  * @private
@@ -98,7 +102,7 @@ DB.prototype.start = function(callback) {
   }
 
   this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer);
-  this.store = levelup(this.dataPath, { db: this.levelupStore });
+  this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles });
   this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this));
 
   this.once('ready', function() {

package.json

@@ -54,8 +54,8 @@
     "commander": "^2.8.1",
     "errno": "^0.1.4",
     "express": "^4.13.3",
-    "leveldown": "^1.4.2",
+    "leveldown": "^1.4.3",
-    "levelup": "^1.2.1",
+    "levelup": "^1.3.1",
     "liftoff": "^2.2.0",
     "memdown": "^1.0.0",
     "mkdirp": "0.5.0",

test/services/address/encoding.unit.js

@@ -0,0 +1,103 @@
+'use strict';
+
+var chai = require('chai');
+var should = chai.should();
+var sinon = require('sinon');
+var bitcorenode = require('../../../');
+var bitcore = require('bitcore-lib');
+var Address = bitcore.Address;
+var Script = bitcore.Script;
+var AddressService = bitcorenode.services.Address;
+var Networks = bitcore.Networks;
+var encoding = require('../../../lib/services/address/encoding');
+
+var mockdb = {
+};
+
+var mocknode = {
+  network: Networks.testnet,
+  datadir: 'testdir',
+  db: mockdb,
+  services: {
+    bitcoind: {
+      on: sinon.stub()
+    }
+  }
+};
+
+describe('Address Service Encoding', function() {
+
+  describe('#encodeSpentIndexSyncKey', function() {
+    it('will encode to 36 bytes (string)', function() {
+      var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex');
+      var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12);
+      key.length.should.equal(36);
+    });
+    it('will be able to decode encoded value', function() {
+      var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7';
+      var txidBuffer = new Buffer(txid, 'hex');
+      var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12);
+      var keyBuffer = new Buffer(key, 'binary');
+      keyBuffer.slice(0, 32).toString('hex').should.equal(txid);
+      var outputIndex = keyBuffer.readUInt32BE(32);
+      outputIndex.should.equal(12);
+    });
+  });
+
+  describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() {
+    var encoded;
+    var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex');
+    it('encode key', function() {
+      encoded = encoding.encodeInputKeyMap(outputTxIdBuffer, 13);
+    });
+    it('decode key', function() {
+      var key = encoding.decodeInputKeyMap(encoded);
+      key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex'));
+      key.outputIndex.should.equal(13);
+    });
+  });
+
+  describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() {
+    var encoded;
+    var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex');
+    it('encode key', function() {
+      encoded = encoding.encodeInputValueMap(inputTxIdBuffer, 7);
+    });
+    it('decode key', function() {
+      var key = encoding.decodeInputValueMap(encoded);
+      key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex'));
+      key.inputIndex.should.equal(7);
+    });
+  });
+
+
+  describe('#extractAddressInfoFromScript', function() {
+    it('pay-to-publickey', function() {
+      var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da');
+      var script = Script.buildPublicKeyOut(pubkey);
+      var info = encoding.extractAddressInfoFromScript(script, Networks.livenet);
+      info.addressType.should.equal(Address.PayToPublicKeyHash);
+      info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147');
+    });
+    it('pay-to-publickeyhash', function() {
+      var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG');
+      var info = encoding.extractAddressInfoFromScript(script, Networks.livenet);
+      info.addressType.should.equal(Address.PayToPublicKeyHash);
+      info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000');
+    });
+    it('pay-to-scripthash', function() {
+      var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL');
+      var info = encoding.extractAddressInfoFromScript(script, Networks.livenet);
+      info.addressType.should.equal(Address.PayToScriptHash);
+      info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000');
+    });
+    it('non-address script type', function() {
+      var buf = new Buffer(40);
+      buf.fill(0);
+      var script = Script('OP_RETURN 40 0x' + buf.toString('hex'));
+      var info = encoding.extractAddressInfoFromScript(script, Networks.livenet);
+      info.should.equal(false);
+    });
+  });
+
+});

test/services/address/history.unit.js

@@ -2,6 +2,7 @@
 
 var should = require('chai').should();
 var sinon = require('sinon');
+var bitcore = require('bitcore-lib');
 var Transaction = require('../../../lib/transaction');
 var AddressHistory = require('../../../lib/services/address/history');
 
@@ -23,8 +24,6 @@ describe('Address Service History', function() {
       history.node.should.equal(node);
       history.options.should.equal(options);
       history.addresses.should.equal(addresses);
-      history.transactionInfo.should.deep.equal([]);
-      history.combinedArray.should.deep.equal([]);
       history.detailedArray.should.deep.equal([]);
     });
     it('will set addresses an array if only sent a string', function() {
@@ -38,501 +37,321 @@ describe('Address Service History', function() {
   });
 
   describe('#get', function() {
-    it('will complete the async each limit series', function(done) {
+    it('will give an error if length of addresses is too long', function(done) {
-      var addresses = [address];
+      var node = {};
+      var options = {};
+      var addresses = [];
+      for (var i = 0; i < 101; i++) {
+        addresses.push(address);
+      }
       var history = new AddressHistory({
-        node: {},
+        node: node,
-        options: {},
+        options: options,
         addresses: addresses
       });
-      var expected = [{}];
+      history.maxAddressesQuery = 100;
-      history.detailedArray = expected;
+      history.get(function(err) {
-      history.combinedArray = [{}];
+        should.exist(err);
-      history.getTransactionInfo = sinon.stub().callsArg(1);
+        err.message.match(/Maximum/);
-      history.combineTransactionInfo = sinon.stub();
+        done();
-      history.sortAndPaginateCombinedArray = sinon.stub();
+      });
-      history.getDetailedInfo = sinon.stub().callsArg(1);
+    });
-      history.sortTransactionsIntoArray = sinon.stub();
+    it('give error from getAddressSummary with one address', function(done) {
-      history.get(function(err, results) {
+      var node = {
-        if (err) {
+        services: {
-          throw err;
+          address: {
+            getAddressSummary: sinon.stub().callsArgWith(2, new Error('test'))
+          }
         }
-        history.getTransactionInfo.callCount.should.equal(1);
-        history.getDetailedInfo.callCount.should.equal(1);
-        history.combineTransactionInfo.callCount.should.equal(1);
-        history.sortAndPaginateCombinedArray.callCount.should.equal(1);
-        results.should.deep.equal({
-          totalCount: 1,
-          items: expected
-        });
-        done();
-      });
-    });
-    it('handle an error from getDetailedInfo', function(done) {
-      var addresses = [address];
-      var history = new AddressHistory({
-        node: {},
-        options: {},
-        addresses: addresses
-      });
-      var expected = [{}];
-      history.sortedArray = expected;
-      history.transactionInfo = [{}];
-      history.getTransactionInfo = sinon.stub().callsArg(1);
-      history.paginateSortedArray = sinon.stub();
-      history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test'));
-      history.get(function(err) {
-        err.message.should.equal('test');
-        done();
-      });
-    });
-    it('handle an error from getTransactionInfo', function(done) {
-      var addresses = [address];
-      var history = new AddressHistory({
-        node: {},
-        options: {},
-        addresses: addresses
-      });
-      var expected = [{}];
-      history.sortedArray = expected;
-      history.transactionInfo = [{}];
-      history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test'));
-      history.get(function(err) {
-        err.message.should.equal('test');
-        done();
-      });
-    });
-  });
-
-  describe('#getTransactionInfo', function() {
-    it('will handle an error from getInputs', function(done) {
-      var history = new AddressHistory({
-        node: {
-          services: {
-            address: {
-              getOutputs: sinon.stub().callsArgWith(2, null, []),
-              getInputs: sinon.stub().callsArgWith(2, new Error('test'))
-            }
-          }
-        },
-        options: {},
-        addresses: []
-      });
-      history.getTransactionInfo(address, function(err) {
-        err.message.should.equal('test');
-        done();
-      });
-    });
-    it('will handle an error from getOutputs', function(done) {
-      var history = new AddressHistory({
-        node: {
-          services: {
-            address: {
-              getOutputs: sinon.stub().callsArgWith(2, new Error('test')),
-              getInputs: sinon.stub().callsArgWith(2, null, [])
-            }
-          }
-        },
-        options: {},
-        addresses: []
-      });
-      history.getTransactionInfo(address, function(err) {
-        err.message.should.equal('test');
-        done();
-      });
-    });
-    it('will call getOutputs and getInputs with the correct options', function() {
-      var startTimestamp = 1438289011844;
-      var endTimestamp = 1438289012412;
-      var expectedArgs = {
-        start: new Date(startTimestamp * 1000),
-        end: new Date(endTimestamp * 1000),
-        queryMempool: true
       };
+      var options = {};
+      var addresses = [address];
       var history = new AddressHistory({
-        node: {
+        node: node,
-          services: {
+        options: options,
-            address: {
+        addresses: addresses
-              getOutputs: sinon.stub().callsArgWith(2, null, []),
-              getInputs: sinon.stub().callsArgWith(2, null, [])
-            }
-          }
-        },
-        options: {
-          start: new Date(startTimestamp * 1000),
-          end: new Date(endTimestamp * 1000),
-          queryMempool: true
-        },
-        addresses: []
       });
-      history.transactionInfo = [{}];
+      history.get(function(err) {
-      history.getTransactionInfo(address, function(err) {
+        should.exist(err);
-        if (err) {
+        err.message.should.equal('test');
-          throw err;
+        done();
-        }
-        history.node.services.address.getOutputs.args[0][1].should.deep.equal(expectedArgs);
-        history.node.services.address.getInputs.args[0][1].should.deep.equal(expectedArgs);
       });
     });
-    it('will handle empty results from getOutputs and getInputs', function() {
+    it('give error from getAddressSummary with multiple addresses', function(done) {
-      var history = new AddressHistory({
+      var node = {
-        node: {
+        services: {
-          services: {
+          address: {
-            address: {
+            getAddressSummary: sinon.stub().callsArgWith(2, new Error('test2'))
-              getOutputs: sinon.stub().callsArgWith(2, null, []),
-              getInputs: sinon.stub().callsArgWith(2, null, [])
-            }
           }
-        },
-        options: {},
-        addresses: []
-      });
-      history.transactionInfo = [{}];
-      history.getTransactionInfo(address, function(err) {
-        if (err) {
-          throw err;
         }
-        history.transactionInfo.length.should.equal(1);
+      };
-        history.node.services.address.getOutputs.args[0][0].should.equal(address);
+      var options = {};
+      var addresses = [address, address];
+      var history = new AddressHistory({
+        node: node,
+        options: options,
+        addresses: addresses
+      });
+      history.get(function(err) {
+        should.exist(err);
+        err.message.should.equal('test2');
+        done();
       });
     });
-    it('will concatenate outputs and inputs', function() {
+    it('will query get address summary directly with one address', function(done) {
-      var history = new AddressHistory({
+      var txids = [];
-        node: {
+      var summary = {
-          services: {
+        txids: txids
-            address: {
+      };
-              getOutputs: sinon.stub().callsArgWith(2, null, [{}]),
+      var node = {
-              getInputs: sinon.stub().callsArgWith(2, null, [{}])
+        services: {
-            }
+          address: {
+            getAddressSummary: sinon.stub().callsArgWith(2, null, summary)
           }
-        },
-        options: {},
-        addresses: []
-      });
-      history.transactionInfo = [{}];
-      history.getTransactionInfo(address, function(err) {
-        if (err) {
-          throw err;
         }
-        history.transactionInfo.length.should.equal(3);
+      };
-        history.node.services.address.getOutputs.args[0][0].should.equal(address);
+      var options = {};
+      var addresses = [address];
+      var history = new AddressHistory({
+        node: node,
+        options: options,
+        addresses: addresses
+      });
+      history._mergeAndSortTxids = sinon.stub();
+      history._paginateWithDetails = sinon.stub().callsArg(1);
+      history.get(function() {
+        history.node.services.address.getAddressSummary.callCount.should.equal(1);
+        history.node.services.address.getAddressSummary.args[0][0].should.equal(address);
+        history.node.services.address.getAddressSummary.args[0][1].should.equal(options);
+        history._paginateWithDetails.callCount.should.equal(1);
+        history._paginateWithDetails.args[0][0].should.equal(txids);
+        history._mergeAndSortTxids.callCount.should.equal(0);
+        done();
+      });
+    });
+    it('will merge multiple summaries with multiple addresses', function(done) {
+      var txids = [];
+      var summary = {
+        txids: txids
+      };
+      var node = {
+        services: {
+          address: {
+            getAddressSummary: sinon.stub().callsArgWith(2, null, summary)
+          }
+        }
+      };
+      var options = {};
+      var addresses = [address, address];
+      var history = new AddressHistory({
+        node: node,
+        options: options,
+        addresses: addresses
+      });
+      history._mergeAndSortTxids = sinon.stub().returns(txids);
+      history._paginateWithDetails = sinon.stub().callsArg(1);
+      history.get(function() {
+        history.node.services.address.getAddressSummary.callCount.should.equal(2);
+        history.node.services.address.getAddressSummary.args[0][0].should.equal(address);
+        history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({
+          fullTxList: true
+        });
+        history._paginateWithDetails.callCount.should.equal(1);
+        history._paginateWithDetails.args[0][0].should.equal(txids);
+        history._mergeAndSortTxids.callCount.should.equal(1);
+        done();
       });
     });
   });
 
-  describe('@sortByHeight', function() {
+  describe('#_paginateWithDetails', function() {
-    it('will sort latest to oldest using height', function() {
+    it('slice txids based on "from" and "to" (3 to 30)', function() {
-      var transactionInfo = [
+      var node = {};
-        {
+      var options = {
-          height: 276328
+        from: 3,
-        },
+        to: 30
-        {
+      };
-          height: 273845,
+      var addresses = [address];
-        },
+      var history = new AddressHistory({
-        {
+        node: node,
-          height: 555655
+        options: options,
-        },
+        addresses: addresses
-        {
+      });
-          height: 325496
+      var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        },
+      sinon.stub(history, 'getDetailedInfo', function(txid, next) {
-        {
+        this.detailedArray.push(txid);
-          height: 329186
+        next();
-        },
+      });
-        {
+      history._paginateWithDetails(txids, function(err, result) {
-          height: 534195
+        result.totalCount.should.equal(11);
-        }
+        result.items.should.deep.equal([7, 6, 5, 4, 3, 2, 1, 0]);
-      ];
+      });
-      transactionInfo.sort(AddressHistory.sortByHeight);
-      transactionInfo[0].height.should.equal(555655);
-      transactionInfo[1].height.should.equal(534195);
-      transactionInfo[2].height.should.equal(329186);
-      transactionInfo[3].height.should.equal(325496);
-      transactionInfo[4].height.should.equal(276328);
-      transactionInfo[5].height.should.equal(273845);
     });
-    it('mempool and tip with time in the future', function() {
+    it('slice txids based on "from" and "to" (0 to 3)', function() {
-      var transactionInfo = [
+      var node = {};
-        {
+      var options = {
-          timestamp: 1442050425439,
+        from: 0,
-          height: 14,
+        to: 3
-        },
+      };
-        {
+      var addresses = [address];
-          timestamp: 1442050424328,
+      var history = new AddressHistory({
-          height: -1
+        node: node,
-        },
+        options: options,
-        {
+        addresses: addresses
-          timestamp: 1442050424429,
+      });
-          height: -1
+      var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        },
+      sinon.stub(history, 'getDetailedInfo', function(txid, next) {
-        {
+        this.detailedArray.push(txid);
-          timestamp: 1442050425439,
+        next();
-          height: 15
+      });
-        }
+      history._paginateWithDetails(txids, function(err, result) {
-      ];
+        result.totalCount.should.equal(11);
-      transactionInfo.sort(AddressHistory.sortByHeight);
+        result.items.should.deep.equal([10, 9, 8]);
-      transactionInfo[0].height.should.equal(-1);
+      });
-      transactionInfo[0].timestamp.should.equal(1442050424429);
-      transactionInfo[1].height.should.equal(-1);
-      transactionInfo[1].timestamp.should.equal(1442050424328);
-      transactionInfo[2].height.should.equal(15);
-      transactionInfo[3].height.should.equal(14);
     });
-    it('tip with time in the future and mempool', function() {
+    it('will given an error if the full details is too long', function() {
-      var transactionInfo = [
+      var node = {};
-        {
+      var options = {
-          timestamp: 1442050425439,
+        from: 0,
-          height: 14,
+        to: 3
-        },
+      };
-        {
+      var addresses = [address];
-          timestamp: 1442050424328,
+      var history = new AddressHistory({
-          height: -1
+        node: node,
-        }
+        options: options,
-      ];
+        addresses: addresses
-      transactionInfo.sort(AddressHistory.sortByHeight);
+      });
-      transactionInfo[0].height.should.equal(-1);
+      var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-      transactionInfo[1].height.should.equal(14);
+      sinon.stub(history, 'getDetailedInfo', function(txid, next) {
+        this.detailedArray.push(txid);
+        next();
+      });
+      history.maxHistoryQueryLength = 1;
+      history._paginateWithDetails(txids, function(err) {
+        should.exist(err);
+        err.message.match(/Maximum/);
+      });
     });
-    it('many transactions in the mempool', function() {
+    it('will give full result without pagination options', function() {
-      var transactionInfo = [
+      var node = {};
-        {
+      var options = {};
-          timestamp: 1442259670462,
+      var addresses = [address];
-          height: -1
+      var history = new AddressHistory({
-        },
+        node: node,
-        {
+        options: options,
-          timestamp: 1442259785114,
+        addresses: addresses
-          height: -1
+      });
-        },
+      var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
-        {
+      sinon.stub(history, 'getDetailedInfo', function(txid, next) {
-          timestamp: 1442259759896,
+        this.detailedArray.push(txid);
-          height: -1
+        next();
-        },
+      });
-        {
+      history._paginateWithDetails(txids, function(err, result) {
-          timestamp: 1442259692601,
+        result.totalCount.should.equal(11);
-          height: -1
+        result.items.should.deep.equal([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]);
-        },
+      });
-        {
-          timestamp: 1442259692601,
-          height: 100
-        },
-        {
-          timestamp: 1442259749463,
-          height: -1
-        },
-        {
-          timestamp: 1442259737719,
-          height: -1
-        },
-        {
-          timestamp: 1442259773138,
-          height: -1,
-        }
-      ];
-      transactionInfo.sort(AddressHistory.sortByHeight);
-      transactionInfo[0].timestamp.should.equal(1442259785114);
-      transactionInfo[1].timestamp.should.equal(1442259773138);
-      transactionInfo[2].timestamp.should.equal(1442259759896);
-      transactionInfo[3].timestamp.should.equal(1442259749463);
-      transactionInfo[4].timestamp.should.equal(1442259737719);
-      transactionInfo[5].timestamp.should.equal(1442259692601);
-      transactionInfo[6].timestamp.should.equal(1442259670462);
-      transactionInfo[7].height.should.equal(100);
-    });
-    it('mempool and mempool', function() {
-      var transactionInfo = [
-        {
-          timestamp: 1442050424328,
-          height: -1
-        },
-        {
-          timestamp: 1442050425439,
-          height: -1,
-        }
-      ];
-      transactionInfo.sort(AddressHistory.sortByHeight);
-      transactionInfo[0].timestamp.should.equal(1442050425439);
-      transactionInfo[1].timestamp.should.equal(1442050424328);
-    });
-    it('mempool and mempool with the same timestamp', function() {
-      var transactionInfo = [
-        {
-          timestamp: 1442050425439,
-          height: -1,
-          txid: '1',
-        },
-        {
-          timestamp: 1442050425439,
-          height: -1,
-          txid: '2'
-        }
-      ];
-      transactionInfo.sort(AddressHistory.sortByHeight);
-      transactionInfo[0].txid.should.equal('1');
-      transactionInfo[1].txid.should.equal('2');
-    });
-    it('matching block heights', function() {
-      var transactionInfo = [
-        {
-          height: 325496,
-          txid: '1',
-        },
-        {
-          height: 325496,
-          txid: '2'
-        }
-      ];
-      transactionInfo.sort(AddressHistory.sortByHeight);
-      transactionInfo[0].txid.should.equal('1');
-      transactionInfo[1].txid.should.equal('2');
     });
   });
 
-  describe('#sortAndPaginateCombinedArray', function() {
+  describe('#_mergeAndSortTxids', function() {
-    it('from 0 to 2', function() {
+    it('will merge and sort multiple summaries', function() {
-      var history = new AddressHistory({
+      var summaries = [
-        node: {},
-        options: {
-          from: 0,
-          to: 2
-        },
-        addresses: []
-      });
-      history.combinedArray = [
         {
-          height: 13
+          totalReceived: 10000000,
+          totalSpent: 0,
+          balance: 10000000,
+          appearances: 2,
+          unconfirmedBalance: 20000000,
+          unconfirmedAppearances: 2,
+          appearanceIds: {
+            '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579': 154,
+            'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce': 120
+          },
+          unconfirmedAppearanceIds: {
+            'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406,
+            'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964
+          }
         },
         {
-          height: 14,
+          totalReceived: 59990000,
-        },
+          totalSpent: 0,
-        {
+          balance: 49990000,
-          height: 12
+          appearances: 3,
+          unconfirmedBalance: 1000000,
+          unconfirmedAppearances: 3,
+          appearanceIds: {
+            'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2': 156,
+            'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47': 152,
+            'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f': 151
+          },
+          unconfirmedAppearanceIds: {
+            'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377,
+            'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363,
+            'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107
+          }
         }
       ];
-      history.sortAndPaginateCombinedArray();
+      var node = {};
-      history.combinedArray.length.should.equal(2);
+      var options = {};
-      history.combinedArray[0].height.should.equal(14);
+      var addresses = [address];
-      history.combinedArray[1].height.should.equal(13);
-    });
-    it('from 0 to 4 (exceeds length)', function() {
       var history = new AddressHistory({
-        node: {},
+        node: node,
-        options: {
+        options: options,
-          from: 0,
+        addresses: addresses
-          to: 4
-        },
-        addresses: []
       });
-      history.combinedArray = [
+      var txids = history._mergeAndSortTxids(summaries);
-        {
+      txids.should.deep.equal([
-          height: 13
+        'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce',
-        },
+        'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f',
-        {
+        'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47',
-          height: 14,
+        '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579',
-        },
+        'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2',
-        {
+        'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345',
-          height: 12
+        'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9',
-        }
+        'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a',
-      ];
+        'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693',
-      history.sortAndPaginateCombinedArray();
+        'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681'
-      history.combinedArray.length.should.equal(3);
+      ]);
-      history.combinedArray[0].height.should.equal(14);
-      history.combinedArray[1].height.should.equal(13);
-      history.combinedArray[2].height.should.equal(12);
-    });
-    it('from 0 to 1', function() {
-      var history = new AddressHistory({
-        node: {},
-        options: {
-          from: 0,
-          to: 1
-        },
-        addresses: []
-      });
-      history.combinedArray = [
-        {
-          height: 13
-        },
-        {
-          height: 14,
-        },
-        {
-          height: 12
-        }
-      ];
-      history.sortAndPaginateCombinedArray();
-      history.combinedArray.length.should.equal(1);
-      history.combinedArray[0].height.should.equal(14);
-    });
-    it('from 2 to 3', function() {
-      var history = new AddressHistory({
-        node: {},
-        options: {
-          from: 2,
-          to: 3
-        },
-        addresses: []
-      });
-      history.combinedArray = [
-        {
-          height: 13
-        },
-        {
-          height: 14,
-        },
-        {
-          height: 12
-        }
-      ];
-      history.sortAndPaginateCombinedArray();
-      history.combinedArray.length.should.equal(1);
-      history.combinedArray[0].height.should.equal(12);
-    });
-    it('from 10 to 20 (out of range)', function() {
-      var history = new AddressHistory({
-        node: {},
-        options: {
-          from: 10,
-          to: 20
-        },
-        addresses: []
-      });
-      history.combinedArray = [
-        {
-          height: 13
-        },
-        {
-          height: 14,
-        },
-        {
-          height: 12
-        }
-      ];
-      history.sortAndPaginateCombinedArray();
-      history.combinedArray.length.should.equal(0);
     });
   });
 
   describe('#getDetailedInfo', function() {
-    it('will add additional information to existing this.transactions', function() {
+    it('will add additional information to existing this.transactions', function(done) {
       var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0';
+      var tx = {
+        populateInputs: sinon.stub().callsArg(2),
+        __height: 20,
+        __timestamp: 1453134151,
+        isCoinbase: sinon.stub().returns(false),
+        getFee: sinon.stub().returns(1000)
+      };
       var history = new AddressHistory({
         node: {
           services: {
             db: {
-              getTransactionWithBlockInfo: sinon.stub()
+              getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, tx),
+              tip: {
+                __height: 300
+              }
             }
           }
         },
         options: {},
         addresses: []
       });
+      history.getAddressDetailsForTransaction = sinon.stub().returns({
+        addresses: {},
+        satoshis: 1000,
+      });
       history.getDetailedInfo(txid, function(err) {
         if (err) {
           throw err;
         }
-        history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0);
+        history.node.services.db.getTransactionWithBlockInfo.callCount.should.equal(1);
+        done();
       });
     });
-    it('will handle error from getTransactionFromBlock', function() {
+    it('will handle error from getTransactionFromBlock', function(done) {
       var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0';
       var history = new AddressHistory({
         node: {
@@ -547,9 +366,10 @@ describe('Address Service History', function() {
       });
       history.getDetailedInfo(txid, function(err) {
         err.message.should.equal('test');
+        done();
       });
     });
-    it('will handle error from populateInputs', function() {
+    it('will handle error from populateInputs', function(done) {
       var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0';
       var history = new AddressHistory({
         node: {
@@ -566,9 +386,10 @@ describe('Address Service History', function() {
       });
       history.getDetailedInfo(txid, function(err) {
         err.message.should.equal('test');
+        done();
       });
     });
-    it('will set this.transactions with correct information', function() {
+    it('will set this.transactions with correct information', function(done) {
       // block #314159
       // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e
       // outputIndex 1
@@ -602,7 +423,7 @@ describe('Address Service History', function() {
           }
         },
         options: {},
-        addresses: []
+        addresses: [txAddress]
       });
       var transactionInfo = {
         addresses: {},
@@ -614,7 +435,7 @@ describe('Address Service History', function() {
       transactionInfo.addresses[txAddress] = {};
       transactionInfo.addresses[txAddress].outputIndexes = [1];
       transactionInfo.addresses[txAddress].inputIndexes = [];
-      history.getDetailedInfo(transactionInfo, function(err) {
+      history.getDetailedInfo(txid, function(err) {
         if (err) {
           throw err;
         }
@@ -629,11 +450,74 @@ describe('Address Service History', function() {
         info.timestamp.should.equal(1407292005);
         info.fees.should.equal(20000);
         info.tx.should.equal(transaction);
+        done();
       });
     });
   });
+
+  describe('#getAddressDetailsForTransaction', function() {
+    it('will calculate details for the transaction', function(done) {
+      /* jshint sub:true */
+      var tx = bitcore.Transaction({
+        'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c',
+        'version': 1,
+        'inputs': [
+          {
+            'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543',
+            'outputIndex': 0,
+            'sequenceNumber': 4294967295,
+            'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924',
+            'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924',
+            'output': {
+              'satoshis': 1000000000,
+              'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+            }
+          }
+        ],
+        'outputs': [
+          {
+            'satoshis': 100000000,
+            'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+          },
+          {
+            'satoshis': 200000000,
+            'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+          },
+          {
+            'satoshis': 50000000,
+            'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+          },
+          {
+            'satoshis': 300000000,
+            'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+          },
+          {
+            'satoshis': 349990000,
+            'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'
+          }
+        ],
+        'nLockTime': 0
+      });
+      var history = new AddressHistory({
+        node: {
+          network: bitcore.Networks.testnet
+        },
+        options: {},
+        addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']
+      });
+      var details = history.getAddressDetailsForTransaction(tx);
+      should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']);
+      details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]);
+      details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([
+        0, 1, 2, 3, 4
+      ]);
+      details.satoshis.should.equal(-10000);
+      done();
+    });
+  });
+
   describe('#getConfirmationsDetail', function() {
-    it('the correct confirmations when included in the tip', function() {
+    it('the correct confirmations when included in the tip', function(done) {
       var history = new AddressHistory({
         node: {
           services: {
@@ -651,30 +535,7 @@ describe('Address Service History', function() {
         __height: 100
       };
       history.getConfirmationsDetail(transaction).should.equal(1);
-    });
+      done();
-  });
-  describe('#getSatoshisDetail', function() {
-    it('subtract inputIndexes satoshis without outputIndexes', function() {
-      var history = new AddressHistory({
-        node: {},
-        options: {},
-        addresses: []
-      });
-      var transaction = {
-        inputs: [
-          {
-            output: {
-              satoshis: 10000
-            }
-          }
-        ]
-      };
-      var txInfo = {
-        addresses: {}
-      };
-      txInfo.addresses[address] = {};
-      txInfo.addresses[address].inputIndexes = [0];
-      history.getSatoshisDetail(transaction, txInfo).should.equal(-10000);
     });
   });
 });