Merge pull request #407 from bitpay/opt/multiple-address-history

Opt/multiple address history

lib/services/address/encoding.js

@@ -19,6 +19,15 @@ exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
   return key.toString('binary');
 };
 
+exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) {
+  var key = Buffer.concat([
+    hashBuffer,
+    hashTypeBuffer,
+  ]);
+  return key.toString('binary');
+};
+
+
 exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) {
   var heightBuffer = new Buffer(4);
   heightBuffer.writeUInt32BE(height);

lib/services/address/history.js

@@ -44,6 +44,7 @@ function AddressHistory(args) {
 AddressHistory.prototype._mergeAndSortTxids = function(summaries) {
   var appearanceIds = {};
   var unconfirmedAppearanceIds = {};
+
   for (var i = 0; i < summaries.length; i++) {
     var summary = summaries[i];
     for (var key in summary.appearanceIds) {
@@ -79,16 +80,19 @@ AddressHistory.prototype.get = function(callback) {
     return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded'));
   }
 
+  var opts = _.clone(this.options);
+  opts.noBalance = true;
+
   if (this.addresses.length === 1) {
     var address = this.addresses[0];
-    self.node.services.address.getAddressSummary(address, this.options, function(err, summary) {
+    self.node.services.address.getAddressSummary(address, opts, function(err, summary) {
       if (err) {
         return callback(err);
       }
       return self._paginateWithDetails.call(self, summary.txids, callback);
     });
   } else {
-    var opts = _.clone(this.options);
+
     opts.fullTxList = true;
     async.mapLimit(
       self.addresses,

lib/services/address/index.js

@@ -57,6 +57,7 @@ var AddressService = function(options) {
   }
   this.mempoolIndex = null; // Used for larger mempool indexes
   this.mempoolSpentIndex = {}; // Used for small quick synchronous lookups
+  this.mempoolAddressIndex = {}; // Used to check if an address is on the spend pool
 };
 
 inherits(AddressService, BaseService);
@@ -70,6 +71,7 @@ AddressService.prototype.start = function(callback) {
   var self = this;
 
   async.series([
+
     function(next) {
       // Flush any existing mempool index
       if (fs.existsSync(self.mempoolIndexPath)) {
@@ -274,6 +276,25 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) {
 
 };
 
+AddressService.prototype._updateAddressIndex = function(key, add) {
+  var currentValue = this.mempoolAddressIndex[key] || 0;
+
+  if(add) {
+    if (currentValue > 0) {
+      this.mempoolAddressIndex[key] = currentValue + 1;
+    } else  {
+      this.mempoolAddressIndex[key] = 1;
+    }
+  } else {
+    if (currentValue <= 1) {
+      delete this.mempoolAddressIndex[key];
+    } else {
+      this.mempoolAddressIndex[key]--;
+    }
+  }
+};
+
+
 /**
  * This function will update the mempool address index with the necessary
  * information for further lookups.
@@ -306,6 +327,10 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) {
       continue;
     }
 
+    var addressIndexKey = encoding.encodeMempoolAddressIndexKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer);
+
+    this._updateAddressIndex(addressIndexKey, add);
+
     // Update output index
     var outputIndexBuffer = new Buffer(4);
     outputIndexBuffer.writeUInt32BE(outputIndex);
@@ -398,6 +423,9 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) {
       value: inputValue
     });
 
+    var addressIndexKey = encoding.encodeMempoolAddressIndexKey(inputHashBuffer, inputHashType);
+
+    this._updateAddressIndex(addressIndexKey, add);
   }
 
   if (!callback) {
@@ -1434,26 +1462,29 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address,
 
     var txid = output.txid;
     var outputIndex = output.outputIndex;
-
-    // Bitcoind's isSpent only works for confirmed transactions
-    var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex);
     result.totalReceived += output.satoshis;
     result.appearanceIds[txid] = output.height;
 
-    if (!spentDB) {
+    if(!options.noBalance) {
-      result.balance += output.satoshis;
-    }
 
-    if (options.queryMempool) {
+      // Bitcoind's isSpent only works for confirmed transactions
-      // Check to see if this output is spent in the mempool and if so
+      var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex);
-      // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta)
+
-      var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(
+      if(!spentDB) {
-        new Buffer(txid, 'hex'), // TODO: get buffer directly
+        result.balance += output.satoshis;
-        outputIndex
+      }
-      );
+
-      var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey];
+      if(options.queryMempool) {
-      if (spentMempool) {
+        // Check to see if this output is spent in the mempool and if so
-        result.unconfirmedBalance -= output.satoshis;
+        // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta)
+        var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(
+          new Buffer(txid, 'hex'), // TODO: get buffer directly
+          outputIndex
+        );
+        var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey];
+        if(spentMempool) {
+          result.unconfirmedBalance -= output.satoshis;
+        }
       }
     }
 
@@ -1505,6 +1536,11 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options,
   var addressStr = address.toString();
   var hashBuffer = address.hashBuffer;
   var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type];
+  var addressIndexKey = encoding.encodeMempoolAddressIndexKey(hashBuffer, hashTypeBuffer);
+
+  if(!this.mempoolAddressIndex[addressIndexKey]) {
+    return callback(null, result);
+  }
 
   async.waterfall([
     function(next) {
@@ -1529,14 +1565,16 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options,
 
           result.unconfirmedAppearanceIds[output.txid] = output.timestamp;
 
-          var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(
+          if(!options.noBalance) {
-            new Buffer(output.txid, 'hex'), // TODO: get buffer directly
+            var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(
-            output.outputIndex
+              new Buffer(output.txid, 'hex'), // TODO: get buffer directly
-          );
+              output.outputIndex
-          var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey];
+            );
-          // Only add this to the balance if it's not spent in the mempool already
+            var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey];
-          if (!spentMempool) {
+            // Only add this to the balance if it's not spent in the mempool already
-            result.unconfirmedBalance += output.satoshis;
+            if(!spentMempool) {
+              result.unconfirmedBalance += output.satoshis;
+            }
           }
         }
         next(null, result);

test/services/address/history.unit.js

@@ -122,7 +122,9 @@ describe('Address Service History', function() {
       history.get(function() {
         history.node.services.address.getAddressSummary.callCount.should.equal(1);
         history.node.services.address.getAddressSummary.args[0][0].should.equal(address);
-        history.node.services.address.getAddressSummary.args[0][1].should.equal(options);
+        history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({
+          noBalance: true
+        });
         history._paginateWithDetails.callCount.should.equal(1);
         history._paginateWithDetails.args[0][0].should.equal(txids);
         history._mergeAndSortTxids.callCount.should.equal(0);
@@ -154,7 +156,8 @@ describe('Address Service History', function() {
         history.node.services.address.getAddressSummary.callCount.should.equal(2);
         history.node.services.address.getAddressSummary.args[0][0].should.equal(address);
         history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({
-          fullTxList: true
+          fullTxList: true,
+          noBalance: true
         });
         history._paginateWithDetails.callCount.should.equal(1);
         history._paginateWithDetails.args[0][0].should.equal(txids);

test/services/address/index.unit.js

@@ -9,6 +9,7 @@ var bitcorenode = require('../../../');
 var AddressService = bitcorenode.services.Address;
 var blockData = require('../../data/livenet-345003.json');
 var bitcore = require('bitcore-lib');
+var _ = bitcore.deps._;
 var memdown = require('memdown');
 var leveldown = require('leveldown');
 var Networks = bitcore.Networks;
@@ -1971,6 +1972,8 @@ describe('Address Service', function() {
       am.mempoolIndex.batch = function(operations, callback) {
         callback.should.be.a('function');
         Object.keys(am.mempoolSpentIndex).length.should.equal(14);
+        Object.keys(am.mempoolAddressIndex).length.should.equal(5);
+        _.values(am.mempoolAddressIndex).should.deep.equal([1,1,12,1,1]);
         for (var i = 0; i < operations.length; i++) {
           operations[i].type.should.equal('put');
         }
@@ -2006,6 +2009,7 @@ describe('Address Service', function() {
         for (var i = 0; i < operations.length; i++) {
           operations[i].type.should.equal('del');
         }
+        Object.keys(am.mempoolAddressIndex).length.should.equal(0);
       };
       am.updateMempoolIndex(tx, false);
     });
@@ -2435,6 +2439,54 @@ describe('Address Service', function() {
     });
   });
 
+
+  describe('#_updateAddressIndex', function() {
+    it('should add using 2 keys', function() {
+      var as = new AddressService({
+          mempoolMemoryIndex: true,
+          node: mocknode
+      });
+
+      _.values(as.mempoolAddressIndex).should.deep.equal([]);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index2', true);
+      as._updateAddressIndex('index2', true);
+      as.mempoolAddressIndex.should.deep.equal({
+        "index1": 4,
+        "index2": 2
+      });
+    });
+
+    it('should add/remove using 2 keys', function() {
+      var as = new AddressService({
+          mempoolMemoryIndex: true,
+          node: mocknode
+      });
+      _.values(as.mempoolAddressIndex).should.deep.equal([]);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', true);
+      as._updateAddressIndex('index1', false);
+
+      as._updateAddressIndex('index2', true);
+      as._updateAddressIndex('index2', true);
+      as._updateAddressIndex('index2', false);
+      as._updateAddressIndex('index2', false);
+      as.mempoolAddressIndex.should.deep.equal({
+        "index1": 3
+      });
+      as._updateAddressIndex('index2', false);
+      as.mempoolAddressIndex.should.deep.equal({
+        "index1": 3
+      });
+    });
+  });
+ 
+
   describe('#_getAddressMempoolSummary', function() {
     it('skip if options not enabled', function(done) {
       var testnode = {
@@ -2527,6 +2579,11 @@ describe('Address Service', function() {
         0
       );
       as.mempoolSpentIndex[spentIndexSyncKey] = true;
+
+      var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type];
+      var addressIndex = encoding.encodeMempoolAddressIndexKey(address.hashBuffer, hashTypeBuffer);
+      as.mempoolAddressIndex[addressIndex] = 1;
+
       as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs);
       as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs);
       as._getAddressMempoolSummary(address, options, resultBase, function(err, result) {