Merge pull request #548 from bitpay/opt/txlist-cache

add txIdList cache

lib/services/address/index.js

@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
+var LRU = require('lru-cache');
+var XXHash = require('xxhash');
+
+
+
+// See rationale about this cache at function getTxList(next)
+const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
+                                                // clients downloading their tx history)
+const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
+const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
+const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -24,6 +35,11 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
+  this._txIdListCache = new LRU({
+    max: TXID_LIST_CACHE_ITEMS,        
+    maxAge: TXID_LIST_CACHE_EXPIRATION 
+  });
+ 
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
+//
+// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
+//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
+  var cacheUsed = false;
 
   options = options || {};
   options.from = options.from || 0;
@@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
-  async.eachLimit(addresses, 4, function(address, next) {
-    self._getAddressTxidHistory(address, options, next);
 
-  }, function(err) {
+  function getTxList(next) {
 
+
+    function hashAddresses(addresses) {
+
+      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
+      // a 32 bits hash is secure enough
+
+      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
+    };
+
+    var calculatedCacheKey;
+
+    // We use the cache ONLY on from > 0 queries.
+    //
+    // Rationale: The a full history is downloaded, the client do
+    // from =0, to=x
+    // then from =x+1 to=y
+    // then [...]
+    // The objective of this cache is to speed up the from>0 queries, and also
+    // "freeze" the txid list during download.
+    //
+    if (options.from >0 ) {
+      
+      let cacheKey  = options.cacheKey;
+      if (!cacheKey) {
+         calculatedCacheKey = hashAddresses(addresses);
+         cacheKey =  calculatedCacheKey;
+      }
+
+      var txIdList  = self._txIdListCache.get(cacheKey);
+      if (txIdList) {
+        options.txIdList = txIdList;
+        cacheUsed = true;
+        return next();
+      }
+    }
+
+    // Get the list from the db
+    async.eachLimit(addresses, 4, function(address, next) {
+      self._getAddressTxidHistory(address, options, next);
+    }, function(err) {
+      if (err) return next(err);
+
+      var list = lodash.uniqBy(options.txIdList, function(x) { 
+        return x.txid + x.height;
+      });
+
+
+      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
+
+      if (list.length > TXID_LIST_CACHE_MIN) {
+        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
+
+        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
+      }
+
+      return next();
+    });
+
+  };
+
+
+  getTxList(function(err) {
     if(err) {
       return callback(err);
     }
 
-    var list = lodash.uniqBy(options.txIdList, function(x) { 
-      return x.txid + x.height;
-    });
-
-
-    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList
+        items: txList,
       };
 
-      callback(null, results);
+      // cacheUsed is returned for testing
+      callback(null, results, cacheUsed);
 
     });
   });

package-lock.json

@@ -159,6 +159,11 @@
         "lodash": "4.17.4"
       }
     },
+    "async-limiter": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
+      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -238,9 +243,9 @@
           "optional": true,
           "requires": {
             "debug": "2.6.8",
-            "engine.io": "3.1.0",
+            "engine.io": "3.1.4",
             "object-assign": "4.1.1",
-            "socket.io-adapter": "1.1.0",
+            "socket.io-adapter": "1.1.1",
             "socket.io-client": "2.0.1",
             "socket.io-parser": "3.1.2"
           }
@@ -256,7 +261,7 @@
             "component-bind": "1.0.0",
             "component-emitter": "1.2.1",
             "debug": "2.6.4",
-            "engine.io-client": "3.1.1",
+            "engine.io-client": "3.1.4",
             "has-cors": "1.1.0",
             "indexof": "0.0.1",
             "object-component": "0.0.3",
@@ -1351,50 +1356,84 @@
       }
     },
     "engine.io": {
-      "version": "3.1.0",
+      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz",
-      "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=",
+      "integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=",
       "optional": true,
       "requires": {
         "accepts": "1.3.3",
         "base64id": "1.0.0",
         "cookie": "0.3.1",
-        "debug": "2.6.8",
+        "debug": "2.6.9",
-        "engine.io-parser": "2.1.1",
+        "engine.io-parser": "2.1.2",
         "uws": "0.14.5",
-        "ws": "2.3.1"
+        "ws": "3.3.3"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
       }
     },
     "engine.io-client": {
-      "version": "3.1.1",
+      "version": "3.1.4",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz",
-      "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=",
+      "integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=",
       "optional": true,
       "requires": {
         "component-emitter": "1.2.1",
         "component-inherit": "0.0.3",
-        "debug": "2.6.8",
+        "debug": "2.6.9",
-        "engine.io-parser": "2.1.1",
+        "engine.io-parser": "2.1.2",
         "has-cors": "1.1.0",
         "indexof": "0.0.1",
-        "parsejson": "0.0.3",
         "parseqs": "0.0.5",
         "parseuri": "0.0.5",
-        "ws": "2.3.1",
+        "ws": "3.3.3",
-        "xmlhttprequest-ssl": "1.5.3",
+        "xmlhttprequest-ssl": "1.5.5",
         "yeast": "0.1.2"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "optional": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        },
+        "xmlhttprequest-ssl": {
+          "version": "1.5.5",
+          "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
+          "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
+          "optional": true
+        }
       }
     },
     "engine.io-parser": {
-      "version": "2.1.1",
+      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
-      "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=",
+      "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
       "requires": {
         "after": "0.8.2",
-        "arraybuffer.slice": "0.0.6",
+        "arraybuffer.slice": "0.0.7",
         "base64-arraybuffer": "0.1.5",
         "blob": "0.0.4",
         "has-binary2": "1.0.2"
+      },
+      "dependencies": {
+        "arraybuffer.slice": {
+          "version": "0.0.7",
+          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
+          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
+        }
       }
     },
     "entities": {
@@ -4048,30 +4087,10 @@
       }
     },
     "socket.io-adapter": {
-      "version": "1.1.0",
+      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
-      "integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=",
+      "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
-      "optional": true,
+      "optional": true
-      "requires": {
-        "debug": "2.3.3"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.3.3",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
-          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
-          "optional": true,
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
-          "optional": true
-        }
-      }
     },
     "socket.io-client": {
       "version": "1.7.4",
@@ -4483,9 +4502,9 @@
       "optional": true
     },
     "ultron": {
-      "version": "1.1.0",
+      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
-      "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
+      "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
     },
     "unc-path-regex": {
       "version": "0.1.2",
@@ -4631,19 +4650,13 @@
       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
     },
     "ws": {
-      "version": "2.3.1",
+      "version": "3.3.3",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
-      "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=",
+      "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
       "requires": {
-        "safe-buffer": "5.0.1",
+        "async-limiter": "1.0.0",
-        "ultron": "1.1.0"
+        "safe-buffer": "5.1.1",
-      },
+        "ultron": "1.1.1"
-      "dependencies": {
-        "safe-buffer": {
-          "version": "5.0.1",
-          "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
-          "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
-        }
       }
     },
     "wtf-8": {
@@ -4661,6 +4674,14 @@
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
       "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
     },
+    "xxhash": {
+      "version": "0.2.4",
+      "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz",
+      "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=",
+      "requires": {
+        "nan": "2.6.2"
+      }
+    },
     "yallist": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",

package.json

@@ -43,12 +43,13 @@
     "levelup": "^2.0.0",
     "liftoff": "^2.2.0",
     "lodash": "^4.17.4",
-    "lru-cache": "^4.0.2",
+    "lru-cache": "^4.1.1",
     "memwatch-next": "^0.3.0",
     "mkdirp": "0.5.0",
     "path-is-absolute": "^1.0.0",
     "socket.io": "^1.4.5",
-    "socket.io-client": "^1.4.5"
+    "socket.io-client": "^1.4.5",
+    "xxhash": "^0.2.4"
   },
   "devDependencies": {
     "chai": "^3.5.0",

test/services/address/index.unit.js

@@ -172,7 +172,125 @@ describe('Address Service', function() {
     });
 
 
+    describe('TxIdList cache', function() {
+      var list, old_getAddressTxidHistory, old_getAddressTxHistory;
 
+      beforeEach(function(done){
+        this.clock = sinon.useFakeTimers();
+        list = [];
+        for(let i=1000; i>0; i--) {
+          list.push({
+            txid: "txid" + i,
+            height: 1000  + i,
+
+          });
+        };
+        old_getAddressTxidHistory = addressService._getAddressTxidHistory;
+        // Note that this stub DOES NOT respect options.from/to as the real function
+        addressService._getAddressTxidHistory = function(addr, options, cb) {
+          options.txIdList = lodash.clone(list);
+          return cb();
+        };
+        old_getAddressTxHistory = addressService._getAddressTxHistory;
+
+        addressService._getAddressTxHistory = function(options, cb) {
+          return cb(null, options.txIdList);
+        };
+
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      afterEach(function(done){
+        this.clock.restore();
+
+        addressService._getAddressTxidHistory = old_getAddressTxHistory;
+        addressService._getAddressTxHistory = old_getAddressTxHistory;
+        done();
+      });
+
+
+      it('should not cache the address txlist history when from =0 ', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          expect(cacheUsed).equal(false);
+          done();
+        });
+      });
+
+      it('should cache the address txlist history', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+
+      it('should retrive cached list using cachekey', function(done) {
+        addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(true);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+   
+
+      it('should expire cache', function(done) {
+        this.clock.tick(35*1000);
+        addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          done();
+        });
+      });
+
+      it('should cache using the address as key', function(done) {
+        addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+          if (err) {
+            return done(err);
+          }
+          expect(cacheUsed).equal(false);
+          expect(res.totalCount).equal(1000);
+          expect(res.items,'txid').to.be.deep.equal(list);
+          addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
+            if (err) {
+              return done(err);
+            }
+            expect(cacheUsed).equal(true);
+            expect(res.totalCount).equal(1000);
+            expect(res.items,'txid').to.be.deep.equal(list);
+            done();
+          });
+        });
+      });
+
+
+    });
   });
 
   describe('#_getAddressTxidHistory', function() {