diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 6f40da3b..97144473 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -13,17 +13,6 @@ var Encoding = require('./encoding'); var Transform = require('stream').Transform; var assert = require('assert'); var utils = require('../../utils'); -var LRU = require('lru-cache'); -var XXHash = require('xxhash'); - - - -// See rationale about this cache at function getTxList(next) -const TXID_LIST_CACHE_ITEMS = 250; // nr of items (this translates to: consecutive - // clients downloading their tx history) -const TXID_LIST_CACHE_EXPIRATION = 1000 * 30; // ms -const TXID_LIST_CACHE_MIN = 100; // Min items to cache -const TXID_LIST_CACHE_SEED = 0x3233DE; // Min items to cache var AddressService = function(options) { @@ -35,11 +24,6 @@ var AddressService = function(options) { this._network = this.node.network; this._db = this.node.services.db; this._mempool = this.node.services.mempool; - this._txIdListCache = new LRU({ - max: TXID_LIST_CACHE_ITEMS, - maxAge: TXID_LIST_CACHE_EXPIRATION - }); - if (this._network === 'livenet') { this._network = 'main'; @@ -65,12 +49,8 @@ AddressService.dependencies = [ // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs, // then I would pass back [tx1, tx2] in that order -// -// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set. -// AddressService.prototype.getAddressHistory = function(addresses, options, callback) { var self = this; - var cacheUsed = false; options = options || {}; options.from = options.from || 0; @@ -85,75 +65,21 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba addresses = [addresses]; } + async.eachLimit(addresses, 4, function(address, next) { + self._getAddressTxidHistory(address, options, next); - function getTxList(next) { + }, function(err) { - - function hashAddresses(addresses) { - - // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime - // a 32 bits hash is secure enough - - return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED); - }; - - var calculatedCacheKey; - - // We use the cache ONLY on from > 0 queries. - // - // Rationale: The a full history is downloaded, the client do - // from =0, to=x - // then from =x+1 to=y - // then [...] - // The objective of this cache is to speed up the from>0 queries, and also - // "freeze" the txid list during download. - // - if (options.from >0 ) { - - let cacheKey = options.cacheKey; - if (!cacheKey) { - calculatedCacheKey = hashAddresses(addresses); - cacheKey = calculatedCacheKey; - } - - var txIdList = self._txIdListCache.get(cacheKey); - if (txIdList) { - options.txIdList = txIdList; - cacheUsed = true; - return next(); - } - } - - // Get the list from the db - async.eachLimit(addresses, 4, function(address, next) { - self._getAddressTxidHistory(address, options, next); - }, function(err) { - if (err) return next(err); - - var list = lodash.uniqBy(options.txIdList, function(x) { - return x.txid + x.height; - }); - - - options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']); - - if (list.length > TXID_LIST_CACHE_MIN) { - calculatedCacheKey = calculatedCacheKey || hashAddresses(addresses); - - self._txIdListCache.set(calculatedCacheKey, options.txIdList); - } - - return next(); - }); - - }; - - - getTxList(function(err) { if(err) { return callback(err); } + var list = lodash.uniqBy(options.txIdList, function(x) { + return x.txid + x.height; + }); + + + options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']); self._getAddressTxHistory(options, function(err, txList) { if (err) { @@ -162,11 +88,10 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba var results = { totalCount: options.txIdList.length || 0, - items: txList, + items: txList }; - // cacheUsed is returned for testing - callback(null, results, cacheUsed); + callback(null, results); }); }); diff --git a/package-lock.json b/package-lock.json index 59692c09..d8f28caa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -159,11 +159,6 @@ "lodash": "4.17.4" } }, - "async-limiter": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", - "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -243,9 +238,9 @@ "optional": true, "requires": { "debug": "2.6.8", - "engine.io": "3.1.4", + "engine.io": "3.1.0", "object-assign": "4.1.1", - "socket.io-adapter": "1.1.1", + "socket.io-adapter": "1.1.0", "socket.io-client": "2.0.1", "socket.io-parser": "3.1.2" } @@ -261,7 +256,7 @@ "component-bind": "1.0.0", "component-emitter": "1.2.1", "debug": "2.6.4", - "engine.io-client": "3.1.4", + "engine.io-client": "3.1.1", "has-cors": "1.1.0", "indexof": "0.0.1", "object-component": "0.0.3", @@ -1356,84 +1351,50 @@ } }, "engine.io": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz", - "integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz", + "integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=", "optional": true, "requires": { "accepts": "1.3.3", "base64id": "1.0.0", "cookie": "0.3.1", - "debug": "2.6.9", - "engine.io-parser": "2.1.2", + "debug": "2.6.8", + "engine.io-parser": "2.1.1", "uws": "0.14.5", - "ws": "3.3.3" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "optional": true, - "requires": { - "ms": "2.0.0" - } - } + "ws": "2.3.1" } }, "engine.io-client": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz", - "integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz", + "integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=", "optional": true, "requires": { "component-emitter": "1.2.1", "component-inherit": "0.0.3", - "debug": "2.6.9", - "engine.io-parser": "2.1.2", + "debug": "2.6.8", + "engine.io-parser": "2.1.1", "has-cors": "1.1.0", "indexof": "0.0.1", + "parsejson": "0.0.3", "parseqs": "0.0.5", "parseuri": "0.0.5", - "ws": "3.3.3", - "xmlhttprequest-ssl": "1.5.5", + "ws": "2.3.1", + "xmlhttprequest-ssl": "1.5.3", "yeast": "0.1.2" - }, - "dependencies": { - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "optional": true, - "requires": { - "ms": "2.0.0" - } - }, - "xmlhttprequest-ssl": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz", - "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=", - "optional": true - } } }, "engine.io-parser": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz", - "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz", + "integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=", "requires": { "after": "0.8.2", - "arraybuffer.slice": "0.0.7", + "arraybuffer.slice": "0.0.6", "base64-arraybuffer": "0.1.5", "blob": "0.0.4", "has-binary2": "1.0.2" - }, - "dependencies": { - "arraybuffer.slice": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz", - "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==" - } } }, "entities": { @@ -4087,10 +4048,30 @@ } }, "socket.io-adapter": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz", - "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=", - "optional": true + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz", + "integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=", + "optional": true, + "requires": { + "debug": "2.3.3" + }, + "dependencies": { + "debug": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz", + "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=", + "optional": true, + "requires": { + "ms": "0.7.2" + } + }, + "ms": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz", + "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=", + "optional": true + } + } }, "socket.io-client": { "version": "1.7.4", @@ -4502,9 +4483,9 @@ "optional": true }, "ultron": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", - "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz", + "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ=" }, "unc-path-regex": { "version": "0.1.2", @@ -4650,13 +4631,19 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz", - "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz", + "integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=", "requires": { - "async-limiter": "1.0.0", - "safe-buffer": "5.1.1", - "ultron": "1.1.1" + "safe-buffer": "5.0.1", + "ultron": "1.1.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz", + "integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c=" + } } }, "wtf-8": { @@ -4674,14 +4661,6 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, - "xxhash": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz", - "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=", - "requires": { - "nan": "2.6.2" - } - }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", diff --git a/package.json b/package.json index a7e9c069..74dfa803 100644 --- a/package.json +++ b/package.json @@ -43,13 +43,12 @@ "levelup": "^2.0.0", "liftoff": "^2.2.0", "lodash": "^4.17.4", - "lru-cache": "^4.1.1", + "lru-cache": "^4.0.2", "memwatch-next": "^0.3.0", "mkdirp": "0.5.0", "path-is-absolute": "^1.0.0", "socket.io": "^1.4.5", - "socket.io-client": "^1.4.5", - "xxhash": "^0.2.4" + "socket.io-client": "^1.4.5" }, "devDependencies": { "chai": "^3.5.0", diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index afa7744b..5ded3800 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -172,125 +172,7 @@ describe('Address Service', function() { }); - describe('TxIdList cache', function() { - var list, old_getAddressTxidHistory, old_getAddressTxHistory; - beforeEach(function(done){ - this.clock = sinon.useFakeTimers(); - list = []; - for(let i=1000; i>0; i--) { - list.push({ - txid: "txid" + i, - height: 1000 + i, - - }); - }; - old_getAddressTxidHistory = addressService._getAddressTxidHistory; - // Note that this stub DOES NOT respect options.from/to as the real function - addressService._getAddressTxidHistory = function(addr, options, cb) { - options.txIdList = lodash.clone(list); - return cb(); - }; - old_getAddressTxHistory = addressService._getAddressTxHistory; - - addressService._getAddressTxHistory = function(options, cb) { - return cb(null, options.txIdList); - }; - - addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - expect(cacheUsed).equal(false); - done(); - }); - }); - - afterEach(function(done){ - this.clock.restore(); - - addressService._getAddressTxidHistory = old_getAddressTxHistory; - addressService._getAddressTxHistory = old_getAddressTxHistory; - done(); - }); - - - it('should not cache the address txlist history when from =0 ', function(done) { - addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - expect(cacheUsed).equal(false); - done(); - }); - }); - - it('should cache the address txlist history', function(done) { - addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(cacheUsed).equal(true); - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - done(); - }); - }); - - - it('should retrive cached list using cachekey', function(done) { - addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(cacheUsed).equal(true); - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - done(); - }); - }); - - - - it('should expire cache', function(done) { - this.clock.tick(35*1000); - addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(cacheUsed).equal(false); - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - done(); - }); - }); - - it('should cache using the address as key', function(done) { - addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(cacheUsed).equal(false); - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) { - if (err) { - return done(err); - } - expect(cacheUsed).equal(true); - expect(res.totalCount).equal(1000); - expect(res.items,'txid').to.be.deep.equal(list); - done(); - }); - }); - }); - - - }); }); describe('#_getAddressTxidHistory', function() {