RanchiMall/flocore-node
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #555 from bitpay/revert-553-revert-548-opt/txlist-cache

Browse Source 
add txIdList cache
This commit is contained in:
Justin Langston 2018-04-12 14:57:54 -04:00 committed by GitHub
commit e65689ab5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 288 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
lib/services/address/index.js
View File

@ -13,6 +13,17 @@ var Encoding = require('./encoding');
var Transform = require('stream').Transform;
var assert = require('assert');
var utils = require('../../utils');
var LRU = require('lru-cache');
var XXHash = require('xxhash');
// See rationale about this cache at function getTxList(next)
const TXID_LIST_CACHE_ITEMS = 250; // nr of items (this translates to: consecutive
// clients downloading their tx history)
const TXID_LIST_CACHE_EXPIRATION = 1000 * 30; // ms
const TXID_LIST_CACHE_MIN = 100; // Min items to cache
const TXID_LIST_CACHE_SEED = 0x3233DE; // Min items to cache
var AddressService = function(options) {
@ -24,6 +35,11 @@ var AddressService = function(options) {
this._network = this.node.network;
this._db = this.node.services.db;
this._mempool = this.node.services.mempool;
this._txIdListCache = new LRU({
max: TXID_LIST_CACHE_ITEMS,
maxAge: TXID_LIST_CACHE_EXPIRATION
});
if (this._network === 'livenet') {
this._network = 'main';
@ -49,8 +65,12 @@ AddressService.dependencies = [
// for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
// in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
// then I would pass back [tx1, tx2] in that order
//
// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
//
AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
var self = this;
var cacheUsed = false;
options = options || {};
options.from = options.from || 0;
@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
addresses = [addresses];
}
async.eachLimit(addresses, 4, function(address, next) {
self._getAddressTxidHistory(address, options, next);
}, function(err) {
function getTxList(next) {
function hashAddresses(addresses) {
// Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
// a 32 bits hash is secure enough
return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
};
var calculatedCacheKey;
// We use the cache ONLY on from > 0 queries.
//
// Rationale: The a full history is downloaded, the client do
// from =0, to=x
// then from =x+1 to=y
// then [...]
// The objective of this cache is to speed up the from>0 queries, and also
// "freeze" the txid list during download.
//
if (options.from >0 ) {
let cacheKey = options.cacheKey;
if (!cacheKey) {
calculatedCacheKey = hashAddresses(addresses);
cacheKey = calculatedCacheKey;
}
var txIdList = self._txIdListCache.get(cacheKey);
if (txIdList) {
options.txIdList = txIdList;
cacheUsed = true;
return next();
}
}
// Get the list from the db
async.eachLimit(addresses, 4, function(address, next) {
self._getAddressTxidHistory(address, options, next);
}, function(err) {
if (err) return next(err);
var list = lodash.uniqBy(options.txIdList, function(x) {
return x.txid + x.height;
});
options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
if (list.length > TXID_LIST_CACHE_MIN) {
calculatedCacheKey = calculatedCacheKey || hashAddresses(addresses);
self._txIdListCache.set(calculatedCacheKey, options.txIdList);
}
return next();
});
};
getTxList(function(err) {
if(err) {
return callback(err);
}
var list = lodash.uniqBy(options.txIdList, function(x) {
return x.txid + x.height;
});
options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
self._getAddressTxHistory(options, function(err, txList) {
if (err) {
@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
var results = {
totalCount: options.txIdList.length || 0,
items: txList
items: txList,
};
callback(null, results);
// cacheUsed is returned for testing
callback(null, results, cacheUsed);
});
});

141
package-lock.json generated
View File

@ -159,6 +159,11 @@
"lodash": "4.17.4"
}
},
"async-limiter": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
"integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg=="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@ -238,9 +243,9 @@
"optional": true,
"requires": {
"debug": "2.6.8",
"engine.io": "3.1.0",
"engine.io": "3.1.4",
"object-assign": "4.1.1",
"socket.io-adapter": "1.1.0",
"socket.io-adapter": "1.1.1",
"socket.io-client": "2.0.1",
"socket.io-parser": "3.1.2"
}
@ -256,7 +261,7 @@
"component-bind": "1.0.0",
"component-emitter": "1.2.1",
"debug": "2.6.4",
"engine.io-client": "3.1.1",
"engine.io-client": "3.1.4",
"has-cors": "1.1.0",
"indexof": "0.0.1",
"object-component": "0.0.3",
@ -1351,50 +1356,84 @@
}
},
"engine.io": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.0.tgz",
"integrity": "sha1-XKQ4486f28kVxKIcjdnhJmcG5X4=",
"version": "3.1.4",
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.4.tgz",
"integrity": "sha1-PQIRtwpVLOhB/8fahiezAamkFi4=",
"optional": true,
"requires": {
"accepts": "1.3.3",
"base64id": "1.0.0",
"cookie": "0.3.1",
"debug": "2.6.8",
"engine.io-parser": "2.1.1",
"debug": "2.6.9",
"engine.io-parser": "2.1.2",
"uws": "0.14.5",
"ws": "2.3.1"
"ws": "3.3.3"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"optional": true,
"requires": {
"ms": "2.0.0"
}
}
}
},
"engine.io-client": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.1.tgz",
"integrity": "sha1-QVqYUrrbFPoAj6PvHjFgjbZ2EyU=",
"version": "3.1.4",
"resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.4.tgz",
"integrity": "sha1-T88TcLRxY70s6b4nM5ckMDUNTqE=",
"optional": true,
"requires": {
"component-emitter": "1.2.1",
"component-inherit": "0.0.3",
"debug": "2.6.8",
"engine.io-parser": "2.1.1",
"debug": "2.6.9",
"engine.io-parser": "2.1.2",
"has-cors": "1.1.0",
"indexof": "0.0.1",
"parsejson": "0.0.3",
"parseqs": "0.0.5",
"parseuri": "0.0.5",
"ws": "2.3.1",
"xmlhttprequest-ssl": "1.5.3",
"ws": "3.3.3",
"xmlhttprequest-ssl": "1.5.5",
"yeast": "0.1.2"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"optional": true,
"requires": {
"ms": "2.0.0"
}
},
"xmlhttprequest-ssl": {
"version": "1.5.5",
"resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
"integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
"optional": true
}
}
},
"engine.io-parser": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.1.tgz",
"integrity": "sha1-4Ps/DgRi9/WLt3waUun1p+JuRmg=",
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
"integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
"requires": {
"after": "0.8.2",
"arraybuffer.slice": "0.0.6",
"arraybuffer.slice": "0.0.7",
"base64-arraybuffer": "0.1.5",
"blob": "0.0.4",
"has-binary2": "1.0.2"
},
"dependencies": {
"arraybuffer.slice": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
"integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
}
}
},
"entities": {
@ -4048,30 +4087,10 @@
}
},
"socket.io-adapter": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.0.tgz",
"integrity": "sha1-x6pGUB3VVsLLiiivj/lcC14dqkw=",
"optional": true,
"requires": {
"debug": "2.3.3"
},
"dependencies": {
"debug": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
"integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
"optional": true,
"requires": {
"ms": "0.7.2"
}
},
"ms": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
"integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
"optional": true
}
}
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
"integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
"optional": true
},
"socket.io-client": {
"version": "1.7.4",
@ -4483,9 +4502,9 @@
"optional": true
},
"ultron": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz",
"integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ="
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
"integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og=="
},
"unc-path-regex": {
"version": "0.1.2",
@ -4631,19 +4650,13 @@
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
},
"ws": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-2.3.1.tgz",
"integrity": "sha1-a5Sz5EfLajY/eF6vlK9jWejoHIA=",
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
"integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
"requires": {
"safe-buffer": "5.0.1",
"ultron": "1.1.0"
},
"dependencies": {
"safe-buffer": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz",
"integrity": "sha1-0mPKVGls2KMGtcplUekt5XkY++c="
}
"async-limiter": "1.0.0",
"safe-buffer": "5.1.1",
"ultron": "1.1.1"
}
},
"wtf-8": {
@ -4661,6 +4674,14 @@
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
"integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
},
"xxhash": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz",
"integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=",
"requires": {
"nan": "2.6.2"
}
},
"yallist": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",

5
package.json
View File

@ -43,12 +43,13 @@
"levelup": "^2.0.0",
"liftoff": "^2.2.0",
"lodash": "^4.17.4",
"lru-cache": "^4.0.2",
"lru-cache": "^4.1.1",
"memwatch-next": "^0.3.0",
"mkdirp": "0.5.0",
"path-is-absolute": "^1.0.0",
"socket.io": "^1.4.5",
"socket.io-client": "^1.4.5"
"socket.io-client": "^1.4.5",
"xxhash": "^0.2.4"
},
"devDependencies": {
"chai": "^3.5.0",

118
test/services/address/index.unit.js
View File

@ -172,7 +172,125 @@ describe('Address Service', function() {
});
describe('TxIdList cache', function() {
var list, old_getAddressTxidHistory, old_getAddressTxHistory;
beforeEach(function(done){
this.clock = sinon.useFakeTimers();
list = [];
for(let i=1000; i>0; i--) {
list.push({
txid: "txid" + i,
height: 1000 + i,
});
};
old_getAddressTxidHistory = addressService._getAddressTxidHistory;
// Note that this stub DOES NOT respect options.from/to as the real function
addressService._getAddressTxidHistory = function(addr, options, cb) {
options.txIdList = lodash.clone(list);
return cb();
};
old_getAddressTxHistory = addressService._getAddressTxHistory;
addressService._getAddressTxHistory = function(options, cb) {
return cb(null, options.txIdList);
};
addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
expect(cacheUsed).equal(false);
done();
});
});
afterEach(function(done){
this.clock.restore();
addressService._getAddressTxidHistory = old_getAddressTxHistory;
addressService._getAddressTxHistory = old_getAddressTxHistory;
done();
});
it('should not cache the address txlist history when from =0 ', function(done) {
addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
expect(cacheUsed).equal(false);
done();
});
});
it('should cache the address txlist history', function(done) {
addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(cacheUsed).equal(true);
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
done();
});
});
it('should retrive cached list using cachekey', function(done) {
addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(cacheUsed).equal(true);
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
done();
});
});
it('should expire cache', function(done) {
this.clock.tick(35*1000);
addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(cacheUsed).equal(false);
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
done();
});
});
it('should cache using the address as key', function(done) {
addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(cacheUsed).equal(false);
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) {
if (err) {
return done(err);
}
expect(cacheUsed).equal(true);
expect(res.totalCount).equal(1000);
expect(res.items,'txid').to.be.deep.equal(list);
done();
});
});
});
});
});
describe('#_getAddressTxidHistory', function() {