diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 849be206..9bc6371a 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -13,6 +13,17 @@ var Encoding = require('./encoding'); var Transform = require('stream').Transform; var assert = require('assert'); var utils = require('../../utils'); +var LRU = require('lru-cache'); +var XXHash = require('xxhash'); + + + +// See rationale about this cache at function getTxList(next) +const TXID_LIST_CACHE_ITEMS = 250; // nr of items (this translates to: consecutive + // clients downloading their tx history) +const TXID_LIST_CACHE_EXPIRATION = 1000 * 30; // ms +const TXID_LIST_CACHE_MIN = 100; // Min items to cache +const TXID_LIST_CACHE_SEED = 0x3233DE; // Min items to cache var AddressService = function(options) { @@ -24,6 +35,11 @@ var AddressService = function(options) { this._network = this.node.network; this._db = this.node.services.db; this._mempool = this.node.services.mempool; + this._txIdListCache = new LRU({ + max: TXID_LIST_CACHE_ITEMS, + maxAge: TXID_LIST_CACHE_EXPIRATION + }); + if (this._network === 'livenet') { this._network = 'main'; @@ -49,8 +65,12 @@ AddressService.dependencies = [ // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs, // then I would pass back [tx1, tx2] in that order +// +// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set. +// AddressService.prototype.getAddressHistory = function(addresses, options, callback) { var self = this; + var cacheUsed = false; options = options || {}; options.from = options.from || 0; @@ -65,31 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba addresses = [addresses]; } - async.eachLimit(addresses, 4, function(address, next) { - self._getAddressTxidHistory(address, options, next); + function getTxList(next) { - }, function(err) { + function hashAddresses(addresses) { + + // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime + // a 32 bits hash is secure enough + + return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED); + }; + + var calculatedCacheKey; + + // We use the cache ONLY on from > 0 queries. + // + // Rationale: The a full history is downloaded, the client do + // from =0, to=x + // then from =x+1 to=y + // then [...] + // The objective of this cache is to speed up the from>0 queries, and also + // "freeze" the txid list during download. + // + if (options.from >0 ) { + + let cacheKey = options.cacheKey; + if (!cacheKey) { + calculatedCacheKey = hashAddresses(addresses); + cacheKey = calculatedCacheKey; + } + + var txIdList = self._txIdListCache.get(cacheKey); + if (txIdList) { + options.txIdList = txIdList; + cacheUsed = true; + return next(); + } + } + + // Get the list from the db + async.eachLimit(addresses, 4, function(address, next) { + self._getAddressTxidHistory(address, options, next); + }, function(err) { + if (err) return next(err); + + var list = lodash.uniqBy(options.txIdList, function(x) { + return x.txid + x.height; + }); + + + options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']); + + if (list.length > TXID_LIST_CACHE_MIN) { + calculatedCacheKey = calculatedCacheKey || hashAddresses(addresses); + + self._txIdListCache.set(calculatedCacheKey, options.txIdList); + } + + return next(); + }); + + }; + + + getTxList(function(err) { if(err) { return callback(err); } - var unique = {}; - var list = []; - - for (let i = 0; i < options.txIdList.length; i++) { - unique[options.txIdList[i].txid + options.txIdList[i].height] = options.txIdList[i]; - } - - for (var prop in unique) { - list.push(unique[prop]); - } - - options.txIdList = list.sort(function(a, b) { - return b.height - a.height; - }); - self._getAddressTxHistory(options, function(err, txList) { if (err) { @@ -98,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba var results = { totalCount: options.txIdList.length || 0, - items: txList + items: txList, }; - callback(null, results); + // cacheUsed is returned for testing + callback(null, results, cacheUsed); }); }); @@ -427,7 +492,6 @@ AddressService.prototype._getAddressTxHistory = function(options, callback) { }; AddressService.prototype._getAddressTxidHistory = function(address, options, callback) { - var self = this; options = options || {}; diff --git a/lib/services/p2p/index.js b/lib/services/p2p/index.js index f7f32a4d..475b8cfb 100644 --- a/lib/services/p2p/index.js +++ b/lib/services/p2p/index.js @@ -10,6 +10,7 @@ var assert = require('assert'); var Bcoin = require('./bcoin'); var BcoinTx = require('fcoin').tx; var Networks = require('flocore-lib').Networks; +var BitcoreRPC = require('bitcoind-rpc'); var LRU = require('lru-cache'); var P2P = function(options) { @@ -21,6 +22,7 @@ var P2P = function(options) { BaseService.call(this, options); this._options = options; + this._initRPC(options); this._initP2P(); this._initPubSub(); this._bcoin = null; @@ -128,26 +130,7 @@ P2P.prototype.getPublishEvents = function() { P2P.prototype.sendTransaction = function(tx, callback) { - var peer = this._getPeer(); - - var bcoinTx; - try { - bcoinTx = BcoinTx.fromRaw(tx, 'hex'); - } catch(e) { - return callback(e); - } - - log.info('P2P Service: sending transaction: ' + bcoinTx.txid()); - - this._outgoingTxs.set(bcoinTx.txid(), bcoinTx); - var inv = p2p.Inventory.forTransaction(bcoinTx.txid()); - var txMessage = this.messages.Inventory([inv]); - - peer.sendMessage(txMessage); - - this._onPeerTx(peer, { transaction: bcoinTx }); - - return callback(null, bcoinTx.txid()); + return this._client.sendRawTransaction(tx, callback); }; @@ -269,6 +252,23 @@ P2P.prototype._initCache = function() { this._inv = LRU(1000); }; +P2P.prototype._initRPC = function (options) { + var port = 7312 + + if (this.node.network === 'testnet') + port = 17312 + + this._config = options.rpc || { + user: 'flocore', + pass: 'flocorepassw123', + host: 'localhost', + protocol: 'http', + port: port + }; + + this._client = new BitcoreRPC(this._config); +} + P2P.prototype._initP2P = function() { this._maxPeers = this._options.maxPeers || 60; this._minPeers = this._options.minPeers || 0; diff --git a/package-lock.json b/package-lock.json index 10fca47c..6750b9b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "flocore-node", - "version": "5.0.0-beta.68", + "version": "5.0.0-beta.69", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -217,8 +217,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base": { "version": "0.11.2", @@ -335,6 +334,11 @@ "safe-buffer": "5.1.1" } }, + "bitcoind-rpc": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/bitcoind-rpc/-/bitcoind-rpc-0.7.2.tgz", + "integrity": "sha512-3alq5pH4/mAdEscucew98ls0s7BEaDvKTuYW34nSkTsmTC7G9R7Xh+ABXOxeVBfG/+rKfyQLmFAm2aRq92tJqw==" + }, "bl": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", @@ -349,6 +353,11 @@ "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz", "integrity": "sha1-vPEwUspURj8w+fx+lbmkdjCpSSE=" }, + "bloom-filter": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/bloom-filter/-/bloom-filter-0.2.0.tgz", + "integrity": "sha1-hNY7v5Fy2DA+ZMH/FuudvzOpgaM=" + }, "bn.js": { "version": "4.11.8", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz", @@ -375,7 +384,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "1.0.0", "concat-map": "0.0.1" @@ -458,6 +466,9 @@ "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", "optional": true }, + "buffers": { + "version": "github:bitpay/node-buffers#04f4c4264e0d105db2b99b786843ed64f23230d8" + }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -566,6 +577,31 @@ } } }, + "cli": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/cli/-/cli-0.4.5.tgz", + "integrity": "sha1-ePlIXNFhtWbppsctcXDEJw6B22E=", + "requires": { + "glob": "7.1.2" + } + }, + "cliff": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/cliff/-/cliff-0.1.10.tgz", + "integrity": "sha1-U74z6p9ZvshWCe4wCsQgdgPlIBM=", + "requires": { + "colors": "1.0.3", + "eyes": "0.1.8", + "winston": "0.8.3" + }, + "dependencies": { + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + } + } + }, "cliui": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", @@ -678,8 +714,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "console-browserify": { "version": "1.1.0", @@ -903,6 +938,11 @@ "sha.js": "2.4.11" } }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, "dashdash": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", @@ -1560,6 +1600,11 @@ "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, "fast-deep-equal": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", @@ -1581,6 +1626,83 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fcoin": { + "version": "1.0.0-beta.20", + "resolved": "https://registry.npmjs.org/fcoin/-/fcoin-1.0.0-beta.20.tgz", + "integrity": "sha512-U448wU+tKx+GwBtmdckQRx1CsMMOLlBkAozhGPuej07ZPrKE1qicHvLSMq4qD4JHmVAVcVLxb40qhAFoorKcKw==", + "requires": { + "bcoin-native": "0.0.23", + "bn.js": "4.11.8", + "elliptic": "6.4.0", + "leveldown": "1.7.2", + "n64": "0.0.18", + "secp256k1": "3.3.0", + "socket.io": "2.0.3", + "socket.io-client": "2.0.3" + }, + "dependencies": { + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=", + "optional": true + }, + "leveldown": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.7.2.tgz", + "integrity": "sha1-XjRnuyfuJGpKe429j7KxYgam64s=", + "optional": true, + "requires": { + "abstract-leveldown": "2.6.3", + "bindings": "1.2.1", + "fast-future": "1.0.2", + "nan": "2.6.2", + "prebuild-install": "2.5.3" + } + }, + "nan": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.6.2.tgz", + "integrity": "sha1-5P805slf37WuzAjeZZb0NgWn20U=", + "optional": true + }, + "socket.io": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.0.3.tgz", + "integrity": "sha1-Q1nwaiSTOua9CHeYr3jGgOrjReM=", + "optional": true, + "requires": { + "debug": "2.6.9", + "engine.io": "3.1.5", + "object-assign": "4.1.1", + "socket.io-adapter": "1.1.1", + "socket.io-client": "2.0.3", + "socket.io-parser": "3.1.3" + } + }, + "socket.io-client": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.0.3.tgz", + "integrity": "sha1-bK9K/5+FsZ/ZG2zhPWmttWT4hzs=", + "optional": true, + "requires": { + "backo2": "1.0.2", + "base64-arraybuffer": "0.1.5", + "component-bind": "1.0.0", + "component-emitter": "1.2.1", + "debug": "2.6.9", + "engine.io-client": "3.1.6", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "object-component": "0.0.3", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "socket.io-parser": "3.1.3", + "to-array": "0.1.4" + } + } + } + }, "fill-keys": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/fill-keys/-/fill-keys-1.0.2.tgz", @@ -1662,9 +1784,9 @@ "integrity": "sha1-Tnmumy6zi/hrO7Vr8+ClaqX8q9c=" }, "flocore-lib": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/flocore-lib/-/flocore-lib-0.15.0.tgz", - "integrity": "sha512-1n8AN/aqjPPVf4xlNPfEZhQjkCUHWEGJ0nBahVvifOVkamLEAJWA3rpkgqhmYaW5p0YnUJ3lCBGjkRzu55AvYA==", + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/flocore-lib/-/flocore-lib-0.15.1.tgz", + "integrity": "sha512-HKPP7EnwHibfp0LQpOkwNcXhg+QmN301oVPnael+jb3qzu7AZQnLrqW1J/Q1yiWuYJcbwNHsbpgAG9bLcByF6w==", "requires": { "bn.js": "2.0.4", "bs58": "2.0.0", @@ -1815,6 +1937,95 @@ } } }, + "flocore-p2p": { + "version": "5.0.0-beta.6", + "resolved": "https://registry.npmjs.org/flocore-p2p/-/flocore-p2p-5.0.0-beta.6.tgz", + "integrity": "sha512-F++fED/rR6cLY/r1qOkVAv4T3IytvxiNJs7rgxDbIP6RgK7DNYn77eJCTbiFf/SZ7n7JEiNJcpgAim4kBnbC5A==", + "requires": { + "bloom-filter": "0.2.0", + "buffers": "github:bitpay/node-buffers#04f4c4264e0d105db2b99b786843ed64f23230d8", + "fcoin": "1.0.0-beta.18", + "flocore-lib": "0.15.1", + "socks5-client": "0.3.6" + }, + "dependencies": { + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=", + "optional": true + }, + "fcoin": { + "version": "1.0.0-beta.18", + "resolved": "https://registry.npmjs.org/fcoin/-/fcoin-1.0.0-beta.18.tgz", + "integrity": "sha512-Kk2SN8xILi6YnyOzf9MqJUsE1KoG/fhSfh43m34OhVJAi4o/1AP7VS6MJ10otgfGsiEj+CQ4pp+CfEG7DUG4xw==", + "requires": { + "bcoin-native": "0.0.23", + "bn.js": "4.11.8", + "elliptic": "6.4.0", + "leveldown": "1.7.2", + "n64": "0.0.18", + "secp256k1": "3.3.0", + "socket.io": "2.0.3", + "socket.io-client": "2.0.3" + } + }, + "leveldown": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-1.7.2.tgz", + "integrity": "sha1-XjRnuyfuJGpKe429j7KxYgam64s=", + "optional": true, + "requires": { + "abstract-leveldown": "2.6.3", + "bindings": "1.2.1", + "fast-future": "1.0.2", + "nan": "2.6.2", + "prebuild-install": "2.5.3" + } + }, + "nan": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.6.2.tgz", + "integrity": "sha1-5P805slf37WuzAjeZZb0NgWn20U=", + "optional": true + }, + "socket.io": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.0.3.tgz", + "integrity": "sha1-Q1nwaiSTOua9CHeYr3jGgOrjReM=", + "optional": true, + "requires": { + "debug": "2.6.9", + "engine.io": "3.1.5", + "object-assign": "4.1.1", + "socket.io-adapter": "1.1.1", + "socket.io-client": "2.0.3", + "socket.io-parser": "3.1.3" + } + }, + "socket.io-client": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.0.3.tgz", + "integrity": "sha1-bK9K/5+FsZ/ZG2zhPWmttWT4hzs=", + "optional": true, + "requires": { + "backo2": "1.0.2", + "base64-arraybuffer": "0.1.5", + "component-bind": "1.0.0", + "component-emitter": "1.2.1", + "debug": "2.6.9", + "engine.io-client": "3.1.6", + "has-cors": "1.1.0", + "indexof": "0.0.1", + "object-component": "0.0.3", + "parseqs": "0.0.5", + "parseuri": "0.0.5", + "socket.io-parser": "3.1.3", + "to-array": "0.1.4" + } + } + } + }, "florincoind-rpc": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/florincoind-rpc/-/florincoind-rpc-0.7.1.tgz", @@ -2060,8 +2271,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "gauge": { "version": "2.7.4", @@ -2115,7 +2325,6 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", - "dev": true, "requires": { "fs.realpath": "1.0.0", "inflight": "1.0.6", @@ -2399,7 +2608,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "1.4.0", "wrappy": "1.0.2" @@ -2420,6 +2628,16 @@ "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz", "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs=" }, + "ipv6": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/ipv6/-/ipv6-3.1.3.tgz", + "integrity": "sha1-TZBk+cLa+g3RC4t9dv/KSq0xs7k=", + "requires": { + "cli": "0.4.5", + "cliff": "0.1.10", + "sprintf": "0.1.5" + } + }, "irregular-plurals": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-1.4.0.tgz", @@ -3127,7 +3345,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "1.1.11" } @@ -3317,6 +3534,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" }, + "network-byte-order": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/network-byte-order/-/network-byte-order-0.2.0.tgz", + "integrity": "sha1-asEb9Ev2ENrt2+kKCaXIF8bg0rM=" + }, "node-abi": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.4.1.tgz", @@ -3592,6 +3814,11 @@ "pinkie": "2.0.4" } }, + "pkginfo": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.3.1.tgz", + "integrity": "sha1-Wyn2qB9wcXFC4J52W76rl7T4HiE=" + }, "plur": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/plur/-/plur-2.1.2.tgz", @@ -4430,6 +4657,15 @@ } } }, + "socks5-client": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/socks5-client/-/socks5-client-0.3.6.tgz", + "integrity": "sha1-QgW1eR8t93zwdSciJVj+TkasovE=", + "requires": { + "ipv6": "3.1.3", + "network-byte-order": "0.2.0" + } + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -4460,6 +4696,11 @@ "extend-shallow": "3.0.2" } }, + "sprintf": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/sprintf/-/sprintf-0.1.5.tgz", + "integrity": "sha1-j4PjmpMXwaUCy324BQ5Rxnn27c8=" + }, "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -4481,6 +4722,11 @@ "tweetnacl": "0.14.5" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "static-extend": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", @@ -4907,6 +5153,32 @@ "dev": true, "optional": true }, + "winston": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/winston/-/winston-0.8.3.tgz", + "integrity": "sha1-ZLar9M0Brcrv1QCTk7HY6L7BnbA=", + "requires": { + "async": "0.2.10", + "colors": "0.6.2", + "cycle": "1.0.3", + "eyes": "0.1.8", + "isstream": "0.1.2", + "pkginfo": "0.3.1", + "stack-trace": "0.0.10" + }, + "dependencies": { + "async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=" + }, + "colors": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.6.2.tgz", + "integrity": "sha1-JCP+ZnisDF2uiFLl0OW+CMmXq8w=" + } + } + }, "wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", @@ -4944,6 +5216,14 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" }, + "xxhash": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/xxhash/-/xxhash-0.2.4.tgz", + "integrity": "sha1-i4pIFiz8zCG5IPpQAmEYfUAhbDk=", + "requires": { + "nan": "2.10.0" + } + }, "yallist": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", diff --git a/package.json b/package.json index d9f82408..d57b120b 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ ], "dependencies": { "async": "^2.5.0", + "bitcoind-rpc": "^0.7.2", "bn.js": "^4.11.8", "body-parser": "^1.13.3", "colors": "^1.1.2", @@ -45,12 +46,13 @@ "levelup": "^2.0.0", "liftoff": "^2.2.0", "lodash": "^4.17.4", - "lru-cache": "^4.0.2", + "lru-cache": "^4.1.1", "memwatch-next": "^0.3.0", "mkdirp": "0.5.0", "path-is-absolute": "^1.0.0", "socket.io": "^1.4.5", - "socket.io-client": "^1.4.5" + "socket.io-client": "^1.4.5", + "xxhash": "^0.2.4" }, "devDependencies": { "chai": "^3.5.0", diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 1fd2ddc1..afa7744b 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -8,6 +8,7 @@ var Encoding = require('../../../lib/services/address/encoding'); var Readable = require('stream').Readable; var EventEmitter = require('events').EventEmitter; var bcoin = require('bcoin'); +var lodash = require('lodash'); describe('Address Service', function() { @@ -54,7 +55,7 @@ describe('Address Service', function() { describe('#getAddressHistory', function() { - it('should get the address history', function(done) { + it('should get the address history (null case)', function(done) { sandbox.stub(addressService, '_getAddressTxidHistory').callsArgWith(2, null, null); sandbox.stub(addressService, '_getAddressTxHistory').callsArgWith(1, null, []); @@ -74,6 +75,222 @@ describe('Address Service', function() { }); }); + it('should get the sorted address history', function(done) { + + var old_getAddressTxidHistory = addressService._getAddressTxidHistory; + addressService._getAddressTxidHistory = function(addr, options, cb) { + options.txIdList = [ + { + txid: "d", + height: 10, + }, + { + txid: "c", + height: 10, + }, + { + txid: "a", + height: 101, + }, + { + txid: "b", + height: 100, + }, + ]; + return cb(); + }; + + + var old_getAddressTxHistory = addressService._getAddressTxHistory; + addressService._getAddressTxHistory = function(options, cb) { + return cb(null, options.txIdList); + }; + + addressService.getAddressHistory(['a', 'b', 'c'], { from: 12, to: 14 }, function(err, res) { + + if (err) { + return done(err); + } + + expect(res.totalCount).equal(4); + expect(lodash.map(res.items,'txid')).to.be.deep.equal(['a','b','c','d']); + + addressService._getAddressTxidHistory = old_getAddressTxHistory; + addressService._getAddressTxHistory = old_getAddressTxHistory; + done(); + }); + }); + + it('should remove duplicated items in history', function(done) { + + var old_getAddressTxidHistory = addressService._getAddressTxidHistory; + addressService._getAddressTxidHistory = function(addr, options, cb) { + options.txIdList = [ + { + txid: "b", + height: 10, + }, + { + txid: "b", + height: 10, + }, + { + txid: "d", + height: 101, + }, + { + txid: "c", + height: 100, + }, + { + txid: "d", + height: 101, + }, + ]; + return cb(); + }; + + + var old_getAddressTxHistory = addressService._getAddressTxHistory; + addressService._getAddressTxHistory = function(options, cb) { + return cb(null, options.txIdList); + }; + + addressService.getAddressHistory(['a', 'b', 'c'], { from: 12, to: 14 }, function(err, res) { + + if (err) { + return done(err); + } + + expect(res.totalCount).equal(3); + expect(lodash.map(res.items,'txid')).to.be.deep.equal(['d','c','b']); + + addressService._getAddressTxidHistory = old_getAddressTxHistory; + addressService._getAddressTxHistory = old_getAddressTxHistory; + done(); + }); + }); + + + describe('TxIdList cache', function() { + var list, old_getAddressTxidHistory, old_getAddressTxHistory; + + beforeEach(function(done){ + this.clock = sinon.useFakeTimers(); + list = []; + for(let i=1000; i>0; i--) { + list.push({ + txid: "txid" + i, + height: 1000 + i, + + }); + }; + old_getAddressTxidHistory = addressService._getAddressTxidHistory; + // Note that this stub DOES NOT respect options.from/to as the real function + addressService._getAddressTxidHistory = function(addr, options, cb) { + options.txIdList = lodash.clone(list); + return cb(); + }; + old_getAddressTxHistory = addressService._getAddressTxHistory; + + addressService._getAddressTxHistory = function(options, cb) { + return cb(null, options.txIdList); + }; + + addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + expect(cacheUsed).equal(false); + done(); + }); + }); + + afterEach(function(done){ + this.clock.restore(); + + addressService._getAddressTxidHistory = old_getAddressTxHistory; + addressService._getAddressTxHistory = old_getAddressTxHistory; + done(); + }); + + + it('should not cache the address txlist history when from =0 ', function(done) { + addressService.getAddressHistory(['a', 'b', 'c'], { from: 0, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + expect(cacheUsed).equal(false); + done(); + }); + }); + + it('should cache the address txlist history', function(done) { + addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(cacheUsed).equal(true); + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + done(); + }); + }); + + + it('should retrive cached list using cachekey', function(done) { + addressService.getAddressHistory([], { from: 1, to: 10, cacheKey: 977282097 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(cacheUsed).equal(true); + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + done(); + }); + }); + + + + it('should expire cache', function(done) { + this.clock.tick(35*1000); + addressService.getAddressHistory(['a', 'b', 'c'], { from: 1, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(cacheUsed).equal(false); + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + done(); + }); + }); + + it('should cache using the address as key', function(done) { + addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(cacheUsed).equal(false); + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + addressService.getAddressHistory(['a', 'b', 'c', 'd'], { from: 1, to: 10 }, function(err, res, cacheUsed) { + if (err) { + return done(err); + } + expect(cacheUsed).equal(true); + expect(res.totalCount).equal(1000); + expect(res.items,'txid').to.be.deep.equal(list); + done(); + }); + }); + }); + + + }); }); describe('#_getAddressTxidHistory', function() {