From 74d3c0212ba3d1d04d912470d7ad099d5a855894 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Sun, 5 Nov 2017 18:14:19 -0500 Subject: [PATCH] Improved getAddressHistory. --- lib/services/address/index.js | 31 +++++---- lib/services/block/index.js | 6 ++ lib/services/db/index.js | 17 +++-- lib/services/transaction/index.js | 2 +- package-lock.json | 111 +++++++++++------------------- package.json | 2 +- 6 files changed, 79 insertions(+), 90 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 263f8c8b..7b91e6da 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -8,6 +8,7 @@ var log = index.log; var bitcore = require('bitcore-lib'); var Unit = bitcore.Unit; var _ = bitcore.deps._; +var lodash = require('lodash'); var Encoding = require('./encoding'); var Transform = require('stream').Transform; var assert = require('assert'); @@ -82,7 +83,6 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba return callback(err); } - txList = utils.dedupByTxid(txList); txList = utils.orderByConfirmations(txList); var results = { @@ -441,16 +441,11 @@ AddressService.prototype._getAddressTxHistory = function(options, callback) { var self = this; - // sort the txids by height ascending - var ids = _.sortBy(options.txIdList, function(item) { - return item.height; - }); - // slice the txids based on pagination needs - ids = ids.slice(options.from, options.to); + var ids = options.txIdList.slice(options.from, options.to); // go and get the actual txs - async.mapLimit(ids, function(id, next) { + async.mapLimit(ids, 4, function(id, next) { if (id.height === -1) { return self._mempool.getMempoolTransaction(id.txid, function(err, tx) { @@ -528,7 +523,7 @@ AddressService.prototype._getAddressTxidHistory = function(address, options, cal txIdTransformStream._transform = function(chunk, enc, callback) { var txInfo = self._encoding.decodeAddressIndexKey(chunk); - options.txIdList.push({ txid: txInfo.txid, height: txInfo.height }); + self._pushTxInfo(txInfo, options); callback(); }; @@ -540,6 +535,16 @@ AddressService.prototype._getAddressTxidHistory = function(address, options, cal }; +AddressService.prototype._pushTxInfo = function(info, options) { + // look back to see if there are dupes of this record + // we can do this because the addresses stream out in order + if (options.txIdList.length > 0 && + info.txid === options.txIdList[options.txIdList.length - 1].txid) { + return; + } + options.txIdList.push({ txid: info.txid, height: info.height }); +}; + AddressService.prototype._removeBlock = function(block, callback) { var self = this; @@ -685,7 +690,7 @@ AddressService.prototype.onReorg = function(args, callback) { return callback(err); } - var operations = _.compact(_.flattenDeep(ops)); + var operations = lodash.compact(lodash.flattenDeep(ops)); callback(null, operations); }); @@ -706,7 +711,7 @@ AddressService.prototype.onBlock = function(block, callback) { operations.push(ops); } - operations = _.flattenDeep(operations); + operations = lodash.flattenDeep(operations); callback(null, operations); }; @@ -800,7 +805,7 @@ AddressService.prototype._processTransaction = function(tx, opts) { return self._processOutput(tx, output, index, _opts); }); - outputOperations = _.compact(_.flattenDeep(outputOperations)); + outputOperations = lodash.compact(lodash.flattenDeep(outputOperations)); assert(outputOperations.length % 2 === 0 && outputOperations.length <= tx.outputs.length * 2, 'Output operations count is not reflective of what should be possible.'); @@ -809,7 +814,7 @@ AddressService.prototype._processTransaction = function(tx, opts) { return self._processInput(tx, input, index, _opts); }); - inputOperations = _.compact(_.flattenDeep(inputOperations)); + inputOperations = lodash.compact(lodash.flattenDeep(inputOperations)); assert(inputOperations.length % 2 === 0 && inputOperations.length <= tx.inputs.length * 2, diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 8d078a8a..df033df3 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -32,6 +32,7 @@ var BlockService = function(options) { this._recentBlockHashesCount = options.recentBlockHashesCount || 144; // block service won't reorg past this point this._recentBlockHashes = new LRU(this._recentBlockHashesCount); this._readAheadBlockCount = options.readAheadBlockCount || 2; // this is the number of blocks to direct the p2p service to read aheead + this._pauseSync = options.pause; }; inherits(BlockService, BaseService); @@ -523,6 +524,11 @@ BlockService.prototype.onHeaders = function(callback) { var self = this; + if (self._pauseSync) { + log.warn('Block Service: pausing sync due to config option.'); + return callback(); + } + // if this service is waiting on block-related callbacks to be fired in the event loop, // then we need to wait for the _processingBlock flag to be set to false. // when this flag is false, we know we aren't waiting on any new blocks or historical blocks diff --git a/lib/services/db/index.js b/lib/services/db/index.js index cdaccf89..854085ff 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -78,8 +78,7 @@ DB.prototype.start = function(callback) { mkdirp.sync(this.dataPath); } - this._store = levelup(this.dataPath, { - db: this.levelupStore, + this._store = levelup(this.levelupStore(this.dataPath), { keyEncoding: 'binary', valueEncoding: 'binary', writeBufferSize: 8 * 1024 * 1024, @@ -242,8 +241,8 @@ DB.prototype.getServiceTip = function(serviceName, callback) { DB.prototype.getPrefix = function(service, callback) { var self = this; - var keyBuf = Buffer.concat([ self._dbPrefix, new Buffer('prefix-', 'utf8'), new Buffer(service, 'utf8') ]); - var unusedBuf = Buffer.concat([ self._dbPrefix, new Buffer('nextUnused', 'utf8') ]); + var keyBuf = Buffer.concat([ self._dbPrefix, new Buffer('prefix-', 'utf8'), new Buffer(service, 'utf8') ]); + var unusedBuf = Buffer.concat([ self._dbPrefix, new Buffer('nextUnused', 'utf8') ]); function getPrefix(next) { @@ -255,6 +254,7 @@ DB.prototype.getPrefix = function(service, callback) { if (!buf) { return next(); } + log.info('Db Service: service prefix for: ' + service + ' is: ' + buf.toString('hex')); callback(null, buf); }); @@ -314,8 +314,13 @@ DB.prototype.getPrefix = function(service, callback) { putPrefix, putUnused ], - callback - ); + function(err, prefix) { + if (err) { + return callback(err); + } + log.info('Db Service: service prefix for: ' + service + ' is: ' + prefix.toString('hex')); + callback(null, prefix); + }); }; module.exports = DB; diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js index cb0a6c19..e0867202 100644 --- a/lib/services/transaction/index.js +++ b/lib/services/transaction/index.js @@ -190,7 +190,7 @@ TransactionService.prototype.setTxMetaInfo = function(tx, options, callback) { tx.__inputValues.forEach(function(val) { - if (val >+ 0) { + if (val > 0) { inputSatoshis += val; } }); diff --git a/package-lock.json b/package-lock.json index b58a3fd1..f5ddf8d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "version": "2.6.1", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-2.6.1.tgz", "integrity": "sha1-+QFKVmm3RkGOFFFo3qSaBErhWQA=", + "optional": true, "requires": { "xtend": "4.0.1" } @@ -1189,14 +1190,6 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, - "deferred-leveldown": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-1.2.2.tgz", - "integrity": "sha512-uukrWD2bguRtXilKt6cAWKyoXrTSMo5m7crUdLfWQmu8kIm88w3QZoUL+6nhpfKVmhHANER6Re3sKoNoZ3IKMA==", - "requires": { - "abstract-leveldown": "2.6.1" - } - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -2569,53 +2562,6 @@ "integrity": "sha1-GwuP+ayceIklBYK3C3ExXZ2m2aM=", "dev": true }, - "level-codec": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-7.0.0.tgz", - "integrity": "sha1-x1W2jQ1E/6Cxy6BEuPgaVaFK05s=" - }, - "level-errors": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-1.0.4.tgz", - "integrity": "sha1-NYXmI5dMc3qTdVSSpDwCZ82kQl8=", - "requires": { - "errno": "0.1.4" - } - }, - "level-iterator-stream": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-1.3.1.tgz", - "integrity": "sha1-5Dt4sagUPm+pek9IXrjqUwNS8u0=", - "requires": { - "inherits": "2.0.3", - "level-errors": "1.0.4", - "readable-stream": "1.1.14", - "xtend": "4.0.1" - }, - "dependencies": { - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" - }, - "readable-stream": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", - "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" - } - } - }, "leveldown": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-2.0.0.tgz", @@ -2649,23 +2595,49 @@ } }, "levelup": { - "version": "1.3.9", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-1.3.9.tgz", - "integrity": "sha512-VVGHfKIlmw8w1XqpGOAGwq6sZm2WwWLmlDcULkKWQXEA5EopA8OBNJ2Ck2v6bdk8HeEZSbCSEgzXadyQFm76sQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/levelup/-/levelup-2.0.0.tgz", + "integrity": "sha512-0yfH17VjEgdeVYYrCBwlLStJEKpzj5j4WRDKjMWkby5zwgY/HcHiH4VukHsyMH4HyectVljpATCD4Pcbk5md6w==", "requires": { - "deferred-leveldown": "1.2.2", - "level-codec": "7.0.0", - "level-errors": "1.0.4", - "level-iterator-stream": "1.3.1", - "prr": "1.0.1", - "semver": "5.4.1", + "deferred-leveldown": "2.0.3", + "level-errors": "1.1.1", + "level-iterator-stream": "2.0.0", "xtend": "4.0.1" }, "dependencies": { - "prr": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=" + "abstract-leveldown": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-3.0.0.tgz", + "integrity": "sha512-KUWx9UWGQD12zsmLNj64/pndaz4iJh/Pj7nopgkfDG6RlCcbMZvT6+9l7dchK4idog2Is8VdC/PvNbFuFmalIQ==", + "requires": { + "xtend": "4.0.1" + } + }, + "deferred-leveldown": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-2.0.3.tgz", + "integrity": "sha512-8c2Hv+vIwKNc7qqy4zE3t5DIsln+FQnudcyjLYstHwLFg7XnXZT/H8gQb8lj6xi8xqGM0Bz633ZWcCkonycBTA==", + "requires": { + "abstract-leveldown": "3.0.0" + } + }, + "level-errors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-1.1.1.tgz", + "integrity": "sha512-9MIIbizlJgWFQ6m45ehVuSrpzFxwJQmZYD6sfmiizhdmWMNUf41mBYpUJEeCslIa3sB4vdsIFCimPdDZkWznwA==", + "requires": { + "errno": "0.1.4" + } + }, + "level-iterator-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-2.0.0.tgz", + "integrity": "sha512-TWOYw8HR5mhj6xwoVLo0yu26RPL6v28KgvhK1kY1CJf9LyL+rJXjx99zhORTYhN9ysOBIH+iaxAiqRteA+C1/g==", + "requires": { + "inherits": "2.0.3", + "readable-stream": "2.3.3", + "xtend": "4.0.1" + } } } }, @@ -3847,7 +3819,8 @@ "semver": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", - "integrity": "sha512-WfG/X9+oATh81XtllIo/I8gOiY9EXRdv1cQdyykeXK17YcUW3EXUAi2To4pcH6nZtJPr7ZOpM5OMyWJZm+8Rsg==" + "integrity": "sha512-WfG/X9+oATh81XtllIo/I8gOiY9EXRdv1cQdyykeXK17YcUW3EXUAi2To4pcH6nZtJPr7ZOpM5OMyWJZm+8Rsg==", + "optional": true }, "send": { "version": "0.15.3", diff --git a/package.json b/package.json index f69e09ab..5196cdf5 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "errno": "^0.1.4", "express": "^4.13.3", "leveldown": "^2.0.0", - "levelup": "^1.3.9", + "levelup": "^2.0.0", "liftoff": "^2.2.0", "lodash": "^4.17.4", "lru-cache": "^4.0.2",