From 40eb4f50aec0d841daea783e3055ed16e570af7e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Dec 2015 00:16:22 -0500 Subject: [PATCH] Address Service: Start to cache `getAddressSummary` based on range of block heights --- lib/services/address/constants.js | 4 + lib/services/address/encoding.js | 59 +++++++++++ lib/services/address/index.js | 167 +++++++++++++++++++++++------- lib/services/db.js | 6 +- 4 files changed, 197 insertions(+), 39 deletions(-) diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js index 7a0cf8dd..dc0b18b7 100644 --- a/lib/services/address/constants.js +++ b/lib/services/address/constants.js @@ -37,5 +37,9 @@ exports.HASH_TYPES_MAP = { exports.SPACER_MIN = new Buffer('00', 'hex'); exports.SPACER_MAX = new Buffer('ff', 'hex'); +// The total number of transactions that an address can receive before it will start +// to cache the summary to disk. +exports.SUMMARY_CACHE_THRESHOLD = 10000; + module.exports = exports; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index 68313754..ca42fc7f 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -160,6 +160,65 @@ exports.decodeInputValueMap = function(buffer) { }; }; +exports.encodeSummaryCacheKey = function(address) { + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_BUFFER[address.type]]); +}; + +exports.decodeSummaryCacheKey = function(buffer, network) { + var hashBuffer = buffer.read(20); + var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')]; + var address = new Address({ + hashBuffer: hashBuffer, + type: type, + network: network + }); + return address; +}; + +exports.encodeSummaryCacheValue = function(cache, tipHeight) { + var buffer = new Buffer(new Array(20)); + buffer.writeUInt32BE(tipHeight); + buffer.writeDoubleBE(cache.result.totalReceived, 4); + buffer.writeDoubleBE(cache.result.balance, 12); + var txidBuffers = []; + for (var key in cache.result.appearanceIds) { + txidBuffers.push(new Buffer(key, 'hex')); + } + var txidsBuffer = Buffer.concat(txidBuffers); + var value = Buffer.concat([buffer, txidsBuffer]); + + return value; +}; + +exports.decodeSummaryCacheValue = function(buffer) { + + var height = buffer.readUInt32BE(); + var totalReceived = buffer.readDoubleBE(4); + var balance = buffer.readDoubleBE(12); + + // read 32 byte chunks until exhausted + var appearanceIds = {}; + var pos = 16; + while(pos < buffer.length) { + var txid = buffer.slice(pos, pos + 32).toString('hex'); + appearanceIds[txid] = true; + pos += 32; + } + + var cache = { + height: height, + result: { + appearanceIds: appearanceIds, + totalReceived: totalReceived, + balance: balance, + unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache + unconfirmedBalance: 0 + } + }; + + return cache; +}; + exports.getAddressInfo = function(addressStr) { var addrObj = bitcore.Address(addressStr); var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index cf37e7e2..665a32dc 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -44,7 +44,10 @@ var AddressService = function(options) { this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this.summaryCacheThreshold = options.summaryCacheThreshold || constants.SUMMARY_CACHE_THRESHOLD; + this._setMempoolIndexPath(); + this._setSummaryCachePath(); if (options.mempoolMemoryIndex) { this.levelupStore = memdown; } else { @@ -74,6 +77,7 @@ AddressService.prototype.start = function(callback) { } }, function(next) { + // Setup new mempool index if (!fs.existsSync(self.mempoolIndexPath)) { mkdirp(self.mempoolIndexPath, next); } else { @@ -87,7 +91,21 @@ AddressService.prototype.start = function(callback) { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', - fillCache: false + fillCache: false, + maxOpenFiles: 200 + }, + next + ); + }, + function(next) { + self.summaryCache = levelup( + self.summaryCachePath, + { + db: self.levelupStore, + keyEncoding: 'binary', + valueEncoding: 'binary', + fillCache: false, + maxOpenFiles: 200 }, next ); @@ -102,21 +120,35 @@ AddressService.prototype.stop = function(callback) { }; /** - * This function will set `this.dataPath` based on `this.node.network`. + * This function will set `this.summaryCachePath` based on `this.node.network`. + * @private + */ +AddressService.prototype._setSummaryCachePath = function() { + this.summaryCachePath = this._getDBPathFor('bitcore-addresssummary.db'); +}; + +/** + * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private */ AddressService.prototype._setMempoolIndexPath = function() { + this.mempoolIndexPath = this._getDBPathFor('bitcore-addressmempool.db'); +}; + +AddressService.prototype._getDBPathFor = function(dbname) { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); + var path; var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { - this.mempoolIndexPath = this.node.datadir + '/bitcore-addressmempool.db'; + path = this.node.datadir + '/' + dbname; } else if (this.node.network === Networks.testnet) { - this.mempoolIndexPath = this.node.datadir + '/testnet3/bitcore-addressmempool.db'; + path = this.node.datadir + '/testnet3/' + dbname; } else if (this.node.network === regtest) { - this.mempoolIndexPath = this.node.datadir + '/regtest/bitcore-addressmempool.db'; + path = this.node.datadir + '/regtest/' + dbname; } else { throw new Error('Unknown network: ' + this.network); } + return path; }; /** @@ -1270,32 +1302,49 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; + var startTime = new Date(); + var address = new Address(addressArg); + var tipHeight = this.node.services.db.tip.__height; async.waterfall([ function(next) { - self._getAddressInputsSummary(address, options, next); + self._getAddressSummaryCache(address, next); }, - function(result, next) { - self._getAddressOutputsSummary(address, options, result, next); + function(cache, next) { + self._getAddressInputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._getAddressOutputsSummary(address, cache, tipHeight, next); + }, + function(cache, next) { + self._saveAddressSummaryCache(address, cache, tipHeight, next); } - ], function(err, result) { + ], function(err, cache) { if (err) { return callback(err); } + var result = cache.result; var confirmedTxids = Object.keys(result.appearanceIds); var unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); var summary = { totalReceived: result.totalReceived, - totalSpent: result.totalSpent, + totalSpent: result.totalReceived - result.balance, balance: result.balance, - unconfirmedBalance: result.unconfirmedBalance, appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, unconfirmedAppearances: unconfirmedTxids.length }; + var timeDelta = new Date() - startTime; + if (timeDelta > 5000) { + var seconds = Math.round(timeDelta / 1000); + log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); + log.warn('Address Summary:', summary); + } + if (!options.noTxList) { var txids = confirmedTxids.concat(unconfirmedTxids); @@ -1315,20 +1364,64 @@ AddressService.prototype.getAddressSummary = function(addressArg, options, callb }; -AddressService.prototype._getAddressInputsSummary = function(address, options, callback) { +AddressService.prototype._saveAddressSummaryCache = function(address, cache, tipHeight, callback) { + var transactionLength = Object.keys(cache.result.appearanceIds).length; + var exceedsCacheThreshold = (transactionLength > this.summaryCacheThreshold); + if (exceedsCacheThreshold) { + log.info('Saving address summary cache for: ' + address.toString() + 'at height: ' + tipHeight); + var key = encoding.encodeSummaryCacheKey(address); + var value = encoding.encodeSummaryCacheValue(cache, tipHeight); + this.summaryCache.put(key, value, function(err) { + if (err) { + return callback(err); + } + callback(null, cache); + }); + } else { + callback(null, cache); + } +}; + +AddressService.prototype._getAddressSummaryCache = function(address, callback) { + var baseCache = { + result: { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + } + }; + var key = encoding.encodeSummaryCacheKey(address); + this.summaryCache.get(key, { + valueEncoding: 'binary', + keyEncoding: 'binary' + }, function(err, buffer) { + if (err instanceof levelup.errors.NotFoundError) { + return callback(null, baseCache); + } else if (err) { + return callback(err); + } + var cache = encoding.decodeSummaryCacheValue(buffer); + callback(null, cache); + }); +}; + +AddressService.prototype._getAddressInputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); var self = this; var error = null; - var result = { - appearanceIds: {}, - unconfirmedAppearanceIds: {}, + + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight }; - var inputsStream = self.createInputsStream(address, options); + var inputsStream = self.createInputsStream(address, opts); inputsStream.on('data', function(input) { var txid = input.txid; - result.appearanceIds[txid] = true; + cache.result.appearanceIds[txid] = true; }); inputsStream.on('error', function(err) { @@ -1347,27 +1440,27 @@ AddressService.prototype._getAddressInputsSummary = function(address, options, c } for(var i = 0; i < mempoolInputs.length; i++) { var input = mempoolInputs[i]; - result.unconfirmedAppearanceIds[input.txid] = true; + cache.result.unconfirmedAppearanceIds[input.txid] = true; } - callback(error, result); + callback(error, cache); }); }); }; -AddressService.prototype._getAddressOutputsSummary = function(address, options, result, callback) { +AddressService.prototype._getAddressOutputsSummary = function(address, cache, tipHeight, callback) { $.checkArgument(address instanceof Address); - $.checkArgument(!_.isUndefined(result) && - !_.isUndefined(result.appearanceIds) && - !_.isUndefined(result.unconfirmedAppearanceIds)); + $.checkArgument(!_.isUndefined(cache.result) && + !_.isUndefined(cache.result.appearanceIds) && + !_.isUndefined(cache.result.unconfirmedAppearanceIds)); var self = this; - var outputStream = self.createOutputsStream(address, options); + var opts = { + start: _.isUndefined(cache.height) ? 0 : cache.height + 1, + end: tipHeight + }; - result.totalReceived = 0; - result.totalSpent = 0; - result.balance = 0; - result.unconfirmedBalance = 0; + var outputStream = self.createOutputsStream(address, opts); outputStream.on('data', function(output) { @@ -1376,13 +1469,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, // Bitcoind's isSpent only works for confirmed transactions var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); - result.totalReceived += output.satoshis; - result.appearanceIds[txid] = true; + cache.result.totalReceived += output.satoshis; + cache.result.appearanceIds[txid] = true; - if (spentDB) { - result.totalSpent += output.satoshis; - } else { - result.balance += output.satoshis; + if (!spentDB) { + cache.result.balance += output.satoshis; } // Check to see if this output is spent in the mempool and if so @@ -1393,7 +1484,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, ); var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; if (spentMempool) { - result.unconfirmedBalance -= output.satoshis; + cache.result.unconfirmedBalance -= output.satoshis; } }); @@ -1418,7 +1509,7 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, for(var i = 0; i < mempoolOutputs.length; i++) { var output = mempoolOutputs[i]; - result.unconfirmedAppearanceIds[output.txid] = true; + cache.result.unconfirmedAppearanceIds[output.txid] = true; var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly @@ -1427,11 +1518,11 @@ AddressService.prototype._getAddressOutputsSummary = function(address, options, var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; // Only add this to the balance if it's not spent in the mempool already if (!spentMempool) { - result.unconfirmedBalance += output.satoshis; + cache.result.unconfirmedBalance += output.satoshis; } } - callback(error, result); + callback(error, cache); }); diff --git a/lib/services/db.js b/lib/services/db.js index 0c655051..c3c3dfd4 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -46,6 +46,8 @@ function DB(options) { this._setDataPath(); + this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; + this.levelupStore = leveldown; if (options.store) { this.levelupStore = options.store; @@ -68,6 +70,8 @@ DB.PREFIXES = { TIP: new Buffer('04', 'hex') }; +DB.DEFAULT_MAX_OPEN_FILES = 200; + /** * This function will set `this.dataPath` based on `this.node.network`. * @private @@ -98,7 +102,7 @@ DB.prototype.start = function(callback) { } this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore }); + this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles }); this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.once('ready', function() {