From c205f781a58255a1ac13629055f95f232a30d3c2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Sep 2015 19:45:49 -0400 Subject: [PATCH 01/10] Pagination for Address History - Reindexing the bitcore-node database is required with this change. - Address indexes are updated to include address and height in spent index so that both inputs and outputs can be queried by address and height using "start" and "stop" to limit the range of the query. - Address history also now supports paginated results using "from" and "to" values that indicate an index in the array. --- integration/regtest-node.js | 373 ++++++++++++ lib/services/address/history.js | 188 ++++++ lib/services/{address.js => address/index.js} | 353 +++++------ test/services/address/history.unit.js | 574 ++++++++++++++++++ .../index.unit.js} | 462 ++++++-------- 5 files changed, 1512 insertions(+), 438 deletions(-) create mode 100644 lib/services/address/history.js rename lib/services/{address.js => address/index.js} (66%) create mode 100644 test/services/address/history.unit.js rename test/services/{address.unit.js => address/index.unit.js} (71%) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index 932b1483..0c41524e 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -24,6 +24,7 @@ var should = chai.should(); var BitcoinRPC = require('bitcoind-rpc'); var index = require('..'); +var Transaction = index.Transaction; var BitcoreNode = index.Node; var AddressService = index.services.Address; var BitcoinService = index.services.Bitcoin; @@ -34,6 +35,8 @@ var client; describe('Node Functionality', function() { + var regtest; + before(function(done) { this.timeout(30000); @@ -51,6 +54,7 @@ describe('Node Functionality', function() { port: 18444, dnsSeeds: [ ] }); + regtest = bitcore.Networks.get('regtest'); var datadir = __dirname + '/data'; @@ -238,4 +242,373 @@ describe('Node Functionality', function() { }); }); }); + + describe('Address Functionality', function() { + var address; + var unspentOutput; + before(function() { + address = testKey.toAddress().toString(); + }); + it('should be able to get the balance of the test address', function(done) { + node.services.address.getBalance(address, false, function(err, balance) { + if (err) { + throw err; + } + balance.should.equal(10 * 1e8); + done(); + }); + }); + it('can get unspent outputs for address', function(done) { + node.services.address.getUnspentOutputs(address, false, function(err, results) { + if (err) { + throw err; + } + results.length.should.equal(1); + unspentOutput = results[0]; + done(); + }); + }); + it('correctly give the history for the address', function(done) { + var options = { + from: 0, + to: 10, + queryMempool: false + }; + node.services.address.getAddressHistory(address, options, function(err, results) { + if (err) { + throw err; + } + results.length.should.equal(1); + var info = results[0]; + info.address.should.equal(address); + info.satoshis.should.equal(10 * 1e8); + info.confirmations.should.equal(3); + info.timestamp.should.be.a('number'); + info.fees.should.be.within(190, 193); + info.outputIndexes.length.should.equal(1); + info.outputIndexes[0].should.be.within(0, 1); + info.inputIndexes.should.deep.equal([]); + info.tx.should.be.an.instanceof(Transaction); + done(); + }); + }); + describe('History', function() { + + this.timeout(20000); + + var testKey2; + var address2; + var testKey3; + var address3; + var testKey4; + var address4; + var testKey5; + var address5; + var testKey6; + var address6; + + before(function(done) { + /* jshint maxstatements: 50 */ + + testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x'); + address2 = testKey2.toAddress().toString(); + + testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt'); + address3 = testKey3.toAddress().toString(); + + testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw'); + address4 = testKey4.toAddress().toString(); + + testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF'); + address5 = testKey5.toAddress().toString(); + + testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF'); + address6 = testKey6.toAddress().toString(); + + var tx = new Transaction(); + tx.from(unspentOutput); + tx.to(address, 1 * 1e8); + tx.to(address, 2 * 1e8); + tx.to(address, 0.5 * 1e8); + tx.to(address, 3 * 1e8); + tx.fee(10000); + tx.change(address); + tx.sign(testKey); + + node.services.bitcoind.sendTransaction(tx.serialize()); + + function mineBlock(next) { + client.generate(1, function(err, response) { + if (err) { + throw err; + } + should.exist(response); + next(); + }); + } + + client.generate(1, function(err, response) { + if (err) { + throw err; + } + should.exist(response); + node.once('synced', function() { + node.services.address.getUnspentOutputs(address, false, function(err, results) { + /* jshint maxstatements: 50 */ + if (err) { + throw err; + } + results.length.should.equal(5); + + async.series([ + function(next) { + var tx2 = new Transaction(); + tx2.from(results[0]); + tx2.to(address2, results[0].satoshis - 10000); + tx2.change(address); + tx2.sign(testKey); + node.services.bitcoind.sendTransaction(tx2.serialize()); + mineBlock(next); + }, function(next) { + var tx3 = new Transaction(); + tx3.from(results[1]); + tx3.to(address3, results[1].satoshis - 10000); + tx3.change(address); + tx3.sign(testKey); + node.services.bitcoind.sendTransaction(tx3.serialize()); + mineBlock(next); + }, function(next) { + var tx4 = new Transaction(); + tx4.from(results[2]); + tx4.to(address4, results[2].satoshis - 10000); + tx4.change(address); + tx4.sign(testKey); + node.services.bitcoind.sendTransaction(tx4.serialize()); + mineBlock(next); + }, function(next) { + var tx5 = new Transaction(); + tx5.from(results[3]); + tx5.from(results[4]); + tx5.to(address5, results[3].satoshis - 10000); + tx5.to(address6, results[4].satoshis - 10000); + tx5.change(address); + tx5.sign(testKey); + node.services.bitcoind.sendTransaction(tx5.serialize()); + mineBlock(next); + } + ], function(err) { + if (err) { + throw err; + } + node.once('synced', function() { + done(); + }); + }); + }); + }); + }); + + }); + + it('five addresses', function(done) { + var addresses = [ + address2, + address3, + address4, + address5, + address6 + ]; + var options = {}; + node.services.address.getAddressHistory(addresses, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(5); + history[0].height.should.equal(157); + history[0].confirmations.should.equal(1); + history[1].height.should.equal(157); + history[2].height.should.equal(156); + history[2].address.should.equal(address4); + history[3].height.should.equal(155); + history[3].address.should.equal(address3); + history[4].height.should.equal(154); + history[4].address.should.equal(address2); + history[4].satoshis.should.equal(99990000); + history[4].confirmations.should.equal(4); + done(); + }); + }); + + it('five addresses (limited by height)', function(done) { + var addresses = [ + address2, + address3, + address4, + address5, + address6 + ]; + var options = { + start: 157, + end: 156 + }; + node.services.address.getAddressHistory(addresses, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(3); + history[0].height.should.equal(157); + history[0].confirmations.should.equal(1); + history[1].height.should.equal(157); + history[2].height.should.equal(156); + history[2].address.should.equal(address4); + done(); + }); + }); + + it('five addresses (paginated by index)', function(done) { + var addresses = [ + address2, + address3, + address4, + address5, + address6 + ]; + var options = { + from: 0, + to: 3 + }; + node.services.address.getAddressHistory(addresses, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(3); + history[0].height.should.equal(157); + history[0].confirmations.should.equal(1); + history[1].height.should.equal(157); + history[2].height.should.equal(156); + history[2].address.should.equal(address4); + done(); + }); + }); + + it('one address with sending and receiving', function(done) { + var addresses = [ + address + ]; + var options = {}; + node.services.address.getAddressHistory(addresses, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(6); + history[0].height.should.equal(157); + history[0].inputIndexes.should.deep.equal([0, 1]); + history[0].outputIndexes.should.deep.equal([2]); + history[0].confirmations.should.equal(1); + history[1].height.should.equal(156); + history[2].height.should.equal(155); + history[3].height.should.equal(154); + history[4].height.should.equal(153); + history[4].satoshis.should.equal(-10000); + history[4].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); + history[4].inputIndexes.should.deep.equal([0]); + history[5].height.should.equal(150); + history[5].satoshis.should.equal(10 * 1e8); + done(); + }); + }); + + describe('Pagination', function() { + it('from 0 to 1', function(done) { + var options = { + from: 0, + to: 1 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(157); + done(); + }); + }); + it('from 1 to 2', function(done) { + var options = { + from: 1, + to: 2 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(156); + done(); + }); + }); + it('from 2 to 3', function(done) { + var options = { + from: 2, + to: 3 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(155); + done(); + }); + }); + it('from 3 to 4', function(done) { + var options = { + from: 3, + to: 4 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(154); + done(); + }); + }); + it('from 4 to 5', function(done) { + var options = { + from: 4, + to: 5 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(153); + history[0].satoshis.should.equal(-10000); + history[0].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); + history[0].inputIndexes.should.deep.equal([0]); + done(); + }); + }); + it('from 5 to 6', function(done) { + var options = { + from: 5, + to: 6 + }; + node.services.address.getAddressHistory(address, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(1); + history[0].height.should.equal(150); + history[0].satoshis.should.equal(10 * 1e8); + done(); + }); + }); + + }); + }); + }); }); diff --git a/lib/services/address/history.js b/lib/services/address/history.js new file mode 100644 index 00000000..ad36b2d6 --- /dev/null +++ b/lib/services/address/history.js @@ -0,0 +1,188 @@ +'use strict'; + +var bitcore = require('bitcore'); +var async = require('async'); +var _ = bitcore.deps._; + +/** + * This represents an instance that keeps track of data over a series of + * asynchronous I/O calls to get the transaction history for a group of + * addresses. History can be queried by start and end block heights to limit large sets + * of results (uses leveldb key streaming). See AddressService.prototype.getAddressHistory + * for complete documentation about options. + */ +function AddressHistory(args) { + this.node = args.node; + this.options = args.options; + + if(Array.isArray(args.addresses)) { + this.addresses = args.addresses; + } else { + this.addresses = [args.addresses]; + } + this.transactionInfo = []; + this.transactions = {}; + this.sortedArray = []; +} + +AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.MAX_TX_QUERIES = 10; + +AddressHistory.prototype.get = function(callback) { + var self = this; + + // TODO check for mempool inputs and outputs by a group of addresses, currently + // each address individually loops through the mempool and does not check input scripts. + + async.eachLimit( + self.addresses, + AddressHistory.MAX_ADDRESS_QUERIES, + function(address, next) { + self.getTransactionInfo(address, next); + }, + function(err) { + if (err) { + return callback(err); + } + + async.eachSeries( + self.transactionInfo, + function(txInfo, next) { + self.getDetailedInfo(txInfo, next); + }, + function(err) { + if (err) { + return callback(err); + } + self.sortTransactionsIntoArray(); + self.paginateSortedArray(); + callback(null, self.sortedArray); + } + ); + } + ); +}; + +AddressHistory.prototype.getTransactionInfo = function(address, next) { + var self = this; + + var args = { + start: self.options.start, + end: self.options.end, + queryMempool: _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool + }; + + var outputs; + var inputs; + + async.parallel([ + function(done) { + self.node.services.address.getOutputs(address, args, function(err, result) { + if (err) { + return done(err); + } + outputs = result; + done(); + }); + }, + function(done) { + self.node.services.address.getInputs(address, args, function(err, result) { + if (err) { + return done(err); + } + inputs = result; + done(); + }); + } + ], function(err) { + if (err) { + return next(err); + } + self.transactionInfo = self.transactionInfo.concat(outputs, inputs); + next(); + }); +}; + +AddressHistory.sortByHeight = function(a, b) { + // TODO consider timestamp for mempool transactions + return a.height < b.height; +}; + +AddressHistory.prototype.paginateSortedArray = function() { + if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { + this.sortedArray = this.sortedArray.slice(this.options.from, this.options.to); + } +}; + +AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { + var self = this; + var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; + + if (self.transactions[txInfo.address] && self.transactions[txInfo.address][txInfo.txid]) { + self.amendDetailedInfoWithSatoshis(txInfo); + setImmediate(next); + } else { + self.node.services.db.getTransactionWithBlockInfo( + txInfo.txid, + queryMempool, + function(err, transaction) { + if (err) { + return next(err); + } + + transaction.populateInputs(self.node.services.db, [], function(err) { + if(err) { + return next(err); + } + var confirmations = 0; + if (transaction.__height >= 0) { + confirmations = self.node.services.db.tip.__height - transaction.__height + 1; + } + + if (!self.transactions[txInfo.address]) { + self.transactions[txInfo.address] = {}; + } + + self.transactions[txInfo.address][txInfo.txid] = { + address: txInfo.address, + satoshis: 0, + height: transaction.__height, + confirmations: confirmations, + timestamp: transaction.__timestamp, + // TODO bitcore should return null instead of throwing error on coinbase + fees: !transaction.isCoinbase() ? transaction.getFee() : null, + outputIndexes: [], + inputIndexes: [], + tx: transaction + }; + + self.amendDetailedInfoWithSatoshis(txInfo); + next(); + }); + } + ); + } +}; + +AddressHistory.prototype.amendDetailedInfoWithSatoshis = function(txInfo) { + var historyItem = this.transactions[txInfo.address][txInfo.txid]; + if (txInfo.outputIndex >= 0) { + historyItem.outputIndexes.push(txInfo.outputIndex); + historyItem.satoshis += txInfo.satoshis; + } else if (txInfo.inputIndex >= 0){ + historyItem.inputIndexes.push(txInfo.inputIndex); + historyItem.satoshis -= historyItem.tx.inputs[txInfo.inputIndex].output.satoshis; + } +}; + +AddressHistory.prototype.sortTransactionsIntoArray = function() { + this.sortedArray = []; + for(var address in this.transactions) { + for(var txid in this.transactions[address]) { + this.sortedArray.push(this.transactions[address][txid]); + } + } + this.sortedArray.sort(AddressHistory.sortByHeight); +}; + +module.exports = AddressHistory; diff --git a/lib/services/address.js b/lib/services/address/index.js similarity index 66% rename from lib/services/address.js rename to lib/services/address/index.js index 5cdf2362..12eda69a 100644 --- a/lib/services/address.js +++ b/lib/services/address/index.js @@ -1,11 +1,10 @@ 'use strict'; -var BaseService = require('../service'); +var BaseService = require('../../service'); var inherits = require('util').inherits; var async = require('async'); -var index = require('../'); +var index = require('../../'); var log = index.log; -var levelup = require('levelup'); var errors = index.errors; var bitcore = require('bitcore'); var $ = bitcore.util.preconditions; @@ -13,6 +12,7 @@ var _ = bitcore.deps._; var EventEmitter = require('events').EventEmitter; var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; +var AddressHistory = require('./history'); var AddressService = function(options) { BaseService.call(this, options); @@ -152,8 +152,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var txmessages = {}; var outputLength = outputs.length; - for (var j = 0; j < outputLength; j++) { - var output = outputs[j]; + for (var outputIndex = 0; outputIndex < outputLength; outputIndex++) { + var output = outputs[outputIndex]; var script = output.script; @@ -170,16 +170,26 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - var outputIndex = j; - // TODO: expose block timestamp as a date object in bitcore? - var timestamp = block.header.timestamp; + // We need to use the height for indexes (and not the timestamp) because the + // the timestamp has unreliable sequential ordering. The next block + // can have a time that is previous to the previous block (however not + // less than the mean of the 11 previous blocks) and not greater than 2 + // hours in the future. var height = block.__height; var addressStr = address.toString(); var scriptHex = output._scriptBuffer.toString('hex'); - var key = [AddressService.PREFIXES.OUTPUTS, addressStr, timestamp, txid, outputIndex].join('-'); - var value = [output.satoshis, scriptHex, height].join(':'); + // To lookup outputs by address and height + var key = [ + AddressService.PREFIXES.OUTPUTS, + addressStr, + height, + txid, + outputIndex + ].join('-'); + + var value = [output.satoshis, scriptHex].join(':'); operations.push({ type: action, @@ -213,13 +223,38 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - for(var k = 0; k < inputs.length; k++) { - var input = inputs[k].toObject(); - operations.push({ - type: action, - key: [AddressService.PREFIXES.SPENTS, input.prevTxId, input.outputIndex].join('-'), - value: [txid, k].join(':') - }); + for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { + + var input = inputs[inputIndex]; + var inputAddress = input.script.toAddress(this.node.network); + + if (inputAddress) { + + var inputObject = input.toObject(); + var inputAddressStr = inputAddress.toString(); + + var height = block.__height; + + // To be able to query inputs by address and spent height + var inputKey = [ + AddressService.PREFIXES.SPENTS, + inputAddressStr, + height, + inputObject.prevTxId, + inputObject.outputIndex + ].join('-'); + + var inputValue = [ + txid, + inputIndex + ].join(':'); + + operations.push({ + type: action, + key: inputKey, + value: inputValue + }); + } } } @@ -323,17 +358,120 @@ AddressService.prototype.getBalance = function(address, queryMempool, callback) }); }; -AddressService.prototype.getOutputs = function(addressStr, queryMempool, callback) { +/** + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Boolean} [options.queryMempool] - Include the mempool in the results + * @param {Function} callback + */ +AddressService.prototype.getInputs = function(addressStr, options, callback) { + var self = this; - var outputs = []; - var key = [AddressService.PREFIXES.OUTPUTS, addressStr].join('-'); + var inputs = []; + var stream; + + if (options.start && options.end) { + + // The positions will be flipped because the end position should be greater + // than the starting position for the stream, and we'll add one to the end key + // so that it's included in the results. + + var endKey = [AddressService.PREFIXES.SPENTS, addressStr, options.start + 1].join('-'); + var startKey = [AddressService.PREFIXES.SPENTS, addressStr, options.end].join('-'); + + stream = this.node.services.db.store.createReadStream({ + start: startKey, + end: endKey + }); + } else { + var allKey = [AddressService.PREFIXES.SPENTS, addressStr].join('-'); + stream = this.node.services.db.store.createReadStream({ + start: allKey, + end: allKey + '~' + }); + } + + stream.on('data', function(data) { + + var key = data.key.split('-'); + var value = data.value.split(':'); + + var blockHeight = Number(key[2]); + + var output = { + address: addressStr, + txid: value[0], + inputIndex: Number(value[1]), + height: blockHeight, + confirmations: self.node.services.db.tip.__height - blockHeight + 1 + }; + + inputs.push(output); - var stream = this.node.services.db.store.createReadStream({ - start: key, - end: key + '~' }); + var error; + + stream.on('error', function(streamError) { + if (streamError) { + error = streamError; + } + }); + + stream.on('close', function() { + if (error) { + return callback(error); + } + + // TODO include results from mempool + + callback(null, inputs); + + }); + + return stream; + +}; + +/** + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Boolean} [options.queryMempool] - Include the mempool in the results + * @param {Function} callback + */ +AddressService.prototype.getOutputs = function(addressStr, options, callback) { + var self = this; + $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); + $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); + + var outputs = []; + var stream; + + if (options.start && options.end) { + + // The positions will be flipped because the end position should be greater + // than the starting position for the stream, and we'll add one to the end key + // so that it's included in the results. + var endKey = [AddressService.PREFIXES.OUTPUTS, addressStr, options.start + 1].join('-'); + var startKey = [AddressService.PREFIXES.OUTPUTS, addressStr, options.end].join('-'); + + stream = this.node.services.db.store.createReadStream({ + start: startKey, + end: endKey + }); + } else { + var allKey = [AddressService.PREFIXES.OUTPUTS, addressStr].join('-'); + stream = this.node.services.db.store.createReadStream({ + start: allKey, + end: allKey + '~' + }); + } + stream.on('data', function(data) { var key = data.key.split('-'); @@ -343,11 +481,10 @@ AddressService.prototype.getOutputs = function(addressStr, queryMempool, callbac address: addressStr, txid: key[3], outputIndex: Number(key[4]), - timestamp: Number(key[2]), + height: Number(key[2]), satoshis: Number(value[0]), script: value[1], - blockHeight: Number(value[2]), - confirmations: self.node.services.db.tip.__height - Number(value[2]) + 1 + confirmations: self.node.services.db.tip.__height - Number(key[2]) + 1 }; outputs.push(output); @@ -367,10 +504,9 @@ AddressService.prototype.getOutputs = function(addressStr, queryMempool, callbac return callback(error); } - if(queryMempool) { + if(options.queryMempool) { outputs = outputs.concat(self.node.services.bitcoind.getMempoolOutputs(addressStr)); } - callback(null, outputs); }); @@ -407,7 +543,7 @@ AddressService.prototype.getUnspentOutputsForAddress = function(address, queryMe var self = this; - this.getOutputs(address, queryMempool, function(err, outputs) { + this.getOutputs(address, {queryMempool: queryMempool}, function(err, outputs) { if (err) { return callback(err); } else if(!outputs.length) { @@ -439,148 +575,25 @@ AddressService.prototype.isSpent = function(output, queryMempool, callback) { }); }; -AddressService.prototype.getSpendInfoForOutput = function(txid, outputIndex, callback) { - var self = this; - - var key = [AddressService.PREFIXES.SPENTS, txid, outputIndex].join('-'); - this.node.services.db.store.get(key, function(err, value) { - if(err) { - return callback(err); - } - - value = value.split(':'); - - var info = { - txid: value[0], - inputIndex: value[1] - }; - - callback(null, info); - }); -}; - -AddressService.prototype.getAddressHistory = function(addresses, queryMempool, callback) { - var self = this; - - if(!Array.isArray(addresses)) { - addresses = [addresses]; - } - - var history = []; - - async.eachSeries(addresses, function(address, next) { - self.getAddressHistoryForAddress(address, queryMempool, function(err, h) { - if(err) { - return next(err); - } - - history = history.concat(h); - next(); - }); - }, function(err) { - callback(err, history); - }); -}; - -AddressService.prototype.getAddressHistoryForAddress = function(address, queryMempool, callback) { - var self = this; - - var txinfos = {}; - - function getTransactionInfo(txid, callback) { - if(txinfos[txid]) { - return callback(null, txinfos[txid]); - } - - self.node.services.db.getTransactionWithBlockInfo(txid, queryMempool, function(err, transaction) { - if(err) { - return callback(err); - } - - transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { - return callback(err); - } - - var confirmations = 0; - if(transaction.__height >= 0) { - confirmations = self.node.services.db.tip.__height - transaction.__height; - confirmations = self.node.services.db.tip.__height - transaction.__height + 1; - } - - txinfos[transaction.hash] = { - address: address, - satoshis: 0, - height: transaction.__height, - confirmations: confirmations, - timestamp: transaction.__timestamp, - // TODO bitcore should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - outputIndexes: [], - inputIndexes: [], - tx: transaction - }; - - callback(null, txinfos[transaction.hash]); - }); - }); - } - - this.getOutputs(address, queryMempool, function(err, outputs) { - if(err) { - return callback(err); - } - - async.eachSeries( - outputs, - function(output, next) { - getTransactionInfo(output.txid, function(err, txinfo) { - if(err) { - return next(err); - } - - txinfo.outputIndexes.push(output.outputIndex); - txinfo.satoshis += output.satoshis; - - self.getSpendInfoForOutput(output.txid, output.outputIndex, function(err, spendInfo) { - if(err instanceof levelup.errors.NotFoundError) { - return next(); - } else if(err) { - return next(err); - } - - getTransactionInfo(spendInfo.txid, function(err, txinfo) { - if(err) { - return next(err); - } - - txinfo.inputIndexes.push(spendInfo.inputIndex); - txinfo.satoshis -= txinfo.tx.inputs[spendInfo.inputIndex].output.satoshis; - next(); - }); - }); - }); - }, - function(err) { - if(err) { - return callback(err); - } - - // convert to array - var history = []; - for(var txid in txinfos) { - history.push(txinfos[txid]); - } - - // sort by height - history.sort(function(a, b) { - return a.height > b.height; - }); - - callback(null, history); - } - ); +/** + * This will give the history for many addresses limited by a range of dates (to limit + * the database lookup times) and/or paginated to limit the results length. + * @param {Array} addresses - An array of addresses + * @param {Object} options - The options to limit the query + * @param {Number} [options.from] - The pagination "from" index + * @param {Number} [options.to] - The pagination "to" index + * @param {Number} [options.start] - The beginning block height + * @param {Number} [options.end] - The ending block height + * @param {Boolean} [options.queryMempool] - Include the mempool in the query + * @param {Function} callback + */ +AddressService.prototype.getAddressHistory = function(addresses, options, callback) { + var history = new AddressHistory({ + node: this.node, + options: options, + addresses: addresses }); + history.get(callback); }; module.exports = AddressService; diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js new file mode 100644 index 00000000..06970b4b --- /dev/null +++ b/test/services/address/history.unit.js @@ -0,0 +1,574 @@ +'use strict'; + +var should = require('chai').should(); +var sinon = require('sinon'); +var Transaction = require('../../../lib/transaction'); +var AddressHistory = require('../../../lib/services/address/history'); + +describe('Address Service History', function() { + + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + + describe('@constructor', function() { + it('will construct a new instance', function() { + var node = {}; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.should.be.instanceof(AddressHistory); + history.node.should.equal(node); + history.options.should.equal(options); + history.addresses.should.equal(addresses); + history.transactions.should.deep.equal({}); + history.transactionInfo.should.deep.equal([]); + history.sortedArray.should.deep.equal([]); + }); + it('will set addresses an array if only sent a string', function() { + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: address + }); + history.addresses.should.deep.equal([address]); + }); + }); + + describe('#get', function() { + it('will complete the async each limit series', function(done) { + var addresses = [address]; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: addresses + }); + var expected = [{}]; + history.sortedArray = expected; + history.transactionInfo = [{}]; + history.getTransactionInfo = sinon.stub().callsArg(1); + history.paginateSortedArray = sinon.stub(); + history.getDetailedInfo = sinon.stub().callsArg(1); + history.sortTransactionsIntoArray = sinon.stub(); + history.get(function(err, results) { + if (err) { + throw err; + } + history.getTransactionInfo.callCount.should.equal(1); + history.getDetailedInfo.callCount.should.equal(1); + history.sortTransactionsIntoArray.callCount.should.equal(1); + history.paginateSortedArray.callCount.should.equal(1); + results.should.equal(expected); + done(); + }); + }); + it('handle an error from getDetailedInfo', function(done) { + var addresses = [address]; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: addresses + }); + var expected = [{}]; + history.sortedArray = expected; + history.transactionInfo = [{}]; + history.getTransactionInfo = sinon.stub().callsArg(1); + history.paginateSortedArray = sinon.stub(); + history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); + history.get(function(err) { + err.message.should.equal('test'); + done(); + }); + }); + it('handle an error from getTransactionInfo', function(done) { + var addresses = [address]; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: addresses + }); + var expected = [{}]; + history.sortedArray = expected; + history.transactionInfo = [{}]; + history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test')); + history.get(function(err) { + err.message.should.equal('test'); + done(); + }); + }); + }); + + describe('#getTransactionInfo', function() { + it('will handle an error from getInputs', function(done) { + var history = new AddressHistory({ + node: { + services: { + address: { + getOutputs: sinon.stub().callsArgWith(2, null, []), + getInputs: sinon.stub().callsArgWith(2, new Error('test')) + } + } + }, + options: {}, + addresses: [] + }); + history.getTransactionInfo(address, function(err) { + err.message.should.equal('test'); + done(); + }); + }); + it('will handle an error from getOutputs', function(done) { + var history = new AddressHistory({ + node: { + services: { + address: { + getOutputs: sinon.stub().callsArgWith(2, new Error('test')), + getInputs: sinon.stub().callsArgWith(2, null, []) + } + } + }, + options: {}, + addresses: [] + }); + history.getTransactionInfo(address, function(err) { + err.message.should.equal('test'); + done(); + }); + }); + it('will call getOutputs and getInputs with the correct options', function() { + var startTimestamp = 1438289011844; + var endTimestamp = 1438289012412; + var expectedArgs = { + start: new Date(startTimestamp * 1000), + end: new Date(endTimestamp * 1000), + queryMempool: true + }; + var history = new AddressHistory({ + node: { + services: { + address: { + getOutputs: sinon.stub().callsArgWith(2, null, []), + getInputs: sinon.stub().callsArgWith(2, null, []) + } + } + }, + options: { + start: new Date(startTimestamp * 1000), + end: new Date(endTimestamp * 1000), + queryMempool: true + }, + addresses: [] + }); + history.transactionInfo = [{}]; + history.getTransactionInfo(address, function(err) { + if (err) { + throw err; + } + history.node.services.address.getOutputs.args[0][1].should.deep.equal(expectedArgs); + history.node.services.address.getInputs.args[0][1].should.deep.equal(expectedArgs); + }); + }); + it('will handle empty results from getOutputs and getInputs', function() { + var history = new AddressHistory({ + node: { + services: { + address: { + getOutputs: sinon.stub().callsArgWith(2, null, []), + getInputs: sinon.stub().callsArgWith(2, null, []) + } + } + }, + options: {}, + addresses: [] + }); + history.transactionInfo = [{}]; + history.getTransactionInfo(address, function(err) { + if (err) { + throw err; + } + history.transactionInfo.length.should.equal(1); + history.node.services.address.getOutputs.args[0][0].should.equal(address); + }); + }); + it('will concatenate outputs and inputs', function() { + var history = new AddressHistory({ + node: { + services: { + address: { + getOutputs: sinon.stub().callsArgWith(2, null, [{}]), + getInputs: sinon.stub().callsArgWith(2, null, [{}]) + } + } + }, + options: {}, + addresses: [] + }); + history.transactionInfo = [{}]; + history.getTransactionInfo(address, function(err) { + if (err) { + throw err; + } + history.transactionInfo.length.should.equal(3); + history.node.services.address.getOutputs.args[0][0].should.equal(address); + }); + }); + }); + + describe('@sortByHeight', function() { + it('will sort latest to oldest using height', function() { + var transactionInfo = [ + { + height: 12 + }, + { + height: 14, + }, + { + height: 13 + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].height.should.equal(14); + transactionInfo[1].height.should.equal(13); + transactionInfo[2].height.should.equal(12); + }); + }); + + describe('#paginateSortedArray', function() { + it('from 0 to 2', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 0, + to: 2 + }, + addresses: [] + }); + history.sortedArray = [ + { + height: 14 + }, + { + height: 13, + }, + { + height: 12 + } + ]; + history.paginateSortedArray(); + history.sortedArray.length.should.equal(2); + history.sortedArray[0].height.should.equal(14); + history.sortedArray[1].height.should.equal(13); + }); + it('from 0 to 4 (exceeds length)', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 0, + to: 4 + }, + addresses: [] + }); + history.sortedArray = [ + { + height: 14 + }, + { + height: 13, + }, + { + height: 12 + } + ]; + history.paginateSortedArray(); + history.sortedArray.length.should.equal(3); + history.sortedArray[0].height.should.equal(14); + history.sortedArray[1].height.should.equal(13); + history.sortedArray[2].height.should.equal(12); + }); + it('from 0 to 1', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 0, + to: 1 + }, + addresses: [] + }); + history.sortedArray = [ + { + height: 14 + }, + { + height: 13, + }, + { + height: 12 + } + ]; + history.paginateSortedArray(); + history.sortedArray.length.should.equal(1); + history.sortedArray[0].height.should.equal(14); + }); + it('from 2 to 3', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 2, + to: 3 + }, + addresses: [] + }); + history.sortedArray = [ + { + height: 14 + }, + { + height: 13, + }, + { + height: 12 + } + ]; + history.paginateSortedArray(); + history.sortedArray.length.should.equal(1); + history.sortedArray[0].height.should.equal(12); + }); + it('from 10 to 20 (out of range)', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 10, + to: 20 + }, + addresses: [] + }); + history.sortedArray = [ + { + height: 14 + }, + { + height: 13, + }, + { + height: 12 + } + ]; + history.paginateSortedArray(); + history.sortedArray.length.should.equal(0); + }); + }); + + describe('#getDetailedInfo', function() { + it('will add additional information to existing this.transactions', function() { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var history = new AddressHistory({ + node: { + services: { + db: { + getTransactionWithBlockInfo: sinon.stub() + } + } + }, + options: {}, + addresses: [] + }); + history.transactions[txid] = {}; + history.amendDetailedInfoWithSatoshis = sinon.stub(); + history.getDetailedInfo(txid, function(err) { + if (err) { + throw err; + } + history.amendDetailedInfoWithSatoshis.callCount.should.equal(1); + history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); + }); + }); + it('will handle error from getTransactionFromBlock', function() { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var history = new AddressHistory({ + node: { + services: { + db: { + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, new Error('test')), + } + } + }, + options: {}, + addresses: [] + }); + history.getDetailedInfo(txid, function(err) { + err.message.should.equal('test'); + }); + }); + it('will handle error from populateInputs', function() { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var history = new AddressHistory({ + node: { + services: { + db: { + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, { + populateInputs: sinon.stub().callsArgWith(2, new Error('test')) + }), + } + } + }, + options: {}, + addresses: [] + }); + history.getDetailedInfo(txid, function(err) { + err.message.should.equal('test'); + }); + }); + it('will set this.transactions with correct information', function() { + // block #314159 + // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e + // outputIndex 1 + var txAddress = '1Cj4UZWnGWAJH1CweTMgPLQMn26WRMfXmo'; + var txString = '0100000001a08ee59fcd5d86fa170abb6d925d62d5c5c476359681b70877c04f270c4ef246000000008a47304402203fb9b476bb0c37c9b9ed5784ebd67ae589492be11d4ae1612be29887e3e4ce750220741ef83781d1b3a5df8c66fa1957ad0398c733005310d7d9b1d8c2310ef4f74c0141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02b0a75fac000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac20badc02000000001976a914809dc14496f99b6deb722cf46d89d22f4beb8efd88ac00000000'; + var previousTxString = '010000000155532fad2869bb951b0bd646a546887f6ee668c4c0ee13bf3f1c4bce6d6e3ed9000000008c4930460221008540795f4ef79b1d2549c400c61155ca5abbf3089c84ad280e1ba6db2a31abce022100d7d162175483d51174d40bba722e721542c924202a0c2970b07e680b51f3a0670141046516ad02713e51ecf23ac9378f1069f9ae98e7de2f2edbf46b7836096e5dce95a05455cc87eaa1db64f39b0c63c0a23a3b8df1453dbd1c8317f967c65223cdf8ffffffff02f0af3caf000000001976a91484b45b9bf3add8f7a0f3daad305fdaf6b73441ea88ac80969800000000001976a91421277e65777760d1f3c7c982ba14ed8f934f005888ac00000000'; + var transaction = new Transaction(); + var previousTransaction = new Transaction(); + previousTransaction.fromString(previousTxString); + var previousTransactionTxid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + transaction.fromString(txString); + var txid = transaction.hash; + transaction.__blockHash = '00000000000000001bb82a7f5973618cfd3185ba1ded04dd852a653f92a27c45'; + transaction.__height = 314159; + transaction.__timestamp = 1407292005; + var history = new AddressHistory({ + node: { + services: { + db: { + tip: { + __height: 314159 + }, + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, transaction), + getTransaction: function(prevTxid, queryMempool, callback) { + prevTxid.should.equal(previousTransactionTxid); + setImmediate(function() { + callback(null, previousTransaction); + }); + } + } + } + }, + options: {}, + addresses: [] + }); + var transactionInfo = { + txid: txid, + timestamp: 1407292005, + outputIndex: 1, + satoshis: 48020000, + address: txAddress + }; + history.getDetailedInfo(transactionInfo, function(err) { + if (err) { + throw err; + } + var info = history.transactions[txAddress][txid]; + info.address.should.equal(txAddress); + info.satoshis.should.equal(48020000); + info.height.should.equal(314159); + info.confirmations.should.equal(1); + info.timestamp.should.equal(1407292005); + info.fees.should.equal(20000); + info.outputIndexes.should.deep.equal([1]); + info.inputIndexes.should.deep.equal([]); + info.tx.should.equal(transaction); + }); + }); + }); + + describe('#amendDetailedInfoWithSatoshis', function() { + it('will amend info with inputIndex and subtract satoshis', function() { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: [] + }); + history.transactions[address] = {}; + history.transactions[address][txid] = { + inputIndexes: [], + satoshis: 10, + tx: { + inputs: [ + { + output: { + satoshis: 3000 + } + } + ] + } + }; + history.amendDetailedInfoWithSatoshis({ + address: address, + txid: txid, + inputIndex: 0 + }); + history.transactions[address][txid].inputIndexes.should.deep.equal([0]); + history.transactions[address][txid].satoshis.should.equal(-2990); + }); + it('will amend info with outputIndex and add satoshis', function() { + var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: [] + }); + history.transactions[address] = {}; + history.transactions[address][txid] = { + outputIndexes: [], + satoshis: 10 + }; + history.amendDetailedInfoWithSatoshis({ + address: address, + txid: txid, + outputIndex: 10, + satoshis: 2000 + }); + history.transactions[address][txid].outputIndexes.should.deep.equal([10]); + history.transactions[address][txid].satoshis.should.equal(2010); + }); + }); + + describe('#sortTransactionIntoArray', function() { + it('will convert this.transactions into an array and sort by height', function() { + var history = new AddressHistory({ + node: {}, + options: { + from: 10, + to: 20 + }, + addresses: [] + }); + history.transactions = { + address1: { + txid1: { + height: 12 + }, + txid2: { + height: 14, + }, + txid3: { + height: 13 + } + }, + address2: { + txid4: { + height: 15 + } + } + }; + history.sortTransactionsIntoArray(); + history.sortedArray.length.should.equal(4); + history.sortedArray[0].height.should.equal(15); + history.sortedArray[1].height.should.equal(14); + history.sortedArray[2].height.should.equal(13); + history.sortedArray[3].height.should.equal(12); + }); + }); +}); diff --git a/test/services/address.unit.js b/test/services/address/index.unit.js similarity index 71% rename from test/services/address.unit.js rename to test/services/address/index.unit.js index 2b7c2a89..79c31287 100644 --- a/test/services/address.unit.js +++ b/test/services/address/index.unit.js @@ -2,14 +2,14 @@ var should = require('chai').should(); var sinon = require('sinon'); -var bitcorenode = require('../../'); +var proxyquire = require('proxyquire'); +var bitcorenode = require('../../../'); var AddressService = bitcorenode.services.Address; -var blockData = require('../data/livenet-345003.json'); +var blockData = require('../../data/livenet-345003.json'); var bitcore = require('bitcore'); var Networks = bitcore.Networks; var EventEmitter = require('events').EventEmitter; var errors = bitcorenode.errors; -var levelup = require('levelup'); var mockdb = { }; @@ -110,7 +110,7 @@ describe('Address Service', function() { { key: { address: '1F1MAvhTKg2VG29w8cXsiSN2PJ8gSsrJw', - timestamp: 1424836934, + height: 345003, txid: 'fdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e16923', outputIndex: 0 }, @@ -122,19 +122,20 @@ describe('Address Service', function() { }, { key: { + address: '1Q8ec8kG7c7HqgK7uSzQyWsX9tzepRcKEL', + height: 345003, prevTxId: '3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a9', prevOutputIndex: 32 }, value: { txid: '5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca', - inputIndex: 0, - timestamp: 1424836934 + inputIndex: 0 } }, { key: { address: '1Ep5LA4T6Y7zaBPiwruUJurjGFvCJHzJhm', - timestamp: 1424836934, + height: 345003, txid: 'e66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d', outputIndex: 1 }, @@ -170,17 +171,17 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('put'); - var expected0 = ['outs', key0.address, key0.timestamp, key0.txid, key0.outputIndex].join('-'); + var expected0 = ['outs', key0.address, key0.height, key0.txid, key0.outputIndex].join('-'); operations[0].key.should.equal(expected0); - operations[0].value.should.equal([value0.satoshis, value0.script, value0.blockHeight].join(':')); + operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); operations[3].type.should.equal('put'); - var expected3 = ['sp', key3.prevTxId, key3.prevOutputIndex].join('-'); + var expected3 = ['sp', key3.address, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-'); operations[3].key.should.equal(expected3); operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); operations[64].type.should.equal('put'); - var expected64 = ['outs', key64.address, key64.timestamp, key64.txid, key64.outputIndex].join('-'); + var expected64 = ['outs', key64.address, key64.height, key64.txid, key64.outputIndex].join('-'); operations[64].key.should.equal(expected64); - operations[64].value.should.equal([value64.satoshis, value64.script, value64.blockHeight].join(':')); + operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); done(); }); }); @@ -196,14 +197,14 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('del'); - operations[0].key.should.equal(['outs', key0.address, key0.timestamp, key0.txid, key0.outputIndex].join('-')); - operations[0].value.should.equal([value0.satoshis, value0.script, value0.blockHeight].join(':')); + operations[0].key.should.equal(['outs', key0.address, key0.height, key0.txid, key0.outputIndex].join('-')); + operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); operations[3].type.should.equal('del'); - operations[3].key.should.equal(['sp', key3.prevTxId, key3.prevOutputIndex].join('-')); + operations[3].key.should.equal(['sp', key3.address, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-')); operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); operations[64].type.should.equal('del'); - operations[64].key.should.equal(['outs', key64.address, key64.timestamp, key64.txid, key64.outputIndex].join('-')); - operations[64].value.should.equal([value64.satoshis, value64.script, value64.blockHeight].join(':')); + operations[64].key.should.equal(['outs', key64.address, key64.height, key64.txid, key64.outputIndex].join('-')); + operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); done(); }); }); @@ -421,6 +422,116 @@ describe('Address Service', function() { }); + describe('#getInputs', function() { + var am; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var db = { + tip: { + __height: 1 + } + }; + var testnode = { + services: { + db: db, + bitcoind: { + on: sinon.stub() + } + } + }; + before(function() { + am = new AddressService({node: testnode}); + }); + + it('will get inputs for an address and timestamp', function(done) { + var testStream = new EventEmitter(); + var args = { + start: 15, + end: 12, + queryMempool: true + }; + var createReadStreamCallCount = 0; + am.node.services.db.store = { + createReadStream: function(ops) { + ops.start.should.equal([AddressService.PREFIXES.SPENTS, address, 12].join('-')); + ops.end.should.equal([AddressService.PREFIXES.SPENTS, address, 16].join('-')); + createReadStreamCallCount++; + return testStream; + } + }; + am.node.services.bitcoind = { + getMempoolInputs: sinon.stub().returns([]) + }; + am.getInputs(address, args, function(err, inputs) { + should.not.exist(err); + inputs.length.should.equal(1); + inputs[0].address.should.equal(address); + inputs[0].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); + inputs[0].inputIndex.should.equal(0); + inputs[0].height.should.equal(15); + done(); + }); + createReadStreamCallCount.should.equal(1); + var data = { + key: ['sp', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), + value: ['3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '0'].join(':') + }; + testStream.emit('data', data); + testStream.emit('close'); + }); + it('should get inputs for address', function(done) { + var testStream = new EventEmitter(); + var args = { + queryMempool: true + }; + var createReadStreamCallCount = 0; + am.node.services.db.store = { + createReadStream: function(ops) { + ops.start.should.equal([AddressService.PREFIXES.SPENTS, address].join('-')); + ops.end.should.equal([AddressService.PREFIXES.SPENTS, address].join('-') + '~'); + createReadStreamCallCount++; + return testStream; + } + }; + am.node.services.bitcoind = { + getMempoolInputs: sinon.stub().returns([]) + }; + am.getInputs(address, args, function(err, inputs) { + should.not.exist(err); + inputs.length.should.equal(1); + inputs[0].address.should.equal(address); + inputs[0].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); + inputs[0].inputIndex.should.equal(0); + inputs[0].height.should.equal(15); + done(); + }); + createReadStreamCallCount.should.equal(1); + var data = { + key: ['sp', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), + value: ['3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '0'].join(':') + }; + testStream.emit('data', data); + testStream.emit('close'); + }); + it('should give an error if the readstream has an error', function(done) { + var testStream = new EventEmitter(); + am.node.services.db.store = { + createReadStream: sinon.stub().returns(testStream) + }; + + am.getOutputs(address, {}, function(err, outputs) { + should.exist(err); + err.message.should.equal('readstreamerror'); + done(); + }); + + testStream.emit('error', new Error('readstreamerror')); + setImmediate(function() { + testStream.emit('close'); + }); + }); + + }); + describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; @@ -437,11 +548,53 @@ describe('Address Service', function() { } } }; + var options = { + queryMempool: true + }; before(function() { am = new AddressService({node: testnode}); }); + it('will get outputs for an address and timestamp', function(done) { + var testStream = new EventEmitter(); + var args = { + start: 15, + end: 12, + queryMempool: true + }; + var createReadStreamCallCount = 0; + am.node.services.db.store = { + createReadStream: function(ops) { + ops.start.should.equal([AddressService.PREFIXES.OUTPUTS, address, 12].join('-')); + ops.end.should.equal([AddressService.PREFIXES.OUTPUTS, address, 16].join('-')); + createReadStreamCallCount++; + return testStream; + } + }; + am.node.services.bitcoind = { + getMempoolOutputs: sinon.stub().returns([]) + }; + am.getOutputs(address, args, function(err, outputs) { + should.not.exist(err); + outputs.length.should.equal(1); + outputs[0].address.should.equal(address); + outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); + outputs[0].outputIndex.should.equal(1); + outputs[0].satoshis.should.equal(4527773864); + outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); + outputs[0].height.should.equal(15); + done(); + }); + createReadStreamCallCount.should.equal(1); + var data = { + key: ['outs', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), + value: ['4527773864', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') + }; + testStream.emit('data', data); + testStream.emit('close'); + }); + it('should get outputs for an address', function(done) { var readStream1 = new EventEmitter(); am.node.services.db.store = { @@ -452,7 +605,7 @@ describe('Address Service', function() { address: '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W', txid: 'aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371', satoshis: 307627737, - script: 'OP_DUP OP_HASH160 f6db95c81dea3d10f0ff8d890927751bf7b203c1 OP_EQUALVERIFY OP_CHECKSIG', + script: '76a914f6db95c81dea3d10f0ff8d890927751bf7b203c188ac', blockHeight: 352532 } ]; @@ -460,38 +613,36 @@ describe('Address Service', function() { getMempoolOutputs: sinon.stub().returns(mempoolOutputs) }; - am.getOutputs(address, true, function(err, outputs) { + am.getOutputs(address, options, function(err, outputs) { should.not.exist(err); outputs.length.should.equal(3); outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); - outputs[0].script.should.equal('OP_DUP OP_HASH160 038a213afdfc551fc658e9a2a58a86e98d69b687 OP_EQUALVERIFY OP_CHECKSIG'); - outputs[0].blockHeight.should.equal(345000); - outputs[0].timestamp.should.equal(1424835319000); + outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); + outputs[0].height.should.equal(345000); outputs[1].address.should.equal(address); outputs[1].txid.should.equal('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'); outputs[1].outputIndex.should.equal(2); outputs[1].satoshis.should.equal(10000); - outputs[1].script.should.equal('OP_DUP OP_HASH160 038a213afdfc551fc658e9a2a58a86e98d69b687 OP_EQUALVERIFY OP_CHECKSIG'); - outputs[1].blockHeight.should.equal(345004); - outputs[1].timestamp.should.equal(1424837300000); + outputs[1].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); + outputs[1].height.should.equal(345004); outputs[2].address.should.equal(address); outputs[2].txid.should.equal('aa2db23f670596e96ed94c405fd11848c8f236d266ee96da37ecd919e53b4371'); - outputs[2].script.should.equal('OP_DUP OP_HASH160 f6db95c81dea3d10f0ff8d890927751bf7b203c1 OP_EQUALVERIFY OP_CHECKSIG'); + outputs[2].script.should.equal('76a914f6db95c81dea3d10f0ff8d890927751bf7b203c188ac'); outputs[2].blockHeight.should.equal(352532); done(); }); var data1 = { - key: ['outs', address, 1424835319000, '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), - value: ['4527773864', 'OP_DUP OP_HASH160 038a213afdfc551fc658e9a2a58a86e98d69b687 OP_EQUALVERIFY OP_CHECKSIG', '345000'].join(':') + key: ['outs', address, 345000, '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), + value: ['4527773864', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') }; var data2 = { - key: ['outs', address, 1424837300000, '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '2'].join('-'), - value: ['10000', 'OP_DUP OP_HASH160 038a213afdfc551fc658e9a2a58a86e98d69b687 OP_EQUALVERIFY OP_CHECKSIG', '345004'].join(':') + key: ['outs', address, 345004, '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '2'].join('-'), + value: ['10000', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') }; readStream1.emit('data', data1); @@ -505,14 +656,14 @@ describe('Address Service', function() { createReadStream: sinon.stub().returns(readStream2) }; - am.getOutputs(address, true, function(err, outputs) { + am.getOutputs(address, options, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); }); readStream2.emit('error', new Error('readstreamerror')); - process.nextTick(function() { + setImmediate(function() { readStream2.emit('close'); }); }); @@ -731,245 +882,20 @@ describe('Address Service', function() { }); }); - describe('#getSpendInfoForOutput', function() { - it('should call store.get the right values', function(done) { - var db = { - store: { - get: sinon.stub().callsArgWith(1, null, 'spendtxid:1') - } - }; - var testnode = { - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({node: testnode}); - am.getSpendInfoForOutput('txid', 3, function(err, info) { - should.not.exist(err); - info.txid.should.equal('spendtxid'); - info.inputIndex.should.equal('1'); - db.store.get.args[0][0].should.equal('sp-txid-3'); - done(); - }); - }); - }); - describe('#getAddressHistory', function() { - var incoming = [ - { - txid: 'tx1', - outputIndex: 0, - spentTx: 'tx2', - inputIndex: 0, - height: 1, - timestamp: 1438289011844, - satoshis: 5000, - getFee: sinon.stub().throws(new Error('inputs not populated')), - isCoinbase: sinon.stub().returns(true) - }, - { - txid: 'tx3', - outputIndex: 1, - height: 3, - timestamp: 1438289031844, - satoshis: 2000, - getFee: sinon.stub().returns(1000), - isCoinbase: sinon.stub().returns(false) - }, - { - txid: 'tx4', - outputIndex: 2, - spentTx: 'tx5', - inputIndex: 1, - height: 4, - timestamp: 1438289041844, - satoshis: 3000, - getFee: sinon.stub().returns(1000), - isCoinbase: sinon.stub().returns(false) - }, - ]; - - var outgoing = [ - { - txid: 'tx2', - height: 2, - timestamp: 1438289021844, - inputs: [ - { - output: { - satoshis: 5000 - } - } - ], - getFee: sinon.stub().returns(1000), - isCoinbase: sinon.stub().returns(false) - }, - { - txid: 'tx5', - height: 5, - timestamp: 1438289051844, - inputs: [ - {}, - { - output: { - satoshis: 3000 - } - } - ], - getFee: sinon.stub().returns(1000), - isCoinbase: sinon.stub().returns(false) + it('will call get on address history instance', function(done) { + function TestAddressHistory(args) { + args.node.should.equal(mocknode); + args.addresses.should.deep.equal([]); + args.options.should.deep.equal({}); } - ]; - - var db = { - tip: { - __height: 1 - }, - getTransactionWithBlockInfo: function(txid, queryMempool, callback) { - var transaction = { - populateInputs: sinon.stub().callsArg(2) - }; - for(var i = 0; i < incoming.length; i++) { - if(incoming[i].txid === txid) { - if(incoming[i].error) { - return callback(new Error(incoming[i].error)); - } - transaction.hash = txid; - transaction.__height = incoming[i].height; - transaction.__timestamp = incoming[i].timestamp; - transaction.getFee = incoming[i].getFee; - transaction.isCoinbase = incoming[i].isCoinbase; - return callback(null, transaction); - } - } - - for(var i = 0; i < outgoing.length; i++) { - if(outgoing[i].txid === txid) { - if(outgoing[i].error) { - return callback(new Error(outgoing[i].error)); - } - transaction.hash = txid; - transaction.__height = outgoing[i].height; - transaction.__timestamp = outgoing[i].timestamp; - transaction.inputs = outgoing[i].inputs; - transaction.getFee = outgoing[i].getFee; - transaction.isCoinbase = outgoing[i].isCoinbase; - return callback(null, transaction); - } - } - callback(new Error('tx ' + txid + ' not found')); - } - }; - var testnode = { - services: { - db: db, - bitcoind: { - on: sinon.stub() - } - } - }; - var am = new AddressService({node: testnode}); - - am.getOutputs = sinon.stub().callsArgWith(2, null, incoming); - am.getSpendInfoForOutput = function(txid, outputIndex, callback) { - for(var i = 0; i < incoming.length; i++) { - if(incoming[i].txid === txid && incoming[i].outputIndex === outputIndex && incoming[i].spentTx) { - if(incoming[i].spendError) { - return callback(new Error(incoming[i].spendError)); - } - return callback(null, { - txid: incoming[i].spentTx, - inputIndex: incoming[i].inputIndex - }); - } - } - - callback(new levelup.errors.NotFoundError()); - }; - - it('should give transaction history for an address', function(done) { - am.getAddressHistory('address', true, function(err, history) { - should.not.exist(err); - history[0].tx.hash.should.equal('tx1'); - history[0].satoshis.should.equal(5000); - history[0].height.should.equal(1); - history[0].timestamp.should.equal(1438289011844); - should.equal(history[0].fees, null); - history[1].tx.hash.should.equal('tx2'); - history[1].satoshis.should.equal(-5000); - history[1].height.should.equal(2); - history[1].timestamp.should.equal(1438289021844); - history[1].fees.should.equal(1000); - history[2].tx.hash.should.equal('tx3'); - history[2].satoshis.should.equal(2000); - history[2].height.should.equal(3); - history[2].timestamp.should.equal(1438289031844); - history[2].fees.should.equal(1000); - history[3].tx.hash.should.equal('tx4'); - history[3].satoshis.should.equal(3000); - history[3].height.should.equal(4); - history[3].timestamp.should.equal(1438289041844); - history[3].fees.should.equal(1000); - history[4].tx.hash.should.equal('tx5'); - history[4].satoshis.should.equal(-3000); - history[4].height.should.equal(5); - history[4].timestamp.should.equal(1438289051844); - history[4].fees.should.equal(1000); - done(); + TestAddressHistory.prototype.get = sinon.stub().callsArg(0); + var TestAddressService = proxyquire('../../../lib/services/address', { + './history': TestAddressHistory }); - }); - - it('should give an error if the second getTransactionInfo gives an error', function(done) { - outgoing[0].error = 'txinfo2err'; - am.getAddressHistory('address', true, function(err, history) { - should.exist(err); - err.message.should.equal('txinfo2err'); - outgoing[0].error = null; - done(); - }); - }); - - it('should give an error if getSpendInfoForOutput gives an error', function(done) { - incoming[0].spendError = 'spenderr'; - am.getAddressHistory('address', true, function(err, history) { - should.exist(err); - err.message.should.equal('spenderr'); - incoming[0].spendError = null; - done(); - }); - }); - - it('should give an error if the first getTransactionInfo gives an error', function(done) { - incoming[1].error = 'txinfo1err'; - am.getAddressHistory('address', true, function(err, history) { - should.exist(err); - err.message.should.equal('txinfo1err'); - incoming[1].error = null; - done(); - }); - }); - - it('should give an error if populateInputs gives an error', function(done) { - var populateStub = sinon.stub().callsArgWith(2, new Error('populateerr')); - sinon.stub(db, 'getTransactionWithBlockInfo').callsArgWith(2, null, { - populateInputs: populateStub - }); - am.getAddressHistory('address', true, function(err, history) { - should.exist(err); - err.message.should.equal('populateerr'); - db.getTransactionWithBlockInfo.restore(); - done(); - }); - }); - - it('should give an error if getOutputs gives an error', function(done) { - am.getOutputs = sinon.stub().callsArgWith(2, new Error('getoutputserr')); - am.getAddressHistory('address', true, function(err, history) { - should.exist(err); - err.message.should.equal('getoutputserr'); + var am = new TestAddressService({node: mocknode}); + am.getAddressHistory([], {}, function(err, history) { + TestAddressHistory.prototype.get.callCount.should.equal(1); done(); }); }); From 1cf34f2dd857a23e249d0af919b3b7a87dbf842e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 11 Sep 2015 20:58:59 -0400 Subject: [PATCH 02/10] Address Block Handling Optimizations - Changes to use ripemd160 hash directly instead of the base58check encoded values - Speeds block handling performance by ~4 times --- lib/services/address/index.js | 203 ++++++++++++++++++---------- test/services/address/index.unit.js | 125 +++++++++-------- 2 files changed, 201 insertions(+), 127 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 12eda69a..82eb4527 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -9,6 +9,7 @@ var errors = index.errors; var bitcore = require('bitcore'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; +var Hash = bitcore.crypto.Hash; var EventEmitter = require('events').EventEmitter; var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; @@ -81,23 +82,21 @@ AddressService.prototype.transactionOutputHandler = function(messages, tx, outpu return; } - // Find the address for the output - var address = script.toAddress(this.node.network); - if (!address && script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); - } else if (!address){ + var addressInfo = this._extractAddressInfoFromScript(script); + if (!addressInfo) { return; } + addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); + // Collect data to publish to address subscribers - if (messages[address]) { - messages[address].outputIndexes.push(outputIndex); + if (messages[addressInfo.hashHex]) { + messages[addressInfo.hashHex].outputIndexes.push(outputIndex); } else { - messages[address] = { + messages[addressInfo.hashHex] = { tx: tx, outputIndexes: [outputIndex], - address: address.toString(), + addressInfo: addressInfo, rejected: rejected }; } @@ -130,6 +129,28 @@ AddressService.prototype.transactionHandler = function(txInfo) { } }; +AddressService.prototype._extractAddressInfoFromScript = function(script) { + var hashBuffer; + var addressType; + if (script.isPublicKeyHashOut()) { + hashBuffer = script.chunks[2].buf; + addressType = Address.PayToPublicKeyHash; + } else if (script.isScriptHashOut()) { + hashBuffer = script.chunks[1].buf; + addressType = Address.PayToScriptHash; + } else if (script.isPublicKeyOut()) { + var pubkey = script.chunks[0].buf; + var address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); + hashBuffer = address.hashBuffer; + } else { + return false; + } + return { + hashBuffer: hashBuffer, + addressType: addressType + }; +}; + AddressService.prototype.blockHandler = function(block, addOutput, callback) { var txs = block.transactions; @@ -162,14 +183,13 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - var address = script.toAddress(this.node.network); - if (!address && script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); - } else if (!address){ + var addressInfo = this._extractAddressInfoFromScript(script); + if (!addressInfo) { continue; } + addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); + // We need to use the height for indexes (and not the timestamp) because the // the timestamp has unreliable sequential ordering. The next block // can have a time that is previous to the previous block (however not @@ -177,18 +197,19 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // hours in the future. var height = block.__height; - var addressStr = address.toString(); var scriptHex = output._scriptBuffer.toString('hex'); // To lookup outputs by address and height var key = [ AddressService.PREFIXES.OUTPUTS, - addressStr, + addressInfo.hashHex, height, txid, outputIndex ].join('-'); + // TODO use buffers directly to save on disk storage + var value = [output.satoshis, scriptHex].join(':'); operations.push({ @@ -198,19 +219,19 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); // Collect data for subscribers - if (txmessages[addressStr]) { - txmessages[addressStr].outputIndexes.push(outputIndex); + if (txmessages[addressInfo.hashHex]) { + txmessages[addressInfo.hashHex].outputIndexes.push(outputIndex); } else { - txmessages[addressStr] = { + txmessages[addressInfo.hashHex] = { tx: tx, height: block.__height, outputIndexes: [outputIndex], - address: addressStr, + addressInfo: addressInfo, timestamp: block.header.timestamp }; } - this.balanceEventHandler(block, address); + this.balanceEventHandler(block, addressInfo); } @@ -226,35 +247,36 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { var input = inputs[inputIndex]; - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { + var inputHashBuffer; - var inputObject = input.toObject(); - var inputAddressStr = inputAddress.toString(); - - var height = block.__height; - - // To be able to query inputs by address and spent height - var inputKey = [ - AddressService.PREFIXES.SPENTS, - inputAddressStr, - height, - inputObject.prevTxId, - inputObject.outputIndex - ].join('-'); - - var inputValue = [ - txid, - inputIndex - ].join(':'); - - operations.push({ - type: action, - key: inputKey, - value: inputValue - }); + if (input.script.isPublicKeyHashIn()) { + inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); + } else if (input.script.isScriptHashIn()) { + inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); + } else { + continue; } + + // To be able to query inputs by address and spent height + var inputKey = [ + AddressService.PREFIXES.SPENTS, + inputHashBuffer.toString('hex'), + block.__height, + input.prevTxId.toString('hex'), + input.outputIndex + ].join('-'); + + var inputValue = [ + txid, + inputIndex + ].join(':'); + + operations.push({ + type: action, + key: inputKey, + value: inputValue + }); } } @@ -266,30 +288,57 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { /** * @param {Object} obj * @param {Transaction} obj.tx - The transaction - * @param {String} [obj.address] - The address for the subscription - * @param {Array} [obj.outputIndexes] - Indexes of the inputs that includes the address - * @param {Array} [obj.inputIndexes] - Indexes of the outputs that includes the address - * @param {Date} [obj.timestamp] - The time of the block the transaction was included - * @param {Number} [obj.height] - The height of the block the transaction was included - * @param {Boolean} [obj.rejected] - If the transaction was not accepted in the mempool + * @param {Object} obj.addressInfo + * @param {String} obj.addressInfo.hashHex - The hex string of address hash for the subscription + * @param {String} obj.addressInfo.hashBuffer - The address hash buffer + * @param {String} obj.addressInfo.addressType - The address type + * @param {Array} obj.outputIndexes - Indexes of the inputs that includes the address + * @param {Array} obj.inputIndexes - Indexes of the outputs that includes the address + * @param {Date} obj.timestamp - The time of the block the transaction was included + * @param {Number} obj.height - The height of the block the transaction was included + * @param {Boolean} obj.rejected - If the transaction was not accepted in the mempool */ AddressService.prototype.transactionEventHandler = function(obj) { - if(this.subscriptions['address/transaction'][obj.address]) { - var emitters = this.subscriptions['address/transaction'][obj.address]; + if(this.subscriptions['address/transaction'][obj.addressInfo.hashHex]) { + var emitters = this.subscriptions['address/transaction'][obj.addressInfo.hashHex]; + var address = new Address({ + hashBuffer: obj.addressInfo.hashBuffer, + network: this.node.network, + type: obj.addressInfo.addressType + }); for(var i = 0; i < emitters.length; i++) { - emitters[i].emit('address/transaction', obj); + emitters[i].emit('address/transaction', { + rejected: obj.rejected, + height: obj.height, + timestamp: obj.timestamp, + inputIndexes: obj.inputIndexes, + outputIndexes: obj.outputIndexes, + address: address, + tx: obj.tx + }); } } }; -AddressService.prototype.balanceEventHandler = function(block, address) { - if(this.subscriptions['address/balance'][address]) { - var emitters = this.subscriptions['address/balance'][address]; +/** + * @param {Block} block + * @param {Object} obj + * @param {String} obj.hashHex + * @param {Buffer} obj.hashBuffer + * @param {String} obj.addressType + */ +AddressService.prototype.balanceEventHandler = function(block, obj) { + if(this.subscriptions['address/balance'][obj.hashHex]) { + var emitters = this.subscriptions['address/balance'][obj.hashHex]; + var address = new Address({ + hashBuffer: obj.hashBuffer, + network: this.node.network, + type: obj.addressType + }); this.getBalance(address, true, function(err, balance) { if(err) { return this.emit(err); } - for(var i = 0; i < emitters.length; i++) { emitters[i].emit('address/balance', address, balance, block); } @@ -302,10 +351,11 @@ AddressService.prototype.subscribe = function(name, emitter, addresses) { $.checkArgument(Array.isArray(addresses), 'Second argument is expected to be an Array of addresses'); for(var i = 0; i < addresses.length; i++) { - if(!this.subscriptions[name][addresses[i]]) { - this.subscriptions[name][addresses[i]] = []; + var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); + if(!this.subscriptions[name][hashHex]) { + this.subscriptions[name][hashHex] = []; } - this.subscriptions[name][addresses[i]].push(emitter); + this.subscriptions[name][hashHex].push(emitter); } }; @@ -318,8 +368,9 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { } for(var i = 0; i < addresses.length; i++) { - if(this.subscriptions[name][addresses[i]]) { - var emitters = this.subscriptions[name][addresses[i]]; + var hashHex = bitcore.Address(addresses[i]).hashBuffer.toString('hex'); + if(this.subscriptions[name][hashHex]) { + var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); if(index > -1) { emitters.splice(index, 1); @@ -331,8 +382,8 @@ AddressService.prototype.unsubscribe = function(name, emitter, addresses) { AddressService.prototype.unsubscribeAll = function(name, emitter) { $.checkArgument(emitter instanceof EventEmitter, 'First argument is expected to be an EventEmitter'); - for(var address in this.subscriptions[name]) { - var emitters = this.subscriptions[name][address]; + for(var hashHex in this.subscriptions[name]) { + var emitters = this.subscriptions[name][hashHex]; var index = emitters.indexOf(emitter); if(index > -1) { emitters.splice(index, 1); @@ -373,21 +424,23 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var inputs = []; var stream; + var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex'); + if (options.start && options.end) { // The positions will be flipped because the end position should be greater // than the starting position for the stream, and we'll add one to the end key // so that it's included in the results. - var endKey = [AddressService.PREFIXES.SPENTS, addressStr, options.start + 1].join('-'); - var startKey = [AddressService.PREFIXES.SPENTS, addressStr, options.end].join('-'); + var endKey = [AddressService.PREFIXES.SPENTS, hashHex, options.start + 1].join('-'); + var startKey = [AddressService.PREFIXES.SPENTS, hashHex, options.end].join('-'); stream = this.node.services.db.store.createReadStream({ start: startKey, end: endKey }); } else { - var allKey = [AddressService.PREFIXES.SPENTS, addressStr].join('-'); + var allKey = [AddressService.PREFIXES.SPENTS, hashHex].join('-'); stream = this.node.services.db.store.createReadStream({ start: allKey, end: allKey + '~' @@ -449,6 +502,8 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); + var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex'); + var outputs = []; var stream; @@ -457,15 +512,15 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { // The positions will be flipped because the end position should be greater // than the starting position for the stream, and we'll add one to the end key // so that it's included in the results. - var endKey = [AddressService.PREFIXES.OUTPUTS, addressStr, options.start + 1].join('-'); - var startKey = [AddressService.PREFIXES.OUTPUTS, addressStr, options.end].join('-'); + var endKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.start + 1].join('-'); + var startKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.end].join('-'); stream = this.node.services.db.store.createReadStream({ start: startKey, end: endKey }); } else { - var allKey = [AddressService.PREFIXES.OUTPUTS, addressStr].join('-'); + var allKey = [AddressService.PREFIXES.OUTPUTS, hashHex].join('-'); stream = this.node.services.db.store.createReadStream({ start: allKey, end: allKey + '~' diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 79c31287..468a75a9 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -74,13 +74,15 @@ describe('Address Service', function() { var am = new AddressService({node: mocknode}); am.node.network = Networks.livenet; var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var hashHex = bitcore.Address(address).hashBuffer.toString('hex'); var messages = {}; am.transactionOutputHandler(messages, tx, 0, true); - should.exist(messages[address]); - var message = messages[address]; + should.exist(messages[hashHex]); + var message = messages[hashHex]; message.tx.should.equal(tx); message.outputIndexes.should.deep.equal([0]); - message.address.should.equal(address); + message.addressInfo.hashBuffer.toString('hex').should.equal(hashHex); + message.addressInfo.hashHex.should.equal(hashHex); message.rejected.should.equal(true); }); }); @@ -109,7 +111,7 @@ describe('Address Service', function() { var data = [ { key: { - address: '1F1MAvhTKg2VG29w8cXsiSN2PJ8gSsrJw', + hashHex: bitcore.Address('1F1MAvhTKg2VG29w8cXsiSN2PJ8gSsrJw').hashBuffer.toString('hex'), height: 345003, txid: 'fdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e16923', outputIndex: 0 @@ -122,7 +124,7 @@ describe('Address Service', function() { }, { key: { - address: '1Q8ec8kG7c7HqgK7uSzQyWsX9tzepRcKEL', + hashHex: bitcore.Address('1Q8ec8kG7c7HqgK7uSzQyWsX9tzepRcKEL').hashBuffer.toString('hex'), height: 345003, prevTxId: '3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a9', prevOutputIndex: 32 @@ -134,7 +136,7 @@ describe('Address Service', function() { }, { key: { - address: '1Ep5LA4T6Y7zaBPiwruUJurjGFvCJHzJhm', + hashHex: bitcore.Address('1Ep5LA4T6Y7zaBPiwruUJurjGFvCJHzJhm').hashBuffer.toString('hex'), height: 345003, txid: 'e66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d', outputIndex: 1 @@ -171,15 +173,15 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('put'); - var expected0 = ['outs', key0.address, key0.height, key0.txid, key0.outputIndex].join('-'); + var expected0 = ['outs', key0.hashHex, key0.height, key0.txid, key0.outputIndex].join('-'); operations[0].key.should.equal(expected0); operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); operations[3].type.should.equal('put'); - var expected3 = ['sp', key3.address, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-'); + var expected3 = ['sp', key3.hashHex, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-'); operations[3].key.should.equal(expected3); operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); operations[64].type.should.equal('put'); - var expected64 = ['outs', key64.address, key64.height, key64.txid, key64.outputIndex].join('-'); + var expected64 = ['outs', key64.hashHex, key64.height, key64.txid, key64.outputIndex].join('-'); operations[64].key.should.equal(expected64); operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); done(); @@ -197,13 +199,13 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('del'); - operations[0].key.should.equal(['outs', key0.address, key0.height, key0.txid, key0.outputIndex].join('-')); + operations[0].key.should.equal(['outs', key0.hashHex, key0.height, key0.txid, key0.outputIndex].join('-')); operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); operations[3].type.should.equal('del'); - operations[3].key.should.equal(['sp', key3.address, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-')); + operations[3].key.should.equal(['sp', key3.hashHex, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-')); operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); operations[64].type.should.equal('del'); - operations[64].key.should.equal(['outs', key64.address, key64.height, key64.txid, key64.outputIndex].join('-')); + operations[64].key.should.equal(['outs', key64.hashHex, key64.height, key64.txid, key64.outputIndex].join('-')); operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); done(); }); @@ -277,16 +279,16 @@ describe('Address Service', function() { it('will emit a transaction if there is a subscriber', function(done) { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); - am.subscriptions['address/transaction'] = { - '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N': [emitter] - }; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + am.subscriptions['address/transaction'] = {}; + am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] = [emitter]; var block = { __height: 0, timestamp: new Date() }; var tx = {}; emitter.on('address/transaction', function(obj) { - obj.address.should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + obj.address.toString().should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); obj.tx.should.equal(tx); obj.timestamp.should.equal(block.timestamp); obj.height.should.equal(block.__height); @@ -294,7 +296,11 @@ describe('Address Service', function() { done(); }); am.transactionEventHandler({ - address: '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N', + addressInfo: { + hashHex: address.hashBuffer.toString('hex'), + hashBuffer: address.hashBuffer, + addressType: address.type + }, height: block.__height, timestamp: block.timestamp, outputIndexes: [1], @@ -307,19 +313,22 @@ describe('Address Service', function() { it('will emit a balance if there is a subscriber', function(done) { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); - am.subscriptions['address/balance'] = { - '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N': [emitter] - }; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] = [emitter]; var block = {}; var balance = 1000; am.getBalance = sinon.stub().callsArgWith(2, null, balance); - emitter.on('address/balance', function(address, bal, b) { - address.should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + emitter.on('address/balance', function(a, bal, b) { + a.toString().should.equal('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); bal.should.equal(balance); b.should.equal(block); done(); }); - am.balanceEventHandler(block, '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + am.balanceEventHandler(block, { + hashHex: address.hashBuffer.toString('hex'), + hashBuffer: address.hashBuffer, + addressType: address.type + }); }); }); @@ -328,34 +337,40 @@ describe('Address Service', function() { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); - var address = '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); var name = 'address/transaction'; am.subscribe(name, emitter, [address]); - am.subscriptions['address/transaction'][address].should.deep.equal([emitter]); + am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter]); - var address2 = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var address2 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); am.subscribe(name, emitter, [address2]); - am.subscriptions['address/transaction'][address2].should.deep.equal([emitter]); + am.subscriptions['address/transaction'][address2.hashBuffer.toString('hex')] + .should.deep.equal([emitter]); var emitter2 = new EventEmitter(); am.subscribe(name, emitter2, [address]); - am.subscriptions['address/transaction'][address].should.deep.equal([emitter, emitter2]); + am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter, emitter2]); }); it('will add an emitter to the subscribers array (balance)', function() { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); var name = 'address/balance'; - var address = '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); am.subscribe(name, emitter, [address]); - am.subscriptions['address/balance'][address].should.deep.equal([emitter]); + am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter]); - var address2 = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var address2 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); am.subscribe(name, emitter, [address2]); - am.subscriptions['address/balance'][address2].should.deep.equal([emitter]); + am.subscriptions['address/balance'][address2.hashBuffer.toString('hex')] + .should.deep.equal([emitter]); var emitter2 = new EventEmitter(); am.subscribe(name, emitter2, [address]); - am.subscriptions['address/balance'][address].should.deep.equal([emitter, emitter2]); + am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter, emitter2]); }); }); @@ -364,35 +379,37 @@ describe('Address Service', function() { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); var emitter2 = new EventEmitter(); - var address = '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'; - am.subscriptions['address/transaction'][address] = [emitter, emitter2]; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] = [emitter, emitter2]; var name = 'address/transaction'; am.unsubscribe(name, emitter, [address]); - am.subscriptions['address/transaction'][address].should.deep.equal([emitter2]); + am.subscriptions['address/transaction'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter2]); }); it('will remove emitter from subscribers array (balance)', function() { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); var emitter2 = new EventEmitter(); - var address = '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'; + var address = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); var name = 'address/balance'; - am.subscriptions['address/balance'][address] = [emitter, emitter2]; + am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] = [emitter, emitter2]; am.unsubscribe(name, emitter, [address]); - am.subscriptions['address/balance'][address].should.deep.equal([emitter2]); + am.subscriptions['address/balance'][address.hashBuffer.toString('hex')] + .should.deep.equal([emitter2]); }); it('should unsubscribe from all addresses if no addresses are specified', function() { var am = new AddressService({node: mocknode}); var emitter = new EventEmitter(); var emitter2 = new EventEmitter(); - am.subscriptions['address/balance'] = { - '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W': [emitter, emitter2], - '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N': [emitter2, emitter] - }; + var address1 = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + var hashHex1 = address1.hashBuffer.toString('hex'); + var address2 = bitcore.Address('1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N'); + var hashHex2 = address2.hashBuffer.toString('hex'); + am.subscriptions['address/balance'][hashHex1] = [emitter, emitter2]; + am.subscriptions['address/balance'][hashHex2] = [emitter2, emitter]; am.unsubscribe('address/balance', emitter); - am.subscriptions['address/balance'].should.deep.equal({ - '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W': [emitter2], - '1DzjESe6SLmAKVPLFMj6Sx1sWki3qt5i8N': [emitter2] - }); + am.subscriptions['address/balance'][hashHex1].should.deep.equal([emitter2]); + am.subscriptions['address/balance'][hashHex2].should.deep.equal([emitter2]); }); }); @@ -425,6 +442,7 @@ describe('Address Service', function() { describe('#getInputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var hashHex = bitcore.Address(address).hashBuffer.toString('hex'); var db = { tip: { __height: 1 @@ -452,8 +470,8 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.SPENTS, address, 12].join('-')); - ops.end.should.equal([AddressService.PREFIXES.SPENTS, address, 16].join('-')); + ops.start.should.equal([AddressService.PREFIXES.SPENTS, hashHex, 12].join('-')); + ops.end.should.equal([AddressService.PREFIXES.SPENTS, hashHex, 16].join('-')); createReadStreamCallCount++; return testStream; } @@ -486,8 +504,8 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.SPENTS, address].join('-')); - ops.end.should.equal([AddressService.PREFIXES.SPENTS, address].join('-') + '~'); + ops.start.should.equal([AddressService.PREFIXES.SPENTS, hashHex].join('-')); + ops.end.should.equal([AddressService.PREFIXES.SPENTS, hashHex].join('-') + '~'); createReadStreamCallCount++; return testStream; } @@ -535,6 +553,7 @@ describe('Address Service', function() { describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var hashHex = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer.toString('hex'); var db = { tip: { __height: 1 @@ -566,8 +585,8 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.OUTPUTS, address, 12].join('-')); - ops.end.should.equal([AddressService.PREFIXES.OUTPUTS, address, 16].join('-')); + ops.start.should.equal([AddressService.PREFIXES.OUTPUTS, hashHex, 12].join('-')); + ops.end.should.equal([AddressService.PREFIXES.OUTPUTS, hashHex, 16].join('-')); createReadStreamCallCount++; return testStream; } From d3641f3b0a09b2576efc8ad12673265aed53ec46 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 09:00:18 -0400 Subject: [PATCH 03/10] Pagination Optimization - Sort and paginate before getting full transaction details. - Only get detailed transaction information for items within the current page. - Improves the performance with large sets of transactions. --- lib/services/address/history.js | 155 +++++++++++--------- test/services/address/history.unit.js | 194 ++++++++++---------------- 2 files changed, 161 insertions(+), 188 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index ad36b2d6..c20bc964 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -21,12 +21,11 @@ function AddressHistory(args) { this.addresses = [args.addresses]; } this.transactionInfo = []; - this.transactions = {}; - this.sortedArray = []; + this.combinedArray = []; + this.detailedArray = []; } AddressHistory.MAX_ADDRESS_QUERIES = 20; -AddressHistory.MAX_TX_QUERIES = 10; AddressHistory.prototype.get = function(callback) { var self = this; @@ -45,8 +44,11 @@ AddressHistory.prototype.get = function(callback) { return callback(err); } + self.combineTransactionInfo(); + self.sortAndPaginateCombinedArray(); + async.eachSeries( - self.transactionInfo, + self.combinedArray, function(txInfo, next) { self.getDetailedInfo(txInfo, next); }, @@ -54,9 +56,7 @@ AddressHistory.prototype.get = function(callback) { if (err) { return callback(err); } - self.sortTransactionsIntoArray(); - self.paginateSortedArray(); - callback(null, self.sortedArray); + callback(null, self.detailedArray); } ); } @@ -103,86 +103,107 @@ AddressHistory.prototype.getTransactionInfo = function(address, next) { }); }; +/** + * This function combines results from getInputs and getOutputs by + * combining inputIndexes and outputIndexes for address transaction + * matching combinations. + */ +AddressHistory.prototype.combineTransactionInfo = function() { + var combinedArrayMap = {}; + this.combinedArray = []; + var l = this.transactionInfo.length; + for(var i = 0; i < l; i++) { + var item = this.transactionInfo[i]; + var mapKey = item.address + item.txid; + if (combinedArrayMap[mapKey] >= 0) { + var combined = this.combinedArray[combinedArrayMap[mapKey]]; + if (item.outputIndex >= 0) { + combined.satoshis += item.satoshis; + combined.outputIndexes.push(item.outputIndex); + } else if (item.inputIndex >= 0) { + combined.inputIndexes.push(item.inputIndex); + } + } else { + item.outputIndexes = []; + item.inputIndexes = []; + if (item.outputIndex >= 0) { + item.outputIndexes.push(item.outputIndex); + delete item.outputIndex; + } else if (item.inputIndex >= 0) { + item.inputIndexes.push(item.inputIndex); + delete item.inputIndex; + } + this.combinedArray.push(item); + combinedArrayMap[mapKey] = this.combinedArray.length - 1; + } + } +}; + +AddressHistory.prototype.sortAndPaginateCombinedArray = function() { + this.combinedArray.sort(AddressHistory.sortByHeight); + if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { + this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); + } +}; + AddressHistory.sortByHeight = function(a, b) { // TODO consider timestamp for mempool transactions return a.height < b.height; }; -AddressHistory.prototype.paginateSortedArray = function() { - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.sortedArray = this.sortedArray.slice(this.options.from, this.options.to); - } -}; - AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { var self = this; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; - if (self.transactions[txInfo.address] && self.transactions[txInfo.address][txInfo.txid]) { - self.amendDetailedInfoWithSatoshis(txInfo); - setImmediate(next); - } else { - self.node.services.db.getTransactionWithBlockInfo( - txInfo.txid, - queryMempool, - function(err, transaction) { - if (err) { + self.node.services.db.getTransactionWithBlockInfo( + txInfo.txid, + queryMempool, + function(err, transaction) { + if (err) { + return next(err); + } + + transaction.populateInputs(self.node.services.db, [], function(err) { + if(err) { return next(err); } - transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { - return next(err); - } - var confirmations = 0; - if (transaction.__height >= 0) { - confirmations = self.node.services.db.tip.__height - transaction.__height + 1; - } - - if (!self.transactions[txInfo.address]) { - self.transactions[txInfo.address] = {}; - } - - self.transactions[txInfo.address][txInfo.txid] = { - address: txInfo.address, - satoshis: 0, - height: transaction.__height, - confirmations: confirmations, - timestamp: transaction.__timestamp, - // TODO bitcore should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - outputIndexes: [], - inputIndexes: [], - tx: transaction - }; - - self.amendDetailedInfoWithSatoshis(txInfo); - next(); + self.detailedArray.push({ + address: txInfo.address, + satoshis: self.getSatoshisDetail(transaction, txInfo), + height: transaction.__height, + confirmations: self.getConfirmationsDetail(transaction), + timestamp: transaction.__timestamp, + // TODO bitcore should return null instead of throwing error on coinbase + fees: !transaction.isCoinbase() ? transaction.getFee() : null, + outputIndexes: txInfo.outputIndexes, + inputIndexes: txInfo.inputIndexes, + tx: transaction }); - } - ); - } + + next(); + }); + } + ); }; -AddressHistory.prototype.amendDetailedInfoWithSatoshis = function(txInfo) { - var historyItem = this.transactions[txInfo.address][txInfo.txid]; - if (txInfo.outputIndex >= 0) { - historyItem.outputIndexes.push(txInfo.outputIndex); - historyItem.satoshis += txInfo.satoshis; - } else if (txInfo.inputIndex >= 0){ - historyItem.inputIndexes.push(txInfo.inputIndex); - historyItem.satoshis -= historyItem.tx.inputs[txInfo.inputIndex].output.satoshis; +AddressHistory.prototype.getConfirmationsDetail = function(transaction) { + var confirmations = 0; + if (transaction.__height >= 0) { + confirmations = this.node.services.db.tip.__height - transaction.__height + 1; } + return confirmations; }; -AddressHistory.prototype.sortTransactionsIntoArray = function() { - this.sortedArray = []; - for(var address in this.transactions) { - for(var txid in this.transactions[address]) { - this.sortedArray.push(this.transactions[address][txid]); +AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { + var satoshis = txInfo.satoshis || 0; + + if (txInfo.inputIndexes.length >= 0) { + for(var j = 0; j < txInfo.inputIndexes.length; j++) { + satoshis -= transaction.inputs[txInfo.inputIndexes[j]].output.satoshis; } } - this.sortedArray.sort(AddressHistory.sortByHeight); + return satoshis; }; module.exports = AddressHistory; diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 06970b4b..61925bfa 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -23,9 +23,9 @@ describe('Address Service History', function() { history.node.should.equal(node); history.options.should.equal(options); history.addresses.should.equal(addresses); - history.transactions.should.deep.equal({}); history.transactionInfo.should.deep.equal([]); - history.sortedArray.should.deep.equal([]); + history.combinedArray.should.deep.equal([]); + history.detailedArray.should.deep.equal([]); }); it('will set addresses an array if only sent a string', function() { var history = new AddressHistory({ @@ -46,10 +46,11 @@ describe('Address Service History', function() { addresses: addresses }); var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; + history.detailedArray = expected; + history.combinedArray = [{}]; history.getTransactionInfo = sinon.stub().callsArg(1); - history.paginateSortedArray = sinon.stub(); + history.combineTransactionInfo = sinon.stub(); + history.sortAndPaginateCombinedArray = sinon.stub(); history.getDetailedInfo = sinon.stub().callsArg(1); history.sortTransactionsIntoArray = sinon.stub(); history.get(function(err, results) { @@ -58,8 +59,8 @@ describe('Address Service History', function() { } history.getTransactionInfo.callCount.should.equal(1); history.getDetailedInfo.callCount.should.equal(1); - history.sortTransactionsIntoArray.callCount.should.equal(1); - history.paginateSortedArray.callCount.should.equal(1); + history.combineTransactionInfo.callCount.should.equal(1); + history.sortAndPaginateCombinedArray.callCount.should.equal(1); results.should.equal(expected); done(); }); @@ -236,7 +237,7 @@ describe('Address Service History', function() { }); }); - describe('#paginateSortedArray', function() { + describe('#sortAndPaginateCombinedArray', function() { it('from 0 to 2', function() { var history = new AddressHistory({ node: {}, @@ -246,21 +247,21 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(2); - history.sortedArray[0].height.should.equal(14); - history.sortedArray[1].height.should.equal(13); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(2); + history.combinedArray[0].height.should.equal(14); + history.combinedArray[1].height.should.equal(13); }); it('from 0 to 4 (exceeds length)', function() { var history = new AddressHistory({ @@ -271,22 +272,22 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(3); - history.sortedArray[0].height.should.equal(14); - history.sortedArray[1].height.should.equal(13); - history.sortedArray[2].height.should.equal(12); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(3); + history.combinedArray[0].height.should.equal(14); + history.combinedArray[1].height.should.equal(13); + history.combinedArray[2].height.should.equal(12); }); it('from 0 to 1', function() { var history = new AddressHistory({ @@ -297,20 +298,20 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(1); - history.sortedArray[0].height.should.equal(14); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(1); + history.combinedArray[0].height.should.equal(14); }); it('from 2 to 3', function() { var history = new AddressHistory({ @@ -321,20 +322,20 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(1); - history.sortedArray[0].height.should.equal(12); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(1); + history.combinedArray[0].height.should.equal(12); }); it('from 10 to 20 (out of range)', function() { var history = new AddressHistory({ @@ -345,19 +346,19 @@ describe('Address Service History', function() { }, addresses: [] }); - history.sortedArray = [ + history.combinedArray = [ { - height: 14 + height: 13 }, { - height: 13, + height: 14, }, { height: 12 } ]; - history.paginateSortedArray(); - history.sortedArray.length.should.equal(0); + history.sortAndPaginateCombinedArray(); + history.combinedArray.length.should.equal(0); }); }); @@ -375,13 +376,10 @@ describe('Address Service History', function() { options: {}, addresses: [] }); - history.transactions[txid] = {}; - history.amendDetailedInfoWithSatoshis = sinon.stub(); history.getDetailedInfo(txid, function(err) { if (err) { throw err; } - history.amendDetailedInfoWithSatoshis.callCount.should.equal(1); history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); }); }); @@ -460,7 +458,8 @@ describe('Address Service History', function() { var transactionInfo = { txid: txid, timestamp: 1407292005, - outputIndex: 1, + outputIndexes: [1], + inputIndexes: [], satoshis: 48020000, address: txAddress }; @@ -468,7 +467,7 @@ describe('Address Service History', function() { if (err) { throw err; } - var info = history.transactions[txAddress][txid]; + var info = history.detailedArray[0]; info.address.should.equal(txAddress); info.satoshis.should.equal(48020000); info.height.should.equal(314159); @@ -481,94 +480,47 @@ describe('Address Service History', function() { }); }); }); - - describe('#amendDetailedInfoWithSatoshis', function() { - it('will amend info with inputIndex and subtract satoshis', function() { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + describe('#getConfirmationsDetail', function() { + it('the correct confirmations when included in the tip', function() { var history = new AddressHistory({ - node: {}, - options: {}, - addresses: [] - }); - history.transactions[address] = {}; - history.transactions[address][txid] = { - inputIndexes: [], - satoshis: 10, - tx: { - inputs: [ - { - output: { - satoshis: 3000 + node: { + services: { + db: { + tip: { + __height: 100 } } - ] - } - }; - history.amendDetailedInfoWithSatoshis({ - address: address, - txid: txid, - inputIndex: 0 + } + }, + options: {}, + addresses: [] }); - history.transactions[address][txid].inputIndexes.should.deep.equal([0]); - history.transactions[address][txid].satoshis.should.equal(-2990); + var transaction = { + __height: 100 + }; + history.getConfirmationsDetail(transaction).should.equal(1); }); - it('will amend info with outputIndex and add satoshis', function() { - var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + }); + describe('#getSatoshisDetail', function() { + it('subtract inputIndexes satoshis without outputIndexes', function() { var history = new AddressHistory({ node: {}, options: {}, addresses: [] }); - history.transactions[address] = {}; - history.transactions[address][txid] = { - outputIndexes: [], - satoshis: 10 - }; - history.amendDetailedInfoWithSatoshis({ - address: address, - txid: txid, - outputIndex: 10, - satoshis: 2000 - }); - history.transactions[address][txid].outputIndexes.should.deep.equal([10]); - history.transactions[address][txid].satoshis.should.equal(2010); - }); - }); - - describe('#sortTransactionIntoArray', function() { - it('will convert this.transactions into an array and sort by height', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 10, - to: 20 - }, - addresses: [] - }); - history.transactions = { - address1: { - txid1: { - height: 12 - }, - txid2: { - height: 14, - }, - txid3: { - height: 13 + var transaction = { + inputs: [ + { + output: { + satoshis: 10000 + } } - }, - address2: { - txid4: { - height: 15 - } - } + ] }; - history.sortTransactionsIntoArray(); - history.sortedArray.length.should.equal(4); - history.sortedArray[0].height.should.equal(15); - history.sortedArray[1].height.should.equal(14); - history.sortedArray[2].height.should.equal(13); - history.sortedArray[3].height.should.equal(12); + var txInfo = { + inputIndexes: [0] + }; + history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); }); }); }); From f88eee5a1cbec2f3033b06e0ace7f47861cd04ed Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 09:01:51 -0400 Subject: [PATCH 04/10] Improve db storage efficiency by ~2 times Encodes and decodes keys and values for leveldb storage more efficiently. --- lib/services/address/index.js | 257 +++++++++++++++++++--------- test/services/address/index.unit.js | 116 ++++--------- 2 files changed, 215 insertions(+), 158 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 82eb4527..e99a3dae 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -10,6 +10,7 @@ var bitcore = require('bitcore'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var Hash = bitcore.crypto.Hash; +var BufferReader = bitcore.encoding.BufferReader; var EventEmitter = require('events').EventEmitter; var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; @@ -34,10 +35,13 @@ AddressService.dependencies = [ ]; AddressService.PREFIXES = { - OUTPUTS: 'outs', - SPENTS: 'sp' + OUTPUTS: new Buffer('32', 'hex'), + SPENTS: new Buffer('33', 'hex') }; +AddressService.SPACER_MIN = new Buffer('00', 'hex'); +AddressService.SPACER_MAX = new Buffer('ff', 'hex'); + AddressService.prototype.getAPIMethods = function() { return [ ['getBalance', this, this.getBalance, 2], @@ -153,6 +157,7 @@ AddressService.prototype._extractAddressInfoFromScript = function(script) { AddressService.prototype.blockHandler = function(block, addOutput, callback) { var txs = block.transactions; + var height = block.__height; var action = 'put'; if (!addOutput) { @@ -188,43 +193,28 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); - // We need to use the height for indexes (and not the timestamp) because the // the timestamp has unreliable sequential ordering. The next block // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var height = block.__height; - - var scriptHex = output._scriptBuffer.toString('hex'); - - // To lookup outputs by address and height - var key = [ - AddressService.PREFIXES.OUTPUTS, - addressInfo.hashHex, - height, - txid, - outputIndex - ].join('-'); - - // TODO use buffers directly to save on disk storage - - var value = [output.satoshis, scriptHex].join(':'); - + var key = this._encodeOutputKey(addressInfo.hashBuffer, height, txid, outputIndex); + var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, key: key, value: value }); + addressInfo.hashHex = addressInfo.hashBuffer.toString('hex'); + // Collect data for subscribers if (txmessages[addressInfo.hashHex]) { txmessages[addressInfo.hashHex].outputIndexes.push(outputIndex); } else { txmessages[addressInfo.hashHex] = { tx: tx, - height: block.__height, + height: height, outputIndexes: [outputIndex], addressInfo: addressInfo, timestamp: block.header.timestamp @@ -247,30 +237,19 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { for(var inputIndex = 0; inputIndex < inputs.length; inputIndex++) { var input = inputs[inputIndex]; - - var inputHashBuffer; + var inputHash; if (input.script.isPublicKeyHashIn()) { - inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); + inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); } else if (input.script.isScriptHashIn()) { - inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); + inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); } else { continue; } // To be able to query inputs by address and spent height - var inputKey = [ - AddressService.PREFIXES.SPENTS, - inputHashBuffer.toString('hex'), - block.__height, - input.prevTxId.toString('hex'), - input.outputIndex - ].join('-'); - - var inputValue = [ - txid, - inputIndex - ].join(':'); + var inputKey = this._encodeInputKey(inputHash, height, input.prevTxId, input.outputIndex); + var inputValue = this._encodeInputValue(txid, inputIndex); operations.push({ type: action, @@ -285,6 +264,104 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); }; +AddressService.prototype._encodeOutputKey = function(hashBuffer, height, txid, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + AddressService.PREFIXES.OUTPUTS, + hashBuffer, + AddressService.SPACER_MIN, + heightBuffer, + new Buffer(txid, 'hex'), //TODO get buffer directly from tx + outputIndexBuffer + ]); + return key; +}; + +AddressService.prototype._decodeOutputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var txid = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + height: height, + txid: txid, + outputIndex: outputIndex + }; +}; + +AddressService.prototype._encodeOutputValue = function(satoshis, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, scriptBuffer]); +}; + +AddressService.prototype._decodeOutputValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var scriptBuffer = buffer.slice(8, buffer.length); + return { + satoshis: satoshis, + scriptBuffer: scriptBuffer + }; +}; + +AddressService.prototype._encodeInputKey = function(hashBuffer, height, prevTxIdBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + AddressService.PREFIXES.SPENTS, + hashBuffer, + AddressService.SPACER_MIN, + heightBuffer, + prevTxIdBuffer, + outputIndexBuffer + ]); +}; + +AddressService.prototype._decodeInputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var prevTxId = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + height: height, + prevTxId: prevTxId, + outputIndex: outputIndex + }; +}; + +AddressService.prototype._encodeInputValue = function(txid, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + new Buffer(txid, 'hex'), + inputIndexBuffer + ]); +}; + +AddressService.prototype._decodeInputValue = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + txid: txid, + inputIndex: inputIndex + }; +}; + /** * @param {Object} obj * @param {Transaction} obj.tx - The transaction @@ -424,45 +501,56 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var inputs = []; var stream; - var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex'); + var hashBuffer = bitcore.Address(addressStr).hashBuffer; if (options.start && options.end) { - // The positions will be flipped because the end position should be greater - // than the starting position for the stream, and we'll add one to the end key - // so that it's included in the results. + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end); - var endKey = [AddressService.PREFIXES.SPENTS, hashHex, options.start + 1].join('-'); - var startKey = [AddressService.PREFIXES.SPENTS, hashHex, options.end].join('-'); + var startBuffer = new Buffer(4); + startBuffer.writeUInt32BE(options.start + 1); stream = this.node.services.db.store.createReadStream({ - start: startKey, - end: endKey + gte: Buffer.concat([ + AddressService.PREFIXES.SPENTS, + hashBuffer, + AddressService.SPACER_MIN, + endBuffer + ]), + lte: Buffer.concat([ + AddressService.PREFIXES.SPENTS, + hashBuffer, + AddressService.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' }); } else { - var allKey = [AddressService.PREFIXES.SPENTS, hashHex].join('-'); + var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer]); stream = this.node.services.db.store.createReadStream({ - start: allKey, - end: allKey + '~' + gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), + lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' }); } stream.on('data', function(data) { - var key = data.key.split('-'); - var value = data.value.split(':'); + var key = self._decodeInputKey(data.key); + var value = self._decodeInputValue(data.value); - var blockHeight = Number(key[2]); - - var output = { + var input = { address: addressStr, - txid: value[0], - inputIndex: Number(value[1]), - height: blockHeight, - confirmations: self.node.services.db.tip.__height - blockHeight + 1 + txid: value.txid.toString('hex'), + inputIndex: value.inputIndex, + height: key.height, + confirmations: self.node.services.db.tip.__height - key.height + 1 }; - inputs.push(output); + inputs.push(input); }); @@ -502,44 +590,57 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - var hashHex = bitcore.Address(addressStr).hashBuffer.toString('hex'); + var hashBuffer = bitcore.Address(addressStr).hashBuffer; var outputs = []; var stream; if (options.start && options.end) { - // The positions will be flipped because the end position should be greater - // than the starting position for the stream, and we'll add one to the end key - // so that it's included in the results. - var endKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.start + 1].join('-'); - var startKey = [AddressService.PREFIXES.OUTPUTS, hashHex, options.end].join('-'); + var startBuffer = new Buffer(4); + startBuffer.writeUInt32BE(options.start + 1); + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end); stream = this.node.services.db.store.createReadStream({ - start: startKey, - end: endKey + gte: Buffer.concat([ + AddressService.PREFIXES.OUTPUTS, + hashBuffer, + AddressService.SPACER_MIN, + endBuffer + ]), + lte: Buffer.concat([ + AddressService.PREFIXES.OUTPUTS, + hashBuffer, + AddressService.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' }); } else { - var allKey = [AddressService.PREFIXES.OUTPUTS, hashHex].join('-'); + var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer]); stream = this.node.services.db.store.createReadStream({ - start: allKey, - end: allKey + '~' + gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), + lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' }); } stream.on('data', function(data) { - var key = data.key.split('-'); - var value = data.value.split(':'); + var key = self._decodeOutputKey(data.key); + var value = self._decodeOutputValue(data.value); var output = { address: addressStr, - txid: key[3], - outputIndex: Number(key[4]), - height: Number(key[2]), - satoshis: Number(value[0]), - script: value[1], - confirmations: self.node.services.db.tip.__height - Number(key[2]) + 1 + txid: key.txid.toString('hex'), + outputIndex: key.outputIndex, + height: key.height, + satoshis: value.satoshis, + script: value.scriptBuffer.toString('hex'), + confirmations: self.node.services.db.tip.__height - key.height + 1 }; outputs.push(output); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 468a75a9..4aac31b5 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -108,53 +108,6 @@ describe('Address Service', function() { var am; var testBlock = bitcore.Block.fromString(blockData); - var data = [ - { - key: { - hashHex: bitcore.Address('1F1MAvhTKg2VG29w8cXsiSN2PJ8gSsrJw').hashBuffer.toString('hex'), - height: 345003, - txid: 'fdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e16923', - outputIndex: 0 - }, - value: { - satoshis: 2502227470, - script: '76a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac', - blockHeight: 345003 - } - }, - { - key: { - hashHex: bitcore.Address('1Q8ec8kG7c7HqgK7uSzQyWsX9tzepRcKEL').hashBuffer.toString('hex'), - height: 345003, - prevTxId: '3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a9', - prevOutputIndex: 32 - }, - value: { - txid: '5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca', - inputIndex: 0 - } - }, - { - key: { - hashHex: bitcore.Address('1Ep5LA4T6Y7zaBPiwruUJurjGFvCJHzJhm').hashBuffer.toString('hex'), - height: 345003, - txid: 'e66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d', - outputIndex: 1 - }, - value: { - satoshis: 3100000, - script: '76a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac', - blockHeight: 345003 - } - } - ]; - var key0 = data[0].key; - var value0 = data[0].value; - var key3 = data[1].key; - var value3 = data[1].value; - var key64 = data[2].key; - var value64 = data[2].value; - before(function() { am = new AddressService({node: mocknode}); am.node.network = Networks.livenet; @@ -173,17 +126,14 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('put'); - var expected0 = ['outs', key0.hashHex, key0.height, key0.txid, key0.outputIndex].join('-'); - operations[0].key.should.equal(expected0); - operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); + operations[0].key.toString('hex').should.equal('3202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); + operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('put'); - var expected3 = ['sp', key3.hashHex, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-'); - operations[3].key.should.equal(expected3); - operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); + operations[3].key.toString('hex').should.equal('33fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); + operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); operations[64].type.should.equal('put'); - var expected64 = ['outs', key64.hashHex, key64.height, key64.txid, key64.outputIndex].join('-'); - operations[64].key.should.equal(expected64); - operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); + operations[64].key.toString('hex').should.equal('329780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[64].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); }); @@ -199,14 +149,14 @@ describe('Address Service', function() { should.not.exist(err); operations.length.should.equal(81); operations[0].type.should.equal('del'); - operations[0].key.should.equal(['outs', key0.hashHex, key0.height, key0.txid, key0.outputIndex].join('-')); - operations[0].value.should.equal([value0.satoshis, value0.script].join(':')); + operations[0].key.toString('hex').should.equal('3202a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b00000543abfdbefe0d064729d85556bd3ab13c3a889b685d042499c02b4aa2064fb1e1692300000000'); + operations[0].value.toString('hex').should.equal('41e2a49ec1c0000076a91402a61d2066d19e9e2fd348a8320b7ebd4dd3ca2b88ac'); operations[3].type.should.equal('del'); - operations[3].key.should.equal(['sp', key3.hashHex, key3.height, key3.prevTxId, key3.prevOutputIndex].join('-')); - operations[3].value.should.equal([value3.txid, value3.inputIndex].join(':')); + operations[3].key.toString('hex').should.equal('33fdbd324b28ea69e49c998816407dc055fb81d06e00000543ab3d7d5d98df753ef2a4f82438513c509e3b11f3e738e94a7234967b03a03123a900000020'); + operations[3].value.toString('hex').should.equal('5780f3ee54889a0717152a01abee9a32cec1b0cdf8d5537a08c7bd9eeb6bfbca00000000'); operations[64].type.should.equal('del'); - operations[64].key.should.equal(['outs', key64.hashHex, key64.height, key64.txid, key64.outputIndex].join('-')); - operations[64].value.should.equal([value64.satoshis, value64.script].join(':')); + operations[64].key.toString('hex').should.equal('329780ccd5356e2acc0ee439ee04e0fe69426c752800000543abe66f3b989c790178de2fc1a5329f94c0d8905d0d3df4e7ecf0115e7f90a6283d00000001'); + operations[64].value.toString('hex').should.equal('4147a6b00000000076a9149780ccd5356e2acc0ee439ee04e0fe69426c752888ac'); done(); }); }); @@ -442,7 +392,7 @@ describe('Address Service', function() { describe('#getInputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashHex = bitcore.Address(address).hashBuffer.toString('hex'); + var hashBuffer = bitcore.Address(address).hashBuffer; var db = { tip: { __height: 1 @@ -470,8 +420,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.SPENTS, hashHex, 12].join('-')); - ops.end.should.equal([AddressService.PREFIXES.SPENTS, hashHex, 16].join('-')); + var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('000000000c', 'hex')]); + ops.gte.toString('hex').should.equal(gte.toString('hex')); + var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('0000000010', 'hex')]); + ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -490,8 +442,8 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: ['sp', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), - value: ['3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '0'].join(':') + key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); testStream.emit('close'); @@ -504,8 +456,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.SPENTS, hashHex].join('-')); - ops.end.should.equal([AddressService.PREFIXES.SPENTS, hashHex].join('-') + '~'); + var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('00', 'hex')]); + ops.gte.toString('hex').should.equal(gte.toString('hex')); + var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, new Buffer('ff', 'hex')]); + ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -524,8 +478,8 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: ['sp', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), - value: ['3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '0'].join(':') + key: new Buffer('33038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); testStream.emit('close'); @@ -553,7 +507,7 @@ describe('Address Service', function() { describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; - var hashHex = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer.toString('hex'); + var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; var db = { tip: { __height: 1 @@ -585,8 +539,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - ops.start.should.equal([AddressService.PREFIXES.OUTPUTS, hashHex, 12].join('-')); - ops.end.should.equal([AddressService.PREFIXES.OUTPUTS, hashHex, 16].join('-')); + var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, new Buffer('000000000c', 'hex')]); + ops.gte.toString('hex').should.equal(gte.toString('hex')); + var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, new Buffer('0000000010', 'hex')]); + ops.lte.toString('hex').should.equal(lte.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -607,8 +563,8 @@ describe('Address Service', function() { }); createReadStreamCallCount.should.equal(1); var data = { - key: ['outs', address, '15', '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), - value: ['4527773864', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') + key: new Buffer('32038a213afdfc551fc658e9a2a58a86e98d69b687000000000f125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; testStream.emit('data', data); testStream.emit('close'); @@ -655,13 +611,13 @@ describe('Address Service', function() { }); var data1 = { - key: ['outs', address, 345000, '125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87', '1'].join('-'), - value: ['4527773864', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') + key: new Buffer('32038a213afdfc551fc658e9a2a58a86e98d69b68700000543a8125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf8700000001', 'hex'), + value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; var data2 = { - key: ['outs', address, 345004, '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', '2'].join('-'), - value: ['10000', '76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'].join(':') + key: new Buffer('32038a213afdfc551fc658e9a2a58a86e98d69b68700000543ac3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000002', 'hex'), + value: new Buffer('40c388000000000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; readStream1.emit('data', data1); From a80b58e00470cb6ffc26eeffe42224921e3edc34 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 16:11:31 -0400 Subject: [PATCH 05/10] Address History: Fix sorting bug --- lib/services/address/history.js | 22 +++- lib/services/address/index.js | 2 +- test/services/address/history.unit.js | 158 +++++++++++++++++++++++++- 3 files changed, 173 insertions(+), 9 deletions(-) diff --git a/lib/services/address/history.js b/lib/services/address/history.js index c20bc964..c5e91c5a 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -147,8 +147,26 @@ AddressHistory.prototype.sortAndPaginateCombinedArray = function() { }; AddressHistory.sortByHeight = function(a, b) { - // TODO consider timestamp for mempool transactions - return a.height < b.height; + if (a.height < 0 && b.height < 0) { + // Both are from the mempool, compare timestamps + if (a.timestamp === b.timestamp) { + return 0; + } else { + return a.timestamp < b.timestamp ? 1 : -1; + } + } else if (a.height < 0 && b.height > 0) { + // A is from the mempool and B is in a block + return -1; + } else if (a.height > 0 && b.height < 0) { + // A is in a block and B is in the mempool + return 1; + } else if (a.height === b.height) { + // The heights are equal + return 0; + } else { + // Otherwise compare heights + return a.height < b.height ? 1 : -1; + } }; AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { diff --git a/lib/services/address/index.js b/lib/services/address/index.js index e99a3dae..7d74adb0 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -732,7 +732,7 @@ AddressService.prototype.isSpent = function(output, queryMempool, callback) { }; /** - * This will give the history for many addresses limited by a range of dates (to limit + * This will give the history for many addresses limited by a range of block heights (to limit * the database lookup times) and/or paginated to limit the results length. * @param {Array} addresses - An array of addresses * @param {Object} options - The options to limit the query diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 61925bfa..9c24e05a 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -221,19 +221,165 @@ describe('Address Service History', function() { it('will sort latest to oldest using height', function() { var transactionInfo = [ { - height: 12 + height: 276328 }, { - height: 14, + height: 273845, }, { - height: 13 + height: 555655 + }, + { + height: 325496 + }, + { + height: 329186 + }, + { + height: 534195 } ]; transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(14); - transactionInfo[1].height.should.equal(13); - transactionInfo[2].height.should.equal(12); + transactionInfo[0].height.should.equal(555655); + transactionInfo[1].height.should.equal(534195); + transactionInfo[2].height.should.equal(329186); + transactionInfo[3].height.should.equal(325496); + transactionInfo[4].height.should.equal(276328); + transactionInfo[5].height.should.equal(273845); + }); + it('mempool and tip with time in the future', function() { + var transactionInfo = [ + { + timestamp: 1442050425439, + height: 14, + }, + { + timestamp: 1442050424328, + height: -1 + }, + { + timestamp: 1442050424429, + height: -1 + }, + { + timestamp: 1442050425439, + height: 15 + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].height.should.equal(-1); + transactionInfo[0].timestamp.should.equal(1442050424429); + transactionInfo[1].height.should.equal(-1); + transactionInfo[1].timestamp.should.equal(1442050424328); + transactionInfo[2].height.should.equal(15); + transactionInfo[3].height.should.equal(14); + }); + it('tip with time in the future and mempool', function() { + var transactionInfo = [ + { + timestamp: 1442050425439, + height: 14, + }, + { + timestamp: 1442050424328, + height: -1 + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].height.should.equal(-1); + transactionInfo[1].height.should.equal(14); + }); + it('many transactions in the mempool', function() { + var transactionInfo = [ + { + timestamp: 1442259670462, + height: -1 + }, + { + timestamp: 1442259785114, + height: -1 + }, + { + timestamp: 1442259759896, + height: -1 + }, + { + timestamp: 1442259692601, + height: -1 + }, + { + timestamp: 1442259692601, + height: 100 + }, + { + timestamp: 1442259749463, + height: -1 + }, + { + timestamp: 1442259737719, + height: -1 + }, + { + timestamp: 1442259773138, + height: -1, + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].timestamp.should.equal(1442259785114); + transactionInfo[1].timestamp.should.equal(1442259773138); + transactionInfo[2].timestamp.should.equal(1442259759896); + transactionInfo[3].timestamp.should.equal(1442259749463); + transactionInfo[4].timestamp.should.equal(1442259737719); + transactionInfo[5].timestamp.should.equal(1442259692601); + transactionInfo[6].timestamp.should.equal(1442259670462); + transactionInfo[7].height.should.equal(100); + }); + it('mempool and mempool', function() { + var transactionInfo = [ + { + timestamp: 1442050424328, + height: -1 + }, + { + timestamp: 1442050425439, + height: -1, + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].timestamp.should.equal(1442050425439); + transactionInfo[1].timestamp.should.equal(1442050424328); + }); + it('mempool and mempool with the same timestamp', function() { + var transactionInfo = [ + { + timestamp: 1442050425439, + height: -1, + txid: '1', + }, + { + timestamp: 1442050425439, + height: -1, + txid: '2' + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].txid.should.equal('1'); + transactionInfo[1].txid.should.equal('2'); + }); + it('matching block heights', function() { + var transactionInfo = [ + { + height: 325496, + txid: '1', + }, + { + height: 325496, + txid: '2' + } + ]; + transactionInfo.sort(AddressHistory.sortByHeight); + transactionInfo[0].txid.should.equal('1'); + transactionInfo[1].txid.should.equal('2'); }); }); From 87a91637436a3d3e139c135ddb8bb46778d8eb5a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 16:28:34 -0400 Subject: [PATCH 06/10] Address History: Docs for "start" and "end" arguments --- lib/services/address/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 7d74adb0..42cd6617 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -738,8 +738,8 @@ AddressService.prototype.isSpent = function(output, queryMempool, callback) { * @param {Object} options - The options to limit the query * @param {Number} [options.from] - The pagination "from" index * @param {Number} [options.to] - The pagination "to" index - * @param {Number} [options.start] - The beginning block height - * @param {Number} [options.end] - The ending block height + * @param {Number} [options.start] - The beginning block height (e.g. 1500 the most recent block height). + * @param {Number} [options.end] - The ending block height (e.g. 0 the older block height, results are inclusive). * @param {Boolean} [options.queryMempool] - Include the mempool in the query * @param {Function} callback */ From b1b40c892e3b911bc24131dbc1bd61eee608f805 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 16:48:28 -0400 Subject: [PATCH 07/10] Address History: Add getAddressHistoryCount method --- integration/regtest-node.js | 14 ++++++++++ lib/services/address/history.js | 19 ++++++++++++++ lib/services/address/index.js | 19 ++++++++++++++ test/services/address/history.unit.js | 37 +++++++++++++++++++++++++++ test/services/address/index.unit.js | 19 ++++++++++++++ 5 files changed, 108 insertions(+) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index 0c41524e..fedb0f84 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -518,6 +518,20 @@ describe('Node Functionality', function() { }); }); + it('total transaction count (sending and receiving)', function(done) { + var addresses = [ + address + ]; + var options = {}; + node.services.address.getAddressHistoryCount(addresses, options, function(err, count) { + if (err) { + throw err; + } + count.should.equal(6); + done(); + }); + }); + describe('Pagination', function() { it('from 0 to 1', function(done) { var options = { diff --git a/lib/services/address/history.js b/lib/services/address/history.js index c5e91c5a..18c97afd 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -27,6 +27,25 @@ function AddressHistory(args) { AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.prototype.getCount = function(callback) { + var self = this; + + async.eachLimit( + self.addresses, + AddressHistory.MAX_ADDRESS_QUERIES, + function(address, next) { + self.getTransactionInfo(address, next); + }, + function(err) { + if (err) { + return callback(err); + } + self.combineTransactionInfo(); + callback(null, self.combinedArray.length); + } + ); +}; + AddressHistory.prototype.get = function(callback) { var self = this; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 42cd6617..af54d3dc 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -731,6 +731,25 @@ AddressService.prototype.isSpent = function(output, queryMempool, callback) { }); }; +/** + * This will give the total count of transactions for a single or several addresses + * limited by a range of block heights. + * @param {Array} addresses - An array of addresses + * @param {Object} options - The options to limit the query + * @param {Number} [options.start] - The beginning block height (e.g. 1500 the most recent block height). + * @param {Number} [options.end] - The ending block height (e.g. 0 the older block height, results are inclusive). + * @param {Boolean} [options.queryMempool] - Include the mempool in the query + * @param {Function} callback + */ +AddressService.prototype.getAddressHistoryCount = function(addresses, options, callback) { + var history = new AddressHistory({ + node: this.node, + options: options, + addresses: addresses + }); + history.getCount(callback); +}; + /** * This will give the history for many addresses limited by a range of block heights (to limit * the database lookup times) and/or paginated to limit the results length. diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 9c24e05a..c0ce648a 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -37,6 +37,43 @@ describe('Address Service History', function() { }); }); + describe('#getCount', function() { + it('will complete the async each limit series', function(done) { + var addresses = [address]; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: addresses + }); + history.getTransactionInfo = sinon.stub().callsArg(1); + history.combineTransactionInfo = sinon.stub(); + history.get(function(err, results) { + if (err) { + throw err; + } + history.getTransactionInfo.callCount.should.equal(1); + history.combineTransactionInfo.callCount.should.equal(1); + done(); + }); + }); + it('handle an error from getTransactionInfo', function(done) { + var addresses = [address]; + var history = new AddressHistory({ + node: {}, + options: {}, + addresses: addresses + }); + var expected = [{}]; + history.sortedArray = expected; + history.transactionInfo = [{}]; + history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test')); + history.get(function(err) { + err.message.should.equal('test'); + done(); + }); + }); + }); + describe('#get', function() { it('will complete the async each limit series', function(done) { var addresses = [address]; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 4aac31b5..6d611b60 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -857,6 +857,25 @@ describe('Address Service', function() { }); }); + describe('#getAddressHistoryCount', function() { + it('will call getCount on address history instance', function(done) { + function TestAddressHistory(args) { + args.node.should.equal(mocknode); + args.addresses.should.deep.equal([]); + args.options.should.deep.equal({}); + } + TestAddressHistory.prototype.getCount = sinon.stub().callsArg(0); + var TestAddressService = proxyquire('../../../lib/services/address', { + './history': TestAddressHistory + }); + var am = new TestAddressService({node: mocknode}); + am.getAddressHistoryCount([], {}, function(err, history) { + TestAddressHistory.prototype.getCount.callCount.should.equal(1); + done(); + }); + }); + }); + describe('#getAddressHistory', function() { it('will call get on address history instance', function(done) { function TestAddressHistory(args) { From e7587564a7eb3aa639ca1e81cfd6312d3d069c43 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 17:29:43 -0400 Subject: [PATCH 08/10] Address History: Add getAddressHistoryCount to available API methods. --- lib/services/address/index.js | 3 ++- test/services/address/index.unit.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/services/address/index.js b/lib/services/address/index.js index af54d3dc..dbf944ce 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -48,7 +48,8 @@ AddressService.prototype.getAPIMethods = function() { ['getOutputs', this, this.getOutputs, 2], ['getUnspentOutputs', this, this.getUnspentOutputs, 2], ['isSpent', this, this.isSpent, 2], - ['getAddressHistory', this, this.getAddressHistory, 2] + ['getAddressHistory', this, this.getAddressHistory, 2], + ['getAddressHistoryCount', this, this.getAddressHistoryCount, 2] ]; }; diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index 6d611b60..f8f8a65c 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -29,7 +29,7 @@ describe('Address Service', function() { it('should return the correct methods', function() { var am = new AddressService({node: mocknode}); var methods = am.getAPIMethods(); - methods.length.should.equal(5); + methods.length.should.equal(6); }); }); From da1513edb28bdca8177fdac34658c00240a38b02 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 17:30:10 -0400 Subject: [PATCH 09/10] Address History: Add additional regtest for getting history by height. --- integration/regtest-node.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index fedb0f84..d28ce990 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -465,6 +465,29 @@ describe('Node Functionality', function() { }); }); + it('five addresses (limited by height 155 to 154)', function(done) { + var addresses = [ + address2, + address3, + address4, + address5, + address6 + ]; + var options = { + start: 155, + end: 154 + }; + node.services.address.getAddressHistory(addresses, options, function(err, history) { + if (err) { + throw err; + } + history.length.should.equal(2); + history[0].height.should.equal(155); + history[1].height.should.equal(154); + done(); + }); + }); + it('five addresses (paginated by index)', function(done) { var addresses = [ address2, From 5fbd8991ed6a0ea6aa19fb97b8ea04706f2529b8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 14 Sep 2015 21:03:06 -0400 Subject: [PATCH 10/10] Address History: Combine addresses so that history is txid unique. --- integration/regtest-node.js | 51 +++++++++++++-------------- lib/services/address/history.js | 41 +++++++++++++-------- test/services/address/history.unit.js | 17 +++++---- 3 files changed, 61 insertions(+), 48 deletions(-) diff --git a/integration/regtest-node.js b/integration/regtest-node.js index d28ce990..1a661a1b 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -280,14 +280,14 @@ describe('Node Functionality', function() { } results.length.should.equal(1); var info = results[0]; - info.address.should.equal(address); + should.exist(info.addresses[address]); + info.addresses[address].outputIndexes.length.should.equal(1); + info.addresses[address].outputIndexes[0].should.be.within(0, 1); + info.addresses[address].inputIndexes.should.deep.equal([]); info.satoshis.should.equal(10 * 1e8); info.confirmations.should.equal(3); info.timestamp.should.be.a('number'); info.fees.should.be.within(190, 193); - info.outputIndexes.length.should.equal(1); - info.outputIndexes[0].should.be.within(0, 1); - info.inputIndexes.should.deep.equal([]); info.tx.should.be.an.instanceof(Transaction); done(); }); @@ -423,18 +423,17 @@ describe('Node Functionality', function() { if (err) { throw err; } - history.length.should.equal(5); + history.length.should.equal(4); history[0].height.should.equal(157); history[0].confirmations.should.equal(1); - history[1].height.should.equal(157); - history[2].height.should.equal(156); - history[2].address.should.equal(address4); - history[3].height.should.equal(155); - history[3].address.should.equal(address3); - history[4].height.should.equal(154); - history[4].address.should.equal(address2); - history[4].satoshis.should.equal(99990000); - history[4].confirmations.should.equal(4); + history[1].height.should.equal(156); + should.exist(history[1].addresses[address4]); + history[2].height.should.equal(155); + should.exist(history[2].addresses[address3]); + history[3].height.should.equal(154); + should.exist(history[3].addresses[address2]); + history[3].satoshis.should.equal(99990000); + history[3].confirmations.should.equal(4); done(); }); }); @@ -455,12 +454,11 @@ describe('Node Functionality', function() { if (err) { throw err; } - history.length.should.equal(3); + history.length.should.equal(2); history[0].height.should.equal(157); history[0].confirmations.should.equal(1); - history[1].height.should.equal(157); - history[2].height.should.equal(156); - history[2].address.should.equal(address4); + history[1].height.should.equal(156); + should.exist(history[1].addresses[address4]); done(); }); }); @@ -507,9 +505,8 @@ describe('Node Functionality', function() { history.length.should.equal(3); history[0].height.should.equal(157); history[0].confirmations.should.equal(1); - history[1].height.should.equal(157); - history[2].height.should.equal(156); - history[2].address.should.equal(address4); + history[1].height.should.equal(156); + should.exist(history[1].addresses[address4]); done(); }); }); @@ -525,16 +522,16 @@ describe('Node Functionality', function() { } history.length.should.equal(6); history[0].height.should.equal(157); - history[0].inputIndexes.should.deep.equal([0, 1]); - history[0].outputIndexes.should.deep.equal([2]); + history[0].addresses[address].inputIndexes.should.deep.equal([0, 1]); + history[0].addresses[address].outputIndexes.should.deep.equal([2]); history[0].confirmations.should.equal(1); history[1].height.should.equal(156); history[2].height.should.equal(155); history[3].height.should.equal(154); history[4].height.should.equal(153); history[4].satoshis.should.equal(-10000); - history[4].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); - history[4].inputIndexes.should.deep.equal([0]); + history[4].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); + history[4].addresses[address].inputIndexes.should.deep.equal([0]); history[5].height.should.equal(150); history[5].satoshis.should.equal(10 * 1e8); done(); @@ -624,8 +621,8 @@ describe('Node Functionality', function() { history.length.should.equal(1); history[0].height.should.equal(153); history[0].satoshis.should.equal(-10000); - history[0].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); - history[0].inputIndexes.should.deep.equal([0]); + history[0].addresses[address].outputIndexes.should.deep.equal([0, 1, 2, 3, 4]); + history[0].addresses[address].inputIndexes.should.deep.equal([0]); done(); }); }); diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 18c97afd..a74e69ce 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -133,25 +133,35 @@ AddressHistory.prototype.combineTransactionInfo = function() { var l = this.transactionInfo.length; for(var i = 0; i < l; i++) { var item = this.transactionInfo[i]; - var mapKey = item.address + item.txid; + var mapKey = item.txid; if (combinedArrayMap[mapKey] >= 0) { var combined = this.combinedArray[combinedArrayMap[mapKey]]; + if (!combined.addresses[item.address]) { + combined.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; + } if (item.outputIndex >= 0) { combined.satoshis += item.satoshis; - combined.outputIndexes.push(item.outputIndex); + combined.addresses[item.address].outputIndexes.push(item.outputIndex); } else if (item.inputIndex >= 0) { - combined.inputIndexes.push(item.inputIndex); + combined.addresses[item.address].inputIndexes.push(item.inputIndex); } } else { - item.outputIndexes = []; - item.inputIndexes = []; + item.addresses = {}; + item.addresses[item.address] = { + outputIndexes: [], + inputIndexes: [] + }; if (item.outputIndex >= 0) { - item.outputIndexes.push(item.outputIndex); - delete item.outputIndex; + item.addresses[item.address].outputIndexes.push(item.outputIndex); } else if (item.inputIndex >= 0) { - item.inputIndexes.push(item.inputIndex); - delete item.inputIndex; + item.addresses[item.address].inputIndexes.push(item.inputIndex); } + delete item.outputIndex; + delete item.inputIndex; + delete item.address; this.combinedArray.push(item); combinedArrayMap[mapKey] = this.combinedArray.length - 1; } @@ -206,15 +216,13 @@ AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { } self.detailedArray.push({ - address: txInfo.address, + addresses: txInfo.addresses, satoshis: self.getSatoshisDetail(transaction, txInfo), height: transaction.__height, confirmations: self.getConfirmationsDetail(transaction), timestamp: transaction.__timestamp, // TODO bitcore should return null instead of throwing error on coinbase fees: !transaction.isCoinbase() ? transaction.getFee() : null, - outputIndexes: txInfo.outputIndexes, - inputIndexes: txInfo.inputIndexes, tx: transaction }); @@ -235,11 +243,14 @@ AddressHistory.prototype.getConfirmationsDetail = function(transaction) { AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { var satoshis = txInfo.satoshis || 0; - if (txInfo.inputIndexes.length >= 0) { - for(var j = 0; j < txInfo.inputIndexes.length; j++) { - satoshis -= transaction.inputs[txInfo.inputIndexes[j]].output.satoshis; + for(var address in txInfo.addresses) { + if (txInfo.addresses[address].inputIndexes.length >= 0) { + for(var j = 0; j < txInfo.addresses[address].inputIndexes.length; j++) { + satoshis -= transaction.inputs[txInfo.addresses[address].inputIndexes[j]].output.satoshis; + } } } + return satoshis; }; diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index c0ce648a..608081e9 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -639,26 +639,29 @@ describe('Address Service History', function() { addresses: [] }); var transactionInfo = { + addresses: {}, txid: txid, timestamp: 1407292005, - outputIndexes: [1], - inputIndexes: [], satoshis: 48020000, address: txAddress }; + transactionInfo.addresses[txAddress] = {}; + transactionInfo.addresses[txAddress].outputIndexes = [1]; + transactionInfo.addresses[txAddress].inputIndexes = []; history.getDetailedInfo(transactionInfo, function(err) { if (err) { throw err; } var info = history.detailedArray[0]; - info.address.should.equal(txAddress); + info.addresses[txAddress].should.deep.equal({ + outputIndexes: [1], + inputIndexes: [] + }); info.satoshis.should.equal(48020000); info.height.should.equal(314159); info.confirmations.should.equal(1); info.timestamp.should.equal(1407292005); info.fees.should.equal(20000); - info.outputIndexes.should.deep.equal([1]); - info.inputIndexes.should.deep.equal([]); info.tx.should.equal(transaction); }); }); @@ -701,8 +704,10 @@ describe('Address Service History', function() { ] }; var txInfo = { - inputIndexes: [0] + addresses: {} }; + txInfo.addresses[address] = {}; + txInfo.addresses[address].inputIndexes = [0]; history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); }); });