diff --git a/integration/regtest-node.js b/integration/regtest-node.js index fb7ff6bf..e05e3039 100644 --- a/integration/regtest-node.js +++ b/integration/regtest-node.js @@ -28,6 +28,7 @@ var Transaction = index.Transaction; var BitcoreNode = index.Node; var AddressService = index.services.Address; var BitcoinService = index.services.Bitcoin; +var encoding = require('../lib/services/address/encoding'); var DBService = index.services.DB; var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG'; var testKey; @@ -43,22 +44,6 @@ describe('Node Functionality', function() { before(function(done) { this.timeout(30000); - // Add the regtest network - bitcore.Networks.remove(bitcore.Networks.testnet); - bitcore.Networks.add({ - name: 'regtest', - alias: 'regtest', - pubkeyhash: 0x6f, - privatekey: 0xef, - scripthash: 0xc4, - xpubkey: 0x043587cf, - xprivkey: 0x04358394, - networkMagic: 0xfabfb5da, - port: 18444, - dnsSeeds: [ ] - }); - regtest = bitcore.Networks.get('regtest'); - var datadir = __dirname + '/data'; testKey = bitcore.PrivateKey(testWIF); @@ -93,6 +78,9 @@ describe('Node Functionality', function() { node = new BitcoreNode(configuration); + regtest = bitcore.Networks.get('regtest'); + should.exist(regtest); + node.on('error', function(err) { log.error(err); }); @@ -208,7 +196,7 @@ describe('Node Functionality', function() { // We need to add a transaction to the mempool so that the next block will // have a different hash as the hash has been invalidated. - client.sendToAddress(testKey.toAddress().toString(), 10, function(err) { + client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) { if (err) { throw err; } @@ -250,7 +238,7 @@ describe('Node Functionality', function() { var address; var unspentOutput; before(function() { - address = testKey.toAddress().toString(); + address = testKey.toAddress(regtest).toString(); }); it('should be able to get the balance of the test address', function(done) { node.services.address.getBalance(address, false, function(err, balance) { @@ -333,19 +321,19 @@ describe('Node Functionality', function() { /* jshint maxstatements: 50 */ testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x'); - address2 = testKey2.toAddress().toString(); + address2 = testKey2.toAddress(regtest).toString(); testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt'); - address3 = testKey3.toAddress().toString(); + address3 = testKey3.toAddress(regtest).toString(); testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw'); - address4 = testKey4.toAddress().toString(); + address4 = testKey4.toAddress(regtest).toString(); testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF'); - address5 = testKey5.toAddress().toString(); + address5 = testKey5.toAddress(regtest).toString(); testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF'); - address6 = testKey6.toAddress().toString(); + address6 = testKey6.toAddress(regtest).toString(); var tx = new Transaction(); tx.from(unspentOutput); @@ -726,7 +714,7 @@ describe('Node Functionality', function() { node.services.bitcoind.sendTransaction(tx.serialize()); setImmediate(function() { - var addrObj = node.services.address._getAddressInfo(address); + var addrObj = encoding.getAddressInfo(address); node.services.address._getOutputsMempool(address, addrObj.hashBuffer, addrObj.hashTypeBuffer, function(err, outs) { if (err) { diff --git a/lib/services/address/constants.js b/lib/services/address/constants.js new file mode 100644 index 00000000..84f9b32a --- /dev/null +++ b/lib/services/address/constants.js @@ -0,0 +1,54 @@ +'use strict'; + +var exports = {}; + +exports.PREFIXES = { + OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height + SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height + SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output +}; + +exports.MEMPREFIXES = { + OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address + SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address + SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output +}; + +// To save space, we're only storing the PubKeyHash or ScriptHash in our index. +// To avoid intentional unspendable collisions, which have been seen on the blockchain, +// we must store the hash type (PK or Script) as well. +exports.HASH_TYPES = { + PUBKEY: new Buffer('01', 'hex'), + REDEEMSCRIPT: new Buffer('02', 'hex') +}; + +// Translates from our enum type back into the hash types returned by +// bitcore-lib/address. +exports.HASH_TYPES_READABLE = { + '01': 'pubkeyhash', + '02': 'scripthash' +}; + +exports.HASH_TYPES_MAP = { + 'pubkeyhash': exports.HASH_TYPES.PUBKEY, + 'scripthash': exports.HASH_TYPES.REDEEMSCRIPT +}; + +exports.SPACER_MIN = new Buffer('00', 'hex'); +exports.SPACER_MAX = new Buffer('ff', 'hex'); +exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex'); +exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex'); +exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex'); +exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex'); + +// The maximum number of inputs that can be queried at once +exports.MAX_INPUTS_QUERY_LENGTH = 50000; +// The maximum number of outputs that can be queried at once +exports.MAX_OUTPUTS_QUERY_LENGTH = 50000; +// The maximum number of transactions that can be queried at once +exports.MAX_HISTORY_QUERY_LENGTH = 100; +// The maximum number of addresses that can be queried at once +exports.MAX_ADDRESSES_QUERY = 10000; + +module.exports = exports; + diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js new file mode 100644 index 00000000..81421a18 --- /dev/null +++ b/lib/services/address/encoding.js @@ -0,0 +1,298 @@ +'use strict'; + +var bitcore = require('bitcore-lib'); +var BufferReader = bitcore.encoding.BufferReader; +var Address = bitcore.Address; +var PublicKey = bitcore.PublicKey; +var constants = require('./constants'); +var $ = bitcore.util.preconditions; + +var exports = {}; + +exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + txidBuffer, + outputIndexBuffer + ]); + return key.toString('binary'); +}; + +exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + var key = Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + txidBuffer, + outputIndexBuffer + ]); + return key; +}; + +exports.decodeOutputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var txid = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + txid: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeOutputValue = function(satoshis, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, scriptBuffer]); +}; + +exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) { + var satoshisBuffer = new Buffer(8); + satoshisBuffer.writeDoubleBE(satoshis); + return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]); +}; + +exports.decodeOutputValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var scriptBuffer = buffer.slice(8, buffer.length); + return { + satoshis: satoshis, + scriptBuffer: scriptBuffer + }; +}; + +exports.decodeOutputMempoolValue = function(buffer) { + var satoshis = buffer.readDoubleBE(0); + var timestamp = buffer.readDoubleBE(8); + var scriptBuffer = buffer.slice(16, buffer.length); + return { + satoshis: satoshis, + timestamp: timestamp, + scriptBuffer: scriptBuffer + }; +}; + +exports.encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height); + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + heightBuffer, + prevTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKey = function(buffer) { + var reader = new BufferReader(buffer); + var prefix = reader.read(1); + var hashBuffer = reader.read(20); + var hashTypeBuffer = reader.read(1); + var spacer = reader.read(1); + var height = reader.readUInt32BE(); + var prevTxId = reader.read(32); + var outputIndex = reader.readUInt32BE(); + return { + prefix: prefix, + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + height: height, + prevTxId: prevTxId, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValue = function(txidBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + txidBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValue = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + txid: txid, + inputIndex: inputIndex + }; +}; + +exports.encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex); + return Buffer.concat([ + constants.PREFIXES.SPENTSMAP, + outputTxIdBuffer, + outputIndexBuffer + ]); +}; + +exports.decodeInputKeyMap = function(buffer) { + var txid = buffer.slice(1, 33); + var outputIndex = buffer.readUInt32BE(33); + return { + outputTxId: txid, + outputIndex: outputIndex + }; +}; + +exports.encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { + var inputIndexBuffer = new Buffer(4); + inputIndexBuffer.writeUInt32BE(inputIndex); + return Buffer.concat([ + inputTxIdBuffer, + inputIndexBuffer + ]); +}; + +exports.decodeInputValueMap = function(buffer) { + var txid = buffer.slice(0, 32); + var inputIndex = buffer.readUInt32BE(32); + return { + inputTxId: txid, + inputIndex: inputIndex + }; +}; + +exports.encodeSummaryCacheKey = function(address) { + return Buffer.concat([address.hashBuffer, constants.HASH_TYPES_MAP[address.type]]); +}; + +exports.decodeSummaryCacheKey = function(buffer, network) { + var hashBuffer = buffer.read(20); + var type = constants.HASH_TYPES_READABLE[buffer.read(20, 2).toString('hex')]; + var address = new Address({ + hashBuffer: hashBuffer, + type: type, + network: network + }); + return address; +}; + +exports.encodeSummaryCacheValue = function(cache, tipHeight, tipHash) { + var tipHashBuffer = new Buffer(tipHash, 'hex'); + var buffer = new Buffer(new Array(20)); + buffer.writeUInt32BE(tipHeight); + buffer.writeDoubleBE(cache.result.totalReceived, 4); + buffer.writeDoubleBE(cache.result.balance, 12); + var txidBuffers = []; + for (var i = 0; i < cache.result.txids.length; i++) { + var buf = new Buffer(new Array(36)); + var txid = cache.result.txids[i]; + buf.write(txid, 'hex'); + buf.writeUInt32BE(cache.result.appearanceIds[txid], 32); + txidBuffers.push(buf); + } + var txidsBuffer = Buffer.concat(txidBuffers); + var value = Buffer.concat([tipHashBuffer, buffer, txidsBuffer]); + + return value; +}; + +exports.decodeSummaryCacheValue = function(buffer) { + + var hash = buffer.slice(0, 32).toString('hex'); + var height = buffer.readUInt32BE(32); + var totalReceived = buffer.readDoubleBE(36); + var balance = buffer.readDoubleBE(44); + + // read 32 byte chunks until exhausted + var appearanceIds = {}; + var txids = []; + var pos = 52; + while(pos < buffer.length) { + var txid = buffer.slice(pos, pos + 32).toString('hex'); + var txidHeight = buffer.readUInt32BE(pos + 32); + txids.push(txid); + appearanceIds[txid] = txidHeight; + pos += 36; + } + + var cache = { + height: height, + hash: hash, + result: { + appearanceIds: appearanceIds, + txids: txids, + totalReceived: totalReceived, + balance: balance, + unconfirmedAppearanceIds: {}, // unconfirmed values are never stored in cache + unconfirmedBalance: 0 + } + }; + + return cache; +}; + +exports.getAddressInfo = function(addressStr) { + var addrObj = bitcore.Address(addressStr); + var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type]; + + return { + hashBuffer: addrObj.hashBuffer, + hashTypeBuffer: hashTypeBuffer, + hashTypeReadable: addrObj.type + }; +}; + +/** + * This function is optimized to return address information about an output script + * without constructing a Bitcore Address instance. + * @param {Script} - An instance of a Bitcore Script + * @param {Network|String} - The network for the address + */ +exports.extractAddressInfoFromScript = function(script, network) { + $.checkArgument(network, 'Second argument is expected to be a network'); + var hashBuffer; + var addressType; + var hashTypeBuffer; + if (script.isPublicKeyHashOut()) { + hashBuffer = script.chunks[2].buf; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + addressType = Address.PayToPublicKeyHash; + } else if (script.isScriptHashOut()) { + hashBuffer = script.chunks[1].buf; + hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + addressType = Address.PayToScriptHash; + } else if (script.isPublicKeyOut()) { + var pubkey = script.chunks[0].buf; + var address = Address.fromPublicKey(new PublicKey(pubkey), network); + hashBuffer = address.hashBuffer; + hashTypeBuffer = constants.HASH_TYPES.PUBKEY; + // pay-to-publickey doesn't have an address, however for compatibility + // purposes, we can create an address + addressType = Address.PayToPublicKeyHash; + } else { + return false; + } + return { + hashBuffer: hashBuffer, + hashTypeBuffer: hashTypeBuffer, + addressType: addressType + }; +}; + +module.exports = exports; diff --git a/lib/services/address/history.js b/lib/services/address/history.js index 61e77158..1d707279 100644 --- a/lib/services/address/history.js +++ b/lib/services/address/history.js @@ -4,6 +4,8 @@ var bitcore = require('bitcore-lib'); var async = require('async'); var _ = bitcore.deps._; +var constants = require('./constants'); + /** * This represents an instance that keeps track of data over a series of * asynchronous I/O calls to get the transaction history for a group of @@ -19,12 +21,51 @@ function AddressHistory(args) { } else { this.addresses = [args.addresses]; } - this.transactionInfo = []; - this.combinedArray = []; + + this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; + this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; + + this.addressStrings = []; + for (var i = 0; i < this.addresses.length; i++) { + var address = this.addresses[i]; + if (address instanceof bitcore.Address) { + this.addressStrings.push(address.toString()); + } else if (_.isString(address)) { + this.addressStrings.push(address); + } else { + throw new TypeError('Addresses are expected to be strings'); + } + } + this.detailedArray = []; } -AddressHistory.MAX_ADDRESS_QUERIES = 20; +AddressHistory.prototype._mergeAndSortTxids = function(summaries) { + var appearanceIds = {}; + var unconfirmedAppearanceIds = {}; + for (var i = 0; i < summaries.length; i++) { + var summary = summaries[i]; + for (var key in summary.appearanceIds) { + appearanceIds[key] = summary.appearanceIds[key]; + delete summary.appearanceIds[key]; + } + for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { + unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey]; + delete summary.unconfirmedAppearanceIds[key]; + } + } + var confirmedTxids = Object.keys(appearanceIds); + confirmedTxids.sort(function(a, b) { + // Confirmed are sorted by height + return appearanceIds[a] - appearanceIds[b]; + }); + var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); + unconfirmedTxids.sort(function(a, b) { + // Unconfirmed are sorted by timestamp + return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; + }); + return confirmedTxids.concat(unconfirmedTxids); +}; /** * This function will give detailed history for the configured @@ -33,170 +74,79 @@ AddressHistory.MAX_ADDRESS_QUERIES = 20; */ AddressHistory.prototype.get = function(callback) { var self = this; - var totalCount; + if (this.addresses.length > this.maxAddressesQuery) { + return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); + } - async.eachLimit( - self.addresses, - AddressHistory.MAX_ADDRESS_QUERIES, - function(address, next) { - self.getTransactionInfo(address, next); + if (this.addresses.length === 1) { + var address = this.addresses[0]; + self.node.services.address.getAddressSummary(address, this.options, function(err, summary) { + if (err) { + return callback(err); + } + return self._paginateWithDetails.call(self, summary.txids, callback); + }); + } else { + var opts = _.clone(this.options); + opts.fullTxList = true; + async.map( + self.addresses, + function(address, next) { + self.node.services.address.getAddressSummary(address, opts, next); + }, + function(err, summaries) { + if (err) { + return callback(err); + } + var txids = self._mergeAndSortTxids(summaries); + return self._paginateWithDetails.call(self, txids, callback); + } + ); + } + +}; + +AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { + var self = this; + var totalCount = allTxids.length; + + // Slice the page starting with the most recent + var txids; + if (self.options.from >= 0 && self.options.to >= 0) { + var fromOffset = totalCount - self.options.from; + var toOffset = totalCount - self.options.to; + txids = allTxids.slice(toOffset, fromOffset); + } else { + txids = allTxids; + } + + // Verify that this query isn't too long + if (txids.length > self.maxHistoryQueryLength) { + return callback(new Error( + 'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' + + self.addresses.join(',') + )); + } + + // Reverse to include most recent at the top + txids.reverse(); + + async.eachSeries( + txids, + function(txid, next) { + self.getDetailedInfo(txid, next); }, function(err) { if (err) { return callback(err); } - - self.combineTransactionInfo(); - totalCount = Number(self.combinedArray.length); - self.sortAndPaginateCombinedArray(); - - async.eachSeries( - self.combinedArray, - function(txInfo, next) { - self.getDetailedInfo(txInfo, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); + callback(null, { + totalCount: totalCount, + items: self.detailedArray + }); } ); -}; -/** - * This function will retrieve input and output information for an address - * and set the property `this.transactionInfo`. - * @param {String} address - A base58check encoded address - * @param {Function} next - */ -AddressHistory.prototype.getTransactionInfo = function(address, next) { - var self = this; - - var args = { - start: self.options.start, - end: self.options.end, - queryMempool: _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool - }; - - var outputs; - var inputs; - - async.parallel([ - function(done) { - self.node.services.address.getOutputs(address, args, function(err, result) { - if (err) { - return done(err); - } - outputs = result; - done(); - }); - }, - function(done) { - self.node.services.address.getInputs(address, args, function(err, result) { - if (err) { - return done(err); - } - inputs = result; - done(); - }); - } - ], function(err) { - if (err) { - return next(err); - } - self.transactionInfo = self.transactionInfo.concat(outputs, inputs); - next(); - }); -}; - -/** - * This function combines results from getInputs and getOutputs at - * `this.transactionInfo` to be "txid" unique at `this.combinedArray`. - */ -AddressHistory.prototype.combineTransactionInfo = function() { - var combinedArrayMap = {}; - this.combinedArray = []; - var l = this.transactionInfo.length; - for(var i = 0; i < l; i++) { - var item = this.transactionInfo[i]; - var mapKey = item.txid; - if (combinedArrayMap[mapKey] >= 0) { - var combined = this.combinedArray[combinedArrayMap[mapKey]]; - if (!combined.addresses[item.address]) { - combined.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - } - if (item.outputIndex >= 0) { - combined.satoshis += item.satoshis; - combined.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - combined.addresses[item.address].inputIndexes.push(item.inputIndex); - } - } else { - item.addresses = {}; - item.addresses[item.address] = { - outputIndexes: [], - inputIndexes: [] - }; - if (item.outputIndex >= 0) { - item.addresses[item.address].outputIndexes.push(item.outputIndex); - } else if (item.inputIndex >= 0) { - item.addresses[item.address].inputIndexes.push(item.inputIndex); - } - delete item.outputIndex; - delete item.inputIndex; - delete item.address; - this.combinedArray.push(item); - combinedArrayMap[mapKey] = this.combinedArray.length - 1; - } - } -}; - -/** - * A helper function to sort and slice/paginate the `combinedArray` - */ -AddressHistory.prototype.sortAndPaginateCombinedArray = function() { - this.combinedArray.sort(AddressHistory.sortByHeight); - if (!_.isUndefined(this.options.from) && !_.isUndefined(this.options.to)) { - this.combinedArray = this.combinedArray.slice(this.options.from, this.options.to); - } -}; - -/** - * A helper sort function to order by height and then by date - * for transactions that are in the mempool. - * @param {Object} a - An item from the `combinedArray` - * @param {Object} b - */ -AddressHistory.sortByHeight = function(a, b) { - if (a.height < 0 && b.height < 0) { - // Both are from the mempool, compare timestamps - if (a.timestamp === b.timestamp) { - return 0; - } else { - return a.timestamp < b.timestamp ? 1 : -1; - } - } else if (a.height < 0 && b.height > 0) { - // A is from the mempool and B is in a block - return -1; - } else if (a.height > 0 && b.height < 0) { - // A is in a block and B is in the mempool - return 1; - } else if (a.height === b.height) { - // The heights are equal - return 0; - } else { - // Otherwise compare heights - return a.height < b.height ? 1 : -1; - } }; /** @@ -205,12 +155,12 @@ AddressHistory.sortByHeight = function(a, b) { * @param {Object} txInfo - An item from the `combinedArray` * @param {Function} next */ -AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { +AddressHistory.prototype.getDetailedInfo = function(txid, next) { var self = this; var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; self.node.services.db.getTransactionWithBlockInfo( - txInfo.txid, + txid, queryMempool, function(err, transaction) { if (err) { @@ -218,13 +168,15 @@ AddressHistory.prototype.getDetailedInfo = function(txInfo, next) { } transaction.populateInputs(self.node.services.db, [], function(err) { - if(err) { + if (err) { return next(err); } + var addressDetails = self.getAddressDetailsForTransaction(transaction); + self.detailedArray.push({ - addresses: txInfo.addresses, - satoshis: self.getSatoshisDetail(transaction, txInfo), + addresses: addressDetails.addresses, + satoshis: addressDetails.satoshis, height: transaction.__height, confirmations: self.getConfirmationsDetail(transaction), timestamp: transaction.__timestamp, @@ -251,23 +203,58 @@ AddressHistory.prototype.getConfirmationsDetail = function(transaction) { return confirmations; }; -/** - * A helper function for `getDetailedInfo` for getting the satoshis. - * @param {Transaction} transaction - A transaction populated with previous outputs - * @param {Object} txInfo - An item from `combinedArray` - */ -AddressHistory.prototype.getSatoshisDetail = function(transaction, txInfo) { - var satoshis = txInfo.satoshis || 0; +AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) { + var result = { + addresses: {}, + satoshis: 0 + }; - for(var address in txInfo.addresses) { - if (txInfo.addresses[address].inputIndexes.length >= 0) { - for(var j = 0; j < txInfo.addresses[address].inputIndexes.length; j++) { - satoshis -= transaction.inputs[txInfo.addresses[address].inputIndexes[j]].output.satoshis; + for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { + var input = transaction.inputs[inputIndex]; + if (!input.script) { + continue; + } + var inputAddress = input.script.toAddress(this.node.network); + if (inputAddress) { + var inputAddressString = inputAddress.toString(); + if (this.addressStrings.indexOf(inputAddressString) >= 0) { + if (!result.addresses[inputAddressString]) { + result.addresses[inputAddressString] = { + inputIndexes: [inputIndex], + outputIndexes: [] + }; + } else { + result.addresses[inputAddressString].inputIndexes.push(inputIndex); + } + result.satoshis -= input.output.satoshis; } } } - return satoshis; + for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { + var output = transaction.outputs[outputIndex]; + if (!output.script) { + continue; + } + var outputAddress = output.script.toAddress(this.node.network); + if (outputAddress) { + var outputAddressString = outputAddress.toString(); + if (this.addressStrings.indexOf(outputAddressString) >= 0) { + if (!result.addresses[outputAddressString]) { + result.addresses[outputAddressString] = { + inputIndexes: [], + outputIndexes: [outputIndex] + }; + } else { + result.addresses[outputAddressString].outputIndexes.push(outputIndex); + } + result.satoshis += output.satoshis; + } + } + } + + return result; + }; module.exports = AddressHistory; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index ef7e3e2c..61eed5ef 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -16,11 +16,14 @@ var memdown = require('memdown'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; var Hash = bitcore.crypto.Hash; -var BufferReader = bitcore.encoding.BufferReader; var EventEmitter = require('events').EventEmitter; -var PublicKey = bitcore.PublicKey; var Address = bitcore.Address; var AddressHistory = require('./history'); +var constants = require('./constants'); +var encoding = require('./encoding'); +var InputsTransformStream = require('./streams/inputs-transform'); +var OutputsTransformStream = require('./streams/outputs-transform'); + /** * The Address Service builds upon the Database Service and the Bitcoin Service to add additional @@ -38,8 +41,13 @@ var AddressService = function(options) { this.subscriptions['address/transaction'] = {}; this.subscriptions['address/balance'] = {}; - this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); - this.node.services.bitcoind.on('txleave', this.transactionLeaveHandler.bind(this)); + this._bitcoindTransactionListener = this.transactionHandler.bind(this); + this._bitcoindTransactionLeaveListener = this.transactionLeaveHandler.bind(this); + this.node.services.bitcoind.on('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.on('txleave', this._bitcoindTransactionLeaveListener); + + this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; + this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this._setMempoolIndexPath(); if (options.mempoolMemoryIndex) { @@ -58,42 +66,6 @@ AddressService.dependencies = [ 'db' ]; -AddressService.PREFIXES = { - OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height - SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height - SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output -}; - -AddressService.MEMPREFIXES = { - OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address - SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address - SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output -}; - -// To save space, we're only storing the PubKeyHash or ScriptHash in our index. -// To avoid intentional unspendable collisions, which have been seen on the blockchain, -// we must store the hash type (PK or Script) as well. -AddressService.HASH_TYPES = { - PUBKEY: new Buffer('01', 'hex'), - REDEEMSCRIPT: new Buffer('02', 'hex') -}; - -// Translates from our enum type back into the hash types returned by -// bitcore-lib/address. -AddressService.HASH_TYPES_READABLE = { - '01': 'pubkeyhash', - '02': 'scripthash' -}; - -// Trnaslates from address types to our enum type. -AddressService.HASH_TYPES_MAP = { - 'pubkeyhash': AddressService.HASH_TYPES.PUBKEY, - 'scripthash': AddressService.HASH_TYPES.REDEEMSCRIPT -}; - -AddressService.SPACER_MIN = new Buffer('00', 'hex'); -AddressService.SPACER_MAX = new Buffer('ff', 'hex'); - AddressService.prototype.start = function(callback) { var self = this; @@ -107,6 +79,7 @@ AddressService.prototype.start = function(callback) { } }, function(next) { + // Setup new mempool index if (!fs.existsSync(self.mempoolIndexPath)) { mkdirp(self.mempoolIndexPath, next); } else { @@ -120,7 +93,8 @@ AddressService.prototype.start = function(callback) { db: self.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary', - fillCache: false + fillCache: false, + maxOpenFiles: 200 }, next ); @@ -131,25 +105,33 @@ AddressService.prototype.start = function(callback) { AddressService.prototype.stop = function(callback) { // TODO Keep track of ongoing db requests before shutting down + this.node.services.bitcoind.removeListener('tx', this._bitcoindTransactionListener); + this.node.services.bitcoind.removeListener('txleave', this._bitcoindTransactionLeaveListener); this.mempoolIndex.close(callback); }; /** - * This function will set `this.dataPath` based on `this.node.network`. + * This function will set `this.mempoolIndexPath` based on `this.node.network`. * @private */ AddressService.prototype._setMempoolIndexPath = function() { + this.mempoolIndexPath = this._getDBPathFor('bitcore-addressmempool.db'); +}; + +AddressService.prototype._getDBPathFor = function(dbname) { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); + var path; var regtest = Networks.get('regtest'); if (this.node.network === Networks.livenet) { - this.mempoolIndexPath = this.node.datadir + '/bitcore-addressmempool.db'; + path = this.node.datadir + '/' + dbname; } else if (this.node.network === Networks.testnet) { - this.mempoolIndexPath = this.node.datadir + '/testnet3/bitcore-addressmempool.db'; + path = this.node.datadir + '/testnet3/' + dbname; } else if (this.node.network === regtest) { - this.mempoolIndexPath = this.node.datadir + '/regtest/bitcore-addressmempool.db'; + path = this.node.datadir + '/regtest/' + dbname; } else { throw new Error('Unknown network: ' + this.network); } + return path; }; /** @@ -205,7 +187,7 @@ AddressService.prototype.transactionOutputHandler = function(messages, tx, outpu return; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { return; } @@ -249,6 +231,18 @@ AddressService.prototype.transactionLeaveHandler = function(txInfo) { AddressService.prototype.transactionHandler = function(txInfo, callback) { var self = this; + if (!callback) { + callback = function(err) { + if (err) { + return log.error(err); + } + }; + } + + if (this.node.stopping) { + return callback(); + } + // Basic transaction format is handled by the daemon // and we can safely assume the buffer is properly formatted. var tx = bitcore.Transaction().fromBuffer(txInfo.buffer); @@ -260,14 +254,6 @@ AddressService.prototype.transactionHandler = function(txInfo, callback) { this.transactionOutputHandler(messages, tx, i, !txInfo.mempool); } - if (!callback) { - callback = function(err) { - if (err) { - return log.error(err); - } - }; - } - function finish(err) { if (err) { return callback(err); @@ -297,6 +283,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { /* jshint maxstatements: 100 */ var operations = []; + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(new Date().getTime()); var action = 'put'; if (!add) { @@ -312,7 +300,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { if (!output.script) { continue; } - var addressInfo = this._extractAddressInfoFromScript(output.script); + var addressInfo = encoding.extractAddressInfoFromScript(output.script, this.node.network); if (!addressInfo) { continue; } @@ -322,14 +310,18 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { outputIndexBuffer.writeUInt32BE(outputIndex); var outKey = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, addressInfo.hashBuffer, addressInfo.hashTypeBuffer, txidBuffer, outputIndexBuffer ]); - var outValue = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var outValue = encoding.encodeOutputMempoolValue( + output.satoshis, + timestampBuffer, + output._scriptBuffer + ); operations.push({ type: action, @@ -347,7 +339,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { inputOutputIndexBuffer.writeUInt32BE(input.outputIndex); // Add an additional small spent index for fast synchronous lookups - var spentIndexSyncKey = this._encodeSpentIndexSyncKey( + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( input.prevTxId, input.outputIndex ); @@ -359,7 +351,7 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { // Add a more detailed spent index with values var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, input.prevTxId, inputOutputIndexBuffer ]); @@ -380,15 +372,15 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { var inputHashType; if (input.script.isPublicKeyHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHashBuffer = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } var inputKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, inputHashBuffer, inputHashType, input.prevTxId, @@ -396,7 +388,8 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { ]); var inputValue = Buffer.concat([ txidBuffer, - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); operations.push({ type: action, @@ -417,42 +410,6 @@ AddressService.prototype.updateMempoolIndex = function(tx, add, callback) { this.mempoolIndex.batch(operations, callback); }; -/** - * This function is optimized to return address information about an output script - * without constructing a Bitcore Address instance. - * @param {Script} - An instance of a Bitcore Script - * @private - */ -AddressService.prototype._extractAddressInfoFromScript = function(script) { - var hashBuffer; - var addressType; - var hashTypeBuffer; - if (script.isPublicKeyHashOut()) { - hashBuffer = script.chunks[2].buf; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - addressType = Address.PayToPublicKeyHash; - } else if (script.isScriptHashOut()) { - hashBuffer = script.chunks[1].buf; - hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - addressType = Address.PayToScriptHash; - } else if (script.isPublicKeyOut()) { - var pubkey = script.chunks[0].buf; - var address = Address.fromPublicKey(new PublicKey(pubkey), this.node.network); - hashBuffer = address.hashBuffer; - hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; - // pay-to-publickey doesn't have an address, however for compatibility - // purposes, we can create an address - addressType = Address.PayToPublicKeyHash; - } else { - return false; - } - return { - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - addressType: addressType - }; -}; - /** * The Database Service will run this function when blocks are connected and * disconnected to the chain during syncing and reorganizations. @@ -494,7 +451,7 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { continue; } - var addressInfo = this._extractAddressInfoFromScript(script); + var addressInfo = encoding.extractAddressInfoFromScript(script, this.node.network); if (!addressInfo) { continue; } @@ -504,9 +461,9 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { // can have a time that is previous to the previous block (however not // less than the mean of the 11 previous blocks) and not greater than 2 // hours in the future. - var key = this._encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, - height, txidBuffer, outputIndex); - var value = this._encodeOutputValue(output.satoshis, output._scriptBuffer); + var key = encoding.encodeOutputKey(addressInfo.hashBuffer, addressInfo.hashTypeBuffer, + height, txidBuffer, outputIndex); + var value = encoding.encodeOutputValue(output.satoshis, output._scriptBuffer); operations.push({ type: action, key: key, @@ -549,10 +506,10 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { if (input.script.isPublicKeyHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[1].buf); - inputHashType = AddressService.HASH_TYPES.PUBKEY; + inputHashType = constants.HASH_TYPES.PUBKEY; } else if (input.script.isScriptHashIn()) { inputHash = Hash.sha256ripemd160(input.script.chunks[input.script.chunks.length - 1].buf); - inputHashType = AddressService.HASH_TYPES.REDEEMSCRIPT; + inputHashType = constants.HASH_TYPES.REDEEMSCRIPT; } else { continue; } @@ -560,8 +517,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { var prevTxIdBuffer = new Buffer(input.prevTxId, 'hex'); // To be able to query inputs by address and spent height - var inputKey = this._encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); - var inputValue = this._encodeInputValue(txidBuffer, inputIndex); + var inputKey = encoding.encodeInputKey(inputHash, inputHashType, height, prevTxIdBuffer, input.outputIndex); + var inputValue = encoding.encodeInputValue(txidBuffer, inputIndex); operations.push({ type: action, @@ -570,8 +527,8 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); // To be able to search for an input spending an output - var inputKeyMap = this._encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); - var inputValueMap = this._encodeInputValueMap(txidBuffer, inputIndex); + var inputKeyMap = encoding.encodeInputKeyMap(prevTxIdBuffer, input.outputIndex); + var inputValueMap = encoding.encodeInputValueMap(txidBuffer, inputIndex); operations.push({ type: action, @@ -587,168 +544,6 @@ AddressService.prototype.blockHandler = function(block, addOutput, callback) { }); }; -AddressService.prototype._encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - txidBuffer, - outputIndexBuffer - ]); - return key.toString('binary'); -}; - -AddressService.prototype._encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - var key = Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - txidBuffer, - outputIndexBuffer - ]); - return key; -}; - -AddressService.prototype._decodeOutputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var txid = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - txid: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeOutputValue = function(satoshis, scriptBuffer) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([satoshisBuffer, scriptBuffer]); -}; - -AddressService.prototype._decodeOutputValue = function(buffer) { - var satoshis = buffer.readDoubleBE(0); - var scriptBuffer = buffer.slice(8, buffer.length); - return { - satoshis: satoshis, - scriptBuffer: scriptBuffer - }; -}; - -AddressService.prototype._encodeInputKey = function(hashBuffer, hashTypeBuffer, height, prevTxIdBuffer, outputIndex) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - heightBuffer, - prevTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKey = function(buffer) { - var reader = new BufferReader(buffer); - var prefix = reader.read(1); - var hashBuffer = reader.read(20); - var hashTypeBuffer = reader.read(1); - var spacer = reader.read(1); - var height = reader.readUInt32BE(); - var prevTxId = reader.read(32); - var outputIndex = reader.readUInt32BE(); - return { - prefix: prefix, - hashBuffer: hashBuffer, - hashTypeBuffer: hashTypeBuffer, - height: height, - prevTxId: prevTxId, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValue = function(txidBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - txidBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValue = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - txid: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._encodeInputKeyMap = function(outputTxIdBuffer, outputIndex) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - return Buffer.concat([ - AddressService.PREFIXES.SPENTSMAP, - outputTxIdBuffer, - outputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputKeyMap = function(buffer) { - var txid = buffer.slice(1, 33); - var outputIndex = buffer.readUInt32BE(33); - return { - outputTxId: txid, - outputIndex: outputIndex - }; -}; - -AddressService.prototype._encodeInputValueMap = function(inputTxIdBuffer, inputIndex) { - var inputIndexBuffer = new Buffer(4); - inputIndexBuffer.writeUInt32BE(inputIndex); - return Buffer.concat([ - inputTxIdBuffer, - inputIndexBuffer - ]); -}; - -AddressService.prototype._decodeInputValueMap = function(buffer) { - var txid = buffer.slice(0, 32); - var inputIndex = buffer.readUInt32BE(32); - return { - inputTxId: txid, - inputIndex: inputIndex - }; -}; - -AddressService.prototype._getAddressInfo = function(addressStr) { - var addrObj = bitcore.Address(addressStr); - var hashTypeBuffer = AddressService.HASH_TYPES_MAP[addrObj.type]; - - return { - hashBuffer: addrObj.hashBuffer, - hashTypeBuffer: hashTypeBuffer, - hashTypeReadable: addrObj.type - }; -}; - /** * This function is responsible for emitting events to any subscribers to the * `address/transaction` event. @@ -926,12 +721,12 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options txidBuffer = new Buffer(txid, 'hex'); } if (options.queryMempool) { - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); if (this.mempoolSpentIndex[spentIndexSyncKey]) { return this._getSpentMempool(txidBuffer, outputIndex, callback); } } - var key = this._encodeInputKeyMap(txidBuffer, outputIndex); + var key = encoding.encodeInputKeyMap(txidBuffer, outputIndex); var dbOptions = { valueEncoding: 'binary', keyEncoding: 'binary' @@ -942,7 +737,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options } else if (err) { return callback(err); } - var value = self._decodeInputValueMap(buffer); + var value = encoding.decodeInputValueMap(buffer); callback(null, { inputTxId: value.inputTxId.toString('hex'), inputIndex: value.inputIndex @@ -950,10 +745,86 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options }); }; +/** + * A streaming equivalent to `getInputs`, and returns a transform stream with data + * emitted in the same format as `getInputs`. + * + * @param {String} addressStr - The relevant address + * @param {Object} options - Additional options for query the outputs + * @param {Number} [options.start] - The relevant start block height + * @param {Number} [options.end] - The relevant end block height + * @param {Function} callback + */ +AddressService.prototype.createInputsStream = function(addressStr, options) { + var inputStream = new InputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); + + var stream = this.createInputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + inputStream.emit('error', err); + inputStream.end(); + }).pipe(inputStream); + + return stream; + +}; + +AddressService.prototype.createInputsDBStream = function(addressStr, options) { + var stream; + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + + if (options.start >= 0 && options.end >= 0) { + + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end, 0); + + var startBuffer = new Buffer(4); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var adjustedStart = options.start + 1; + startBuffer.writeUInt32BE(adjustedStart, 0); + + stream = this.node.services.db.store.createReadStream({ + gt: Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + endBuffer + ]), + lt: Buffer.concat([ + constants.PREFIXES.SPENTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } else { + var allKey = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); + stream = this.node.services.db.store.createReadStream({ + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } + + return stream; +}; + /** * Will give inputs that spend previous outputs for an address as an object with: * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction input * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -971,67 +842,20 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var self = this; var inputs = []; - var stream; - var addrObj = this._getAddressInfo(addressStr); + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; - if (!hashTypeBuffer) { - return callback(new Error('Unknown address type: ' + addrObj.hashTypeReadable + ' for address: ' + addressStr)); - } - if (options.start && options.end) { - - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); - - var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); - - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - endBuffer - ]), - lte: Buffer.concat([ - AddressService.PREFIXES.SPENTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } - - stream.on('data', function(data) { - - var key = self._decodeInputKey(data.key); - var value = self._decodeInputValue(data.value); - - var input = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: value.txid.toString('hex'), - inputIndex: value.inputIndex, - height: key.height, - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; + var stream = this.createInputsStream(addressStr, options); + stream.on('data', function(input) { inputs.push(input); - + if (inputs.length > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for address '+ addressStr); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + stream.end(); + } }); var error; @@ -1042,7 +866,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('finish', function() { if (error) { return callback(error); } @@ -1071,16 +895,16 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' @@ -1089,15 +913,17 @@ AddressService.prototype._getInputsMempool = function(addressStr, hashBuffer, ha stream.on('data', function(data) { var txid = data.value.slice(0, 32); var inputIndex = data.value.readUInt32BE(32); - var output = { + var timestamp = data.value.readDoubleBE(36); + var input = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer inputIndex: inputIndex, + timestamp: timestamp, height: -1, confirmations: 0 }; - mempoolInputs.push(output); + mempoolInputs.push(input); }); var error; @@ -1121,7 +947,7 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var spentIndexKey = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTSMAP, + constants.MEMPREFIXES.SPENTSMAP, txidBuffer, outputIndexBuffer ]); @@ -1142,10 +968,79 @@ AddressService.prototype._getSpentMempool = function(txidBuffer, outputIndex, ca ); }; +AddressService.prototype.createOutputsStream = function(addressStr, options) { + var outputStream = new OutputsTransformStream({ + address: new Address(addressStr, this.node.network), + tipHeight: this.node.services.db.tip.__height + }); + + var stream = this.createOutputsDBStream(addressStr, options) + .on('error', function(err) { + // Forward the error + outputStream.emit('error', err); + outputStream.end(); + }) + .pipe(outputStream); + + return stream; + +}; + +AddressService.prototype.createOutputsDBStream = function(addressStr, options) { + + var addrObj = encoding.getAddressInfo(addressStr); + var hashBuffer = addrObj.hashBuffer; + var hashTypeBuffer = addrObj.hashTypeBuffer; + var stream; + + if (options.start >= 0 && options.end >= 0) { + + var endBuffer = new Buffer(4); + endBuffer.writeUInt32BE(options.end, 0); + + var startBuffer = new Buffer(4); + // Because the key has additional data following it, we don't have an ability + // to use "gte" or "lte" we can only use "gt" and "lt", we therefore need to adjust the number + // to be one value larger to include it. + var startAdjusted = options.start + 1; + startBuffer.writeUInt32BE(startAdjusted, 0); + + stream = this.node.services.db.store.createReadStream({ + gt: Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + endBuffer + ]), + lt: Buffer.concat([ + constants.PREFIXES.OUTPUTS, + hashBuffer, + hashTypeBuffer, + constants.SPACER_MIN, + startBuffer + ]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } else { + var allKey = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); + stream = this.node.services.db.store.createReadStream({ + gt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MIN]), + lt: Buffer.concat([allKey, constants.SPACER_HEIGHT_MAX]), + valueEncoding: 'binary', + keyEncoding: 'binary' + }); + } + + return stream; + +}; + /** * Will give outputs for an address as an object with: * address - The base58check encoded address - * hashType - The type of the address, e.g. 'pubkeyhash' or 'scripthash' + * hashtype - The type of the address, e.g. 'pubkeyhash' or 'scripthash' * txid - A string of the transaction hash * outputIndex - A number of corresponding transaction output * height - The height of the block the transaction was included, will be -1 for mempool transactions @@ -1165,7 +1060,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - var addrObj = this._getAddressInfo(addressStr); + var addrObj = encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; if (!hashTypeBuffer) { @@ -1173,61 +1068,15 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } var outputs = []; - var stream; - - if (options.start && options.end) { - - var startBuffer = new Buffer(4); - startBuffer.writeUInt32BE(options.start + 1); - var endBuffer = new Buffer(4); - endBuffer.writeUInt32BE(options.end); - - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - endBuffer - ]), - lte: Buffer.concat([ - AddressService.PREFIXES.OUTPUTS, - hashBuffer, - hashTypeBuffer, - AddressService.SPACER_MIN, - startBuffer - ]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } else { - var allKey = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer]); - stream = this.node.services.db.store.createReadStream({ - gte: Buffer.concat([allKey, AddressService.SPACER_MIN]), - lte: Buffer.concat([allKey, AddressService.SPACER_MAX]), - valueEncoding: 'binary', - keyEncoding: 'binary' - }); - } + var stream = this.createOutputsStream(addressStr, options); stream.on('data', function(data) { - - var key = self._decodeOutputKey(data.key); - var value = self._decodeOutputValue(data.value); - - var output = { - address: addressStr, - hashType: addrObj.hashTypeReadable, - txid: key.txid.toString('hex'), //TODO use a buffer - outputIndex: key.outputIndex, - height: key.height, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: self.node.services.db.tip.__height - key.height + 1 - }; - - outputs.push(output); - + outputs.push(data); + if (outputs.length > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for address ' + addressStr); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + stream.end(); + } }); var error; @@ -1238,7 +1087,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { } }); - stream.on('close', function() { + stream.on('finish', function() { if (error) { return callback(error); } @@ -1266,32 +1115,34 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h var stream = self.mempoolIndex.createReadStream({ gte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MIN + constants.SPACER_MIN ]), lte: Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, - AddressService.SPACER_MAX + constants.SPACER_MAX ]), valueEncoding: 'binary', keyEncoding: 'binary' }); stream.on('data', function(data) { - // Format of data: prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 + // Format of data: + // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 var txid = data.key.slice(22, 54); var outputIndex = data.key.readUInt32BE(54); - var value = self._decodeOutputValue(data.value); + var value = encoding.decodeOutputMempoolValue(data.value); var output = { address: addressStr, - hashType: AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], + hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], txid: txid.toString('hex'), //TODO use a buffer outputIndex: outputIndex, height: -1, + timestamp: value.timestamp, satoshis: value.satoshis, script: value.scriptBuffer.toString('hex'), //TODO use a buffer confirmations: 0 @@ -1407,7 +1258,7 @@ AddressService.prototype.isSpent = function(output, options, callback) { var spent = self.node.services.bitcoind.isSpent(txid, output.outputIndex); if (!spent && queryMempool) { var txidBuffer = new Buffer(txid, 'hex'); - var spentIndexSyncKey = this._encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); spent = self.mempoolSpentIndex[spentIndexSyncKey] ? true : false; } setImmediate(function() { @@ -1474,111 +1325,248 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba * @param {Boolean} [options.noTxList] - if set, txid array will not be included * @param {Function} callback */ -AddressService.prototype.getAddressSummary = function(address, options, callback) { +AddressService.prototype.getAddressSummary = function(addressArg, options, callback) { var self = this; - var opt = { - queryMempool: true + var startTime = new Date(); + var address = new Address(addressArg); + + if (_.isUndefined(options.queryMempool)) { + options.queryMempool = true; + } + + async.waterfall([ + function(next) { + self._getAddressConfirmedSummary(address, options, next); + }, + function(result, next) { + self._getAddressMempoolSummary(address, options, result, next); + }, + function(result, next) { + self._setAndSortTxidsFromAppearanceIds(result, next); + } + ], function(err, result) { + if (err) { + return callback(err); + } + + var summary = self._transformAddressSummaryFromResult(result, options); + + var timeDelta = new Date() - startTime; + if (timeDelta > 5000) { + var seconds = Math.round(timeDelta / 1000); + log.warn('Slow (' + seconds + 's) getAddressSummary request for address: ' + address.toString()); + } + + callback(null, summary); + + }); + +}; + +AddressService.prototype._getAddressConfirmedSummary = function(address, options, callback) { + var self = this; + var baseResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 }; - var outputs; - var inputs; + async.waterfall([ + function(next) { + self._getAddressConfirmedInputsSummary(address, baseResult, options, next); + }, + function(result, next) { + self._getAddressConfirmedOutputsSummary(address, result, options, next); + } + ], callback); - async.parallel( - [ - function(next) { - self.getInputs(address, opt, function(err, ins) { - inputs = ins; - next(err); - }); - }, - function(next) { - self.getOutputs(address, opt, function(err, outs) { - outputs = outs; - next(err); - }); - } - ], - function(err) { - if(err) { - return callback(err); +}; + +AddressService.prototype._getAddressConfirmedInputsSummary = function(address, result, options, callback) { + $.checkArgument(address instanceof Address); + var self = this; + var error = null; + var count = 0; + + var inputsStream = self.createInputsStream(address, options); + inputsStream.on('data', function(input) { + var txid = input.txid; + result.appearanceIds[txid] = input.height; + + count++; + + if (count > self.maxInputsQueryLength) { + log.warn('Tried to query too many inputs (' + self.maxInputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of inputs (' + self.maxInputsQueryLength + ') per query reached'); + inputsStream.end(); + } + + }); + + inputsStream.on('error', function(err) { + error = err; + }); + + inputsStream.on('end', function() { + if (error) { + return callback(error); + } + callback(null, result); + }); +}; + +AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, result, options, callback) { + $.checkArgument(address instanceof Address); + $.checkArgument(!_.isUndefined(result) && + !_.isUndefined(result.appearanceIds) && + !_.isUndefined(result.unconfirmedAppearanceIds)); + + var self = this; + var count = 0; + + var outputStream = self.createOutputsStream(address, options); + + outputStream.on('data', function(output) { + + var txid = output.txid; + var outputIndex = output.outputIndex; + + // Bitcoind's isSpent only works for confirmed transactions + var spentDB = self.node.services.bitcoind.isSpent(txid, outputIndex); + result.totalReceived += output.satoshis; + result.appearanceIds[txid] = output.height; + + if (!spentDB) { + result.balance += output.satoshis; + } + + if (options.queryMempool) { + // Check to see if this output is spent in the mempool and if so + // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(txid, 'hex'), // TODO: get buffer directly + outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (spentMempool) { + result.unconfirmedBalance -= output.satoshis; } + } - var totalReceived = 0; - var totalSpent = 0; - var balance = 0; - var unconfirmedBalance = 0; - var appearanceIds = {}; - var unconfirmedAppearanceIds = {}; - var txids = []; + count++; - for(var i = 0; i < outputs.length; i++) { - // Bitcoind's isSpent only works for confirmed transactions - var spentDB = self.node.services.bitcoind.isSpent(outputs[i].txid, outputs[i].outputIndex); - var spentIndexSyncKey = self._encodeSpentIndexSyncKey( - new Buffer(outputs[i].txid, 'hex'), // TODO: get buffer directly - outputs[i].outputIndex - ); - var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + if (count > self.maxOutputsQueryLength) { + log.warn('Tried to query too many outputs (' + self.maxOutputsQueryLength + ') for summary of address ' + address.toString()); + error = new Error('Maximum number of outputs (' + self.maxOutputsQueryLength + ') per query reached'); + outputStream.end(); + } - txids.push(outputs[i]); + }); - if(outputs[i].confirmations) { - totalReceived += outputs[i].satoshis; - balance += outputs[i].satoshis; - appearanceIds[outputs[i].txid] = true; - } else { - unconfirmedBalance += outputs[i].satoshis; - unconfirmedAppearanceIds[outputs[i].txid] = true; + var error = null; + + outputStream.on('error', function(err) { + error = err; + }); + + outputStream.on('end', function() { + if (error) { + return callback(error); + } + callback(null, result); + }); + +}; + +AddressService.prototype._setAndSortTxidsFromAppearanceIds = function(result, callback) { + result.txids = Object.keys(result.appearanceIds); + result.txids.sort(function(a, b) { + return result.appearanceIds[a] - result.appearanceIds[b]; + }); + result.unconfirmedTxids = Object.keys(result.unconfirmedAppearanceIds); + result.unconfirmedTxids.sort(function(a, b) { + return result.unconfirmedAppearanceIds[a] - result.unconfirmedAppearanceIds[b]; + }); + callback(null, result); +}; + +AddressService.prototype._getAddressMempoolSummary = function(address, options, result, callback) { + var self = this; + + // Skip if the options do not want to include the mempool + if (!options.queryMempool) { + return callback(null, result); + } + + var addressStr = address.toString(); + var hashBuffer = address.hashBuffer; + var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; + + async.waterfall([ + function(next) { + self._getInputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolInputs) { + if (err) { + return next(err); } + for(var i = 0; i < mempoolInputs.length; i++) { + var input = mempoolInputs[i]; + result.unconfirmedAppearanceIds[input.txid] = input.timestamp; + } + next(null, result); + }); - if(spentDB || spentMempool) { - if(spentDB) { - totalSpent += outputs[i].satoshis; - balance -= outputs[i].satoshis; - } else if(!outputs[i].confirmations) { - unconfirmedBalance -= outputs[i].satoshis; + }, function(result, next) { + self._getOutputsMempool(addressStr, hashBuffer, hashTypeBuffer, function(err, mempoolOutputs) { + if (err) { + return next(err); + } + for(var i = 0; i < mempoolOutputs.length; i++) { + var output = mempoolOutputs[i]; + + result.unconfirmedAppearanceIds[output.txid] = output.timestamp; + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(output.txid, 'hex'), // TODO: get buffer directly + output.outputIndex + ); + var spentMempool = self.mempoolSpentIndex[spentIndexSyncKey]; + // Only add this to the balance if it's not spent in the mempool already + if (!spentMempool) { + result.unconfirmedBalance += output.satoshis; } } - } - - for(var j = 0; j < inputs.length; j++) { - if (inputs[j].confirmations) { - appearanceIds[inputs[j].txid] = true; - } else { - unconfirmedAppearanceIds[outputs[j].txid] = true; - } - } - - var summary = { - totalReceived: totalReceived, - totalSpent: totalSpent, - balance: balance, - unconfirmedBalance: unconfirmedBalance, - appearances: Object.keys(appearanceIds).length, - unconfirmedAppearances: Object.keys(unconfirmedAppearanceIds).length - }; - - if(!options.noTxList) { - for(var i = 0; i < inputs.length; i++) { - txids.push(inputs[i]); - } - - // sort by height - txids = txids.sort(function(a, b) { - return a.height > b.height ? 1 : -1; - }).map(function(obj) { - return obj.txid; - }).filter(function(value, index, self) { - return self.indexOf(value) === index; - }); - - summary.txids = txids; - } - - callback(null, summary); + next(null, result); + }); } - ); + ], callback); +}; + +AddressService.prototype._transformAddressSummaryFromResult = function(result, options) { + + var confirmedTxids = result.txids; + var unconfirmedTxids = result.unconfirmedTxids; + + var summary = { + totalReceived: result.totalReceived, + totalSpent: result.totalReceived - result.balance, + balance: result.balance, + appearances: confirmedTxids.length, + unconfirmedBalance: result.unconfirmedBalance, + unconfirmedAppearances: unconfirmedTxids.length + }; + + if (options.fullTxList) { + summary.appearanceIds = result.appearanceIds; + summary.unconfirmedAppearanceIds = result.unconfirmedAppearanceIds; + } else if (!options.noTxList) { + summary.txids = confirmedTxids.concat(unconfirmedTxids); + } + + return summary; + }; module.exports = AddressService; diff --git a/lib/services/address/streams/inputs-transform.js b/lib/services/address/streams/inputs-transform.js new file mode 100644 index 00000000..8b8f71d3 --- /dev/null +++ b/lib/services/address/streams/inputs-transform.js @@ -0,0 +1,40 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function InputsTransformStream(options) { + $.checkArgument(options.address instanceof bitcore.Address); + Transform.call(this, { + objectMode: true + }); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(InputsTransformStream, Transform); + +InputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeInputKey(chunk.key); + var value = encodingUtil.decodeInputValue(chunk.value); + + var input = { + address: this._addressStr, + hashType: this._address.type, + txid: value.txid.toString('hex'), + inputIndex: value.inputIndex, + height: key.height, + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(input); + callback(); + +}; + +module.exports = InputsTransformStream; diff --git a/lib/services/address/streams/outputs-transform.js b/lib/services/address/streams/outputs-transform.js new file mode 100644 index 00000000..b9c8e8d3 --- /dev/null +++ b/lib/services/address/streams/outputs-transform.js @@ -0,0 +1,42 @@ +'use strict'; + +var Transform = require('stream').Transform; +var inherits = require('util').inherits; +var bitcore = require('bitcore-lib'); +var encodingUtil = require('../encoding'); +var $ = bitcore.util.preconditions; + +function OutputsTransformStream(options) { + Transform.call(this, { + objectMode: true + }); + $.checkArgument(options.address instanceof bitcore.Address); + this._address = options.address; + this._addressStr = this._address.toString(); + this._tipHeight = options.tipHeight; +} +inherits(OutputsTransformStream, Transform); + +OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) { + var self = this; + + var key = encodingUtil.decodeOutputKey(chunk.key); + var value = encodingUtil.decodeOutputValue(chunk.value); + + var output = { + address: this._addressStr, + hashType: this._address.type, + txid: key.txid.toString('hex'), //TODO use a buffer + outputIndex: key.outputIndex, + height: key.height, + satoshis: value.satoshis, + script: value.scriptBuffer.toString('hex'), //TODO use a buffer + confirmations: this._tipHeight - key.height + 1 + }; + + self.push(output); + callback(); + +}; + +module.exports = OutputsTransformStream; diff --git a/lib/services/db.js b/lib/services/db.js index 0c655051..c3c3dfd4 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -46,6 +46,8 @@ function DB(options) { this._setDataPath(); + this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; + this.levelupStore = leveldown; if (options.store) { this.levelupStore = options.store; @@ -68,6 +70,8 @@ DB.PREFIXES = { TIP: new Buffer('04', 'hex') }; +DB.DEFAULT_MAX_OPEN_FILES = 200; + /** * This function will set `this.dataPath` based on `this.node.network`. * @private @@ -98,7 +102,7 @@ DB.prototype.start = function(callback) { } this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore }); + this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles }); this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); this.once('ready', function() { diff --git a/package.json b/package.json index a92577a2..33ad01af 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,8 @@ "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", - "leveldown": "^1.4.2", - "levelup": "^1.2.1", + "leveldown": "^1.4.3", + "levelup": "^1.3.1", "liftoff": "^2.2.0", "memdown": "^1.0.0", "mkdirp": "0.5.0", diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js new file mode 100644 index 00000000..e5ba7376 --- /dev/null +++ b/test/services/address/encoding.unit.js @@ -0,0 +1,103 @@ +'use strict'; + +var chai = require('chai'); +var should = chai.should(); +var sinon = require('sinon'); +var bitcorenode = require('../../../'); +var bitcore = require('bitcore-lib'); +var Address = bitcore.Address; +var Script = bitcore.Script; +var AddressService = bitcorenode.services.Address; +var Networks = bitcore.Networks; +var encoding = require('../../../lib/services/address/encoding'); + +var mockdb = { +}; + +var mocknode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub() + } + } +}; + +describe('Address Service Encoding', function() { + + describe('#encodeSpentIndexSyncKey', function() { + it('will encode to 36 bytes (string)', function() { + var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + key.length.should.equal(36); + }); + it('will be able to decode encoded value', function() { + var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; + var txidBuffer = new Buffer(txid, 'hex'); + var key = encoding.encodeSpentIndexSyncKey(txidBuffer, 12); + var keyBuffer = new Buffer(key, 'binary'); + keyBuffer.slice(0, 32).toString('hex').should.equal(txid); + var outputIndex = keyBuffer.readUInt32BE(32); + outputIndex.should.equal(12); + }); + }); + + describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { + var encoded; + var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputKeyMap(outputTxIdBuffer, 13); + }); + it('decode key', function() { + var key = encoding.decodeInputKeyMap(encoded); + key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); + key.outputIndex.should.equal(13); + }); + }); + + describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { + var encoded; + var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); + it('encode key', function() { + encoded = encoding.encodeInputValueMap(inputTxIdBuffer, 7); + }); + it('decode key', function() { + var key = encoding.decodeInputValueMap(encoded); + key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); + key.inputIndex.should.equal(7); + }); + }); + + + describe('#extractAddressInfoFromScript', function() { + it('pay-to-publickey', function() { + var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); + var script = Script.buildPublicKeyOut(pubkey); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); + }); + it('pay-to-publickeyhash', function() { + var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToPublicKeyHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('pay-to-scripthash', function() { + var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.addressType.should.equal(Address.PayToScriptHash); + info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); + }); + it('non-address script type', function() { + var buf = new Buffer(40); + buf.fill(0); + var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); + var info = encoding.extractAddressInfoFromScript(script, Networks.livenet); + info.should.equal(false); + }); + }); + +}); diff --git a/test/services/address/history.unit.js b/test/services/address/history.unit.js index 8092a2f0..6ddb7369 100644 --- a/test/services/address/history.unit.js +++ b/test/services/address/history.unit.js @@ -2,6 +2,7 @@ var should = require('chai').should(); var sinon = require('sinon'); +var bitcore = require('bitcore-lib'); var Transaction = require('../../../lib/transaction'); var AddressHistory = require('../../../lib/services/address/history'); @@ -23,8 +24,6 @@ describe('Address Service History', function() { history.node.should.equal(node); history.options.should.equal(options); history.addresses.should.equal(addresses); - history.transactionInfo.should.deep.equal([]); - history.combinedArray.should.deep.equal([]); history.detailedArray.should.deep.equal([]); }); it('will set addresses an array if only sent a string', function() { @@ -38,501 +37,321 @@ describe('Address Service History', function() { }); describe('#get', function() { - it('will complete the async each limit series', function(done) { - var addresses = [address]; + it('will give an error if length of addresses is too long', function(done) { + var node = {}; + var options = {}; + var addresses = []; + for (var i = 0; i < 101; i++) { + addresses.push(address); + } var history = new AddressHistory({ - node: {}, - options: {}, + node: node, + options: options, addresses: addresses }); - var expected = [{}]; - history.detailedArray = expected; - history.combinedArray = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.combineTransactionInfo = sinon.stub(); - history.sortAndPaginateCombinedArray = sinon.stub(); - history.getDetailedInfo = sinon.stub().callsArg(1); - history.sortTransactionsIntoArray = sinon.stub(); - history.get(function(err, results) { - if (err) { - throw err; + history.maxAddressesQuery = 100; + history.get(function(err) { + should.exist(err); + err.message.match(/Maximum/); + done(); + }); + }); + it('give error from getAddressSummary with one address', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test')) + } } - history.getTransactionInfo.callCount.should.equal(1); - history.getDetailedInfo.callCount.should.equal(1); - history.combineTransactionInfo.callCount.should.equal(1); - history.sortAndPaginateCombinedArray.callCount.should.equal(1); - results.should.deep.equal({ - totalCount: 1, - items: expected - }); - done(); - }); - }); - it('handle an error from getDetailedInfo', function(done) { - var addresses = [address]; - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArg(1); - history.paginateSortedArray = sinon.stub(); - history.getDetailedInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('handle an error from getTransactionInfo', function(done) { - var addresses = [address]; - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: addresses - }); - var expected = [{}]; - history.sortedArray = expected; - history.transactionInfo = [{}]; - history.getTransactionInfo = sinon.stub().callsArgWith(1, new Error('test')); - history.get(function(err) { - err.message.should.equal('test'); - done(); - }); - }); - }); - - describe('#getTransactionInfo', function() { - it('will handle an error from getInputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, new Error('test')) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will handle an error from getOutputs', function(done) { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, new Error('test')), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: {}, - addresses: [] - }); - history.getTransactionInfo(address, function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will call getOutputs and getInputs with the correct options', function() { - var startTimestamp = 1438289011844; - var endTimestamp = 1438289012412; - var expectedArgs = { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true }; + var options = {}; + var addresses = [address]; var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } - } - }, - options: { - start: new Date(startTimestamp * 1000), - end: new Date(endTimestamp * 1000), - queryMempool: true - }, - addresses: [] + node: node, + options: options, + addresses: addresses }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; - } - history.node.services.address.getOutputs.args[0][1].should.deep.equal(expectedArgs); - history.node.services.address.getInputs.args[0][1].should.deep.equal(expectedArgs); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); }); }); - it('will handle empty results from getOutputs and getInputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, []), - getInputs: sinon.stub().callsArgWith(2, null, []) - } + it('give error from getAddressSummary with multiple addresses', function(done) { + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, new Error('test2')) } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; } - history.transactionInfo.length.should.equal(1); - history.node.services.address.getOutputs.args[0][0].should.equal(address); + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history.get(function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); }); }); - it('will concatenate outputs and inputs', function() { - var history = new AddressHistory({ - node: { - services: { - address: { - getOutputs: sinon.stub().callsArgWith(2, null, [{}]), - getInputs: sinon.stub().callsArgWith(2, null, [{}]) - } + it('will query get address summary directly with one address', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) } - }, - options: {}, - addresses: [] - }); - history.transactionInfo = [{}]; - history.getTransactionInfo(address, function(err) { - if (err) { - throw err; } - history.transactionInfo.length.should.equal(3); - history.node.services.address.getOutputs.args[0][0].should.equal(address); + }; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub(); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(1); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.equal(options); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(0); + done(); + }); + }); + it('will merge multiple summaries with multiple addresses', function(done) { + var txids = []; + var summary = { + txids: txids + }; + var node = { + services: { + address: { + getAddressSummary: sinon.stub().callsArgWith(2, null, summary) + } + } + }; + var options = {}; + var addresses = [address, address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + history._mergeAndSortTxids = sinon.stub().returns(txids); + history._paginateWithDetails = sinon.stub().callsArg(1); + history.get(function() { + history.node.services.address.getAddressSummary.callCount.should.equal(2); + history.node.services.address.getAddressSummary.args[0][0].should.equal(address); + history.node.services.address.getAddressSummary.args[0][1].should.deep.equal({ + fullTxList: true + }); + history._paginateWithDetails.callCount.should.equal(1); + history._paginateWithDetails.args[0][0].should.equal(txids); + history._mergeAndSortTxids.callCount.should.equal(1); + done(); }); }); }); - describe('@sortByHeight', function() { - it('will sort latest to oldest using height', function() { - var transactionInfo = [ - { - height: 276328 - }, - { - height: 273845, - }, - { - height: 555655 - }, - { - height: 325496 - }, - { - height: 329186 - }, - { - height: 534195 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(555655); - transactionInfo[1].height.should.equal(534195); - transactionInfo[2].height.should.equal(329186); - transactionInfo[3].height.should.equal(325496); - transactionInfo[4].height.should.equal(276328); - transactionInfo[5].height.should.equal(273845); + describe('#_paginateWithDetails', function() { + it('slice txids based on "from" and "to" (3 to 30)', function() { + var node = {}; + var options = { + from: 3, + to: 30 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([7, 6, 5, 4, 3, 2, 1, 0]); + }); }); - it('mempool and tip with time in the future', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050424429, - height: -1 - }, - { - timestamp: 1442050425439, - height: 15 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[0].timestamp.should.equal(1442050424429); - transactionInfo[1].height.should.equal(-1); - transactionInfo[1].timestamp.should.equal(1442050424328); - transactionInfo[2].height.should.equal(15); - transactionInfo[3].height.should.equal(14); + it('slice txids based on "from" and "to" (0 to 3)', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8]); + }); }); - it('tip with time in the future and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: 14, - }, - { - timestamp: 1442050424328, - height: -1 - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].height.should.equal(-1); - transactionInfo[1].height.should.equal(14); + it('will given an error if the full details is too long', function() { + var node = {}; + var options = { + from: 0, + to: 3 + }; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history.maxHistoryQueryLength = 1; + history._paginateWithDetails(txids, function(err) { + should.exist(err); + err.message.match(/Maximum/); + }); }); - it('many transactions in the mempool', function() { - var transactionInfo = [ - { - timestamp: 1442259670462, - height: -1 - }, - { - timestamp: 1442259785114, - height: -1 - }, - { - timestamp: 1442259759896, - height: -1 - }, - { - timestamp: 1442259692601, - height: -1 - }, - { - timestamp: 1442259692601, - height: 100 - }, - { - timestamp: 1442259749463, - height: -1 - }, - { - timestamp: 1442259737719, - height: -1 - }, - { - timestamp: 1442259773138, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442259785114); - transactionInfo[1].timestamp.should.equal(1442259773138); - transactionInfo[2].timestamp.should.equal(1442259759896); - transactionInfo[3].timestamp.should.equal(1442259749463); - transactionInfo[4].timestamp.should.equal(1442259737719); - transactionInfo[5].timestamp.should.equal(1442259692601); - transactionInfo[6].timestamp.should.equal(1442259670462); - transactionInfo[7].height.should.equal(100); - }); - it('mempool and mempool', function() { - var transactionInfo = [ - { - timestamp: 1442050424328, - height: -1 - }, - { - timestamp: 1442050425439, - height: -1, - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].timestamp.should.equal(1442050425439); - transactionInfo[1].timestamp.should.equal(1442050424328); - }); - it('mempool and mempool with the same timestamp', function() { - var transactionInfo = [ - { - timestamp: 1442050425439, - height: -1, - txid: '1', - }, - { - timestamp: 1442050425439, - height: -1, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); - }); - it('matching block heights', function() { - var transactionInfo = [ - { - height: 325496, - txid: '1', - }, - { - height: 325496, - txid: '2' - } - ]; - transactionInfo.sort(AddressHistory.sortByHeight); - transactionInfo[0].txid.should.equal('1'); - transactionInfo[1].txid.should.equal('2'); + it('will give full result without pagination options', function() { + var node = {}; + var options = {}; + var addresses = [address]; + var history = new AddressHistory({ + node: node, + options: options, + addresses: addresses + }); + var txids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + sinon.stub(history, 'getDetailedInfo', function(txid, next) { + this.detailedArray.push(txid); + next(); + }); + history._paginateWithDetails(txids, function(err, result) { + result.totalCount.should.equal(11); + result.items.should.deep.equal([10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]); + }); }); }); - describe('#sortAndPaginateCombinedArray', function() { - it('from 0 to 2', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 2 - }, - addresses: [] - }); - history.combinedArray = [ + describe('#_mergeAndSortTxids', function() { + it('will merge and sort multiple summaries', function() { + var summaries = [ { - height: 13 + totalReceived: 10000000, + totalSpent: 0, + balance: 10000000, + appearances: 2, + unconfirmedBalance: 20000000, + unconfirmedAppearances: 2, + appearanceIds: { + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579': 154, + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce': 120 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964 + } }, { - height: 14, - }, - { - height: 12 + totalReceived: 59990000, + totalSpent: 0, + balance: 49990000, + appearances: 3, + unconfirmedBalance: 1000000, + unconfirmedAppearances: 3, + appearanceIds: { + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2': 156, + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47': 152, + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f': 151 + }, + unconfirmedAppearanceIds: { + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 + } } ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(2); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - }); - it('from 0 to 4 (exceeds length)', function() { + var node = {}; + var options = {}; + var addresses = [address]; var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 4 - }, - addresses: [] + node: node, + options: options, + addresses: addresses }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(3); - history.combinedArray[0].height.should.equal(14); - history.combinedArray[1].height.should.equal(13); - history.combinedArray[2].height.should.equal(12); - }); - it('from 0 to 1', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 0, - to: 1 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(14); - }); - it('from 2 to 3', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 2, - to: 3 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(1); - history.combinedArray[0].height.should.equal(12); - }); - it('from 10 to 20 (out of range)', function() { - var history = new AddressHistory({ - node: {}, - options: { - from: 10, - to: 20 - }, - addresses: [] - }); - history.combinedArray = [ - { - height: 13 - }, - { - height: 14, - }, - { - height: 12 - } - ]; - history.sortAndPaginateCombinedArray(); - history.combinedArray.length.should.equal(0); + var txids = history._mergeAndSortTxids(summaries); + txids.should.deep.equal([ + 'e9dcf22807db77ac0276b03cc2d3a8b03c4837db8ac6650501ef45af1c807cce', + 'f637384e9f81f18767ea50e00bce58fc9848b6588a1130529eebba22a410155f', + 'f3c1ba3ef86a0420d6102e40e2cfc8682632ab95d09d86a27f5d466b9fa9da47', + '56fafeb01961831b926558d040c246b97709fd700adcaa916541270583e8e579', + 'bc992ad772eb02864db07ef248d31fb3c6826d25f1153ebf8c79df9b7f70fcf2', + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345', + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9', + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a', + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693', + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681' + ]); }); }); describe('#getDetailedInfo', function() { - it('will add additional information to existing this.transactions', function() { + it('will add additional information to existing this.transactions', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; + var tx = { + populateInputs: sinon.stub().callsArg(2), + __height: 20, + __timestamp: 1453134151, + isCoinbase: sinon.stub().returns(false), + getFee: sinon.stub().returns(1000) + }; var history = new AddressHistory({ node: { services: { db: { - getTransactionWithBlockInfo: sinon.stub() + getTransactionWithBlockInfo: sinon.stub().callsArgWith(2, null, tx), + tip: { + __height: 300 + } } } }, options: {}, addresses: [] }); + history.getAddressDetailsForTransaction = sinon.stub().returns({ + addresses: {}, + satoshis: 1000, + }); history.getDetailedInfo(txid, function(err) { if (err) { throw err; } - history.node.services.db.getTransactionsWithBlockInfo.callCount.should.equal(0); + history.node.services.db.getTransactionWithBlockInfo.callCount.should.equal(1); + done(); }); }); - it('will handle error from getTransactionFromBlock', function() { + it('will handle error from getTransactionFromBlock', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -547,9 +366,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will handle error from populateInputs', function() { + it('will handle error from populateInputs', function(done) { var txid = '46f24e0c274fc07708b781963576c4c5d5625d926dbb0a17fa865dcd9fe58ea0'; var history = new AddressHistory({ node: { @@ -566,9 +386,10 @@ describe('Address Service History', function() { }); history.getDetailedInfo(txid, function(err) { err.message.should.equal('test'); + done(); }); }); - it('will set this.transactions with correct information', function() { + it('will set this.transactions with correct information', function(done) { // block #314159 // txid 30169e8bf78bc27c4014a7aba3862c60e2e3cce19e52f1909c8255e4b7b3174e // outputIndex 1 @@ -602,7 +423,7 @@ describe('Address Service History', function() { } }, options: {}, - addresses: [] + addresses: [txAddress] }); var transactionInfo = { addresses: {}, @@ -614,7 +435,7 @@ describe('Address Service History', function() { transactionInfo.addresses[txAddress] = {}; transactionInfo.addresses[txAddress].outputIndexes = [1]; transactionInfo.addresses[txAddress].inputIndexes = []; - history.getDetailedInfo(transactionInfo, function(err) { + history.getDetailedInfo(txid, function(err) { if (err) { throw err; } @@ -629,11 +450,74 @@ describe('Address Service History', function() { info.timestamp.should.equal(1407292005); info.fees.should.equal(20000); info.tx.should.equal(transaction); + done(); }); }); }); + + describe('#getAddressDetailsForTransaction', function() { + it('will calculate details for the transaction', function(done) { + /* jshint sub:true */ + var tx = bitcore.Transaction({ + 'hash': 'b12b3ae8489c5a566b629a3c62ce4c51c3870af550fb5dc77d715b669a91343c', + 'version': 1, + 'inputs': [ + { + 'prevTxId': 'a2b7ea824a92f4a4944686e67ec1001bc8785348b8c111c226f782084077b543', + 'outputIndex': 0, + 'sequenceNumber': 4294967295, + 'script': '47304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301210229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'scriptString': '71 0x304402201b81c933297241960a57ae1b2952863b965ac8c9ec7466ff0b715712d27548d50220576e115b63864f003889443525f47c7cf0bc1e2b5108398da085b221f267ba2301 33 0x0229766f1afa25ca499a51f8e01c292b0255a21a41bb6685564a1607a811ffe924', + 'output': { + 'satoshis': 1000000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + } + ], + 'outputs': [ + { + 'satoshis': 100000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 200000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 50000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 300000000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + }, + { + 'satoshis': 349990000, + 'script': '76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac' + } + ], + 'nLockTime': 0 + }); + var history = new AddressHistory({ + node: { + network: bitcore.Networks.testnet + }, + options: {}, + addresses: ['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'] + }); + var details = history.getAddressDetailsForTransaction(tx); + should.exist(details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW']); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].inputIndexes.should.deep.equal([0]); + details.addresses['mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW'].outputIndexes.should.deep.equal([ + 0, 1, 2, 3, 4 + ]); + details.satoshis.should.equal(-10000); + done(); + }); + }); + describe('#getConfirmationsDetail', function() { - it('the correct confirmations when included in the tip', function() { + it('the correct confirmations when included in the tip', function(done) { var history = new AddressHistory({ node: { services: { @@ -651,30 +535,7 @@ describe('Address Service History', function() { __height: 100 }; history.getConfirmationsDetail(transaction).should.equal(1); - }); - }); - describe('#getSatoshisDetail', function() { - it('subtract inputIndexes satoshis without outputIndexes', function() { - var history = new AddressHistory({ - node: {}, - options: {}, - addresses: [] - }); - var transaction = { - inputs: [ - { - output: { - satoshis: 10000 - } - } - ] - }; - var txInfo = { - addresses: {} - }; - txInfo.addresses[address] = {}; - txInfo.addresses[address].inputIndexes = [0]; - history.getSatoshisDetail(transaction, txInfo).should.equal(-10000); + done(); }); }); }); diff --git a/test/services/address/index.unit.js b/test/services/address/index.unit.js index e7460a1d..a35614df 100644 --- a/test/services/address/index.unit.js +++ b/test/services/address/index.unit.js @@ -2,6 +2,8 @@ var should = require('chai').should(); var sinon = require('sinon'); +var stream = require('stream'); +var levelup = require('levelup'); var proxyquire = require('proxyquire'); var bitcorenode = require('../../../'); var AddressService = bitcorenode.services.Address; @@ -9,13 +11,15 @@ var blockData = require('../../data/livenet-345003.json'); var bitcore = require('bitcore-lib'); var memdown = require('memdown'); var leveldown = require('leveldown'); -var Script = bitcore.Script; -var Address = bitcore.Address; var Networks = bitcore.Networks; var EventEmitter = require('events').EventEmitter; var errors = bitcorenode.errors; var Transaction = require('../../../lib/transaction'); var txData = require('../../data/transaction.json'); +var index = require('../../../lib'); +var log = index.log; +var constants = require('../../../lib/services/address/constants'); +var encoding = require('../../../lib/services/address/encoding'); var mockdb = { }; @@ -96,7 +100,8 @@ describe('Address Service', function() { done(); }); }); - it('start levelup db for mempool index', function(done) { + it('start levelup db for mempool', function(done) { + var levelupStub = sinon.stub().callsArg(2); var TestAddressService = proxyquire('../../../lib/services/address', { 'fs': { existsSync: sinon.stub().returns(true) @@ -104,14 +109,7 @@ describe('Address Service', function() { 'leveldown': { destroy: sinon.stub().callsArgWith(1, null) }, - 'levelup': function(dbPath, options, callback) { - dbPath.should.equal('testdir/testnet3/bitcore-addressmempool.db'); - options.db.should.equal(memdown); - options.keyEncoding.should.equal('binary'); - options.valueEncoding.should.equal('binary'); - options.fillCache.should.equal(false); - setImmediate(callback); - }, + 'levelup': levelupStub, 'mkdirp': sinon.stub().callsArgWith(1, null) }); var am = new TestAddressService({ @@ -119,6 +117,14 @@ describe('Address Service', function() { node: mocknode }); am.start(function() { + levelupStub.callCount.should.equal(1); + var dbPath1 = levelupStub.args[0][0]; + dbPath1.should.equal('testdir/testnet3/bitcore-addressmempool.db'); + var options = levelupStub.args[0][1]; + options.db.should.equal(memdown); + options.keyEncoding.should.equal('binary'); + options.valueEncoding.should.equal('binary'); + options.fillCache.should.equal(false); done(); }); }); @@ -186,14 +192,26 @@ describe('Address Service', function() { describe('#stop', function() { it('will close mempool levelup', function(done) { + var testnode = { + network: Networks.testnet, + datadir: 'testdir', + db: mockdb, + services: { + bitcoind: { + on: sinon.stub(), + removeListener: sinon.stub() + } + } + }; var am = new AddressService({ mempoolMemoryIndex: true, - node: mocknode + node: testnode }); am.mempoolIndex = {}; am.mempoolIndex.close = sinon.stub().callsArg(0); am.stop(function() { am.mempoolIndex.close.callCount.should.equal(1); + am.node.services.bitcoind.removeListener.callCount.should.equal(2); done(); }); }); @@ -253,7 +271,7 @@ describe('Address Service', function() { }); it('should load the db with regtest', function() { // Switch to use regtest - // Networks.remove(Networks.testnet); + Networks.remove(Networks.testnet); Networks.add({ name: 'regtest', alias: 'regtest', @@ -387,43 +405,6 @@ describe('Address Service', function() { }); }); - describe('#_extractAddressInfoFromScript', function() { - var am; - before(function() { - am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - am.node.network = Networks.livenet; - }); - it('pay-to-publickey', function() { - var pubkey = new bitcore.PublicKey('022df8750480ad5b26950b25c7ba79d3e37d75f640f8e5d9bcd5b150a0f85014da'); - var script = Script.buildPublicKeyOut(pubkey); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('9674af7395592ec5d91573aa8d6557de55f60147'); - }); - it('pay-to-publickeyhash', function() { - var script = Script('OP_DUP OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUALVERIFY OP_CHECKSIG'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToPublicKeyHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('pay-to-scripthash', function() { - var script = Script('OP_HASH160 20 0x0000000000000000000000000000000000000000 OP_EQUAL'); - var info = am._extractAddressInfoFromScript(script); - info.addressType.should.equal(Address.PayToScriptHash); - info.hashBuffer.toString('hex').should.equal('0000000000000000000000000000000000000000'); - }); - it('non-address script type', function() { - var buf = new Buffer(40); - buf.fill(0); - var script = Script('OP_RETURN 40 0x' + buf.toString('hex')); - var info = am._extractAddressInfoFromScript(script); - info.should.equal(false); - }); - }); - describe('#blockHandler', function() { var am; var testBlock = bitcore.Block.fromString(blockData); @@ -524,6 +505,7 @@ describe('Address Service', function() { var testnode = { datadir: 'testdir', db: db, + network: Networks.testnet, services: { bitcoind: { on: sinon.stub() @@ -559,73 +541,6 @@ describe('Address Service', function() { }); }); - describe('#_encodeSpentIndexSyncKey', function() { - it('will encode to 36 bytes (string)', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txidBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - key.length.should.equal(36); - }); - it('will be able to decode encoded value', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var txid = '3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7'; - var txidBuffer = new Buffer(txid, 'hex'); - var key = am._encodeSpentIndexSyncKey(txidBuffer, 12); - var keyBuffer = new Buffer(key, 'binary'); - keyBuffer.slice(0, 32).toString('hex').should.equal(txid); - var outputIndex = keyBuffer.readUInt32BE(32); - outputIndex.should.equal(12); - }); - }); - - describe('#_encodeInputKeyMap/#_decodeInputKeyMap roundtrip', function() { - var encoded; - var outputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputKeyMap(outputTxIdBuffer, 13); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputKeyMap(encoded); - key.outputTxId.toString('hex').should.equal(outputTxIdBuffer.toString('hex')); - key.outputIndex.should.equal(13); - }); - }); - - describe('#_encodeInputValueMap/#_decodeInputValueMap roundtrip', function() { - var encoded; - var inputTxIdBuffer = new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae7', 'hex'); - it('encode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - encoded = am._encodeInputValueMap(inputTxIdBuffer, 7); - }); - it('decode key', function() { - var am = new AddressService({ - mempoolMemoryIndex: true, - node: mocknode - }); - var key = am._decodeInputValueMap(encoded); - key.inputTxId.toString('hex').should.equal(inputTxIdBuffer.toString('hex')); - key.inputIndex.should.equal(7); - }); - }); - describe('#transactionEventHandler', function() { it('will emit a transaction if there is a subscriber', function(done) { var am = new AddressService({ @@ -817,18 +732,127 @@ describe('Address Service', function() { }); + describe('#createInputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createInputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df72'); + data.inputIndex.should.equal(1); + data.height.should.equal(157); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('030b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d786cfeae288d74aaf9f51f215f9882e7bd7bc18af7a550683c4d7c6962f6372900000004', 'hex'), + value: new Buffer('7b94e3c39386845ea383b8e726b20b5172ccd3ef9be008bbb133e3b63f07df7200000001', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createInputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createInputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '03038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getInputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -845,14 +869,18 @@ describe('Address Service', function() { }); it('will add mempool inputs on close', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var db = { store: { createReadStream: sinon.stub().returns(testStream) + }, + tip: { + __height: 10 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -883,10 +911,11 @@ describe('Address Service', function() { inputs[0].height.should.equal(-1); done(); }); - testStream.emit('close'); + testStream.push(null); }); it('will get inputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -895,12 +924,12 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -924,20 +953,21 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get inputs for address', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { queryMempool: true }; var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('00', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ff', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('0000000000', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.SPENTS, hashBuffer, hashTypeBuffer, new Buffer('ffffffffff', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -960,15 +990,16 @@ describe('Address Service', function() { value: new Buffer('3b6bc2939d1a70ce04bc4f619ee32608fbff5e565c1f9b02e4eaa97959c59ae700000000', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should give an error if the readstream has an error', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(testStream) }; - am.getOutputs(address, {}, function(err, outputs) { + am.getInputs(address, {}, function(err, outputs) { should.exist(err); err.message.should.equal('readstreamerror'); done(); @@ -976,7 +1007,7 @@ describe('Address Service', function() { testStream.emit('error', new Error('readstreamerror')); setImmediate(function() { - testStream.emit('close'); + testStream.push(null); }); }); @@ -986,7 +1017,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1025,39 +1056,44 @@ describe('Address Service', function() { }); }); it('it will parse data', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; am.mempoolIndex = {}; am.mempoolIndex.createReadStream = sinon.stub().returns(testStream); - am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, outputs) { + var nowTime = new Date().getTime(); + + am._getInputsMempool(address, hashBuffer, hashTypeBuffer, function(err, inputs) { should.not.exist(err); - outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].txid.should.equal(txid); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); - outputs[0].inputIndex.should.equal(5); - outputs[0].height.should.equal(-1); - outputs[0].confirmations.should.equal(0); + inputs.length.should.equal(1); + var input = inputs[0]; + input.address.should.equal(address); + input.txid.should.equal(txid); + input.hashType.should.equal('pubkeyhash'); + input.hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + input.inputIndex.should.equal(5); + input.height.should.equal(-1); + input.confirmations.should.equal(0); + input.timestamp.should.equal(nowTime); done(); }); var txid = '5d32f0fff6871c377e00c16f48ebb5e89c723d0b9dd25f68fdda70c3392bee61'; var inputIndex = 5; var inputIndexBuffer = new Buffer(4); + var timestampBuffer = new Buffer(new Array(8)); + timestampBuffer.writeDoubleBE(nowTime); inputIndexBuffer.writeUInt32BE(inputIndex); var valueData = Buffer.concat([ new Buffer(txid, 'hex'), - inputIndexBuffer + inputIndexBuffer, + timestampBuffer ]); - // Note: key is not used currently testStream.emit('data', { value: valueData }); - setImmediate(function() { - testStream.emit('close'); - }); + testStream.emit('close'); }); }); @@ -1105,18 +1141,129 @@ describe('Address Service', function() { }); }); + describe('#createOutputsStream', function() { + it('transform stream from buffer into object', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + tip: { + __height: 157 + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + addressService.createOutputsDBStream = sinon.stub().returns(streamStub); + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsStream(address, {}); + testStream.once('data', function(data) { + data.address.should.equal('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'); + data.hashType.should.equal('pubkeyhash'); + data.txid.should.equal('4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f9'); + data.outputIndex.should.equal(2); + data.height.should.equal(157); + data.satoshis.should.equal(10000); + data.script.toString('hex').should.equal('76a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac'); + data.confirmations.should.equal(1); + done(); + }); + streamStub.emit('data', { + key: new Buffer('020b2f0a0c31bfe0406b0ccc1381fdbe311946dadc01000000009d4078b72b09391f5146e2c564f5847d49b179f9946b253f780f65b140d46ef6f900000002', 'hex'), + value: new Buffer('40c388000000000076a9140b2f0a0c31bfe0406b0ccc1381fdbe311946dadc88ac', 'hex') + }); + streamStub.emit('end'); + }); + }); + + describe('#createOutputsDBStream', function() { + it('will stream all keys', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b68701ffffffffff'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + it('will stream keys based on a range of block heights', function() { + var streamStub = sinon.stub().returns({}); + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + }, + db: { + store: { + createReadStream: streamStub + } + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + start: 1, + end: 0 + }; + var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; + var testStream = addressService.createOutputsDBStream(address, options); + should.exist(testStream); + streamStub.callCount.should.equal(1); + var expectedGt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000000'; + // The expected "lt" value should be one value above the start value, due + // to the keys having additional data following it and can't be "equal". + var expectedLt = '02038a213afdfc551fc658e9a2a58a86e98d69b687010000000002'; + streamStub.args[0][0].gt.toString('hex').should.equal(expectedGt); + streamStub.args[0][0].lt.toString('hex').should.equal(expectedLt); + }); + }); + describe('#getOutputs', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address('1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W').hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 } }; var testnode = { - network: Networks.testnet, + network: Networks.livenet, datadir: 'testdir', services: { db: db, @@ -1137,7 +1284,8 @@ describe('Address Service', function() { }); it('will get outputs for an address and timestamp', function(done) { - var testStream = new EventEmitter(); + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1146,10 +1294,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1161,7 +1309,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('76a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac'); @@ -1174,11 +1322,12 @@ describe('Address Service', function() { value: new Buffer('41f0de058a80000076a914038a213afdfc551fc658e9a2a58a86e98d69b68788ac', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should get outputs for an address', function(done) { - var readStream1 = new EventEmitter(); + var readStream1 = new stream.Readable(); + readStream1._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream1) }; @@ -1234,11 +1383,12 @@ describe('Address Service', function() { readStream1.emit('data', data1); readStream1.emit('data', data2); - readStream1.emit('close'); + readStream1.push(null); }); it('should give an error if the readstream has an error', function(done) { - var readStream2 = new EventEmitter(); + var readStream2 = new stream.Readable(); + readStream2._read = function() { /* do nothing */ }; am.node.services.db.store = { createReadStream: sinon.stub().returns(readStream2) }; @@ -1251,7 +1401,7 @@ describe('Address Service', function() { readStream2.emit('error', new Error('readstreamerror')); setImmediate(function() { - readStream2.emit('close'); + readStream2.push(null); }); }); @@ -1261,8 +1411,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1271,10 +1422,10 @@ describe('Address Service', function() { var createReadStreamCallCount = 0; am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1286,7 +1437,7 @@ describe('Address Service', function() { outputs[0].address.should.equal(address); outputs[0].txid.should.equal('125dd0e50fc732d67c37b6c56be7f9dc00b6859cebf982ee2cc83ed2d604bf87'); outputs[0].hashType.should.equal('scripthash'); - outputs[0].hashType.should.equal(AddressService.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); + outputs[0].hashType.should.equal(constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')]); outputs[0].outputIndex.should.equal(1); outputs[0].satoshis.should.equal(4527773864); outputs[0].script.should.equal('a914038a213afdfc551fc658e9a2a58a86e98d69b68787'); @@ -1301,7 +1452,7 @@ describe('Address Service', function() { value: new Buffer('41f0de058a800000a914038a213afdfc551fc658e9a2a58a86e98d69b68787', 'hex') }; testStream.emit('data', data); - testStream.emit('close'); + testStream.push(null); }); it('should not print outputs for a p2pkh address, if the output was sent to a p2sh redeemScript', function(done) { @@ -1310,8 +1461,9 @@ describe('Address Service', function() { // See https://github.com/bitpay/bitcore-node/issues/377 var address = '321jRYeWBrLBWr2j1KYnAFGico3GUdd5q7'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.REDEEMSCRIPT; - var testStream = new EventEmitter(); + var hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT; + var testStream = new stream.Readable(); + testStream._read = function() { /* do nothing */ }; var args = { start: 15, end: 12, @@ -1322,10 +1474,10 @@ describe('Address Service', function() { // Verifying that the db query is looking for a redeemScript, *not* a p2pkh am.node.services.db.store = { createReadStream: function(ops) { - var gte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); - ops.gte.toString('hex').should.equal(gte.toString('hex')); - var lte = Buffer.concat([AddressService.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); - ops.lte.toString('hex').should.equal(lte.toString('hex')); + var gt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('000000000c', 'hex')]); + ops.gt.toString('hex').should.equal(gt.toString('hex')); + var lt = Buffer.concat([constants.PREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, new Buffer('0000000010', 'hex')]); + ops.lt.toString('hex').should.equal(lt.toString('hex')); createReadStreamCallCount++; return testStream; } @@ -1337,7 +1489,7 @@ describe('Address Service', function() { done(); }); createReadStreamCallCount.should.equal(1); - testStream.emit('close'); + testStream.push(null); }); }); @@ -1345,7 +1497,7 @@ describe('Address Service', function() { var am; var address = '1KiW1A4dx1oRgLHtDtBjcunUGkYtFgZ1W'; var hashBuffer = bitcore.Address(address).hashBuffer; - var hashTypeBuffer = AddressService.HASH_TYPES.PUBKEY; + var hashTypeBuffer = constants.HASH_TYPES.PUBKEY; var db = { tip: { __height: 1 @@ -1391,14 +1543,16 @@ describe('Address Service', function() { throw err; } outputs.length.should.equal(1); - outputs[0].address.should.equal(address); - outputs[0].hashType.should.equal('pubkeyhash'); - outputs[0].txid.should.equal(txid); - outputs[0].outputIndex.should.equal(outputIndex); - outputs[0].height.should.equal(-1); - outputs[0].satoshis.should.equal(3); - outputs[0].script.should.equal('ac'); - outputs[0].confirmations.should.equal(0); + var output = outputs[0]; + output.address.should.equal(address); + output.hashType.should.equal('pubkeyhash'); + output.txid.should.equal(txid); + output.outputIndex.should.equal(outputIndex); + output.height.should.equal(-1); + output.satoshis.should.equal(3); + output.script.should.equal('ac'); + output.timestamp.should.equal(1452696715750); + output.confirmations.should.equal(0); done(); }); @@ -1408,7 +1562,7 @@ describe('Address Service', function() { var outputIndexBuffer = new Buffer(4); outputIndexBuffer.writeUInt32BE(outputIndex); var keyData = Buffer.concat([ - AddressService.MEMPREFIXES.OUTPUTS, + constants.MEMPREFIXES.OUTPUTS, hashBuffer, hashTypeBuffer, txidBuffer, @@ -1417,6 +1571,7 @@ describe('Address Service', function() { var valueData = Buffer.concat([ new Buffer('4008000000000000', 'hex'), + new Buffer('427523b78c1e6000', 'hex'), new Buffer('ac', 'hex') ]); @@ -1803,12 +1958,18 @@ describe('Address Service', function() { describe('#updateMempoolIndex/#removeMempoolIndex', function() { var am; var tx = Transaction().fromBuffer(txBuf); + var clock; - before(function() { + beforeEach(function() { am = new AddressService({ mempoolMemoryIndex: true, node: mocknode }); + clock = sinon.useFakeTimers(); + }); + + afterEach(function() { + clock.restore(); }); it('will update the input and output indexes', function() { @@ -1819,12 +1980,18 @@ describe('Address Service', function() { for (var i = 0; i < operations.length; i++) { operations[i].type.should.equal('put'); } - var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002'; + var nowTime = new Date().getTime(); + var nowTimeBuffer = new Buffer(8); + nowTimeBuffer.writeDoubleBE(nowTime); + var expectedValue = '45202ffdeb8344af4dec07cddf0478485dc65cc7d08303e45959630c89b51ea200000002' + + nowTimeBuffer.toString('hex'); operations[7].value.toString('hex').should.equal(expectedValue); var matches = 0; + + for (var j = 0; j < operations.length; j++) { var match = Buffer.concat([ - AddressService.MEMPREFIXES.SPENTS, + constants.MEMPREFIXES.SPENTS, bitcore.Address('1JT7KDYwT9JY9o2vyqcKNSJgTWeKfV3ui8').hashBuffer ]).toString('hex'); @@ -1850,88 +2017,594 @@ describe('Address Service', function() { }); }); + describe('#getAddressSummary', function() { - var node = { - datadir: 'testdir', - network: Networks.testnet, - services: { - bitcoind: { - isSpent: sinon.stub().returns(false), - on: sinon.spy() - } - } - }; - var inputs = [ - { - 'txid': '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - 'height': -1, - 'confirmations': 0, - 'addresses': { - 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2': { - 'outputIndexes': [], - 'inputIndexes': [ - 3 - ] + var clock; + beforeEach(function() { + clock = sinon.useFakeTimers(); + sinon.stub(log, 'warn'); + }); + afterEach(function() { + clock.restore(); + log.warn.restore(); + }); + it('will handle error from _getAddressConfirmedSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() } }, - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2' - } - ]; - - var outputs = [ - { - 'address': 'mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', - 'txid': '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', - 'outputIndex': 0, - 'height': 556351, - 'satoshis': 3487110, - 'script': '76a914653b58493c2208481e0902a8ffb97b8112b13fe188ac', - 'confirmations': 13190 - } - ]; - - var as = new AddressService({ - mempoolMemoryIndex: true, - node: node - }); - as.getInputs = sinon.stub().callsArgWith(2, null, inputs); - as.getOutputs = sinon.stub().callsArgWith(2, null, outputs); - var key = Buffer.concat([ - new Buffer('689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5', 'hex'), - new Buffer(Array(4)) - ]).toString('binary'); - as.mempoolSpentIndex = {}; - as.mempoolSpentIndex[key] = true; - it('should handle unconfirmed and confirmed outputs and inputs', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - summary.txids.should.deep.equal( - [ - '9f183412de12a6c1943fc86c390174c1cde38d709217fdb59dcf540230fa58a6', - '689e9f543fa4aa5b2daa3b5bb65f9a00ad5aa1a2e9e1fc4e11061d85f2aa9bc5' - ] - ); + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, new Error('test')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); done(); }); }); - it('noTxList should not include txids array', function(done) { - as.getAddressSummary('mpkDdnLq26djg17s6cYknjnysAm3QwRzu2', {noTxList: true}, function(err, summary) { - should.not.exist(err); - summary.totalReceived.should.equal(3487110); - summary.totalSpent.should.equal(0); - summary.balance.should.equal(3487110); - summary.unconfirmedBalance.should.equal(0); - summary.appearances.should.equal(1); - summary.unconfirmedAppearances.should.equal(1); - should.not.exist(summary.txids); + it('will handle error from _getAddressMempoolSummary', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArg(2); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(2, new Error('test2')); + addressService.getAddressSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test2'); + done(); + }); + }); + it('will pass cache and summary between functions correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function(err, sum) { + addressService._getAddressConfirmedSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.callCount.should.equal(1); + addressService._getAddressMempoolSummary.args[0][2].should.equal(cache); + addressService._setAndSortTxidsFromAppearanceIds.callCount.should.equal(1); + addressService._setAndSortTxidsFromAppearanceIds.args[0][0].should.equal(cache); + addressService._transformAddressSummaryFromResult.callCount.should.equal(1); + addressService._transformAddressSummaryFromResult.args[0][0].should.equal(cache); + sum.should.equal(summary); + done(); + }); + }); + it('will log if there is a slow query', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var addressService = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var address = '12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'; + var options = {}; + var cache = {}; + var summary = {}; + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressConfirmedSummary = sinon.stub().callsArgWith(2, null, cache); + addressService._getAddressMempoolSummary = sinon.stub().callsArgWith(3, null, cache); + addressService._setAndSortTxidsFromAppearanceIds = sinon.stub().callsArgWith(1, null, cache); + addressService._transformAddressSummaryFromResult = sinon.stub().returns(summary); + addressService.getAddressSummary(address, options, function() { + log.warn.callCount.should.equal(1); + done(); + }); + clock.tick(6000); + }); + }); + + describe('#_getAddressConfirmedSummary', function() { + it('will pass arguments correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedSummary(address, options, function(err) { + if (err) { + return done(err); + } + var expectedResult = { + appearanceIds: {}, + totalReceived: 0, + balance: 0, + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + as._getAddressConfirmedInputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedInputsSummary.args[0][1].should.deep.equal(expectedResult); + as._getAddressConfirmedInputsSummary.args[0][2].should.deep.equal(options); + as._getAddressConfirmedOutputsSummary.args[0][0].should.equal(address); + as._getAddressConfirmedOutputsSummary.args[0][1].should.deep.equal(result); + as._getAddressConfirmedOutputsSummary.args[0][2].should.equal(options); + done(); + }); + }); + it('will pass error correctly (inputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will pass error correctly (outputs)', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = {}; + as._getAddressConfirmedInputsSummary = sinon.stub().callsArgWith(3, null, result); + as._getAddressConfirmedOutputsSummary = sinon.stub().callsArgWith(3, new Error('test')); + as._getAddressConfirmedSummary(address, options, function(err) { + should.exist(err); + err.message.should.equal('test'); done(); }); }); }); + + describe('#_getAddressConfirmedInputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: {} + }; + var options = {}; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, result, options, function(err, result) { + if (err) { + return done(err); + } + result.appearanceIds[txid].should.equal(10); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var cache = {}; + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createInputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedInputsSummary(address, cache, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe('#_getAddressConfirmedOutputsSummary', function() { + it('will stream inputs and collect txids', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub(), + isSpent: sinon.stub().returns(false) + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + }; + + var options = { + queryMempool: true + }; + var txid = 'f2cfc19d13f0c12199f70e420d84e2b3b1d4e499702aa9d737f8c24559c9ec47'; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + + as.createOutputsStream = sinon.stub().returns(streamStub); + + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey(new Buffer(txid, 'hex'), 2); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { + if (err) { + return done(err); + } + result.appearanceIds[txid].should.equal(10); + result.balance.should.equal(1000); + result.totalReceived.should.equal(1000); + result.unconfirmedBalance.should.equal(-1000); + done(); + }); + + streamStub.emit('data', { + txid: txid, + height: 10, + outputIndex: 2, + satoshis: 1000 + }); + streamStub.push(null); + }); + it('handle stream error', function(done) { + var streamStub = new stream.Readable(); + streamStub._read = function() { /* do nothing */ }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: {}, + unconfirmedAppearanceIds: {}, + balance: 0, + totalReceived: 0, + unconfirmedBalance: 0 + }; + + var options = {}; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + as.createOutputsStream = sinon.stub().returns(streamStub); + as._getAddressConfirmedOutputsSummary(address, result, options, function(err, cache) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + + streamStub.emit('error', new Error('test')); + streamStub.push(null); + }); + }); + + describe('#_setAndSortTxidsFromAppearanceIds', function() { + it('will sort correctly', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var result = { + appearanceIds: { + '22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be': 200, + '1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93': 100, + '206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505': 101 + }, + unconfirmedAppearanceIds: { + 'ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681': 1452898347406, + 'ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693': 1452898331964, + 'f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345': 1452897902377, + 'edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a': 1452897971363, + 'f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9': 1452897923107 + } + }; + as._setAndSortTxidsFromAppearanceIds(result, function(err, result) { + if (err) { + return done(err); + } + should.exist(result.txids); + result.txids[0].should.equal('1c413601acbd608240fc635b95886c3c1f76ec8589c3392a58b5715ceb618e93'); + result.txids[1].should.equal('206d3834c010d46a2cf478cb1c5fe252be41f683c8a738e3ebe27f1aae67f505'); + result.txids[2].should.equal('22488dbb99aed86e7081ac480e3459fa40ccab7ee18bef98b84b3cdce6bf05be'); + result.unconfirmedTxids[0].should.equal('f71bccef3a8f5609c7f016154922adbfe0194a96fb17a798c24077c18d0a9345'); + result.unconfirmedTxids[1].should.equal('f35e7e2a2334e845946f3eaca76890d9a68f4393ccc9fe37a0c2fb035f66d2e9'); + result.unconfirmedTxids[2].should.equal('edc080f2084eed362aa488ccc873a24c378dc0979aa29b05767517b70569414a'); + result.unconfirmedTxids[3].should.equal('ed11a08e3102f9610bda44c80c46781d97936a4290691d87244b1b345b39a693'); + result.unconfirmedTxids[4].should.equal('ec94d845c603f292a93b7c829811ac624b76e52b351617ca5a758e9d61a11681'); + done(); + }); + }); + }); + + describe('#_getAddressMempoolSummary', function() { + it('skip if options not enabled', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = {}; + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + Object.keys(result.unconfirmedAppearanceIds).length.should.equal(0); + result.unconfirmedBalance.should.equal(0); + done(); + }); + }); + it('include all txids and balance from inputs and outputs', function(done) { + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var resultBase = { + unconfirmedAppearanceIds: {}, + unconfirmedBalance: 0 + }; + var address = new bitcore.Address('12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX'); + var options = { + queryMempool: true + }; + var mempoolInputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + inputIndex: 0, + timestamp: 1452874536321, + height: -1, + confirmations: 0 + } + ]; + var mempoolOutputs = [ + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + }, + { + address: '3NbU8XzUgKyuCgYgZEKsBtUvkTm2r7Xgwj', + hashType: 'scripthash', + txid: '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + outputIndex: 0, + height: -1, + timestamp: 1452874521466, + satoshis: 131368318, + script: '76a9148c66db6e9f74b1db9c400eaa2aed3743417f38e688ac', + confirmations: 0 + } + ]; + var spentIndexSyncKey = encoding.encodeSpentIndexSyncKey( + new Buffer(mempoolOutputs[1].txid, 'hex'), + 0 + ); + as.mempoolSpentIndex[spentIndexSyncKey] = true; + as._getInputsMempool = sinon.stub().callsArgWith(3, null, mempoolInputs); + as._getOutputsMempool = sinon.stub().callsArgWith(3, null, mempoolOutputs); + as._getAddressMempoolSummary(address, options, resultBase, function(err, result) { + if (err) { + return done(err); + } + var txid1 = '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8'; + var txid2 = '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d'; + var txid3 = '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247'; + result.unconfirmedAppearanceIds[txid1].should.equal(1452874536321); + result.unconfirmedAppearanceIds[txid2].should.equal(1452874521466); + result.unconfirmedAppearanceIds[txid3].should.equal(1452874521466); + result.unconfirmedBalance.should.equal(131368318); + done(); + }); + }); + }); + + describe('#_transformAddressSummaryFromResult', function() { + var result = { + totalReceived: 1000000, + balance: 500000, + txids: [ + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8', + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92' + ], + appearanceIds: { + 'b1bfa8dbbde790cb46b9763ef3407c1a21c8264b67bfe224f462ec0e1f569e92': 100000, + '70d9d441d7409aace8e0ffe24ff0190407b2fcb405799a266e0327017288d1f8': 200000 + }, + unconfirmedAppearanceIds: { + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d': 1452874536321, + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247': 1452874521466 + }, + unconfirmedTxids: [ + '57b7842afc97a2b46575b490839df46e9273524c6ea59ba62e1e86477cf25247', + '35fafaf572341798b2ce2858755afa7c8800bb6b1e885d3e030b81255b5e172d' + ], + unconfirmedBalance: 500000 + }; + var testnode = { + services: { + bitcoind: { + on: sinon.stub() + } + }, + datadir: 'testdir' + }; + it('will transform result into summary', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = {}; + var summary = as._transformAddressSummaryFromResult(result, options); + summary.totalReceived.should.equal(1000000); + summary.totalSpent.should.equal(500000); + summary.balance.should.equal(500000); + summary.appearances.should.equal(2); + summary.unconfirmedAppearances.should.equal(2); + summary.unconfirmedBalance.should.equal(500000); + summary.txids.length.should.equal(4); + }); + it('will omit txlist', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + noTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.not.exist(summary.txids); + }); + it('will include full appearance ids', function() { + var as = new AddressService({ + mempoolMemoryIndex: true, + node: testnode + }); + var options = { + fullTxList: true + }; + var summary = as._transformAddressSummaryFromResult(result, options); + should.exist(summary.appearanceIds); + should.exist(summary.unconfirmedAppearanceIds); + }); + }); + });