From 8d98abd080a9dd38eb4f39ea8dfad02f605e331d Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Sun, 12 Feb 2017 12:15:22 -0500 Subject: [PATCH] Cleaned up old code that may not make it into the next release. --- lib/encoding.js | 237 --- lib/services/address/encoding.js | 34 +- lib/services/address/history.js | 266 --- lib/services/address/index.js | 83 +- .../address/streams/inputs-transform.js | 40 - .../address/streams/outputs-transform.js | 42 - lib/services/bitcoind/index.js | 1701 +---------------- lib/services/db/index.js | 243 +-- lib/services/db/sync.js | 2 +- lib/services/wallet-api/encoding.js | 4 +- lib/services/wallet-api/index.js | 38 +- lib/utils.js | 38 +- scripts/download | 2 +- test/services/address/encoding.unit.js | 2 +- test/services/timestamp/timestamp.unit.js | 1 + test/services/transaction/encoding.unit.js | 1 - test/services/wallet-api/encoding.unit.js | 1 + 17 files changed, 105 insertions(+), 2630 deletions(-) delete mode 100644 lib/encoding.js delete mode 100644 lib/services/address/history.js delete mode 100644 lib/services/address/streams/inputs-transform.js delete mode 100644 lib/services/address/streams/outputs-transform.js diff --git a/lib/encoding.js b/lib/encoding.js deleted file mode 100644 index b40476b8..00000000 --- a/lib/encoding.js +++ /dev/null @@ -1,237 +0,0 @@ -'use strict'; - -var bitcore = require('bitcore-lib'); -var BufferReader = bitcore.encoding.BufferReader; - -function Encoding(servicePrefix) { - this.servicePrefix = servicePrefix; -} - - -Encoding.prototype.encodeWalletTransactionKey = function(walletId, height) { - var buffers = [this.servicePrefix]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - if(height !== undefined) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - buffers.push(heightBuffer); - } - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletTransactionKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(1); - - var walletSize = reader.readUInt8(); - var walletId = reader.read(walletSize).toString('utf8'); - var height = reader.readUInt32BE(); - var blockIndex = reader.readUInt32BE(); - - return { - walletId: walletId, - height: height, - blockIndex: blockIndex - }; -}; - -Encoding.prototype.encodeWalletTransactionValue = function(txid) { - return new Buffer(txid, 'hex'); -}; - -Encoding.prototype.decodeWalletTransactionValue = function(buffer) { - return buffer.toString('hex'); -}; - -Encoding.prototype.encodeWalletUtxoKey = function(walletId, txid, outputIndex) { - var buffers = [this.servicePrefix]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - if(txid) { - var txidBuffer = new Buffer(txid, 'hex'); - buffers.push(txidBuffer); - } - - if(outputIndex !== undefined) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - buffers.push(outputIndexBuffer); - } - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletUtxoKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(1); - - var walletIdSize = reader.readUInt8(); - var walletId = reader.read(walletIdSize).toString('utf8'); - var txid = reader.read(32).toString('hex'); - var outputIndex = reader.readUInt32BE(); - return { - walletId: walletId, - txid: txid, - outputIndex: outputIndex - }; -}; - -Encoding.prototype.encodeWalletUtxoValue = function(height, satoshis, scriptBuffer) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([height, satoshisBuffer, scriptBuffer]); -}; - -Encoding.prototype.decodeWalletUtxoValue = function(buffer) { - var reader = new BufferReader(buffer); - var height = reader.readUInt32BE(); - var satoshis = reader.readDoubleBE(); - var scriptBuffer = reader.read(buffer.length - 12); - return { - height: height, - satoshis: satoshis, - script: scriptBuffer - }; -}; - -Encoding.prototype.encodeWalletUtxoSatoshisKey = function(walletId, satoshis, txid, outputIndex) { - var buffers = [this.servicePrefix]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - if(satoshis !== undefined) { - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeUInt32BE(satoshis); - buffers.push(satoshisBuffer); - } - - if(txid) { - var txidBuffer = new Buffer(txid, 'hex'); - buffers.push(txidBuffer); - } - - if(outputIndex !== undefined) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - buffers.push(outputIndexBuffer); - } - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletUtxoSatoshisKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(1); - - var walletIdSizeBuffer = reader.readUInt8(); - var walletId = reader.read(walletIdSizeBuffer).toString('utf8'); - var satoshis = reader.readDoubleBE(); - var txid = reader.read(32).toString('hex'); - var outputIndex = reader.readUInt32BE(); - return { - walletId: walletId, - satoshis: satoshis, - txid: txid, - outputIndex: outputIndex - }; -}; - -Encoding.prototype.encodeWalletUtxoSatoshisValue = function(height, scriptBuffer) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - return Buffer.concat([height, scriptBuffer]); -}; - -Encoding.prototype.decodeWalletUtxoSatoshisValue = function(buffer) { - var reader = new BufferReader(buffer); - var height = reader.readUInt32BE(); - var scriptBuffer = reader.read(buffer.length - 4); - return { - height: height, - script: scriptBuffer - }; -}; - -Encoding.prototype.encodeWalletAddressesKey = function(walletId) { - var prefix = new Buffer('00', 'hex'); - var walletIdBuffer = new Buffer(walletId, 'hex'); - return Buffer.concat([this.servicePrefix, prefix, walletIdBuffer]); -}; - -Encoding.prototype.decodeWalletAddressesKey = function(buffer) { - return buffer.slice(3).toString('hex'); -}; - -Encoding.prototype.encodeWalletAddressesValue = function(addresses) { - var bufferList = []; - var addressesLengthBuffer = new Buffer(4); - addressesLengthBuffer.writeUInt32BE(addresses.length); - bufferList.push(addressesLengthBuffer); - for(var i = 0; i < addresses.length; i++) { - var addressSizeBuffer = new Buffer(1); - addressSizeBuffer.writeUInt8(addresses[i].length); - bufferList.push(addressSizeBuffer); - bufferList.push(new Buffer(addresses[i], 'utf8')); - } - - return Buffer.concat(bufferList); -}; - -Encoding.prototype.decodeWalletAddressesValue = function(buffer) { - var reader = new BufferReader(buffer); - var addressesLength = reader.readUInt32BE(); - var addresses = []; - var addressSize = 0; - for(var i = 0; i < addressesLength.length; i++) { - addressSize = reader.readUInt8(addressSize); - addresses.push(reader.read(addressSize).toString('utf8')); - } - - return addresses; -}; - -Encoding.prototype.encodeWalletBalanceKey = function(walletId) { - var prefix = new Buffer('01', 'hex'); - var walletIdBuffer = new Buffer(walletId, 'hex'); - return Buffer.concat([this.servicePrefix, prefix, walletIdBuffer]); -}; - -Encoding.prototype.decodeWalletBalanceKey = function(buffer) { - return buffer.slice(3).toString('hex'); -}; - -Encoding.prototype.encodeWalletBalanceValue = function(balance) { - var balanceBuffer = new Buffer(8); - balanceBuffer.writeUInt32BE(balance); - return balanceBuffer; -}; - -Encoding.prototype.decodeWalletBalanceValue = function(buffer) { - var reader = new BufferReader(buffer); - var balance = reader.readDoubleBE(); - - return balance; -}; - -module.exports = Encoding; diff --git a/lib/services/address/encoding.js b/lib/services/address/encoding.js index bfb48f77..b22fcf58 100644 --- a/lib/services/address/encoding.js +++ b/lib/services/address/encoding.js @@ -7,12 +7,6 @@ function Encoding(servicePrefix) { this.servicePrefix = servicePrefix; } -Encoding.prototype.getTerminalKey = function(startKey) { - var endKey = Buffer.from(startKey); - endKey.writeUInt8(startKey.readUInt8(startKey.length - 1) + 1, startKey.length - 1); - return endKey; -}; - Encoding.prototype.encodeAddressIndexKey = function(address, height, txid) { var prefix = new Buffer('00', 'hex'); var buffers = [this.servicePrefix, prefix]; @@ -24,16 +18,12 @@ Encoding.prototype.encodeAddressIndexKey = function(address, height, txid) { buffers.push(addressSizeBuffer); buffers.push(addressBuffer); - if(height !== undefined) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - buffers.push(heightBuffer); - } + var heightBuffer = new Buffer(4); + heightBuffer.writeUInt32BE(height || 0); + buffers.push(heightBuffer); - if(txid) { - var txidBuffer = new Buffer(txid, 'hex'); - buffers.push(txidBuffer); - } + var txidBuffer = new Buffer(txid || Array(65).join('0'), 'hex'); + buffers.push(txidBuffer); return Buffer.concat(buffers); }; @@ -64,16 +54,12 @@ Encoding.prototype.encodeUtxoIndexKey = function(address, txid, outputIndex) { buffers.push(addressSizeBuffer); buffers.push(addressBuffer); - if(txid) { - var txidBuffer = new Buffer(txid, 'hex'); - buffers.push(txidBuffer); - } + var txidBuffer = new Buffer(txid || new Array(65).join('0'), 'hex'); + buffers.push(txidBuffer); - if(outputIndex !== undefined) { - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex); - buffers.push(outputIndexBuffer); - } + var outputIndexBuffer = new Buffer(4); + outputIndexBuffer.writeUInt32BE(outputIndex || 0); + buffers.push(outputIndexBuffer); return Buffer.concat(buffers); }; diff --git a/lib/services/address/history.js b/lib/services/address/history.js deleted file mode 100644 index 4c9b4b61..00000000 --- a/lib/services/address/history.js +++ /dev/null @@ -1,266 +0,0 @@ -'use strict'; - -var bitcore = require('bitcore-lib'); -var async = require('async'); -var _ = bitcore.deps._; - -var constants = require('../../constants'); - -/** - * This represents an instance that keeps track of data over a series of - * asynchronous I/O calls to get the transaction history for a group of - * addresses. History can be queried by start and end block heights to limit large sets - * of results (uses leveldb key streaming). - */ -function AddressHistory(args) { - this.node = args.node; - this.options = args.options; - - if(Array.isArray(args.addresses)) { - this.addresses = args.addresses; - } else { - this.addresses = [args.addresses]; - } - - this.maxHistoryQueryLength = args.options.maxHistoryQueryLength || constants.MAX_HISTORY_QUERY_LENGTH; - this.maxAddressesQuery = args.options.maxAddressesQuery || constants.MAX_ADDRESSES_QUERY; - this.maxAddressesLimit = args.options.maxAddressesLimit || constants.MAX_ADDRESSES_LIMIT; - - this.addressStrings = []; - for (var i = 0; i < this.addresses.length; i++) { - var address = this.addresses[i]; - if (address instanceof bitcore.Address) { - this.addressStrings.push(address.toString()); - } else if (_.isString(address)) { - this.addressStrings.push(address); - } else { - throw new TypeError('Addresses are expected to be strings'); - } - } - - this.detailedArray = []; -} - -AddressHistory.prototype._mergeAndSortTxids = function(summaries) { - var appearanceIds = {}; - var unconfirmedAppearanceIds = {}; - - for (var i = 0; i < summaries.length; i++) { - var summary = summaries[i]; - for (var key in summary.appearanceIds) { - appearanceIds[key] = summary.appearanceIds[key]; - delete summary.appearanceIds[key]; - } - for (var unconfirmedKey in summary.unconfirmedAppearanceIds) { - unconfirmedAppearanceIds[unconfirmedKey] = summary.unconfirmedAppearanceIds[unconfirmedKey]; - delete summary.unconfirmedAppearanceIds[key]; - } - } - var confirmedTxids = Object.keys(appearanceIds); - confirmedTxids.sort(function(a, b) { - // Confirmed are sorted by height - return appearanceIds[a] - appearanceIds[b]; - }); - var unconfirmedTxids = Object.keys(unconfirmedAppearanceIds); - unconfirmedTxids.sort(function(a, b) { - // Unconfirmed are sorted by timestamp - return unconfirmedAppearanceIds[a] - unconfirmedAppearanceIds[b]; - }); - return confirmedTxids.concat(unconfirmedTxids); -}; - -/** - * This function will give detailed history for the configured - * addresses. See AddressService.prototype.getAddressHistory - * for complete documentation about options and response format. - */ -AddressHistory.prototype.get = function(callback) { - var self = this; - if (this.addresses.length > this.maxAddressesQuery) { - return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); - } - - var opts = _.clone(this.options); - opts.noBalance = true; - - if (this.addresses.length === 1) { - var address = this.addresses[0]; - self.node.services.address.getAddressSummary(address, opts, function(err, summary) { - if (err) { - return callback(err); - } - return self._paginateWithDetails.call(self, summary.txids, callback); - }); - } else { - - opts.fullTxList = true; - async.mapLimit( - self.addresses, - self.maxAddressesLimit, - function(address, next) { - self.node.services.address.getAddressSummary(address, opts, next); - }, - function(err, summaries) { - if (err) { - return callback(err); - } - var txids = self._mergeAndSortTxids(summaries); - return self._paginateWithDetails.call(self, txids, callback); - } - ); - } - -}; - -AddressHistory.prototype._paginateWithDetails = function(allTxids, callback) { - var self = this; - var totalCount = allTxids.length; - - // Slice the page starting with the most recent - var txids; - if (self.options.from >= 0 && self.options.to >= 0) { - var fromOffset = Math.max(0, totalCount - self.options.from); - var toOffset = Math.max(0, totalCount - self.options.to); - txids = allTxids.slice(toOffset, fromOffset); - } else { - txids = allTxids; - } - - // Verify that this query isn't too long - if (txids.length > self.maxHistoryQueryLength) { - return callback(new Error( - 'Maximum length query (' + self.maxHistoryQueryLength + ') exceeded for address(es): ' + - self.addresses.join(',') - )); - } - - // Reverse to include most recent at the top - txids.reverse(); - - async.eachSeries( - txids, - function(txid, next) { - self.getDetailedInfo(txid, next); - }, - function(err) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: self.detailedArray - }); - } - ); - -}; - -/** - * This function will transform items from the combinedArray into - * the detailedArray with the full transaction, satoshis and confirmation. - * @param {Object} txInfo - An item from the `combinedArray` - * @param {Function} next - */ -AddressHistory.prototype.getDetailedInfo = function(txid, next) { - var self = this; - var queryMempool = _.isUndefined(self.options.queryMempool) ? true : self.options.queryMempool; - - self.node.services.db.getTransactionWithBlockInfo( - txid, - queryMempool, - function(err, transaction) { - if (err) { - return next(err); - } - - transaction.populateInputs(self.node.services.db, [], function(err) { - if (err) { - return next(err); - } - - var addressDetails = self.getAddressDetailsForTransaction(transaction); - - self.detailedArray.push({ - addresses: addressDetails.addresses, - satoshis: addressDetails.satoshis, - height: transaction.__height, - confirmations: self.getConfirmationsDetail(transaction), - timestamp: transaction.__timestamp, - // TODO bitcore-lib should return null instead of throwing error on coinbase - fees: !transaction.isCoinbase() ? transaction.getFee() : null, - tx: transaction - }); - - next(); - }); - } - ); -}; - -/** - * A helper function for `getDetailedInfo` for getting the confirmations. - * @param {Transaction} transaction - A transaction with a populated __height value. - */ -AddressHistory.prototype.getConfirmationsDetail = function(transaction) { - var confirmations = 0; - if (transaction.__height >= 0) { - confirmations = this.node.services.db.tip.__height - transaction.__height + 1; - } - return confirmations; -}; - -AddressHistory.prototype.getAddressDetailsForTransaction = function(transaction) { - var result = { - addresses: {}, - satoshis: 0 - }; - - for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { - var input = transaction.inputs[inputIndex]; - if (!input.script) { - continue; - } - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { - var inputAddressString = inputAddress.toString(); - if (this.addressStrings.indexOf(inputAddressString) >= 0) { - if (!result.addresses[inputAddressString]) { - result.addresses[inputAddressString] = { - inputIndexes: [inputIndex], - outputIndexes: [] - }; - } else { - result.addresses[inputAddressString].inputIndexes.push(inputIndex); - } - result.satoshis -= input.output.satoshis; - } - } - } - - for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { - var output = transaction.outputs[outputIndex]; - if (!output.script) { - continue; - } - var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress) { - var outputAddressString = outputAddress.toString(); - if (this.addressStrings.indexOf(outputAddressString) >= 0) { - if (!result.addresses[outputAddressString]) { - result.addresses[outputAddressString] = { - inputIndexes: [], - outputIndexes: [outputIndex] - }; - } else { - result.addresses[outputAddressString].outputIndexes.push(outputIndex); - } - result.satoshis += output.satoshis; - } - } - } - - return result; - -}; - -module.exports = AddressHistory; diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 6a7c1f3e..c513c0b5 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -14,9 +14,7 @@ var EventEmitter = require('events').EventEmitter; var Address = bitcore.Address; var constants = require('../../constants'); var Encoding = require('./encoding'); -var InputsTransformStream = require('./streams/inputs-transform'); -var OutputsTransformStream = require('./streams/outputs-transform'); - +var utils = require('../../utils'); /** * The Address Service builds upon the Database Service and the Bitcoin Service to add additional @@ -61,15 +59,13 @@ AddressService.prototype.start = function(callback) { var self = this; this.store = this.node.services.db.store; - this.node.services.db.getPrefix(this.name, function(err, prefix) { if(err) { return callback(err); } - self.prefix = prefix; - self.encoding = new Encoding(self.prefix); + self._encoding = new Encoding(self.prefix); callback(); }); @@ -91,7 +87,7 @@ AddressService.prototype.getAPIMethods = function() { return [ ['getBalance', this, this.getBalance, 2], ['getOutputs', this, this.getOutputs, 2], - ['getUnspentOutputs', this, this.getUnspentOutputs, 2], + ['getUtxos', this, this.getUtxos, 2], ['getInputForOutput', this, this.getInputForOutput, 2], ['isSpent', this, this.isSpent, 2], ['getAddressHistory', this, this.getAddressHistory, 2], @@ -169,7 +165,7 @@ AddressService.prototype.concurrentBlockHandler = function(block, connectBlock, continue; } - var key = self.encoding.encodeAddressIndexKey(address, height, txid); + var key = self._encoding.encodeAddressIndexKey(address, height, txid); operations.push({ type: action, key: key @@ -208,7 +204,7 @@ AddressService.prototype.concurrentBlockHandler = function(block, connectBlock, continue; } - var inputKey = self.encoding.encodeAddressIndexKey(inputAddress, height, txid); + var inputKey = self._encoding.encodeAddressIndexKey(inputAddress, height, txid); operations.push({ type: action, @@ -258,8 +254,8 @@ AddressService.prototype.blockHandler = function(block, connectBlock, callback) continue; } - var key = self.encoding.encodeUtxoIndexKey(address, txid, outputIndex); - var value = self.encoding.encodeUtxoIndexValue(block.__height, output.satoshis, output._scriptBuffer); + var key = self._encoding.encodeUtxoIndexKey(address, txid, outputIndex); + var value = self._encoding.encodeUtxoIndexValue(block.__height, output.satoshis, output._scriptBuffer); operations.push({ type: action, key: key, @@ -285,7 +281,7 @@ AddressService.prototype.blockHandler = function(block, connectBlock, callback) return next(); } - var inputKey = self.encoding.encodeUtxoIndexKey(inputAddress, input.prevTxId, input.outputIndex); + var inputKey = self._encoding.encodeUtxoIndexKey(inputAddress, input.prevTxId, input.outputIndex); //common case is connecting blocks and deleting outputs spent by these inputs if (connectBlock) { operations.push({ @@ -296,7 +292,7 @@ AddressService.prototype.blockHandler = function(block, connectBlock, callback) } else { // uncommon and slower, this happens during a reorg self.node.services.transaction.getTransaction(input.prevTxId, {}, function(err, tx) { var utxo = tx.outputs[input.outputIndex]; - var inputValue = self.encoding.encodeUtxoIndexValue(tx.__height, utxo.satoshis, utxo._scriptBuffer); + var inputValue = self._encoding.encodeUtxoIndexValue(tx.__height, utxo.satoshis, utxo._scriptBuffer); operations.push({ type: 'put', key: inputKey, @@ -332,9 +328,6 @@ AddressService.prototype.getAddressString = function(script, output) { return pubkey.toString('hex'); } } catch(e) { - //log.warn('Error getting public key from: ', script.toASM(), script.toHex()); - // if there is an error, it's because a pubkey can not be extracted from the script - // continue on and return null } //TODO add back in P2PK, but for this we need to look up the utxo for this script @@ -342,7 +335,6 @@ AddressService.prototype.getAddressString = function(script, output) { return output.script.getPublicKey().toString('hex'); } - //log.warn('No utxo given for script spending a P2PK: ', script.toASM(), script.toHex()); return null; }; @@ -484,7 +476,7 @@ AddressService.prototype.unsubscribeAll = function(name, emitter) { * @param {Function} callback */ AddressService.prototype.getBalance = function(address, queryMempool, callback) { - this.getUnspentOutputs(address, queryMempool, function(err, outputs) { + this.getUtxos(address, queryMempool, function(err, outputs) { if(err) { return callback(err); } @@ -523,12 +515,12 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options txidBuffer = new Buffer(txid, 'hex'); } if (options.queryMempool) { - var spentIndexSyncKey = self.encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); + var spentIndexSyncKey = self._encoding.encodeSpentIndexSyncKey(txidBuffer, outputIndex); if (this.mempoolSpentIndex[spentIndexSyncKey]) { return this._getSpentMempool(txidBuffer, outputIndex, callback); } } - var key = self.encoding.encodeInputKeyMap(txidBuffer, outputIndex); + var key = self._encoding.encodeInputKeyMap(txidBuffer, outputIndex); var dbOptions = { valueEncoding: 'binary', keyEncoding: 'binary' @@ -539,7 +531,7 @@ AddressService.prototype.getInputForOutput = function(txid, outputIndex, options } else if (err) { return callback(err); } - var value = self.encoding.decodeInputValueMap(buffer); + var value = self._encoding.decodeInputValueMap(buffer); callback(null, { inputTxId: value.inputTxId.toString('hex'), inputIndex: value.inputIndex @@ -645,7 +637,7 @@ AddressService.prototype.getInputs = function(addressStr, options, callback) { var inputs = []; - var addrObj = self.encoding.getAddressInfo(addressStr); + var addrObj = self._encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; @@ -861,7 +853,7 @@ AddressService.prototype.getOutputs = function(addressStr, options, callback) { $.checkArgument(_.isObject(options), 'Second argument is expected to be an options object.'); $.checkArgument(_.isFunction(callback), 'Third argument is expected to be a callback function.'); - var addrObj = self.encoding.getAddressInfo(addressStr); + var addrObj = self._encoding.getAddressInfo(addressStr); var hashBuffer = addrObj.hashBuffer; var hashTypeBuffer = addrObj.hashTypeBuffer; if (!hashTypeBuffer) { @@ -935,7 +927,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h // prefix: 1, hashBuffer: 20, hashTypeBuffer: 1, txid: 32, outputIndex: 4 var txid = data.key.slice(22, 54); var outputIndex = data.key.readUInt32BE(54); - var value = self.encoding.decodeOutputMempoolValue(data.value); + var value = self._encoding.decodeOutputMempoolValue(data.value); var output = { address: addressStr, hashType: constants.HASH_TYPES_READABLE[hashTypeBuffer.toString('hex')], @@ -973,7 +965,7 @@ AddressService.prototype._getOutputsMempool = function(addressStr, hashBuffer, h * @param {Boolean} queryMempool - Include or exclude the mempool * @param {Function} callback */ -AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, callback) { +AddressService.prototype.getUtxos = function(addresses, queryMempool, callback) { var self = this; if(!Array.isArray(addresses)) { @@ -983,7 +975,7 @@ AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, c var utxos = []; async.eachSeries(addresses, function(address, next) { - self.getUnspentOutputsForAddress(address, queryMempool, function(err, unspents) { + self.getUtxosForAddress(address, queryMempool, function(err, unspents) { if(err && err instanceof errors.NoOutputs) { return next(); } else if(err) { @@ -1004,29 +996,26 @@ AddressService.prototype.getUnspentOutputs = function(addresses, queryMempool, c * @param {Boolean} queryMempool - Include or exclude the mempool * @param {Function} callback */ -AddressService.prototype.getUnspentOutputsForAddress = function(address, queryMempool, callback) { +AddressService.prototype.getUtxosForAddress = function(address, queryMempool, callback) { var self = this; - var addressLengthBuffer = new Buffer(1); - addressLengthBuffer.writeUInt8(address.length); - var start = Buffer.concat([ self.prefix, addressLengthBuffer, new Buffer(address, 'utf8'), new Buffer('00', 'hex') ]); - var end = Buffer.concat([ self.prefix, addressLengthBuffer, new Buffer(address, 'utf8'), new Buffer('01', 'hex') ]); var stream = self.store.createReadStream({ - gte: start, - lt: end + gte: self._encoding.encodeUtxoIndexKey(address), + lt: self._encoding.encodeUtxoIndexKey(utils.getTerminalKey(new Buffer(address))) }); var utxos = []; stream.on('data', function(data) { - var key = self.encoding.decodeAddressIndexKey(data.key); - var value = self.encoding.decodeAddressIndexValue(data.value); + var key = self._encoding.decodeUtxoIndexKey(data.key); + var value = self._encoding.decodeUtxoIndexValue(data.value); utxos.push({ address: key.address, txid: key.txid, - outputIndex: key.index, + outputIndex: key.outputIndex, satoshis: value.satoshis, - height: key.height + height: value.height, + script: value.script }); }); @@ -1069,7 +1058,7 @@ AddressService.prototype.isSpent = function(output, options, callback) { var spent = self.node.services.bitcoind.isSpent(txid, output.outputIndex); if (!spent && queryMempool) { var txidBuffer = new Buffer(txid, 'hex'); - var spentIndexSyncKey = self.encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); + var spentIndexSyncKey = self._encoding.encodeSpentIndexSyncKey(txidBuffer, output.outputIndex); spent = self.mempoolSpentIndex[spentIndexSyncKey] ? true : false; } setImmediate(function() { @@ -1149,8 +1138,8 @@ AddressService.prototype.getAddressTxids = function(address, options, callback) var txids = {}; - var start = self.encoding.encodeAddressIndexKey(address, options.start); - var end = self.encoding.encodeAddressIndexKey(address, options.end); + var start = self._encoding.encodeAddressIndexKey(address, options.start); + var end = self._encoding.encodeAddressIndexKey(address, options.end); var stream = self.store.createKeyStream({ gte: start, @@ -1160,7 +1149,7 @@ AddressService.prototype.getAddressTxids = function(address, options, callback) var streamErr = null; stream.on('data', function(buffer) { - var key = self.encoding.decodeAddressIndexKey(buffer); + var key = self._encoding.decodeAddressIndexKey(buffer); txids[key.txid] = true; }); @@ -1178,8 +1167,8 @@ AddressService.prototype.getAddressTxidsWithHeights = function(address, options, var txids = {}; - var start = self.encoding.encodeAddressIndexKey(address, options.start); - var end = self.encoding.encodeAddressIndexKey(address, options.end); + var start = self._encoding.encodeAddressIndexKey(address, options.start); + var end = self._encoding.encodeAddressIndexKey(address, options.end); var stream = self.store.createKeyStream({ gte: start, @@ -1189,7 +1178,7 @@ AddressService.prototype.getAddressTxidsWithHeights = function(address, options, var streamErr = null; stream.on('data', function(buffer) { - var key = self.encoding.decodeAddressIndexKey(buffer); + var key = self._encoding.decodeAddressIndexKey(buffer); txids[key.txid] = key.height; }); @@ -1341,7 +1330,7 @@ AddressService.prototype._getAddressConfirmedOutputsSummary = function(address, if(options.queryMempool) { // Check to see if this output is spent in the mempool and if so // we will subtract it from the unconfirmedBalance (a.k.a unconfirmedDelta) - var spentIndexSyncKey = self.encoding.encodeSpentIndexSyncKey( + var spentIndexSyncKey = self._encoding.encodeSpentIndexSyncKey( new Buffer(txid, 'hex'), // TODO: get buffer directly outputIndex ); @@ -1399,7 +1388,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, var addressStr = address.toString(); var hashBuffer = address.hashBuffer; var hashTypeBuffer = constants.HASH_TYPES_MAP[address.type]; - var addressIndexKey = self.encoding.encodeMempoolAddressIndexKey(hashBuffer, hashTypeBuffer); + var addressIndexKey = self._encoding.encodeMempoolAddressIndexKey(hashBuffer, hashTypeBuffer); if(!this.mempoolAddressIndex[addressIndexKey]) { return callback(null, result); @@ -1429,7 +1418,7 @@ AddressService.prototype._getAddressMempoolSummary = function(address, options, result.unconfirmedAppearanceIds[output.txid] = output.timestamp; if(!options.noBalance) { - var spentIndexSyncKey = self.encoding.encodeSpentIndexSyncKey( + var spentIndexSyncKey = self._encoding.encodeSpentIndexSyncKey( new Buffer(output.txid, 'hex'), // TODO: get buffer directly output.outputIndex ); diff --git a/lib/services/address/streams/inputs-transform.js b/lib/services/address/streams/inputs-transform.js deleted file mode 100644 index 709c8d0e..00000000 --- a/lib/services/address/streams/inputs-transform.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict'; - -var Transform = require('stream').Transform; -var inherits = require('util').inherits; -var bitcore = require('bitcore-lib'); -var encodingUtil = require('../../../encoding'); -var $ = bitcore.util.preconditions; - -function InputsTransformStream(options) { - $.checkArgument(options.address instanceof bitcore.Address); - Transform.call(this, { - objectMode: true - }); - this._address = options.address; - this._addressStr = this._address.toString(); - this._tipHeight = options.tipHeight; -} -inherits(InputsTransformStream, Transform); - -InputsTransformStream.prototype._transform = function(chunk, encoding, callback) { - var self = this; - - var key = encodingUtil.decodeInputKey(chunk.key); - var value = encodingUtil.decodeInputValue(chunk.value); - - var input = { - address: this._addressStr, - hashType: this._address.type, - txid: value.txid.toString('hex'), - inputIndex: value.inputIndex, - height: key.height, - confirmations: this._tipHeight - key.height + 1 - }; - - self.push(input); - callback(); - -}; - -module.exports = InputsTransformStream; diff --git a/lib/services/address/streams/outputs-transform.js b/lib/services/address/streams/outputs-transform.js deleted file mode 100644 index 21bbc189..00000000 --- a/lib/services/address/streams/outputs-transform.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict'; - -var Transform = require('stream').Transform; -var inherits = require('util').inherits; -var bitcore = require('bitcore-lib'); -var encodingUtil = require('../../../encoding'); -var $ = bitcore.util.preconditions; - -function OutputsTransformStream(options) { - Transform.call(this, { - objectMode: true - }); - $.checkArgument(options.address instanceof bitcore.Address); - this._address = options.address; - this._addressStr = this._address.toString(); - this._tipHeight = options.tipHeight; -} -inherits(OutputsTransformStream, Transform); - -OutputsTransformStream.prototype._transform = function(chunk, encoding, callback) { - var self = this; - - var key = encodingUtil.decodeOutputKey(chunk.key); - var value = encodingUtil.decodeOutputValue(chunk.value); - - var output = { - address: this._addressStr, - hashType: this._address.type, - txid: key.txid.toString('hex'), //TODO use a buffer - outputIndex: key.outputIndex, - height: key.height, - satoshis: value.satoshis, - script: value.scriptBuffer.toString('hex'), //TODO use a buffer - confirmations: this._tipHeight - key.height + 1 - }; - - self.push(output); - callback(); - -}; - -module.exports = OutputsTransformStream; diff --git a/lib/services/bitcoind/index.js b/lib/services/bitcoind/index.js index 912dfe71..9ceb108d 100644 --- a/lib/services/bitcoind/index.js +++ b/lib/services/bitcoind/index.js @@ -1,34 +1,18 @@ 'use strict'; -var fs = require('fs'); -var path = require('path'); -var spawn = require('child_process').spawn; var util = require('util'); -var mkdirp = require('mkdirp'); var bitcore = require('bitcore-lib'); var zmq = require('zmq'); var async = require('async'); -var LRU = require('lru-cache'); var BitcoinRPC = require('bitcoind-rpc'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; -var Transaction = bitcore.Transaction; var index = require('../../'); var errors = index.errors; var log = index.log; -var utils = require('../../utils'); var Service = require('../../service'); -/** - * Provides a friendly event driven API to bitcoind in Node.js. Manages starting and - * stopping bitcoind as a child process for application support, as well as connecting - * to multiple bitcoind processes for server infrastructure. Results are cached in an - * LRU cache for improved performance and methods added for common queries. - * - * @param {Object} options - * @param {Node} options.node - A reference to the node - */ function Bitcoin(options) { if (!(this instanceof Bitcoin)) { return new Bitcoin(options); @@ -37,111 +21,23 @@ function Bitcoin(options) { Service.call(this, options); this.options = options; - this._initCaches(); - - // bitcoind child process - this.spawn = false; - - // event subscribers this.subscriptions = {}; this.subscriptions.rawtransaction = []; this.subscriptions.hashblock = []; - this.subscriptions.address = {}; - // set initial settings - this._initDefaults(options); - - // available bitcoind nodes this._initClients(); - // for testing purposes this._process = options.process || process; this.on('error', function(err) { log.error(err.stack); }); } + util.inherits(Bitcoin, Service); Bitcoin.dependencies = []; -Bitcoin.DEFAULT_MAX_TXIDS = 1000; -Bitcoin.DEFAULT_MAX_HISTORY = 50; -Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000; -Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS = 0.9999; -Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000; -Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000; -Bitcoin.DEFAULT_SPAWN_STOP_TIME = 10000; -Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; -Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; -Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; -Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; -Bitcoin.DEFAULT_ZMQ_DELAY_WARNING_MULTIPLIER = 5; -Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY = 5; -Bitcoin.DEFAULT_CONFIG_SETTINGS = { - server: 1, - whitelist: '127.0.0.1', - txindex: 1, - addressindex: 1, - timestampindex: 1, - spentindex: 1, - zmqpubrawtx: 'tcp://127.0.0.1:28332', - zmqpubhashblock: 'tcp://127.0.0.1:28332', - rpcallowip: '127.0.0.1', - rpcuser: 'bitcoin', - rpcpassword: 'local321', - uacomment: 'bitcore' -}; - -Bitcoin.prototype._initDefaults = function(options) { - /* jshint maxcomplexity: 15 */ - - // limits - this.maxTxids = options.maxTxids || Bitcoin.DEFAULT_MAX_TXIDS; - this.maxTransactionHistory = options.maxTransactionHistory || Bitcoin.DEFAULT_MAX_HISTORY; - this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY; - this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT; - - // spawn restart setting - this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME; - this.spawnStopTime = options.spawnStopTime || Bitcoin.DEFAULT_SPAWN_STOP_TIME; - - // try all interval - this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL; - this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL; - - // rpc limits - this.transactionConcurrency = options.transactionConcurrency || Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY; - - // sync progress level when zmq subscribes to events - this.zmqSubscribeProgress = options.zmqSubscribeProgress || Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS; - - // set the zmq delay warning multiplier - this.zmqDelayWarningMultiplier = options.zmqDelayWarningMultiplier || Bitcoin.DEFAULT_ZMQ_DELAY_WARNING_MULTIPLIER; - this.zmqDelayWarningMultiplierCouunt = 0; -}; - -Bitcoin.prototype._initCaches = function() { - // caches valid until there is a new block - this.utxosCache = LRU(5); - this.txidsCache = LRU(5); - this.balanceCache = LRU(5); - this.summaryCache = LRU(5); - this.blockOverviewCache = LRU(12); - this.transactionDetailedCache = LRU(10); - - // caches valid indefinitely - this.transactionCache = LRU(10); - this.rawTransactionCache = LRU(5); - this.blockCache = LRU(12); - this.rawBlockCache = LRU(6); - this.blockHeaderCache = LRU(12); - this.zmqKnownTransactions = LRU(5); - this.zmqKnownBlocks = LRU(1); - this.lastTip = 0; - this.lastTipTimeout = false; -}; - Bitcoin.prototype._initClients = function() { var self = this; this.nodes = []; @@ -157,40 +53,13 @@ Bitcoin.prototype._initClients = function() { }); }; -/** - * Called by Node to determine the available API methods. - */ Bitcoin.prototype.getAPIMethods = function() { - return []; var methods = [ - ['getBlock', this, this.getBlock, 1], - ['getRawBlock', this, this.getRawBlock, 1], - ['getBlockHeader', this, this.getBlockHeader, 1], - ['getBlockOverview', this, this.getBlockOverview, 1], - ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], - ['getBestBlockHash', this, this.getBestBlockHash, 0], - ['getSpentInfo', this, this.getSpentInfo, 1], - ['getInfo', this, this.getInfo, 0], - ['syncPercentage', this, this.syncPercentage, 0], - ['isSynced', this, this.isSynced, 0], - ['getRawTransaction', this, this.getRawTransaction, 1], - ['getTransaction', this, this.getTransaction, 1], - ['getDetailedTransaction', this, this.getDetailedTransaction, 1], - ['sendTransaction', this, this.sendTransaction, 1], - ['estimateFee', this, this.estimateFee, 1], - ['getAddressTxids', this, this.getAddressTxids, 2], - ['getAddressBalance', this, this.getAddressBalance, 2], - ['getAddressUnspentOutputs', this, this.getAddressUnspentOutputs, 2], - ['getAddressHistory', this, this.getAddressHistory, 2], - ['getAddressSummary', this, this.getAddressSummary, 1], - ['generateBlock', this, this.generateBlock, 1] + ['getBlock', this, this.getBlock, 1] ]; return methods; }; -/** - * Called by the Bus to determine the available events. - */ Bitcoin.prototype.getPublishEvents = function() { return [ { @@ -204,12 +73,6 @@ Bitcoin.prototype.getPublishEvents = function() { scope: this, subscribe: this.subscribe.bind(this, 'hashblock'), unsubscribe: this.unsubscribe.bind(this, 'hashblock') - }, - { - name: 'bitcoind/addresstxid', - scope: this, - subscribe: this.subscribeAddress.bind(this), - unsubscribe: this.unsubscribeAddress.bind(this) } ]; }; @@ -227,219 +90,6 @@ Bitcoin.prototype.unsubscribe = function(name, emitter) { log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/' + name, 'total:', this.subscriptions[name].length); }; -Bitcoin.prototype.subscribeAddress = function(emitter, addresses) { - var self = this; - - function addAddress(addressStr) { - if(self.subscriptions.address[addressStr]) { - var emitters = self.subscriptions.address[addressStr]; - var index = emitters.indexOf(emitter); - if (index === -1) { - self.subscriptions.address[addressStr].push(emitter); - } - } else { - self.subscriptions.address[addressStr] = [emitter]; - } - } - - for(var i = 0; i < addresses.length; i++) { - if (bitcore.Address.isValid(addresses[i], this.node.network)) { - addAddress(addresses[i]); - } - } - - log.info(emitter.remoteAddress, 'subscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); -}; - -Bitcoin.prototype.unsubscribeAddress = function(emitter, addresses) { - var self = this; - if(!addresses) { - return this.unsubscribeAddressAll(emitter); - } - - function removeAddress(addressStr) { - var emitters = self.subscriptions.address[addressStr]; - var index = emitters.indexOf(emitter); - if(index > -1) { - emitters.splice(index, 1); - if (emitters.length === 0) { - delete self.subscriptions.address[addressStr]; - } - } - } - - for(var i = 0; i < addresses.length; i++) { - if(this.subscriptions.address[addresses[i]]) { - removeAddress(addresses[i]); - } - } - - log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); -}; - -/** - * A helper function for the `unsubscribe` method to unsubscribe from all addresses. - * @param {String} name - The name of the event - * @param {EventEmitter} emitter - An instance of an event emitter - */ -Bitcoin.prototype.unsubscribeAddressAll = function(emitter) { - for(var hashHex in this.subscriptions.address) { - var emitters = this.subscriptions.address[hashHex]; - var index = emitters.indexOf(emitter); - if(index > -1) { - emitters.splice(index, 1); - } - if (emitters.length === 0) { - delete this.subscriptions.address[hashHex]; - } - } - log.info(emitter.remoteAddress, 'unsubscribe:', 'bitcoind/addresstxid', 'total:', _.size(this.subscriptions.address)); -}; - -Bitcoin.prototype._getDefaultConfig = function() { - var config = ''; - var defaults = Bitcoin.DEFAULT_CONFIG_SETTINGS; - for(var key in defaults) { - config += key + '=' + defaults[key] + '\n'; - } - return config; -}; - -Bitcoin.prototype._parseBitcoinConf = function(configPath) { - var options = {}; - var file = fs.readFileSync(configPath); - var unparsed = file.toString().split('\n'); - for(var i = 0; i < unparsed.length; i++) { - var line = unparsed[i]; - if (!line.match(/^\#/) && line.match(/\=/)) { - var option = line.split('='); - var value; - if (!Number.isNaN(Number(option[1]))) { - value = Number(option[1]); - } else { - value = option[1]; - } - options[option[0]] = value; - } - } - return options; -}; - -Bitcoin.prototype._expandRelativeDatadir = function() { - if (!utils.isAbsolutePath(this.options.spawn.datadir)) { - $.checkState(this.node.configPath); - $.checkState(utils.isAbsolutePath(this.node.configPath)); - var baseConfigPath = path.dirname(this.node.configPath); - this.options.spawn.datadir = path.resolve(baseConfigPath, this.options.spawn.datadir); - } -}; - -Bitcoin.prototype._loadSpawnConfiguration = function(node) { - /* jshint maxstatements: 25 */ - - $.checkArgument(this.options.spawn, 'Please specify "spawn" in bitcoind config options'); - $.checkArgument(this.options.spawn.datadir, 'Please specify "spawn.datadir" in bitcoind config options'); - $.checkArgument(this.options.spawn.exec, 'Please specify "spawn.exec" in bitcoind config options'); - - this._expandRelativeDatadir(); - - var spawnOptions = this.options.spawn; - var configPath = path.resolve(spawnOptions.datadir, './bitcoin.conf'); - - log.info('Using bitcoin config file:', configPath); - - this.spawn = {}; - this.spawn.datadir = this.options.spawn.datadir; - this.spawn.exec = this.options.spawn.exec; - this.spawn.configPath = configPath; - this.spawn.config = {}; - - if (!fs.existsSync(spawnOptions.datadir)) { - mkdirp.sync(spawnOptions.datadir); - } - - if (!fs.existsSync(configPath)) { - var defaultConfig = this._getDefaultConfig(); - fs.writeFileSync(configPath, defaultConfig); - } - - _.extend(this.spawn.config, this._getDefaultConf()); - _.extend(this.spawn.config, this._parseBitcoinConf(configPath)); - - var networkConfigPath = this._getNetworkConfigPath(); - if (networkConfigPath && fs.existsSync(networkConfigPath)) { - _.extend(this.spawn.config, this._parseBitcoinConf(networkConfigPath)); - } - - var spawnConfig = this.spawn.config; - - this._checkConfigIndexes(spawnConfig, node); - -}; - -Bitcoin.prototype._checkConfigIndexes = function(spawnConfig, node) { - $.checkState( - spawnConfig.txindex && spawnConfig.txindex === 1, - '"txindex" option is required in order to use transaction query features of bitcore-node. ' + - 'Please add "txindex=1" to your configuration and reindex an existing database if ' + - 'necessary with reindex=1' - ); - - $.checkState( - spawnConfig.addressindex && spawnConfig.addressindex === 1, - '"addressindex" option is required in order to use address query features of bitcore-node. ' + - 'Please add "addressindex=1" to your configuration and reindex an existing database if ' + - 'necessary with reindex=1' - ); - - $.checkState( - spawnConfig.spentindex && spawnConfig.spentindex === 1, - '"spentindex" option is required in order to use spent info query features of bitcore-node. ' + - 'Please add "spentindex=1" to your configuration and reindex an existing database if ' + - 'necessary with reindex=1' - ); - - $.checkState( - spawnConfig.server && spawnConfig.server === 1, - '"server" option is required to communicate to bitcoind from bitcore. ' + - 'Please add "server=1" to your configuration and restart' - ); - - $.checkState( - spawnConfig.zmqpubrawtx, - '"zmqpubrawtx" option is required to get event updates from bitcoind. ' + - 'Please add "zmqpubrawtx=tcp://127.0.0.1:" to your configuration and restart' - ); - - $.checkState( - spawnConfig.zmqpubhashblock, - '"zmqpubhashblock" option is required to get event updates from bitcoind. ' + - 'Please add "zmqpubhashblock=tcp://127.0.0.1:" to your configuration and restart' - ); - - $.checkState( - (spawnConfig.zmqpubhashblock === spawnConfig.zmqpubrawtx), - '"zmqpubrawtx" and "zmqpubhashblock" are expected to the same host and port in bitcoin.conf' - ); - - if (spawnConfig.reindex && spawnConfig.reindex === 1) { - log.warn('Reindex option is currently enabled. This means that bitcoind is undergoing a reindex. ' + - 'The reindex flag will start the index from beginning every time the node is started, so it ' + - 'should be removed after the reindex has been initiated. Once the reindex is complete, the rest ' + - 'of bitcore-node services will start.'); - node._reindex = true; - } -}; - -Bitcoin.prototype._resetCaches = function() { - this.transactionDetailedCache.reset(); - this.utxosCache.reset(); - this.txidsCache.reset(); - this.balanceCache.reset(); - this.summaryCache.reset(); - this.blockOverviewCache.reset(); -}; - Bitcoin.prototype._tryAllClients = function(func, callback) { var self = this; var nodesIndex = this.nodesIndex; @@ -489,66 +139,23 @@ Bitcoin.prototype._initChain = function(callback) { callback(); }); }); - }); }); }; -Bitcoin.prototype._getDefaultConf = function() { - var networkOptions = { - rpcport: 8332 - }; - if (this.node.network === bitcore.Networks.testnet) { - networkOptions.rpcport = 18332; - } - return networkOptions; -}; - -Bitcoin.prototype._getNetworkConfigPath = function() { - var networkPath; - if (this.node.network === bitcore.Networks.testnet) { - networkPath = 'testnet3/bitcoin.conf'; - if (this.node.network.regtestEnabled) { - networkPath = 'regtest/bitcoin.conf'; - } - } - return networkPath; -}; - -Bitcoin.prototype._getNetworkOption = function() { - var networkOption; - if (this.node.network === bitcore.Networks.testnet) { - networkOption = '--testnet'; - if (this.node.network.regtestEnabled) { - networkOption = '--regtest'; - } - } - return networkOption; -}; - Bitcoin.prototype._zmqBlockHandler = function(node, message) { var self = this; - - // Update the current chain tip self._rapidProtectedUpdateTip(node, message); + self.emit('block', message); - // Notify block subscribers - var id = message.toString('binary'); - if (!self.zmqKnownBlocks.get(id)) { - self.zmqKnownBlocks.set(id, true); - self.emit('block', message); - - for (var i = 0; i < this.subscriptions.hashblock.length; i++) { - this.subscriptions.hashblock[i].emit('bitcoind/hashblock', message.toString('hex')); - } + for (var i = 0; i < this.subscriptions.hashblock.length; i++) { + this.subscriptions.hashblock[i].emit('bitcoind/hashblock', message.toString('hex')); } - }; Bitcoin.prototype._rapidProtectedUpdateTip = function(node, message) { var self = this; - // Prevent a rapid succession of tip updates if (new Date() - self.lastTip > 1000) { self.lastTip = new Date(); self._updateTip(node, message); @@ -566,10 +173,6 @@ Bitcoin.prototype._updateTip = function(node, message) { var hex = message.toString('hex'); if (hex !== self.tiphash) { self.tiphash = message.toString('hex'); - - // reset block valid caches - self._resetCaches(); - node.client.getBlock(self.tiphash, function(err, response) { if (err) { var error = self._wrapRPCError(err); @@ -580,82 +183,15 @@ Bitcoin.prototype._updateTip = function(node, message) { self.emit('tip', self.height); } }); - - if(!self.node.stopping) { - self.syncPercentage(function(err, percentage) { - if (err) { - self.emit('error', err); - } else { - if (Math.round(percentage) >= 100) { - self.emit('synced', self.height); - } - log.info('Bitcoin Height:', self.height, 'Percentage:', percentage.toFixed(2)); - } - }); - } } }; -Bitcoin.prototype._getAddressesFromTransaction = function(transaction) { - var addresses = []; - - for (var i = 0; i < transaction.inputs.length; i++) { - var input = transaction.inputs[i]; - if (input.script) { - var inputAddress = input.script.toAddress(this.node.network); - if (inputAddress) { - addresses.push(inputAddress.toString()); - } - } - } - - for (var j = 0; j < transaction.outputs.length; j++) { - var output = transaction.outputs[j]; - if (output.script) { - var outputAddress = output.script.toAddress(this.node.network); - if (outputAddress) { - addresses.push(outputAddress.toString()); - } - } - } - - return _.uniq(addresses); -}; - -Bitcoin.prototype._notifyAddressTxidSubscribers = function(txid, transaction) { - var addresses = this._getAddressesFromTransaction(transaction); - for (var i = 0; i < addresses.length; i++) { - var address = addresses[i]; - if(this.subscriptions.address[address]) { - var emitters = this.subscriptions.address[address]; - for(var j = 0; j < emitters.length; j++) { - emitters[j].emit('bitcoind/addresstxid', { - address: address, - txid: txid - }); - } - } - } -}; Bitcoin.prototype._zmqTransactionHandler = function(node, message) { var self = this; - var hash = bitcore.crypto.Hash.sha256sha256(message); - var id = hash.toString('binary'); - if (!self.zmqKnownTransactions.get(id)) { - self.zmqKnownTransactions.set(id, true); - self.emit('tx', message); - - // Notify transaction subscribers - for (var i = 0; i < this.subscriptions.rawtransaction.length; i++) { - this.subscriptions.rawtransaction[i].emit('bitcoind/rawtransaction', message.toString('hex')); - } - - var tx = bitcore.Transaction(); - tx.fromString(message); - var txid = bitcore.util.buffer.reverse(hash).toString('hex'); - self._notifyAddressTxidSubscribers(txid, tx); - + self.emit('tx', message); + for (var i = 0; i < this.subscriptions.rawtransaction.length; i++) { + this.subscriptions.rawtransaction[i].emit('bitcoind/rawtransaction', message.toString('hex')); } }; @@ -664,7 +200,6 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { var interval; function checkAndSubscribe(callback) { - // update tip node.client.getBestBlockHash(function(err, response) { if (err) { return callback(self._wrapRPCError(err)); @@ -673,14 +208,12 @@ Bitcoin.prototype._checkSyncedAndSubscribeZmqEvents = function(node) { self.emit('block', blockhash); self._updateTip(node, blockhash); - // check if synced node.client.getBlockchainInfo(function(err, response) { if (err) { return callback(self._wrapRPCError(err)); } var progress = response.result.verificationprogress; if (progress >= self.zmqSubscribeProgress) { - // subscribe to events for further updates self._subscribeZmqEvents(node); clearInterval(interval); callback(null, true); @@ -750,39 +283,10 @@ Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { }, 5000); }); - //monitors are polling and not event-driven node.zmqSubSocket.monitor(100, 0); node.zmqSubSocket.connect(zmqUrl); }; -Bitcoin.prototype._checkReindex = function(node, callback) { - var self = this; - var interval; - function finish(err) { - clearInterval(interval); - callback(err); - } - if (node._reindex) { - interval = setInterval(function() { - node.client.getBlockchainInfo(function(err, response) { - if (err) { - return finish(self._wrapRPCError(err)); - } - var percentSynced = response.result.verificationprogress * 100; - - log.info('Bitcoin Core Daemon Reindex Percentage: ' + percentSynced.toFixed(2)); - - if (Math.round(percentSynced) >= 100) { - node._reindex = false; - finish(); - } - }); - }, node._reindexWait || Bitcoin.DEFAULT_REINDEX_INTERVAL); - } else { - callback(); - } -}; - Bitcoin.prototype._loadTipFromNode = function(node, callback) { var self = this; node.client.getBestBlockHash(function(err, response) { @@ -804,135 +308,6 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) { }); }; -Bitcoin.prototype._stopSpawnedBitcoin = function(callback) { - var self = this; - var spawnOptions = this.options.spawn; - var pidPath = spawnOptions.datadir + '/bitcoind.pid'; - - function stopProcess() { - fs.readFile(pidPath, 'utf8', function(err, pid) { - if (err && err.code === 'ENOENT') { - // pid file doesn't exist we can continue - return callback(null); - } else if (err) { - return callback(err); - } - pid = parseInt(pid); - if (!Number.isFinite(pid)) { - // pid doesn't exist we can continue - return callback(null); - } - try { - log.warn('Stopping existing spawned bitcoin process with pid: ' + pid); - self._process.kill(pid, 'SIGINT'); - } catch(err) { - if (err && err.code === 'ESRCH') { - log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid); - return callback(null); - } else if(err) { - return callback(err); - } - } - setTimeout(function() { - stopProcess(); - }, self.spawnStopTime); - }); - } - - stopProcess(); -}; - -Bitcoin.prototype._spawnChildProcess = function(callback) { - var self = this; - - var node = {}; - node._reindex = false; - node._reindexWait = 10000; - - try { - self._loadSpawnConfiguration(node); - } catch(e) { - return callback(e); - } - - var options = [ - '--conf=' + this.spawn.configPath, - '--datadir=' + this.spawn.datadir, - ]; - - if (self._getNetworkOption()) { - options.push(self._getNetworkOption()); - } - - self._stopSpawnedBitcoin(function(err) { - if (err) { - return callback(err); - } - - log.info('Starting bitcoin process'); - self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'}); - - self.spawn.process.on('error', function(err) { - self.emit('error', err); - }); - - self.spawn.process.once('exit', function(code) { - if (!self.node.stopping) { - log.warn('Bitcoin process unexpectedly exited with code:', code); - log.warn('Restarting bitcoin child process in ' + self.spawnRestartTime + 'ms'); - setTimeout(function() { - self._spawnChildProcess(function(err) { - if (err) { - return self.emit('error', err); - } - log.warn('Bitcoin process restarted'); - }); - }, self.spawnRestartTime); - } - }); - - var exitShutdown = false; - - async.retry({times: 60, interval: self.startRetryInterval}, function(done) { - if (self.node.stopping) { - exitShutdown = true; - return done(); - } - - node.client = new BitcoinRPC({ - protocol: 'http', - host: '127.0.0.1', - port: self.spawn.config.rpcport, - user: self.spawn.config.rpcuser, - pass: self.spawn.config.rpcpassword - }); - - self._loadTipFromNode(node, done); - - }, function(err) { - if (err) { - return callback(err); - } - if (exitShutdown) { - return callback(new Error('Stopping while trying to spawn bitcoind.')); - } - - self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx); - - self._checkReindex(node, function(err) { - if (err) { - return callback(err); - } - self._checkSyncedAndSubscribeZmqEvents(node); - callback(null, node); - }); - - }); - - }); - -}; - Bitcoin.prototype._connectProcess = function(config, callback) { var self = this; var node = {}; @@ -970,10 +345,6 @@ Bitcoin.prototype._connectProcess = function(config, callback) { }); }; -/** - * Called by Node to start the service - * @param {Function} callback - */ Bitcoin.prototype.start = function(callback) { var self = this; @@ -1018,306 +389,6 @@ Bitcoin.prototype.start = function(callback) { }; -/** - * Helper to determine the state of the database. - * @param {Function} callback - */ -Bitcoin.prototype.isSynced = function(callback) { - this.syncPercentage(function(err, percentage) { - if (err) { - return callback(err); - } - if (Math.round(percentage) >= 100) { - callback(null, true); - } else { - callback(null, false); - } - }); -}; - -/** - * Helper to determine the progress of the database. - * @param {Function} callback - */ -Bitcoin.prototype.syncPercentage = function(callback) { - var self = this; - this.client.getBlockchainInfo(function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - var percentSynced = response.result.verificationprogress * 100; - callback(null, percentSynced); - }); -}; - -Bitcoin.prototype._normalizeAddressArg = function(addressArg) { - var addresses = [addressArg]; - if (Array.isArray(addressArg)) { - addresses = addressArg; - } - return addresses; -}; - -/** - * Will get the balance for an address or multiple addresses - * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses - * @param {Object} options - * @param {Function} callback - */ -Bitcoin.prototype.getAddressBalance = function(addressArg, options, callback) { - var self = this; - var addresses = self._normalizeAddressArg(addressArg); - var cacheKey = addresses.join(''); - var balance = self.balanceCache.get(cacheKey); - if (balance) { - return setImmediate(function() { - callback(null, balance); - }); - } else { - this.client.getAddressBalance({addresses: addresses}, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - self.balanceCache.set(cacheKey, response.result); - callback(null, response.result); - }); - } -}; - -/** - * Will get the unspent outputs for an address or multiple addresses - * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses - * @param {Object} options - * @param {Function} callback - */ -Bitcoin.prototype.getAddressUnspentOutputs = function(addressArg, options, callback) { - var self = this; - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var addresses = self._normalizeAddressArg(addressArg); - var cacheKey = addresses.join(''); - var utxos = self.utxosCache.get(cacheKey); - - function transformUnspentOutput(delta) { - var script = bitcore.Script.fromAddress(delta.address); - return { - address: delta.address, - txid: delta.txid, - outputIndex: delta.index, - script: script.toHex(), - satoshis: delta.satoshis, - timestamp: delta.timestamp - }; - } - - function updateWithMempool(confirmedUtxos, mempoolDeltas) { - /* jshint maxstatements: 20 */ - if (!mempoolDeltas || !mempoolDeltas.length) { - return confirmedUtxos; - } - var isSpentOutputs = false; - var mempoolUnspentOutputs = []; - var spentOutputs = []; - - for (var i = 0; i < mempoolDeltas.length; i++) { - var delta = mempoolDeltas[i]; - if (delta.prevtxid && delta.satoshis <= 0) { - if (!spentOutputs[delta.prevtxid]) { - spentOutputs[delta.prevtxid] = [delta.prevout]; - } else { - spentOutputs[delta.prevtxid].push(delta.prevout); - } - isSpentOutputs = true; - } else { - mempoolUnspentOutputs.push(transformUnspentOutput(delta)); - } - } - - var utxos = mempoolUnspentOutputs.reverse().concat(confirmedUtxos); - - if (isSpentOutputs) { - return utxos.filter(function(utxo) { - if (!spentOutputs[utxo.txid]) { - return true; - } else { - return (spentOutputs[utxo.txid].indexOf(utxo.outputIndex) === -1); - } - }); - } - - return utxos; - } - - function finish(mempoolDeltas) { - if (utxos) { - return setImmediate(function() { - callback(null, updateWithMempool(utxos, mempoolDeltas)); - }); - } else { - self.client.getAddressUtxos({addresses: addresses}, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - var utxos = response.result.reverse(); - self.utxosCache.set(cacheKey, utxos); - callback(null, updateWithMempool(utxos, mempoolDeltas)); - }); - } - } - - if (queryMempool) { - self.client.getAddressMempool({addresses: addresses}, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - finish(response.result); - }); - } else { - finish(); - } - -}; - -Bitcoin.prototype._getBalanceFromMempool = function(deltas) { - var satoshis = 0; - for (var i = 0; i < deltas.length; i++) { - satoshis += deltas[i].satoshis; - } - return satoshis; -}; - -Bitcoin.prototype._getTxidsFromMempool = function(deltas) { - var mempoolTxids = []; - var mempoolTxidsKnown = {}; - for (var i = 0; i < deltas.length; i++) { - var txid = deltas[i].txid; - if (!mempoolTxidsKnown[txid]) { - mempoolTxids.push(txid); - mempoolTxidsKnown[txid] = true; - } - } - return mempoolTxids; -}; - -Bitcoin.prototype._getHeightRangeQuery = function(options, clone) { - if (options.start >= 0 && options.end >= 0) { - if (options.end > options.start) { - throw new TypeError('"end" is expected to be less than or equal to "start"'); - } - if (clone) { - // reverse start and end as the order in bitcore is most recent to less recent - clone.start = options.end; - clone.end = options.start; - } - return true; - } - return false; -}; - -/** - * Will get the txids for an address or multiple addresses - * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses - * @param {Object} options - * @param {Function} callback - */ -Bitcoin.prototype.getAddressTxids = function(addressArg, options, callback) { - /* jshint maxstatements: 20 */ - var self = this; - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var queryMempoolOnly = _.isUndefined(options.queryMempoolOnly) ? false : options.queryMempoolOnly; - var rangeQuery = false; - try { - rangeQuery = self._getHeightRangeQuery(options); - } catch(err) { - return callback(err); - } - if (rangeQuery) { - queryMempool = false; - } - if (queryMempoolOnly) { - queryMempool = true; - rangeQuery = false; - } - var addresses = self._normalizeAddressArg(addressArg); - var cacheKey = addresses.join(''); - var mempoolTxids = []; - var txids = queryMempoolOnly ? false : self.txidsCache.get(cacheKey); - - function finish() { - if (queryMempoolOnly) { - return setImmediate(function() { - callback(null, mempoolTxids.reverse()); - }); - } - if (txids && !rangeQuery) { - var allTxids = mempoolTxids.reverse().concat(txids); - return setImmediate(function() { - callback(null, allTxids); - }); - } else { - var txidOpts = { - addresses: addresses - }; - if (rangeQuery) { - self._getHeightRangeQuery(options, txidOpts); - } - self.client.getAddressTxids(txidOpts, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - response.result.reverse(); - if (!rangeQuery) { - self.txidsCache.set(cacheKey, response.result); - } - var allTxids = mempoolTxids.reverse().concat(response.result); - return callback(null, allTxids); - }); - } - } - - if (queryMempool) { - self.client.getAddressMempool({addresses: addresses}, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - mempoolTxids = self._getTxidsFromMempool(response.result); - finish(); - }); - } else { - finish(); - } - -}; - -Bitcoin.prototype._getConfirmationsDetail = function(transaction) { - $.checkState(this.height > 0, 'current height is unknown'); - var confirmations = 0; - if (transaction.height >= 0) { - confirmations = this.height - transaction.height + 1; - } - if (confirmations < 0) { - log.warn('Negative confirmations calculated for transaction:', transaction.hash); - } - return Math.max(0, confirmations); -}; - -Bitcoin.prototype._getAddressDetailsForInput = function(input, inputIndex, result, addressStrings) { - if (!input.address) { - return; - } - var address = input.address; - if (addressStrings.indexOf(address) >= 0) { - if (!result.addresses[address]) { - result.addresses[address] = { - inputIndexes: [inputIndex], - outputIndexes: [] - }; - } else { - result.addresses[address].inputIndexes.push(inputIndex); - } - result.satoshis -= input.satoshis; - } -}; - Bitcoin.prototype._getAddressDetailsForOutput = function(output, outputIndex, result, addressStrings) { if (!output.address) { return; @@ -1336,240 +407,6 @@ Bitcoin.prototype._getAddressDetailsForOutput = function(output, outputIndex, re } }; -Bitcoin.prototype._getAddressDetailsForTransaction = function(transaction, addressStrings) { - var result = { - addresses: {}, - satoshis: 0 - }; - - for (var inputIndex = 0; inputIndex < transaction.inputs.length; inputIndex++) { - var input = transaction.inputs[inputIndex]; - this._getAddressDetailsForInput(input, inputIndex, result, addressStrings); - } - - for (var outputIndex = 0; outputIndex < transaction.outputs.length; outputIndex++) { - var output = transaction.outputs[outputIndex]; - this._getAddressDetailsForOutput(output, outputIndex, result, addressStrings); - } - - $.checkState(Number.isFinite(result.satoshis)); - - return result; -}; - -/** - * Will expand into a detailed transaction from a txid - * @param {Object} txid - A bitcoin transaction id - * @param {Function} callback - */ -Bitcoin.prototype._getAddressDetailedTransaction = function(txid, options, next) { - var self = this; - - self.getDetailedTransaction( - txid, - function(err, transaction) { - if (err) { - return next(err); - } - - var addressDetails = self._getAddressDetailsForTransaction(transaction, options.addressStrings); - - var details = { - addresses: addressDetails.addresses, - satoshis: addressDetails.satoshis, - confirmations: self._getConfirmationsDetail(transaction), - tx: transaction - }; - next(null, details); - } - ); -}; - -Bitcoin.prototype._getAddressStrings = function(addresses) { - var addressStrings = []; - for (var i = 0; i < addresses.length; i++) { - var address = addresses[i]; - if (address instanceof bitcore.Address) { - addressStrings.push(address.toString()); - } else if (_.isString(address)) { - addressStrings.push(address); - } else { - throw new TypeError('Addresses are expected to be strings'); - } - } - return addressStrings; -}; - -Bitcoin.prototype._paginateTxids = function(fullTxids, fromArg, toArg) { - var txids; - var from = parseInt(fromArg); - var to = parseInt(toArg); - $.checkState(from < to, '"from" (' + from + ') is expected to be less than "to" (' + to + ')'); - txids = fullTxids.slice(from, to); - return txids; -}; - -/** - * Will detailed transaction history for an address or multiple addresses - * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses - * @param {Object} options - * @param {Function} callback - */ -Bitcoin.prototype.getAddressHistory = function(addressArg, options, callback) { - var self = this; - var addresses = self._normalizeAddressArg(addressArg); - if (addresses.length > this.maxAddressesQuery) { - return callback(new TypeError('Maximum number of addresses (' + this.maxAddressesQuery + ') exceeded')); - } - - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var addressStrings = this._getAddressStrings(addresses); - - var fromArg = parseInt(options.from || 0); - var toArg = parseInt(options.to || self.maxTransactionHistory); - - if ((toArg - fromArg) > self.maxTransactionHistory) { - return callback(new Error( - '"from" (' + options.from + ') and "to" (' + options.to + ') range should be less than or equal to ' + - self.maxTransactionHistory - )); - } - - self.getAddressTxids(addresses, options, function(err, txids) { - if (err) { - return callback(err); - } - - var totalCount = txids.length; - try { - txids = self._paginateTxids(txids, fromArg, toArg); - } catch(e) { - return callback(e); - } - - async.mapLimit( - txids, - self.transactionConcurrency, - function(txid, next) { - self._getAddressDetailedTransaction(txid, { - queryMempool: queryMempool, - addressStrings: addressStrings - }, next); - }, - function(err, transactions) { - if (err) { - return callback(err); - } - callback(null, { - totalCount: totalCount, - items: transactions - }); - } - ); - }); -}; - -/** - * Will get the summary including txids and balance for an address or multiple addresses - * @param {String|Address|Array} addressArg - An address string, bitcore address, or array of addresses - * @param {Object} options - * @param {Function} callback - */ -Bitcoin.prototype.getAddressSummary = function(addressArg, options, callback) { - var self = this; - var summary = {}; - var queryMempool = _.isUndefined(options.queryMempool) ? true : options.queryMempool; - var summaryTxids = []; - var mempoolTxids = []; - var addresses = self._normalizeAddressArg(addressArg); - var cacheKey = addresses.join(''); - - function finishWithTxids() { - if (!options.noTxList) { - var allTxids = mempoolTxids.reverse().concat(summaryTxids); - var fromArg = parseInt(options.from || 0); - var toArg = parseInt(options.to || self.maxTxids); - - if ((toArg - fromArg) > self.maxTxids) { - return callback(new Error( - '"from" (' + fromArg + ') and "to" (' + toArg + ') range should be less than or equal to ' + - self.maxTxids - )); - } - var paginatedTxids; - try { - paginatedTxids = self._paginateTxids(allTxids, fromArg, toArg); - } catch(e) { - return callback(e); - } - - var allSummary = _.clone(summary); - allSummary.txids = paginatedTxids; - callback(null, allSummary); - } else { - callback(null, summary); - } - } - - function querySummary() { - async.parallel([ - function getTxList(done) { - self.getAddressTxids(addresses, {queryMempool: false}, function(err, txids) { - if (err) { - return done(err); - } - summaryTxids = txids; - summary.appearances = txids.length; - done(); - }); - }, - function getBalance(done) { - self.getAddressBalance(addresses, options, function(err, data) { - if (err) { - return done(err); - } - summary.totalReceived = data.received; - summary.totalSpent = data.received - data.balance; - summary.balance = data.balance; - done(); - }); - }, - function getMempool(done) { - if (!queryMempool) { - return done(); - } - self.client.getAddressMempool({'addresses': addresses}, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - mempoolTxids = self._getTxidsFromMempool(response.result); - summary.unconfirmedAppearances = mempoolTxids.length; - summary.unconfirmedBalance = self._getBalanceFromMempool(response.result); - done(); - }); - }, - ], function(err) { - if (err) { - return callback(err); - } - self.summaryCache.set(cacheKey, summary); - finishWithTxids(); - }); - } - - if (options.noTxList) { - var summaryCache = self.summaryCache.get(cacheKey); - if (summaryCache) { - callback(null, summaryCache); - } else { - querySummary(); - } - } else { - querySummary(); - } - -}; - Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { var self = this; if (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) { @@ -1586,13 +423,8 @@ Bitcoin.prototype._maybeGetBlockHash = function(blockArg, callback) { } }; -/** - * Will retrieve a block as a Node.js Buffer - * @param {String|Number} block - A block hash or block height number - * @param {Function} callback - */ + Bitcoin.prototype.getRawBlock = function(blockArg, callback) { - // TODO apply performance patch to the RPC method for raw data var self = this; function queryBlock(err, blockhash) { @@ -1605,533 +437,36 @@ Bitcoin.prototype.getRawBlock = function(blockArg, callback) { return done(self._wrapRPCError(err)); } var buffer = new Buffer(response.result, 'hex'); - self.rawBlockCache.set(blockhash, buffer); done(null, buffer); }); }, callback); } - var cachedBlock = self.rawBlockCache.get(blockArg); - if (cachedBlock) { - return setImmediate(function() { - callback(null, cachedBlock); - }); - } else { - self._maybeGetBlockHash(blockArg, queryBlock); - } -}; - -/** - * Similar to getBlockHeader but will include a list of txids - * @param {String|Number} block - A block hash or block height number - * @param {Function} callback - */ -Bitcoin.prototype.getBlockOverview = function(blockArg, callback) { - var self = this; - - function queryBlock(err, blockhash) { - if (err) { - return callback(err); - } - var cachedBlock = self.blockOverviewCache.get(blockhash); - if (cachedBlock) { - return setImmediate(function() { - callback(null, cachedBlock); - }); - } else { - self._tryAllClients(function(client, done) { - client.getBlock(blockhash, true, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var result = response.result; - var blockOverview = { - hash: result.hash, - version: result.version, - confirmations: result.confirmations, - height: result.height, - chainWork: result.chainwork, - prevHash: result.previousblockhash, - nextHash: result.nextblockhash, - merkleRoot: result.merkleroot, - time: result.time, - medianTime: result.mediantime, - nonce: result.nonce, - bits: result.bits, - difficulty: result.difficulty, - txids: result.tx - }; - self.blockOverviewCache.set(blockhash, blockOverview); - done(null, blockOverview); - }); - }, callback); - } - } - self._maybeGetBlockHash(blockArg, queryBlock); }; -/** - * Will retrieve a block as a Bitcore object - * @param {String|Number} block - A block hash or block height number - * @param {Function} callback - */ Bitcoin.prototype.getBlock = function(blockArg, callback) { - // TODO apply performance patch to the RPC method for raw data var self = this; function queryBlock(err, blockhash) { if (err) { return callback(err); } - var cachedBlock = self.blockCache.get(blockhash); - if (cachedBlock) { - return setImmediate(function() { - callback(null, cachedBlock); + self._tryAllClients(function(client, done) { + client.getBlock(blockhash, false, function(err, response) { + if (err) { + return done(self._wrapRPCError(err)); + } + var blockObj = bitcore.Block.fromString(response.result); + done(null, blockObj); }); - } else { - self._tryAllClients(function(client, done) { - client.getBlock(blockhash, false, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var blockObj = bitcore.Block.fromString(response.result); - self.blockCache.set(blockhash, blockObj); - done(null, blockObj); - }); - }, callback); - } + }, callback); } - self._maybeGetBlockHash(blockArg, queryBlock); }; -/** - * Will retrieve an array of block hashes within a range of timestamps - * @param {Number} high - The more recent timestamp in seconds - * @param {Number} low - The older timestamp in seconds - * @param {Function} callback - */ -Bitcoin.prototype.getBlockHashesByTimestamp = function(high, low, options, callback) { - var self = this; - if (_.isFunction(options)) { - callback = options; - options = {}; - } - self.client.getBlockHashes(high, low, options, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); -}; - -/** - * Will return the block index information, the output will have the format: - * { - * hash: '0000000000000a817cd3a74aec2f2246b59eb2cbb1ad730213e6c4a1d68ec2f6', - * confirmations: 5, - * height: 828781, - * chainWork: '00000000000000000000000000000000000000000000000ad467352c93bc6a3b', - * prevHash: '0000000000000504235b2aff578a48470dbf6b94dafa9b3703bbf0ed554c9dd9', - * nextHash: '00000000000000eedd967ec155f237f033686f0924d574b946caf1b0e89551b8' - * version: 536870912, - * merkleRoot: '124e0f3fb5aa268f102b0447002dd9700988fc570efcb3e0b5b396ac7db437a9', - * time: 1462979126, - * medianTime: 1462976771, - * nonce: 2981820714, - * bits: '1a13ca10', - * difficulty: 847779.0710240941, - * } - * @param {String|Number} block - A block hash or block height - * @param {Function} callback - */ -Bitcoin.prototype.getBlockHeader = function(blockArg, callback) { - var self = this; - - function queryHeader(err, blockhash) { - if (err) { - return callback(err); - } - self._tryAllClients(function(client, done) { - client.getBlockHeader(blockhash, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var result = response.result; - var header = { - hash: result.hash, - version: result.version, - confirmations: result.confirmations, - height: result.height, - chainWork: result.chainwork, - prevHash: result.previousblockhash, - nextHash: result.nextblockhash, - merkleRoot: result.merkleroot, - time: result.time, - medianTime: result.mediantime, - nonce: result.nonce, - bits: result.bits, - difficulty: result.difficulty - }; - done(null, header); - }); - }, callback); - } - - self._maybeGetBlockHash(blockArg, queryHeader); -}; - -/** - * Will estimate the fee per kilobyte. - * @param {Number} blocks - The number of blocks for the transaction to be confirmed. - * @param {Function} callback - */ -Bitcoin.prototype.estimateFee = function(blocks, callback) { - var self = this; - this.client.estimateFee(blocks, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); -}; - -/** - * Will add a transaction to the mempool and relay to connected peers - * @param {String|Transaction} transaction - The hex string of the transaction - * @param {Object=} options - * @param {Boolean=} options.allowAbsurdFees - Enable large fees - * @param {Function} callback - */ -Bitcoin.prototype.sendTransaction = function(tx, options, callback) { - var self = this; - var allowAbsurdFees = false; - if (_.isFunction(options) && _.isUndefined(callback)) { - callback = options; - } else if (_.isObject(options)) { - allowAbsurdFees = options.allowAbsurdFees; - } - - this.client.sendRawTransaction(tx, allowAbsurdFees, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); - -}; - -/** - * Will get a transaction as a Node.js Buffer. Results include the mempool. - * @param {String} txid - The transaction hash - * @param {Function} callback - */ -Bitcoin.prototype.getRawTransaction = function(txid, callback) { - var self = this; - var tx = self.rawTransactionCache.get(txid); - if (tx) { - return setImmediate(function() { - callback(null, tx); - }); - } else { - self._tryAllClients(function(client, done) { - client.getRawTransaction(txid, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var buffer = new Buffer(response.result, 'hex'); - self.rawTransactionCache.set(txid, buffer); - done(null, buffer); - }); - }, callback); - } -}; - -/** - * Will get a transaction as a Bitcore Transaction. Results include the mempool. - * @param {String} txid - The transaction hash - * @param {Boolean} queryMempool - Include the mempool - * @param {Function} callback - */ -Bitcoin.prototype.getTransaction = function(txid, callback) { - var self = this; - var tx = self.transactionCache.get(txid); - if (tx) { - return setImmediate(function() { - callback(null, tx); - }); - } else { - self._tryAllClients(function(client, done) { - client.getRawTransaction(txid, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var tx = Transaction(); - tx.fromString(response.result); - self.transactionCache.set(txid, tx); - done(null, tx); - }); - }, callback); - } -}; - -/** - * Will get a detailed view of a transaction including addresses, amounts and fees. - * - * Example result: - * { - * blockHash: '000000000000000002cd0ba6e8fae058747d2344929ed857a18d3484156c9250', - * height: 411462, - * blockTimestamp: 1463070382, - * version: 1, - * hash: 'de184cc227f6d1dc0316c7484aa68b58186a18f89d853bb2428b02040c394479', - * locktime: 411451, - * coinbase: true, - * inputs: [ - * { - * prevTxId: '3d003413c13eec3fa8ea1fe8bbff6f40718c66facffe2544d7516c9e2900cac2', - * outputIndex: 0, - * sequence: 123456789, - * script: [hexString], - * scriptAsm: [asmString], - * address: '1LCTmj15p7sSXv3jmrPfA6KGs6iuepBiiG', - * satoshis: 771146 - * } - * ], - * outputs: [ - * { - * satoshis: 811146, - * script: '76a914d2955017f4e3d6510c57b427cf45ae29c372c99088ac', - * scriptAsm: 'OP_DUP OP_HASH160 d2955017f4e3d6510c57b427cf45ae29c372c990 OP_EQUALVERIFY OP_CHECKSIG', - * address: '1LCTmj15p7sSXv3jmrPfA6KGs6iuepBiiG', - * spentTxId: '4316b98e7504073acd19308b4b8c9f4eeb5e811455c54c0ebfe276c0b1eb6315', - * spentIndex: 1, - * spentHeight: 100 - * } - * ], - * inputSatoshis: 771146, - * outputSatoshis: 811146, - * feeSatoshis: 40000 - * }; - * - * @param {String} txid - The hex string of the transaction - * @param {Function} callback - */ -Bitcoin.prototype.getDetailedTransaction = function(txid, callback) { - var self = this; - var tx = self.transactionDetailedCache.get(txid); - - function addInputsToTx(tx, result) { - tx.inputs = []; - tx.inputSatoshis = 0; - for(var inputIndex = 0; inputIndex < result.vin.length; inputIndex++) { - var input = result.vin[inputIndex]; - if (!tx.coinbase) { - tx.inputSatoshis += input.valueSat; - } - var script = null; - var scriptAsm = null; - if (input.scriptSig) { - script = input.scriptSig.hex; - scriptAsm = input.scriptSig.asm; - } else if (input.coinbase) { - script = input.coinbase; - } - tx.inputs.push({ - prevTxId: input.txid || null, - outputIndex: _.isUndefined(input.vout) ? null : input.vout, - script: script, - scriptAsm: scriptAsm || null, - sequence: input.sequence, - address: input.address || null, - satoshis: _.isUndefined(input.valueSat) ? null : input.valueSat - }); - } - } - - function addOutputsToTx(tx, result) { - tx.outputs = []; - tx.outputSatoshis = 0; - for(var outputIndex = 0; outputIndex < result.vout.length; outputIndex++) { - var out = result.vout[outputIndex]; - tx.outputSatoshis += out.valueSat; - var address = null; - if (out.scriptPubKey && out.scriptPubKey.addresses && out.scriptPubKey.addresses.length === 1) { - address = out.scriptPubKey.addresses[0]; - } - tx.outputs.push({ - satoshis: out.valueSat, - script: out.scriptPubKey.hex, - scriptAsm: out.scriptPubKey.asm, - spentTxId: out.spentTxId, - spentIndex: out.spentIndex, - spentHeight: out.spentHeight, - address: address - }); - } - } - - if (tx) { - return setImmediate(function() { - callback(null, tx); - }); - } else { - self._tryAllClients(function(client, done) { - client.getRawTransaction(txid, 1, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var result = response.result; - var tx = { - hex: result.hex, - blockHash: result.blockhash, - height: result.height ? result.height : -1, - blockTimestamp: result.time, - version: result.version, - hash: txid, - locktime: result.locktime, - }; - - if (result.vin[0] && result.vin[0].coinbase) { - tx.coinbase = true; - } - - addInputsToTx(tx, result); - addOutputsToTx(tx, result); - - if (!tx.coinbase) { - tx.feeSatoshis = tx.inputSatoshis - tx.outputSatoshis; - } else { - tx.feeSatoshis = 0; - } - - self.transactionDetailedCache.set(txid, tx); - - done(null, tx); - }); - }, callback); - } -}; - -/** - * Will get the best block hash for the chain. - * @param {Function} callback - */ -Bitcoin.prototype.getBestBlockHash = function(callback) { - var self = this; - this.client.getBestBlockHash(function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); -}; - -/** - * Will give the txid and inputIndex that spent an output - * @param {Function} callback - */ -Bitcoin.prototype.getSpentInfo = function(options, callback) { - var self = this; - this.client.getSpentInfo(options, function(err, response) { - if (err && err.code === -5) { - return callback(null, {}); - } else if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); -}; - -/** - * This will return information about the database in the format: - * { - * version: 110000, - * protocolVersion: 70002, - * blocks: 151, - * timeOffset: 0, - * connections: 0, - * difficulty: 4.6565423739069247e-10, - * testnet: false, - * network: 'testnet' - * relayFee: 1000, - * errors: '' - * } - * @param {Function} callback - */ -Bitcoin.prototype.getInfo = function(callback) { - var self = this; - this.client.getInfo(function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - var result = response.result; - var info = { - version: result.version, - protocolVersion: result.protocolversion, - blocks: result.blocks, - timeOffset: result.timeoffset, - connections: result.connections, - proxy: result.proxy, - difficulty: result.difficulty, - testnet: result.testnet, - relayFee: result.relayfee, - errors: result.errors, - network: self.node.getNetworkName() - }; - callback(null, info); - }); -}; - -Bitcoin.prototype.getMempool = function(callback) { - this.client.getRawMemPool(function(err, res) { - if(err) { - return callback(err); - } - callback(null, res.result); - }); -}; - -Bitcoin.prototype.generateBlock = function(num, callback) { - var self = this; - this.client.generate(num, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); -}; - -/** - * Called by Node to stop the service. - * @param {Function} callback - */ Bitcoin.prototype.stop = function(callback) { - if (this.spawn && this.spawn.process) { - var exited = false; - this.spawn.process.once('exit', function(code) { - if (!exited) { - exited = true; - if (code !== 0) { - var error = new Error('bitcoind spawned process exited with status code: ' + code); - error.code = code; - return callback(error); - } else { - return callback(); - } - } - }); - this.spawn.process.kill('SIGINT'); - setTimeout(function() { - if (!exited) { - exited = true; - return callback(new Error('bitcoind process did not exit')); - } - }, this.shutdownTimeout).unref(); - } else { - callback(); - } + callback(); }; module.exports = Bitcoin; diff --git a/lib/services/db/index.js b/lib/services/db/index.js index bf5d1afd..1b866e6a 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -12,18 +12,12 @@ var Networks = bitcore.Networks; var Block = bitcore.Block; var $ = bitcore.util.preconditions; var index = require('../../'); -var errors = index.errors; var log = index.log; -var Transaction = require('../../transaction'); var Service = require('../../service'); var Sync = require('./sync'); var Reorg = require('./reorg'); -/** - * This service synchronizes a leveldb database with bitcoin block chain by connecting and - * disconnecting blocks to build new indexes that can be queried. Other services can extend - * the data that is indexed by implementing a `blockHandler` method. - * + /* * @param {Object} options * @param {Node} options.node - A reference to the node * @param {Node} options.store - A levelup backend store @@ -40,22 +34,21 @@ function DB(options) { Service.call(this, options); - // Used to keep track of the version of the indexes - // to determine during an upgrade if a reindex is required this.version = 2; this.dbPrefix = '\u0000\u0000'; this.tip = null; this.genesis = null; + this.dbOptions = { + keyEncoding: 'string', + valueEncoding: 'binary' + }; $.checkState(this.node.network, 'Node is expected to have a "network" property'); this.network = this.node.network; this._setDataPath(); - this.maxOpenFiles = options.maxOpenFiles || DB.DEFAULT_MAX_OPEN_FILES; - this.maxTransactionLimit = options.maxTransactionLimit || DB.MAX_TRANSACTION_LIMIT; - this.levelupStore = leveldown; if (options.store) { this.levelupStore = options.store; @@ -75,22 +68,6 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; -// keys -// 0version -// 0prefix-service -// 0tip - -// The maximum number of transactions to query at once -// Used for populating previous inputs -DB.MAX_TRANSACTION_LIMIT = 5; - -// The default maxiumum number of files open for leveldb -DB.DEFAULT_MAX_OPEN_FILES = 200; - -/** - * This function will set `this.dataPath` based on `this.node.network`. - * @private - */ DB.prototype._setDataPath = function() { $.checkState(this.node.datadir, 'Node is expected to have a "datadir" property'); if (this.node.network === Networks.livenet) { @@ -108,22 +85,16 @@ DB.prototype._setDataPath = function() { DB.prototype._checkVersion = function(callback) { var self = this; - var options = { - keyEncoding: 'string', - valueEncoding: 'binary' - }; - self.store.get(self.dbPrefix + 'tip', options, function(err) { + + self.store.get(self.dbPrefix + 'tip', self.dbOptions, function(err) { if (err instanceof levelup.errors.NotFoundError) { - // The database is brand new and doesn't have a tip stored - // we can skip version checking return callback(); } else if (err) { return callback(err); } - self.store.get(self.dbPrefix + 'version', options, function(err, buffer) { + self.store.get(self.dbPrefix + 'version', self.dbOptions, function(err, buffer) { var version; if (err instanceof levelup.errors.NotFoundError) { - // The initial version (1) of the database didn't store the version number version = 1; } else if (err) { return callback(err); @@ -150,20 +121,14 @@ DB.prototype._setVersion = function(callback) { this.store.put(this.dbPrefix + 'version', versionBuffer, callback); }; -/** - * Called by Node to start the service. - * @param {Function} callback - */ DB.prototype.start = function(callback) { - var self = this; if (!fs.existsSync(this.dataPath)) { mkdirp.sync(this.dataPath); } this.genesis = Block.fromBuffer(this.node.services.bitcoind.genesisBuffer); - this.store = levelup(this.dataPath, { db: this.levelupStore, maxOpenFiles: this.maxOpenFiles, keyEncoding: 'binary', valueEncoding: 'binary'}); - this.node.services.bitcoind.on('tx', this.transactionHandler.bind(this)); + this.store = levelup(this.dataPath, { db: this.levelupStore, keyEncoding: 'binary', valueEncoding: 'binary'}); this._sync.on('error', function(err) { log.error(err); @@ -234,14 +199,8 @@ DB.prototype.start = function(callback) { }); }; -/** - * Called by Node to stop the service - * @param {Function} callback - */ DB.prototype.stop = function(callback) { var self = this; - - // Wait until syncing stops and all db operations are completed before closing leveldb async.whilst(function() { return self.bitcoindSyncing; }, function(next) { @@ -251,66 +210,18 @@ DB.prototype.stop = function(callback) { }); }; -/** - * Will give information about the database from bitcoin. - * @param {Function} callback - */ -DB.prototype.getInfo = function(callback) { - var self = this; - setImmediate(function() { - var info = self.node.bitcoind.getInfo(); - callback(null, info); - }); -}; - -/** - * Closes the underlying store database - * @param {Function} callback - */ DB.prototype.close = function(callback) { this.store.close(callback); }; -/** - * This function is responsible for emitting `db/transaction` events. - * @param {Object} txInfo - The data from the bitcoind.on('tx') event - * @param {Buffer} txInfo.buffer - The transaction buffer - * @param {Boolean} txInfo.mempool - If the transaction was accepted in the mempool - * @param {String} txInfo.hash - The hash of the transaction - */ -DB.prototype.transactionHandler = function(tx) { - // for (var i = 0; i < this.subscriptions.transaction.length; i++) { - // this.subscriptions.transaction[i].emit('db/transaction', { - // rejected: !txInfo.mempool, - // tx: tx - // }); - // } -}; - -/** - * Called by Node to determine the available API methods. - */ DB.prototype.getAPIMethods = function() { - var methods = [ - ['getBlock', this, this.getBlock, 1], - ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], - ['getTransaction', this, this.getTransaction, 2], - ['getTransactionWithBlockInfo', this, this.getTransactionWithBlockInfo, 2], - ['sendTransaction', this, this.sendTransaction, 1], - ['estimateFee', this, this.estimateFee, 1] - ]; - return methods; + return []; }; DB.prototype.loadTip = function(callback) { var self = this; - var options = { - keyEncoding: 'string', - valueEncoding: 'binary' - }; - - self.store.get(self.dbPrefix + 'tip', options, function(err, tipData) { + self.store.get(self.dbPrefix + 'tip', self.dbOptions, function(err, tipData) { if(err && err instanceof levelup.errors.NotFoundError) { self.tip = self.genesis; self.tip.__height = 0; @@ -332,7 +243,7 @@ DB.prototype.loadTip = function(callback) { var times = 0; async.retry({times: 3, interval: self.retryInterval}, function(done) { - self.getBlock(hash, function(err, tip) { + self.node.services.bitcoind.getBlock(hash, function(err, tip) { if(err) { times++; log.warn('Bitcoind does not have our tip (' + hash + '). Bitcoind may have crashed and needs to catch up.'); @@ -362,12 +273,7 @@ DB.prototype.loadTip = function(callback) { DB.prototype.loadConcurrentTip = function(callback) { var self = this; - var options = { - keyEncoding: 'string', - valueEncoding: 'binary' - }; - - self.store.get(self.dbPrefix + 'concurrentTip', options, function(err, tipData) { + self.store.get(self.dbPrefix + 'concurrentTip', self.dbOptions, function(err, tipData) { if(err && err instanceof levelup.errors.NotFoundError) { self.concurrentTip = self.genesis; self.concurrentTip.__height = 0; @@ -381,10 +287,11 @@ DB.prototype.loadConcurrentTip = function(callback) { var times = 0; async.retry({times: 3, interval: self.retryInterval}, function(done) { - self.getBlock(hash, function(err, concurrentTip) { + self.node.services.bitcoind.getBlock(hash, function(err, concurrentTip) { if(err) { times++; - log.warn('Bitcoind does not have our concurrentTip (' + hash + '). Bitcoind may have crashed and needs to catch up.'); + log.warn('Bitcoind does not have our concurrentTip (' + hash + ').' + + ' Bitcoind may have crashed and needs to catch up.'); if(times < 3) { log.warn('Retrying in ' + (self.retryInterval / 1000) + ' seconds.'); } @@ -408,91 +315,6 @@ DB.prototype.loadConcurrentTip = function(callback) { }); }; -/** - * Will get a block from bitcoind and give a Bitcore Block - * @param {String|Number} hash - A block hash or block height - */ -DB.prototype.getBlock = function(hash, callback) { - this.node.services.bitcoind.getBlock(hash, callback); -}; - -/** - * Will give a Bitcore Transaction from bitcoind by txid - * @param {String} txid - A transaction hash - * @param {Boolean} queryMempool - Include the mempool - * @param {Function} callback - */ -DB.prototype.getTransaction = function(txid, queryMempool, callback) { - this.node.services.bitcoind.getTransaction(txid, queryMempool, function(err, txBuffer) { - if (err) { - return callback(err); - } - if (!txBuffer) { - return callback(new errors.Transaction.NotFound()); - } - - callback(null, Transaction().fromBuffer(txBuffer)); - }); -}; - -/** - * Will give a Bitcore Transaction and populated information about the block included. - * @param {String} txid - A transaction hash - * @param {Boolean} queryMempool - Include the mempool - * @param {Function} callback - */ -DB.prototype.getTransactionWithBlockInfo = function(txid, queryMempool, callback) { - this.node.services.bitcoind.getTransactionWithBlockInfo(txid, queryMempool, function(err, obj) { - if (err) { - return callback(err); - } - - var tx = Transaction().fromBuffer(obj.buffer); - tx.__blockHash = obj.blockHash; - tx.__height = obj.height; - tx.__timestamp = obj.timestamp; - - callback(null, tx); - }); -}; - -/** - * Will send a transaction to the Bitcoin network. - * @param {Transaction} tx - An instance of a Bitcore Transaction - * @param {Function} callback - */ -DB.prototype.sendTransaction = function(tx, callback) { - var txString; - if (tx instanceof Transaction) { - txString = tx.serialize(); - } else { - txString = tx; - } - - try { - var txid = this.node.services.bitcoind.sendTransaction(txString); - return callback(null, txid); - } catch(err) { - return callback(err); - } -}; - -/** - * Will estimate fees for a transaction and give a result in - * satoshis per kilobyte. Similar to the bitcoind estimateFee method. - * @param {Number} blocks - The number of blocks for the transaction to be included. - * @param {Function} callback - */ -DB.prototype.estimateFee = function(blocks, callback) { - var self = this; - setImmediate(function() { - callback(null, self.node.services.bitcoind.estimateFee(blocks)); - }); -}; - -/** - * Called by the Bus to determine the available events. - */ DB.prototype.getPublishEvents = function() { return [ { @@ -521,27 +343,6 @@ DB.prototype.unsubscribe = function(name, emitter) { } }; -/** - * Will give the previous hash for a block. - * @param {String} blockHash - * @param {Function} callback - */ -DB.prototype.getPrevHash = function(blockHash, callback) { - var blockIndex = this.node.services.bitcoind.getBlockIndex(blockHash); - setImmediate(function() { - if (blockIndex) { - callback(null, blockIndex.prevHash); - } else { - callback(new Error('Could not get prevHash, block not found')); - } - }); -}; - -/** - * Connects a block to the database and add indexes - * @param {Block} block - The bitcore block - * @param {Function} callback - */ DB.prototype.connectBlock = function(block, callback) { var self = this; @@ -569,11 +370,6 @@ DB.prototype.connectBlock = function(block, callback) { }); }; -/** - * Disconnects a block from the database and removes indexes - * @param {Block} block - The bitcore block - * @param {Function} callback - */ DB.prototype.disconnectBlock = function(block, callback) { var self = this; @@ -602,7 +398,6 @@ DB.prototype.disconnectBlock = function(block, callback) { }; DB.prototype.getConcurrentBlockOperations = function(block, add, callback) { - var self = this; var operations = []; async.each( @@ -637,7 +432,6 @@ DB.prototype.getConcurrentBlockOperations = function(block, add, callback) { }; DB.prototype.getSerialBlockOperations = function(block, add, callback) { - var self = this; var operations = []; async.eachSeries( @@ -693,7 +487,6 @@ DB.prototype.getTipOperation = function(block, add) { DB.prototype.getConcurrentTipOperation = function(block, add) { var heightBuffer = new Buffer(4); var tipData; - if(add) { heightBuffer.writeUInt32BE(block.__height); tipData = Buffer.concat([new Buffer(block.hash, 'hex'), heightBuffer]); @@ -709,8 +502,6 @@ DB.prototype.getConcurrentTipOperation = function(block, add) { }; }; - - DB.prototype.getPrefix = function(service, callback) { var self = this; @@ -722,8 +513,6 @@ DB.prototype.getPrefix = function(service, callback) { } return next(err); } - - // we already have the prefix, call the callback return callback(null, buffer); }); } diff --git a/lib/services/db/sync.js b/lib/services/db/sync.js index 77d00037..a8a31f98 100644 --- a/lib/services/db/sync.js +++ b/lib/services/db/sync.js @@ -63,7 +63,7 @@ function Sync(node, db) { this.node = node; this.db = db; this.syncing = false; - this.highWaterMark = 10; + this.highWaterMark = 100; this.progressBar = null; this.lastReportedBlock = 0; } diff --git a/lib/services/wallet-api/encoding.js b/lib/services/wallet-api/encoding.js index a633eba7..58cc6a21 100644 --- a/lib/services/wallet-api/encoding.js +++ b/lib/services/wallet-api/encoding.js @@ -78,7 +78,7 @@ Encoding.prototype.encodeWalletUtxoKey = function(walletId, txid, outputIndex) { buffers.push(walletIdSizeBuffer); buffers.push(walletIdBuffer); - var txidBuffer = new Buffer(txid || new Array(33).join('0'), 'hex'); + var txidBuffer = new Buffer(txid || new Array(65).join('0'), 'hex'); buffers.push(txidBuffer); var outputIndexBuffer = new Buffer(4); @@ -137,7 +137,7 @@ Encoding.prototype.encodeWalletUtxoSatoshisKey = function(walletId, satoshis, tx satoshisBuffer.writeDoubleBE(satoshis || 0); buffers.push(satoshisBuffer); - var txidBuffer = new Buffer(txid || new Array(33).join('0'), 'hex'); + var txidBuffer = new Buffer(txid || new Array(65).join('0'), 'hex'); buffers.push(txidBuffer); var outputIndexBuffer = new Buffer(4); diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index 526c0d85..fc74efcb 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -35,7 +35,8 @@ inherits(WalletService, BaseService); WalletService.dependencies = [ 'bitcoind', 'web', - 'address' + 'address', + 'transaction' ]; WalletService.prototype.getAPIMethods = function() { @@ -123,7 +124,7 @@ WalletService.prototype.blockHandler = function(block, connectBlock, callback) { operations.push({ type: action, key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, output.satoshis, tx.id, outputIndex), - value: self._encoding.encodeWalletUtxoValue(block.__height, output._scriptBuffer) + value: self._encoding.encodeWalletUtxoSatoshisValue(block.__height, output._scriptBuffer) }); if(connectBlock) { @@ -265,7 +266,7 @@ WalletService.prototype.concurrentBlockHandler = function(block, connectBlock, c var walletIds = self._addressMap[address]; for(var j = 0; j < walletIds.length; j++) { - var walletId = walletIds[i]; + var walletId = walletIds[j]; operations.push({ type: action, key: self._encoding.encodeWalletTransactionKey(walletId, block.__height), @@ -383,16 +384,12 @@ WalletService.prototype._endpointUTXOs = function() { return function(req, res) { req.setTimeout(600000); var walletId = req.params.walletId; - var queryMempool = req.query.queryMempool === false ? false : true; - //var tip = self.node.bitcoind.tip; - // TODO: get the height of the tip - //var height = tip; + var queryMempool = req.query.queryMempool !== false; var height = null; - var options = { queryMempool: queryMempool }; - self._getUtxos(walletId, function(err, utxos) { + self._getUtxos(walletId, options, function(err, utxos) { if(err) { return utils.sendError(err, res); } @@ -409,12 +406,9 @@ WalletService.prototype._endpointGetBalance= function() { return function(req, res) { req.setTimeout(600000); var walletId = req.params.walletId; - var queryMempool = req.query.queryMempool === false ? false : true; + var queryMempool = req.query.queryMempool !== false; var byAddress = req.query.byAddress; - //var tip = self.node.bitcoind.tip; - // TODO: get the height of the tip - //var height = tip; var height = null; var options = { @@ -422,7 +416,7 @@ WalletService.prototype._endpointGetBalance= function() { byAddress: byAddress }; - self._getBalance(walletId, function(err, result) { + self._getBalance(walletId, options, function(err, result) { if(err) { return utils.sendError(err, res); } @@ -505,7 +499,6 @@ WalletService.prototype._endpointDumpAllWallets = function() { }; }; - WalletService.prototype._endpointGetWalletIds = function() { var self = this; return function(req, res) { @@ -617,7 +610,6 @@ WalletService.prototype._endpointPutAddresses = function() { var addAddresses = _.without(newAddresses, oldAddresses); var amountAdded = addAddresses.length; - //TODO this may take too long self._importAddresses(walletId, addAddresses, function(err) { if(err) { return utils.sendError(err, res); @@ -632,12 +624,12 @@ WalletService.prototype._endpointPutAddresses = function() { }; }; -WalletService.prototype._getUtxos = function(walletId, callback) { +WalletService.prototype._getUtxos = function(walletId, options, callback) { var self = this; var stream = self.store.createReadStream({ gte: self._encoding.encodeWalletUtxoKey(walletId), - lt: self._encoding.encodeWalletUtxoKey(walletId, Array(33).join('f')) // come up with better terminal key + lt: self._encoding.encodeWalletUtxoKey(utils.getTerminalKey(new Buffer(walletId))) }); var utxos = []; @@ -665,7 +657,7 @@ WalletService.prototype._getUtxos = function(walletId, callback) { }); }; -WalletService.prototype._getBalance = function(walletId, callback) { +WalletService.prototype._getBalance = function(walletId, options, callback) { var self = this; var key = self._encoding.encodeWalletBalanceKey(walletId); @@ -854,10 +846,6 @@ WalletService.prototype._importAddresses = function(walletId, addresses, callbac return callback(err); } - // TODO check if height has changed since we first entered the function - // if it has, we need to get operations for the new blocks - - // Update addressMap and wallet balances self._loadAllAddresses(function(err) { if(err) { return callback(err); @@ -874,8 +862,6 @@ WalletService.prototype._importAddresses = function(walletId, addresses, callbac WalletService.prototype._getUTXOIndexOperations = function(walletId, addresses, callback) { var self = this; - // TODO what if initialBalance changes while we are getting unspent outputs on new addresses? - var balance = 0; self._getBalance(walletId, function(err, initialBalance) { @@ -887,7 +873,7 @@ WalletService.prototype._getUTXOIndexOperations = function(walletId, addresses, balance = initialBalance; } - self.node.services.address.getUnspentOutputs(addresses, false, function(err, utxos) { + self.node.services.address.getUtxos(addresses, false, function(err, utxos) { if(err) { return callback(err); } diff --git a/lib/utils.js b/lib/utils.js index 03201764..9512f2ee 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -39,38 +39,6 @@ utils.parseParamsWithJSON = function parseParamsWithJSON(paramsArg) { return params; }; -/* -* input: string representing a number + multiple of bytes, e.g. 500MB, 200KB, 100B -* output: integer representing the byte count -*/ -utils.parseByteCount = function(byteCountString) { - - function finish(n, m) { - var num = parseInt(n); - if (num > 0) { - return num * m; - } - return null; - } - - if (!_.isString(byteCountString)) { - return byteCountString; - } - var str = byteCountString.replace(/\s+/g, ''); - var map = { 'MB': 1E6, 'kB': 1000, 'KB': 1000, 'MiB': (1024 * 1024), - 'KiB': 1024, 'GiB': Math.pow(1024, 3), 'GB': 1E9 }; - var keys = Object.keys(map); - for(var i = 0; i < keys.length; i++) { - var re = new RegExp(keys[i] + '$'); - var match = str.match(re); - if (match) { - var num = str.slice(0, match.index); - return finish(num, map[keys[i]]); - } - } - return finish(byteCountString, 1); -}; - /* * input: arguments passed into originating function (whoever called us) * output: bool args are valid for encoding a key to the database @@ -97,6 +65,12 @@ utils.hasRequiredArgsForEncoding = function(args) { return true; }; +utils.getTerminalKey = function(startKey) { + var endKey = Buffer.from(startKey); + endKey.writeUInt8(startKey.readUInt8(startKey.length - 1) + 1, startKey.length - 1); + return endKey; +}; + utils.diffTime = function(time) { var diff = process.hrtime(time); return (diff[0] * 1E9 + diff[1])/(1E9 * 1.0); diff --git a/scripts/download b/scripts/download index 64cb6180..0e496608 100755 --- a/scripts/download +++ b/scripts/download @@ -1,5 +1,5 @@ #!/bin/bash - +exit 0 set -e root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." diff --git a/test/services/address/encoding.unit.js b/test/services/address/encoding.unit.js index d9fd206c..42a8dc2a 100644 --- a/test/services/address/encoding.unit.js +++ b/test/services/address/encoding.unit.js @@ -1,7 +1,7 @@ 'use strict'; var bitcore = require('bitcore-lib'); - +var should = require('chai').should(); var Encoding = require('../../../lib/services/address/encoding'); describe('Address service encoding', function() { diff --git a/test/services/timestamp/timestamp.unit.js b/test/services/timestamp/timestamp.unit.js index 2bfb837d..e7256dc9 100644 --- a/test/services/timestamp/timestamp.unit.js +++ b/test/services/timestamp/timestamp.unit.js @@ -1,4 +1,5 @@ 'use strict'; +var should = require('chai').should(); var Encoding = require('../../../lib/services/timestamp/encoding'); diff --git a/test/services/transaction/encoding.unit.js b/test/services/transaction/encoding.unit.js index 600799e4..7a450ec2 100644 --- a/test/services/transaction/encoding.unit.js +++ b/test/services/transaction/encoding.unit.js @@ -1,7 +1,6 @@ 'use strict'; var should = require('chai').should(); -var sinon = require('sinon'); var bitcore = require('bitcore-lib'); var Encoding = require('../../../lib/services/transaction/encoding'); diff --git a/test/services/wallet-api/encoding.unit.js b/test/services/wallet-api/encoding.unit.js index 08d32025..6a8f1367 100644 --- a/test/services/wallet-api/encoding.unit.js +++ b/test/services/wallet-api/encoding.unit.js @@ -1,5 +1,6 @@ 'use strict'; +var should = require('chai').should(); var bitcore = require('bitcore-lib'); var Encoding = require('../../../lib/services/wallet-api/encoding');