From 98ea0524059f2a2d8d3b08a0bf863b3910e9c8fa Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 14 Jul 2017 16:14:24 -0400 Subject: [PATCH] wip --- lib/services/address/index.js | 328 +++---- lib/services/block/index.js | 109 +-- lib/services/mempool/index.js | 7 +- lib/services/timestamp/index.js | 140 ++- lib/services/transaction/index.js | 49 +- lib/services/wallet-api/encoding.js | 246 ------ lib/services/wallet-api/index.js | 1157 ------------------------- lib/services/wallet-api/utils.js | 765 ---------------- lib/services/wallet-api/validators.js | 225 ----- test_bcoin.js | 21 - 10 files changed, 362 insertions(+), 2685 deletions(-) delete mode 100644 lib/services/wallet-api/encoding.js delete mode 100644 lib/services/wallet-api/index.js delete mode 100644 lib/services/wallet-api/utils.js delete mode 100644 lib/services/wallet-api/validators.js delete mode 100644 test_bcoin.js diff --git a/lib/services/address/index.js b/lib/services/address/index.js index c6a553f4..58094249 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -1,6 +1,5 @@ 'use strict'; -var assert = require('assert'); var BaseService = require('../../service'); var inherits = require('util').inherits; var async = require('async'); @@ -24,12 +23,92 @@ inherits(AddressService, BaseService); AddressService.dependencies = [ 'bitcoind', 'db', + 'block', 'transaction' ]; +// ---- public function prototypes +AddressService.prototype.getBalance = function(address, queryMempool, callback) { + this.getUtxos(address, queryMempool, function(err, outputs) { + if(err) { + return callback(err); + } + + var satoshis = outputs.map(function(output) { + return output.satoshis; + }); + + var sum = satoshis.reduce(function(a, b) { + return a + b; + }, 0); + + return callback(null, sum); + }); +}; + + +AddressService.prototype.getUtxos = function(addresses, queryMempool, callback) { + var self = this; + + if(!Array.isArray(addresses)) { + addresses = [addresses]; + } + + var utxos = []; + + async.eachSeries(addresses, function(address, next) { + self.getUtxosForAddress(address, queryMempool, function(err, unspents) { + if(err && err instanceof errors.NoOutputs) { + return next(); + } else if(err) { + return next(err); + } + + utxos = utxos.concat(unspents); + next(); + }); + }, function(err) { + callback(err, utxos); + }); +}; + +AddressService.prototype.getUtxosForAddress = function(address, queryMempool, callback) { + + var self = this; + + var stream = self.db.createReadStream({ + gte: self._encoding.encodeUtxoIndexKey(address), + lt: self._encoding.encodeUtxoIndexKey(utils.getTerminalKey(new Buffer(address))) + }); + + var utxos = []; + stream.on('data', function(data) { + var key = self._encoding.decodeUtxoIndexKey(data.key); + var value = self._encoding.decodeUtxoIndexValue(data.value); + utxos.push({ + address: key.address, + txid: key.txid, + outputIndex: key.outputIndex, + satoshis: value.satoshis, + height: value.height, + script: value.script + }); + }); + + stream.on('end', function() { + return callback(null, utxos); + }); + stream.on('error', function(err) { + if(err) { + return callback(err); + } + }); +}; + AddressService.prototype.start = function(callback) { var self = this; + self._setListeners(); this.db = this.node.services.db; this.db.getPrefix(this.name, function(err, prefix) { @@ -60,23 +139,26 @@ AddressService.prototype.getAPIMethods = function() { AddressService.prototype.getAddressBalance = function(addresses, options, callback) { var self = this; - var addresses = self._normalizeAddressArg(addressArg); - var cacheKey = addresses.join(''); - var balance = self.balanceCache.get(cacheKey); + addresses = utils.normalizeAddressArg(addresses); + var balance = 0; + + async.eachLimit(addresses, 4, function(address, next) { + + var start = self._encoding.encodeUtxoIndexKey(address); + var criteria = { + gte: start, + lte: Buffer.concat([ start.slice(-36), new Buffer(new Array(73).join('f'), 'hex') ]) + }; + + var stream = this._db.createReadStream(criteria); + stream.on('data', function(data) { - if (balance) { - return setImmediate(function() { - callback(null, balance); }); - } else { - this.client.getAddressBalance({addresses: addresses}, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - self.balanceCache.set(cacheKey, response.result); - callback(null, response.result); + stream.on('error', function(err) { }); - } + stream.on('end', function() { + }); + }); }; @@ -271,6 +353,87 @@ AddressService.prototype.getAddressUnspentOutputs = function(address, options, c }; AddressService.prototype.syncPercentage = function(callback) { + return callback(null, ((this._tip.height / this._block.getBestBlockHeight()) * 100).toFixed(2) + '%'); +}; + + +AddressService.prototype.getAddressTxidsWithHeights = function(address, options, callback) { + var self = this; + + var opts = options || {}; + var txids = {}; + + var start = self._encoding.encodeAddressIndexKey(address, opts.start || 0); + var end = Buffer.concat([ start.slice(0, -36), new Buffer((opts.end || 'ffffffff'), 'hex') ]); + + var stream = self.db.createKeyStream({ + gte: start, + lt: end + }); + + var streamErr = null; + + stream.on('data', function(buffer) { + var key = self._encoding.decodeAddressIndexKey(buffer); + txids[key.txid] = key.height; + }); + + stream.on('end', function() { + callback(streamErr, txids); + }); + + stream.on('error', function(err) { + streamErr = err; + }); +}; + +// ---- private function prototypes +AddressService.prototype._setListeners = function() { + + var self = this; + + self._db.on('error', self._onDbError.bind(self)); + self.on('reorg', self._handleReorg.bind(self)); + +}; + +AddressService.prototype._startSubscriptions = function() { + + if (this._subscribed) { + return; + } + + this._subscribed = true; + if (!this._bus) { + this._bus = this.node.openBus({remoteAddress: 'localhost'}); + } + + this._bus.on('block/block', this._onBlock.bind(this)); + this._bus.subscribe('block/block'); +}; + +AddressService.prototype._onBlock = function(block) { + var self = this; + + var operations = []; + + block.transactions.forEach(function(tx) { + operations.concat(self._processTransaction(tx, { block: block, connect: connect })); + }); + + if (operations && operations.length > 0) { + + self._db.batch(operations, function(err) { + + if(err) { + log.error('Address Service: Error saving block with hash: ' + block.hash); + this._db.emit('error', err); + return; + } + + log.debug('Address Service: Success saving block hash ' + block.hash); + }); + } }; AddressService.prototype._processInput = function(opts, input) { @@ -290,7 +453,6 @@ AddressService.prototype._processInput = function(opts, input) { }]; // prev utxo - // TODO: ensure this is a good link backward var rec = { type: opts.action, key: this._encoding.encodeUtxoIndexKey(address, input.prevTxId.toString('hex'), input.outputIndex) @@ -362,140 +524,6 @@ AddressService.prototype._processTransaction = function(opts, tx) { }; -AddressService.prototype.onBlock = function(block, connect) { - - var self = this; - - var operations = []; - - block.transactions.forEach(function(tx) { - operations.concat(self._processTransaction(tx, { block: block, connect: connect })); - }); - - if (operations && operations.length > 0) { - - self._db.batch(operations, function(err) { - - if(err) { - log.error('Address Service: Error saving block with hash: ' + block.hash); - this._db.emit('error', err); - return; - } - - log.debug('Address Service: Success saving block hash ' + block.hash); - }); - } - -}; - -AddressService.prototype.getBalance = function(address, queryMempool, callback) { - this.getUtxos(address, queryMempool, function(err, outputs) { - if(err) { - return callback(err); - } - - var satoshis = outputs.map(function(output) { - return output.satoshis; - }); - - var sum = satoshis.reduce(function(a, b) { - return a + b; - }, 0); - - return callback(null, sum); - }); -}; - - -AddressService.prototype.getUtxos = function(addresses, queryMempool, callback) { - var self = this; - - if(!Array.isArray(addresses)) { - addresses = [addresses]; - } - - var utxos = []; - - async.eachSeries(addresses, function(address, next) { - self.getUtxosForAddress(address, queryMempool, function(err, unspents) { - if(err && err instanceof errors.NoOutputs) { - return next(); - } else if(err) { - return next(err); - } - - utxos = utxos.concat(unspents); - next(); - }); - }, function(err) { - callback(err, utxos); - }); -}; - -AddressService.prototype.getUtxosForAddress = function(address, queryMempool, callback) { - - var self = this; - - var stream = self.db.createReadStream({ - gte: self._encoding.encodeUtxoIndexKey(address), - lt: self._encoding.encodeUtxoIndexKey(utils.getTerminalKey(new Buffer(address))) - }); - - var utxos = []; - stream.on('data', function(data) { - var key = self._encoding.decodeUtxoIndexKey(data.key); - var value = self._encoding.decodeUtxoIndexValue(data.value); - utxos.push({ - address: key.address, - txid: key.txid, - outputIndex: key.outputIndex, - satoshis: value.satoshis, - height: value.height, - script: value.script - }); - }); - - stream.on('end', function() { - return callback(null, utxos); - }); - stream.on('error', function(err) { - if(err) { - return callback(err); - } - }); -}; - -AddressService.prototype.getAddressTxidsWithHeights = function(address, options, callback) { - var self = this; - - var opts = options || {}; - var txids = {}; - - var start = self._encoding.encodeAddressIndexKey(address, opts.start || 0); //the start and end must be the same length - var end = Buffer.concat([ start.slice(0, -36), new Buffer((opts.end || 'ffffffff'), 'hex') ]); - - var stream = self.db.createKeyStream({ - gte: start, - lt: end - }); - - var streamErr = null; - - stream.on('data', function(buffer) { - var key = self._encoding.decodeAddressIndexKey(buffer); - assert(key.txid.length === 64, 'AddressService, Txid: ' + key.txid + ' with length: ' + key.txid.length + ' does not resemble a txid.'); - txids[key.txid] = key.height; - }); - - stream.on('end', function() { - callback(streamErr, txids); - }); - - stream.on('error', function(err) { - streamErr = err; - }); -}; - module.exports = AddressService; diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 02728d10..71f2bc84 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -29,7 +29,7 @@ var BlockService = function(options) { // meta is [{ chainwork: chainwork, hash: hash }] this._meta = []; - // this is the in-memory full/raw block cache + // in-memory full/raw block cache this._blockQueue = LRU({ max: 50 * (1 * 1024 * 1024), // 50 MB of blocks, length: function(n) { @@ -39,6 +39,7 @@ var BlockService = function(options) { // keep track of out-of-order blocks, this is a list of chains (which are lists themselves) // e.g. [ [ block5, block4 ], [ block8, block7 ] ]; + // TODO: persist this to disk, we can't hold too many blocks in memory this._incompleteChains = []; // list of all chain tips, including main chain and any chains that were orphaned after a reorg this._chainTips = []; @@ -61,14 +62,13 @@ BlockService.prototype.getAPIMethods = function() { ['getRawBlock', this, this.getRawBlock, 1], ['getBlockHeader', this, this.getBlockHeader, 1], ['getBlockOverview', this, this.getBlockOverview, 1], - ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], ['getBestBlockHash', this, this.getBestBlockHash, 0] ]; return methods; }; -BlockService.prototype.getBestBlockHash = function(callback) { - callback(this._meta[this._meta.length - 1].hash); +BlockService.prototype.getBestBlockHash = function() { + return this._meta[this._meta.length - 1].hash; }; BlockService.prototype.getBlock = function(hash, callback) { @@ -81,27 +81,10 @@ BlockService.prototype.getBlock = function(hash, callback) { }); }; -BlockService.prototype.getBlockHashesByTimestamp = function(high, low, options, callback) { - - var self = this; - if (_.isFunction(options)) { - callback = options; - options = {}; - } - - self.client.getBlockHashes(high, low, options, function(err, response) { - if (err) { - return callback(self._wrapRPCError(err)); - } - callback(null, response.result); - }); - -}; - BlockService.prototype._getHash = function(blockArg) { - (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) && - this._meta[blockArg] ? meta.hash : null; + return (_.isNumber(blockArg) || (blockArg.length < 40 && /^[0-9]+$/.test(blockArg))) && + this._meta[blockArg] ? this._meta[blockArg] : null; }; @@ -109,8 +92,11 @@ BlockService.prototype.getBlockHeader = function(blockArg, callback) { blockArg = this._getHash(blockArg); - // by hash - this._getBlock(meta.hash, function(err, block) { + if (!blockArg) { + return callback(); + } + + this._getBlock(blockArg, function(err, block) { if(err) { return callback(err); @@ -126,49 +112,40 @@ BlockService.prototype.getBlockHeader = function(blockArg, callback) { }; -BlockService.prototype.getBlockOverview = function(hash, callback) { - var self = this; +BlockService.prototype._getBlock = function(hash, callback) { + var block = this._blockQueue(hash); + if (block) { + return callback(null, block); + } + this._db.get(this._encoding.encodeBlockKey(hash), callback); +}; + +BlockService.prototype.getBlockOverview = function(hash, callback) { + + this._getBlock(hash, function(err, block) { - function queryBlock(err, blockhash) { if (err) { return callback(err); } - var cachedBlock = self.blockOverviewCache.get(blockhash); - if (cachedBlock) { - return setImmediate(function() { - callback(null, cachedBlock); - }); - } else { - self._tryAllClients(function(client, done) { - client.getBlock(blockhash, true, function(err, response) { - if (err) { - return done(self._wrapRPCError(err)); - } - var result = response.result; - var blockOverview = { - hash: result.hash, - version: result.version, - confirmations: result.confirmations, - height: result.height, - chainWork: result.chainwork, - prevHash: result.previousblockhash, - nextHash: result.nextblockhash, - merkleRoot: result.merkleroot, - time: result.time, - medianTime: result.mediantime, - nonce: result.nonce, - bits: result.bits, - difficulty: result.difficulty, - txids: result.tx - }; - self.blockOverviewCache.set(blockhash, blockOverview); - done(null, blockOverview); - }); - }, callback); - } - } - self._maybeGetBlockHash(blockArg, queryBlock); + var blockOverview = { + hash: block.hash, + version: block.header.version, + confirmations: null, + height: null, + chainWork: null, + prevHash: utils.reverseBufferToString(block.header.prevHash), + nextHash: null, + merkleRoot: block.header.merkleroot, + time: null, + medianTime: null, + nonce: block.header.nonce, + bits: block.header.bits, + difficulty: null, + txids: null + }; + callback(null, blockOverview); + }); }; @@ -415,7 +392,6 @@ BlockService.prototype._getIncompleteChainIndexes = function(block) { }; BlockService.prototype._handleReorg = function(block) { - this._reorging = true; log.warn('Chain reorganization detected! Our current block tip is: ' + this._tip.hash + ' the current block: ' + block.hash + '.'); @@ -431,9 +407,12 @@ BlockService.prototype._handleReorg = function(block) { } log.warn('A common ancestor block was found to at hash: ' + commonAncestor + '.'); - this._broadcast(this.subscriptions.reorg, 'block/reorg', [block, commonAncestor]); + this._broadcast(this.subscriptions.reorg, 'block/reorg', [commonAncestor, [block]]); + this._onReorg(commonAncestor, [block]); this._reorging = false; +}; +BlockService.prototype._onReorg = function(commonAncestor, newBlockList) { }; BlockService.prototype._isChainReorganizing = function(block) { diff --git a/lib/services/mempool/index.js b/lib/services/mempool/index.js index 4088efc9..f6afec4c 100644 --- a/lib/services/mempool/index.js +++ b/lib/services/mempool/index.js @@ -1,7 +1,6 @@ 'use strict'; var BaseService = require('../../service'); var util = require('util'); -var bitcore = require('bitcore-lib'); var Encoding = require('./encoding'); var index = require('../../index'); var log = index.log; @@ -49,9 +48,9 @@ MempoolService.prototype._setListeners = function() { }; MempoolService.prototype._startSubscriptions = function() { - var bus = self.node.openBus({ remoteAddress: 'localhost' }); - bus.on('transaction/transaction', this._onTransaction.bind(self)); - bus.subscribe('tranaction/transaction'); + var bus = this.node.openBus({ remoteAddress: 'localhost' }); + bus.on('p2p/transaction', this._onTransaction.bind(this)); + bus.subscribe('p2p/transaction'); }; MempoolService.prototype._onTransaction = function(tx) { diff --git a/lib/services/timestamp/index.js b/lib/services/timestamp/index.js index 1af0a022..e255030a 100644 --- a/lib/services/timestamp/index.js +++ b/lib/services/timestamp/index.js @@ -1,17 +1,15 @@ 'use strict'; var Encoding = require('./encoding'); -var BaseService = require('../../service'); + var inherits = require('util').inherits; var LRU = require('lru-cache'); var utils = require('../../../lib/utils'); function TimestampService(options) { BaseService.call(this, options); - this.currentBlock = null; - this.currentTimestamp = null; - this._createConcurrencyCache(); - this._concurrencyCache.set(new Array(65).join('0'), { valueItem: 0 }); + this._db = this.node.services.db; + this._tip = null; } inherits(TimestampService, BaseService); @@ -20,69 +18,111 @@ TimestampService.dependencies = [ 'db', 'block' ]; TimestampService.prototype.getAPIMethods = function() { return [ + ['getBlockHashesByTimestamp', this, this.getBlockHashesByTimestamp, 2], ['syncPercentage', this, this.syncPercentage, 0] ]; }; + +TimestampService.prototype.syncPercentage = function(callback) { +}; + +TimestampService.prototype.getBlockHashesByTimestamp = function(callback) { +}; + TimestampService.prototype.start = function(callback) { var self = this; + self._setListeners(); - this.db = this.node.services.db; + self._db.getPrefix(self.name, function(err, prefix) { - this.node.services.db.getPrefix(this.name, function(err, prefix) { if(err) { return callback(err); } - self.prefix = prefix; - self.encoding = new Encoding(self.prefix); - callback(); + self._prefix = prefix; + self._encoding = new Encoding(self._prefix); + + self._db.getServiceTip(self.name, function(err, tip) { + + if (err) { + return callback(err); + } + + self._tip = tip; + self._startSubscriptions(); + callback(); + + }); }); }; +TimestampService.prototype._startSubscriptions = function() { + + if (this._subscribed) { + return; + } + + this._subscribed = true; + if (!this._bus) { + this._bus = this.node.openBus({remoteAddress: 'localhost'}); + } + + this._bus.on('block/block', this._onBlock.bind(this)); + this._bus.subscribe('block/block'); +}; + +BlockService.prototype._sync = function() { + + if (--this._p2pBlockCallsNeeded > 0) { + + log.info('Blocks download progress: ' + this._numCompleted + '/' + + this._numNeeded + ' (' + (this._numCompleted/this._numNeeded*100).toFixed(2) + '%)'); + this._p2p.getBlocks({ startHash: this._latestBlockHash }); + return; + + } + +}; +TimestampService.prototype._setListeners = function() { + + var self = this; + + self._db.on('error', self._onDbError.bind(self)); + self.on('reorg', self._handleReorg.bind(self)); + +}; + +TimestampService.prototype._setTip = function(tip) { + log.debug('Timestamp Service: Setting tip to height: ' + tip.height); + log.debug('Timestamp Service: Setting tip to hash: ' + tip.hash); + this._tip = tip; + this._db.setServiceTip('block', this._tip); +}; + TimestampService.prototype.stop = function(callback) { setImmediate(callback); }; -TimestampService.prototype.concurrentBlockHandler = function(block, connectBlock, callback) { +TimestampService.prototype._onBlock = function(block) { - var self = this; - - var action = connectBlock ? 'put' : 'del'; - - var filter = function(newBlockTime, prevBlockTime) { - if (newBlockTime <= prevBlockTime) { - return prevBlockTime + 1; - } - return newBlockTime; - }; var prevHash = utils.reverseBufferToString(block.header.prevHash); - var hash = block.hash; - var queue = self._retrieveCachedItems(hash, block.header.timestamp, prevHash, filter); var operations = []; - if (queue.length === 0) { - return callback(null, queue); - } - - for(var i = 0; i < queue.length; i++) { - - var item = queue[i]; - operations = operations.concat([ - { - type: action, - key: self.encoding.encodeTimestampBlockKey(item.value), - value: self.encoding.encodeTimestampBlockValue(item.key) - }, - { - type: action, - key: self.encoding.encodeBlockTimestampKey(item.key), - value: self.encoding.encodeBlockTimestampValue(item.value) - } - ]); - } + operations = operations.concat([ + { + type: action, + key: self.encoding.encodeTimestampBlockKey(item.value), + value: self.encoding.encodeTimestampBlockValue(item.key) + }, + { + type: action, + key: self.encoding.encodeBlockTimestampKey(item.key), + value: self.encoding.encodeBlockTimestampValue(item.value) + } + ]); callback(null, operations); @@ -119,4 +159,18 @@ TimestampService.prototype._getValue = function(key, callback) { }); }; + +TimestampService.prototype._onReorg = function(commonAncestor, newBlockList) { +}; + +TimestampService.prototype.getBlockHashesByTimestamp = function(high, low, options, callback) { + + var self = this; + if (_.isFunction(options)) { + callback = options; + options = {}; + } + +}; + module.exports = TimestampService; diff --git a/lib/services/transaction/index.js b/lib/services/transaction/index.js index 45a229dd..530e7d84 100644 --- a/lib/services/transaction/index.js +++ b/lib/services/transaction/index.js @@ -17,6 +17,7 @@ inherits(TransactionService, BaseService); TransactionService.dependencies = [ 'db', + 'block', 'timestamp', 'mempool' ]; @@ -32,6 +33,9 @@ TransactionService.prototype.getAPIMethods = function() { ]; }; +TransactionService.prototype.getSpentInfo = function(txid, callback) { +}; + TransactionService.prototype.getRawTransaction = function(txid, callback) { this.getTransaction(txid, function(err, tx) { if (err) { @@ -45,14 +49,6 @@ TransactionService.prototype.getDetailedTransaction = TransactionService.prototy var self = this; - assert(txid.length === 64, 'Transaction, Txid: ' + txid + ' with length: ' + txid.length + ' does not resemble a txid.'); - - if(self.currentTransactions[txid]) { - return setImmediate(function() { - callback(null, self.currentTransactions[txid]); - }); - } - var key = self.encoding.encodeTransactionKey(txid); async.waterfall([ @@ -144,8 +140,24 @@ TransactionService.prototype.sendTransaction = function(tx, callback) { this._p2p.sendTransaction(tx, callback); }; +TransactionService.prototype._setListeners = function() { + + var self = this; + + self._db.on('error', self._onDbError.bind(self)); + self.on('reorg', self._onReorg.bind(self)); + +}; + +TransactionService.prototype._onDbError = function(error) { +}; + +TransactionService.prototype._onReorg = function(commonAncestor, newBlockList) { +}; + TransactionService.prototype.start = function(callback) { var self = this; + self._setListeners(); self._db.getPrefix(self.name, function(err, prefix) { @@ -153,7 +165,7 @@ TransactionService.prototype.start = function(callback) { return callback(err); } - self._db.getServiceTip('transaction', function(err, tip) { + self._db.getServiceTip(self.name, function(err, tip) { if (err) { return callback(err); @@ -162,12 +174,31 @@ TransactionService.prototype.start = function(callback) { self._tip = tip; self.prefix = prefix; self.encoding = new Encoding(self.prefix); + self._startSubscriptions(); callback(); }); }); }; +TransactionService.prototype._onTransaction = function(transaction) { +}; + +TransactionService.prototype._startSubscriptions = function() { + + if (this._subscribed) { + return; + } + + this._subscribed = true; + if (!this._bus) { + this._bus = this.node.openBus({remoteAddress: 'localhost'}); + } + + this._bus.on('block/block', this._onTransaction.bind(this)); + this._bus.subscribe('block/block'); +}; + TransactionService.prototype.stop = function(callback) { setImmediate(callback); }; diff --git a/lib/services/wallet-api/encoding.js b/lib/services/wallet-api/encoding.js deleted file mode 100644 index 52b62919..00000000 --- a/lib/services/wallet-api/encoding.js +++ /dev/null @@ -1,246 +0,0 @@ -'use strict'; - -var bitcore = require('bitcore-lib'); -var BufferReader = bitcore.encoding.BufferReader; - -function Encoding(servicePrefix) { - this.servicePrefix = servicePrefix; - this.subKeyMap = { - transaction: { - fn: this.encodeWalletTransactionKey, - buffer: new Buffer('00', 'hex') - }, - addresses: { - fn: this.encodeWalletAddressesKey, - buffer: new Buffer('01', 'hex') - }, - utxo: { - fn: this.encodeWalletUtxoKey, - buffer: new Buffer('02', 'hex') - }, - utxoSat: { - fn: this.encodeWalletUtxoSatoshisKey, - buffer: new Buffer('03', 'hex') - }, - balance: { - fn: this.encodeWalletBalanceKey, - buffer: new Buffer('04', 'hex') - } - }; -} - -Encoding.prototype.encodeWalletTransactionKey = function(walletId, height, txid) { - var buffers = [this.servicePrefix, this.subKeyMap.transaction.buffer]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height || 0); - buffers.push(heightBuffer); - - var txidBuffer = new Buffer((txid || new Array(65).join('0')), 'hex'); - buffers.push(txidBuffer); - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletTransactionKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(3); - - var walletSize = reader.readUInt8(); - var walletId = reader.read(walletSize).toString('utf8'); - var height = reader.readUInt32BE(); - - var txid = reader.read(32); - - return { - walletId: walletId, - height: height, - txid: txid - }; -}; - -Encoding.prototype.encodeWalletUtxoKey = function(walletId, txid, outputIndex) { - var buffers = [this.servicePrefix, this.subKeyMap.utxo.buffer]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - var txidBuffer = new Buffer(txid || new Array(65).join('0'), 'hex'); - buffers.push(txidBuffer); - - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex || 0); - buffers.push(outputIndexBuffer); - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletUtxoKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(3); - - var walletIdSize = reader.readUInt8(); - var walletId = reader.read(walletIdSize).toString('utf8'); - var txid = reader.read(32).toString('hex'); - var outputIndex = reader.readUInt32BE(); - return { - walletId: walletId, - txid: txid, - outputIndex: outputIndex - }; -}; - -Encoding.prototype.encodeWalletUtxoValue = function(height, satoshis, scriptBuffer) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis); - return Buffer.concat([heightBuffer, satoshisBuffer, scriptBuffer]); -}; - -Encoding.prototype.decodeWalletUtxoValue = function(buffer) { - var reader = new BufferReader(buffer); - var height = reader.readUInt32BE(); - var satoshis = buffer.readDoubleBE(4); - var scriptBuffer = buffer.slice(12); - return { - height: height, - satoshis: satoshis, - script: scriptBuffer - }; -}; - -Encoding.prototype.encodeWalletUtxoSatoshisKey = function(walletId, satoshis, txid, outputIndex) { - var buffers = [this.servicePrefix, this.subKeyMap.utxoSat.buffer]; - - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - - buffers.push(walletIdSizeBuffer); - buffers.push(walletIdBuffer); - - var satoshisBuffer = new Buffer(8); - satoshisBuffer.writeDoubleBE(satoshis || 0); - buffers.push(satoshisBuffer); - - var txidBuffer = new Buffer(txid || new Array(65).join('0'), 'hex'); - buffers.push(txidBuffer); - - var outputIndexBuffer = new Buffer(4); - outputIndexBuffer.writeUInt32BE(outputIndex || 0); - buffers.push(outputIndexBuffer); - - return Buffer.concat(buffers); -}; - -Encoding.prototype.decodeWalletUtxoSatoshisKey = function(buffer) { - var walletIdSize = buffer.readUInt8(3); - var walletId = buffer.slice(4, walletIdSize + 4).toString('utf8'); - var satoshis = buffer.readDoubleBE(walletIdSize + 4); - - var txid = buffer.slice(walletIdSize + 12, walletIdSize + 44).toString('hex'); - var outputIndex = buffer.readUInt32BE(walletIdSize + 44); - return { - walletId: walletId, - satoshis: satoshis, - txid: txid, - outputIndex: outputIndex - }; -}; - -Encoding.prototype.encodeWalletUtxoSatoshisValue = function(height, scriptBuffer) { - var heightBuffer = new Buffer(4); - heightBuffer.writeUInt32BE(height); - return Buffer.concat([heightBuffer, scriptBuffer]); -}; - -Encoding.prototype.decodeWalletUtxoSatoshisValue = function(buffer) { - var reader = new BufferReader(buffer); - var height = reader.readUInt32BE(); - var scriptBuffer = reader.read(buffer.length - 4); - return { - height: height, - script: scriptBuffer - }; -}; - -Encoding.prototype.encodeWalletAddressesKey = function(walletId) { - var prefix = this.subKeyMap.addresses.buffer; - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - return Buffer.concat([this.servicePrefix, prefix, walletIdSizeBuffer, walletIdBuffer]); -}; - -Encoding.prototype.decodeWalletAddressesKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(3); - var walletSize = reader.readUInt8(); - return reader.read(walletSize).toString('utf8'); -}; - -Encoding.prototype.encodeWalletAddressesValue = function(addresses) { - var bufferList = []; - var addressesLengthBuffer = new Buffer(4); - addressesLengthBuffer.writeUInt32BE(addresses.length); - bufferList.push(addressesLengthBuffer); - for(var i = 0; i < addresses.length; i++) { - var addressSizeBuffer = new Buffer(1); - addressSizeBuffer.writeUInt8(addresses[i].length); - bufferList.push(addressSizeBuffer); - bufferList.push(new Buffer(addresses[i], 'utf8')); - } - - return Buffer.concat(bufferList); -}; - -Encoding.prototype.decodeWalletAddressesValue = function(buffer) { - var reader = new BufferReader(buffer); - var addressesLength = reader.readUInt32BE(); - var addresses = []; - for(var i = 0; i < addressesLength; i++) { - var addressSize = reader.readUInt8(); - addresses.push(reader.read(addressSize).toString('utf8')); - } - return addresses; -}; - -Encoding.prototype.encodeWalletBalanceKey = function(walletId) { - var prefix = this.subKeyMap.balance.buffer; - var walletIdSizeBuffer = new Buffer(1); - walletIdSizeBuffer.writeUInt8(walletId.length); - var walletIdBuffer = new Buffer(walletId, 'utf8'); - return Buffer.concat([this.servicePrefix, prefix, walletIdSizeBuffer, walletIdBuffer]); -}; - -Encoding.prototype.decodeWalletBalanceKey = function(buffer) { - var reader = new BufferReader(buffer); - reader.read(3); - var walletSize = reader.readUInt8(); - return reader.read(walletSize).toString('utf8'); -}; - -Encoding.prototype.encodeWalletBalanceValue = function(balance) { - var balanceBuffer = new Buffer(8); - balanceBuffer.writeDoubleBE(balance); - return balanceBuffer; -}; - -Encoding.prototype.decodeWalletBalanceValue = function(buffer) { - return buffer.readDoubleBE(); -}; - -module.exports = Encoding; - diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js deleted file mode 100644 index 80bfe519..00000000 --- a/lib/services/wallet-api/index.js +++ /dev/null @@ -1,1157 +0,0 @@ -'use strict'; - -var async = require('async'); -var assert = require('assert'); -var BaseService = require('../../service'); -var inherits = require('util').inherits; -var index = require('../../'); -var log = index.log; -var multer = require('multer'); -var storage = multer.memoryStorage(); -var upload = multer({ storage: storage }); -var validators = require('./validators'); -var mainUtils = require('../../utils'); -var utils = require('./utils'); -var _ = require('lodash'); -var bodyParser = require('body-parser'); -var LRU = require('lru-cache'); -var Encoding = require('./encoding'); -var bitcore = require('bitcore-lib'); -var Input = bitcore.Transaction.Input; -var Unit = bitcore.Unit; -var Transform = require('stream').Transform; - -var WalletService = function(options) { - BaseService.call(this, options); - - this._MAX_QUEUE = 20; - this._jobs = LRU({ - max: this._MAX_QUEUE, - maxAge: 86400000 * 3 //3 days - }); - - this._addressMap = {}; - this.balances = {}; - - this.db = this.node.services.db; -}; - -inherits(WalletService, BaseService); - -WalletService.dependencies = [ - 'web', - 'address', - 'transaction', - 'timestamp' -]; - -WalletService.prototype.getAPIMethods = function() { - return []; -}; - -WalletService.prototype.start = function(callback) { - var self = this; - - - self.node.services.db.getPrefix(self.name, function(err, servicePrefix) { - - if(err) { - return callback(err); - } - - self.servicePrefix = servicePrefix; - self._encoding = new Encoding(self.servicePrefix); - - self._loadAllAddresses(function(err) { - - if(err) { - return callback(err); - } - - self.setListeners(); - self._loadAllBalances(callback); - }); - - }); -}; - -WalletService.prototype._setListeners = function() { - this._startSubscriptions(); -}; - -WalletService.prototype._startSubscriptions = function() { - - var self = this; - - if (self._subscribed) { - return; - } - - self._subscribed = true; - self.bus = self.node.openBus({remoteAddress: 'localhost'}); - - self.bus.on('block/block', self._onBlock.bind(self)); - self.bus.subscribe('block/block'); -}; - -WalletService.prototype._onBlock = function(block) { -}; - -WalletService.prototype.stop = function(callback) { - setImmediate(callback); -}; - -WalletService.prototype.getPublishEvents = function() { - return []; -}; - - -WalletService.prototype.getAddressString = function(io) { - - var address = io.script.toAddress(this.node.network); - - if(address) { - return address.toString(); - } - - try { - var pubkey = io.script.getPublicKey(); - if(pubkey) { - return pubkey.toString('hex'); - } - } catch(e) {} - -}; - -WalletService.prototype._checkAddresses = function() { - return Object.keys(this._addressMap).length > 0; -}; - - var self = this; - - self._addressMap = {}; - - var start = self._encoding.encodeWalletAddressesKey('00'); - var end = self._encoding.encodeWalletAddressesKey(Array(65).join('f')); - - var stream = self.db.createReadStream({ - gte: start, - lt: end - }); - - var streamErr = null; - - stream.on('data', function(data) { - var key = self._encoding.decodeWalletAddressesKey(data.key); - var value = self._encoding.decodeWalletAddressesValue(data.value); - value.forEach(function(address) { - if(!self._addressMap[address]) { - self._addressMap[address] = []; - } - - self._addressMap[address].push(key); - }); - }); - - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('end', function() { - callback(streamErr); - }); -}; - -WalletService.prototype._loadAllBalances = function(callback) { - var self = this; - - self._balances = {}; - - var start = self._encoding.encodeWalletBalanceKey('00'); - var end = self._encoding.encodeWalletBalanceKey(Array(65).join('f')); - - var stream = self.db.createReadStream({ - gte: start, - lt: end - }); - - var streamErr = null; - - stream.on('data', function(data) { - var walletId = self._encoding.decodeWalletBalanceKey(data.key); - var balance = self._encoding.decodeWalletBalanceValue(data.value); - - self._balances[walletId] = balance; - }); - - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('end', function() { - callback(streamErr); - }); -}; - - -WalletService.prototype._endpointUTXOs = function() { - var self = this; - return function(req, res) { - req.setTimeout(600000); - var walletId = req.params.walletId; - var queryMempool = req.query.queryMempool !== false; - var height = self.node.services.db.tip.__height; - var options = { - queryMempool: queryMempool - }; - self.db.pauseSync(function() { - self._getUtxos(walletId, options, function(err, utxos) { - if(err) { - return utils.sendError(err, res); - } - self.db.resumeSync(); - res.status(200).jsonp({ - utxos: utxos, - height: height - }); - }); - }); - }; -}; - -WalletService.prototype._endpointGetBalance= function() { - var self = this; - return function(req, res) { - req.setTimeout(600000); - var walletId = req.params.walletId; - var queryMempool = req.query.queryMempool !== false; - var byAddress = req.query.byAddress; - - var options = { - queryMempool: queryMempool, - byAddress: byAddress - }; - - self.db.pauseSync(function() { - self._getBalance(walletId, options, function(err, result) { - if(err) { - return utils.sendError(err, res); - } - self.db.resumeSync(); - res.status(200).jsonp({ - satoshis: result, - height: self.node.services.db.tip.__height, - hash: self.node.services.db.tip.hash - }); - }); - }); - }; -}; - -WalletService.prototype._endpointRemoveWallet = function() { - var self = this; - return function(req, res) { - var walletId = req.params.walletId; - - self.db.pauseSync(function() { - self._removeWallet(walletId, function(err, numRecords) { - if(err) { - return utils.sendError(err, res); - } - self.db.resumeSync(); - res.status(200).jsonp({ - walletId: walletId, - numberRemoved: numRecords - }); - }); - }); - }; -}; - -WalletService.prototype._endpointRemoveAllWallets = function() { - var self = this; - return function(req, res) { - - self.db.pauseSync(function() { - self._removeAllWallets(function(err, numRecords) { - if(err) { - return utils.sendError(err, res); - } - self.db.resumeSync(); - res.status(200).jsonp({ - numberRemoved: numRecords - }); - }); - }); - }; -}; - -WalletService.prototype._endpointGetAddresses = function() { - var self = this; - return function(req, res) { - var walletId = req.params.walletId; - - self.db.pauseSync(function() { - self._getAddresses(walletId, function(err, addresses) { - self.db.resumeSync(); - if(err) { - return utils.sendError(err, res); - } - - if(!addresses) { - return res.status(404).send('Not found'); - } - - res.status(200).jsonp({ - addresses: addresses.length - }); - }); - }); - }; -}; - -WalletService.prototype._endpointDumpAllWallets = function() { - var self = this; - return function(req, res) { - var keys = []; - - var start = new Buffer(self.servicePrefix); - var end = new Buffer.concat([start, new Buffer('ff', 'hex')]); - - var stream = self.db.createKeyStream({ - gte: start, - lt: end - }); - - var streamErr = null; - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('data', function(data) { - keys.push(data); - }); - - stream.on('end', function() { - if(streamErr) { - return utils.sendError(streamErr, res); - } - var resultsMap = keys.map(function(key) { - return key.toString('hex'); - }); - res.status(200).jsonp({ - result: resultsMap - }); - }); - }; -}; - -WalletService.prototype._endpointGetWalletIds = function() { - var self = this; - return function(req, res) { - var start = new Buffer.concat([self.servicePrefix, new Buffer(self._encoding.subKeyMap.addresses.buffer)]); - var end = new Buffer.concat([start, new Buffer('ff', 'hex')]); - var stream = self.db.createKeyStream({ - gte: start, - lt: end - }); - var walletIds = []; - - var streamErr; - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('data', function(data) { - walletIds.push(self._encoding.decodeWalletAddressesKey(data)); - }); - - stream.on('end', function() { - if(streamErr) { - return utils.sendError(streamErr, res); - } - res.status(200).jsonp({ - walletIds: walletIds - }); - }); - }; -}; - -WalletService.prototype._endpointRegisterWallet = function() { - var self = this; - return function(req, res) { - var walletId = req.params.walletId; - if (!walletId) { - walletId = utils.getWalletId(); - } - self._createWallet(walletId, function(err) { - if(err) { - return utils.sendError(err, res); - } - res.status(201).jsonp({ - walletId: walletId - }); - }); - }; -}; - -WalletService.prototype._endpointResyncAddresses = function() { - - var self = this; - - return function(req, res) { - var walletId = req.params.walletId; - - - if (!walletId) { - return utils.sendError(new Error('WalletId must be given.'), res); - } - - if (!self._isJobQueueReady()) { - return utils.sendError(new Error('Job queue is currently overloaded, please try again later.'), res); - } - - self.db.pauseSync(function() { - - self._getAddresses(walletId, function(err, oldAddresses) { - - if(err) { - return utils.sendError(err, res); - } - - if(!oldAddresses) { - return res.status(404).send('Not found'); - } - - self._removeWallet(walletId, function(err) { - - if(err) { - return utils.sendError(err, res); - } - - self._createWallet(walletId, function() { - - var jobId = utils.generateJobId(); - self._importAddresses(walletId, oldAddresses, jobId, self._jobCompletionCallback.bind(self)); - res.status(200).jsonp({jobId: jobId}); - - }); - }); - }); - }); - }; -}; - -WalletService.prototype._endpointPostAddresses = function() { - var self = this; - return function(req, res) { - - var addresses = req.addresses; - if (!addresses || !addresses.length) { - return utils.sendError(new Error('addresses are required when creating a wallet.'), res); - } - var walletId = req.params.walletId; - if (!walletId) { - return utils.sendError(new Error('WalletId must be given.'), res); - } - if (!self._isJobQueueReady()) { - return utils.sendError(new Error('Job queue is currently overloaded, please try again later.'), res); - } - - var jobId = utils.generateJobId(); - - self.db.pauseSync(function() { - - self._importAddresses(walletId, addresses, jobId, self._jobCompletionCallback.bind(self)); - res.status(200).jsonp({jobId: jobId}); - - }); - }; -}; - -WalletService.prototype._endpointGetTransactions = function() { - - var self = this; - - return function(req, res) { - - var walletId = req.params.walletId; - - self.db.pauseSync(function() { - self._processStartEndOptions(req, function(err, heights) { - - if(err) { - return utils.sendError(err, res); - } - - var options = { - start: heights[0] || 0, - end : heights[1] || 0xffffffff, - self: self, - walletId: walletId - }; - - var missingTxidCount = 0; - var transform = new Transform({ objectMode: true, highWaterMark: 1000000 }); - //txids are sent in and the actual tx's are found here - transform._transform = function(chunk, enc, callback) { - - var txid = self._encoding.decodeWalletTransactionKey(chunk).txid.toString('hex'); - - if (txid.length !== 64 || txid === '0000000000000000000000000000000000000000000000000000000000000000') { - missingTxidCount++; - log.error('missingTxidCount: ', missingTxidCount); - return callback(); - } - - self._getTransactionFromDb(options, txid, function(err, tx) { - - if(err) { - log.error(err); - transform.unpipe(); - return callback(); - } - - var formattedTx = utils.toJSONL(self._formatTransaction(tx)); - transform.push(formattedTx); - callback(); - - }); - - }; - - transform._flush = function(callback) { - self.db.resumeSync(); - callback(); - }; - - var encodingFn = self._encoding.encodeWalletTransactionKey.bind(self._encoding); - var stream = self.db.createKeyStream(self._getSearchParams(encodingFn, options)); - - stream.on('close', function() { - stream.unpipe(); - }); - - stream.pipe(transform).pipe(res); - }); - }); - }; -}; - -WalletService.prototype._formatTransactions = function(txs) { - return txs.forEach(this._formatTransaction); -}; - -WalletService.prototype._formatTransaction = function(tx) { - var obj = tx.toObject(); - - for(var i = 0; i < tx.inputs.length; i++) { - obj.inputs[i].inputSatoshis = tx.__inputValues[i]; - } - obj.height = tx.__height; - obj.timestamp = tx.__timestamp; - return obj; -}; - -WalletService.prototype._endpointPutAddresses = function() { - var self = this; - return function(req, res) { - if (!self._isJobQueueReady()) { - return utils.sendError(new Error('Job Queue is full, current job limit: ' + self._MAX_QUEUE), res); - } - - var newAddresses = req.body; - - if(!Array.isArray(req.body)) { - return utils.sendError(new Error('Must PUT an array'), res); - } - - var walletId = req.params.walletId; - if (!walletId) { - return utils.sendError(new Error('WalletId must be given.'), res); - } - - self.db.pauseSync(function() { - - self._getAddresses(walletId, function(err, oldAddresses) { - - if(err) { - return utils.sendError(err, res); - } - - if(!oldAddresses) { - return res.status(404).send('Not found'); - } - - var addAddresses = _.without(newAddresses, oldAddresses); - - var jobId = utils.generateJobId(); - self._importAddresses(walletId, addAddresses, jobId, self._jobCompletionCallback.bind(self)); - res.status(200).jsonp({jobId: jobId}); - - }); - }); - - }; -}; - -WalletService.prototype._getUtxos = function(walletId, options, callback) { - var self = this; - - var stream = self.db.createReadStream({ - gte: self._encoding.encodeWalletUtxoKey(walletId), - lt: self._encoding.encodeWalletUtxoKey(mainUtils.getTerminalKey(new Buffer(walletId))) - }); - - var utxos = []; - var streamErr = null; - - stream.on('data', function(data) { - var key = self._encoding.decodeWalletUtxoKey(data.key); - var value = self._encoding.decodeWalletUtxoValue(data.value); - utxos.push({ - txid: key.txid, - vout: key.outputIndex, - height: value.height, - satoshis: value.satoshis, - scriptPubKey: value.script.toString('hex') - }); - }); - - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('end', function() { - callback(streamErr, utxos); - }); -}; - -WalletService.prototype._getBalance = function(walletId, options, callback) { - - var self = this; - - var key = self._encoding.encodeWalletBalanceKey(walletId); - - self.db.get(key, function(err, buffer) { - - if(err) { - return callback(err); - } - - callback(null, self._encoding.decodeWalletBalanceValue(buffer)); - }); - -}; - -WalletService.prototype._getSearchParams = function(fn, options) { - return { - gte: fn.call(this, options.walletId, options.start), - lt: Buffer.concat([ fn.call(this, options.walletId, options.end).slice(0, -32), new Buffer('ff', 'hex') ]) - }; -}; - -WalletService.prototype._getTransactionFromDb = function(options, txid, callback) { - - var self = options.self; - - self.node.services.transaction.getTransaction(txid.toString('hex'), options, function(err, tx) { - - if(err) { - return callback(err); - } - - if (tx.__inputValues) { - return callback(null, tx); - } - - async.mapLimit(tx.inputs, 8, function(input, next) { - - self.node.services.transaction.getTransaction(input.prevTxId.toString('hex'), options, function(err, tx) { - - if(err) { - return next(err); - } - - next(null, tx.outputs[input.outputIndex].satoshis); - }); - - }, function(err, inputValues) { - - if(err) { - return callback(err); - } - - tx.__inputValues = inputValues; - callback(null, tx); - - }); - }); - -}; - -WalletService.prototype._removeWallet = function(walletId, callback) { - - var self = this; - async.map(Object.keys(self._encoding.subKeyMap), function(prefix, next) { - - var keys = []; - - var start = self._encoding.subKeyMap[prefix].fn.call(self._encoding, walletId); - var end = new Buffer.concat([ - self._encoding.subKeyMap[prefix] - .fn.call(self._encoding, walletId), - new Buffer('ff', 'hex')]); - - var stream = self.db.createKeyStream({ - gte: start, - lt: end - }); - - var streamErr = null; - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('data', function(data) { - keys.push(data); - }); - - stream.on('end', function() { - next(streamErr, keys); - }); - - }, function(err, results) { - if(err) { - return callback(err); - } - results = _.flatten(results); - var operations = []; - for(var i = 0; i < results.length; i++) { - operations.push({ - type: 'del', - key: results[i] - }); - } - self.db.batch(operations, function(err) { - if(err) { - return callback(err); - } - callback(null, operations.length); - }); - }); -}; - -WalletService.prototype._removeAllWallets = function(callback) { - var self = this; - var operations = []; - - var start = self._encoding.servicePrefix; - var end = new Buffer.concat([ start, new Buffer('ff', 'hex') ]); - - var stream = self.db.createKeyStream({ - gte: start, - lte: end - }); - - var streamErr = null; - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('data', function(data) { - operations.push({ type: 'del', key: data }); - }); - - stream.on('end', function() { - self.db.batch(operations, function(err) { - if(err) { - return callback(err); - } - callback(null, operations.length); - }); - }); -}; - -WalletService.prototype._getAddresses = function(walletId, callback) { - var self = this; - var key = self._encoding.encodeWalletAddressesKey(walletId); - self.db.get(key, function(err, value) { - if(err) { - return callback(err); - } - if (!value) { - return callback(null, []); - } - callback(null, self._encoding.decodeWalletAddressesValue(value)); - }); -}; - -WalletService.prototype._createWallet = function(walletId, callback) { - var self = this; - var key = self._encoding.encodeWalletAddressesKey(walletId); - self.db.get(key, function(err) { - if (err && ((/notfound/i).test(err) || err.notFound)) { - var value = self._encoding.encodeWalletAddressesValue([]); - return self.db.put(key, value, callback); - } - callback(); - }); -}; - -WalletService.prototype._isJobQueueReady = function() { - - var self = this; - - self._jobs.rforEach(function(value, key) { - if ((value.status === 'complete' || value.status === 'error') && value.reported) { - self._jobs.del(key); - } - }); - - return self._jobs.length < self._MAX_QUEUE; - -}; - -WalletService.prototype._jobCompletionCallback = function(err, results) { - - this.db.resumeSync(); - - log.info('Completed job: ', results.jobId); - - var jobId = results.jobId; - var job = this._jobs.get(jobId); - - if (!job) { - log.error('ERROR: Could not locate job id: ' + jobId + - ' in the list of jobs. It may have been purged already although it should not have.'); - return; - } - - job.progress = 1.0; - job.endtime = Date.now(); - - if (err) { - job.status = 'error'; - job.message = err.message; - return; - } - - job.status = 'complete'; - job.message = results; - job.reported = false; -}; - -WalletService.prototype._importAddresses = function(walletId, addresses, jobId, callback) { - var self = this; - - var jobResults = { jobId: jobId }; - - var job = { - starttime: Date.now(), - fn: 'importAddresses', - progress: 0, - projectedendtime: null - }; - - this._jobs.set(jobId, job); - - - self._getAddresses(walletId, function(err, oldAddresses) { - if(err) { - return callback(err, jobResults); - } - - log.info('loaded existing addresses, count: ', oldAddresses.length); - async.parallel( - [ - self._getUTXOIndexOperations.bind(self, walletId, addresses, jobId), - self._getTxidIndexOperations.bind(self, walletId, addresses, jobId) - ], - function(err, results) { - if(err) { - return callback(err, jobResults); - } - - var now = Date.now(); - job.progress = 0.50; - job.projectedendtime = now + (now - job.starttime); - - var operations = results[0].concat(results[1]); - - operations.push({ - type: 'put', - key: self._encoding.encodeWalletAddressesKey(walletId), - value: self._encoding.encodeWalletAddressesValue(oldAddresses.concat(addresses)) - }); - - self.db.batch(operations, function(err) { - if(err) { - return callback(err, jobResults); - } - - self._loadAllAddresses(function(err) { - if(err) { - return callback(err, jobResults); - } - - self._loadAllBalances(function(err) { - if(err) { - return callback(err, jobResults); - } - callback(null, jobResults); - }); - }); - }); - } - ); - }); -}; - -WalletService.prototype._getUTXOIndexOperations = function(walletId, addresses, jobId, callback) { - var self = this; - - var balance = 0; - - self._getBalance(walletId, {}, function(err, initialBalance) { - if(err && !err.notFound) { - return callback(err); - } - - if(initialBalance) { - balance = initialBalance; - } - - log.info('Initial balance of walletId: ' + walletId + ' is: ' + Unit.fromSatoshis(balance).toBTC() + ' BTC.'); - log.info('Starting to gather utxos for walletId: ' + walletId); - self.node.services.address.getUtxos(addresses, false, function(err, utxos) { - if(err) { - return callback(err); - } - - log.info('completed gathering utxos: ', utxos.length); - var operations = []; - - for(var i = 0; i < utxos.length; i++) { - var utxo = utxos[i]; - - balance += utxo.satoshis; - - operations.push({ - type: 'put', - key: self._encoding.encodeWalletUtxoKey(walletId, utxo.txid, utxo.outputIndex), - value: self._encoding.encodeWalletUtxoValue(utxo.height, utxo.satoshis, utxo.script) - }); - - operations.push({ - type: 'put', - key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, utxo.satoshis, utxo.txid, utxo.outputIndex), - value: self._encoding.encodeWalletUtxoSatoshisValue(utxo.height, utxo.script) - }); - } - - operations.push({ - type: 'put', - key: self._encoding.encodeWalletBalanceKey(walletId), - value: self._encoding.encodeWalletBalanceValue(balance) - }); - - log.info('Final balance for walletId: ' + walletId + ' is: ' + Unit.fromSatoshis(balance).toBTC() + ' BTC.'); - callback(null, operations); - }); - }); -}; - -WalletService.prototype._getTxidIndexOperations = function(walletId, addresses, jobId, callback) { - var self = this; - var txids = {}; - - var logCount = 0; - async.eachLimit(addresses, 10, function(address, next) { - self.node.services.address.getAddressTxidsWithHeights(address, null, function(err, tmpTxids) { - if(err) { - return next(err); - } - if (logCount++ % 1000 === 0) { - log.info('loaded address txids, total count: ', Object.keys(txids).length); - } - txids = _.merge(txids, tmpTxids); - return next(); - }); - }, function(err) { - if(err) { - return callback(err); - } - - var operations = Object.keys(txids).map(function(txid) { - assert(txid.length === 64, 'WalletService, Txid: ' + txid + ' with length: ' + txid.length + ' does not resemble a txid.'); - return { - type: 'put', - key: self._encoding.encodeWalletTransactionKey(walletId, txids[txid], txid) - }; - }); - - callback(null, operations); - }); -}; - -WalletService.prototype._storeAddresses = function(walletId, addresses, callback) { - var key = this._encoding.encodeWalletAddressesKey(walletId); - var value = this._encoding.encodeWalletValue(addresses); - this.db.put(key, value, callback); -}; - -WalletService.prototype._storeBalance = function(walletId, balance, callback) { - var key = this._encoding.encodeWalletBalanceKey(walletId); - var value = this._encoding.encodeWalletBalanceValue(balance); - this.db.put(key, value, callback); -}; - -WalletService.prototype._processStartEndOptions = function(req, callback) { - var self = this; - - if (req.query.start >= 0 && req.query.end >= 0) { - - var heights = []; - self.node.services.timestamp.getBlockHeights([ - utils.normalizeTimeStamp(req.query.start), - utils.normalizeTimeStamp(req.query.end) - ], - - function(err, hashTuple) { - if(err) { - return callback(err); - } - - hashTuple.forEach(function(hash) { - self.node.services.bitcoind._tryAllClients(function(client, done) { - client.getBlock(hash, function(err, response) { - if (err) { - return callback(err); - } - done(null, heights.push(response.result.height)); - }); - }, function(err) { - if(err) { - return callback(err); - } - if (heights.length > 1) { - callback(null, heights); - } - }); - }); - }); - } else { - - setImmediate(function() { - callback(null, [req.query.start, req.query.end]); - }); - - } -}; - -WalletService.prototype._endpointJobs = function() { - - var self = this; - - return function(req, res) { - - var count = 0; - self._jobs.rforEach(function(value) { - if ((value.state === 'complete' || value.state === 'error') && value.reported) { - count++; - } - }); - - res.status(200).jsonp({ jobCount: self._jobs.length - count }); - }; - -}; - -WalletService.prototype._endpointJobStatus = function() { - - var self = this; - - return function(req, res) { - var jobId = req.params.jobId; - var job = self._jobs.get(jobId); - if (!jobId || !job) { - return utils.sendError(new Error('Job not found. ' + - 'The job results may have been purged to make room for new jobs.'), res); - } - job.reported = true; - return res.status(201).jsonp(job); - }; - -}; - -WalletService.prototype._setupReadOnlyRoutes = function(app) { - var s = this; - - app.get('/wallets/:walletId/utxos', - s._endpointUTXOs() - ); - app.get('/wallets/:walletId/balance', - s._endpointGetBalance() - ); - app.get('/wallets/dump', - s._endpointDumpAllWallets() - ); - app.get('/wallets/:walletId', - s._endpointGetAddresses() - ); - app.get('/wallets/:walletId/transactions', - s._endpointGetTransactions() - ); - app.get('/wallets', - s._endpointGetWalletIds() - ); - app.get('/jobs/:jobId', - s._endpointJobStatus() - ); - app.get('/jobs', - s._endpointJobs() - ); -}; - -WalletService.prototype._setupWriteRoutes = function(app) { - var s = this; - var v = validators; - - app.post('/wallets/:walletId', - s._endpointRegisterWallet() - ); - app.delete('/wallets/:walletId', - s._endpointRemoveWallet() - ); - app.delete('/wallets/', - s._endpointRemoveAllWallets() - ); - app.put('/wallets/:walletId/addresses', - s._endpointPutAddresses() - ); - app.post('/wallets/:walletId/addresses', - upload.single('addresses'), - v.checkAddresses, - s._endpointPostAddresses() - ); - app.put('/wallets/:walletId/addresses/resync', - s._endpointResyncAddresses() - ); -}; - - -WalletService.prototype.setupRoutes = function(app) { - - app.use(bodyParser.json()); - this._setupReadOnlyRoutes(app); - this._setupWriteRoutes(app); - -}; - -WalletService.prototype.getRoutePrefix = function() { - return 'wallet-api'; -}; - -module.exports = WalletService; - diff --git a/lib/services/wallet-api/utils.js b/lib/services/wallet-api/utils.js deleted file mode 100644 index 4a0326ee..00000000 --- a/lib/services/wallet-api/utils.js +++ /dev/null @@ -1,765 +0,0 @@ -'use strict'; - -var Writable = require('stream').Writable; -var assert = require('assert'); -var crypto = require('crypto'); -var fs = require('fs'); -var inherits = require('util').inherits; -var path = require('path'); -var spawn = require('child_process').spawn; - -var BitcoinRPC = require('bitcoind-rpc'); -var _ = require('lodash'); -var async = require('async'); -var bitcore = require('bitcore-lib'); -var mkdirp = require('mkdirp'); -var ttyread = require('ttyread'); - -var exports = {}; - -exports.isInteger = function(value) { - return typeof value === 'number' && - isFinite(value) && - Math.floor(value) === value; -}; - -exports.normalizeTimeStamp = function(value) { - if (value > 0xffffffff) { - value = Math.round(value/1000); - } - return value; -}; -/** - * Will create a directory if it does not already exist. - * - * @param {String} directory - An absolute path to the directory - * @param {Function} callback - */ -exports.setupDirectory = function(directory, callback) { - fs.access(directory, function(err) { - if (err && err.code === 'ENOENT') { - return mkdirp(directory, callback); - } else if (err) { - return callback(err); - } - callback(); - }); -}; - -/** - * This will split a range of numbers "a" to "b" by sections - * of the length "max". - * - * Example: - * > var range = utils.splitRange(1, 10, 3); - * > [[1, 3], [4, 6], [7, 9], [10, 10]] - * - * @param {Number} a - The start index (lesser) - * @param {Number} b - The end index (greater) - * @param {Number} max - The maximum section length - */ -exports.splitRange = function(a, b, max) { - assert(b > a, '"b" is expected to be greater than "a"'); - var sections = []; - var delta = b - a; - var first = a; - var last = a; - - var length = Math.floor(delta / max); - for (var i = 0; i < length; i++) { - last = first + max - 1; - sections.push([first, last]); - first += max; - } - - if (last <= b) { - sections.push([first, b]); - } - - return sections; -}; - -/** - * getFileStream: Checks for the file's existence and returns a readable stream or stdin - * @param {String} path - The path to the file - * @param {Function} callback - */ -exports.getFileStream = function(filePath, callback) { - callback(null, fs.createReadStream(filePath)); -}; - -exports.readWalletDatFile = function(filePath, network, callback) { - assert(_.isString(network), 'Network expected to be a string.'); - var datadir = path.dirname(filePath).replace(/(\/testnet3|\/regtest)$/, ''); - var name = path.basename(filePath); - var options = ['-datadir=' + datadir, '-wallet=' + name]; - if (network === 'testnet') { - options.push('-testnet'); - } else if (network === 'regtest') { - options.push('-regtest'); - } - // TODO use ../node_modules/.bin/wallet-utility - var exec = path.resolve(__dirname, '../node_modules/bitcore-node/bin/bitcoin-0.12.1/bin/wallet-utility'); - var wallet = spawn(exec, options); - - var result = ''; - - wallet.stdout.on('data', function(data) { - result += data.toString('utf8'); - }); - - var error; - - wallet.stderr.on('data', function(data) { - error = data.toString(); - }); - - wallet.on('close', function(code) { - if (code === 0) { - var addresses; - try { - addresses = JSON.parse(result); - addresses = addresses.map(function(entry) { - return entry.addr ? entry.addr : entry; - }); - } catch(err) { - return callback(err); - } - return callback(null, addresses); - } else if (error) { - return callback(new Error(error)); - } else { - var message = 'wallet-utility exited (' + code + '): ' + result; - return callback(new Error(message)); - } - }); -}; - -exports.readWalletFile = function(filePath, network, callback) { - if (/\.dat$/.test(filePath)) { - exports.readWalletDatFile(filePath, network, callback); - } else { - exports.getFileStream(filePath, callback); - } -}; - -/** - * This will split an array into smaller arrays by size - * - * @param {Array} array - * @param {Number} size - The length of resulting smaller arrays - */ -exports.splitArray = function(array, size) { - var results = []; - while (array.length) { - results.push(array.splice(0, size)); - } - return results; -}; - -/** - * Utility to get the remote ip address from cloudflare headers. - * - * @param {Object} req - An express request object - */ -exports.getRemoteAddress = function(req) { - if (req.headers['cf-connecting-ip']) { - return req.headers['cf-connecting-ip']; - } - return req.socket.remoteAddress; -}; - -/** - * A middleware to enable CORS - * - * @param {Object} req - An express request object - * @param {Object} res - An express response object - * @param {Function} next - */ -exports.enableCORS = function(req, res, next) { - res.header('access-control-allow-origin', '*'); - res.header('access-control-allow-methods', 'GET, HEAD, PUT, POST, OPTIONS'); - var allowed = [ - 'origin', - 'x-requested-with', - 'content-type', - 'accept', - 'content-length', - 'cache-control', - 'cf-connecting-ip' - ]; - res.header('access-control-allow-headers', allowed.join(', ')); - - var method = req.method && req.method.toUpperCase && req.method.toUpperCase(); - - if (method === 'OPTIONS') { - res.statusCode = 204; - res.end(); - } else { - next(); - } -}; - -/** - * Will send error to express response - * - * @param {Error} err - error object - * @param {Object} res - express response object - */ -exports.sendError = function(err, res) { - if (err.statusCode) { - res.status(err.statusCode).send(err.message); - } else { - console.error(err.stack); - res.status(503).send(err.message); - } -}; - -/** - * Will create a writeable logger stream - * - * @param {Function} logger - Function to log information - * @returns {Stream} - */ -exports.createLogStream = function(logger) { - function Log(options) { - Writable.call(this, options); - } - inherits(Log, Writable); - - Log.prototype._write = function (chunk, enc, callback) { - logger(chunk.slice(0, chunk.length - 1)); // remove new line and pass to logger - callback(); - }; - var stream = new Log(); - - return stream; -}; - -exports.getWalletId = exports.generateJobId = function() { - return crypto.randomBytes(16).toString('hex'); -}; - -exports.getClients = function(clientsConfig) { - var clients = []; - for (var i = 0; i < clientsConfig.length; i++) { - var config = clientsConfig[i]; - var remoteClient = new BitcoinRPC({ - protocol: config.rpcprotocol || 'http', - host: config.rpchost || '127.0.0.1', - port: config.rpcport, - user: config.rpcuser, - pass: config.rpcpassword, - rejectUnauthorized: _.isUndefined(config.rpcstrict) ? true : config.rpcstrict - }); - clients.push(remoteClient); - } - return clients; -}; - -exports.setClients = function(obj, clients) { - obj._clients = clients; - obj._clientsIndex = 0; - Object.defineProperty(obj, 'clients', { - get: function() { - var client = obj._clients[obj._clientsIndex]; - obj._clientsIndex = (obj._clientsIndex + 1) % obj._clients.length; - return client; - }, - enumerable: true, - configurable: false - }); -}; - -exports.tryAllClients = function(obj, func, options, callback) { - if (_.isFunction(options)) { - callback = options; - options = {}; - } - var clientIndex = obj._clientsIndex; - var retry = function(done) { - var client = obj._clients[clientIndex]; - clientIndex = (clientIndex + 1) % obj._clients.length; - func(client, done); - }; - async.retry({times: obj._clients.length, interval: options.interval || 1000}, retry, callback); -}; - -exports.wrapRPCError = function(errObj) { - var err = new Error(errObj.message); - err.code = errObj.code; - return err; -}; - -var PUBKEYHASH = new Buffer('01', 'hex'); -var SCRIPTHASH = new Buffer('02', 'hex'); - -exports.getAddressTypeString = function(bufferArg) { - var buffer = bufferArg; - if (!Buffer.isBuffer(bufferArg)) { - buffer = new Buffer(bufferArg, 'hex'); - } - var type = buffer.slice(0, 1); - if (type.compare(PUBKEYHASH) === 0) { - return 'pubkeyhash'; - } else if (type.compare(SCRIPTHASH) === 0) { - return 'scripthash'; - } else { - throw new TypeError('Unknown address type'); - } -}; - -exports.getAddressTypeBuffer = function(address) { - var type; - if (address.type === 'pubkeyhash') { - type = PUBKEYHASH; - } else if (address.type === 'scripthash') { - type = SCRIPTHASH; - } else { - throw new TypeError('Unknown address type'); - } - return type; -}; - -exports.splitBuffer = function(buffer, size) { - var pos = 0; - var buffers = []; - while (pos < buffer.length) { - buffers.push(buffer.slice(pos, pos + size)); - pos += size; - } - return buffers; -}; - -exports.exitWorker = function(worker, timeout, callback) { - assert(worker, '"worker" is expected to be defined'); - var exited = false; - worker.once('exit', function(code) { - if (!exited) { - exited = true; - if (code !== 0) { - var error = new Error('Worker did not exit cleanly: ' + code); - error.code = code; - return callback(error); - } else { - return callback(); - } - } - }); - worker.kill('SIGINT'); - setTimeout(function() { - if (!exited) { - exited = true; - worker.kill('SIGKILL'); - return callback(new Error('Worker exit timeout, force shutdown')); - } - }, timeout).unref(); -}; - -exports.timestampToISOString = function(timestamp) { - return new Date(this.toIntIfNumberLike(timestamp) * 1000).toISOString(); -}; - -exports.satoshisToBitcoin = function(satoshis) { - return satoshis / 100000000; -}; - -exports.getPassphrase = function(callback) { - ttyread('Enter passphrase: ', {silent: true}, callback); -}; - -exports.acquirePassphrase = function(callback) { - var first; - var second; - async.doWhilst(function(next) { - ttyread('Enter passphrase: ', {silent: true}, function(err, result) { - if (err) { - return callback(err); - } - first = result; - ttyread('Re-enter passphrase: ', {silent: true}, function(err, result) { - second = result; - next(); - }); - }); - }, function() { - if (first !== second) { - console.log('Passphrases do not match, please re-enter.'); - return true; - } - return false; - }, function(err) { - if (err) { - return callback(err); - } - callback(null, first); - }); -}; - -/* - Important notes: - - How the encryption/decryption schemes work. - 1. The user's passphrase and salt are hashed using scrypt algorithm. You must store the salt. - On modern hardware this hashing function should take 1-2 seconds. - 2. The resulting hash is 48 bytes. The first 32 bytes of this hash is the "key" and the last - 16 bytes is the "iv" to decrypt the master key using AES256-cbc. - 3. The plaintext "master key" is always 32 bytes and should be as random as possible. - You may pass in the plaintext master key to encryptSecret -or- /dev/random will be consulted. - 4. The cipherText of the master key must be stored just like the salt. For added security, you - might store the cipherText of the master key separate from the cipherText. - For example, if an attacker discovers your passphrase and salt (the most likely scenario), they would - still require the cipherText of the master key in order to decrypt the cipherText of your private keys. - Storing your encrypted master key on another device would be a better choice than keeping your salt, - the cipherText of your master key and the cipherText of your private keys on the same computer system. - 5. The plaintext master key is then used to encrypt/decrypt the bitcoin private keys. The private keys' - corresponding public key is used as the IV for the procedure. - - - Specific notes regarding how private keys are transferred from a traditional "wallet.dat" file used with - Bitcoin Core's Wallet: - - 1. Bitcoin Core's Wallet uses Berkeley DB version 4.8 to store secp256k1 elliptic curve private keys in WIF format. - 2. The same Berkeley DB, internally called "main", also stores compressed public keys for the above private keys, - the master keys used to encrypt the above private keys and bitcoin transaction details relevant to those private keys - 3. The underlying data structure for the Berkeley database is the B-Tree (balanced tree). This is a key-value data - structure, therefore the database is a key-value database. - Berkeley DB documentation also refers to this as "key-record" - This means that the data contained in this B-Tree is organized for high speed retrieval based on a key. - In other words the database is optimized for lookups. - 4. The filename for this database file is called "wallet.dat" historically, - but you can rename it to whatever suits you - -*/ -//this function depends on the derivation method and its params that were originally used to hash the passphrase -//this could be SHA512, scrypt, etc. -exports.sha512KDF = function(passphrase, salt, derivationOptions, callback) { - if (!derivationOptions || derivationOptions.method !== 0 || !derivationOptions.rounds) { - return callback(new Error('SHA512 KDF method was called for, ' + - 'yet the derivations options for it were not supplied.')); - } - var rounds = derivationOptions.rounds || 1; - //if salt was sent in as a string, we will have to assume the default encoding type - if (!Buffer.isBuffer(salt)) { - salt = new Buffer(salt, 'utf-8'); - } - var derivation = Buffer.concat([new Buffer(''), new Buffer(passphrase), salt]); - for(var i = 0; i < rounds; i++) { - derivation = crypto.createHash('sha512').update(derivation).digest(); - } - callback(null, derivation); -}; - -exports.hashPassphrase = function() { - return exports.sha512KDF; -}; - -exports.decryptPrivateKey = function(opts, callback) { - exports.decryptSecret(opts, function(err, masterKey) { - if(err) { - return callback(err); - } - opts.cipherText = opts.pkCipherText; - //decrypt the private here using the plainText master key as the "key" - //and the double sha256 compressed pub key as the "IV" - opts.key = masterKey; - opts.iv = bitcore.crypto.Hash.sha256sha256(new Buffer(opts.pubkey, 'hex')); - exports.decrypt(opts, function(err, privateKey) { - if(err) { - return callback(err); - } - callback(null, privateKey); - }); - }); -}; - -//call decryptSecret first -exports.encryptPrivateKeys = function(opts, callback) { - if (!opts.masterKey || !opts.keys) { - return callback(new Error('A decrypted master key, ' + - 'compressed public keys and private keys are required for encryption.')); - } - if (!Buffer.isBuffer(opts.masterKey)) { //we'll have to assume the master key is utf-8 encoded - opts.masterKey = new Buffer(opts.masterKey); - } - assert(opts.masterKey.length === 32, 'Master Key must be 32 bytes in length, ' + - 'if you have a hex string, please pass master key in as a buffer'); - //if the master key is not 32 bytes, then take the sha256 hash - var ret = []; - async.mapLimit(opts.keys, 5, function(key, next) { - var iv = bitcore.crypto.Hash.sha256sha256(new Buffer(key.pubKey, 'hex')).slice(0, 16); - //do we want to encrypt WIF's or RAW private keys or does it matter? - exports.encrypt({ - secret: key.privKey, - iv: iv, - key: opts.masterKey - }, next); - }, function(err, results) { - if(err) { - return callback(err); - } - for(var i = 0; i < results.length; i++) { - ret.push({ - cipherText: results[i], - checkHash: bitcore.crypto.Hash.sha256(new Buffer(opts.keys[i].pubKey + results[i])).toString('hex'), - type: 'encrypted private key', - pubKey: opts.keys[i].pubKey - }); - } - callback(null, ret); - }); -}; - -exports.encrypt = function(opts, callback) { - if (!opts.key || - !opts.iv || - !opts.secret || - opts.key.length !== 32 || - opts.iv.length !== 16 || - opts.secret.length < 1) { - return callback(new Error('Key, IV, and something to encrypt is required.')); - } - var cipher = crypto.createCipheriv('aes-256-cbc', opts.key, opts.iv); - var cipherText; - try { - cipherText = Buffer.concat([cipher.update(opts.secret), cipher.final()]).toString('hex'); - } catch(e) { - return callback(e); - } - return callback(null, cipherText); - -}; -exports.encryptSecret = function(opts, callback) { - var hashFunc = exports.hashPassphrase(opts.derivationOptions); - hashFunc(opts.passphrase, opts.salt, opts.derivationOptions, function(err, hashedPassphrase) { - if (err) { - return callback(err); - } - var secret = opts.secret || crypto.randomBytes(32); - assert(Buffer.isBuffer(secret), 'secret is expected to be a buffer'); - secret = bitcore.crypto.Hash.sha256sha256(secret); - var firstHalf = hashedPassphrase.slice(0, 32); //AES256-cbc shared key - var secondHalf = hashedPassphrase.slice(32, 48); //AES256-cbc IV, for cbc mode, the IV will be 16 bytes - exports.encrypt({ - secret: secret, - key: firstHalf, - iv: secondHalf - }, callback); - }); -}; - -exports.decryptSecret = function(opts, callback) { - var hashFunc = exports.hashPassphrase(opts.derivationOptions); - hashFunc(opts.passphrase, opts.salt, opts.derivationOptions, function(err, hashedPassphrase) { - if (err) { - return callback(err); - } - opts.key = hashedPassphrase; - exports.decrypt(opts, callback); - }); -}; - -exports.decrypt = function(opts, callback) { - if (!Buffer.isBuffer(opts.key)) { - opts.key = new Buffer(opts.key, 'hex'); - } - var secondHalf; - if (opts.iv) { - secondHalf = opts.iv.slice(0, 16); - } else { - secondHalf = opts.key.slice(32, 48); //AES256-cbc IV - } - var cipherText = new Buffer(opts.cipherText, 'hex'); - var firstHalf = opts.key.slice(0, 32); //AES256-cbc shared key - var AESDecipher = crypto.createDecipheriv('aes-256-cbc', firstHalf, secondHalf); - var plainText; - try { - plainText = Buffer.concat([AESDecipher.update(cipherText), AESDecipher.final()]).toString('hex'); - } catch(e) { - return callback(e); - } - callback(null, plainText); -}; - -exports.confirm = function(question, callback) { - ttyread(question + ' (y/N): ', function(err, answer) { - if (err) { - return callback(err, false); - } - if (answer === 'y') { - return callback(null, true); - } - callback(null, false); - }); -}; - -exports.encryptSecretWithPassphrase = function(secret, callback) { - exports.acquirePassphrase(function(err, passphrase) { - if (err) { - return callback(err); - } - var salt = crypto.randomBytes(32).toString('hex'); - exports.encryptSecret({ - secret: secret, - passphrase: passphrase, - salt: salt - }, function(err, cipherText) { - if (err) { - return callback(err); - } - callback(null, cipherText, salt); - }); - }); -}; - -exports.generateNonce = function() { - var nonce = new Buffer(new Array(12)); - nonce.writeDoubleBE(Date.now()); - nonce.writeUInt32BE(process.hrtime()[1], 8); - return nonce; -}; - -exports.generateHashForRequest = function(method, url, nonce) { - nonce = nonce || new Buffer(0); - assert(Buffer.isBuffer(nonce), 'nonce must a buffer'); - var dataToSign = Buffer.concat([nonce, new Buffer(method), new Buffer(url)]); - return bitcore.crypto.Hash.sha256sha256(dataToSign); -}; - -exports.getWalletIdFromName = function(walletName) { - if (!Buffer.isBuffer(walletName)) { - walletName = new Buffer(walletName, 'utf8'); - } - return bitcore.crypto.Hash.sha256sha256(walletName).toString('hex'); -}; - -exports.isRangeMoreThan = function(a, b) { - if (a && !b) { - return true; - } - if (!a && !b) { - return false; - } - if (!a && b) { - return false; - } - if (a.height > b.height) { - return true; - } else if (a.height < b.height) { - return false; - } else { - return a.index > b.index; - } -}; - -exports.toHexBuffer = function(a) { - if (!Buffer.isBuffer(a)) { - a = new Buffer(a, 'hex'); - } - return a; -}; - -exports.toIntIfNumberLike = function(a) { - if (!/[^\d]+/.test(a)) { - return parseInt(a); - } - return a; -}; - -exports.delimitedStringParse = function(delim, str) { - function tryJSONparse(str) { - try { - return JSON.parse(str); - } catch(e) { - return false; - } - } - var ret = []; - - if (delim === null) { - return tryJSONparse(str); - } - - var list = str.split(delim); - for(var i = 0; i < list.length; i++) { - ret.push(tryJSONparse(list[i])); - } - ret = _.compact(ret); - return ret.length === 0 ? false : ret; - -}; - -exports.diffTime = function(time) { - var diff = process.hrtime(time); - return (diff[0] * 1E9 + diff[1])/(1E9 * 1.0); -}; - -/* -* input: string representing a number + multiple of bytes, e.g. 500MB, 200KB, 100B -* output: integer representing the byte count -*/ -exports.parseByteCount = function(byteCountString) { - - function finish(n, m) { - var num = parseInt(n); - if (num > 0) { - return num * m; - } - return null; - } - - if (!_.isString(byteCountString)) { - return byteCountString; - } - var str = byteCountString.replace(/\s+/g, ''); - var map = { 'MB': 1E6, 'kB': 1000, 'KB': 1000, 'MiB': (1024 * 1024), - 'KiB': 1024, 'GiB': Math.pow(1024, 3), 'GB': 1E9 }; - var keys = Object.keys(map); - for(var i = 0; i < keys.length; i++) { - var re = new RegExp(keys[i] + '$'); - var match = str.match(re); - if (match) { - var num = str.slice(0, match.index); - return finish(num, map[keys[i]]); - } - } - return finish(byteCountString, 1); -}; - -/* - * input: arguments passed into originating function (whoever called us) - * output: bool args are valid for encoding a key to the database -*/ -exports.hasRequiredArgsForEncoding = function(args) { - function exists(arg) { - return !(arg === null || arg === undefined); - } - - if (!exists(args[0])) { - return false; - } - - var pastArgMissing; - - for(var i = 1; i < args.length; i++) { - var argMissing = exists(args[i]); - if (argMissing && pastArgMissing) { - return false; - } - pastArgMissing = argMissing; - } - - return true; -}; - -exports.toJSONL = function(obj) { - //this should be a standard obj that JSON.stringify will handle - //general newlines within key values or data values are not permitted - //this is intended to be used for bitcoin tx's that don't have newlines - //within keys or values themselves - var str = JSON.stringify(obj); - str = str.replace(/\n/g, ''); - return str + '\n'; -}; - -module.exports = exports; diff --git a/lib/services/wallet-api/validators.js b/lib/services/wallet-api/validators.js deleted file mode 100644 index 9b162b8c..00000000 --- a/lib/services/wallet-api/validators.js +++ /dev/null @@ -1,225 +0,0 @@ -'use strict'; - -var assert = require('assert'); - -var bitcore = require('bitcore-lib'); -var _ = require('lodash'); -var utils = require('./utils'); - -var MAX_INT = 0xffffffff; // Math.pow(2, 32) - 1 - -exports.sanitizeRangeOptions = function(options) { - if (!options) { - options = {}; - } - options.height = options.height || 0; - options.index = options.index || 0; - - if (!options.limit) { - options.limit = 10; - } else if (options.limit > 500) { - throw new Error('Limit exceeds maximum'); - } - - assert(bitcore.util.js.isNaturalNumber(options.height), '"height" is expected to be a natural number'); - assert(bitcore.util.js.isNaturalNumber(options.index), '"index" is expected to be a natural number'); - assert(bitcore.util.js.isNaturalNumber(options.limit), '"limit" is expected to be a natural number'); - - assert(options.limit <= 500, '"limit" exceeds maximum'); - - if (options.end) { - assert(bitcore.util.js.isNaturalNumber(options.end.height), '"end height" is expected to be a natural number'); - } - return options; -}; - -exports.checkRangeParams = function(req, res, next) { - assert(req.bitcoinHeight, '"bitcoinHeight" is expected to be set on the request'); - - var range = { - height: parseInt(req.query.height), - index: parseInt(req.query.index), - limit: parseInt(req.query.limit), - end: { - height: req.bitcoinHeight, - index: MAX_INT - } - }; - - if (req.query.end) { - range.end.height = parseInt(req.query.end) || req.bitcoinHeight; - } - - try { - range = exports.sanitizeRangeOptions(range); - } catch(e) { - return utils.sendError({ - message: 'Invalid params: ' + e.message, - statusCode: 400 - }, res); - } - - assert(range.height <= range.end.height, '\'Height\' param required to be less than \'End\' param.'); - req.range = range; - next(); -}; - -exports.checkAddress = function(req, res, next) { - var address; - var addressStr; - - if (req.body.address) { - addressStr = req.body.address; - } else { - addressStr = req.params.address; - } - - if(!addressStr) { - return utils.sendError({ - message: 'Address param is expected', - statusCode: 400 - }, res); - } - - assert(req.network, '"network" is expected to be set on the request'); - - try { - address = new bitcore.Address(addressStr, req.network); - } catch(e) { - return utils.sendError({ - message: 'Invalid address: ' + e.message, - statusCode: 400 - }, res); - } - - req.address = address; - next(); -}; - -exports.checkWalletId = function(req, res, next) { - - if (!req.params.walletId) { - return utils.sendError({ - message: 'Wallet id is expected', - statusCode: 400 - }, res); - } - - if (req.params.walletId.length !== 64 || !bitcore.util.js.isHexa(req.params.walletId)) { - return utils.sendError({ - message: 'Wallet id is expected to be a hexadecimal string with length of 64', - statusCode: 400 - }, res); - } - - req.walletId = new Buffer(req.params.walletId, 'hex'); - next(); - -}; - -exports.checkAddresses = function(req, res, next) { - - if (!req.file && req.body) { - req.addresses = req.body; - return next(); - } - - if (!req.file || !req.file.buffer) { - generateError(406, 'Content-Type must be set to multipart/form' + - ' and addresses key and value must be given.'); - return; - } - var buf = req.file.buffer; - var bufString = buf.toString(); - if (bufString.slice(-1) === ',') { - var bufString = '[' + bufString.slice(0,-1) + ']'; - } - req.addresses = parse(bufString); - if (!req.addresses) { - generateError(415, 'Could not parse addresses buffer into something meaningful.'); - return; - } - next(); - - function generateError(status, msg) { - res.status(status).jsonp({ - error: msg - }); - } - - //we are able to deal with json/jsonl, possibly others - function parse(string) { - var ret = false; - var delims = [null, '\n', ' ']; - for(var i = 0; i < delims.length; i++) { - ret = utils.delimitedStringParse(delims[i], string); - if (_.isArray(ret)) { - return ret; - } - } - return ret; - } -}; - -exports.checkAuthHeaders = function(req, res) { - var identity = req.header('x-identity'); - var signature = req.header('x-signature'); - var nonce = req.header('x-nonce'); - if (identity && (identity.length > 130 || !bitcore.util.js.isHexa(identity))) { - utils.sendError({ - message: 'x-identity is expected to be a hexadecimal string with length of less than 131', - statusCode: 400 - }, res); - return false; - } - if (signature && (signature.length > 142 || !bitcore.util.js.isHexa(signature))) { - utils.sendError({ - message: 'x-signature is expected to be a hexadecimal string with length of less than 143', - statusCode: 400 - }, res); - return false; - } - if (nonce && (nonce.length > 128 || nonce.length % 2 !== 0 || !bitcore.util.js.isHexa(nonce))) { - utils.sendError({ - message: 'x-nonce is expected to be a hexadecimal string with length of less than 129', - statusCode: 400 - }, res); - return false; - } - return true; -}; - -exports.checkDate = function(dateStrings) { - var errors = []; - if (!Array.isArray(dateStrings)) { - dateStrings = [dateStrings]; - } - for(var i = 0; i < dateStrings.length; i++) { - internalDateCheck(dateStrings[i]); - } - - function internalDateCheck(dateString) { - var date = new Date(utils.toIntIfNumberLike(dateString)); - if (date.toString() === 'Invalid Date') { - errors.push('The date supplied: \'' + dateString + - '\' is not a valid date string. A valid date could be: \'2016-09-01\'.'); - } - } - return errors; -}; - -exports.checkDateFunction = function(callback) { - var self = this; - return function() { - var args = Array.prototype.slice.call(arguments); - var errors = self.checkDate([args[1], args[2]]); - if (errors.length > 0) { - args.unshift(errors); - } else { - args.unshift(null); - } - callback.apply(null, args); - }; -}; - -module.exports = exports; diff --git a/test_bcoin.js b/test_bcoin.js deleted file mode 100644 index 24bca856..00000000 --- a/test_bcoin.js +++ /dev/null @@ -1,21 +0,0 @@ -var bcoin = require('bcoin').set('main'); - -var node = bcoin.fullnode({ - checkpoints: true, - // Primary wallet passphrase - logLevel: 'info' -}); - -// We get a lot of errors sometimes, -// usually from peers hanging up on us. -// Just ignore them for now. -node.on('error', function(err) { - console.log(err); -}); - -// Start the node -node.open().then(function() { - node.connect().then(function() { - node.startSync(); - }); -});