From 86186e6147997bc7ca81bc7d826ac29b4c8f9964 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Tue, 10 Jan 2017 21:24:12 -0500 Subject: [PATCH 01/10] Small fixes for service requiring and warnings. --- lib/scaffold/start.js | 25 ++++++++++++++++--------- lib/services/bitcoind.js | 13 +++++++++++-- package.json | 1 + 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index e8654575..9dd8a645 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -4,6 +4,7 @@ var path = require('path'); var BitcoreNode = require('../node'); var index = require('../'); var bitcore = require('bitcore-lib'); +var fs = require('fs'); var _ = bitcore.deps._; var log = index.log; var shuttingDown = false; @@ -136,16 +137,22 @@ function checkService(service) { function loadModule(req, service) { try { // first try in the built-in bitcore-node services directory - service.module = req(path.resolve(__dirname, '../services/' + service.name)); - } catch(e) { - - // check if the package.json specifies a specific file to use - var servicePackage = req(service.name + '/package.json'); - var serviceModule = service.name; - if (servicePackage.bitcoreNode) { - serviceModule = service.name + '/' + servicePackage.bitcoreNode; + var serviceFile = path.resolve(__dirname, '../services/' + service.name); + if (fs.existsSync(serviceFile + '.js')) { + // if the file exists, we can require it, then if there is a problem, catch and display the error + service.module = req(serviceFile); + } else { + // check if the package.json specifies a specific file to use + var servicePackage = req(service.name + '/package.json'); + var serviceModule = service.name; + if (servicePackage.bitcoreNode) { + serviceModule = service.name + '/' + servicePackage.bitcoreNode; + } + service.module = req(serviceModule); } - service.module = req(serviceModule); + } catch(e) { + log.error(e.stack); + process.exit(-1); } } diff --git a/lib/services/bitcoind.js b/lib/services/bitcoind.js index 21480251..2f6ba0ed 100644 --- a/lib/services/bitcoind.js +++ b/lib/services/bitcoind.js @@ -76,6 +76,7 @@ Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000; Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000; Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000; Bitcoin.DEFAULT_TIP_UPDATE_INTERVAL = 15000; +Bitcoin.DEFAULT_ZMQ_DELAY_WARNING_MULTIPLIER = 5; Bitcoin.DEFAULT_TRANSACTION_CONCURRENCY = 5; Bitcoin.DEFAULT_CONFIG_SETTINGS = { server: 1, @@ -114,6 +115,10 @@ Bitcoin.prototype._initDefaults = function(options) { // sync progress level when zmq subscribes to events this.zmqSubscribeProgress = options.zmqSubscribeProgress || Bitcoin.DEFAULT_ZMQ_SUBSCRIBE_PROGRESS; + + // set the zmq delay warning multiplier + this.zmqDelayWarningMultiplier = options.zmqDelayWarningMultiplier || Bitcoin.DEFAULT_ZMQ_DELAY_WARNING_MULTIPLIER; + this.zmqDelayWarningMultiplierCouunt = 0; }; Bitcoin.prototype._initCaches = function() { @@ -725,7 +730,10 @@ Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { }); node.zmqSubSocket.on('connect_delay', function(fd, endPoint) { - log.warn('ZMQ connection delay:', endPoint); + if (this.zmqDelayWarningMultiplierCouunt++ >= this.zmqDelayWarningMultiplier) { + log.warn('ZMQ connection delay:', endPoint); + this.zmqDelayWarningMultiplierCouunt = 0; + } }); node.zmqSubSocket.on('disconnect', function(fd, endPoint) { @@ -739,7 +747,8 @@ Bitcoin.prototype._initZmqSubSocket = function(node, zmqUrl) { }, 5000); }); - node.zmqSubSocket.monitor(500, 0); + //monitors are polling and not event-driven + node.zmqSubSocket.monitor(100, 0); node.zmqSubSocket.connect(zmqUrl); }; diff --git a/package.json b/package.json index 4e02b7cf..58ecf91f 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "errno": "^0.1.4", "express": "^4.13.3", "liftoff": "^2.2.0", + "lmdb": "rvagg/lmdb#5ad819f77714925248dfecb0ba2174080dba949f", "lru-cache": "^4.0.1", "mkdirp": "0.5.0", "path-is-absolute": "^1.0.0", From e6d569620bc0b8a6cdafea3ec3692a15637d68cd Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 11 Jan 2017 17:01:48 -0500 Subject: [PATCH 02/10] Implemented basic wallet db and calls to bitcoind. --- lib/scaffold/start.js | 20 +- lib/services/wallet/index.js | 242 +++++++++++ lib/services/wallet/utils.js | 691 ++++++++++++++++++++++++++++++ lib/services/wallet/validators.js | 246 +++++++++++ package.json | 6 +- 5 files changed, 1191 insertions(+), 14 deletions(-) create mode 100644 lib/services/wallet/index.js create mode 100644 lib/services/wallet/utils.js create mode 100644 lib/services/wallet/validators.js diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 9dd8a645..a0815918 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -138,21 +138,15 @@ function loadModule(req, service) { try { // first try in the built-in bitcore-node services directory var serviceFile = path.resolve(__dirname, '../services/' + service.name); - if (fs.existsSync(serviceFile + '.js')) { - // if the file exists, we can require it, then if there is a problem, catch and display the error - service.module = req(serviceFile); - } else { - // check if the package.json specifies a specific file to use - var servicePackage = req(service.name + '/package.json'); - var serviceModule = service.name; - if (servicePackage.bitcoreNode) { - serviceModule = service.name + '/' + servicePackage.bitcoreNode; - } - service.module = req(serviceModule); - } + service.module = req(serviceFile); } catch(e) { log.error(e.stack); - process.exit(-1); + var servicePackage = req(service.name + '/package.json'); + var serviceModule = service.name; + if (servicePackage.bitcoreNode) { + serviceModule = service.name + '/' + servicePackage.bitcoreNode; + } + service.module = req(serviceModule); } } diff --git a/lib/services/wallet/index.js b/lib/services/wallet/index.js new file mode 100644 index 00000000..35ddca8b --- /dev/null +++ b/lib/services/wallet/index.js @@ -0,0 +1,242 @@ +'use strict'; + +var BaseService = require('../../service'); +var inherits = require('util').inherits; +var index = require('../../'); +var log = index.log; +var errors = index.errors; +var bitcore = require('bitcore-lib'); +var Networks = bitcore.Networks; +var levelup = require('levelup'); +var leveldown = require('leveldown'); +var multer = require('multer'); +var storage = multer.memoryStorage(); +var upload = multer({ storage: storage }); +var validators = require('./validators'); +var utils = require('./utils'); +/** + * The Address Service builds upon the Database Service and the Bitcoin Service to add additional + * functionality for getting information by base58check encoded addresses. This includes getting the + * balance for an address, the history for a collection of addresses, and unspent outputs for + * constructing transactions. This is typically the core functionality for building a wallet. + * @param {Object} options + * @param {Node} options.node - An instance of the node + * @param {String} options.name - An optional name of the service + */ +var WalletService = function(options) { + BaseService.call(this, options); + this._dbOptions = { + keyEncoding: 'string', + valueEncoding: 'json' + }; + this._db = levelup(options.dbPath, this._dbOptions); +}; + +inherits(WalletService, BaseService); + +WalletService.dependencies = [ + 'bitcoind', + 'web' +]; + +/** + * Called by the Node to get the available API methods for this service, + * that can be exposed over the JSON-RPC interface. + */ +WalletService.prototype.getAPIMethods = function() { + return [ + //['getWalletInfo', this, this.getInfo, 0] + //['getWalletBalance', this, this.getWalletBalance, 2], + //['getBlockTimestampInfo', this, this.getBlockTimestampInfo, 2], + //['addWalletAddresses', this, this., 2], + //['addWalletAddress', this, this.addWalletAddress, 2], + //['addWallet', this, this.addWallet, 2], + //['getWalletUtxos', this, this.getWalletUtxos, 2], + //['getWalletRawTransactions', this, this.getWalletRawTransactions, 1] + //['getWalletTxids', this, this.getWalletTxids, 1] + //['getWalletTransactions', this, this.getWalletTransactions, 1] + ]; +}; +WalletService.prototype.start = function(callback) { + setImmediate(callback); +}; + +WalletService.prototype.stop = function(callback) { + setImmediate(callback); +}; + +/** + * Called by the Bus to get the available events for this service. + */ +WalletService.prototype.getPublishEvents = function() { + return []; +}; + +WalletService.prototype._endpointUTXOs = function() { + var self = this; + return function(req, res) { + var walletId = req.params.walletId; + //var tip = self.node.bitcoind.tip; + // TODO: get the height of the tip + //var height = tip; + var height = null; + self._getUtxos(walletId, height, function(err, utxos) { + if(err) { + return utils.sendError(err); + } + res.status(200).jsonp({ + utxos: utxos, + height: height + }); + }); + }; +}; + +WalletService.prototype._endpointGetBalance= function() { + var self = this; + return function(req, res) { + var walletId = req.params.walletId; + //var tip = self.node.bitcoind.tip; + // TODO: get the height of the tip + //var height = tip; + var height = null; + self._getBalance(walletId, height, function(err, balance) { + if(err) { + return utils.sendError(err); + } + res.status(200).jsonp({ + balance: balance, + height: height + }); + }); + }; +}; + +WalletService.prototype._endpointGetAddresses = function() { + var self = this; + return function(req, res) { + var walletId = req.params.walletId; + + self._getAddresses(walletId, function(err, addresses) { + if(err) { + return utils.sendError(err); + } + res.status(200).jsonp({ + addresses: addresses + }); + }); + }; +}; +WalletService.prototype._endpointPostAddresses = function() { + var self = this; + return function(req, res) { + var addresses = req.addresses; + var walletId = utils.getWalletId(); + self._storeAddresses(walletId, addresses, function(err, hash) { + if(err) { + return utils.sendError(err, res); + } + res.status(201).jsonp({ + walletId: walletId, + }); + }); + }; +}; + +WalletService.prototype._getUtxos = function(walletId, height, callback) { + // TODO get the balance only to this height + var self = this; + self._getAddresses(walletId, function(err, addresses) { + if(err) { + return callback(err); + } + + self.node.services.bitcoind.getAddressUnspentOutputs(addresses, {queryMempool: false}, callback); + }); +}; +WalletService.prototype._getBalance = function(walletId, height, callback) { + // TODO get the balance only to this height + var self = this; + self._getAddresses(walletId, function(err, addresses) { + if(err) { + return callback(err); + } + self.node.services.bitcoind.getAddressUnspentOutputs(addresses, { + queryMempool: false + }, function(err, utxos) { + if(err) { + return callback(err); + } + var balance = 0; + utxos.forEach(function(utxo) { + balance += utxo.satoshis; + }); + callback(null, balance); + }); + }); +}; +WalletService.prototype._getAddresses = function(walletId, callback) { + this._db.get(walletId, callback); +}; +WalletService.prototype._storeAddresses = function(walletId, addresses, callback) { + this._db.put(walletId, addresses, callback); +}; + +WalletService.prototype._endpointGetInfo = function() { + return function(req, res) { + res.jsonp({result: 'ok'}); + }; +}; +WalletService.prototype.setupRoutes = function(app, express) { + var s = this; + var v = validators; + app.get('/info', + s._endpointGetInfo() + ); + //app.get('/wallets/:walletId/txids', + // v.checkWalletId, + // v.checkRangeParams, + // s._endpointTxids() + //); + //app.get('/wallets/:walletId/transactions', + // v.checkWalletId, + // v.checkRangeParams, + // s._endpointTransactions() + //); + //app.get('/wallets/:walletId/rawtransactions', + // v.checkWalletId, + // v.checkRangeParams, + // s._endpointRawTransactions() + //); + app.get('/wallets/:walletId/utxos', + s._endpointUTXOs() + ); + app.get('/wallets/:walletId/balance', + s._endpointGetBalance() + ); + app.get('/wallets/:walletId', + s._endpointGetAddresses() + ); + app.post('/wallets/addresses', + upload.single('addresses'), + v.checkAddresses, + s._endpointPostAddresses() + ); + //app.put('/wallets/:walletId', + // v.checkWalletId, + // s._endpointPutWallet() + //); + //app.get('/info/timestamps', + // s._endpointGetHeightsFromTimestamps() + //); + //app.get('/jobs/:jobId', + // s._endpointJobStatus() + //); + //app.use(s._endpointNotFound()); +}; + +WalletService.prototype.getRoutePrefix = function() { + return 'wallet'; +}; +module.exports = WalletService; + diff --git a/lib/services/wallet/utils.js b/lib/services/wallet/utils.js new file mode 100644 index 00000000..5eae4143 --- /dev/null +++ b/lib/services/wallet/utils.js @@ -0,0 +1,691 @@ +'use strict'; + +var Writable = require('stream').Writable; +var assert = require('assert'); +var crypto = require('crypto'); +var fs = require('fs'); +var inherits = require('util').inherits; +var path = require('path'); +var spawn = require('child_process').spawn; + +var BitcoinRPC = require('bitcoind-rpc'); +var _ = require('lodash'); +var async = require('async'); +var bitcore = require('bitcore-lib'); +var mkdirp = require('mkdirp'); +var ttyread = require('ttyread'); + +var exports = {}; + +exports.isInteger = function(value) { + return typeof value === 'number' && + isFinite(value) && + Math.floor(value) === value; +}; + +/** + * Will create a directory if it does not already exist. + * + * @param {String} directory - An absolute path to the directory + * @param {Function} callback + */ +exports.setupDirectory = function(directory, callback) { + fs.access(directory, function(err) { + if (err && err.code === 'ENOENT') { + return mkdirp(directory, callback); + } else if (err) { + return callback(err); + } + callback(); + }); +}; + +/** + * This will split a range of numbers "a" to "b" by sections + * of the length "max". + * + * Example: + * > var range = utils.splitRange(1, 10, 3); + * > [[1, 3], [4, 6], [7, 9], [10, 10]] + * + * @param {Number} a - The start index (lesser) + * @param {Number} b - The end index (greater) + * @param {Number} max - The maximum section length + */ +exports.splitRange = function(a, b, max) { + assert(b > a, '"b" is expected to be greater than "a"'); + var sections = []; + var delta = b - a; + var first = a; + var last = a; + + var length = Math.floor(delta / max); + for (var i = 0; i < length; i++) { + last = first + max - 1; + sections.push([first, last]); + first += max; + } + + if (last <= b) { + sections.push([first, b]); + } + + return sections; +}; + +/** + * getFileStream: Checks for the file's existence and returns a readable stream or stdin + * @param {String} path - The path to the file + * @param {Function} callback + */ +exports.getFileStream = function(filePath, callback) { + callback(null, fs.createReadStream(filePath)); +}; + +exports.readWalletDatFile = function(filePath, network, callback) { + assert(_.isString(network), 'Network expected to be a string.'); + var datadir = path.dirname(filePath).replace(/(\/testnet3|\/regtest)$/, ''); + var name = path.basename(filePath); + var options = ['-datadir=' + datadir, '-wallet=' + name]; + if (network === 'testnet') { + options.push('-testnet'); + } else if (network === 'regtest') { + options.push('-regtest'); + } + // TODO use ../node_modules/.bin/wallet-utility + var exec = path.resolve(__dirname, '../node_modules/bitcore-node/bin/bitcoin-0.12.1/bin/wallet-utility'); + var wallet = spawn(exec, options); + + var result = ''; + + wallet.stdout.on('data', function(data) { + result += data.toString('utf8'); + }); + + var error; + + wallet.stderr.on('data', function(data) { + error = data.toString(); + }); + + wallet.on('close', function(code) { + if (code === 0) { + var addresses; + try { + addresses = JSON.parse(result); + addresses = addresses.map(function(entry) { + return entry.addr ? entry.addr : entry; + }); + } catch(err) { + return callback(err); + } + return callback(null, addresses); + } else if (error) { + return callback(new Error(error)); + } else { + var message = 'wallet-utility exited (' + code + '): ' + result; + return callback(new Error(message)); + } + }); +}; + +exports.readWalletFile = function(filePath, network, callback) { + if (/\.dat$/.test(filePath)) { + exports.readWalletDatFile(filePath, network, callback); + } else { + exports.getFileStream(filePath, callback); + } +}; + +/** + * This will split an array into smaller arrays by size + * + * @param {Array} array + * @param {Number} size - The length of resulting smaller arrays + */ +exports.splitArray = function(array, size) { + var results = []; + while (array.length) { + results.push(array.splice(0, size)); + } + return results; +}; + +/** + * Utility to get the remote ip address from cloudflare headers. + * + * @param {Object} req - An express request object + */ +exports.getRemoteAddress = function(req) { + if (req.headers['cf-connecting-ip']) { + return req.headers['cf-connecting-ip']; + } + return req.socket.remoteAddress; +}; + +/** + * A middleware to enable CORS + * + * @param {Object} req - An express request object + * @param {Object} res - An express response object + * @param {Function} next + */ +exports.enableCORS = function(req, res, next) { + res.header('access-control-allow-origin', '*'); + res.header('access-control-allow-methods', 'GET, HEAD, PUT, POST, OPTIONS'); + var allowed = [ + 'origin', + 'x-requested-with', + 'content-type', + 'accept', + 'content-length', + 'cache-control', + 'cf-connecting-ip' + ]; + res.header('access-control-allow-headers', allowed.join(', ')); + + var method = req.method && req.method.toUpperCase && req.method.toUpperCase(); + + if (method === 'OPTIONS') { + res.statusCode = 204; + res.end(); + } else { + next(); + } +}; + +/** + * Will send error to express response + * + * @param {Error} err - error object + * @param {Object} res - express response object + */ +exports.sendError = function(err, res) { + if (err.statusCode) { + res.status(err.statusCode).send(err.message); + } else { + console.error(err.stack); + res.status(503).send(err.message); + } +}; + +/** + * Will create a writeable logger stream + * + * @param {Function} logger - Function to log information + * @returns {Stream} + */ +exports.createLogStream = function(logger) { + function Log(options) { + Writable.call(this, options); + } + inherits(Log, Writable); + + Log.prototype._write = function (chunk, enc, callback) { + logger(chunk.slice(0, chunk.length - 1)); // remove new line and pass to logger + callback(); + }; + var stream = new Log(); + + return stream; +}; + +exports.getWalletId = function() { + return crypto.randomBytes(16).toString('hex'); +}; + +exports.getClients = function(clientsConfig) { + var clients = []; + for (var i = 0; i < clientsConfig.length; i++) { + var config = clientsConfig[i]; + var remoteClient = new BitcoinRPC({ + protocol: config.rpcprotocol || 'http', + host: config.rpchost || '127.0.0.1', + port: config.rpcport, + user: config.rpcuser, + pass: config.rpcpassword, + rejectUnauthorized: _.isUndefined(config.rpcstrict) ? true : config.rpcstrict + }); + clients.push(remoteClient); + } + return clients; +}; + +exports.setClients = function(obj, clients) { + obj._clients = clients; + obj._clientsIndex = 0; + Object.defineProperty(obj, 'clients', { + get: function() { + var client = obj._clients[obj._clientsIndex]; + obj._clientsIndex = (obj._clientsIndex + 1) % obj._clients.length; + return client; + }, + enumerable: true, + configurable: false + }); +}; + +exports.tryAllClients = function(obj, func, options, callback) { + if (_.isFunction(options)) { + callback = options; + options = {}; + } + var clientIndex = obj._clientsIndex; + var retry = function(done) { + var client = obj._clients[clientIndex]; + clientIndex = (clientIndex + 1) % obj._clients.length; + func(client, done); + }; + async.retry({times: obj._clients.length, interval: options.interval || 1000}, retry, callback); +}; + +exports.wrapRPCError = function(errObj) { + var err = new Error(errObj.message); + err.code = errObj.code; + return err; +}; + +var PUBKEYHASH = new Buffer('01', 'hex'); +var SCRIPTHASH = new Buffer('02', 'hex'); + +exports.getAddressTypeString = function(bufferArg) { + var buffer = bufferArg; + if (!Buffer.isBuffer(bufferArg)) { + buffer = new Buffer(bufferArg, 'hex'); + } + var type = buffer.slice(0, 1); + if (type.compare(PUBKEYHASH) === 0) { + return 'pubkeyhash'; + } else if (type.compare(SCRIPTHASH) === 0) { + return 'scripthash'; + } else { + throw new TypeError('Unknown address type'); + } +}; + +exports.getAddressTypeBuffer = function(address) { + var type; + if (address.type === 'pubkeyhash') { + type = PUBKEYHASH; + } else if (address.type === 'scripthash') { + type = SCRIPTHASH; + } else { + throw new TypeError('Unknown address type'); + } + return type; +}; + +exports.splitBuffer = function(buffer, size) { + var pos = 0; + var buffers = []; + while (pos < buffer.length) { + buffers.push(buffer.slice(pos, pos + size)); + pos += size; + } + return buffers; +}; + +exports.exitWorker = function(worker, timeout, callback) { + assert(worker, '"worker" is expected to be defined'); + var exited = false; + worker.once('exit', function(code) { + if (!exited) { + exited = true; + if (code !== 0) { + var error = new Error('Worker did not exit cleanly: ' + code); + error.code = code; + return callback(error); + } else { + return callback(); + } + } + }); + worker.kill('SIGINT'); + setTimeout(function() { + if (!exited) { + exited = true; + worker.kill('SIGKILL'); + return callback(new Error('Worker exit timeout, force shutdown')); + } + }, timeout).unref(); +}; + +exports.timestampToISOString = function(timestamp) { + return new Date(this.toIntIfNumberLike(timestamp) * 1000).toISOString(); +}; + +exports.satoshisToBitcoin = function(satoshis) { + return satoshis / 100000000; +}; + +exports.getPassphrase = function(callback) { + ttyread('Enter passphrase: ', {silent: true}, callback); +}; + +exports.acquirePassphrase = function(callback) { + var first; + var second; + async.doWhilst(function(next) { + ttyread('Enter passphrase: ', {silent: true}, function(err, result) { + if (err) { + return callback(err); + } + first = result; + ttyread('Re-enter passphrase: ', {silent: true}, function(err, result) { + second = result; + next(); + }); + }); + }, function() { + if (first !== second) { + console.log('Passphrases do not match, please re-enter.'); + return true; + } + return false; + }, function(err) { + if (err) { + return callback(err); + } + callback(null, first); + }); +}; + +/* + Important notes: + + How the encryption/decryption schemes work. + 1. The user's passphrase and salt are hashed using scrypt algorithm. You must store the salt. + On modern hardware this hashing function should take 1-2 seconds. + 2. The resulting hash is 48 bytes. The first 32 bytes of this hash is the "key" and the last + 16 bytes is the "iv" to decrypt the master key using AES256-cbc. + 3. The plaintext "master key" is always 32 bytes and should be as random as possible. + You may pass in the plaintext master key to encryptSecret -or- /dev/random will be consulted. + 4. The cipherText of the master key must be stored just like the salt. For added security, you + might store the cipherText of the master key separate from the cipherText. + For example, if an attacker discovers your passphrase and salt (the most likely scenario), they would + still require the cipherText of the master key in order to decrypt the cipherText of your private keys. + Storing your encrypted master key on another device would be a better choice than keeping your salt, + the cipherText of your master key and the cipherText of your private keys on the same computer system. + 5. The plaintext master key is then used to encrypt/decrypt the bitcoin private keys. The private keys' + corresponding public key is used as the IV for the procedure. + + + Specific notes regarding how private keys are transferred from a traditional "wallet.dat" file used with + Bitcoin Core's Wallet: + + 1. Bitcoin Core's Wallet uses Berkeley DB version 4.8 to store secp256k1 elliptic curve private keys in WIF format. + 2. The same Berkeley DB, internally called "main", also stores compressed public keys for the above private keys, + the master keys used to encrypt the above private keys and bitcoin transaction details relevant to those private keys + 3. The underlying data structure for the Berkeley database is the B-Tree (balanced tree). This is a key-value data + structure, therefore the database is a key-value database. + Berkeley DB documentation also refers to this as "key-record" + This means that the data contained in this B-Tree is organized for high speed retrieval based on a key. + In other words the database is optimized for lookups. + 4. The filename for this database file is called "wallet.dat" historically, + but you can rename it to whatever suits you + +*/ +//this function depends on the derivation method and its params that were originally used to hash the passphrase +//this could be SHA512, scrypt, etc. +exports.sha512KDF = function(passphrase, salt, derivationOptions, callback) { + if (!derivationOptions || derivationOptions.method !== 0 || !derivationOptions.rounds) { + return callback(new Error('SHA512 KDF method was called for, ' + + 'yet the derivations options for it were not supplied.')); + } + var rounds = derivationOptions.rounds || 1; + //if salt was sent in as a string, we will have to assume the default encoding type + if (!Buffer.isBuffer(salt)) { + salt = new Buffer(salt, 'utf-8'); + } + var derivation = Buffer.concat([new Buffer(''), new Buffer(passphrase), salt]); + for(var i = 0; i < rounds; i++) { + derivation = crypto.createHash('sha512').update(derivation).digest(); + } + callback(null, derivation); +}; + +exports.hashPassphrase = function(opts) { + return exports.sha512KDF; +}; + +exports.decryptPrivateKey = function(opts, callback) { + exports.decryptSecret(opts, function(err, masterKey) { + if(err) { + return callback(err); + } + opts.cipherText = opts.pkCipherText; + //decrypt the private here using the plainText master key as the "key" + //and the double sha256 compressed pub key as the "IV" + opts.key = masterKey; + opts.iv = bitcore.crypto.Hash.sha256sha256(new Buffer(opts.pubkey, 'hex')); + exports.decrypt(opts, function(err, privateKey) { + if(err) { + return callback(err); + } + callback(null, privateKey); + }); + }); +}; + +//call decryptSecret first +exports.encryptPrivateKeys = function(opts, callback) { + if (!opts.masterKey || !opts.keys) { + return callback(new Error('A decrypted master key, ' + + 'compressed public keys and private keys are required for encryption.')); + } + if (!Buffer.isBuffer(opts.masterKey)) { //we'll have to assume the master key is utf-8 encoded + opts.masterKey = new Buffer(opts.masterKey); + } + assert(opts.masterKey.length === 32, 'Master Key must be 32 bytes in length, ' + + 'if you have a hex string, please pass master key in as a buffer'); + //if the master key is not 32 bytes, then take the sha256 hash + var ret = []; + async.mapLimit(opts.keys, 5, function(key, next) { + var iv = bitcore.crypto.Hash.sha256sha256(new Buffer(key.pubKey, 'hex')).slice(0, 16); + //do we want to encrypt WIF's or RAW private keys or does it matter? + exports.encrypt({ + secret: key.privKey, + iv: iv, + key: opts.masterKey + }, next); + }, function(err, results) { + if(err) { + return callback(err); + } + for(var i = 0; i < results.length; i++) { + ret.push({ + cipherText: results[i], + checkHash: bitcore.crypto.Hash.sha256(new Buffer(opts.keys[i].pubKey + results[i])).toString('hex'), + type: 'encrypted private key', + pubKey: opts.keys[i].pubKey + }); + } + callback(null, ret); + }); +}; + +exports.encrypt = function(opts, callback) { + if (!opts.key || + !opts.iv || + !opts.secret || + opts.key.length !== 32 || + opts.iv.length !== 16 || + opts.secret.length < 1) { + return callback(new Error('Key, IV, and something to encrypt is required.')); + } + var cipher = crypto.createCipheriv('aes-256-cbc', opts.key, opts.iv); + var cipherText; + try { + cipherText = Buffer.concat([cipher.update(opts.secret), cipher.final()]).toString('hex'); + } catch(e) { + return callback(e); + } + return callback(null, cipherText); + +}; +exports.encryptSecret = function(opts, callback) { + var hashFunc = exports.hashPassphrase(opts.derivationOptions); + hashFunc(opts.passphrase, opts.salt, opts.derivationOptions, function(err, hashedPassphrase) { + if (err) { + return callback(err); + } + var secret = opts.secret || crypto.randomBytes(32); + assert(Buffer.isBuffer(secret), 'secret is expected to be a buffer'); + secret = bitcore.crypto.Hash.sha256sha256(secret); + var firstHalf = hashedPassphrase.slice(0, 32); //AES256-cbc shared key + var secondHalf = hashedPassphrase.slice(32, 48); //AES256-cbc IV, for cbc mode, the IV will be 16 bytes + exports.encrypt({ + secret: secret, + key: firstHalf, + iv: secondHalf + }, callback); + }); +}; + +exports.decryptSecret = function(opts, callback) { + var hashFunc = exports.hashPassphrase(opts.derivationOptions); + hashFunc(opts.passphrase, opts.salt, opts.derivationOptions, function(err, hashedPassphrase) { + if (err) { + return callback(err); + } + opts.key = hashedPassphrase; + exports.decrypt(opts, callback); + }); +}; + +exports.decrypt = function(opts, callback) { + if (!Buffer.isBuffer(opts.key)) { + opts.key = new Buffer(opts.key, 'hex'); + } + var secondHalf; + if (opts.iv) { + secondHalf = opts.iv.slice(0, 16); + } else { + secondHalf = opts.key.slice(32, 48); //AES256-cbc IV + } + var cipherText = new Buffer(opts.cipherText, 'hex'); + var firstHalf = opts.key.slice(0, 32); //AES256-cbc shared key + var AESDecipher = crypto.createDecipheriv('aes-256-cbc', firstHalf, secondHalf); + var plainText; + try { + plainText = Buffer.concat([AESDecipher.update(cipherText), AESDecipher.final()]).toString('hex'); + } catch(e) { + return callback(e); + } + callback(null, plainText); +}; + +exports.confirm = function(question, callback) { + ttyread(question + ' (y/N): ', function(err, answer) { + if (err) { + return callback(err, false); + } + if (answer === 'y') { + return callback(null, true); + } + callback(null, false); + }); +}; + +exports.encryptSecretWithPassphrase = function(secret, callback) { + exports.acquirePassphrase(function(err, passphrase) { + if (err) { + return callback(err); + } + var salt = crypto.randomBytes(32).toString('hex'); + exports.encryptSecret({ + secret: secret, + passphrase: passphrase, + salt: salt + }, function(err, cipherText) { + if (err) { + return callback(err); + } + callback(null, cipherText, salt); + }); + }); +}; + +exports.generateNonce = function() { + var nonce = new Buffer(new Array(12)); + nonce.writeDoubleBE(Date.now()); + nonce.writeUInt32BE(process.hrtime()[1], 8); + return nonce; +}; + +exports.generateHashForRequest = function(method, url, nonce) { + nonce = nonce || new Buffer(0); + assert(Buffer.isBuffer(nonce), 'nonce must a buffer'); + var dataToSign = Buffer.concat([nonce, new Buffer(method), new Buffer(url)]); + return bitcore.crypto.Hash.sha256sha256(dataToSign); +}; + +exports.getWalletIdFromName = function(walletName) { + if (!Buffer.isBuffer(walletName)) { + walletName = new Buffer(walletName, 'utf8'); + } + return bitcore.crypto.Hash.sha256sha256(walletName).toString('hex'); +}; + +exports.isRangeMoreThan = function(a, b) { + if (a && !b) { + return true; + } + if (!a && !b) { + return false; + } + if (!a && b) { + return false; + } + if (a.height > b.height) { + return true; + } else if (a.height < b.height) { + return false; + } else { + return a.index > b.index; + } +}; + +exports.toHexBuffer = function(a) { + if (!Buffer.isBuffer(a)) { + a = new Buffer(a, 'hex'); + } + return a; +}; + +exports.toIntIfNumberLike = function(a) { + if (!/[^\d]+/.test(a)) { + return parseInt(a); + } + return a; +}; + +exports.delimitedStringParse = function(delim, str) { + var ret = []; + + if (delim === null) { + return tryJSONparse(str); + } + + var list = str.split(delim); + for(var i = 0; i < list.length; i++) { + ret.push(tryJSONparse(list[i])); + } + ret = _.compact(ret); + return ret.length === 0 ? false : ret; + + function tryJSONparse(str) { + try { + return JSON.parse(str); + } catch(e) { + return false; + } + } +}; + +exports.diffTime = function(time) { + var diff = process.hrtime(time); + return (diff[0] * 1E9 + diff[1])/(1E9 * 1.0); +} + +module.exports = exports; diff --git a/lib/services/wallet/validators.js b/lib/services/wallet/validators.js new file mode 100644 index 00000000..b2ef0bf1 --- /dev/null +++ b/lib/services/wallet/validators.js @@ -0,0 +1,246 @@ +'use strict'; + +var assert = require('assert'); + +var bitcore = require('bitcore-lib'); +var _ = require('lodash'); +var utils = require('./utils'); + +var MAX_INT = 0xffffffff; // Math.pow(2, 32) - 1 + +exports.sanitizeRangeOptions = function(options) { + if (!options) { + options = {}; + } + options.height = options.height || 0; + options.index = options.index || 0; + + if (!options.limit) { + options.limit = 10; + } else if (options.limit > 500) { + throw new Error('Limit exceeds maximum'); + } + + assert(bitcore.util.js.isNaturalNumber(options.height), '"height" is expected to be a natural number'); + assert(bitcore.util.js.isNaturalNumber(options.index), '"index" is expected to be a natural number'); + assert(bitcore.util.js.isNaturalNumber(options.limit), '"limit" is expected to be a natural number'); + + assert(options.limit <= 500, '"limit" exceeds maximum'); + + if (options.end) { + assert(bitcore.util.js.isNaturalNumber(options.end.height), '"end height" is expected to be a natural number'); + } + return options; +}; + +exports.checkRangeParams = function(req, res, next) { + assert(req.bitcoinHeight, '"bitcoinHeight" is expected to be set on the request'); + + var range = { + height: parseInt(req.query.height), + index: parseInt(req.query.index), + limit: parseInt(req.query.limit), + end: { + height: req.bitcoinHeight, + index: MAX_INT + } + }; + + if (req.query.end) { + range.end.height = parseInt(req.query.end) || req.bitcoinHeight; + } + + try { + range = exports.sanitizeRangeOptions(range); + } catch(e) { + return utils.sendError({ + message: 'Invalid params: ' + e.message, + statusCode: 400 + }, res); + } + + assert(range.height <= range.end.height, '\'Height\' param required to be less than \'End\' param.'); + req.range = range; + next(); +}; + +exports.checkAddress = function(req, res, next) { + var address; + var addressStr; + + if (req.body.address) { + addressStr = req.body.address; + } else { + addressStr = req.params.address; + } + + if(!addressStr) { + return utils.sendError({ + message: 'Address param is expected', + statusCode: 400 + }, res); + } + + assert(req.network, '"network" is expected to be set on the request'); + + try { + address = new bitcore.Address(addressStr, req.network); + } catch(e) { + return utils.sendError({ + message: 'Invalid address: ' + e.message, + statusCode: 400 + }, res); + } + + req.address = address; + next(); +}; + +//exports.checkAddresses = function(req, res, next) { +// var addresses = []; +// +// if (!req.body.addresses || !req.body.addresses.length || !Array.isArray(req.body.addresses)) { +// return utils.sendError({ +// message: 'Addresses param is expected', +// statusCode: 400 +// }, res); +// } +// +// assert(req.network, '"network" is expected to be set on the request'); +// +// for (var i = 0; i < req.body.addresses.length; i++) { +// var address; +// try { +// address = new bitcore.Address(req.body.addresses[i], req.network); +// } catch(e) { +// return utils.sendError({ +// message: 'Invalid address: ' + e.message, +// statusCode: 400 +// }, res); +// } +// addresses.push(address); +// } +// +// req.addresses = addresses; +// next(); +//}; + +exports.checkWalletId = function(req, res, next) { + + if (!req.params.walletId) { + return utils.sendError({ + message: 'Wallet id is expected', + statusCode: 400 + }, res); + } + + if (req.params.walletId.length !== 64 || !bitcore.util.js.isHexa(req.params.walletId)) { + return utils.sendError({ + message: 'Wallet id is expected to be a hexadecimal string with length of 64', + statusCode: 400 + }, res); + } + + req.walletId = new Buffer(req.params.walletId, 'hex'); + next(); + +}; + +exports.checkAddresses = function(req, res, next) { + + if (!req.file || !req.file.buffer) { + generateError(406, 'Content-Type must be set to multipart/form' + + ' and addresses key and value must be given.'); + return; + } + var buf = req.file.buffer; + var bufString = buf.toString(); + req.addresses = parse(bufString); + if (!req.addresses) { + generateError(415, 'Could not parse addresses buffer into something meaningful.'); + return; + } + next(); + + function generateError(status, msg) { + res.status(status).jsonp({ + error: msg + }); + } + + //we are able to deal with json/jsonl, possibly others + function parse(string) { + var ret = false; + var delims = [null, '\n', ' ']; + for(var i = 0; i < delims.length; i++) { + ret = utils.delimitedStringParse(delims[i], string); + if (_.isArray(ret)) { + return ret; + } + } + return ret; + } +}; + +exports.checkAuthHeaders = function(req, res) { + var identity = req.header('x-identity'); + var signature = req.header('x-signature'); + var nonce = req.header('x-nonce'); + if (identity && (identity.length > 130 || !bitcore.util.js.isHexa(identity))) { + utils.sendError({ + message: 'x-identity is expected to be a hexadecimal string with length of less than 131', + statusCode: 400 + }, res); + return false; + } + if (signature && (signature.length > 142 || !bitcore.util.js.isHexa(signature))) { + utils.sendError({ + message: 'x-signature is expected to be a hexadecimal string with length of less than 143', + statusCode: 400 + }, res); + return false; + } + if (nonce && (nonce.length > 128 || nonce.length % 2 !== 0 || !bitcore.util.js.isHexa(nonce))) { + utils.sendError({ + message: 'x-nonce is expected to be a hexadecimal string with length of less than 129', + statusCode: 400 + }, res); + return false; + } + return true; +}; + +exports.checkDate = function(dateStrings) { + var errors = []; + if (!Array.isArray(dateStrings)) { + dateStrings = [dateStrings]; + } + for(var i = 0; i < dateStrings.length; i++) { + internalDateCheck(dateStrings[i]); + } + + function internalDateCheck(dateString) { + var date = new Date(utils.toIntIfNumberLike(dateString)); + if (date.toString() === 'Invalid Date') { + errors.push('The date supplied: \'' + dateString + + '\' is not a valid date string. A valid date could be: \'2016-09-01\'.'); + } + } + return errors; +}; + +exports.checkDateFunction = function(callback) { + var self = this; + return function() { + var args = Array.prototype.slice.call(arguments); + var errors = self.checkDate([args[1], args[2]]); + if (errors.length > 0) { + args.unshift(errors); + } else { + args.unshift(null); + } + callback.apply(null, args); + }; +}; + +module.exports = exports; diff --git a/package.json b/package.json index 58ecf91f..8f5d6136 100644 --- a/package.json +++ b/package.json @@ -52,14 +52,18 @@ "commander": "^2.8.1", "errno": "^0.1.4", "express": "^4.13.3", + "leveldown": "", + "levelup": "^1.3.3", "liftoff": "^2.2.0", - "lmdb": "rvagg/lmdb#5ad819f77714925248dfecb0ba2174080dba949f", + "lodash": "^4.17.4", "lru-cache": "^4.0.1", "mkdirp": "0.5.0", + "multer": "^1.2.1", "path-is-absolute": "^1.0.0", "semver": "^5.0.1", "socket.io": "^1.4.5", "socket.io-client": "^1.4.5", + "ttyread": "^1.0.2", "zmq": "^2.14.0" }, "optionalDependencies": { From 44cb188c611fe13a15bf19a6d67f9e765481367b Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Wed, 11 Jan 2017 19:03:44 -0500 Subject: [PATCH 03/10] clean up. --- lib/services/wallet/index.js | 63 +++++------------------------------- 1 file changed, 8 insertions(+), 55 deletions(-) diff --git a/lib/services/wallet/index.js b/lib/services/wallet/index.js index 35ddca8b..590f9384 100644 --- a/lib/services/wallet/index.js +++ b/lib/services/wallet/index.js @@ -14,15 +14,7 @@ var storage = multer.memoryStorage(); var upload = multer({ storage: storage }); var validators = require('./validators'); var utils = require('./utils'); -/** - * The Address Service builds upon the Database Service and the Bitcoin Service to add additional - * functionality for getting information by base58check encoded addresses. This includes getting the - * balance for an address, the history for a collection of addresses, and unspent outputs for - * constructing transactions. This is typically the core functionality for building a wallet. - * @param {Object} options - * @param {Node} options.node - An instance of the node - * @param {String} options.name - An optional name of the service - */ + var WalletService = function(options) { BaseService.call(this, options); this._dbOptions = { @@ -39,23 +31,8 @@ WalletService.dependencies = [ 'web' ]; -/** - * Called by the Node to get the available API methods for this service, - * that can be exposed over the JSON-RPC interface. - */ WalletService.prototype.getAPIMethods = function() { - return [ - //['getWalletInfo', this, this.getInfo, 0] - //['getWalletBalance', this, this.getWalletBalance, 2], - //['getBlockTimestampInfo', this, this.getBlockTimestampInfo, 2], - //['addWalletAddresses', this, this., 2], - //['addWalletAddress', this, this.addWalletAddress, 2], - //['addWallet', this, this.addWallet, 2], - //['getWalletUtxos', this, this.getWalletUtxos, 2], - //['getWalletRawTransactions', this, this.getWalletRawTransactions, 1] - //['getWalletTxids', this, this.getWalletTxids, 1] - //['getWalletTransactions', this, this.getWalletTransactions, 1] - ]; + return []; }; WalletService.prototype.start = function(callback) { setImmediate(callback); @@ -65,9 +42,6 @@ WalletService.prototype.stop = function(callback) { setImmediate(callback); }; -/** - * Called by the Bus to get the available events for this service. - */ WalletService.prototype.getPublishEvents = function() { return []; }; @@ -127,6 +101,7 @@ WalletService.prototype._endpointGetAddresses = function() { }); }; }; + WalletService.prototype._endpointPostAddresses = function() { var self = this; return function(req, res) { @@ -154,6 +129,7 @@ WalletService.prototype._getUtxos = function(walletId, height, callback) { self.node.services.bitcoind.getAddressUnspentOutputs(addresses, {queryMempool: false}, callback); }); }; + WalletService.prototype._getBalance = function(walletId, height, callback) { // TODO get the balance only to this height var self = this; @@ -175,9 +151,11 @@ WalletService.prototype._getBalance = function(walletId, height, callback) { }); }); }; + WalletService.prototype._getAddresses = function(walletId, callback) { this._db.get(walletId, callback); }; + WalletService.prototype._storeAddresses = function(walletId, addresses, callback) { this._db.put(walletId, addresses, callback); }; @@ -193,21 +171,6 @@ WalletService.prototype.setupRoutes = function(app, express) { app.get('/info', s._endpointGetInfo() ); - //app.get('/wallets/:walletId/txids', - // v.checkWalletId, - // v.checkRangeParams, - // s._endpointTxids() - //); - //app.get('/wallets/:walletId/transactions', - // v.checkWalletId, - // v.checkRangeParams, - // s._endpointTransactions() - //); - //app.get('/wallets/:walletId/rawtransactions', - // v.checkWalletId, - // v.checkRangeParams, - // s._endpointRawTransactions() - //); app.get('/wallets/:walletId/utxos', s._endpointUTXOs() ); @@ -222,21 +185,11 @@ WalletService.prototype.setupRoutes = function(app, express) { v.checkAddresses, s._endpointPostAddresses() ); - //app.put('/wallets/:walletId', - // v.checkWalletId, - // s._endpointPutWallet() - //); - //app.get('/info/timestamps', - // s._endpointGetHeightsFromTimestamps() - //); - //app.get('/jobs/:jobId', - // s._endpointJobStatus() - //); - //app.use(s._endpointNotFound()); }; WalletService.prototype.getRoutePrefix = function() { - return 'wallet'; + return 'bws'; }; + module.exports = WalletService; From 789c18a6dfc8b7d33aebdcd085e301bb22938466 Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Thu, 12 Jan 2017 15:56:45 -0500 Subject: [PATCH 04/10] add put endpoint, rename module to wallet-api --- lib/services/{wallet => wallet-api}/index.js | 82 ++++++++++++++++--- lib/services/{wallet => wallet-api}/utils.js | 0 .../{wallet => wallet-api}/validators.js | 3 + 3 files changed, 74 insertions(+), 11 deletions(-) rename lib/services/{wallet => wallet-api}/index.js (71%) rename lib/services/{wallet => wallet-api}/utils.js (100%) rename lib/services/{wallet => wallet-api}/validators.js (99%) diff --git a/lib/services/wallet/index.js b/lib/services/wallet-api/index.js similarity index 71% rename from lib/services/wallet/index.js rename to lib/services/wallet-api/index.js index 590f9384..8885e7a1 100644 --- a/lib/services/wallet/index.js +++ b/lib/services/wallet-api/index.js @@ -14,6 +14,8 @@ var storage = multer.memoryStorage(); var upload = multer({ storage: storage }); var validators = require('./validators'); var utils = require('./utils'); +var _ = require('lodash'); +var bodyParser = require('body-parser'); var WalletService = function(options) { BaseService.call(this, options); @@ -50,11 +52,12 @@ WalletService.prototype._endpointUTXOs = function() { var self = this; return function(req, res) { var walletId = req.params.walletId; + var queryMempool = req.query.queryMempool === false ? false : true; //var tip = self.node.bitcoind.tip; // TODO: get the height of the tip //var height = tip; var height = null; - self._getUtxos(walletId, height, function(err, utxos) { + self._getUtxos(walletId, height, options, function(err, utxos) { if(err) { return utils.sendError(err); } @@ -70,11 +73,18 @@ WalletService.prototype._endpointGetBalance= function() { var self = this; return function(req, res) { var walletId = req.params.walletId; + var queryMempool = req.query.queryMempool === false ? false : true; + //var tip = self.node.bitcoind.tip; // TODO: get the height of the tip //var height = tip; var height = null; - self._getBalance(walletId, height, function(err, balance) { + + var options = { + queryMempool: queryMempool + }; + + self._getBalance(walletId, height, options, function(err, balance) { if(err) { return utils.sendError(err); } @@ -93,8 +103,13 @@ WalletService.prototype._endpointGetAddresses = function() { self._getAddresses(walletId, function(err, addresses) { if(err) { - return utils.sendError(err); + return utils.sendError(err, res); } + + if(!addresses) { + return res.status(404).send('Not found'); + } + res.status(200).jsonp({ addresses: addresses }); @@ -118,7 +133,45 @@ WalletService.prototype._endpointPostAddresses = function() { }; }; -WalletService.prototype._getUtxos = function(walletId, height, callback) { +WalletService.prototype._endpointPutAddresses = function() { + var self = this; + return function(req, res) { + var newAddresses = req.body; + + if(!Array.isArray(req.body)) { + return utils.sendError(new Error('Must PUT an array'), res); + } + + var walletId = req.params.walletId; + + self._getAddresses(walletId, function(err, oldAddresses) { + if(err) { + return utils.sendError(err, res); + } + + if(!oldAddresses) { + return res.status(404).send('Not found'); + } + + var allAddresses = _.union(oldAddresses, newAddresses); + + var amountAdded = allAddresses.length - oldAddresses.length; + + self._storeAddresses(walletId, allAddresses, function(err) { + if(err) { + return utils.sendError(err, res); + } + + res.status(200).jsonp({ + walletId: walletId, + amountAdded: amountAdded + }); + }); + }); + }; +}; + +WalletService.prototype._getUtxos = function(walletId, height, options, callback) { // TODO get the balance only to this height var self = this; self._getAddresses(walletId, function(err, addresses) { @@ -126,20 +179,20 @@ WalletService.prototype._getUtxos = function(walletId, height, callback) { return callback(err); } - self.node.services.bitcoind.getAddressUnspentOutputs(addresses, {queryMempool: false}, callback); + self.node.services.bitcoind.getAddressUnspentOutputs(addresses, options, callback); }); }; -WalletService.prototype._getBalance = function(walletId, height, callback) { +WalletService.prototype._getBalance = function(walletId, height, options, callback) { // TODO get the balance only to this height var self = this; self._getAddresses(walletId, function(err, addresses) { if(err) { return callback(err); } - self.node.services.bitcoind.getAddressUnspentOutputs(addresses, { - queryMempool: false - }, function(err, utxos) { + + + self.node.services.bitcoind.getAddressUnspentOutputs(addresses, options, function(err, utxos) { if(err) { return callback(err); } @@ -168,6 +221,9 @@ WalletService.prototype._endpointGetInfo = function() { WalletService.prototype.setupRoutes = function(app, express) { var s = this; var v = validators; + + app.use(bodyParser.json()); + app.get('/info', s._endpointGetInfo() ); @@ -180,15 +236,19 @@ WalletService.prototype.setupRoutes = function(app, express) { app.get('/wallets/:walletId', s._endpointGetAddresses() ); - app.post('/wallets/addresses', + app.put('/wallets/:walletId/addresses', + s._endpointPutAddresses() + ); + app.post('/wallets', upload.single('addresses'), v.checkAddresses, s._endpointPostAddresses() ); + }; WalletService.prototype.getRoutePrefix = function() { - return 'bws'; + return 'wallet-api'; }; module.exports = WalletService; diff --git a/lib/services/wallet/utils.js b/lib/services/wallet-api/utils.js similarity index 100% rename from lib/services/wallet/utils.js rename to lib/services/wallet-api/utils.js diff --git a/lib/services/wallet/validators.js b/lib/services/wallet-api/validators.js similarity index 99% rename from lib/services/wallet/validators.js rename to lib/services/wallet-api/validators.js index b2ef0bf1..3b3fdac7 100644 --- a/lib/services/wallet/validators.js +++ b/lib/services/wallet-api/validators.js @@ -147,6 +147,9 @@ exports.checkWalletId = function(req, res, next) { }; exports.checkAddresses = function(req, res, next) { + if(req.body) { + return next(); + } if (!req.file || !req.file.buffer) { generateError(406, 'Content-Type must be set to multipart/form' + From 57b3f5d67f89a04591d7cfc30419476d89726448 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 13 Jan 2017 13:44:38 -0500 Subject: [PATCH 05/10] Added GetTransactions --- lib/services/wallet-api/index.js | 66 +++++++++++++++++++++++++++ lib/services/wallet-api/validators.js | 4 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index 8885e7a1..1f4a3c34 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -1,5 +1,6 @@ 'use strict'; +var async = require('async'); var BaseService = require('../../service'); var inherits = require('util').inherits; var index = require('../../'); @@ -133,6 +134,29 @@ WalletService.prototype._endpointPostAddresses = function() { }; }; +WalletService.prototype._endpointGetTransactions = function() { + var self = this; + return function(req, res) { + req.setTimeout(600000); + var walletId = req.params.walletId; + var options = { + start: req.query.start, + end : req.query.end, + from: req.query.from, + to: req.query.to + }; + self._getTransactions(walletId, options, function(err, transactions, totalCount) { + if(err) { + return utils.sendError(err, res); + } + res.status(200).jsonp({ + transactions: transactions, + totalCount: totalCount + }); + }); + }; +}; + WalletService.prototype._endpointPutAddresses = function() { var self = this; return function(req, res) { @@ -205,6 +229,45 @@ WalletService.prototype._getBalance = function(walletId, height, options, callba }); }; +WalletService.prototype._chunkAdresses = function(addresses) { + var maxLength = this.node.services.bitcoind.maxAddressesQuery; + var groups = []; + var groupsCount = Math.ceil(addresses.length / maxLength); + for(var i = 0; i < groupsCount; i++) { + groups.push(addresses.slice(i * maxLength, Math.min(maxLength * (i + 1), addresses.length))); + } + return groups; +}; + +WalletService.prototype._getTransactions = function(walletId, options, callback) { + var self = this; + self._getAddresses(walletId, function(err, addresses) { + if(err) { + return callback(err); + } + if (!addresses) { + return callback(new Error('wallet not found')); + } + var addressGroups = self._chunkAdresses(addresses); + var transactions = []; + async.eachSeries(addressGroups, function(addresses, next) { + self.node.services.bitcoind.getAddressHistory(addresses, options, function(err, history) { + if(err) { + return callback(err); + } + var groupTransactions = history.items.map(function(item) { + return item.tx; + }); + transactions = _.union(transactions, groupTransactions); + next(); + }); + + }, function(err) { + callback(null, transactions, transactions.length); + }); + }); +}; + WalletService.prototype._getAddresses = function(walletId, callback) { this._db.get(walletId, callback); }; @@ -239,6 +302,9 @@ WalletService.prototype.setupRoutes = function(app, express) { app.put('/wallets/:walletId/addresses', s._endpointPutAddresses() ); + app.get('/wallets/:walletId/transactions', + s._endpointGetTransactions() + ); app.post('/wallets', upload.single('addresses'), v.checkAddresses, diff --git a/lib/services/wallet-api/validators.js b/lib/services/wallet-api/validators.js index 3b3fdac7..2377ee87 100644 --- a/lib/services/wallet-api/validators.js +++ b/lib/services/wallet-api/validators.js @@ -147,7 +147,9 @@ exports.checkWalletId = function(req, res, next) { }; exports.checkAddresses = function(req, res, next) { - if(req.body) { + + if (!req.file && req.body) { + req.addresses = req.body; return next(); } From af2b75c0d0e287c3c10e94d7889050544d8bbbe3 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 13 Jan 2017 14:27:15 -0500 Subject: [PATCH 06/10] Added pagination. --- lib/services/wallet-api/index.js | 71 +++++++++++++++++++++++--------- package.json | 2 +- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index 1f4a3c34..d3b3c649 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -17,6 +17,7 @@ var validators = require('./validators'); var utils = require('./utils'); var _ = require('lodash'); var bodyParser = require('body-parser'); +var LRU = require('lru-cache'); var WalletService = function(options) { BaseService.call(this, options); @@ -25,6 +26,13 @@ var WalletService = function(options) { valueEncoding: 'json' }; this._db = levelup(options.dbPath, this._dbOptions); + this._cache = LRU({ + max: 500 * 1024 * 1024, + length: function(n, key) { + return Buffer.byteLength(n, 'utf8'); + }, + maxAge: 30 * 60 * 1000 + }); }; inherits(WalletService, BaseService); @@ -241,31 +249,54 @@ WalletService.prototype._chunkAdresses = function(addresses) { WalletService.prototype._getTransactions = function(walletId, options, callback) { var self = this; - self._getAddresses(walletId, function(err, addresses) { - if(err) { - return callback(err); - } - if (!addresses) { - return callback(new Error('wallet not found')); - } - var addressGroups = self._chunkAdresses(addresses); - var transactions = []; - async.eachSeries(addressGroups, function(addresses, next) { - self.node.services.bitcoind.getAddressHistory(addresses, options, function(err, history) { + var transactions = []; + var opts = { + start: options.start, + end: options.end + }; + var key = walletId + opts.start + opts.end; + if (!self._cache.peek(key)) { + self._getAddresses(walletId, function(err, addresses) { + if(err) { + return callback(err); + } + if (!addresses) { + return callback(new Error('wallet not found')); + } + var addressGroups = self._chunkAdresses(addresses); + async.eachSeries(addressGroups, function(addresses, next) { + self.node.services.bitcoind.getAddressHistory(addresses, opts, function(err, history) { + if(err) { + return next(err); + } + var groupTransactions = history.items.map(function(item) { + return item.tx; + }); + transactions = _.union(transactions, groupTransactions); + next(); + }); + }, function(err) { if(err) { return callback(err); } - var groupTransactions = history.items.map(function(item) { - return item.tx; - }); - transactions = _.union(transactions, groupTransactions); - next(); + self._cache.set(key, JSON.stringify(transactions)); + finish(); }); - - }, function(err) { - callback(null, transactions, transactions.length); }); - }); + } else { + try { + transactions = JSON.parse(self._cache.get(key)); + finish(); + } catch(e) { + self._cache.del(key); + return callback(e); + } + } + function finish() { + var from = options.from || 0; + var to = options.to || transactions.length; + callback(null, transactions.slice(from, to), transactions.length); + } }; WalletService.prototype._getAddresses = function(walletId, callback) { diff --git a/package.json b/package.json index 8f5d6136..6cfd90ea 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "levelup": "^1.3.3", "liftoff": "^2.2.0", "lodash": "^4.17.4", - "lru-cache": "^4.0.1", + "lru-cache": "^4.0.2", "mkdirp": "0.5.0", "multer": "^1.2.1", "path-is-absolute": "^1.0.0", From 05ddd43dfcd51ce781eda9c51066ec9b3b780948 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Fri, 13 Jan 2017 14:45:36 -0500 Subject: [PATCH 07/10] Fixed options on getUtxos --- lib/services/wallet-api/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index d3b3c649..1da0615f 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -66,6 +66,10 @@ WalletService.prototype._endpointUTXOs = function() { // TODO: get the height of the tip //var height = tip; var height = null; + + var options = { + queryMempool: queryMempool + }; self._getUtxos(walletId, height, options, function(err, utxos) { if(err) { return utils.sendError(err); From 3f558f6acee4ce940df2a2579f10e6239b74769a Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 16 Jan 2017 19:23:17 -0500 Subject: [PATCH 08/10] Added balance by address. --- lib/services/wallet-api/index.js | 39 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index 1da0615f..457964d7 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -87,6 +87,7 @@ WalletService.prototype._endpointGetBalance= function() { return function(req, res) { var walletId = req.params.walletId; var queryMempool = req.query.queryMempool === false ? false : true; + var byAddress = req.query.byAddress; //var tip = self.node.bitcoind.tip; // TODO: get the height of the tip @@ -94,17 +95,15 @@ WalletService.prototype._endpointGetBalance= function() { var height = null; var options = { - queryMempool: queryMempool + queryMempool: queryMempool, + byAddress: byAddress }; - self._getBalance(walletId, height, options, function(err, balance) { + self._getBalance(walletId, height, options, function(err, result) { if(err) { return utils.sendError(err); } - res.status(200).jsonp({ - balance: balance, - height: height - }); + res.status(200).jsonp(result); }); }; }; @@ -227,20 +226,44 @@ WalletService.prototype._getBalance = function(walletId, height, options, callba return callback(err); } - self.node.services.bitcoind.getAddressUnspentOutputs(addresses, options, function(err, utxos) { if(err) { return callback(err); } + if (options.byAddress) { + var result = self._getBalanceByAddress(addresses, utxos); + return callback(null, { + addresses: result, + height: null + }); + } var balance = 0; utxos.forEach(function(utxo) { balance += utxo.satoshis; }); - callback(null, balance); + callback(null, { + balance: balance, + height: null + }); }); }); }; +WalletService.prototype._getBalanceByAddress = function(addresses, utxos) { + var res = {}; + utxos.forEach(function(utxo) { + if (res[utxo.address]) { + res[utxo.address] += utxo.satoshis; + } else { + res[utxo.address] = utxo.satoshis; + } + }); + addresses.forEach(function(address) { + res[address] = res[address] || 0; + }); + return res; +}; + WalletService.prototype._chunkAdresses = function(addresses) { var maxLength = this.node.services.bitcoind.maxAddressesQuery; var groups = []; From d115580ae984dd33a08530c06c3ed3c1338e6c37 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 16 Jan 2017 19:32:30 -0500 Subject: [PATCH 09/10] Fixed sendError call. --- lib/services/wallet-api/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index 457964d7..abfe3233 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -72,7 +72,7 @@ WalletService.prototype._endpointUTXOs = function() { }; self._getUtxos(walletId, height, options, function(err, utxos) { if(err) { - return utils.sendError(err); + return utils.sendError(err, res); } res.status(200).jsonp({ utxos: utxos, @@ -101,7 +101,7 @@ WalletService.prototype._endpointGetBalance= function() { self._getBalance(walletId, height, options, function(err, result) { if(err) { - return utils.sendError(err); + return utils.sendError(err, res); } res.status(200).jsonp(result); }); From 885023894479e0d604c4ea1068b7fe077350a574 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 16 Jan 2017 19:41:27 -0500 Subject: [PATCH 10/10] Increased timeout for endpoints. --- lib/services/wallet-api/index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index abfe3233..a907e3c8 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -60,6 +60,7 @@ WalletService.prototype.getPublishEvents = function() { WalletService.prototype._endpointUTXOs = function() { var self = this; return function(req, res) { + req.setTimeout(600000); var walletId = req.params.walletId; var queryMempool = req.query.queryMempool === false ? false : true; //var tip = self.node.bitcoind.tip; @@ -85,6 +86,7 @@ WalletService.prototype._endpointUTXOs = function() { WalletService.prototype._endpointGetBalance= function() { var self = this; return function(req, res) { + req.setTimeout(600000); var walletId = req.params.walletId; var queryMempool = req.query.queryMempool === false ? false : true; var byAddress = req.query.byAddress;