From 3fe2c3ea1600bf72cc7b401c9dbda5825050143a Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Mon, 1 May 2017 11:22:46 -0400 Subject: [PATCH] wip --- lib/services/address/index.js | 12 - lib/services/db/index.js | 68 ++-- lib/services/wallet-api/index.js | 406 ++++++++++++++---------- lib/services/web/index.js | 1 + lib/utils.js | 8 +- regtest/utils.js | 349 ++++++++++++++++++++ regtest/wallet.js | 527 ++++++++----------------------- 7 files changed, 758 insertions(+), 613 deletions(-) create mode 100644 regtest/utils.js diff --git a/lib/services/address/index.js b/lib/services/address/index.js index 4865f3fe..783764b0 100644 --- a/lib/services/address/index.js +++ b/lib/services/address/index.js @@ -28,23 +28,11 @@ var utils = require('../../utils'); var AddressService = function(options) { BaseService.call(this, options); - // this.subscriptions = {}; - // this.subscriptions['address/transaction'] = {}; - // this.subscriptions['address/balance'] = {}; - - // this._bitcoindTransactionListener = this.transactionHandler.bind(this); - // this._bitcoindTransactionLeaveListener = this.transactionLeaveHandler.bind(this); - // this.node.services.bitcoind.on('tx', this._bitcoindTransactionListener); - // this.node.services.bitcoind.on('txleave', this._bitcoindTransactionLeaveListener); - this.maxInputsQueryLength = options.maxInputsQueryLength || constants.MAX_INPUTS_QUERY_LENGTH; this.maxOutputsQueryLength = options.maxOutputsQueryLength || constants.MAX_OUTPUTS_QUERY_LENGTH; this.concurrency = options.concurrency || 20; - // this.mempoolIndex = null; // Used for larger mempool indexes - // this.mempoolSpentIndex = {}; // Used for small quick synchronous lookups - // this.mempoolAddressIndex = {}; // Used to check if an address is on the spend pool }; inherits(AddressService, BaseService); diff --git a/lib/services/db/index.js b/lib/services/db/index.js index 16120197..011df27e 100644 --- a/lib/services/db/index.js +++ b/lib/services/db/index.js @@ -161,12 +161,15 @@ DB.prototype.start = function(callback) { }); }); - self._sync.on('synced', function() { + self._sync.once('synced', function() { + self.syncing = false; + self.node.services.bitcoind.on('tip', function(height) { log.info('New tip at height: ' + height + ' hash: ' + self.node.services.bitcoind.tiphash); self._sync.sync(); }); + log.info('Initial sync complete'); }); @@ -185,7 +188,6 @@ DB.prototype.start = function(callback) { self.loadTip(self.loadConcurrentTip.bind(self, finish)); }); - //TODO remove! setImmediate(function() { self._checkVersion(self._setVersion.bind(self, callback)); }); @@ -215,11 +217,12 @@ DB.prototype.loadTip = function(callback) { var self = this; self.store.get(self.dbPrefix + 'tip', self.dbOptions, function(err, tipData) { + if(err && err instanceof levelup.errors.NotFoundError) { + self.tip = self.genesis; self.tip.__height = 0; - // we need to wait for all the services to become ready, - // then we can proceed with connecting blocks here + self.connectBlock(self.genesis, function(err) { if(err) { return callback(err); @@ -228,40 +231,41 @@ DB.prototype.loadTip = function(callback) { self.emit('addblock', self.genesis); callback(); }); - return; + } else if(err) { return callback(err); - } + } else { - var hash = tipData.slice(0, 32).toString('hex'); - var height = tipData.readUInt32BE(32); + var hash = tipData.slice(0, 32).toString('hex'); + var height = tipData.readUInt32BE(32); - var times = 0; - async.retry({times: 3, interval: self.retryInterval}, function(done) { - self.node.services.bitcoind.getBlock(hash, function(err, tip) { - if(err) { - times++; - log.warn('Bitcoind does not have our tip (' + hash + '). Bitcoind may have crashed and needs to catch up.'); - if(times < 3) { - log.warn('Retrying in ' + (self.retryInterval / 1000) + ' seconds.'); + var times = 0; + async.retry({times: 3, interval: self.retryInterval}, function(done) { + self.node.services.bitcoind.getBlock(hash, function(err, tip) { + if(err) { + times++; + log.warn('Bitcoind does not have our tip (' + hash + '). Bitcoind may have crashed and needs to catch up.'); + if(times < 3) { + log.warn('Retrying in ' + (self.retryInterval / 1000) + ' seconds.'); + } + return done(err); } - return done(err); + + done(null, tip); + }); + }, function(err, tip) { + if(err) { + log.warn('Giving up after 3 tries. Please report this bug to https://github.com/bitpay/bitcore-node/issues'); + log.warn('Please reindex your database.'); + return callback(err); } - done(null, tip); + tip.__height = height; + self.tip = tip; + + callback(); }); - }, function(err, tip) { - if(err) { - log.warn('Giving up after 3 tries. Please report this bug to https://github.com/bitpay/bitcore-node/issues'); - log.warn('Please reindex your database.'); - return callback(err); - } - - tip.__height = height; - self.tip = tip; - - callback(); - }); + } }); }; @@ -433,6 +437,10 @@ DB.prototype.getSerialBlockOperations = function(block, add, callback) { async.eachSeries( this.node.services, function(mod, next) { +//console.log('s***********************'); +//console.log('here'); +//console.log(mod.name, block.__height); +//console.log('e***********************'); if(mod.blockHandler) { $.checkArgument(typeof mod.blockHandler === 'function', 'blockHandler must be a function'); diff --git a/lib/services/wallet-api/index.js b/lib/services/wallet-api/index.js index a30a1e74..8e4977f2 100644 --- a/lib/services/wallet-api/index.js +++ b/lib/services/wallet-api/index.js @@ -85,64 +85,61 @@ WalletService.prototype.getPublishEvents = function() { }; -WalletService.prototype.getAddressString = function(script, output) { - var address = script.toAddress(); +WalletService.prototype.getAddressString = function(io) { + + var address = io.script.toAddress(this.node.network); + if(address) { return address.toString(); } try { - var pubkey = script.getPublicKey(); + var pubkey = io.script.getPublicKey(); if(pubkey) { return pubkey.toString('hex'); } - } catch(e) { - //log.warn('Error getting public key from: ', script.toASM(), script.toHex()); - // if there is an error, it's because a pubkey can not be extracted from the script - // continue on and return null - } + } catch(e) {} - //TODO add back in P2PK, but for this we need to look up the utxo for this script - if(output && output.script && output.script.isPublicKeyOut()) { - return output.script.getPublicKey().toString('hex'); - } +}; - //log.warn('No utxo given for script spending a P2PK: ', script.toASM(), script.toHex()); - return null; +WalletService.prototype._checkAddresses = function() { + return Object.keys(this._addressMap).length > 0; }; WalletService.prototype.blockHandler = function(block, connectBlock, callback) { var opts = { block: block, connectBlock: connectBlock, - fnProcessIO: this._processSerialIO, serial: true }; -console.log(block.__height); this._blockHandler(opts, callback); }; WalletService.prototype.concurrentBlockHandler = function(block, connectBlock, callback) { var opts = { block: block, - connectBlock: connectBlock, - fnProcessIO: this._processConcurrentIO + connectBlock: connectBlock }; this._blockHandler(opts, callback); }; WalletService.prototype._blockHandler = function(opts, callback) { + var self = this; + if (!self._checkAddresses()) { + return setImmediate(function() { + callback(null, []); + }); + } - var txs = opts.block.transactions; - - async.mapSeries(txs, function(tx, next) { + async.mapSeries(opts.block.transactions, function(tx, next) { self._processTransaction(opts, tx, next); }, function(err, operations) { if(err) { return callback(err); } - callback(null, _.compact(operations)); + var ret = _.compact(_.flattenDeep(operations)); + callback(null, ret); }); }; @@ -150,10 +147,6 @@ WalletService.prototype._blockHandler = function(opts, callback) { WalletService.prototype._processTransaction = function(opts, tx, callback) { var self = this; - if(tx.isCoinbase()) { - return callback(); - } - tx.outputs.forEach(function(output, index) { output.index = index; }); @@ -170,23 +163,25 @@ WalletService.prototype._processTransaction = function(opts, tx, callback) { if(err) { return callback(err); } - callback(null, _.compact(operations)); + callback(null, operations); }); }; WalletService.prototype._processConcurrentIO = function(opts, tx, io, callback) { - var walletIds = this._getWalletIdsFromScript(io.script); + var self = this; + var walletIds = self._getWalletIdsFromScript(io); + if (!walletIds) { return callback(); } - var actions = this._getActions(opts.connectBlock); + var actions = self._getActions(opts.connectBlock); - var operations = walletIds.forEach(function(walletId) { + var operations = walletIds.map(function(walletId) { return { type: actions[0], - key: this._encoding.encodeWalletTransactionKey(walletId, opts.block.__height, tx.id) + key: self._encoding.encodeWalletTransactionKey(walletId, opts.block.__height, tx.id) }; }); @@ -196,29 +191,23 @@ WalletService.prototype._processConcurrentIO = function(opts, tx, io, callback) }; - WalletService.prototype._processSerialIO = function(opts, tx, io, callback) { var fn = this._processSerialOutput; if (io instanceof Input) { fn = this._processSerialInput; } - fn(opts, tx. io, callback); + fn.call(this, opts, tx, io, callback); }; -WalletService.prototype._getWalletIdsFromScript= function(script) { +WalletService.prototype._getWalletIdsFromScript = function(io) { - if(!script) { + if(!io.script) { log.debug('Invalid script'); return; } - var address = this.getAddressString(script); + return this._addressMap[this.getAddressString(io)]; - if(!address || !this._addressMap[address]) { - return; - } - - return this._addressMap[address]; }; WalletService.prototype._getActions = function(connect) { @@ -234,43 +223,45 @@ WalletService.prototype._getActions = function(connect) { WalletService.prototype._processSerialOutput = function(opts, tx, output, callback) { var self = this; - var walletIds = self._getWalletIdsFromScript(output.script); + var walletIds = self._getWalletIdsFromScript(output); if (!walletIds) { return callback(); } var actions = self._getActions(opts.connectBlock); - var walletIdsNeedingUpdate = {}; - async.mapSeries(walletIds, function(walletId, next) { - walletIdsNeedingUpdate[walletId] = true; + self.balances[walletId] = self.balances[walletId] || 0; + self.balances[walletId] += opts.connectBlock ? output.satoshis : (-1 * output.satoshis); - var operations = [{ - type: actions[0], - key: self._encoding.encodeWalletUtxoKey(walletId, tx.id, output.index), - value: self._encoding.encodeWalletUtxoValue(opts.block.__height, output.satoshis, output._scriptBuffer) - }, - { - type: actions[0], - key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, output.satoshis, tx.id, output.index), - value: self._encoding.encodeWalletUtxoSatoshisValue(opts.block.__height, output._scriptBuffer) - }]; - - if(opts.connectBlock) { - self.balances[walletId] += output.satoshis; - } else { - self.balances[walletId] -= output.satoshis; - } + var operations = [ + { + type: actions[0], + key: self._encoding.encodeWalletUtxoKey(walletId, tx.id, output.index), + value: self._encoding.encodeWalletUtxoValue(opts.block.__height, output.satoshis, output._scriptBuffer) + }, + { + type: actions[0], + key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, output.satoshis, tx.id, output.index), + value: self._encoding.encodeWalletUtxoSatoshisValue(opts.block.__height, output._scriptBuffer) + }, + { + type: 'put', + key: self._encoding.encodeWalletBalanceKey(walletId), + value: self._encoding.encodeWalletBalanceValue(self.balances[walletId]) + } + ]; next(null, operations); + }, function(err, operations) { if(err) { return callback(err); } + callback(null, operations); }); @@ -281,19 +272,20 @@ WalletService.prototype._processSerialInput = function(opts, tx, input, callback var self = this; - var actions = self._getActions(opts.connectBlock); + //we may not have walletIds to update but this input may be spending a pay-to-pub-key utxo + //so we must get the spending tx to check on that + var walletIds = input.script && input.script.isPublicKeyIn() ? + ['p2pk'] : + self._getWalletIdsFromScript(input); - var walletIds = self._getWalletIdsFromScript(input.script); - - if (walletIds) { + if (!walletIds) { return callback(); } - var walletIdsNeedingUpdate = []; + + var actions = self._getActions(opts.connectBlock); async.mapSeries(walletIds, function(walletId, next) { - walletIdsNeedingUpdate[walletId] = true; - self.node.services.transaction.getTransaction(input.prevTxId, {}, function(err, tx) { if(err) { @@ -302,7 +294,22 @@ WalletService.prototype._processSerialInput = function(opts, tx, input, callback var utxo = tx.outputs[input.outputIndex]; - var operations = [{ + if (walletId === 'p2pk') { + + var pubKey = utxo.script.getPublicKey().toString('hex'); + walletId = self._addressMap[pubKey]; + + if (!walletId) { + return next(); + } + + } + + self.balances[walletId] = self.balances[walletId] || 0; + self.balances[walletId] += opts.connectBlock ? (-1 * utxo.satoshis) : utxo.satoshis; + + var operations = [ + { type: actions[1], key: self._encoding.encodeWalletUtxoKey(walletId, input.prevTxId, input.outputIndex), value: self._encoding.encodeWalletUtxoValue(tx.__height, utxo.satoshis, utxo._scriptBuffer) @@ -311,13 +318,13 @@ WalletService.prototype._processSerialInput = function(opts, tx, input, callback type: actions[1], key: self._encoding.encodeWalletUtxoSatoshisKey(walletId, utxo.satoshis, tx.id, input.outputIndex), value: self._encoding.encodeWalletUtxoSatoshisValue(tx.__height, utxo._scriptBuffer) - }]; - - if(self.connectBlock) { - self.balances[walletId] -= utxo.satoshis; - } else { - self.balances[walletId] += utxo.satoshis; - } + }, + { + type: 'put', + key: self._encoding.encodeWalletBalanceKey(walletId), + value: self._encoding.encodeWalletBalanceValue(self.balances[walletId]) + } + ]; next(null, operations); @@ -327,6 +334,7 @@ WalletService.prototype._processSerialInput = function(opts, tx, input, callback if(err) { return callback(err); } + callback(null, operations); }); @@ -589,6 +597,7 @@ WalletService.prototype._endpointRegisterWallet = function() { WalletService.prototype._endpointPostAddresses = function() { var self = this; return function(req, res) { + var addresses = req.addresses; if (!addresses || !addresses.length) { return utils.sendError(new Error('addresses are required when creating a wallet.'), res); @@ -602,7 +611,9 @@ WalletService.prototype._endpointPostAddresses = function() { } var jobId = utils.generateJobId(); + self._importAddresses(walletId, addresses, jobId, self._jobCompletionCallback.bind(self)); + res.status(200).jsonp({jobId: jobId}); }; }; @@ -634,7 +645,9 @@ WalletService.prototype._endpointGetTransactions = function() { var rs = new Readable(); transactions.forEach(function(transaction) { - rs.push(utils.toJSONL(self._formatTransaction(transaction))); + if (transaction) { + rs.push(utils.toJSONL(self._formatTransaction(transaction))); + } }); rs.push(null); @@ -729,101 +742,150 @@ WalletService.prototype._getUtxos = function(walletId, options, callback) { }; WalletService.prototype._getBalance = function(walletId, options, callback) { + var self = this; var key = self._encoding.encodeWalletBalanceKey(walletId); self.store.get(key, function(err, buffer) { + if(err) { return callback(err); } + callback(null, self._encoding.decodeWalletBalanceValue(buffer)); }); + }; WalletService.prototype._chunkAdresses = function(addresses) { + var maxLength = this.node.services.bitcoind.maxAddressesQuery; var groups = []; var groupsCount = Math.ceil(addresses.length / maxLength); + for(var i = 0; i < groupsCount; i++) { groups.push(addresses.slice(i * maxLength, Math.min(maxLength * (i + 1), addresses.length))); } + return groups; + +}; + +WalletService.prototype._getSearchParams = function(fn, options) { + return { + gte: fn.call(this, options.walletId, options.start), + lt: Buffer.concat([ fn.call(this, options.walletId, options.end).slice(0, -32), new Buffer('ff', 'hex') ]) + }; +}; + +WalletService.prototype._getTxidsFromDb = function(options, callback) { + + var self = this; + var txids = []; + var encodingFn = self._encoding.encodeWalletTransactionKey.bind(self._encoding); + + var stream = self.store.createKeyStream(self._getSearchParams(encodingFn, options)); + + var streamErr; + stream.on('error', function(err) { + streamErr = err; + }); + + stream.on('data', function(data) { + txids.push(self._encoding.decodeWalletTransactionKey(data).txid); + }); + + stream.on('end', function() { + self._getTransactionsFromDb(txids, options, callback); + }); + +}; + +WalletService.prototype._getTransactionsFromDb = function(txids, options, callback) { + + var self = this; + async.mapLimit(txids, 10, function(txid, next) { + + self.node.services.transaction.getTransaction(txid, options, next); + + }, function(err, txs) { + + if(err) { + return callback(err); + } + + self._cache.set(options.key, JSON.stringify(self._formatTransactions(txs))); + + if (!options.queryMempool) { + + options.to = options.to || txs.length; + return callback(null, txs.slice(options.from, options.to), txs.length); + + } + + options.txs = txs; + self._getTransactionsFromMempool(options, callback); + + }); +}; + +WalletService.prototype._getTransactionsFromMempool = function(options, callback) { + + var self = this; + + self._getAddresses(options.walletId, function(err, addresses) { + + if(err) { + return callback(err); + } + + self.mempool.getTransactionsByAddresses(addresses, function(err, mempoolTxs) { + if(err) { + return callback(err); + } + + var txs = options.txs.concat(mempoolTxs); + callback(null, txs.slice(options.from, options.to), txs.length); + + }); + }); }; WalletService.prototype._getTransactions = function(walletId, options, callback) { + var self = this; - var txids = []; + + var start = options.start || 0; + var end = options.end || 0xffffffff; + var opts = { - start: options.start || 0, - end: options.end || Math.pow(2, 32) - 1 + start: start, + end: end, + from: options.from || 0, + to: options.to || 0, + walletId: walletId, + key: walletId + start + end }; - var key = walletId + opts.start + opts.end; - var transactions; - function finish(transactions) { - var from = options.from || 0; - var to = options.to || transactions.length; - if (!options.queryMempool) { - return callback(null, transactions.slice(from, to), transactions.length); - } - self._getAddresses(walletId, function(err, addresses) { - if(err) { - return callback(err); - } - self.mempool.getTransactionsByAddresses(addresses, function(err, mempoolTxs) { - if(err) { - return callback(err); - } - transactions = transactions.concat(mempoolTxs); - callback(null, transactions.slice(from, to), transactions.length); - }); - }); + if (!self._cache.peek(opts.key)) { + + return self._getTxidsFromDb(opts, callback); + } - function mapTxids(txids) { - async.mapLimit(txids, 10, function(txid, next) { - self.node.services.transaction.getTransaction(txid, options, next); - }, function(err, transactions) { - if(err) { - return callback(err); - } - self._cache.set(key, JSON.stringify(self._formatTransactions(transactions))); - finish(transactions); - }); + try { + + opts.txs = JSON.parse(self._cache.get(opts.key)); + self._getTransactionsFromMempool(opts, callback); + + } catch(e) { + + self._cache.del(opts.key); + return callback(e); + } - if (!self._cache.peek(key)) { - var start = self._encoding.encodeWalletTransactionKey(walletId, opts.start); - var end = Buffer.concat([ - self._encoding.encodeWalletTransactionKey(walletId, opts.end) - .slice(0, -32), new Buffer('ff', 'hex') ]); - var stream = self.store.createKeyStream({ - gte: start, - lte: end - }); - - var streamErr; - stream.on('error', function(err) { - streamErr = err; - }); - - stream.on('data', function(data) { - txids.push(self._encoding.decodeWalletTransactionKey(data).txid); - }); - - stream.on('end', function() { - mapTxids(txids); - }); - } else { - try { - transactions = JSON.parse(self._cache.get(key)); - finish(transactions); - } catch(e) { - self._cache.del(key); - return callback(e); - } - } }; WalletService.prototype._removeWallet = function(walletId, callback) { @@ -973,7 +1035,6 @@ WalletService.prototype._jobCompletionCallback = function(err, results) { job.reported = false; }; -//TODO: if this is running as a job, then the whole process can be moved to another CPU WalletService.prototype._importAddresses = function(walletId, addresses, jobId, callback) { var self = this; @@ -1219,16 +1280,18 @@ WalletService.prototype._endpointJobStatus = function() { }; WalletService.prototype._endpointGetInfo = function() { + var self = this; return function(req, res) { - res.jsonp({result: 'ok'}); + res.jsonp({ + result: 'ok', + height: self.node.services.db.tip.__height, + hash: self.node.services.db.tip.hash + }); }; }; -WalletService.prototype.setupRoutes = function(app) { +WalletService.prototype._setupReadOnlyRoutes = function(app) { var s = this; - var v = validators; - - app.use(bodyParser.json()); app.get('/info', s._endpointGetInfo() @@ -1245,23 +1308,6 @@ WalletService.prototype.setupRoutes = function(app) { app.get('/wallets/:walletId', s._endpointGetAddresses() ); - app.post('/wallets/:walletId', - s._endpointRegisterWallet() - ); - app.delete('/wallets/:walletId', - s._endpointRemoveWallet() - ); - app.delete('/wallets/', - s._endpointRemoveAllWallets() - ); - app.put('/wallets/:walletId/addresses', - s._endpointPutAddresses() - ); - app.post('/wallets/:walletId/addresses', - upload.single('addresses'), - v.checkAddresses, - s._endpointPostAddresses() - ); app.get('/wallets/:walletId/transactions', s._endpointGetTransactions() ); @@ -1279,6 +1325,38 @@ WalletService.prototype.setupRoutes = function(app) { ); }; +WalletService.prototype._setupWriteRoutes = function(app) { + var s = this; + var v = validators; + + app.post('/wallets/:walletId', + s._endpointRegisterWallet() + ); + app.delete('/wallets/:walletId', + s._endpointRemoveWallet() + ); + app.delete('/wallets/', + s._endpointRemoveAllWallets() + ); + app.put('/wallets/:walletId/addresses', + s._endpointPutAddresses() + ); + app.post('/wallets/:walletId/addresses', + upload.single('addresses'), + v.checkAddresses, + s._endpointPostAddresses() + ); +}; + + +WalletService.prototype.setupRoutes = function(app) { + + app.use(bodyParser.json()); + this._setupReadOnlyRoutes(app); + this._setupWriteRoutes(app); + +}; + WalletService.prototype.getRoutePrefix = function() { return 'wallet-api'; }; diff --git a/lib/services/web/index.js b/lib/services/web/index.js index ab6d8c07..443c913b 100644 --- a/lib/services/web/index.js +++ b/lib/services/web/index.js @@ -50,6 +50,7 @@ var WebService = function(options) { self.server.listen(self.port); self.createMethodsMap(); }); + BaseService.call(this, options); }; inherits(WebService, BaseService); diff --git a/lib/utils.js b/lib/utils.js index 9512f2ee..97d5928f 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -3,11 +3,11 @@ var MAX_SAFE_INTEGER = 0x1fffffffffffff; // 2 ^ 53 - 1 var utils = {}; -utils.isHash = function isHash(value) { +utils.isHash = function(value) { return typeof value === 'string' && value.length === 64 && /^[0-9a-fA-F]+$/.test(value); }; -utils.isSafeNatural = function isSafeNatural(value) { +utils.isSafeNatural = function(value) { return typeof value === 'number' && isFinite(value) && Math.floor(value) === value && @@ -15,7 +15,7 @@ utils.isSafeNatural = function isSafeNatural(value) { value <= MAX_SAFE_INTEGER; }; -utils.startAtZero = function startAtZero(obj, key) { +utils.startAtZero = function(obj, key) { if (!obj.hasOwnProperty(key)) { obj[key] = 0; } @@ -26,7 +26,7 @@ if (!utils.isAbsolutePath) { utils.isAbsolutePath = require('path-is-absolute'); } -utils.parseParamsWithJSON = function parseParamsWithJSON(paramsArg) { +utils.parseParamsWithJSON = function(paramsArg) { var params = paramsArg.map(function(paramArg) { var param; try { diff --git a/regtest/utils.js b/regtest/utils.js new file mode 100644 index 00000000..4c5fefe3 --- /dev/null +++ b/regtest/utils.js @@ -0,0 +1,349 @@ +'use strict'; + +var bitcore = require('bitcore-lib'); +var _ = require('lodash'); +var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); +var fs = require('fs'); +var async = require('async'); +var spawn = require('child_process').spawn; +var http = require('http'); +var Unit = bitcore.Unit; +var Transaction = bitcore.Transaction; +var PrivateKey = bitcore.PrivateKey; +var crypto = require('crypto'); + +var utils = {}; + +utils.writeConfigFile = function(fileStr, obj) { + fs.writeFileSync(fileStr, JSON.stringify(obj)); +}; + +utils.toArgs = function(opts) { + return Object.keys(opts).map(function(key) { + return '-' + key + '=' + opts[key]; + }); +}; + +utils.waitForService = function(task, callback) { + var retryOpts = { times: 20, interval: 1000 }; + async.retry(retryOpts, task, callback); +}; + +utils.queryBitcoreNode = function(httpOpts, callback) { + var error; + var request = http.request(httpOpts, function(res) { + + if (res.statusCode !== 200 && res.statusCode !== 201) { + if (error) { + return; + } + return callback(res.statusCode); + } + + var resError; + var resData = ''; + + res.on('error', function(e) { + resError = e; + }); + + res.on('data', function(data) { + resData += data; + }); + + res.on('end', function() { + if (error) { + return; + } + if (httpOpts.errorFilter) { + return callback(httpOpts.errorFilter(resError, resData)); + } + callback(resError, resData); + }); + + }); + + request.on('error', function(e) { + error = e; + callback(error); + }); + + request.write(httpOpts.body || ''); + request.end(); +}; + +utils.waitForBitcoreNode = function(callback) { + + bitcore.process.stdout.on('data', function(data) { + if (debug) { + console.log(data.toString()); + } + }); + + bitcore.process.stderr.on('data', function(data) { + console.log(data.toString()); + }); + + var errorFilter = function(err, res) { + try { + if (JSON.parse(res).height === blockHeight) { + return; + } + return res; + } catch(e) { + return e; + } + }; + + var httpOpts = getHttpOpts({ path: '/wallet-api/info', errorFilter: errorFilter }); + + waitForService(queryBitcoreNode.bind(this, httpOpts), callback); +}; + +utils.waitForBitcoinReady = function(callback) { + waitForService(function(callback) { + rpc.generate(initialHeight, function(err, res) { + if (err || (res && res.error)) { + return callback('keep trying'); + } + blockHeight += initialHeight; + callback(); + }); + }, function(err) { + if(err) { + return callback(err); + } + callback(); + }, callback); +} + +utils.initializeAndStartService = function(opts, callback) { + rimraf(opts.datadir, function(err) { + if(err) { + return callback(err); + } + mkdirp(opts.datadir, function(err) { + if(err) { + return callback(err); + } + if (opts.configFile) { + writeConfigFile(opts.configFile.file, opts.configFile.conf); + } + var args = _.isArray(opts.args) ? opts.args : toArgs(opts.args); + opts.process = spawn(opts.exec, args, opts.opts); + callback(); + }); + }); +} + +utils.startBitcoreNode = function(callback) { + initializeAndStartService(bitcore, callback); +} + +utils.startBitcoind = function(callback) { + initializeAndStartService(bitcoin, callback); +} + +utils.unlockWallet = function(callback) { + rpc.walletPassPhrase(walletPassphrase, 3000, function(err) { + if(err && err.code !== -15) { + return callback(err); + } + callback(); + }); +} + +utils.getPrivateKeysWithABalance = function(callback) { + rpc.listUnspent(function(err, res) { + if(err) { + return callback(err); + } + + var utxos = []; + for(var i = 0; i < res.result.length; i++) { + if (res.result[i].amount > 1) { + utxos.push(res.result[i]); + } + } + if (utxos.length <= 0) { + return callback(new Error('no utxos available')); + } + async.mapLimit(utxos, 8, function(utxo, callback) { + rpc.dumpPrivKey(utxo.address, function(err, res) { + if(err) { + return callback(err); + } + var privKey = res.result; + callback(null, { utxo: utxo, privKey: privKey }); + }); + }, function(err, utxos) { + if(err) { + return callback(err); + } + callback(null, utxos); + }); + }); +} + +utils.generateSpendingTxs = function(utxos) { + return utxos.map(function(utxo) { + txCount++; + var toPrivKey = new PrivateKey('testnet'); //external addresses + var changePrivKey = new PrivateKey('testnet'); //our wallet keys + var utxoSatoshis = Unit.fromBTC(utxo.utxo.amount).satoshis; + var satsToPrivKey = Math.round(utxoSatoshis / 2); + var tx = new Transaction(); + + tx.from(utxo.utxo); + tx.to(toPrivKey.toAddress().toString(), satsToPrivKey); + tx.fee(fee); + tx.change(changePrivKey.toAddress().toString()); + tx.sign(utxo.privKey); + + walletPrivKeys.push(changePrivKey); + satoshisReceived += Unit.fromBTC(utxo.utxo.amount).toSatoshis() - (satsToPrivKey + fee); + return tx; + }); +} + +utils.setupInitialTxs = function(callback) { + getPrivateKeysWithABalance(function(err, utxos) { + if(err) { + return callback(err); + } + initialTxs = generateSpendingTxs(utxos); + callback(); + }); +} + +utils.sendTxs = function(callback) { + async.eachOfSeries(initialTxs, sendTx, callback); +} + +utils.sendTx = function(tx, index, callback) { + rpc.sendRawTransaction(tx.serialize(), function(err) { + if (err) { + return callback(err); + } + var mod = index % 2; + if (mod === 1) { + blockHeight++; + rpc.generate(1, callback); + } else { + callback(); + } + }); +} + +utils.getHttpOpts = function(opts) { + return Object.assign({ + path: opts.path, + method: opts.method || 'GET', + body: opts.body, + headers: { + 'Content-Type': 'application/json', + 'Content-Length': opts.length || 0 + }, + errorFilter: opts.errorFilter + }, bitcore.httpOpts); +} + +utils.registerWallet = function(callback) { + var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId, method: 'POST' }); + queryBitcoreNode(httpOpts, callback); +} + +utils.uploadWallet = function(callback) { + var addresses = JSON.stringify(walletPrivKeys.map(function(privKey) { + if (privKey.privKey) { + return privKey.pubKey.toString(); + } + return privKey.toAddress().toString(); + })); + var httpOpts = getHttpOpts({ + path: '/wallet-api/wallets/' + walletId + '/addresses', + method: 'POST', + body: addresses, + length: addresses.length + }); + async.waterfall([ queryBitcoreNode.bind(this, httpOpts) ], function(err, res) { + if (err) { + return callback(err); + } + var job = JSON.parse(res); + + Object.keys(job).should.deep.equal(['jobId']); + + var httpOpts = getHttpOpts({ path: '/wallet-api/jobs/' + job.jobId }); + + async.retry({ times: 10, interval: 1000 }, function(next) { + queryBitcoreNode(httpOpts, function(err, res) { + if (err) { + return next(err); + } + var result = JSON.parse(res); + if (result.status === 'complete') { + return next(); + } + next(res); + }); + + }, function(err) { + if(err) { + return callback(err); + } + callback(); + }); + }); +} + +utils.getListOfTxs = function(callback) { + var end = Date.now() + 86400000; + var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/transactions?start=0&end=' + end }); + queryBitcoreNode(httpOpts, function(err, res) { + if(err) { + return callback(err); + } + var results = []; + res.split('\n').forEach(function(result) { + if (result.length > 0) { + return results.push(JSON.parse(result)); + } + }); + + var map = initialTxs.map(function(tx) { + return tx.serialize(); + }); + + results.forEach(function(result) { + var tx = new Transaction(result); + map.splice(map.indexOf(tx.uncheckedSerialize()), 1); + }); + + map.length.should.equal(0); + results.length.should.equal(initialTxs.length); + callback(); + }); +} + +utils.initGlobals = function() { + walletPassphrase = 'test'; + txCount = 0; + blockHeight = 0; + walletPrivKeys = []; + initialTxs = []; + fee = 100000; + feesReceived = 0; + satoshisSent = 0; + walletId = crypto.createHash('sha256').update('test').digest('hex'); + satoshisReceived = 0; +} + +utils.cleanup = function(callback) { + bitcore.process.kill(); + bitcoin.process.kill(); + setTimeout(callback, 2000); +} + +module.exports = utils; diff --git a/regtest/wallet.js b/regtest/wallet.js index b46144a7..dffa02c4 100644 --- a/regtest/wallet.js +++ b/regtest/wallet.js @@ -1,23 +1,14 @@ 'use strict'; -var _ = require('lodash'); -var mkdirp = require('mkdirp'); -var rimraf = require('rimraf'); var chai = require('chai'); var should = chai.should(); -var spawn = require('child_process').spawn; var async = require('async'); var bitcore = require('bitcore-lib'); -var Unit = bitcore.Unit; -var Transaction = bitcore.Transaction; -var PrivateKey = bitcore.PrivateKey; var BitcoinRPC = require('bitcoind-rpc'); var path = require('path'); -var fs = require('fs'); -var http = require('http'); -var crypto = require('crypto'); +var utils = require('utils'); -var debug = true; +var debug = false; var bitcoreDataDir = '/tmp/bitcore'; var bitcoinDataDir = '/tmp/bitcoin'; @@ -92,414 +83,144 @@ var bitcore = { }; var rpc = new BitcoinRPC(rpcConfig); -var walletPassphrase = 'test'; - -var numberOfStartingTxs = 49; //this should be an even number of txs -var txCount = 0; -var blockHeight = 0; - -var walletPrivKeys = []; -var initialTxs = []; -var fee = 100000; -var walletId = crypto.createHash('sha256').update('test').digest('hex'); -var satoshisReceived = 0; +var walletPassphrase, txCount, blockHeight, walletPrivKeys, +initialTxs, fee, walletId, satoshisReceived, satoshisSent, feesReceived; +var initialHeight = 150; describe('Wallet Operations', function() { this.timeout(60000); - after(function(done) { - bitcore.process.kill(); - bitcoin.process.kill(); - setTimeout(done, 2000); - }); + after(cleanup); - before(function(done) { - async.series([ - startBitcoind, - waitForBitcoinReady, - unlockWallet, - setupInitialTxs, //generate a set of transactions to get us a predictable history - startBitcoreNode, - waitForBitcoreNode - ], done); - }); + describe('Register and Upload', function() { - it('should register wallet', function(done) { - - var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId, method: 'POST' }); - queryBitcoreNode(httpOpts, function(err, res) { - if (err) { - return done(err); - } - res.should.deep.equal(JSON.stringify({ - walletId: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08' - })); - done(); + before(function(done) { + initGlobals(); + async.series([ + startBitcoind, + waitForBitcoinReady, + unlockWallet, + setupInitialTxs, + startBitcoreNode, + waitForBitcoreNode + ], done); }); + + it('should register wallet', function(done) { + registerWallet(function(err, res) { + if (err) { + return done(err); + } + res.should.deep.equal(JSON.stringify({ + walletId: '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08' + })); + done(); + }); + }); + + it('should upload a wallet', function(done) { + uploadWallet(done); + }); + }); - it('should upload a wallet', function(done) { - var addresses = JSON.stringify(walletPrivKeys.map(function(privKey) { - return privKey.toAddress().toString(); - })); - var httpOpts = getHttpOpts({ - path: '/wallet-api/wallets/' + walletId + '/addresses', - method: 'POST', - body: addresses, - length: addresses.length - }); - async.waterfall([ queryBitcoreNode.bind(this, httpOpts) ], function(err, res) { - if (err) { - return done(err); - } - var job = JSON.parse(res); + describe('Load addresses at genesis block', function() { - Object.keys(job).should.deep.equal(['jobId']); - - var httpOpts = getHttpOpts({ path: '/wallet-api/jobs/' + job.jobId }); - - async.retry({ times: 10, interval: 1000 }, function(next) { - queryBitcoreNode(httpOpts, function(err, res) { - if (err) { - return next(err); - } - var result = JSON.parse(res); - if (result.status === 'complete') { - return next(); - } - next(res); - }); - - }, function(err) { + before(function(done) { + sendTxs(function(err) { if(err) { return done(err); } + waitForBitcoreNode(done); + }); + }); + + it('should get a list of transactions', function(done) { + + getListOfTxs(done); + + }); + + }); + + describe('Load addresses after syncing the blockchain', function() { + + before(function(done) { + initGlobals(); + async.series([ + cleanup, + startBitcoind, + waitForBitcoinReady, + unlockWallet, + setupInitialTxs, + sendTxs, + startBitcoreNode, + waitForBitcoreNode, + registerWallet, + uploadWallet + ], done); + }); + + it('should get list of transactions', function(done) { + + getListOfTxs(done); + + }); + + it('should get the balance of a wallet', function(done) { + var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/balance' }); + queryBitcoreNode(httpOpts, function(err, res) { + if(err) { + return done(err); + } + var results = JSON.parse(res); + results.satoshis.should.equal(satoshisReceived); + done(); + }); + + }); + + it('should get the set of utxos for the wallet', function(done) { + var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/utxos' }); + queryBitcoreNode(httpOpts, function(err, res) { + if(err) { + return done(err); + } + var results = JSON.parse(res); + var balance = 0; + + results.utxos.forEach(function(utxo) { + balance += utxo.satoshis; + }); + results.height.should.equal(blockHeight); + balance.should.equal(satoshisReceived); + done(); + }); + }); + + it('should get the list of jobs', function(done) { + var httpOpts = getHttpOpts({ path: '/wallet-api/jobs' }); + queryBitcoreNode(httpOpts, function(err, res) { + if(err) { + return done(err); + } + var results = JSON.parse(res); + results.jobCount.should.equal(1); + done(); + }); + }); + + it('should remove all wallets', function(done) { + var httpOpts = getHttpOpts({ path: '/wallet-api/wallets', method: 'DELETE' }); + queryBitcoreNode(httpOpts, function(err, res) { + if(err) { + return done(err); + } + var results = JSON.parse(res); + results.numberRemoved.should.equal(152); done(); }); }); }); - - it('should get a list of transactions', function(done) { - var end = Date.now() + 86400000; - var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/transactions?start=0&end=' + end }); - queryBitcoreNode(httpOpts, function(err, res) { - if(err) { - return done(err); - } - //jsonl is returned, so there will be a newline at the end - var results = res.split('\n').filter(function(result) { - return result.length > 0; - }); - var map = initialTxs.map(function(tx) { - return tx.serialize(); - }); - for(var i = 0; i < results.length; i++) { - var result = results[i]; - var tx = new Transaction(JSON.parse(result)); - map.splice(map.indexOf(tx.uncheckedSerialize()), 1); - } - map.length.should.equal(0); - results.length.should.equal(numberOfStartingTxs); - done(); - }); - }); - - it('should get the balance of a wallet', function(done) { - var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/balance' }); - queryBitcoreNode(httpOpts, function(err, res) { - if(err) { - return done(err); - } - var results = JSON.parse(res); - results.satoshis.should.equal(satoshisReceived); - done(); - }); - - }); - - it('should get the set of utxos for the wallet', function(done) { - var httpOpts = getHttpOpts({ path: '/wallet-api/wallets/' + walletId + '/utxos' }); - queryBitcoreNode(httpOpts, function(err, res) { - if(err) { - return done(err); - } - var results = JSON.parse(res); - // all starting txs were spending to our wallet - results.utxos.length.should.equal(numberOfStartingTxs); - var map = initialTxs.map(function(tx) { - return tx.txid; - }); - var balance = 0; - for(var i = 0; i < results.utxos.length; i++) { - var result = results.utxos[i]; - balance += result.satoshis; - map.splice(map.indexOf(result.txid), 1); - } - map.length.should.equal(0); - results.height.should.equal(blockHeight); - balance.should.equal(satoshisReceived); - done(); - }); - }); - - it('should get the list of jobs', function(done) { - var httpOpts = getHttpOpts({ path: '/wallet-api/jobs' }); - queryBitcoreNode(httpOpts, function(err, res) { - if(err) { - return done(err); - } - var results = JSON.parse(res); - results.jobCount.should.equal(1); - done(); - }); - }); - - it('should remove all wallets', function(done) { - var httpOpts = getHttpOpts({ path: '/wallet-api/wallets', method: 'DELETE' }); - queryBitcoreNode(httpOpts, function(err, res) { - if(err) { - return done(err); - } - //walletTransactionKey = 1, walletUtxoKey = 1, walletUtxoSatoshis = 1 <-- multiples of numberOfStartingTxs - //walletAddresses = 1, walletBalance = 1 <-- one record per index - var results = JSON.parse(res); - results.numberRemoved.should.equal((numberOfStartingTxs * 3) + 2); - done(); - }); - }); }); - -function writeConfigFile(fileStr, obj) { - fs.writeFileSync(fileStr, JSON.stringify(obj)); -} - -function toArgs(opts) { - return Object.keys(opts).map(function(key) { - return '-' + key + '=' + opts[key]; - }); -} - -function waitForService(task, next) { - var retryOpts = { times: 20, interval: 1000 }; - async.retry(retryOpts, task, next); -} - -function queryBitcoreNode(httpOpts, next) { - var error; - var request = http.request(httpOpts, function(res) { - - if (res.statusCode !== 200 && res.statusCode !== 201) { - if (error) { - return; - } - return next(res.statusCode); - } - - var resError; - var resData = ''; - - res.on('error', function(e) { - resError = e; - }); - - res.on('data', function(data) { - resData += data; - }); - - res.on('end', function() { - if (error) { - return; - } - if (httpOpts.errorFilter) { - return next(httpOpts.errorFilter(resError, resData)); - } - next(resError, resData); - }); - - }); - - request.on('error', function(e) { - error = e; - next(error); - }); - - request.write(httpOpts.body || ''); - request.end(); -} - -function waitForBitcoreNode(next) { - bitcore.process.stdout.on('data', function(data) { - if (debug) { - console.log(data.toString()); - } - }); - bitcore.process.stderr.on('data', function(data) { - console.log(data.toString()); - }); - var errorFilter = function(err, res) { - if (err || (res && !JSON.parse(res).result)) { - return 'still syncing'; - } - }; - - var httpOpts = getHttpOpts({ path: '/wallet-api/issynced', errorFilter: errorFilter }); - - waitForService(queryBitcoreNode.bind(this, httpOpts), next); -} - -function waitForBitcoinReady(next) { - waitForService(function(next) { - rpc.generate(150, function(err, res) { - if (err || (res && res.error)) { - return next('keep trying'); - } - blockHeight += 150; - next(); - }); - }, function(err) { - if(err) { - return next(err); - } - next(); - }, next); -} - -function initializeAndStartService(opts, next) { - rimraf(opts.datadir, function(err) { - if(err) { - return next(err); - } - mkdirp(opts.datadir, function(err) { - if(err) { - return next(err); - } - if (opts.configFile) { - writeConfigFile(opts.configFile.file, opts.configFile.conf); - } - var args = _.isArray(opts.args) ? opts.args : toArgs(opts.args); - opts.process = spawn(opts.exec, args, opts.opts); - next(); - }); - }); -} - -function startBitcoreNode(next) { - initializeAndStartService(bitcore, next); -} - -function startBitcoind(next) { - initializeAndStartService(bitcoin, next); -} - -function unlockWallet(next) { - rpc.walletPassPhrase(walletPassphrase, 3000, function(err) { - if(err && err.code !== -15) { - return next(err); - } - next(); - }); -} - -function getPrivateKeyWithABalance(next) { - rpc.listUnspent(function(err, res) { - if(err) { - return next(err); - } - - var utxo; - for(var i = 0; i < res.result.length; i++) { - if (res.result[i].amount > 1) { - utxo = res.result[i]; - break; - } - } - if (!utxo) { - return next(new Error('no utxos available')); - } - rpc.dumpPrivKey(utxo.address, function(err, res) { - if(err) { - return next(err); - } - var privKey = res.result; - next(null, privKey, utxo); - }); - }); -} - -function generateSpendingTx(privKey, utxo) { - txCount++; - var toPrivKey = new PrivateKey('testnet'); //external addresses - var changePrivKey = new PrivateKey('testnet'); //our wallet keys - var utxoSatoshis = Unit.fromBTC(utxo.amount).satoshis; - var satsToPrivKey = Math.round(utxoSatoshis / 2); - var tx = new Transaction(); - - tx.from(utxo); - tx.to(toPrivKey.toAddress().toString(), satsToPrivKey); - tx.fee(fee); - tx.change(changePrivKey.toAddress().toString()); - tx.sign(privKey); - - walletPrivKeys.push(changePrivKey); - satoshisReceived += Unit.fromBTC(utxo.amount).toSatoshis() - (satsToPrivKey + fee); - return tx; -} - -function setupInitialTx(index, next) { - getPrivateKeyWithABalance(function(err, privKey, utxo) { - if(err) { - return next(err); - } - var tx = generateSpendingTx(privKey, utxo); - sendTx(tx, (index % 2 === 0 ? 0 : 1), function(err, tx) { - if(err) { - return next(err); - } - initialTxs.push(tx); - next(); - }); - }); -} - -function setupInitialTxs(next) { - async.timesSeries(numberOfStartingTxs, setupInitialTx, function(err) { - if(err) { - return next(err); - } - blockHeight++; - rpc.generate(1, next); - }); -} - -function sendTx(tx, generateBlocks, next) { - rpc.sendRawTransaction(tx.serialize(), function(err) { - if(err) { - return next(err); - } - if (generateBlocks) { - blockHeight += generateBlocks; - rpc.generate(generateBlocks, function(err) { - if(err) { - return next(err); - } - next(null, tx); - }); - } else { - next(null, tx); - } - }); -} - -function getHttpOpts(opts) { - return Object.assign({ - path: opts.path, - method: opts.method || 'GET', - body: opts.body, - headers: { - 'Content-Type': 'application/json', - 'Content-Length': opts.length || 0 - }, - errorFilter: opts.errorFilter - }, bitcore.httpOpts); -}