From 8730ca61483947f67759c9961773af152942e5a1 Mon Sep 17 00:00:00 2001 From: Chris Kleeschulte Date: Thu, 25 May 2017 23:11:55 -0400 Subject: [PATCH] wip --- lib/scaffold/start.js | 1 + lib/services/block/block_handler.js | 1 + lib/services/block/index.js | 9 +- lib/services/utxo/encoding.js | 1 + lib/services/utxo/index.js | 88 ++++++++++--------- lib/utils.js | 16 ++++ regtest/utxo.js | 130 ++++++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 42 deletions(-) create mode 100644 regtest/utxo.js diff --git a/lib/scaffold/start.js b/lib/scaffold/start.js index 696c17f6..0ca6ea64 100644 --- a/lib/scaffold/start.js +++ b/lib/scaffold/start.js @@ -156,6 +156,7 @@ function lookInBuiltInPath(req, service) { var serviceFile = path.resolve(__dirname, '../services/' + service.name); return req(serviceFile); } catch(e) { + throw e; } } diff --git a/lib/services/block/block_handler.js b/lib/services/block/block_handler.js index c6e98afd..3048cfc8 100644 --- a/lib/services/block/block_handler.js +++ b/lib/services/block/block_handler.js @@ -121,6 +121,7 @@ BlockHandler.prototype._onFinish = function() { var self = this; self.syncing = false; +console.log('done'); self.emit('synced'); diff --git a/lib/services/block/index.js b/lib/services/block/index.js index 0b558f2a..e01e7f83 100644 --- a/lib/services/block/index.js +++ b/lib/services/block/index.js @@ -501,8 +501,8 @@ BlockService.prototype._loadTips = function(callback) { BlockService.prototype._detectReorg = function(blocks, callback) { var self = this; - var tipHash = self.tip.hash; - var tipHeight = self.tip.__height; + var tipHash = self.reorgHash || self.tip.hash; + var tipHeight = self.reorgHeight || self.tip.__height; var forkedHash; for(var i = 0; i < blocks.length; i++) { @@ -517,7 +517,8 @@ BlockService.prototype._detectReorg = function(blocks, callback) { break; } - tipHash = prevHash; + tipHash = blocks[i].hash; + tipHeight = blocks[i].__height; } if (forkedHash) { @@ -526,6 +527,8 @@ BlockService.prototype._detectReorg = function(blocks, callback) { }); } + self.reorgHash = tipHash; + self.reorgHeight = tipHeight; callback(); }; diff --git a/lib/services/utxo/encoding.js b/lib/services/utxo/encoding.js index d2b418ac..2343b1fd 100644 --- a/lib/services/utxo/encoding.js +++ b/lib/services/utxo/encoding.js @@ -1,5 +1,6 @@ 'use strict'; +var bitcore = require('bitcore-lib'); var BufferReader = bitcore.encoding.BufferReader; function Encoding(servicePrefix) { diff --git a/lib/services/utxo/index.js b/lib/services/utxo/index.js index d8e66ca0..81575a09 100644 --- a/lib/services/utxo/index.js +++ b/lib/services/utxo/index.js @@ -5,9 +5,13 @@ var inherits = require('util').inherits; var Encoding = require('./encoding'); var LRU = require('lru-cache'); +const REORG_BUFFER = 6; + function UtxoService(options) { BaseService.call(this, options); + this._operations = []; this._createCache({ max: 500000, dispose: this._getUtxoOperations.bind(this) }); + this._exclusionIndexes = []; } inherits(UtxoService, BaseService); @@ -35,83 +39,84 @@ UtxoService.prototype.stop = function(callback) { } }; -UtxoService.prototype._processInputs = function(inputs, block, connect) { - - var ret = []; +UtxoService.prototype._processInputs = function(tx, inputs, connect) { + var operations = []; for(var i = 0; i < inputs.length; i++) { var input = inputs[i]; var key = input.prevHash + input.outputIndex; if (input.prevTxId === tx.hash) { - ret.push(i); + this._exlusionIndexes.push(i); continue; } - if (connect) { - self._removeSpentOutput(key); - } else { - self._readdUnspentOutput(key); - } + var ops = this._moveOutput(key, connect); + operations = operations.concat(ops); } - return ret; + return operations; + }; -UtxoService.prototype._processOutputs = function(outputs, exclusions, block, connect) { +UtxoService.prototype._processOutputs = function(tx, outputs, block, connect) { + var operations = []; for(var i = 0; i < outputs.length; i++) { var output = outputs[i]; var key = tx.hash + i; - if (exclusions.indexOf(i) > -1) { + if (this._exclusionIndexes.indexOf(i) > -1) { continue; } - self._setCache(key, block, output); + if (connect) { + //when the cache is full, we will write out. + return this._setCache(key, block, output); + } + + return this._cache.del(key); } }; -UtxoService.prototype._setCache = function(key, block, output) { +UtxoService.prototype._setCache = function(key, block, output, value) { - self._cache.set(key, { + if (!value) { + value = { output: output, height: block.__height, - hash: block.hash, - }); // key = 36 bytes, value = (8 + 25ish) + 36 = 69 bytes + hash: block.hash + }; + } + + this._cache.set(key, value); // key = 36 bytes, value = (8 + 25ish) + 36 = 69 bytes }; -UtxoService.prototype._removeSpentOutput = function(key) { +UtxoService.prototype._moveOutput = function(key, connect) { - var output = self._cache.peek(key); - - // we don't want nuke out our - if (!output) { + if (connect) { + self._cache.del(key); return { action: 'del', key: key }; } + // this should only happen during a reorg, hopefully this is an infrequent occurence + // the ramifications are that comsumers of this data will need to make an additional + // lookup of the tx index. We are ok with trade-off for performance. + return { action: 'put', key: key, value: null }; + }; -UtxoService.prototype._readdSpentOutput = function(key) { - - var output = self._cache.get(key); - - if (!output) { - } -}; -/* -connect: - 1. for all txs, for each input in the tx, remove the output that this input is spending - -*/ UtxoService.prototype.blockHandler = function(block, connect) { var self = this; + + self._currentBlockHeight = block.__height; + self._exclusionIndexes.length = 0; var operations = []; for(var i = 0; i < block.transactions.length; i++) { @@ -119,20 +124,25 @@ UtxoService.prototype.blockHandler = function(block, connect) { var tx = block.transactions[i]; var inputs = tx.inputs; var outputs = tx.outputs; - var inputOperations = {}; if (!tx.isCoinbase()) { - inputOperations = self._processInputs(inputs, block, connect); + operations = self._processInputs(tx, inputs, connect).concat(operations); } - var outputRes = self._processOutputs(outputs, inputRes.exclusions || [], block, connect); - operations = operations.concat(inputRes.operations || []).concat(outputRes.operations); + self._processOutputs(tx, outputs, block, connect); } + operations = this._operations.concat(operations); + this._operations.length = 0; + return operations; }; UtxoService.prototype._getUtxoOperations = function(key, value) { + if (value.height + REORG_BUFFER >= self._currentHeight) { + log.error('Writing utxos to the database before ' + REORG_BUFFER + ' confirmation blocks.' + + ' The internal cache might be too small or the system does not have enough memory.'); + } this._operations.push({ action: 'put', key: this._getOperationsKey(key, value), @@ -141,7 +151,7 @@ UtxoService.prototype._getUtxoOperations = function(key, value) { }; UtxoService.prototype._getOperationsKey = function(key, value) { - var address = utils.getAddressFromScript(value.output.script); + var address = utils.getAddressStringFromScript(value.output.script, this.node.network); return self.encoding.encodeUtxoIndexKey(address, key.slice(0, 32), parseInt(key.slice(32))); }; diff --git a/lib/utils.js b/lib/utils.js index e39559a1..ea5535de 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -88,4 +88,20 @@ utils.getIpAddressInfo = function(ipStr) { //does this string have colons or periods? }; +utils.getAddressStringFromScript = function(script, network) { + var address = script.toAddress(network); + + if(address) { + return address.toString(); + } + + try { + var pubkey = script.getPublicKey(); + if(pubkey) { + return pubkey.toString('hex'); + } + } catch(e) {} + +}; + module.exports = utils; diff --git a/regtest/utxo.js b/regtest/utxo.js new file mode 100644 index 00000000..10035c2a --- /dev/null +++ b/regtest/utxo.js @@ -0,0 +1,130 @@ +'use strict'; + +var chai = require('chai'); +var should = chai.should(); +var async = require('async'); +var BitcoinRPC = require('bitcoind-rpc'); +var path = require('path'); +var utils = require('./utils'); +var crypto = require('crypto'); + +var debug = true; +var bitcoreDataDir = '/tmp/bitcore'; +var bitcoinDataDir = '/tmp/bitcoin'; + +var rpcConfig = { + protocol: 'http', + user: 'bitcoin', + pass: 'local321', + host: '127.0.0.1', + port: '58332', + rejectUnauthorized: false +}; + +var bitcoin = { + args: { + datadir: bitcoinDataDir, + listen: 0, + regtest: 1, + server: 1, + rpcuser: rpcConfig.user, + rpcpassword: rpcConfig.pass, + rpcport: rpcConfig.port, + zmqpubrawtx: 'tcp://127.0.0.1:38332', + zmqpubhashblock: 'tcp://127.0.0.1:38332' + }, + datadir: bitcoinDataDir, + exec: 'bitcoind', //if this isn't on your PATH, then provide the absolute path, e.g. /usr/local/bin/bitcoind + process: null +}; + +var bitcore = { + configFile: { + file: bitcoreDataDir + '/bitcore-node.json', + conf: { + network: 'regtest', + port: 53001, + datadir: bitcoreDataDir, + services: [ + 'bitcoind', + 'db', + 'timestamp', + 'web', + 'block', + 'utxo' + ], + servicesConfig: { + bitcoind: { + connect: [ + { + rpcconnect: rpcConfig.host, + rpcport: rpcConfig.port, + rpcuser: rpcConfig.user, + rpcpassword: rpcConfig.pass, + zmqpubrawtx: bitcoin.args.zmqpubrawtx + } + ] + } + } + } + }, + httpOpts: { + protocol: 'http:', + hostname: 'localhost', + port: 53001, + }, + opts: { cwd: bitcoreDataDir }, + datadir: bitcoreDataDir, + exec: path.resolve(__dirname, '../bin/bitcore-node'), + args: ['start'], + process: null +}; + +var opts = { + debug: debug, + bitcore: bitcore, + bitcoin: bitcoin, + bitcoinDataDir: bitcoinDataDir, + bitcoreDataDir: bitcoreDataDir, + rpc: new BitcoinRPC(rpcConfig), + walletPassphrase: 'test', + txCount: 0, + blockHeight: 0, + walletPrivKeys: [], + initialTxs: [], + fee: 100000, + feesReceived: 0, + satoshisSent: 0, + walletId: crypto.createHash('sha256').update('test').digest('hex'), + satoshisReceived: 0, + initialHeight: 150 +}; + +describe('Utxo Operations', function() { + + this.timeout(60000); + + var self = this; + + after(function(done) { + utils.cleanup(self.opts, done); + }); + + before(function(done) { + self.opts = Object.assign({}, opts); + async.series([ + utils.startBitcoind.bind(utils, self.opts), + utils.waitForBitcoinReady.bind(utils, self.opts), + utils.unlockWallet.bind(utils, self.opts), + utils.setupInitialTxs.bind(utils, self.opts), + utils.startBitcoreNode.bind(utils, self.opts), + utils.waitForBitcoreNode.bind(utils, self.opts) + ], done); + }); + + it('should index utxos', function(done) { + done(); + }); + +}); +