wip

lib/scaffold/start.js

@@ -156,6 +156,7 @@ function lookInBuiltInPath(req, service) {
     var serviceFile = path.resolve(__dirname, '../services/' + service.name);
     return req(serviceFile);
   } catch(e) {
+    throw e;
   }
 }
 

lib/services/block/block_handler.js

@@ -121,6 +121,7 @@ BlockHandler.prototype._onFinish = function() {
 
   var self = this;
   self.syncing = false;
+console.log('done');
 
   self.emit('synced');
 

lib/services/block/index.js

@@ -501,8 +501,8 @@ BlockService.prototype._loadTips = function(callback) {
 BlockService.prototype._detectReorg = function(blocks, callback) {
 
   var self = this;
-  var tipHash = self.tip.hash;
+  var tipHash = self.reorgHash || self.tip.hash;
-  var tipHeight = self.tip.__height;
+  var tipHeight = self.reorgHeight || self.tip.__height;
   var forkedHash;
 
   for(var i = 0; i < blocks.length; i++) {
@@ -517,7 +517,8 @@ BlockService.prototype._detectReorg = function(blocks, callback) {
       break;
     }
 
-    tipHash = prevHash;
+    tipHash = blocks[i].hash;
+    tipHeight = blocks[i].__height;
   }
 
   if (forkedHash) {
@@ -526,6 +527,8 @@ BlockService.prototype._detectReorg = function(blocks, callback) {
     });
   }
 
+  self.reorgHash = tipHash;
+  self.reorgHeight = tipHeight;
   callback();
 
 };

lib/services/utxo/encoding.js

@@ -1,5 +1,6 @@
 'use strict';
 
+var bitcore = require('bitcore-lib');
 var BufferReader = bitcore.encoding.BufferReader;
 
 function Encoding(servicePrefix) {

lib/services/utxo/index.js

@@ -5,9 +5,13 @@ var inherits = require('util').inherits;
 var Encoding = require('./encoding');
 var LRU = require('lru-cache');
 
+const REORG_BUFFER = 6;
+
 function UtxoService(options) {
   BaseService.call(this, options);
+  this._operations = [];
   this._createCache({ max: 500000, dispose: this._getUtxoOperations.bind(this) });
+  this._exclusionIndexes = [];
 }
 
 inherits(UtxoService, BaseService);
@@ -35,83 +39,84 @@ UtxoService.prototype.stop = function(callback) {
   }
 };
 
-UtxoService.prototype._processInputs = function(inputs, block, connect) {
+UtxoService.prototype._processInputs = function(tx, inputs, connect) {
-
-  var ret = [];
 
+  var operations = [];
   for(var i = 0; i < inputs.length; i++) {
     var input = inputs[i];
 
     var key = input.prevHash + input.outputIndex;
 
     if (input.prevTxId === tx.hash) {
-      ret.push(i);
+      this._exlusionIndexes.push(i);
       continue;
     }
 
-    if (connect) {
+    var ops = this._moveOutput(key, connect);
-      self._removeSpentOutput(key);
+    operations = operations.concat(ops);
-    } else {
-      self._readdUnspentOutput(key);
-    }
   }
 
-  return ret;
+  return operations;
+
 };
 
-UtxoService.prototype._processOutputs = function(outputs, exclusions, block, connect) {
+UtxoService.prototype._processOutputs = function(tx, outputs, block, connect) {
 
+  var operations = [];
   for(var i = 0; i < outputs.length; i++) {
 
     var output = outputs[i];
     var key = tx.hash + i;
 
-    if (exclusions.indexOf(i) > -1) {
+    if (this._exclusionIndexes.indexOf(i) > -1) {
       continue;
     }
 
-    self._setCache(key, block, output);
+    if (connect) {
+      //when the cache is full, we will write out.
+      return this._setCache(key, block, output);
+    }
+
+    return this._cache.del(key);
 
   }
 
 };
 
-UtxoService.prototype._setCache = function(key, block, output) {
+UtxoService.prototype._setCache = function(key, block, output, value) {
 
-    self._cache.set(key, {
+  if (!value) {
+    value = {
       output: output,
       height: block.__height,
-      hash: block.hash,
+      hash: block.hash
-    }); // key = 36 bytes, value = (8 + 25ish) + 36 = 69 bytes
+    };
+  }
+
+  this._cache.set(key, value); // key = 36 bytes, value = (8 + 25ish) + 36 = 69 bytes
 
 };
 
-UtxoService.prototype._removeSpentOutput = function(key) {
+UtxoService.prototype._moveOutput = function(key, connect) {
 
-  var output = self._cache.peek(key);
+  if (connect) {
-
+    self._cache.del(key);
-  // we don't want nuke out our
-  if (!output) {
     return { action: 'del', key: key };
   }
 
+  // this should only happen during a reorg, hopefully this is an infrequent occurence
+  // the ramifications are that comsumers of this data will need to make an additional
+  // lookup of the tx index. We are ok with trade-off for performance.
+  return { action: 'put', key: key, value: null };
+
 };
 
-UtxoService.prototype._readdSpentOutput = function(key) {
-
-  var output = self._cache.get(key);
-
-  if (!output) {
-  }
-};
-/*
-connect:
-  1. for all txs, for each input in the tx, remove the output that this input is spending
-
-*/
 UtxoService.prototype.blockHandler = function(block, connect) {
 
   var self = this;
+
+  self._currentBlockHeight = block.__height;
+  self._exclusionIndexes.length = 0;
   var operations = [];
 
   for(var i = 0; i < block.transactions.length; i++) {
@@ -119,20 +124,25 @@ UtxoService.prototype.blockHandler = function(block, connect) {
     var tx = block.transactions[i];
     var inputs = tx.inputs;
     var outputs = tx.outputs;
-    var inputOperations = {};
 
     if (!tx.isCoinbase()) {
-      inputOperations = self._processInputs(inputs, block, connect);
+      operations = self._processInputs(tx, inputs, connect).concat(operations);
     }
 
-    var outputRes = self._processOutputs(outputs, inputRes.exclusions || [], block, connect);
+    self._processOutputs(tx, outputs, block, connect);
-    operations = operations.concat(inputRes.operations || []).concat(outputRes.operations);
   }
 
+  operations = this._operations.concat(operations);
+  this._operations.length = 0;
+
   return operations;
 };
 
 UtxoService.prototype._getUtxoOperations = function(key, value) {
+  if (value.height + REORG_BUFFER >= self._currentHeight) {
+    log.error('Writing utxos to the database before ' + REORG_BUFFER + ' confirmation blocks.' +
+     ' The internal cache might be too small or the system does not have enough memory.');
+  }
   this._operations.push({
     action: 'put',
     key: this._getOperationsKey(key, value),
@@ -141,7 +151,7 @@ UtxoService.prototype._getUtxoOperations = function(key, value) {
 };
 
 UtxoService.prototype._getOperationsKey = function(key, value) {
-  var address = utils.getAddressFromScript(value.output.script);
+  var address = utils.getAddressStringFromScript(value.output.script, this.node.network);
   return self.encoding.encodeUtxoIndexKey(address, key.slice(0, 32), parseInt(key.slice(32)));
 };
 

lib/utils.js

@@ -88,4 +88,20 @@ utils.getIpAddressInfo = function(ipStr) {
   //does this string have colons or periods?
 };
 
+utils.getAddressStringFromScript = function(script, network) {
+  var address = script.toAddress(network);
+
+  if(address) {
+    return address.toString();
+  }
+
+  try {
+    var pubkey = script.getPublicKey();
+    if(pubkey) {
+      return pubkey.toString('hex');
+    }
+  } catch(e) {}
+
+};
+
 module.exports = utils;

regtest/utxo.js

@@ -0,0 +1,130 @@
+'use strict';
+
+var chai = require('chai');
+var should = chai.should();
+var async = require('async');
+var BitcoinRPC = require('bitcoind-rpc');
+var path = require('path');
+var utils = require('./utils');
+var crypto = require('crypto');
+
+var debug = true;
+var bitcoreDataDir = '/tmp/bitcore';
+var bitcoinDataDir = '/tmp/bitcoin';
+
+var rpcConfig = {
+  protocol: 'http',
+  user: 'bitcoin',
+  pass: 'local321',
+  host: '127.0.0.1',
+  port: '58332',
+  rejectUnauthorized: false
+};
+
+var bitcoin = {
+  args: {
+    datadir: bitcoinDataDir,
+    listen: 0,
+    regtest: 1,
+    server: 1,
+    rpcuser: rpcConfig.user,
+    rpcpassword: rpcConfig.pass,
+    rpcport: rpcConfig.port,
+    zmqpubrawtx: 'tcp://127.0.0.1:38332',
+    zmqpubhashblock: 'tcp://127.0.0.1:38332'
+  },
+  datadir: bitcoinDataDir,
+  exec: 'bitcoind', //if this isn't on your PATH, then provide the absolute path, e.g. /usr/local/bin/bitcoind
+  process: null
+};
+
+var bitcore = {
+  configFile: {
+    file: bitcoreDataDir + '/bitcore-node.json',
+    conf: {
+      network: 'regtest',
+      port: 53001,
+      datadir: bitcoreDataDir,
+      services: [
+        'bitcoind',
+        'db',
+        'timestamp',
+        'web',
+        'block',
+        'utxo'
+      ],
+      servicesConfig: {
+        bitcoind: {
+          connect: [
+            {
+              rpcconnect: rpcConfig.host,
+              rpcport: rpcConfig.port,
+              rpcuser: rpcConfig.user,
+              rpcpassword: rpcConfig.pass,
+              zmqpubrawtx: bitcoin.args.zmqpubrawtx
+            }
+          ]
+        }
+      }
+    }
+  },
+  httpOpts: {
+    protocol: 'http:',
+    hostname: 'localhost',
+    port: 53001,
+  },
+  opts: { cwd: bitcoreDataDir },
+  datadir: bitcoreDataDir,
+  exec: path.resolve(__dirname, '../bin/bitcore-node'),
+  args: ['start'],
+  process: null
+};
+
+var opts = {
+  debug: debug,
+  bitcore: bitcore,
+  bitcoin: bitcoin,
+  bitcoinDataDir: bitcoinDataDir,
+  bitcoreDataDir: bitcoreDataDir,
+  rpc: new BitcoinRPC(rpcConfig),
+  walletPassphrase: 'test',
+  txCount: 0,
+  blockHeight: 0,
+  walletPrivKeys: [],
+  initialTxs: [],
+  fee: 100000,
+  feesReceived: 0,
+  satoshisSent: 0,
+  walletId: crypto.createHash('sha256').update('test').digest('hex'),
+  satoshisReceived: 0,
+  initialHeight: 150
+};
+
+describe('Utxo Operations', function() {
+
+  this.timeout(60000);
+
+  var self = this;
+
+  after(function(done) {
+    utils.cleanup(self.opts, done);
+  });
+
+  before(function(done) {
+    self.opts = Object.assign({}, opts);
+    async.series([
+      utils.startBitcoind.bind(utils, self.opts),
+      utils.waitForBitcoinReady.bind(utils, self.opts),
+      utils.unlockWallet.bind(utils, self.opts),
+      utils.setupInitialTxs.bind(utils, self.opts),
+      utils.startBitcoreNode.bind(utils, self.opts),
+      utils.waitForBitcoreNode.bind(utils, self.opts)
+    ], done);
+  });
+
+  it('should index utxos', function(done) {
+    done();
+  });
+
+});
+