add ignore Cache

README.md

@@ -26,15 +26,17 @@ Alternatively, a total resync can be made, running `$ util/sync.js -D`
 ## IMPORTANT: v0.2 Caching schema
 
 In v0.2 a new cache schema has been introduced. Only information from transactions with
-SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There 
+INSIGHT_SAFE_CONFIRMATIONS+ settings will be cached (by default SAFE_CONFIRMATIONS=6). There 
 are 3 different caches:
  * nr. of confirmations 
  * transaction spent information
  * scriptPubKey for unspent transactions
 
-Cache data is only completed on request, i.e., only after accessing the required data for
+Cache data is only filled on request, i.e., only after accessing the required data for
 the first time, the information is cached, there is not pre-caching procedure.
 
+To ignore cache, use INSIGHT_IGNORE_CACHE;
+
 ## Prerequisites
 
 * **bitcoind** - Download and Install [Bitcoin](http://bitcoin.org/en/download)
@@ -91,7 +93,9 @@ BITCOIND_PASS         # RPC password
 BITCOIND_DATADIR      # bitcoind datadir for livenet, or datadir/testnet3 for testnet
 INSIGHT_NETWORK [= 'livenet' | 'testnet']
 INSIGHT_DB            # Path where to store insight's internal DB. (defaults to $HOME/.insight)
-SAFE_CONFIRMATIONS=6  # Nr. of confirmation needed to start caching transaction information   
+INSIGHT_SAFE_CONFIRMATIONS=6  # Nr. of confirmation needed to start caching transaction information   
+INSIGHT_IGNORE_CACHE  # True to ignore cache of spents in transaction, with more than INSIGHT_SAFE_CONFIRMATIONS confirmations. This is useful for tracking double spents for old transactions.
+
 ```
 
 Make sure that bitcoind is configured to [accept incoming connections using 'rpcallowip'](https://en.bitcoin.it/wiki/Running_Bitcoin).

app/controllers/addresses.js

@@ -53,7 +53,7 @@ exports.show = function(req, res, next) {
       } else {
         return res.jsonp(a.getObj());
       }
-    }, {txLimit: req.query.noTxList?0:-1});
+    }, {txLimit: req.query.noTxList?0:-1, ignoreCache: req.param('nocache')});
   }
 };
 
@@ -68,7 +68,7 @@ exports.utxo = function(req, res, next) {
       else {
         return res.jsonp(a.unspent);
       }
-    }, {onlyUnspent: 1});
+    }, {onlyUnspent:1, ignoreCache: req.param('nocache')});
   }
 };
 
@@ -81,7 +81,7 @@ exports.multiutxo = function(req, res, next) {
         if (err) callback(err);
         utxos = utxos.concat(a.unspent);
         callback();
-      }, {onlyUnspent:1});
+      }, {onlyUnspent:1, ignoreCache: req.param('nocache')});
     }, function(err) { // finished callback
       if (err) return common.handleErrors(err, res);
       res.jsonp(utxos);
@@ -99,7 +99,7 @@ exports.balance = function(req, res, next) {
       } else {
         return res.jsonp(a.balanceSat);
       }
-    });
+    }, {ignoreCache: req.param('nocache')});
 };
 
 exports.totalReceived = function(req, res, next) {
@@ -111,7 +111,7 @@ exports.totalReceived = function(req, res, next) {
       } else {
         return res.jsonp(a.totalReceivedSat);
       }
-    });
+    }, {ignoreCache: req.param('nocache')});
 };
 
 exports.totalSent = function(req, res, next) {
@@ -123,7 +123,7 @@ exports.totalSent = function(req, res, next) {
       } else {
         return res.jsonp(a.totalSentSat);
       }
-    });
+    }, {ignoreCache: req.param('nocache')});
 };
 
 exports.unconfirmedBalance = function(req, res, next) {
@@ -135,5 +135,5 @@ exports.unconfirmedBalance = function(req, res, next) {
       } else {
         return res.jsonp(a.unconfirmedBalanceSat);
       }
-    });
+    }, {ignoreCache: req.param('nocache')});
 };

app/models/Address.js

@@ -149,10 +149,15 @@ Address.prototype.update = function(next, opts) {
   if (!self.addrStr) return next();
   opts = opts || {};
 
+  if (! ('ignoreCache' in opts) )
+    opts.ignoreCache = config.ignoreCache;
+
+  // should collect txList from address?
   var txList  = opts.txLimit === 0 ? null: [];
+
   var tDb   = TransactionDb;
   var bDb   = BlockDb;
-  tDb.fromAddr(self.addrStr, function(err,txOut){
+  tDb.fromAddr(self.addrStr, opts, function(err,txOut){
     if (err) return next(err);
 
     bDb.fillConfirmations(txOut, function(err) {
@@ -184,7 +189,9 @@ Address.prototype.update = function(next, opts) {
           txOut.forEach(function(txItem){
             self._addTxItem(txItem, txList);
           });
-          if (txList) self.transactions = txList;
+          if (txList) 
+            self.transactions = txList;
+
           return next();
         }
       });

config/config.js

@@ -60,7 +60,8 @@ if (!dataDir) {
 }
 dataDir += network === 'testnet' ? 'testnet3' : '';
 
-var safeConfirmations = process.env.SAFE_CONFIRMATIONS || 6;
+var safeConfirmations = process.env.INSIGHT_SAFE_CONFIRMATIONS || 6;
+var ignoreCache      = process.env.INSIGHT_IGNORE_CACHE || 0;
 
 
 var bitcoindConf = {
@@ -88,7 +89,8 @@ console.log(
   # Configuration:\n\
 \t\tNetwork: %s\tINSIGHT_NETWORK\n\
 \t\tDatabase Path:  %s\tINSIGHT_DB\n\
-\t\tSafe Confirmations:  %s\tSAFE_CONFIRMATIONS\n\
+\t\tSafe Confirmations:  %s\tINSIGHT_SAFE_CONFIRMATIONS\n\
+\t\tIgnore Cache:  %s\tINSIGHT_IGNORE_CACHE\n\
  # Bicoind Connection configuration:\n\
 \t\tRPC Username: %s\tBITCOIND_USER\n\
 \t\tRPC Password: %s\tBITCOIND_PASS\n\
@@ -102,7 +104,7 @@ console.log(
  $ INSIGHT_NETWORK="testnet" BITCOIND_HOST="123.123.123.123" ./insight.js\
 \n\n',
 version,
-network, home, safeConfirmations,
+network, home, safeConfirmations, ignoreCache?'yes':'no',
 bitcoindConf.user,
 bitcoindConf.pass?'Yes(hidden)':'No',
 bitcoindConf.protocol,
@@ -139,4 +141,5 @@ module.exports = {
     segmentio: process.env.INSIGHT_SEGMENTIO_KEY
   },
   safeConfirmations: safeConfirmations, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations
+  ignoreCache: ignoreCache,
 };

lib/TransactionDb.js

@@ -201,7 +201,6 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) {
         i.value = ret.valueSat / util.COIN;
         valueIn += i.valueSat;
 
-console.log('[TransactionDb.js.204:ret:]',ret); //TODO
         if (ret.multipleSpentAttempt || !ret.spentTxId ||
           (ret.spentTxId && ret.spentTxId !== txInfo.txid)
         ) {
@@ -209,7 +208,6 @@ console.log('[TransactionDb.js.204:ret:]',ret); //TODO
             ret.multipleSpentAttempts.forEach(function(mul) {
               if (mul.spentTxId !== txInfo.txid) {
 
-console.log('[TransactionDb.js.210]'); //TODO
                 i.doubleSpentTxID = ret.spentTxId;
                 i.doubleSpentIndex = ret.spentIndex;
               }
@@ -218,7 +216,6 @@ console.log('[TransactionDb.js.210]'); //TODO
             i.dbError = 'Input spent not registered';
           } else {
 
-console.log('[TransactionDb.js.219]'); //TODO
             i.doubleSpentTxID = ret.spentTxId;
             i.doubleSpentIndex = ret.spentIndex;
           }
@@ -409,7 +406,7 @@ TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) {
 
 
 
-TransactionDb.prototype._parseAddrData = function(data) {
+TransactionDb.prototype._parseAddrData = function(data, ignoreCache) {
   var k = data.key.split('-');
   var v = data.value.split(':');
 // console.log('[TransactionDb.js.375]',data.key,data.value); //TODO
@@ -425,7 +422,7 @@ TransactionDb.prototype._parseAddrData = function(data) {
   //  v[1]== isConfirmedCached
   //  v[2]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs
   //  v[3]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey
-  if (v[1]){
+  if (v[1] && !ignoreCache){
     item.isConfirmed = 1;
     item.isConfirmedCached = 1;
     // console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO
@@ -448,7 +445,8 @@ TransactionDb.prototype._parseAddrData = function(data) {
   return item;
 };
 
-TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) {
+TransactionDb.prototype.fromAddr = function(addr, opts, cb) {
+  opts = opts || {};
   var self = this;
   var k = ADDR_PREFIX + addr + '-';
   var ret = [];
@@ -456,10 +454,10 @@ TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) {
   db.createReadStream({
     start: k,
     end: k + '~',
-    limit: txLimit>0 ? txLimit: -1,   // -1 means not limit
+    limit: opts.txLimit>0 ? opts.txLimit: -1,   // -1 means not limit
   })
     .on('data', function(data) {
-      ret.push(self._parseAddrData(data));
+      ret.push(self._parseAddrData(data, opts.ignoreCache));
     })
     .on('error', cb)
     .on('end', function() {

test/integration/addrCache.js

@@ -93,6 +93,17 @@ describe('Address cache ', function() {
       return done();
     },{txLimit:0});
   });
+  it('cache case 2 w ignore cache', function(done) {
+    var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb);
+    a.update(function(err) {
+      if (err) done(err);
+      a.balance.should.equal(0, 'balance');
+      a.totalReceived.should.equal(1376000, 'totalReceived');
+      a.txApperances.should.equal(8003, 'txApperances');
+      return done();
+    },{txLimit:0, ignoreCache:1});
+  });
+
 });