Merge pull request #479 from matiu/prod2-fix-dead-cache

only cache addresses with balanceSat=0

app/controllers/addresses.js

@@ -12,6 +12,8 @@ var async = require('async');
 var MAX_BATCH_SIZE = 100;
 var RPC_CONCURRENCY = 5;
 
+var SIZE_TO_ENABLE_DEAD_CACHE = 500;
+
 var tDb = require('../../lib/TransactionDb').default();
 
 var checkSync = function(req, res) {
@@ -47,8 +49,7 @@ var getAddrs = function(req, res, next) {
     var addrStrs = req.param('addrs');
     var s = addrStrs.split(',');
     if (s.length === 0) return as;
-    var enableDeadAddresses = s.length > 100;
-console.log('[addresses.js.50:enableDeadAddresses:]',enableDeadAddresses); //TODO
+    var enableDeadAddresses = s.length > SIZE_TO_ENABLE_DEAD_CACHE;
     for (var i = 0; i < s.length; i++) {
       var a = new Address(s[i], enableDeadAddresses);
       as.push(a);
@@ -191,6 +192,19 @@ exports.multitxs = function(req, res, next) {
       // no longer at bitcoind (for example a double spend)
 
       var transactions = _.compact(_.pluck(txs, 'info'));
+      //rm not used items
+      _.each(transactions, function(t) {
+        t.vin = _.map(t.vin, function(i) {
+          return _.pick(i, ['addr', 'valueSat']);
+        });
+        t.vout = _.map(t.vout, function(o) {
+          return _.pick(o, ['scriptPubKey', 'value']);
+        });
+        delete t.locktime;
+        delete t.version;
+      });
+
+
       transactions = {
         totalItems: nbTxs,
         from: +from,
@@ -207,7 +221,7 @@ exports.multitxs = function(req, res, next) {
 
   if (cache[addrStrs] && from > 0) {
     //logtime('Cache hit');
-    txs =cache[addrStrs]; 
+    txs = cache[addrStrs];
     return processTxs(txs, from, to, function(err, transactions) {
       //logtime('After process Txs');
       if (err) return common.handleErrors(err, res)
@@ -243,12 +257,12 @@ exports.multitxs = function(req, res, next) {
       });
 
       if (!cache[addrStrs] || from == 0) {
-        cache[addrStrs] = txs; 
+        cache[addrStrs] = txs;
         // 5 min. just to purge memory. Cache is overwritten in from=0 requests.
-        setTimeout(function(){
-          console.log('Deleting cache:', addrStrs.substr(0,20));
+        setTimeout(function() {
+          console.log('Deleting cache:', addrStrs.substr(0, 20));
           delete cache[addrStrs];
-        }, 5 * 60 * 1000); 
+        }, 5 * 60 * 1000);
       }
 
       processTxs(txs, from, to, function(err, transactions) {

app/models/Address.js

@@ -13,14 +13,15 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb').
 var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default();
 var config = require('../../config/config');
 var CONCURRENCY = 5;
-var DAYS_TO_DEAD = 2;
+var DAYS_TO_DEAD = 40;
+var MAX_CACHE_KEYS = 50000;
 
 var deadCache = {};
 
 function Address(addrStr, deadCacheEnable) {
 
   if (deadCacheEnable && deadCache[addrStr]) {
-    console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached);
+    //    console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached);
     return deadCache[addrStr];
   }
 
@@ -97,12 +98,25 @@ Address.deleteDeadCache = function(addrStr) {
 
 
 Address.prototype.setCache = function() {
+
   this.cached = true;
   deadCache[this.addrStr] = this;
 
-  console.log('%%%%%%%% setting DEAD cache for ', this.addrStr, this.unspent.length, this.transactions.length); //TODO
-  console.log('%%%%%%%% cache size:', _.keys(deadCache).length); //TODO
+  var size = _.keys(deadCache).length;
 
+  console.log('%%%%%%%% cache size:', size); //TODO
+  if (size > MAX_CACHE_KEYS) {
+    console.log('%%%%%%%% deleting ~ 20% of the entries...');
+
+    var skip = _.random(4);
+
+    for (var prop in deadCache)
+      if (!(skip++ % 5))
+        delete deadCache[prop];
+
+    size = _.keys(deadCache).length;
+    console.log('%%%%%%%% cache size after delete:', size); //TODO
+  }
   // TODO expire it...
 };
 
@@ -200,17 +214,11 @@ Address.prototype.update = function(next, opts) {
     return cb('Bad params');
 
   if (!opts.ignoreCache && this.cached) {
-
-console.log('[Address.js.203] CACHED????', this.addrStr, opts.onlyUnspent, opts.includeTxInfo); //TODO
-
-
     if (opts.onlyUnspent && this.unspent) {
-console.log('[Address.js.206] YES (unspent)'); //TODO
       return next();
     }
 
-    if (opts.includeTxInfo && this.transactions.length) {
-console.log('[Address.js.206] YES (TXS)'); //TODO
+    if (opts.includeTxInfo && this.transactions.length && this.balanceSat == 0) {
       return next();
     }
   }
@@ -232,7 +240,7 @@ console.log('[Address.js.206] YES (TXS)'); //TODO
         if (err) return next(err);
         if (opts.onlyUnspent) {
 
-          var unspent = _.filter(txOut,function(x) {
+          var unspent = _.filter(txOut, function(x) {
             return !x.spentTxId;
           });
 
@@ -250,17 +258,17 @@ console.log('[Address.js.206] YES (TXS)'); //TODO
                 confirmations: x.isConfirmedCached ? (config.safeConfirmations) : x.confirmations,
                 confirmationsFromCache: !!x.isConfirmedCached,
               };
-            }), 'scriptPubKey');;
+            }), 'scriptPubKey');
 
             if (self.deadCacheEnable && txOut.length && !self.unspent.length) {
-// console.log('$$$$$$$$$$$$$$$$$$$$$$$$$$$    ',self.addrStr); //TODO
-// console.log('[Address.js.242] NO UNSPENT:', self.addrStr, txOut.length); //TODO
+              // console.log('$$$$$$$$$$$$$$$$$$$$$$$$$$$    ',self.addrStr); //TODO
+              // console.log('[Address.js.242] NO UNSPENT:', self.addrStr, txOut.length); //TODO
               // Asumes that addresses are ordered by Ts;
-              lastUsage = txOut[txOut.length-1].spentTs || now;
+              lastUsage = txOut[txOut.length - 1].spentTs || now;
 
-              var daysOld = (now-lastUsage) / (3600 * 24);
-// console.log('[Address.js.253:dayOlds:]',daysOld); //TODO
-              var isOldEnough = daysOld >  DAYS_TO_DEAD;
+              var daysOld = (now - lastUsage) / (3600 * 24);
+              // console.log('[Address.js.253:dayOlds:]',daysOld); //TODO
+              var isOldEnough = daysOld > DAYS_TO_DEAD;
 
               // console.log('[Address.js.246:isOldEnough:]', isOldEnough, lastUsage, now); //TODO
 
@@ -272,18 +280,13 @@ console.log('[Address.js.206] YES (TXS)'); //TODO
           });
         } else {
 
-console.log('[Address.js.273]'); //TODO
           txOut.forEach(function(txItem) {
             self._addTxItem(txItem, txList, opts.includeTxInfo);
           });
           if (txList)
             self.transactions = txList;
 
-
-console.log('[Address.js.279]', self.addrStr, self.deadCacheEnable , self.cached); //TODO
-          if (self.deadCacheEnable && self.cached) {
-
-console.log('[Address.js.281] WASS DEAD ALREADY! CACHING HISTORY'); //TODO
+          if (self.deadCacheEnable && self.cached && self.balanceSat == 0) {
             self.setCache();
           }