Merge pull request #463 from matiu/opt/dead-addr

Opt/dead addr

app/controllers/addresses.js

@@ -12,6 +12,8 @@ var async = require('async');
 var MAX_BATCH_SIZE = 100;
 var RPC_CONCURRENCY = 5;
 
+var SIZE_TO_ENABLE_DEAD_CACHE = 500;
+
 var tDb = require('../../lib/TransactionDb').default();
 
 var checkSync = function(req, res) {
@@ -47,8 +49,9 @@ var getAddrs = function(req, res, next) {
     var addrStrs = req.param('addrs');
     var s = addrStrs.split(',');
     if (s.length === 0) return as;
+    var enableDeadAddresses = s.length > SIZE_TO_ENABLE_DEAD_CACHE;
     for (var i = 0; i < s.length; i++) {
-      var a = new Address(s[i]);
+      var a = new Address(s[i], enableDeadAddresses);
       as.push(a);
     }
   } catch (e) {
@@ -189,6 +192,19 @@ exports.multitxs = function(req, res, next) {
       // no longer at bitcoind (for example a double spend)
 
       var transactions = _.compact(_.pluck(txs, 'info'));
+      //rm not used items
+      _.each(transactions, function(t) {
+        t.vin = _.map(t.vin, function(i) {
+          return _.pick(i, ['addr', 'valueSat']);
+        });
+        t.vout = _.map(t.vout, function(o) {
+          return _.pick(o, ['scriptPubKey', 'value']);
+        });
+        delete t.locktime;
+        delete t.version;
+      });
+
+
       transactions = {
         totalItems: nbTxs,
         from: +from,
@@ -205,7 +221,7 @@ exports.multitxs = function(req, res, next) {
 
   if (cache[addrStrs] && from > 0) {
     //logtime('Cache hit');
-    txs =cache[addrStrs]; 
+    txs = cache[addrStrs];
     return processTxs(txs, from, to, function(err, transactions) {
       //logtime('After process Txs');
       if (err) return common.handleErrors(err, res)
@@ -241,12 +257,12 @@ exports.multitxs = function(req, res, next) {
       });
 
       if (!cache[addrStrs] || from == 0) {
-        cache[addrStrs] = txs; 
+        cache[addrStrs] = txs;
         // 5 min. just to purge memory. Cache is overwritten in from=0 requests.
-        setTimeout(function(){
-          console.log('Deleting cache');
+        setTimeout(function() {
+          console.log('Deleting cache:', addrStrs.substr(0, 20));
           delete cache[addrStrs];
-        }, 5 * 60 * 1000); 
+        }, 5 * 60 * 1000);
       }
 
       processTxs(txs, from, to, function(err, transactions) {

app/models/Address.js

@@ -13,8 +13,20 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb').
 var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default();
 var config = require('../../config/config');
 var CONCURRENCY = 5;
+var DAYS_TO_DEAD = 40;
+var MAX_CACHE_KEYS = 10000;
+
+var deadCache = {};
+
+function Address(addrStr, deadCacheEnable) {
+
+  if (deadCacheEnable && deadCache[addrStr]) {
+//    console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached);
+    return deadCache[addrStr];
+  }
+
+  this.deadCacheEnable = deadCacheEnable;
 
-function Address(addrStr) {
   this.balanceSat = 0;
   this.totalReceivedSat = 0;
   this.totalSentSat = 0;
@@ -76,6 +88,38 @@ function Address(addrStr) {
 
 }
 
+
+Address.deleteDeadCache = function(addrStr) {
+  if (deadCache[addrStr]) {
+    console.log('Deleting Dead Address Cache', addrStr);
+    delete deadCache[addrStr];
+  }
+};
+
+
+Address.prototype.setCache = function() {
+  this.cached = true;
+  deadCache[this.addrStr] = this;
+
+  var size = _.keys(deadCache).length;
+
+  console.log('%%%%%%%% cache size:', size); //TODO
+  if (size > MAX_CACHE_KEYS) {
+    console.log('%%%%%%%% deleting ~ 20% of the entries...');
+
+    var skip = _.random(4);
+
+    for (var prop in deadCache)
+        if ( !( skip++ % 5 ) )
+          delete deadCache[prop];
+
+    size = _.keys(deadCache).length;
+    console.log('%%%%%%%% cache size:', size); //TODO
+  }
+  // TODO expire it...
+};
+
+
 Address.prototype.getObj = function() {
   // Normalize json address
   return {
@@ -165,8 +209,22 @@ Address.prototype.update = function(next, opts) {
   if (!('ignoreCache' in opts))
     opts.ignoreCache = config.ignoreCache;
 
+  if (opts.onlyUnspent && opts.includeTxInfo)
+    return cb('Bad params');
+
+  if (!opts.ignoreCache && this.cached) {
+    if (opts.onlyUnspent && this.unspent) {
+      return next();
+    }
+
+    if (opts.includeTxInfo && this.transactions.length) {
+      return next();
+    }
+  }
+
   // should collect txList from address?
   var txList = opts.txLimit === 0 ? null : [];
+  var lastUsage, now = Date.now() / 1000;
 
   var tDb = TransactionDb;
   var bDb = BlockDb;
@@ -180,13 +238,15 @@ Address.prototype.update = function(next, opts) {
         // console.log('[Address.js.161:txOut:]',txOut); //TODO
         if (err) return next(err);
         if (opts.onlyUnspent) {
-          txOut = txOut.filter(function(x) {
+
+          var unspent = _.filter(txOut,function(x) {
             return !x.spentTxId;
           });
-          tDb.fillScriptPubKey(txOut, function() {
+
+          tDb.fillScriptPubKey(unspent, function() {
             //_.filter will filterout unspend without scriptPubkey
             //(probably from double spends)
-            self.unspent = _.filter(txOut.map(function(x) {
+            self.unspent = _.filter(unspent.map(function(x) {
               return {
                 address: self.addrStr,
                 txid: x.txid,
@@ -198,14 +258,36 @@ Address.prototype.update = function(next, opts) {
                 confirmationsFromCache: !!x.isConfirmedCached,
               };
             }), 'scriptPubKey');;
+
+            if (self.deadCacheEnable && txOut.length && !self.unspent.length) {
+// console.log('$$$$$$$$$$$$$$$$$$$$$$$$$$$    ',self.addrStr); //TODO
+// console.log('[Address.js.242] NO UNSPENT:', self.addrStr, txOut.length); //TODO
+              // Asumes that addresses are ordered by Ts;
+              lastUsage = txOut[txOut.length-1].spentTs || now;
+
+              var daysOld = (now-lastUsage) / (3600 * 24);
+// console.log('[Address.js.253:dayOlds:]',daysOld); //TODO
+              var isOldEnough = daysOld >  DAYS_TO_DEAD;
+
+              // console.log('[Address.js.246:isOldEnough:]', isOldEnough, lastUsage, now); //TODO
+
+              if (isOldEnough) {
+                self.setCache();
+              }
+            }
             return next();
           });
         } else {
+
           txOut.forEach(function(txItem) {
             self._addTxItem(txItem, txList, opts.includeTxInfo);
           });
           if (txList)
             self.transactions = txList;
+          if (self.deadCacheEnable && self.cached) {
+            self.setCache();
+          }
+
           return next();
         }
       });

lib/Sync.js

@@ -6,6 +6,8 @@ var config        = imports.config || require('../config/config');
 var bitcore       = require('bitcore');
 var networks      = bitcore.networks;
 var async         = require('async');
+var _ = require('lodash');
+var Address = require('../app/models/Address');
 
 var logger = require('./logger').logger;
 var d      = logger.log;
@@ -293,7 +295,14 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) {
 
 //Store unconfirmed TXs
 Sync.prototype.storeTx = function(tx, cb) {
-  this.txDb.add(tx, cb);
+  this.txDb.add(tx, function(err, related) {
+    if (related) {
+      _.each(related, function(v,k){
+        Address.deleteDeadCache(k);
+      });
+    }
+    return cb(err, related);
+  });
 };