diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index ca2fc3a..92570b2 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -12,6 +12,8 @@ var async = require('async'); var MAX_BATCH_SIZE = 100; var RPC_CONCURRENCY = 5; +var SIZE_TO_ENABLE_DEAD_CACHE = 500; + var tDb = require('../../lib/TransactionDb').default(); var checkSync = function(req, res) { @@ -47,8 +49,9 @@ var getAddrs = function(req, res, next) { var addrStrs = req.param('addrs'); var s = addrStrs.split(','); if (s.length === 0) return as; + var enableDeadAddresses = s.length > SIZE_TO_ENABLE_DEAD_CACHE; for (var i = 0; i < s.length; i++) { - var a = new Address(s[i]); + var a = new Address(s[i], enableDeadAddresses); as.push(a); } } catch (e) { @@ -189,6 +192,19 @@ exports.multitxs = function(req, res, next) { // no longer at bitcoind (for example a double spend) var transactions = _.compact(_.pluck(txs, 'info')); + //rm not used items + _.each(transactions, function(t) { + t.vin = _.map(t.vin, function(i) { + return _.pick(i, ['addr', 'valueSat']); + }); + t.vout = _.map(t.vout, function(o) { + return _.pick(o, ['scriptPubKey', 'value']); + }); + delete t.locktime; + delete t.version; + }); + + transactions = { totalItems: nbTxs, from: +from, @@ -205,7 +221,7 @@ exports.multitxs = function(req, res, next) { if (cache[addrStrs] && from > 0) { //logtime('Cache hit'); - txs =cache[addrStrs]; + txs = cache[addrStrs]; return processTxs(txs, from, to, function(err, transactions) { //logtime('After process Txs'); if (err) return common.handleErrors(err, res) @@ -241,12 +257,12 @@ exports.multitxs = function(req, res, next) { }); if (!cache[addrStrs] || from == 0) { - cache[addrStrs] = txs; + cache[addrStrs] = txs; // 5 min. just to purge memory. Cache is overwritten in from=0 requests. - setTimeout(function(){ - console.log('Deleting cache'); + setTimeout(function() { + console.log('Deleting cache:', addrStrs.substr(0, 20)); delete cache[addrStrs]; - }, 5 * 60 * 1000); + }, 5 * 60 * 1000); } processTxs(txs, from, to, function(err, transactions) { diff --git a/app/models/Address.js b/app/models/Address.js index 780daf7..a218563 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,8 +13,20 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; +var DAYS_TO_DEAD = 40; +var MAX_CACHE_KEYS = 10000; + +var deadCache = {}; + +function Address(addrStr, deadCacheEnable) { + + if (deadCacheEnable && deadCache[addrStr]) { +// console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached); + return deadCache[addrStr]; + } + + this.deadCacheEnable = deadCacheEnable; -function Address(addrStr) { this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -76,6 +88,38 @@ function Address(addrStr) { } + +Address.deleteDeadCache = function(addrStr) { + if (deadCache[addrStr]) { + console.log('Deleting Dead Address Cache', addrStr); + delete deadCache[addrStr]; + } +}; + + +Address.prototype.setCache = function() { + this.cached = true; + deadCache[this.addrStr] = this; + + var size = _.keys(deadCache).length; + + console.log('%%%%%%%% cache size:', size); //TODO + if (size > MAX_CACHE_KEYS) { + console.log('%%%%%%%% deleting ~ 20% of the entries...'); + + var skip = _.random(4); + + for (var prop in deadCache) + if ( !( skip++ % 5 ) ) + delete deadCache[prop]; + + size = _.keys(deadCache).length; + console.log('%%%%%%%% cache size:', size); //TODO + } + // TODO expire it... +}; + + Address.prototype.getObj = function() { // Normalize json address return { @@ -165,8 +209,22 @@ Address.prototype.update = function(next, opts) { if (!('ignoreCache' in opts)) opts.ignoreCache = config.ignoreCache; + if (opts.onlyUnspent && opts.includeTxInfo) + return cb('Bad params'); + + if (!opts.ignoreCache && this.cached) { + if (opts.onlyUnspent && this.unspent) { + return next(); + } + + if (opts.includeTxInfo && this.transactions.length) { + return next(); + } + } + // should collect txList from address? var txList = opts.txLimit === 0 ? null : []; + var lastUsage, now = Date.now() / 1000; var tDb = TransactionDb; var bDb = BlockDb; @@ -180,13 +238,15 @@ Address.prototype.update = function(next, opts) { // console.log('[Address.js.161:txOut:]',txOut); //TODO if (err) return next(err); if (opts.onlyUnspent) { - txOut = txOut.filter(function(x) { + + var unspent = _.filter(txOut,function(x) { return !x.spentTxId; }); - tDb.fillScriptPubKey(txOut, function() { + + tDb.fillScriptPubKey(unspent, function() { //_.filter will filterout unspend without scriptPubkey //(probably from double spends) - self.unspent = _.filter(txOut.map(function(x) { + self.unspent = _.filter(unspent.map(function(x) { return { address: self.addrStr, txid: x.txid, @@ -198,14 +258,36 @@ Address.prototype.update = function(next, opts) { confirmationsFromCache: !!x.isConfirmedCached, }; }), 'scriptPubKey');; + + if (self.deadCacheEnable && txOut.length && !self.unspent.length) { +// console.log('$$$$$$$$$$$$$$$$$$$$$$$$$$$ ',self.addrStr); //TODO +// console.log('[Address.js.242] NO UNSPENT:', self.addrStr, txOut.length); //TODO + // Asumes that addresses are ordered by Ts; + lastUsage = txOut[txOut.length-1].spentTs || now; + + var daysOld = (now-lastUsage) / (3600 * 24); +// console.log('[Address.js.253:dayOlds:]',daysOld); //TODO + var isOldEnough = daysOld > DAYS_TO_DEAD; + + // console.log('[Address.js.246:isOldEnough:]', isOldEnough, lastUsage, now); //TODO + + if (isOldEnough) { + self.setCache(); + } + } return next(); }); } else { + txOut.forEach(function(txItem) { self._addTxItem(txItem, txList, opts.includeTxInfo); }); if (txList) self.transactions = txList; + if (self.deadCacheEnable && self.cached) { + self.setCache(); + } + return next(); } }); diff --git a/lib/Sync.js b/lib/Sync.js index 36b0fa7..5795f4d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -6,6 +6,8 @@ var config = imports.config || require('../config/config'); var bitcore = require('bitcore'); var networks = bitcore.networks; var async = require('async'); +var _ = require('lodash'); +var Address = require('../app/models/Address'); var logger = require('./logger').logger; var d = logger.log; @@ -293,7 +295,14 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { //Store unconfirmed TXs Sync.prototype.storeTx = function(tx, cb) { - this.txDb.add(tx, cb); + this.txDb.add(tx, function(err, related) { + if (related) { + _.each(related, function(v,k){ + Address.deleteDeadCache(k); + }); + } + return cb(err, related); + }); };