From 2dc2c4106c156e15367870837f47a229e981aa4e Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Mon, 29 Feb 2016 16:35:28 -0300 Subject: [PATCH 1/6] WIP --- app/controllers/addresses.js | 6 ++- app/models/Address.js | 89 ++++++++++++++++++++++++++++++++++-- lib/Sync.js | 11 ++++- 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index ca2fc3a..1a660b9 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -47,8 +47,10 @@ var getAddrs = function(req, res, next) { var addrStrs = req.param('addrs'); var s = addrStrs.split(','); if (s.length === 0) return as; + var enableDeadAddresses = s.length > 100; +console.log('[addresses.js.50:enableDeadAddresses:]',enableDeadAddresses); //TODO for (var i = 0; i < s.length; i++) { - var a = new Address(s[i]); + var a = new Address(s[i], enableDeadAddresses); as.push(a); } } catch (e) { @@ -244,7 +246,7 @@ exports.multitxs = function(req, res, next) { cache[addrStrs] = txs; // 5 min. just to purge memory. Cache is overwritten in from=0 requests. setTimeout(function(){ - console.log('Deleting cache'); + console.log('Deleting cache:', addrStrs.substr(0,20)); delete cache[addrStrs]; }, 5 * 60 * 1000); } diff --git a/app/models/Address.js b/app/models/Address.js index 780daf7..305faf5 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,8 +13,19 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; +var DAYS_TO_DEAD = 2; + +var deadCache = {}; + +function Address(addrStr, deadCacheEnable) { + + if (deadCacheEnable && deadCache[addrStr]) { + console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached); + return deadCache[addrStr]; + } + + this.deadCacheEnable = deadCacheEnable; -function Address(addrStr) { this.balanceSat = 0; this.totalReceivedSat = 0; this.totalSentSat = 0; @@ -76,6 +87,26 @@ function Address(addrStr) { } + +Address.deleteDeadCache = function(addrStr) { + if (deadCache[addrStr]) { + console.log('Deleting Dead Address Cache', addrStr); + delete deadCache[addrStr]; + } +}; + + +Address.prototype.setCache = function() { + this.cached = true; + deadCache[this.addrStr] = this; + + console.log('%%%%%%%% setting DEAD cache for ', this.addrStr, this.unspent.length, this.transactions.length); //TODO + console.log('%%%%%%%% cache size:', _.keys(deadCache).length); //TODO + + // TODO expire it... +}; + + Address.prototype.getObj = function() { // Normalize json address return { @@ -165,8 +196,28 @@ Address.prototype.update = function(next, opts) { if (!('ignoreCache' in opts)) opts.ignoreCache = config.ignoreCache; + if (opts.onlyUnspent && opts.includeTxInfo) + return cb('Bad params'); + + if (!opts.ignoreCache && this.cached) { + +console.log('[Address.js.203] CACHED????', this.addrStr, opts.onlyUnspent, opts.includeTxInfo); //TODO + + + if (opts.onlyUnspent && this.unspent) { +console.log('[Address.js.206] YES (unspent)'); //TODO + return next(); + } + + if (opts.includeTxInfo && this.transactions.length) { +console.log('[Address.js.206] YES (TXS)'); //TODO + return next(); + } + } + // should collect txList from address? var txList = opts.txLimit === 0 ? null : []; + var lastUsage, now = Date.now() / 1000; var tDb = TransactionDb; var bDb = BlockDb; @@ -180,13 +231,15 @@ Address.prototype.update = function(next, opts) { // console.log('[Address.js.161:txOut:]',txOut); //TODO if (err) return next(err); if (opts.onlyUnspent) { - txOut = txOut.filter(function(x) { + + var unspent = _.filter(txOut,function(x) { return !x.spentTxId; }); - tDb.fillScriptPubKey(txOut, function() { + + tDb.fillScriptPubKey(unspent, function() { //_.filter will filterout unspend without scriptPubkey //(probably from double spends) - self.unspent = _.filter(txOut.map(function(x) { + self.unspent = _.filter(unspent.map(function(x) { return { address: self.addrStr, txid: x.txid, @@ -198,14 +251,42 @@ Address.prototype.update = function(next, opts) { confirmationsFromCache: !!x.isConfirmedCached, }; }), 'scriptPubKey');; + + if (self.deadCacheEnable && txOut.length && !self.unspent.length) { +// console.log('$$$$$$$$$$$$$$$$$$$$$$$$$$$ ',self.addrStr); //TODO +// console.log('[Address.js.242] NO UNSPENT:', self.addrStr, txOut.length); //TODO + // Asumes that addresses are ordered by Ts; + lastUsage = txOut[txOut.length-1].spentTs || now; + + var daysOld = (now-lastUsage) / (3600 * 24); +// console.log('[Address.js.253:dayOlds:]',daysOld); //TODO + var isOldEnough = daysOld > DAYS_TO_DEAD; + + // console.log('[Address.js.246:isOldEnough:]', isOldEnough, lastUsage, now); //TODO + + if (isOldEnough) { + self.setCache(); + } + } return next(); }); } else { + +console.log('[Address.js.273]'); //TODO txOut.forEach(function(txItem) { self._addTxItem(txItem, txList, opts.includeTxInfo); }); if (txList) self.transactions = txList; + + +console.log('[Address.js.279]', self.addrStr, self.deadCacheEnable , self.cached); //TODO + if (self.deadCacheEnable && self.cached) { + +console.log('[Address.js.281] WASS DEAD ALREADY! CACHING HISTORY'); //TODO + self.setCache(); + } + return next(); } }); diff --git a/lib/Sync.js b/lib/Sync.js index 36b0fa7..5795f4d 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -6,6 +6,8 @@ var config = imports.config || require('../config/config'); var bitcore = require('bitcore'); var networks = bitcore.networks; var async = require('async'); +var _ = require('lodash'); +var Address = require('../app/models/Address'); var logger = require('./logger').logger; var d = logger.log; @@ -293,7 +295,14 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { //Store unconfirmed TXs Sync.prototype.storeTx = function(tx, cb) { - this.txDb.add(tx, cb); + this.txDb.add(tx, function(err, related) { + if (related) { + _.each(related, function(v,k){ + Address.deleteDeadCache(k); + }); + } + return cb(err, related); + }); }; From f23f114e9e56639ce4d710d933ce096174c5dc37 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 1 Mar 2016 16:25:53 -0300 Subject: [PATCH 2/6] rm logs --- app/controllers/addresses.js | 24 ++++++++++++++++++------ app/models/Address.js | 18 ++---------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 1a660b9..e348470 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -48,7 +48,6 @@ var getAddrs = function(req, res, next) { var s = addrStrs.split(','); if (s.length === 0) return as; var enableDeadAddresses = s.length > 100; -console.log('[addresses.js.50:enableDeadAddresses:]',enableDeadAddresses); //TODO for (var i = 0; i < s.length; i++) { var a = new Address(s[i], enableDeadAddresses); as.push(a); @@ -191,6 +190,19 @@ exports.multitxs = function(req, res, next) { // no longer at bitcoind (for example a double spend) var transactions = _.compact(_.pluck(txs, 'info')); + //rm not used items + _.each(transactions, function(t) { + t.vin = _.map(t.vin, function(i) { + return _.pick(i, ['addr', 'valueSat']); + }); + t.vout = _.map(t.vout, function(o) { + return _.pick(o, ['scriptPubKey', 'value']); + }); + delete t.locktime; + delete t.version; + }); + + transactions = { totalItems: nbTxs, from: +from, @@ -207,7 +219,7 @@ exports.multitxs = function(req, res, next) { if (cache[addrStrs] && from > 0) { //logtime('Cache hit'); - txs =cache[addrStrs]; + txs = cache[addrStrs]; return processTxs(txs, from, to, function(err, transactions) { //logtime('After process Txs'); if (err) return common.handleErrors(err, res) @@ -243,12 +255,12 @@ exports.multitxs = function(req, res, next) { }); if (!cache[addrStrs] || from == 0) { - cache[addrStrs] = txs; + cache[addrStrs] = txs; // 5 min. just to purge memory. Cache is overwritten in from=0 requests. - setTimeout(function(){ - console.log('Deleting cache:', addrStrs.substr(0,20)); + setTimeout(function() { + console.log('Deleting cache:', addrStrs.substr(0, 20)); delete cache[addrStrs]; - }, 5 * 60 * 1000); + }, 5 * 60 * 1000); } processTxs(txs, from, to, function(err, transactions) { diff --git a/app/models/Address.js b/app/models/Address.js index 305faf5..0e15163 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,14 +13,14 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; -var DAYS_TO_DEAD = 2; +var DAYS_TO_DEAD = 40; var deadCache = {}; function Address(addrStr, deadCacheEnable) { if (deadCacheEnable && deadCache[addrStr]) { - console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached); +// console.log('DEAD CACHE HIT:', addrStr, deadCache[addrStr].cached); return deadCache[addrStr]; } @@ -100,9 +100,7 @@ Address.prototype.setCache = function() { this.cached = true; deadCache[this.addrStr] = this; - console.log('%%%%%%%% setting DEAD cache for ', this.addrStr, this.unspent.length, this.transactions.length); //TODO console.log('%%%%%%%% cache size:', _.keys(deadCache).length); //TODO - // TODO expire it... }; @@ -200,17 +198,11 @@ Address.prototype.update = function(next, opts) { return cb('Bad params'); if (!opts.ignoreCache && this.cached) { - -console.log('[Address.js.203] CACHED????', this.addrStr, opts.onlyUnspent, opts.includeTxInfo); //TODO - - if (opts.onlyUnspent && this.unspent) { -console.log('[Address.js.206] YES (unspent)'); //TODO return next(); } if (opts.includeTxInfo && this.transactions.length) { -console.log('[Address.js.206] YES (TXS)'); //TODO return next(); } } @@ -272,18 +264,12 @@ console.log('[Address.js.206] YES (TXS)'); //TODO }); } else { -console.log('[Address.js.273]'); //TODO txOut.forEach(function(txItem) { self._addTxItem(txItem, txList, opts.includeTxInfo); }); if (txList) self.transactions = txList; - - -console.log('[Address.js.279]', self.addrStr, self.deadCacheEnable , self.cached); //TODO if (self.deadCacheEnable && self.cached) { - -console.log('[Address.js.281] WASS DEAD ALREADY! CACHING HISTORY'); //TODO self.setCache(); } From 0dc39ce84baea997ef5cca821e5abff97a365f12 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 1 Mar 2016 17:21:41 -0300 Subject: [PATCH 3/6] . --- app/controllers/addresses.js | 3 ++- app/models/Address.js | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index e348470..90618dd 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -11,6 +11,7 @@ var async = require('async'); var MAX_BATCH_SIZE = 100; var RPC_CONCURRENCY = 5; +var SIZE_TO_ENABLE_DEAD_CACHE=100; var tDb = require('../../lib/TransactionDb').default(); @@ -47,7 +48,7 @@ var getAddrs = function(req, res, next) { var addrStrs = req.param('addrs'); var s = addrStrs.split(','); if (s.length === 0) return as; - var enableDeadAddresses = s.length > 100; + var enableDeadAddresses = s.length > SIZE_TO_ENABLE_DEAD_CACHE; for (var i = 0; i < s.length; i++) { var a = new Address(s[i], enableDeadAddresses); as.push(a); diff --git a/app/models/Address.js b/app/models/Address.js index 0e15163..98d10a8 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,7 +13,9 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; -var DAYS_TO_DEAD = 40; +//var DAYS_TO_DEAD = 40; +var DAYS_TO_DEAD = 1; +var MAX_CACHE_KEYS = 100; var deadCache = {}; @@ -100,7 +102,18 @@ Address.prototype.setCache = function() { this.cached = true; deadCache[this.addrStr] = this; - console.log('%%%%%%%% cache size:', _.keys(deadCache).length); //TODO + var size = _.keys(deadCache).length; + + console.log('%%%%%%%% cache size:', size); //TODO + if (size > MAX_CACHE_KEYS) { + console.log('%%%%%%%% deleting ~ 20% of the entries...'); + + for (var prop in deadCache) + if (Math.random() < 0.2) + delete deadCache[prop]; + + console.log('%%%%%%%% cache size:', size); //TODO + } // TODO expire it... }; From 7b0dfeccbc0dacd922bb2fda28c96fee8f650cae Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 1 Mar 2016 17:40:40 -0300 Subject: [PATCH 4/6] update limit --- app/controllers/addresses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 90618dd..3af264f 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -11,7 +11,7 @@ var async = require('async'); var MAX_BATCH_SIZE = 100; var RPC_CONCURRENCY = 5; -var SIZE_TO_ENABLE_DEAD_CACHE=100; +var SIZE_TO_ENABLE_DEAD_CACHE = 500; var tDb = require('../../lib/TransactionDb').default(); From 873fee2fb3eab23364fde6ebe6dc5b9e71ca9c12 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Tue, 1 Mar 2016 18:45:42 -0300 Subject: [PATCH 5/6] faster cache purge --- app/controllers/addresses.js | 1 + app/models/Address.js | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 3af264f..92570b2 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -11,6 +11,7 @@ var async = require('async'); var MAX_BATCH_SIZE = 100; var RPC_CONCURRENCY = 5; + var SIZE_TO_ENABLE_DEAD_CACHE = 500; var tDb = require('../../lib/TransactionDb').default(); diff --git a/app/models/Address.js b/app/models/Address.js index 98d10a8..9252689 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -13,9 +13,8 @@ var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb'). var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); var config = require('../../config/config'); var CONCURRENCY = 5; -//var DAYS_TO_DEAD = 40; -var DAYS_TO_DEAD = 1; -var MAX_CACHE_KEYS = 100; +var DAYS_TO_DEAD = 40; +var MAX_CACHE_KEYS = 10000; var deadCache = {}; @@ -108,10 +107,13 @@ Address.prototype.setCache = function() { if (size > MAX_CACHE_KEYS) { console.log('%%%%%%%% deleting ~ 20% of the entries...'); + var skip = _.random(4), i=0; + for (var prop in deadCache) - if (Math.random() < 0.2) + if ( !( skip++ % 5 ) ) delete deadCache[prop]; + size = _.keys(deadCache).length; console.log('%%%%%%%% cache size:', size); //TODO } // TODO expire it... From 530b4407e8de440e4b0dc1bb602211ff71e1acb6 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 3 Mar 2016 09:54:35 -0300 Subject: [PATCH 6/6] rm unused var --- app/models/Address.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/Address.js b/app/models/Address.js index 9252689..a218563 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -107,7 +107,7 @@ Address.prototype.setCache = function() { if (size > MAX_CACHE_KEYS) { console.log('%%%%%%%% deleting ~ 20% of the entries...'); - var skip = _.random(4), i=0; + var skip = _.random(4); for (var prop in deadCache) if ( !( skip++ % 5 ) )