From e7700b374a8ca557af5fc371ab10fd6551e577d0 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Wed, 27 Aug 2014 15:41:07 -0300
Subject: [PATCH 1/9] add cron

---
 package.json | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/package.json b/package.json
index a9150aa..ec43f30 100644
--- a/package.json
+++ b/package.json
@@ -59,10 +59,10 @@
     "buffertools": "*",
     "commander": "^2.3.0",
     "connect-ratelimit": "git://github.com/dharmafly/connect-ratelimit.git#0550eff209c54f35078f46445000797fa942ab97",
+    "cron": "^1.0.4",
     "express": "~3.4.7",
     "glob": "*",
-    "leveldown": "~0.10.0",
-    "levelup": "~0.19.0",
+    "microtime": "^0.6.0",
     "moment": "~2.5.0",
     "preconditions": "^1.0.7",
     "should": "~2.1.1",
@@ -71,8 +71,7 @@
     "socket.io-client": "1.0.6",
     "soop": "=0.1.5",
     "winston": "*",
-    "xmlhttprequest": "~1.6.0",
-    "microtime": "^0.6.0"
+    "xmlhttprequest": "~1.6.0"
   },
   "devDependencies": {
     "chai": "*",

From c36c7d4ab47cdce4d1d720dc851138a61f299383 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 11:36:25 -0300
Subject: [PATCH 2/9] starting with cleaner plugin

---
 config/config.js              | 5 +++++
 insight.js                    | 4 ++++
 lib/MessageDb.js              | 4 ++++
 plugins/cleaner.js            | 9 +++++++++
 plugins/config-cleaner.js     | 3 +++
 plugins/config-mailbox.js     | 3 +++
 plugins/config-ratelimiter.js | 3 +++
 7 files changed, 31 insertions(+)
 create mode 100644 plugins/cleaner.js
 create mode 100644 plugins/config-cleaner.js
 create mode 100644 plugins/config-mailbox.js
 create mode 100644 plugins/config-ratelimiter.js

diff --git a/config/config.js b/config/config.js
index 3496723..a639f2d 100644
--- a/config/config.js
+++ b/config/config.js
@@ -76,6 +76,7 @@ var bitcoindConf = {
   disableAgent: true
 };
 
+var enableCleaner = process.env.ENABLE_CLEANER === 'true';
 var enableMailbox = process.env.ENABLE_MAILBOX === 'true';
 var enableRatelimiter = process.env.ENABLE_RATELIMITER === 'true';
 var loggerLevel = process.env.LOGGER_LEVEL || 'info';
@@ -92,8 +93,12 @@ if (!fs.existsSync(db)) {
 }
 
 module.exports = {
+  enableCleaner: enableCleaner,
+  cleaner: require('../plugins/config-cleaner.js'),
   enableMailbox: enableMailbox,
+  mailbox: require('../plugins/config-mailbox.js'),
   enableRatelimiter: enableRatelimiter,
+  ratelimiter: require('../plugins/config-ratelimiter.js'),
   loggerLevel: loggerLevel,
   enableHTTPS: enableHTTPS,
   version: version,
diff --git a/insight.js b/insight.js
index 5fd9011..2b810a1 100755
--- a/insight.js
+++ b/insight.js
@@ -135,6 +135,10 @@ if (config.enableMailbox) {
   require('./plugins/mailbox').init(ios, config.mailbox);
 }
 
+if (config.enableCleaner) {
+  require('./plugins/cleaner').init(config.cleaner);
+}
+
 
 
 // express settings
diff --git a/lib/MessageDb.js b/lib/MessageDb.js
index a7afe88..75ca0ec 100644
--- a/lib/MessageDb.js
+++ b/lib/MessageDb.js
@@ -116,4 +116,8 @@ MessageDb.prototype.getMessages = function(to, lower_ts, upper_ts, cb) {
     });
 };
 
+MessageDb.prototype.remove = function(ts) {
+  
+};
+
 module.exports = soop(MessageDb);
diff --git a/plugins/cleaner.js b/plugins/cleaner.js
new file mode 100644
index 0000000..ba8321d
--- /dev/null
+++ b/plugins/cleaner.js
@@ -0,0 +1,9 @@
+var microtime = require('microtime');
+var mdb = require('../lib/MessageDb').default();
+var logger = require('../lib/logger').logger;
+var preconditions = require('preconditions').singleton();
+
+module.exports.init = function(config) {
+  logger.info('Using cleaner plugin');
+};
+
diff --git a/plugins/config-cleaner.js b/plugins/config-cleaner.js
new file mode 100644
index 0000000..f4d6253
--- /dev/null
+++ b/plugins/config-cleaner.js
@@ -0,0 +1,3 @@
+module.exports = {
+
+};
diff --git a/plugins/config-mailbox.js b/plugins/config-mailbox.js
new file mode 100644
index 0000000..f4d6253
--- /dev/null
+++ b/plugins/config-mailbox.js
@@ -0,0 +1,3 @@
+module.exports = {
+
+};
diff --git a/plugins/config-ratelimiter.js b/plugins/config-ratelimiter.js
new file mode 100644
index 0000000..f4d6253
--- /dev/null
+++ b/plugins/config-ratelimiter.js
@@ -0,0 +1,3 @@
+module.exports = {
+
+};

From 1ce26a9a6deaf63ccb259d33322290849cd5a46b Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 14:09:58 -0300
Subject: [PATCH 3/9] cleaner plugin: dependencies

---
 plugins/cleaner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/cleaner.js b/plugins/cleaner.js
index ba8321d..9609557 100644
--- a/plugins/cleaner.js
+++ b/plugins/cleaner.js
@@ -1,7 +1,7 @@
-var microtime = require('microtime');
 var mdb = require('../lib/MessageDb').default();
 var logger = require('../lib/logger').logger;
 var preconditions = require('preconditions').singleton();
+var cron = require('cron');
 
 module.exports.init = function(config) {
   logger.info('Using cleaner plugin');

From 349972a047eea38367b6f3bb8a20e8dabe647001 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 14:34:20 -0300
Subject: [PATCH 4/9] cleaner plugin: basic wires

---
 plugins/cleaner.js        | 14 +++++++++++++-
 plugins/config-cleaner.js |  2 ++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/plugins/cleaner.js b/plugins/cleaner.js
index 9609557..bb00719 100644
--- a/plugins/cleaner.js
+++ b/plugins/cleaner.js
@@ -2,8 +2,20 @@ var mdb = require('../lib/MessageDb').default();
 var logger = require('../lib/logger').logger;
 var preconditions = require('preconditions').singleton();
 var cron = require('cron');
+var CronJob = cron.CronJob;
 
+
+var doClean = function() {
+  
+}
+  
 module.exports.init = function(config) {
   logger.info('Using cleaner plugin');
+  logger.info(config);
+  var job = new CronJob({
+    cronTime: config.cronTime || '0 * * * *',
+    onTick: doClean,
+    start: true
+  });
+  doClean();
 };
-
diff --git a/plugins/config-cleaner.js b/plugins/config-cleaner.js
index f4d6253..b550cf3 100644
--- a/plugins/config-cleaner.js
+++ b/plugins/config-cleaner.js
@@ -1,3 +1,5 @@
 module.exports = {
 
+  cronTime: '* * * * *'
+
 };

From fe682c173f5781629022b1c28b520d9cef754843 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 17:33:46 -0300
Subject: [PATCH 5/9] cleaner plugin: optimize getMessages

---
 lib/MessageDb.js | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/MessageDb.js b/lib/MessageDb.js
index 75ca0ec..fd77530 100644
--- a/lib/MessageDb.js
+++ b/lib/MessageDb.js
@@ -97,10 +97,10 @@ MessageDb.prototype.getMessages = function(to, lower_ts, upper_ts, cb) {
   var list = [];
   lower_ts = lower_ts || 1;
   var opts = {
-    end: messageKey(to, lower_ts),
-    start: messageKey(to, upper_ts),
+    start: messageKey(to, lower_ts),
+    end: messageKey(to, upper_ts),
     // limit: limit, TODO
-    reverse: true,
+    reverse: false,
   };
 
   db.createReadStream(opts)
@@ -112,12 +112,13 @@ MessageDb.prototype.getMessages = function(to, lower_ts, upper_ts, cb) {
       return cb(err);
     })
     .on('end', function() {
-      return cb(null, list.reverse());
+      return cb(null, list);
     });
 };
 
-MessageDb.prototype.remove = function(ts) {
-  
+MessageDb.prototype.removeUpTo = function(ts) {
+  var opts = {};
+  db.createKeyStream(opts);
 };
 
 module.exports = soop(MessageDb);

From 8dc4af654b9ac6f5d0923b01c771695039d10f0e Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 18:13:37 -0300
Subject: [PATCH 6/9] cleaner plugin: optimize getMessages

---
 lib/MessageDb.js       | 25 +++++++++++++++++++------
 test/test.MessageDb.js |  4 ++++
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/lib/MessageDb.js b/lib/MessageDb.js
index fd77530..832087c 100644
--- a/lib/MessageDb.js
+++ b/lib/MessageDb.js
@@ -83,13 +83,21 @@ MessageDb.prototype.authenticate = function(m) {
   return AuthMessage._verify(frompubkey, sig, encrypted);
 };
 
+MessageDb.parseKey = function(key) {
+  var ret = {};
+  var spl = key.split('-');
+  
+  ret.to = spl[1];
+  ret.ts = +spl[2];
+
+  return ret;
+};
+
 MessageDb.fromStorage = function(data) {
-  var spl = data.key.split('-');
-  var to = spl[1];
-  var ts = +spl[2];
+  var parsed = MessageDb.parseKey(data.key);
   var message = data.value;
-  message.ts = ts;
-  message.to = to;
+  message.ts = parsed.ts;
+  message.to = parsed.to;
   return message;
 };
 
@@ -117,8 +125,13 @@ MessageDb.prototype.getMessages = function(to, lower_ts, upper_ts, cb) {
 };
 
 MessageDb.prototype.removeUpTo = function(ts) {
+  preconditions.checkArgument(ts);
+  preconditions.checkArgument(typeof ts === 'number');
   var opts = {};
-  db.createKeyStream(opts);
+  db.createKeyStream(opts)
+    .on('data', function(key) {
+      console.log('key=', key)
+    });
 };
 
 module.exports = soop(MessageDb);
diff --git a/test/test.MessageDb.js b/test/test.MessageDb.js
index efc1006..492c454 100644
--- a/test/test.MessageDb.js
+++ b/test/test.MessageDb.js
@@ -108,6 +108,10 @@ describe('MessageDb', function() {
       done();
     });
   });
+  it('should be able #removeUpTo', function() {
+    var mdb = sharedMDB;
+    mdb.removeUpTo(microtime.now());
+  });
   it('should be able to close instance', function() {
     var mdb = new MessageDb(opts);
     mdb.close();

From 02659021d3960ea08049709c7e8294e0c74d8dc0 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 18:42:52 -0300
Subject: [PATCH 7/9] cleaner plugin: removeUpTo

---
 lib/MessageDb.js       | 41 +++++++++++++++++++++++++++++++++--------
 test/test.MessageDb.js | 17 +++++++++++++++--
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/lib/MessageDb.js b/lib/MessageDb.js
index 832087c..ce6be5b 100644
--- a/lib/MessageDb.js
+++ b/lib/MessageDb.js
@@ -86,7 +86,7 @@ MessageDb.prototype.authenticate = function(m) {
 MessageDb.parseKey = function(key) {
   var ret = {};
   var spl = key.split('-');
-  
+
   ret.to = spl[1];
   ret.ts = +spl[2];
 
@@ -124,14 +124,39 @@ MessageDb.prototype.getMessages = function(to, lower_ts, upper_ts, cb) {
     });
 };
 
-MessageDb.prototype.removeUpTo = function(ts) {
-  preconditions.checkArgument(ts);
-  preconditions.checkArgument(typeof ts === 'number');
-  var opts = {};
-  db.createKeyStream(opts)
-    .on('data', function(key) {
-      console.log('key=', key)
+MessageDb.prototype.getAll = function(cb) {
+  var list = [];
+  db.createReadStream()
+    .on('data', function(data) {
+      list.push(MessageDb.fromStorage(data));
+    })
+    .on('end', function() {
+      return cb(null, list);
     });
 };
 
+MessageDb.prototype.removeUpTo = function(ts, cb) {
+  preconditions.checkArgument(ts);
+  preconditions.checkArgument(typeof ts === 'number');
+  var opts = {};
+  var dels = [];
+  db.createKeyStream(opts)
+    .on('data', function(key) {
+      var parsed = MessageDb.parseKey(key);
+      if (parsed.ts < ts) {
+        dels.push({
+          type: 'del',
+          key: key
+        });
+      }
+    })
+    .on('end', function() {
+      db.batch(dels, function(err) {
+        if (err) return cb(err);
+        else cb(null, dels.length);
+      })
+    });
+
+};
+
 module.exports = soop(MessageDb);
diff --git a/test/test.MessageDb.js b/test/test.MessageDb.js
index 492c454..960fc84 100644
--- a/test/test.MessageDb.js
+++ b/test/test.MessageDb.js
@@ -108,9 +108,22 @@ describe('MessageDb', function() {
       done();
     });
   });
-  it('should be able #removeUpTo', function() {
+  it('should be able #removeUpTo', function(done) {
     var mdb = sharedMDB;
-    mdb.removeUpTo(microtime.now());
+    var upper_ts = microtime.now();
+    mdb.addMessage(message, function(err) {
+      expect(err).to.not.exist;
+      mdb.removeUpTo(upper_ts, function(err, n) {
+        expect(err).to.not.exist;
+        n.should.equal(4);
+        mdb.getAll(function(error, all) {
+          expect(error).to.not.exist;
+          all.length.should.equal(1);
+          done();
+        });
+
+      });
+    });
   });
   it('should be able to close instance', function() {
     var mdb = new MessageDb(opts);

From a9f65adb77cf7fb0c3b1b4c42f7d8865c7768ce0 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 19:47:59 -0300
Subject: [PATCH 8/9] cleaner plugin: first version

---
 plugins/cleaner.js        | 13 +++++++------
 plugins/config-cleaner.js |  3 ++-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/plugins/cleaner.js b/plugins/cleaner.js
index bb00719..b8ab326 100644
--- a/plugins/cleaner.js
+++ b/plugins/cleaner.js
@@ -1,21 +1,22 @@
 var mdb = require('../lib/MessageDb').default();
 var logger = require('../lib/logger').logger;
 var preconditions = require('preconditions').singleton();
+var microtime = require('microtime');
 var cron = require('cron');
 var CronJob = cron.CronJob;
 
 
-var doClean = function() {
-  
-}
-  
 module.exports.init = function(config) {
   logger.info('Using cleaner plugin');
   logger.info(config);
   var job = new CronJob({
     cronTime: config.cronTime || '0 * * * *',
-    onTick: doClean,
+    onTick: function() {
+      var limit = microtime.now() - 1000 * 1000 * config.threshold;
+      mdb.removeUpTo(limit, function(err, n) {
+        logger.verbose('Ran cleaner task, removed ' + n);
+      });
+    },
     start: true
   });
-  doClean();
 };
diff --git a/plugins/config-cleaner.js b/plugins/config-cleaner.js
index b550cf3..e0dafac 100644
--- a/plugins/config-cleaner.js
+++ b/plugins/config-cleaner.js
@@ -1,5 +1,6 @@
 module.exports = {
 
-  cronTime: '* * * * *'
+  cronTime: '* * * * *',
+  threshold: 10, // in seconds
 
 };

From 9f1da23e453c41da5e1bcc4a688e19af5402e8c4 Mon Sep 17 00:00:00 2001
From: Manuel Araoz <manuelaraoz@gmail.com>
Date: Mon, 1 Sep 2014 20:23:23 -0300
Subject: [PATCH 9/9] cleaner plugin: manual tests OK

---
 lib/MessageDb.js          |  7 +++++++
 plugins/cleaner.js        | 23 +++++++++++++----------
 plugins/config-cleaner.js |  2 +-
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/lib/MessageDb.js b/lib/MessageDb.js
index ce6be5b..20b2c8c 100644
--- a/lib/MessageDb.js
+++ b/lib/MessageDb.js
@@ -130,6 +130,9 @@ MessageDb.prototype.getAll = function(cb) {
     .on('data', function(data) {
       list.push(MessageDb.fromStorage(data));
     })
+    .on('error', function(err) {
+      return cb(err);
+    })
     .on('end', function() {
       return cb(null, list);
     });
@@ -144,12 +147,16 @@ MessageDb.prototype.removeUpTo = function(ts, cb) {
     .on('data', function(key) {
       var parsed = MessageDb.parseKey(key);
       if (parsed.ts < ts) {
+        logger.verbose('Deleting message ' + key);
         dels.push({
           type: 'del',
           key: key
         });
       }
     })
+    .on('error', function(err) {
+      return cb(err);
+    })
     .on('end', function() {
       db.batch(dels, function(err) {
         if (err) return cb(err);
diff --git a/plugins/cleaner.js b/plugins/cleaner.js
index b8ab326..9323f65 100644
--- a/plugins/cleaner.js
+++ b/plugins/cleaner.js
@@ -7,16 +7,19 @@ var CronJob = cron.CronJob;
 
 
 module.exports.init = function(config) {
-  logger.info('Using cleaner plugin');
-  logger.info(config);
+  var cronTime = config.cronTime || '0 * * * *';
+  logger.info('Using cleaner plugin with cronTime ' + cronTime);
+  var onTick = function() {
+    var limit = microtime.now() - 1000 * 1000 * config.threshold;
+    mdb.removeUpTo(limit, function(err, n) {
+      if (err) logger.error(err);
+      else logger.info('Ran cleaner task, removed ' + n);
+    });
+  };
   var job = new CronJob({
-    cronTime: config.cronTime || '0 * * * *',
-    onTick: function() {
-      var limit = microtime.now() - 1000 * 1000 * config.threshold;
-      mdb.removeUpTo(limit, function(err, n) {
-        logger.verbose('Ran cleaner task, removed ' + n);
-      });
-    },
-    start: true
+    cronTime: cronTime,
+    onTick: onTick
   });
+  onTick();
+  job.start();
 };
diff --git a/plugins/config-cleaner.js b/plugins/config-cleaner.js
index e0dafac..0948f68 100644
--- a/plugins/config-cleaner.js
+++ b/plugins/config-cleaner.js
@@ -1,6 +1,6 @@
 module.exports = {
 
   cronTime: '* * * * *',
-  threshold: 10, // in seconds
+  threshold: 2*24*60*60, // 2 days, in seconds
 
 };