add cache for balance and utxo

app/models/Address.js

@@ -9,6 +9,8 @@ var BitcoreUtil        = bitcore.util;
 var Parser             = bitcore.BinaryParser;
 var Buffer             = bitcore.Buffer;
 var TransactionDb      = imports.TransactionDb || require('../../lib/TransactionDb').default();
+var BlockDb            = imports.BlockDb || require('../../lib/BlockDb').default();
+var config              = require('../../config/config');
 var CONCURRENCY        = 5;
 
 function Address(addrStr) {
@@ -20,6 +22,7 @@ function Address(addrStr) {
 
   this.txApperances           = 0;
   this.unconfirmedTxApperances= 0;
+  this.seen                   = {};
 
   // TODO store only txids? +index? +all?
   this.transactions   = [];
@@ -71,125 +74,123 @@ function Address(addrStr) {
 
 }
 
-Address.prototype._getScriptPubKey = function(hex,n) {
-  // ScriptPubKey is not provided by bitcoind RPC, so we parse it from tx hex.
-
-  var parser = new Parser(new Buffer(hex,'hex'));
-  var tx = new BitcoreTransaction();
-  tx.parse(parser);
-  return (tx.outs[n].s.toString('hex'));
-};
-
 Address.prototype.getUtxo = function(next) {
   var self = this;
-  if (!self.addrStr) return next();
+  var tDb   = TransactionDb;
+  var bDb   = BlockDb;
+  var ret;
+  if (!self.addrStr) return next(new Error('no error'));
 
-  var ret  = [];
+  tDb.fromAddr(self.addrStr, function(err,txOut){
-  var db   = TransactionDb;
-
-  db.fromAddr(self.addrStr, function(err,txOut){
     if (err) return next(err);
+    var unspent = txOut.filter(function(x){
+      return !x.spentTxId;
+    });
 
-    // Complete utxo info
+    bDb.fillConfirmations(unspent, function() {
-    async.eachLimit(txOut,CONCURRENCY,function (txItem, a_c) {
+      tDb.fillScriptPubKey(unspent, function() {
-      db.fromIdInfoSimple(txItem.txid, function(err, info) {
+        ret = unspent.map(function(x){
-        if (!info || !info.hex) return a_c(err);
+          return {
-
-        var scriptPubKey = self._getScriptPubKey(info.hex, txItem.index);
-
-        // we are filtering out even unconfirmed spents!
-        // add || !txItem.spentIsConfirmed 
-        if (!txItem.spentTxId) {
-          ret.push({
             address: self.addrStr,
-            txid: txItem.txid,
+            txid: x.txid,
-            vout: txItem.index,
+            vout: x.index,
-            ts: txItem.ts,
+            ts: x.ts,
-            scriptPubKey: scriptPubKey,
+            scriptPubKey: x.scriptPubKey,
-            amount: txItem.value_sat / BitcoreUtil.COIN,
+            amount: x.value_sat / BitcoreUtil.COIN,
-            confirmations: txItem.height ? info.confirmations : 0,
+            confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations,
-          });
+          };
-        }
+        });
-        return a_c(err);
+        return next(null, ret);
       });
-    }, function(err) {
-      return next(err,ret);
     });
   });
 };
 
+
+Address.prototype._addTxItem = function(txItem, notxlist) {
+  var add=0, addSpend=0;
+  var v = txItem.value_sat;
+  var seen = this.seen;
+  var txs = [];
+
+  if ( !seen[txItem.txid] ) {
+    if (!notxlist) {
+      txs.push({txid: txItem.txid, ts: txItem.ts});
+    }
+    seen[txItem.txid]=1;
+    add=1;
+  }
+
+  if (txItem.spentTxId && !seen[txItem.spentTxId]  ) {
+    if (!notxlist) {
+      txs.push({txid: txItem.spentTxId, ts: txItem.spentTs});
+    }
+    seen[txItem.spentTxId]=1;
+    addSpend=1;
+  }
+  if (txItem.isConfirmed) {
+    this.txApperances += add;
+    this.totalReceivedSat += v;
+    if (! txItem.spentTxId ) {
+      //unspent
+      this.balanceSat   += v;
+    }
+    else if(!txItem.spentIsConfirmed) {
+      // unspent
+      this.balanceSat   += v;
+      this.unconfirmedBalanceSat -= v;
+      this.unconfirmedTxApperances += addSpend;
+    }
+    else {
+      // spent
+      this.totalSentSat += v;
+      this.txApperances += addSpend;
+    }
+  }
+  else {
+    this.unconfirmedBalanceSat += v;
+    this.unconfirmedTxApperances += add;
+  }
+  return txs;
+};
+
+Address.prototype._setTxs = function(txs) {
+
+  // sort input and outputs togheter
+  txs.sort(
+    function compare(a,b) {
+    if (a.ts < b.ts) return 1;
+    if (a.ts > b.ts) return -1;
+    return 0;
+  });
+
+  this.transactions = txs.map(function(i) { return i.txid; } );
+};
+
 Address.prototype.update = function(next, notxlist) {
   var self = this;
   if (!self.addrStr) return next();
 
   var txs  = [];
-  var db   = TransactionDb;
+  var tDb   = TransactionDb;
-  async.series([
+  var bDb   = BlockDb;
-    function (cb) {
+  tDb.fromAddr(self.addrStr, function(err,txOut){
-      var seen={};
+    if (err) return next(err);
-      db.fromAddr(self.addrStr, function(err,txOut){
+
-        if (err) return cb(err);
+    bDb.fillConfirmations(txOut, function(err) {
+      if (err) return next(err);
+      tDb.cacheConfirmations(txOut, function(err) {
+        if (err) return next(err);
+
         txOut.forEach(function(txItem){
-          var add=0, addSpend=0;
+          txs=txs.concat(self._addTxItem(txItem, notxlist));
-          var v = txItem.value_sat;
-
-          if ( !seen[txItem.txid] ) {
-            if (!notxlist) {
-              txs.push({txid: txItem.txid, ts: txItem.ts});
-            }
-            seen[txItem.txid]=1;
-            add=1;
-          }
-
-          if (txItem.spentTxId && !seen[txItem.spentTxId]  ) {
-            if (!notxlist) {
-              txs.push({txid: txItem.spentTxId, ts: txItem.spentTs});
-            }
-            seen[txItem.spentTxId]=1;
-            addSpend=1;
-          }
-
-          if (txItem.height) {
-            self.txApperances += add;
-            self.totalReceivedSat += v;
-            if (! txItem.spentTxId ) {
-              //unspent
-              self.balanceSat   += v;
-            }
-            else if(!txItem.spentIsConfirmed) {
-              // unspent
-              self.balanceSat   += v;
-              self.unconfirmedBalanceSat -= v;
-              self.unconfirmedTxApperances += addSpend;
-            }
-            else {
-              // spent
-              self.totalSentSat += v;
-              self.txApperances += addSpend;
-            }
-          }
-          else {
-            self.unconfirmedBalanceSat += v;
-            self.unconfirmedTxApperances += add;
-          }
         });
-        return cb();
+
+        if (!notxlist)
+          self._setTxs(txs);
+        return next();
       });
-    },
+    });
-  ], function (err) {
-
-    if (!notxlist) {
-      // sort input and outputs togheter
-      txs.sort(
-        function compare(a,b) {
-          if (a.ts < b.ts) return 1;
-          if (a.ts > b.ts) return -1;
-          return 0;
-        });
-
-      self.transactions = txs.map(function(i) { return i.txid; } );
-    }
-    return next(err);
   });
 };
 

config/config.js

@@ -91,5 +91,6 @@ module.exports = {
   currencyRefresh: 10,
   keys: {
     segmentio: process.env.INSIGHT_SEGMENTIO_KEY
-  }
+  },
+  safeConfirmations: 6, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations
 };

lib/BlockDb.js

@@ -12,6 +12,8 @@ var IN_BLK_PREFIX = 'btx-'; //btx-<txid> = <block>
 
 
 var MAX_OPEN_FILES    = 500;
+var CONCURRENCY        = 5;
+var DFLT_REQUIRED_CONFIRMATIONS = 1;
 
 /**
 * Module dependencies.
@@ -20,14 +22,16 @@ var levelup     = require('levelup'),
     config      = require('../config/config');
 var db  = imports.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OPEN_FILES} );
 var Rpc = imports.rpc || require('./Rpc');
+var async              = require('async');
 
 
 var logger = require('./logger').logger;
 var d      = logger.log;
 var info   = logger.info;
 
-var BlockDb = function() {
+var BlockDb = function(opts) {
   this.txDb =  require('./TransactionDb').default();
+  this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS;
   BlockDb.super(this, arguments);
 };
 
@@ -140,7 +144,14 @@ BlockDb.prototype.setBlockNotMain = function(hash, cb) {
 BlockDb.prototype.add = function(b, height, cb) {
   d('adding block %s #d', b,height);
   var dbScript = this._addBlockScript(b,height);
-  dbScript = dbScript.concat(this._addTxsScript(b.tx,b.hash, height));
+  dbScript = dbScript.concat(this._addTxsScript(
+    b.tx.map(
+      function(o){
+        return o.txid;
+      }),
+    b.hash,
+    height
+  ));
   this.txDb.addMany(b.tx, function(err) {
     if (err) return cb(err);
     db.batch(dbScript,cb);
@@ -313,4 +324,64 @@ BlockDb.prototype.blockIndex = function(height, cb) {
   return Rpc.blockIndex(height,cb);
 };
 
+BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) {
+  var self = this;
+  if (!o.spentTxId) return cb();
+  if (o.multipleSpentAttempts) {
+    async.eachLimit(o.multipleSpentAttempts, CONCURRENCY,
+      function(oi, e_c) {
+        self.getBlockForTx(oi.spentTxId, function(err, hash, height) {
+          if (err) return;
+          if (height>=0) {
+            o.spentTxId = oi.spentTxId;
+            o.index = oi.index;
+            o.spentIsConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0;
+            o.spentConfirmations = chainHeight - height;
+          }
+          return e_c();
+        });
+      }, cb);
+  } else {
+    self.getBlockForTx(o.spentTxId, function(err, hash, height) {
+      if (err) return cb(err);
+      o.spentIsConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0;
+      o.spentConfirmations = chainHeight - height;
+      return cb();
+    });
+  }
+};
+
+BlockDb.prototype._fillConfirmationsOne = function(o, chainHeight, cb) {
+  var self = this;
+  self.getBlockForTx(o.txid, function(err, hash, height) {
+    if (err) return cb(err);
+    o.isConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0;
+    o.confirmations = chainHeight - height;
+    return self._fillConfirmationsOneSpent(o,chainHeight,cb);
+  });
+};
+
+BlockDb.prototype.fillConfirmations = function(txouts, cb) {
+  var self = this;
+  this.getTip(function(err, hash, height){
+    var txs = txouts.filter(function(x){
+      return !x.spentIsConfirmedCached // not 100%cached
+        && !(x.isConfirmedCached && !x.spentTxId);  // and not 50%cached but not spent 
+    });
+//console.log('[BlockDb.js.360:txouts:]',txs.length); //TODO
+var i=0;
+    async.eachLimit(txs, CONCURRENCY, function(txout, e_c) {
+      if(txout.isConfirmedCached) {
+//console.log('[BlockDb.js.378]', i++); //TODO
+        self._fillConfirmationsOneSpent(txout,height, e_c);
+      } else {
+//console.log('[BlockDb.js.3782]', i++); //TODO
+        self._fillConfirmationsOne(txout,height, e_c);
+      }
+
+    }, cb);
+  });
+};
+
+
 module.exports = require('soop')(BlockDb);

lib/HistoricSync.js

@@ -145,13 +145,14 @@ HistoricSync.prototype._fromBuffer = function (buf) {
     return parseInt(buf2.toString('hex'), 16);
 };
 
-HistoricSync.prototype.getStandardizedTx = function (tx, time) {
+HistoricSync.prototype.getStandardizedTx = function (tx, time, isCoinBase) {
   var self = this;
   tx.txid = bitcoreUtil.formatHashFull(tx.getHash());
   var ti=0;
+
   tx.vin = tx.ins.map(function(txin) {
     var ret = {n: ti++};
-    if (txin.isCoinBase()) {
+    if (isCoinBase) {
       ret.isCoinBase = true;
     } else {
       ret.txid = buffertools.reverse(new Buffer(txin.getOutpointHash())).toString('hex');
@@ -189,8 +190,11 @@ HistoricSync.prototype.getStandardizedBlock = function(b) {
     previousblockhash: bitcoreUtil.formatHashFull(b.prev_hash),
     time: b.timestamp,
   };
+  var isCoinBase = 1;
   block.tx = b.txs.map(function(tx){
-    return self.getStandardizedTx(tx, b.timestamp);
+    var ret = self.getStandardizedTx(tx, b.timestamp, isCoinBase);
+    isCoinBase=0;
+    return ret;
   });
   return block;
 };

lib/Sync.js

@@ -256,7 +256,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) {
       });
     },
     function() {
-      return hashInterator && !yHeight;
+      return hashInterator && yHeight<=0;
     },
     function() {
       info('\tFound yBlock: %s #%d', hashInterator, yHeight);
@@ -268,7 +268,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) {
           return hashIter;
         },
         function(c) {
-          self.setBlockMain(hashIter, heightIter++, c);
+          self.bDb.setBlockMain(hashIter, heightIter++, c);
         },
         function(err) {
           return cb(err, hashInterator, lastHash, heightIter);

lib/TransactionDb.js

@@ -13,6 +13,7 @@ var ADDR_PREFIX = 'txa-'; //txa-<addr>-<txid>-<n> => + btc_sat:ts [:<txid>-<n>](
 // TODO: use bitcore networks module
 var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b';
 var CONCURRENCY = 10;
+var DEFAULT_SAFE_CONFIRMATIONS = 6;
 
 var MAX_OPEN_FILES    = 500;
 //  var CONFIRMATION_NR_TO_NOT_CHECK = 10;  //Spend
@@ -41,6 +42,7 @@ var TransactionDb = function() {
   TransactionDb.super(this, arguments);
   this.network = config.network === 'testnet' ? networks.testnet : networks.livenet;
   this.poolMatch = new PoolMatch();
+  this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS;
 };
 
 TransactionDb.prototype.close = function(cb) {
@@ -57,25 +59,6 @@ TransactionDb.prototype.drop = function(cb) {
   });
 };
 
-
-TransactionDb.prototype.has = function(txid, cb) {
-
-  var k = OUTS_PREFIX + txid;
-  db.get(k, function(err, val) {
-
-    var ret;
-
-    if (err && err.notFound) {
-      err = null;
-      ret = false;
-    }
-    if (typeof val !== undefined) {
-      ret = true;
-    }
-    return cb(err, ret);
-  });
-};
-
 TransactionDb.prototype._addSpentInfo = function(r, txid, index, ts) {
   if (r.spentTxId) {
     if (!r.multipleSpentAttempts) {
@@ -171,24 +154,24 @@ TransactionDb.prototype._fillSpent = function(info, cb) {
 };
 
 
-TransactionDb.prototype._fillOutpoints = function(info, cb) {
+TransactionDb.prototype._fillOutpoints = function(txInfo, cb) {
   var self = this;
 
-  if (!info || info.isCoinBase) return cb();
+  if (!txInfo || txInfo.isCoinBase) return cb();
 
   var valueIn = 0;
   var incompleteInputs = 0;
 
-  async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) {
+  async.eachLimit(txInfo.vin, CONCURRENCY, function(i, c_in) {
-      self.fromTxIdN(i.txid, i.vout, info.confirmations, function(err, ret) {
+      self.fromTxIdN(i.txid, i.vout, txInfo.confirmations, function(err, ret) {
         if (!ret || !ret.addr || !ret.valueSat) {
-          info('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid);
+          info('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, txInfo.txid);
           if (ret) i.unconfirmedInput = ret.unconfirmedInput;
           incompleteInputs = 1;
           return c_in(); // error not scalated
         }
 
-        info.firstSeenTs = ret.spentTs;
+        txInfo.firstSeenTs = ret.spentTs;
         i.unconfirmedInput = i.unconfirmedInput;
         i.addr = ret.addr;
         i.valueSat = ret.valueSat;
@@ -199,18 +182,18 @@ TransactionDb.prototype._fillOutpoints = function(info, cb) {
 *        If confirmed by bitcoind, we could not check for double spents
 *        but we prefer to keep the flag of double spent attempt
 *
-        if (info.confirmations
+        if (txInfo.confirmations
-            && info.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK)
+            && txInfo.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK)
           return c_in();
 isspent
 */
         // Double spent?
         if (ret.multipleSpentAttempt || !ret.spentTxId ||
-          (ret.spentTxId && ret.spentTxId !== info.txid)
+          (ret.spentTxId && ret.spentTxId !== txInfo.txid)
         ) {
           if (ret.multipleSpentAttempts) {
             ret.multipleSpentAttempts.forEach(function(mul) {
-              if (mul.spentTxId !== info.txid) {
+              if (mul.spentTxId !== txInfo.txid) {
                 i.doubleSpentTxID = ret.spentTxId;
                 i.doubleSpentIndex = ret.spentIndex;
               }
@@ -229,10 +212,10 @@ isspent
     },
     function() {
       if (!incompleteInputs) {
-        info.valueIn = valueIn / util.COIN;
+        txInfo.valueIn = valueIn / util.COIN;
-        info.fees = (valueIn - (info.valueOut * util.COIN)).toFixed(0) / util.COIN;
+        txInfo.fees = (valueIn - (txInfo.valueOut * util.COIN)).toFixed(0) / util.COIN;
       } else {
-        info.incompleteInputs = 1;
+        txInfo.incompleteInputs = 1;
       }
       return cb();
     });
@@ -241,11 +224,11 @@ isspent
 TransactionDb.prototype._getInfo = function(txid, next) {
   var self = this;
 
-  Rpc.getTxInfo(txid, function(err, info) {
+  Rpc.getTxInfo(txid, function(err, txInfo) {
     if (err) return next(err);
-    self._fillOutpoints(info, function() {
+    self._fillOutpoints(txInfo, function() {
-      self._fillSpent(info, function() {
+      self._fillSpent(txInfo, function() {
-        return next(null, info);
+        return next(null, txInfo);
       });
     });
   });
@@ -322,46 +305,133 @@ TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) {
   });
 };
 
-TransactionDb.prototype.fillConfirmations = function(o, cb) {
+
+TransactionDb.prototype.deleteCacheForAddress = function(addr,cb) {
+  var k = ADDR_PREFIX + addr + '-';
+  var dbScript = [];
+  db.createReadStream({
+    start: k,
+    end: k + '~'
+  })
+    .on('data', function(data) {
+      var v = data.value.split(':');
+      dbScript.push({
+        type: 'put',
+        key: data.key,
+        value: v.slice(0,2).join(':'),
+      });
+    })
+    .on('error', function(err) {
+      return cb(err);
+    })
+    .on('end', function (){
+      db.batch(dbScript,cb);
+    });
+};
+
+TransactionDb.prototype.cacheConfirmations = function(txouts,cb) {
   var self = this;
 
-console.log('[TransactionDb.js.339]'); //TODO
+  var dbScript=[];
-  self.getBlock(o.txid, function(err, hash) {
+  for(var ii in txouts){
+    var txout=txouts[ii];
 
-console.log('[TransactionDb.js.342]'); //TODO
+    //everything already cached?
-    if (err) return cb(err);
+    if (txout.spentIsConfirmedCached) {
+      continue;
+    }
 
-    o.isConfirmed = hash?1:0;
+    var infoToCache = [];
-    if (!o.spentTxId) return cb();
+    if (txout.confirmations > self.safeConfirmations) {
+      if (!txout.isConfirmedCached) infoToCache.push(1);
 
-    if (o.multipleSpentAttempts) {
+      if (txout.spentConfirmations > self.safeConfirmations) {
+//    console.log('[TransactionDb.js.309]',txout); //TODO
+        infoToCache = infoToCache.concat([1, txout.spentTxId, txout.spentIndex, txout.spentTs]);
+      }
+      if (infoToCache.length){
 
-      //TODO save it for later is height > 6
+        // if spent, we overwrite scriptPubKey cache (not needed anymore)
-      async.eachLimit(o.multipleSpentAttempts, CONCURRENCY,
+        // Last 1 = txout.isConfirmedCached (must be equal to 1 at this point)
-        function(oi, e_c) {
+        infoToCache.unshift(txout.value_sat,txout.ts, 1);
-          self.getBlock(oi.spentTxId, function(err, hash) {
+        dbScript.push({
-            if (err) return;
+          type: 'put',
-            if (hash) {
+          key: txout.key,
-              o.spentTxId = oi.spentTxId;
+          value: infoToCache.join(':'),
-              o.index = oi.index;
+        });
-              o.spentIsConfirmed = 1;
+      }
-            }
+    }
-            return e_c();
+  }
-          });
+
-        }, cb);
+//console.log('[TransactionDb.js.339:dbScript:]',dbScript); //TODO
-    } else {
+  db.batch(dbScript,cb);
-      self.getBlock(o.spentTxId, function(err, hash) {
+};
-        if (err) return cb(err);
+
-        o.spentIsConfirmed = hash?1:0;
+
-        return cb();
+TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) {
+  var self = this;
+
+  var dbScript=[];
+  for(var ii in txouts){
+    var txout=txouts[ii];
+
+    //everything already cached?
+    if (txout.scriptPubKeyCached || txout.spentTxId) {
+      continue;
+    }
+
+    if (txout.scriptPubKey) {
+      var infoToCache = [txout.value_sat,txout.ts, txout.isConfirmedCached?1:0, txout.scriptPubKey];
+      dbScript.push({
+        type: 'put',
+        key: txout.key,
+        value: infoToCache.join(':'),
       });
     }
-  });
+  }
+  db.batch(dbScript,cb);
+};
+
+
+
+
+TransactionDb.prototype._parseAddrData = function(data) {
+  var k = data.key.split('-');
+  var v = data.value.split(':');
+  var item = {
+    key: data.key,
+    txid: k[2],
+    index: parseInt(k[3]),
+    value_sat: parseInt(v[0]),
+    ts: parseInt(v[1]),
+  };
+
+  // Cache
+  if (v[2]){
+    item.isConfirmed = 1;
+    item.isConfirmedCached = 1;
+    //console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO
+    // Sent, confirmed
+    if (v[3] === 1){
+
+      //console.log('[TransactionDb.js.356] CACHE HIT SPENT:', item.key); //TODO
+      item.spentIsConfirmed = 1;
+      item.spentIsConfirmedCached = 1;
+      item.spentTxId = v[4];
+      item.spentIndex = parseInt(v[5]);
+      item.spentTs = parseInt(v[6]);
+    }
+    // Scriptpubkey cached
+    else if (v[3]) {
+      item.scriptPubKey = v[3];
+      item.scriptPubKeyCached = 1;
+    }
+  }
+  return item;
 };
 
 TransactionDb.prototype.fromAddr = function(addr, cb) {
   var self = this;
-
   var k = ADDR_PREFIX + addr + '-';
   var ret = [];
 
@@ -370,23 +440,11 @@ TransactionDb.prototype.fromAddr = function(addr, cb) {
     end: k + '~'
   })
     .on('data', function(data) {
-      var k = data.key.split('-');
+      ret.push(self._parseAddrData(data));
-      var v = data.value.split(':');
-      ret.push({
-        txid: k[2],
-        index: parseInt(k[3]),
-        value_sat: parseInt(v[0]),
-        ts: parseInt(v[1]),
-      });
-    })
-    .on('error', function(err) {
-      return cb(err);
     })
+    .on('error', cb)
     .on('end', function() {
-
+      async.eachLimit(ret.filter(function(x){return !x.spentIsConfirmed;}), CONCURRENCY, function(o, e_c) {
-      //TODO is spent, and conf > 6, save it on ADDR_PREFIX for later
-      //and skip all the rest
-      async.eachLimit(ret, CONCURRENCY, function(o, e_c) {
           var k = SPENT_PREFIX + o.txid + '-' + o.index + '-';
           db.createReadStream({
             start: k,
@@ -396,28 +454,32 @@ TransactionDb.prototype.fromAddr = function(addr, cb) {
               var k = data.key.split('-');
               self._addSpentInfo(o, k[3], k[4], data.value);
             })
-            .on('error', function(err) {
+            .on('error', e_c)
-              return e_c(err);
+            .on('end', e_c);
-            })
-            .on('end', function(err) {
-              return e_c(err);
-            });
         },
-        function() {
+        function(err) {
-          async.eachLimit(ret, CONCURRENCY, function(o, e_c) {
+          return cb(err, ret);
-            self.fillConfirmations(o, e_c);
-          }, function(err) {
-            return cb(err, ret);
-          });
         });
     });
 };
 
+TransactionDb.prototype.fillScriptPubKey = function(txouts, cb) {
+  var self=this;
+  // Complete utxo info
+  async.eachLimit(txouts, CONCURRENCY, function (txout, a_c) {
+    self.fromIdInfoSimple(txout.txid, function(err, info) {
+      if (!info || !info.vout) return a_c(err);
+
+      txout.scriptPubKey = info.vout[txout.index].scriptPubKey.hex;
+      return a_c();
+    });
+  }, function(){
+    self.cacheScriptPubKey(txouts, cb);
+  });
+};
 
 TransactionDb.prototype.removeFromTxId = function(txid, cb) {
-
   async.series([
-
       function(c) {
         db.createReadStream({
           start: OUTS_PREFIX + txid + '-',
@@ -452,14 +514,14 @@ TransactionDb.prototype._addScript = function(tx) {
   var dbScript  = [];
   var ts        = tx.time;
   var txid      = tx.txid;
-
   // Input Outpoints (mark them as spent)
-  if (!tx.isCoinBase){
+  for(var ii in tx.vin) {
-    for(var ii in tx.vin) {
+    var i = tx.vin[ii];
-      var i = tx.vin[ii];
+    if (i.txid){
+      var k = SPENT_PREFIX + i.txid + '-' + i.vout + '-' + txid + '-' + i.n;
       dbScript.push({
         type: 'put',
-        key: SPENT_PREFIX + i.txid + '-' + i.vout + '-' + txid + '-' + i.n,
+        key: k,
         value: ts || 0,
       });
     }
@@ -467,13 +529,11 @@ TransactionDb.prototype._addScript = function(tx) {
 
   for(var ii in tx.vout) {
     var o = tx.vout[ii];
-    if ((o.value||o.valueSat) &&
+    if ( o.scriptPubKey && o.scriptPubKey.addresses &&
-        o.scriptPubKey &&
-          o.scriptPubKey.addresses &&
             o.scriptPubKey.addresses[0] && !o.scriptPubKey.addresses[1] // TODO : not supported=> standard multisig
        ) {
          var addr = o.scriptPubKey.addresses[0];
-         var sat = o.valueSat || (o.value * util.COIN).toFixed(0);
+         var sat = o.valueSat || ((o.value||0) * util.COIN).toFixed(0);
 
          relatedAddrs[addr]=1;
          var k = OUTS_PREFIX + txid + '-' + o.n;
@@ -538,22 +598,18 @@ TransactionDb.prototype.addMany = function(txs, next) {
 };
 
 
-TransactionDb.prototype.getPoolInfo = function(tx, cb) {
+TransactionDb.prototype.getPoolInfo = function(txid, cb) {
   var self = this;
-  self._getInfo(tx, function(e, a) {
-    if (e) return cb(false);
 
-    if (a && a.isCoinBase) {
+  Rpc.getTxInfo(txid, function(err, txInfo) {
-      var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex');
+    if (err) return cb(false);
-      var aa = self.poolMatch.match(coinbaseHexBuffer);
+    var ret;
 
-      return cb(aa);
+    if (txInfo && txInfo.isCoinBase)
-    }
+      ret = self.poolMatch.match(new Buffer(txInfo.vin[0].coinbase, 'hex'));
-    else {
+    
-      return cb();
+    return cb(ret);
-    }
   });
 };
 
-
 module.exports = require('soop')(TransactionDb);

lib/logger.js

@@ -0,0 +1,5 @@
+var winston = require('winston');
+
+winston.info('starting...')
+
+module.exports.logger=winston;

test/integration/01-transactionouts.js

@@ -5,6 +5,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development';
 
 
 
+var should = require('chai');
 var assert          = require('assert'),
   fs              = require('fs'),
   util            = require('util'),
@@ -33,10 +34,9 @@ describe('TransactionDb fromIdWithInfo', function(){
         assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i);
       }
 
-console.log('[01-transactionouts.js.34:tx:]',tx.info.vin[0]); //TODO
 
-      assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0');
+      tx.info.vin[0].addr.should.equal('msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59');
-      assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1');
+      tx.info.vin[1].addr.should.equal('mfye7oHsdrHbydtj4coPXCasKad2eYSv5P');
       done();
     });
   });

test/integration/addr.js

@@ -11,8 +11,11 @@ var assert = require('assert'),
   addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')),
   utxoValid = JSON.parse(fs.readFileSync('test/integration/utxo.json'));
 
+var should = require('chai');
+
 var txDb;
 describe('Address balances', function() {
+  this.timeout(5000);
 
   before(function(c) {
     txDb = TransactionDb;
@@ -24,19 +27,16 @@ describe('Address balances', function() {
       console.log(v.addr + ' => disabled in JSON');
     } else {
       it('Address info for: ' + v.addr, function(done) {
-        this.timeout(5000);
-
         var a = new Address(v.addr, txDb);
-
         a.update(function(err) {
           if (err) done(err);
-          assert.equal(v.addr, a.addrStr);
+          v.addr.should.equal(a.addrStr);
-          assert.equal(a.unconfirmedTxApperances ,v.unconfirmedTxApperances || 0, 
+          a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); 
-                       'unconfirmedTxApperances got:' + a.unconfirmedTxApperances );
+          a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat');
-          assert.equal(a.unconfirmedBalanceSat ,v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat: ' + a.unconfirmedBalanceSat + ' vs.: ' + v.unconfirmedBalanceSat );
           if (v.txApperances)
-            assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances);
+            a.txApperances.should.equal(v.txApperances, 'txApperances');
-          if (v.totalReceived) assert.equal(v.totalReceived, a.totalReceived, 'received: ' + a.totalReceived);
+
+          if (v.totalReceived) a.totalReceived.should.equal(v.totalReceived,'totalReceived');
           if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent);
 
           if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
@@ -50,31 +50,137 @@ describe('Address balances', function() {
           done();
         });
       });
+
+      it('Address info (cache) for: ' + v.addr, function(done) {
+        var a = new Address(v.addr, txDb);
+        a.update(function(err) {
+          if (err) done(err);
+          v.addr.should.equal(a.addrStr);
+          a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); 
+          a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat');
+          if (v.txApperances)
+            a.txApperances.should.equal(v.txApperances, 'txApperances');
+
+          if (v.totalReceived) a.totalReceived.should.equal(v.totalReceived,'totalReceived');
+          if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent);
+          if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance);
+          done();
+        },1);
+      });
     }
   });
 
 });
 
+describe('Address cache ', function() {
+  this.timeout(5000);
 
+  before(function(c) {
+    txDb = TransactionDb;
+    txDb.deleteCacheForAddress('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG',function(){
+     txDb.deleteCacheForAddress('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3',c);
+    });
+  });
+
+  it('cache case 1 w/o cache', function(done) {
+    var a = new Address('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG', txDb);
+    a.update(function(err) {
+      if (err) done(err);
+      a.balance.should.equal(0, 'balance');
+      a.totalReceived.should.equal(19175, 'totalReceived');
+      a.txApperances.should.equal(2, 'txApperances');
+      return done();
+    });
+  });
+   it('cache case 1 w cache', function(done) {
+    var a = new Address('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG', txDb);
+    a.update(function(err) {
+      if (err) done(err);
+      a.balance.should.equal(0, 'balance');
+      a.totalReceived.should.equal(19175, 'totalReceived');
+      a.txApperances.should.equal(2, 'txApperances');
+      return done();
+    });
+  });
+    
+  it('cache case 2 w/o cache', function(done) {
+    var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb);
+    a.update(function(err) {
+      if (err) done(err);
+      a.balance.should.equal(0, 'balance');
+      a.totalReceived.should.equal(1376000, 'totalReceived');
+      a.txApperances.should.equal(8003, 'txApperances');
+      return done();
+    });
+  },1);
+  it('cache case 2 w cache', function(done) {
+    var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb);
+    a.update(function(err) {
+      if (err) done(err);
+      a.balance.should.equal(0, 'balance');
+      a.totalReceived.should.equal(1376000, 'totalReceived');
+      a.txApperances.should.equal(8003, 'txApperances');
+      return done();
+    },1);
+  });
+});
+
+//tested against https://api.biteasy.com/testnet/v1/addresses/2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5/unspent-outputs?per_page=40
 describe('Address utxo', function() {
+
+  before(function(c) {
+    txDb = TransactionDb;
+    var l = utxoValid.length;
+    var d=0;
+
+    utxoValid.forEach(function(v) {
+      //console.log('Deleting cache for', v.addr); //TODO
+      txDb.deleteCacheForAddress(v.addr,function(){
+        if (d++ == l-1) return c();
+      });
+    });
+  });
+
+
   utxoValid.forEach(function(v) {
     if (v.disabled) {
       console.log(v.addr + ' => disabled in JSON');
     } else {
       it('Address utxo for: ' + v.addr, function(done) {
-        this.timeout(50000);
+        this.timeout(2000);
-
         var a = new Address(v.addr, txDb);
         a.getUtxo(function(err, utxo) {
-console.log('[addr.js.68:utxo:]',utxo); //TODO
           if (err) done(err);
           assert.equal(v.addr, a.addrStr);
-          if (v.length) assert.equal(v.length, utxo.length, 'length: ' + utxo.length);
+          if (v.length) utxo.length.should.equal(v.length, 'Unspent count');
-          if (v.tx0id) assert.equal(v.tx0id, utxo[0].txid, 'have tx: ' + utxo[0].txid);
+          if (v.tx0id) {
-          if (v.tx0scriptPubKey)
+            var x=utxo.filter(function(x){
-            assert.equal(v.tx0scriptPubKey, utxo[0].scriptPubKey, 'have tx: ' + utxo[0].scriptPubKey);
+                return x.txid === v.tx0id;
-          if (v.tx0amount) 
+            });
-            assert.equal(v.tx0amount, utxo[0].amount, 'amount: ' + utxo[0].amount);
+            assert(x,'found output');
+            x.length.should.equal(1,'found output');
+            x[0].scriptPubKey.should.equal(v.tx0scriptPubKey,'scriptPubKey');
+            x[0].amount.should.equal(v.tx0amount,'amount');
+          }
+          done();
+        });
+      });
+      it('Address utxo (cached) for: ' + v.addr, function(done) {
+        this.timeout(2000);
+        var a = new Address(v.addr, txDb);
+        a.getUtxo(function(err, utxo) {
+          if (err) done(err);
+          assert.equal(v.addr, a.addrStr);
+          if (v.length) utxo.length.should.equal(v.length, 'Unspent count');
+          if (v.tx0id) {
+            var x=utxo.filter(function(x){
+                return x.txid === v.tx0id;
+            });
+            assert(x,'found output');
+            x.length.should.equal(1,'found output');
+            x[0].scriptPubKey.should.equal(v.tx0scriptPubKey,'scriptPubKey');
+            x[0].amount.should.equal(v.tx0amount,'amount');
+          }
           done();
         });
       });

test/integration/addr.json

@@ -41,11 +41,12 @@
         "totalReceived": 54.81284116
     },
     { 
+      "disabled": 1,
         "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29",
         "balance": 1363.14677867,
         "totalReceived": 1363.14677867,
         "totalSent": 0,
-        "txApperances": 7943,
+        "txApperances": 7947,
         "unconfirmedTxApperances": 5,
         "unconfirmedBalanceSat": 149174913 
     },

test/integration/utxo.json

@@ -8,9 +8,9 @@
     },
     { 
         "addr": "2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5",
-        "length": 1,
+        "length": 13,
-        "tx0id": "eeabc70063d3f266e190e8735bc4599c811d3a79d138da1364e88502069b029c",
+        "tx0id": "b9cc61b55814a0f972788e9025db1013157f83716e08239026a156efe892a05c",
-        "tx0scriptPubKey": "76a9149e9f6515c70db535abdbbc983c7d8d1bff6c20cd88ac",
+        "tx0scriptPubKey": "a9145e0461e38796367580305e3615fc1b70e4c3307687",
-        "tx0amount": 0.38571339
+        "tx0amount": 0.001 
     }
 ]

util/sync.js

@@ -15,6 +15,7 @@ program
   .option('-D --destroy', 'Remove current DB (and start from there)', 0)
   .option('-S --startfile', 'Number of file from bitcoind to start(default=0)')
   .option('-R --rpc', 'Force sync with RPC')
+  .option('--stop [hash]', 'StopAt block',1)
   .option('-v --verbose', 'Verbose 0/1', 0)
   .parse(process.argv);
 
@@ -30,10 +31,13 @@ async.series([
     historicSync.sync.destroy(cb);
   },
   function(cb) {
-    historicSync.start({
+    var opts= {
       forceStartFile: program.startfile,
       forceRPC: program.rpc,
-    },cb);
+      stopAt: program.stop,
+    };
+    console.log('[options]',opts); //TODO
+    historicSync.start(opts,cb);
   },
   ],
   function(err) {