Merge pull request #303 from matiu/feature/refactor-rpc-classes

Feature/refactor rpc classes

lib/BlockDb.js

@@ -5,23 +5,21 @@ require('classtool');
 
 function spec(b) {
 
-  var TIMESTAMP_PREFIX  = 'bts-'; // b-ts-<ts> => <hash>
+  var TIMESTAMP_PREFIX  = 'bts-';     // b-ts-<ts> => <hash>
   var PREV_PREFIX       = 'bpr-';     // b-prev-<hash> => <prev_hash> 
   var NEXT_PREFIX       = 'bne-';     // b-next-<hash> => <next_hash> 
   var MAIN_PREFIX       = 'bma-';     // b-main-<hash> => 1/0
-  var TIP               = 'bti-';      // last block on the chain
+  var TIP               = 'bti-';     // last block on the chain
+  var LAST_FILE_INDEX   = 'file-';     // last processed file index
 
 
   /**
   * Module dependencies.
   */
-  var RpcClient   = require('bitcore/RpcClient').class(),
+  var levelup     = require('levelup'),
-      util        = require('bitcore/util/util'),
-      levelup     = require('levelup'),
-      BitcoreBlock= require('bitcore/Block').class(),
       config      = require('../config/config');
-  var db   = b.db || levelup(config.leveldb + '/blocks');
+  var db  = b.db || levelup(config.leveldb + '/blocks');
-  var rpc  = b.rpc || new RpcClient(config.bitcoind);
+  var Rpc = b.rpc || require('./Rpc').class();
 
   var BlockDb = function() {
   };
@@ -80,6 +78,24 @@ function spec(b) {
     });
   };
 
+
+  BlockDb.prototype.setLastFileIndex = function(idx, cb) {
+    var self = this;
+    if (this.lastFileIndexSaved === idx) return cb();
+
+    db.put(LAST_FILE_INDEX, idx, function(err) {
+      self.lastFileIndexSaved = idx;
+      return cb(err);
+    });
+  };
+
+  BlockDb.prototype.getLastFileIndex = function(cb) {
+    db.get(LAST_FILE_INDEX, function(err,val) {
+      if (err && err.notFound) { err = null; val = null;}
+      return cb(err,val);
+    });
+  };
+
   BlockDb.prototype.getNext = function(hash, cb) {
     db.get(NEXT_PREFIX + hash, function(err,val) {
       if (err && err.notFound) { err = null; val = null;}
@@ -138,13 +154,8 @@ function spec(b) {
   BlockDb.prototype.fromHashWithInfo = function(hash, cb) {
     var self = this;
 
-    rpc.getBlock(hash, function(err, info) {
+    Rpc.getBlock(hash, function(err, info) {
-      // Not found?
+      if (err || !info) return cb(err);
-      if (err && err.code === -5) return cb();
-      if (err) return cb(err);
-
-      if (info.result.height)
-        info.result.reward =  BitcoreBlock.getBlockValue(info.result.height) / util.COIN ;
 
       self.isMain(hash, function(err, val) {
         if (err) return cb(err);
@@ -182,12 +193,7 @@ function spec(b) {
   };
 
   BlockDb.prototype.blockIndex = function(height, cb) {
-    var rpc  = new RpcClient(config.bitcoind);
+    return Rpc.blockIndex(height,cb);
-    rpc.getBlockHash(height, function(err, bh){
-      if (err) return cb(err);
-
-      cb(null, { blockHash: bh.result });
-    });
   };
 
   return BlockDb;

lib/HistoricSync.js

@@ -14,7 +14,8 @@ function spec() {
   var Sync = require('./Sync').class();
   var sockets = require('../app/controllers/socket.js');
   var BlockExtractor = require('./BlockExtractor.js').class();
-	//var Deserialize = require('bitcore/Deserialize');
+  //  var bitcoreUtil = require('bitcore/util/util');
+	//  var Deserialize = require('bitcore/Deserialize');
 
 
   var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:';
@@ -122,11 +123,10 @@ function spec() {
     if (self.opts.shouldBroadcastSync) {
       sockets.broadcastSyncInfo(self.info());
     }
-
+//
-//TODO
+//   if (self.syncPercentage > 10) {
-// if (self.syncPercentage > 10) {
+//     process.exit(-1);
-//   process.exit(-1);
+//   }
-// }
   };
 
   HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) {
@@ -229,7 +229,6 @@ function spec() {
     //get Info
     self.blockExtractor.getNextBlock(function(err, b) {
       if (err || ! b) return cb(err);
-
       blockInfo = b.getStandardizedObject(b.txs, self.network);
       // blockInfo.curWork = Deserialize.intFromCompact(b.bits);
       // We keep the RPC field names
@@ -277,19 +276,24 @@ function spec() {
       }
 
       self.sync.storeTipBlock(blockInfo, function(err) {
-        if (blockInfo && blockInfo.hash) {
-          self.syncedBlocks++;
-        } else
-          self.status = 'finished';
-
         if (err) {
           self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]',
                                     blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks));
+          return cb(err);
         }
-        return cb(err);
+
+        self.sync.bDb.setLastFileIndex(self.blockExtractor.currentFileIndex, function(err) {
+          if (err) return cb(err);
+
+          if (blockInfo && blockInfo.hash) {
+            self.syncedBlocks++;
+          } else
+            self.status = 'finished';
+
+          return cb(err);
+        });
       });
     });
-  
   };
 
 
@@ -307,7 +311,7 @@ function spec() {
   };
 
 
-  HistoricSync.prototype.getBlockCount = function(cb) {
+  HistoricSync.prototype.updateBlockCount = function(cb) {
     var self = this;
 
     if (self.blockChainHeight) return cb();
@@ -319,64 +323,110 @@ function spec() {
     });
   };
 
-  HistoricSync.prototype.importHistory = function(scanOpts, next) {
+  HistoricSync.prototype.smartImport = function(scanOpts, next) {
     var self = this;
 
+    var genesis, count;
     var lastBlock;
     var tip;
 
     async.series([
-    function(cb) {
+      function(s_c) {
-      if (scanOpts.destroy) {
+        if (!scanOpts.destroy) return s_c();
+        
         p('Deleting DB...');
-        return self.sync.destroy(cb);
+        return self.sync.destroy(s_c);
-      }
+      },
-      return cb();
+      function(s_c) {
-    },
+        self.sync.bDb.has(self.genesis, function(err, b) {
-    // We are not using getBestBlockHash, because is not available in all clients
+          genesis = b;
-    function (cb) { return self.getBlockCount(cb); },
+          return s_c(err);
-    function(cb) {
+        });
-      if (!scanOpts.reverse) return cb();
+      },
-      self.rpc.getBlockHash(self.blockChainHeight, function(err, res) {
+      function(s_c) {
-        if (err) return cb(err);
+        self.countNotOrphan(function(err, c) {
-        lastBlock = res.result;
+          count = c;
-        return cb();
+          return s_c(err);
-      });
+        });
-    },
+      },
-    function(cb) {
+      function(s_c) {
-      if (!scanOpts.reverse) return cb();
+        if (!config.bitcoind.dataDir) return s_c();
-      self.sync.bDb.getTip(function(err, res) {
+        if (scanOpts.startFile) {
-        if (err) return cb(err);
+          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-        tip = res;
+          self.blockExtractor.currentFileIndex = scanOpts.startFile;
+          return s_c();
+        }
+        self.sync.bDb.getLastFileIndex(function(err, idx) {
+          self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
+          if (idx) self.blockExtractor.currentFileIndex = idx;
+          return s_c(err);
+        });
+      },
+      function(s_c) {
+        self.updateBlockCount(s_c);
+      },
 
-        console.log('Old Tip:', tip);
+      // define sync strategy
-        return cb();
+      function(s_c) {
-      });
+        if (!genesis || scanOpts.destroy || count < self.blockChainHeight * 0.9 ) {
-    },
+
- 
+          // Full sync.
-    function(cb) {
+          if (!genesis)
-      if (scanOpts.reverse) {
+            p('Could not find Genesis block. Running FULL SYNC');
+          else
+            p('Less that 90% of current blockchain is stored. Running FULL SYNC',
+                parseInt(count/self.blockChainHeight*100));
+
+          if (config.bitcoind.dataDir) {
+            p('bitcoind dataDir configured...importing blocks from .dat files');
+            p('Starting from file: ' + self.blockExtractor.currentFileIndex);
+            scanOpts.fromFiles = true;
+          }
+          else {
+            scanOpts.reverse = true;
+          }
+        }
+        else {
+          p('Genesis block found. Syncing upto old TIP.');
+          p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks');
+          scanOpts.reverse = true;
+        }
+
+        if (!scanOpts.reverse) return s_c();
+
+        self.rpc.getBlockHash(self.blockChainHeight, function(err, res) {
+          if (err) return s_c(err);
+          lastBlock = res.result;
+          return s_c();
+        });
+      },
+      function(s_c) {
+        if (!scanOpts.reverse) return s_c();
+        self.sync.bDb.getTip(function(err, res) {
+          if (err) return s_c(err);
+          tip = res;
+
+          console.log('Old Tip:', tip);
+          return s_c();
+        });
+      },
+      function(s_c) {
+        if (!scanOpts.reverse) return s_c();
         self.countNotOrphan(function(err, count) {
-          if (err) return cb(err);
+          if (err) return s_c(err);
 
           self.syncedBlocks =  count || 0;
-          return cb();
+          return s_c();
         });
-      }
+      }],
-      else {
+      function(err) {
-          return cb();
+      // SETUP Sync params
-      }
+      var start, end;
-    },
-    ], function(err) {
       if (err) {
         self.setError(err);
         return next(err, 0);
       }
 
-
-      // SETUP Sync params
-      var start, end;
-
       if (!self.step) {
         var step = parseInt( (self.blockChainHeight - self.syncedBlocks) / 1000);
 
@@ -428,46 +478,6 @@ function spec() {
           return next(err);
         });
       }
-
-    });
-  };
-
-  // upto if we have genesis block?
-  HistoricSync.prototype.smartImport = function(scanOpts, next) {
-    var self = this;
-
-    self.sync.bDb.has(self.genesis, function(err, b) {
-      if (err) return next(err);
-      self.countNotOrphan(function(err, count) {
-        if (err) return next(err);
-          self.getBlockCount(function(err) {
-            if (err) return next(err);
-
-            if (!b || scanOpts.destroy || count < self.blockChainHeight * 0.8 ) {
-
-              if (!b)
-                p('Could not find Genesis block. Running FULL SYNC');
-              else
-                p('Less that 80% of current blockchain is stored. Running FULL SYNC',
-                  parseInt(count/self.blockChainHeight*100));
-
-              if (config.bitcoind.dataDir) {
-                p('bitcoind dataDir configured...importing blocks from .dat files');
-                scanOpts.fromFiles = true;
-                self.blockExtractor = new BlockExtractor(config.bitcoind.dataDir, config.network);
-              }
-              else {
-                scanOpts.reverse = true;
-              }
-            }
-            else {
-              p('Genesis block found. Syncing upto old TIP.');
-              p('Got ' + count + ' out of ' + self.blockChainHeight + ' blocks');
-              scanOpts.reverse = true;
-            }
-            return self.importHistory(scanOpts, next);
-        });
-      });
     });
   };
 

lib/Rpc.js

@@ -0,0 +1,103 @@
+'use strict';
+
+require('classtool');
+
+
+function spec(b) {
+  var RpcClient       = require('bitcore/RpcClient').class(),
+      BitcoreTransaction = require('bitcore/Transaction').class(),
+      BitcoreBlock    = require('bitcore/Block').class(),
+      bitcoreUtil     = require('bitcore/util/util'),
+      util         = require('util'),
+      config       = require('../config/config');
+
+   var  bitcoreRpc  = b.bitcoreRpc || new RpcClient(config.bitcoind);
+
+  function Rpc() {
+  }
+
+  Rpc._parseRpcResult = function(info) {
+    var b  = new Buffer(info.hex,'hex');
+    var tx = new BitcoreTransaction();
+    tx.parse(b);
+
+    // Inputs
+    if (tx.isCoinBase())  {
+      info.isCoinBase = true;
+    }
+
+    var n =0;
+    info.vin.forEach(function(i) {
+      i.n = n++;
+    });
+
+    // Outputs
+    var valueOutSat = 0;
+    info.vout.forEach( function(o) {
+      valueOutSat += o.value * bitcoreUtil.COIN;
+    });
+    info.valueOut = parseInt(valueOutSat) / bitcoreUtil.COIN;
+    info.size     = b.length;
+
+    return info;
+  };
+
+
+  Rpc.errMsg = function(err) {
+    var e = err;
+    e.message += util.format(' [Host: %s:%d User:%s Using password:%s]',
+                             bitcoreRpc.host,
+                             bitcoreRpc.port,
+                             bitcoreRpc.user,
+                             bitcoreRpc.pass?'yes':'no'
+                            );
+    return e;
+  };
+
+  Rpc.getRpcInfo = function(txid,  cb) {
+    var self = this;
+
+    bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) {
+
+      // Not found?
+      if (err && err.code === -5) return cb();
+      if (err) return cb(self.errMsg(err));
+
+      var info = self._parseRpcResult(txInfo.result);
+
+      return cb(null,info);
+    });
+  };
+
+
+  Rpc.blockIndex = function(height, cb) {
+    var self = this;
+
+    bitcoreRpc.getBlockHash(height, function(err, bh){
+      if (err) return cb(self.errMsg(err));
+      cb(null, { blockHash: bh.result });
+    });
+  };
+
+  Rpc.getBlock = function(hash, cb) {
+    var self = this;
+
+    bitcoreRpc.getBlock(hash, function(err,info) {
+      // Not found?
+      if (err && err.code === -5) return cb();
+      if (err) return cb(self.errMsg(err));
+
+
+      if (info.result.height)
+        info.result.reward =  BitcoreBlock.getBlockValue(info.result.height) / util.COIN ;
+
+      return cb(err,info);
+    });
+  };
+
+
+  return Rpc;
+}
+module.defineClass(spec);
+
+

lib/Sync.js

@@ -143,7 +143,6 @@ function spec() {
       function(err) {
         if (!err) self._handleBroadcast(b.hash, null, updatedAddrs);
         if (err && err.toString().match(/WARN/) ) {
-          console.log(err);
           err=null;
         }
         return cb(err);

lib/TransactionDb.js

@@ -25,7 +25,7 @@ function spec(b) {
   /**
   * Module dependencies.
   */
-  var TransactionRpc = require('./TransactionRpc').class(),
+  var Rpc = b.rpc || require('./Rpc').class(),
       util        = require('bitcore/util/util'),
       levelup     = require('levelup'),
       async       = require('async'),
@@ -224,7 +224,7 @@ function spec(b) {
   TransactionDb.prototype._getInfo = function(txid, next) {
     var self  = this;
 
-    TransactionRpc.getRpcInfo(txid, function(err, info) {
+    Rpc.getRpcInfo(txid, function(err, info) {
       if (err) return next(err);
 
       self._fillOutpoints(info, function() {
@@ -573,7 +573,7 @@ function spec(b) {
         // TODO: parse it from networks.genesisTX?
         if (t === genesisTXID) return each_cb();
 
-        TransactionRpc.getRpcInfo(t, function(err, inInfo) {
+        Rpc.getRpcInfo(t, function(err, inInfo) {
           if (!inInfo) return each_cb(err);
 
           return self.add(inInfo, blockHash, each_cb);

lib/TransactionRpc.js

@@ -1,62 +0,0 @@
-'use strict';
-
-require('classtool');
-
-
-function spec(b) {
-  var RpcClient       = require('bitcore/RpcClient').class(),
-      BitcoreTransaction = require('bitcore/Transaction').class(),
-      util         = require('bitcore/util/util'),
-      config       = require('../config/config');
-
-   var  rpc  = b.rpc || new RpcClient(config.bitcoind);
-
-  function TransactionRpc() {
-  }
-
-  TransactionRpc._parseRpcResult = function(info) {
-    var b  = new Buffer(info.hex,'hex');
-    var tx = new BitcoreTransaction();
-    tx.parse(b);
-
-    // Inputs
-    if (tx.isCoinBase())  {
-      info.isCoinBase = true;
-    }
-
-    var n =0;
-    info.vin.forEach(function(i) {
-      i.n = n++;
-    });
-
-    // Outputs
-    var valueOutSat = 0;
-    info.vout.forEach( function(o) {
-      valueOutSat += o.value * util.COIN;
-    });
-    info.valueOut = parseInt(valueOutSat) / util.COIN;
-    info.size     = b.length;
-
-    return info;
-  };
-
-  TransactionRpc.getRpcInfo = function(txid,  cb) {
-    var Self = this;
-
-    rpc.getRawTransaction(txid, 1, function(err, txInfo) {
-
-      // Not found?
-      if (err && err.code === -5) return cb();
-      if (err) return cb(err);
-
-      var info = Self._parseRpcResult(txInfo.result);
-
-      return cb(null,info);
-    });
-  };
-
-  return TransactionRpc;
-}
-module.defineClass(spec);
-
-

util/sync.js

@@ -12,12 +12,8 @@ var async = require('async');
 
 program
   .version(SYNC_VERSION)
-  .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet')
   .option('-D --destroy', 'Remove current DB (and start from there)', 0)
-  .option('-R --reverse', 'Sync backwards', 0)
+  .option('-S --startfile', 'Number of file from bitcoind to start(default=0)')
-  .option('-U --uptoexisting', 'Sync only until old Tip block is found', 0)
-  .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0)
-  .option('-S --smart', 'genesis stored? uptoexisting = 1, fromFiles=1 [default]', true)
   .option('-v --verbose', 'Verbose 0/1', 0)
   .parse(process.argv);
 
@@ -33,19 +29,10 @@ async.series([
     historicSync.init(program, cb);
   },
   function(cb) {
-
+    historicSync.smartImport({
-    if (typeof program.smart === 'undefined' || parseInt(program.smart) ) {
+      destroy: program.destroy,
-      historicSync.smartImport({
+      startFile: program.startfile,
-        destroy: program.destroy,
+    },cb);
-      },cb);
-    }
-    else {
-      historicSync.importHistory({
-        destroy: program.destroy,
-        reverse: program.reverse,
-        fromFiles: program.fromfiles,
-      }, cb);
-    }
   },
   ],
   function(err) {