diff --git a/app/models/Block.js b/app/models/Block.js index 709fbcf8..e4d84d38 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -27,6 +27,7 @@ var BlockSchema = new Schema({ }, time: Number, nextBlockHash: String, + isOrphan: Boolean, }); /** diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 84c8614c..fdad11e9 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { } else { if ( !i.coinbase ) { - console.log ('TX: %s,%d could not parse INPUT', txid, i.n); + console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n); } return next_in(); } @@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) { }, next_out); } else { - console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n); + console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n); return next_out(); } }, diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index fdadfccb..fd64c678 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -13,14 +13,16 @@ function spec() { var Sync = require('./Sync').class(); function HistoricSync(opts) { - this.block_count= 0; - this.block_total= 0; this.network = config.network === 'testnet' ? networks.testnet: networks.livenet; var genesisHashReversed = new Buffer(32); this.network.genesisBlock.hash.copy(genesisHashReversed); this.genesis = genesisHashReversed.reverse().toString('hex'); this.sync = new Sync(opts); + + //available status: new / syncing / finished / aborted + this.status = 'new'; + this.syncInfo = {}; } function p() { @@ -32,8 +34,9 @@ function spec() { console.log.apply(this, args); } - var progress_bar = function(string, current, total) { - p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); + var printProgress = function(i) { + var per = parseInt(100 * i.syncedBlocks / i.blocksToSync); + p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per)); }; HistoricSync.prototype.init = function(opts,cb) { @@ -47,7 +50,6 @@ function spec() { }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) { - var self = this; // recursion end. @@ -71,8 +73,11 @@ function spec() { }, //show some (inacurate) status function(c) { - if (self.block_count % 1000 === 1) { - progress_bar('sync status:', self.block_count, self.block_total); + var step = parseInt(self.syncInfo.blocksToSync / 100); + if (step < 10) step = 10; + + if (self.syncInfo.syncedBlocks % step === 1) { + printProgress(self.syncInfo); } return c(); }, @@ -113,15 +118,30 @@ function spec() { ], function (err){ - if (err) - p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count); + if (err) { + self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks); + self.status = 'aborted'; + p(self.err); + } - if (opts.uptoexisting && existed) { - p('DONE. Found existing block: ', blockHash); - return cb(err); + else { + self.err = null; + self.status = 'syncing'; + } + + if (opts.uptoexisting && existed ) { + if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) { + self.status = 'finished'; + p('DONE. Found existing block: ', blockHash); + return cb(err); + } + else { + p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks); + } } if (blockEnd && blockEnd === blockHash) { + self.status = 'finished'; p('DONE. Found END block: ', blockHash); return cb(err); } @@ -129,7 +149,7 @@ function spec() { // Continue if (blockInfo && blockInfo.result) { - self.block_count++; + self.syncInfo.syncedBlocks++; if (opts.prev && blockInfo.result.previousblockhash) { return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb); } @@ -144,11 +164,10 @@ function spec() { HistoricSync.prototype.import_history = function(opts, next) { var self = this; - var retry_attemps = 100; var retry_secs = 2; - var block_best; - var block_height; + var bestBlock; + var blockChainHeight; async.series([ function(cb) { @@ -175,41 +194,56 @@ function spec() { return cb(); } }, - function(cb) { - self.rpc.getInfo(function(err, res) { - if (err) return cb(err); - self.block_total = res.result.blocks; - return cb(); - }); - }, // We are not using getBestBlockHash, because is not available in all clients function(cb) { if (!opts.reverse) return cb(); self.rpc.getBlockCount(function(err, res) { if (err) return cb(err); - block_height = res.result; + blockChainHeight = res.result; return cb(); }); }, function(cb) { if (!opts.reverse) return cb(); - self.rpc.getBlockHash(block_height, function(err, res) { + self.rpc.getBlockHash(blockChainHeight, function(err, res) { if (err) return cb(err); - block_best = res.result; + bestBlock = res.result; + return cb(); }); }, + function(cb) { + // This is only to inform progress. + if (!opts.uptoexisting) { + self.rpc.getInfo(function(err, res) { + if (err) return cb(err); + self.syncInfo.blocksToSync = res.result.blocks; + return cb(); + }); + } + else { + // should be isOrphan = true or null to be more accurate. + Block.count({ isOrphan: null}, function(err, count) { + if (err) return cb(err); + + self.syncInfo.blocksToSync = blockChainHeight - count; + if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1; + return cb(); + }); + } + }, ], function(err) { + var start, end; function sync() { if (opts.reverse) { - start = block_best; + start = bestBlock; end = self.genesis; opts.prev = true; } @@ -219,25 +253,38 @@ function spec() { opts.next = true; } + self.syncInfo = util._extend(self.syncInfo, { + start: start, + isStartGenesis: start === self.genesis, + end: end, + isEndGenesis: end === self.genesis, + scanningForward: opts.next, + scanningBackward: opts.prev, + uptoexisting: opts.uptoexisting, + syncedBlocks: 0, + }); + p('Starting from: ', start); p(' to : ', end); p(' opts: ', JSON.stringify(opts)); self.getPrevNextBlock( start, end, opts , function(err) { - if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){ + if (err && err.message.match(/ECONNREFUSED/)){ setTimeout(function() { p('Retrying in %d secs', retry_secs); sync(); }, retry_secs * 1000); } else - return next(err, self.block_count); + return next(err); }); } + if (!err) sync(); - else + else { return next(err, 0); + } }); }; @@ -246,6 +293,7 @@ function spec() { var self = this; Block.findOne({hash:self.genesis}, function(err, b){ + if (err) return next(err); @@ -253,7 +301,7 @@ function spec() { p('Could not find Genesis block. Running FULL SYNC'); } else { - p('Genesis block found. Syncing upto know blocks.'); + p('Genesis block found. Syncing upto known blocks.'); } var opts = { diff --git a/lib/Sync.js b/lib/Sync.js index ff2ffb8f..8561d4a4 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -15,6 +15,48 @@ function spec() { this.tx_count = 0; } + Sync.prototype.init = function(opts, cb) { + var that = this; + + that.opts = opts; + + if (!(opts && opts.skip_db_connection)) { + + + if (!mongoose.connection.readyState == 1) { + mongoose.connect(config.db, function(err) { + if (err) { + console.log('CRITICAL ERROR: connecting to mongoDB:',err); + return (err); + } + }); + } + + that.db = mongoose.connection; + + that.db.on('error', function(err) { + console.log('MongoDB ERROR:' + err); + return cb(err); + }); + + that.db.on('disconnect', function(err) { + console.log('MongoDB disconnect:' + err); + return cb(err); + }); + + return that.db.once('open', function(err) { + return cb(err); + }); + } + else return cb(); + }; + + Sync.prototype.close = function() { + if ( this.db && this.db.readyState ) { + this.db.close(); + } + }; + Sync.prototype.storeBlock = function(block, cb) { var that = this; @@ -55,40 +97,6 @@ function spec() { return cb(err); }); }; - - - Sync.prototype.init = function(opts, cb) { - var that = this; - - that.opts = opts; - - if (!(opts && opts.skip_db_connection)) { - if (!mongoose.connection) { - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - } - - this.db = mongoose.connection; - - this.db.on('error', function(err) { - console.log('connection error:' + err); - mongoose.disconnect(); - }); - - this.db.on('disconnect', function(err) { - console.log('disconnect:' + err); - mongoose.connect(config.db, {server: {auto_reconnect: true}} ); - }); - - return that.db.once('open', cb); - } - else return cb(); - }; - - Sync.prototype.close = function() { - if (!(this.opts && this.opts.skip_db_connection)) { - this.db.close(); - } - }; return Sync; } module.defineClass(spec); diff --git a/server.js b/server.js index 84e659a4..38f607fb 100644 --- a/server.js +++ b/server.js @@ -50,8 +50,11 @@ if (!config.disableHistoricSync) { skip_db_connection: true, networkName: config.network }, function() { - hs.smart_import(function(){ - console.log('[historic_sync] finished!'); + hs.smart_import(function(err){ + var txt= 'ended.'; + if (err) txt = 'ABORTED with error: ' + err.message; + + console.log('[historic_sync] ' + txt, hs.syncInfo); }); }); } diff --git a/util/sync.js b/util/sync.js index beccf0f9..01205d9d 100755 --- a/util/sync.js +++ b/util/sync.js @@ -29,7 +29,6 @@ if (program.remove) { } */ - async.series([ function(cb) { historicSync.init(program, cb); @@ -46,18 +45,14 @@ async.series([ }, cb); } }, - function(cb) { - historicSync.close(); - return cb(); - }, ], - function(err, count) { + function(err) { + historicSync.close(); if (err) { console.log('CRITICAL ERROR: ', err); } else { - console.log('Finished. [%d blocks synced]', count[1]); + console.log('Finished.\n Status:\n', historicSync.syncInfo); } - return; });