RanchiMall/flocore-node
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better smart syncs

Browse Source
This commit is contained in:
Matias Alejo Garcia 2014-01-18 18:28:24 -03:00
parent c8adefddd9
commit 4d23191116
6 changed files with 132 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/models/Block.js
View File

@ -27,6 +27,7 @@ var BlockSchema = new Schema({
},
time: Number,
nextBlockHash: String,
isOrphan: Boolean,
});
/**

4
app/models/Transaction.js
View File

@ -142,7 +142,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
}
else {
if ( !i.coinbase ) {
console.log ('TX: %s,%d could not parse INPUT', txid, i.n);
console.log ('WARN in TX: %s: could not parse INPUT %d', txid, i.n);
}
return next_in();
}
@ -165,7 +165,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, time, cb) {
}, next_out);
}
else {
console.log ('TX: %s,%d could not parse OUTPUT', txid, o.n);
console.log ('WARN in TX: %s could not parse OUTPUT %d', txid, o.n);
return next_out();
}
},

110
lib/HistoricSync.js
View File

@ -13,14 +13,16 @@ function spec() {
var Sync = require('./Sync').class();
function HistoricSync(opts) {
this.block_count= 0;
this.block_total= 0;
this.network = config.network === 'testnet' ? networks.testnet: networks.livenet;
var genesisHashReversed = new Buffer(32);
this.network.genesisBlock.hash.copy(genesisHashReversed);
this.genesis = genesisHashReversed.reverse().toString('hex');
this.sync = new Sync(opts);
//available status: new / syncing / finished / aborted
this.status = 'new';
this.syncInfo = {};
}
function p() {
@ -32,8 +34,9 @@ function spec() {
console.log.apply(this, args);
}
var progress_bar = function(string, current, total) {
p(util.format('%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total)));
var printProgress = function(i) {
var per = parseInt(100 * i.syncedBlocks / i.blocksToSync);
p(util.format('status: %d/%d [%d%%]', i.syncedBlocks, i.blocksToSync, per));
};
HistoricSync.prototype.init = function(opts,cb) {
@ -47,7 +50,6 @@ function spec() {
};
HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, opts, cb) {
var self = this;
// recursion end.
@ -71,8 +73,11 @@ function spec() {
},
//show some (inacurate) status
function(c) {
if (self.block_count % 1000 === 1) {
progress_bar('sync status:', self.block_count, self.block_total);
var step = parseInt(self.syncInfo.blocksToSync / 100);
if (step < 10) step = 10;
if (self.syncInfo.syncedBlocks % step === 1) {
printProgress(self.syncInfo);
}
return c();
},
@ -113,15 +118,30 @@ function spec() {
],
function (err){
if (err)
p('ERROR: @%s: %s [count: block_count: %d]', blockHash, err, self.block_count);
if (err) {
self.err = util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncInfo.syncedBlocks);
self.status = 'aborted';
p(self.err);
}
if (opts.uptoexisting && existed) {
p('DONE. Found existing block: ', blockHash);
return cb(err);
else {
self.err = null;
self.status = 'syncing';
}
if (opts.uptoexisting && existed ) {
if (self.syncInfo.blocksToSync <= self.syncInfo.syncedBlocks) {
self.status = 'finished';
p('DONE. Found existing block: ', blockHash);
return cb(err);
}
else {
p('WARN found target block\n\tbut blockChain Height is still higher that ours. Previous light sync must be interrupted.\n\tWill keep syncing.', self.syncInfo.syncedBlocks);
}
}
if (blockEnd && blockEnd === blockHash) {
self.status = 'finished';
p('DONE. Found END block: ', blockHash);
return cb(err);
}
@ -129,7 +149,7 @@ function spec() {
// Continue
if (blockInfo && blockInfo.result) {
self.block_count++;
self.syncInfo.syncedBlocks++;
if (opts.prev && blockInfo.result.previousblockhash) {
return self.getPrevNextBlock(blockInfo.result.previousblockhash, blockEnd, opts, cb);
}
@ -144,11 +164,10 @@ function spec() {
HistoricSync.prototype.import_history = function(opts, next) {
var self = this;
var retry_attemps = 100;
var retry_secs = 2;
var block_best;
var block_height;
var bestBlock;
var blockChainHeight;
async.series([
function(cb) {
@ -175,41 +194,56 @@ function spec() {
return cb();
}
},
function(cb) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
self.block_total = res.result.blocks;
return cb();
});
},
// We are not using getBestBlockHash, because is not available in all clients
function(cb) {
if (!opts.reverse) return cb();
self.rpc.getBlockCount(function(err, res) {
if (err) return cb(err);
block_height = res.result;
blockChainHeight = res.result;
return cb();
});
},
function(cb) {
if (!opts.reverse) return cb();
self.rpc.getBlockHash(block_height, function(err, res) {
self.rpc.getBlockHash(blockChainHeight, function(err, res) {
if (err) return cb(err);
block_best = res.result;
bestBlock = res.result;
return cb();
});
},
function(cb) {
// This is only to inform progress.
if (!opts.uptoexisting) {
self.rpc.getInfo(function(err, res) {
if (err) return cb(err);
self.syncInfo.blocksToSync = res.result.blocks;
return cb();
});
}
else {
// should be isOrphan = true or null to be more accurate.
Block.count({ isOrphan: null}, function(err, count) {
if (err) return cb(err);
self.syncInfo.blocksToSync = blockChainHeight - count;
if (self.syncInfo.blocksToSync < 1) self.syncInfo.blocksToSync = 1;
return cb();
});
}
},
],
function(err) {
var start, end;
function sync() {
if (opts.reverse) {
start = block_best;
start = bestBlock;
end = self.genesis;
opts.prev = true;
}
@ -219,25 +253,38 @@ function spec() {
opts.next = true;
}
self.syncInfo = util._extend(self.syncInfo, {
start: start,
isStartGenesis: start === self.genesis,
end: end,
isEndGenesis: end === self.genesis,
scanningForward: opts.next,
scanningBackward: opts.prev,
uptoexisting: opts.uptoexisting,
syncedBlocks: 0,
});
p('Starting from: ', start);
p(' to : ', end);
p(' opts: ', JSON.stringify(opts));
self.getPrevNextBlock( start, end, opts , function(err) {
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
if (err && err.message.match(/ECONNREFUSED/)){
setTimeout(function() {
p('Retrying in %d secs', retry_secs);
sync();
}, retry_secs * 1000);
}
else
return next(err, self.block_count);
return next(err);
});
}
if (!err)
sync();
else
else {
return next(err, 0);
}
});
};
@ -246,6 +293,7 @@ function spec() {
var self = this;
Block.findOne({hash:self.genesis}, function(err, b){
if (err) return next(err);
@ -253,7 +301,7 @@ function spec() {
p('Could not find Genesis block. Running FULL SYNC');
}
else {
p('Genesis block found. Syncing upto know blocks.');
p('Genesis block found. Syncing upto known blocks.');
}
var opts = {

76
lib/Sync.js
View File

@ -15,6 +15,48 @@ function spec() {
this.tx_count = 0;
}
Sync.prototype.init = function(opts, cb) {
var that = this;
that.opts = opts;
if (!(opts && opts.skip_db_connection)) {
if (!mongoose.connection.readyState == 1) {
mongoose.connect(config.db, function(err) {
if (err) {
console.log('CRITICAL ERROR: connecting to mongoDB:',err);
return (err);
}
});
}
that.db = mongoose.connection;
that.db.on('error', function(err) {
console.log('MongoDB ERROR:' + err);
return cb(err);
});
that.db.on('disconnect', function(err) {
console.log('MongoDB disconnect:' + err);
return cb(err);
});
return that.db.once('open', function(err) {
return cb(err);
});
}
else return cb();
};
Sync.prototype.close = function() {
if ( this.db && this.db.readyState ) {
this.db.close();
}
};
Sync.prototype.storeBlock = function(block, cb) {
var that = this;
@ -55,40 +97,6 @@ function spec() {
return cb(err);
});
};
Sync.prototype.init = function(opts, cb) {
var that = this;
that.opts = opts;
if (!(opts && opts.skip_db_connection)) {
if (!mongoose.connection) {
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
}
this.db = mongoose.connection;
this.db.on('error', function(err) {
console.log('connection error:' + err);
mongoose.disconnect();
});
this.db.on('disconnect', function(err) {
console.log('disconnect:' + err);
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
});
return that.db.once('open', cb);
}
else return cb();
};
Sync.prototype.close = function() {
if (!(this.opts && this.opts.skip_db_connection)) {
this.db.close();
}
};
return Sync;
}
module.defineClass(spec);

7
server.js
View File

@ -50,8 +50,11 @@ if (!config.disableHistoricSync) {
skip_db_connection: true,
networkName: config.network
}, function() {
hs.smart_import(function(){
console.log('[historic_sync] finished!');
hs.smart_import(function(err){
var txt= 'ended.';
if (err) txt = 'ABORTED with error: ' + err.message;
console.log('[historic_sync] ' + txt, hs.syncInfo);
});
});
}

11
util/sync.js
View File

@ -29,7 +29,6 @@ if (program.remove) {
}
*/
async.series([
function(cb) {
historicSync.init(program, cb);
@ -46,18 +45,14 @@ async.series([
}, cb);
}
},
function(cb) {
historicSync.close();
return cb();
},
],
function(err, count) {
function(err) {
historicSync.close();
if (err) {
console.log('CRITICAL ERROR: ', err);
}
else {
console.log('Finished. [%d blocks synced]', count[1]);
console.log('Finished.\n Status:\n', historicSync.syncInfo);
}
return;
});