diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 1d30e316..bf572419 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -326,6 +326,7 @@ BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) { if (o.multipleSpentAttempts) { async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, function(oi, e_c) { + // Only one will be confirmed self.getBlockForTx(oi.spentTxId, function(err, hash, height) { if (err) return; if (height>=0) { @@ -417,7 +418,17 @@ BlockDb.prototype.migrateV02cleanup = function(cb) { end: k + '~' }) .pipe(d.createWriteStream({type:'del'})) - .on('close',cb); + .on('close', function(err){ + if (err) return cb(err); + var k = 'txa-'; + var d = self.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close', cb); + }); }); }; diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 2e92bd3a..3c698246 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -9,62 +9,53 @@ var bitcore = require('bitcore'), async = require('async'); function BlockExtractor(dataDir, network) { - - var self = this; var path = dataDir + '/blocks/blk*.dat'; - self.dataDir = dataDir; - self.files = glob.sync(path); - self.nfiles = self.files.length; + this.dataDir = dataDir; + this.files = glob.sync(path); + this.nfiles = this.files.length; - if (self.nfiles === 0) + if (this.nfiles === 0) throw new Error('Could not find block files at: ' + path); - self.currentFileIndex = 0; - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; - self.network = network === 'testnet' ? networks.testnet: networks.livenet; - self.magic = self.network.magic.toString('hex'); + this.currentFileIndex = 0; + this.isCurrentRead = false; + this.currentBuffer = null; + this.currentParser = null; + this.network = network === 'testnet' ? networks.testnet: networks.livenet; + this.magic = this.network.magic.toString('hex'); } BlockExtractor.prototype.currentFile = function() { - var self = this; - - return self.files[self.currentFileIndex]; + return this.files[this.currentFileIndex]; }; BlockExtractor.prototype.nextFile = function() { - var self = this; - - if (self.currentFileIndex < 0) return false; + if (this.currentFileIndex < 0) return false; var ret = true; - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; + this.isCurrentRead = false; + this.currentBuffer = null; + this.currentParser = null; - if (self.currentFileIndex < self.nfiles - 1) { - self.currentFileIndex++; + if (this.currentFileIndex < this.nfiles - 1) { + this.currentFileIndex++; } else { - self.currentFileIndex=-1; + this.currentFileIndex=-1; ret = false; } return ret; }; BlockExtractor.prototype.readCurrentFileSync = function() { - var self = this; + if (this.currentFileIndex < 0 || this.isCurrentRead) return; - if (self.currentFileIndex < 0 || self.isCurrentRead) return; + this.isCurrentRead = true; - - self.isCurrentRead = true; - - var fname = self.currentFile(); + var fname = this.currentFile(); if (!fname) return; @@ -81,80 +72,70 @@ BlockExtractor.prototype.readCurrentFileSync = function() { fs.readSync(fd, buffer, 0, size, 0); - self.currentBuffer = buffer; - self.currentParser = new Parser(buffer); + this.currentBuffer = buffer; + this.currentParser = new Parser(buffer); }; -BlockExtractor.prototype.getNextBlock = function(cb) { - var self = this; +BlockExtractor.prototype._getMagic = function() { + if (!this.currentParser) + return null; + + var byte0 = this.currentParser ? this.currentParser.buffer(1).toString('hex') : null; + + + + // Grab 3 bytes from block without removing them + var p = this.currentParser.pos; + var bytes123 = this.currentParser.subject.toString('hex',p,p+3); + var magic = byte0 + bytes123; + + if (magic !=='00000000' && magic !== this.magic) { + if(this.errorCount++ > 4) + throw new Error('CRITICAL ERROR: Magic number mismatch: ' + + magic + '!=' + this.magic); + magic=null; + } + + if (magic==='00000000') + magic =null; + + return magic; +}; + +BlockExtractor.prototype.getNextBlock = function(cb) { var b; var magic; - async.series([ - function (a_cb) { + var isFinished = 0; - async.whilst( - function() { - return (!magic || magic === '00000000'); - }, - function(w_cb) { - magic = null; + while(!magic && !isFinished) { + this.readCurrentFileSync(); + magic= this._getMagic(); - self.readCurrentFileSync(); - if (self.currentFileIndex < 0) return cb(); + if (!this.currentParser || this.currentParser.eof() ) { - var byte0 = self.currentParser ? self.currentParser.buffer(1).toString('hex') : null; + if (this.nextFile()) { + console.log('Moving forward to file:' + this.currentFile() ); + magic = null; + } else { + console.log('Finished all files'); + isFinished = 1; + } + } + } + if (isFinished) + return cb(); - if (byte0) { - // Grab 3 bytes from block without removing them - var p = self.currentParser.pos; - var bytes123 = self.currentParser.subject.toString('hex',p,p+3); - magic = byte0 + bytes123; + // Remove 3 bytes from magic and spacer + this.currentParser.buffer(3+4); - if (magic !=='00000000' && magic !== self.magic) { - - if (self.errorCount++ > 4) - return cb(new Error('CRITICAL ERROR: Magic number mismatch: ' + - magic + '!=' + self.magic)); - - magic=null; - } - } - - if (!self.currentParser || self.currentParser.eof() ) { - if (self.nextFile()) - console.log('Moving forward to file:' + self.currentFile() ); - else - console.log('Finished all files'); - - magic = null; - return w_cb(); - } - else { - return w_cb(); - } - }, a_cb); - }, - function (a_cb) { - if (!magic) return a_cb(); - // Remove 3 bytes from magic and spacer - self.currentParser.buffer(3+4); - return a_cb(); - }, - function (a_cb) { - if (!magic) return a_cb(); - - b = new Block(); - b.parse(self.currentParser); - b.getHash(); - self.errorCount=0; - return a_cb(); - }, - ], function(err) { - return cb(err,b); - }); + b = new Block(); + b.parse(this.currentParser); + b.getHash(); + this.errorCount=0; + return cb(null,b); }; module.exports = require('soop')(BlockExtractor); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 41251b6f..e9282a68 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -15,6 +15,10 @@ var bitcoreUtil = bitcore.util; var logger = require('./logger').logger; var info = logger.info; var error = logger.error; +var PERCENTAGE_TO_START_FROM_RPC = 1.1; + +// TODO TODO TODO +//var PERCENTAGE_TO_START_FROM_RPC = 0.98; // var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -244,7 +248,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { var self = this; if ( opts.forceRPC || !config.bitcoind.dataDir || - self.height > self.blockChainHeight * 0.9) return next(); + self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next(); try { @@ -279,6 +283,9 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { }); }); }, function(err){ + console.log('\tFOUND Starting Block!'); + + // TODO SET HEIGHT return next(err); }); }); @@ -394,6 +401,7 @@ HistoricSync.prototype.start = function(opts, next) { function (w_cb) { self.getFn(function(err,blockInfo) { if (err) return w_cb(self.setError(err)); + if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) { self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) { if (err) return w_cb(self.setError(err)); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 03f8fe83..22374319 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -11,7 +11,6 @@ var SPENT_PREFIX = 'txs-'; //txs---- = ts // to sum up addr balance (only outs, spents are gotten later) var ADDR_PREFIX = 'txa2-'; //txa---- // tsr = 1e13-js_timestamp - // => + btc_sat [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs] // |balance:txApperances @@ -201,6 +200,26 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) { i.valueSat = ret.valueSat; i.value = ret.valueSat / util.COIN; valueIn += i.valueSat; + + if (ret.multipleSpentAttempt || !ret.spentTxId || + (ret.spentTxId && ret.spentTxId !== info.txid) + ) { + if (ret.multipleSpentAttempts) { + ret.multipleSpentAttempts.forEach(function(mul) { + if (mul.spentTxId !== info.txid) { + i.doubleSpentTxID = ret.spentTxId; + i.doubleSpentIndex = ret.spentIndex; + } + }); + } else if (!ret.spentTxId) { + i.dbError = 'Input spent not registered'; + } else { + i.doubleSpentTxID = ret.spentTxId; + i.doubleSpentIndex = ret.spentIndex; + } + } else { + i.doubleSpentTxID = null; + } return c_in(); }); }, @@ -640,22 +659,26 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) { TransactionDb.prototype.checkVersion02 = function(cb) { var k = 'txa-'; + var isV2=1; db.createReadStream({ start: k, - end: k + '~' + end: k + '~', + limit: 1, }) .on('data', function(data) { -console.log('[TransactionDb.js.689]',data); //TODO - return cb(0); + isV2=0; }) .on('end', function (){ - return cb(1); + return cb(isV2); }); }; TransactionDb.prototype.migrateV02 = function(cb) { var k = 'txa-'; var dbScript = []; + var c=0; + var c2=0; + var N=50000; db.createReadStream({ start: k, end: k + '~' @@ -668,12 +691,19 @@ TransactionDb.prototype.migrateV02 = function(cb) { key: ADDR_PREFIX + k[1] + '-' + (END_OF_WORLD_TS - parseInt(v[1])) + '-' + k[3] + '-' + [4], value: v[0], }); + if (c++>N) { + console.log('\t%dM txs processed', ((c2+=N)/1e6).toFixed(3)); //TODO + db.batch(dbScript,function () { + c=0; + dbScript=[]; + }); + } }) .on('error', function(err) { return cb(err); }) .on('end', function (){ - db.batch(dbScript,cb); + return cb(); }); }; diff --git a/util/updateToV0.2.js b/util/upgradeV0.2js similarity index 100% rename from util/updateToV0.2.js rename to util/upgradeV0.2js