diff --git a/app/controllers/addresses.js b/app/controllers/addresses.js index 089990a4..48b5f34a 100644 --- a/app/controllers/addresses.js +++ b/app/controllers/addresses.js @@ -53,7 +53,7 @@ exports.show = function(req, res, next) { } else { return res.jsonp(a.getObj()); } - }, {noTxList: req.query.noTxList}); + }, {txLimit: req.query.noTxList?0:-1}); } }; diff --git a/app/models/Address.js b/app/models/Address.js index 8629c6ab..193bfe5f 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -104,13 +104,13 @@ Address.prototype._addTxItem = function(txItem, txList) { add=1; if (txList) - txList.push({txid: txItem.txid, ts: txItem.ts}); + txList.push(txItem.txid); } // Spent tx if (txItem.spentTxId && !seen[txItem.spentTxId] ) { if (txList) { - txList.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + txList.push(txItem.spentTxId); } seen[txItem.spentTxId]=1; addSpend=1; @@ -140,29 +140,16 @@ Address.prototype._addTxItem = function(txItem, txList) { } }; -Address.prototype._setTxs = function(txs) { - - // sort input and outputs togheter - txs.sort( - function compare(a,b) { - if (a.ts < b.ts) return 1; - if (a.ts > b.ts) return -1; - return 0; - }); - - this.transactions = txs.map(function(i) { return i.txid; } ); -}; - // opts are -// .noTxList // .onlyUnspent -// .noSortTxs +// .txLimit (=0 -> no txs, => -1 no limit) +// Address.prototype.update = function(next, opts) { var self = this; if (!self.addrStr) return next(); opts = opts || {}; - var txList = opts.noTxList ? null : []; + var txList = opts.txLimit === 0 ? null: []; var tDb = TransactionDb; var bDb = BlockDb; tDb.fromAddr(self.addrStr, function(err,txOut){ @@ -172,8 +159,8 @@ Address.prototype.update = function(next, opts) { if (err) return next(err); tDb.cacheConfirmations(txOut, function(err) { +// console.log('[Address.js.161:txOut:]',txOut); //TODO if (err) return next(err); - if (opts.onlyUnspent) { txOut = txOut.filter(function(x){ return !x.spentTxId; @@ -197,32 +184,13 @@ Address.prototype.update = function(next, opts) { txOut.forEach(function(txItem){ self._addTxItem(txItem, txList); }); - - if (txList && !opts.noSortTxs) - self._setTxs(txList); + if (txList) self.transactions = txList; return next(); } }); }); }); }; -Address.prototype.getUtxo = function(next) { - var self = this; - var tDb = TransactionDb; - var bDb = BlockDb; - var ret; - if (!self.addrStr) return next(new Error('no error')); - - tDb.fromAddr(self.addrStr, function(err,txOut){ - if (err) return next(err); - var unspent = txOut.filter(function(x){ - return !x.spentTxId; - }); - - bDb.fillConfirmations(unspent, function() { - }); - }); -}; module.exports = require('soop')(Address); diff --git a/lib/BlockDb.js b/lib/BlockDb.js index a172c0e8..bf572419 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -283,7 +283,7 @@ BlockDb.prototype.fromHashWithInfo = function(hash, cb) { self.getHeight(hash, function(err, height) { if (err) return cb(err); - info.isMainChain = height ? true : false; + info.isMainChain = height>=0 ? true : false; return cb(null, { hash: hash, @@ -326,6 +326,7 @@ BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) { if (o.multipleSpentAttempts) { async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, function(oi, e_c) { + // Only one will be confirmed self.getBlockForTx(oi.spentTxId, function(err, hash, height) { if (err) return; if (height>=0) { @@ -417,7 +418,17 @@ BlockDb.prototype.migrateV02cleanup = function(cb) { end: k + '~' }) .pipe(d.createWriteStream({type:'del'})) - .on('close',cb); + .on('close', function(err){ + if (err) return cb(err); + var k = 'txa-'; + var d = self.txDb._db; + d.createReadStream({ + start: k, + end: k + '~' + }) + .pipe(d.createWriteStream({type:'del'})) + .on('close', cb); + }); }); }; diff --git a/lib/BlockExtractor.js b/lib/BlockExtractor.js index 2e92bd3a..3c698246 100644 --- a/lib/BlockExtractor.js +++ b/lib/BlockExtractor.js @@ -9,62 +9,53 @@ var bitcore = require('bitcore'), async = require('async'); function BlockExtractor(dataDir, network) { - - var self = this; var path = dataDir + '/blocks/blk*.dat'; - self.dataDir = dataDir; - self.files = glob.sync(path); - self.nfiles = self.files.length; + this.dataDir = dataDir; + this.files = glob.sync(path); + this.nfiles = this.files.length; - if (self.nfiles === 0) + if (this.nfiles === 0) throw new Error('Could not find block files at: ' + path); - self.currentFileIndex = 0; - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; - self.network = network === 'testnet' ? networks.testnet: networks.livenet; - self.magic = self.network.magic.toString('hex'); + this.currentFileIndex = 0; + this.isCurrentRead = false; + this.currentBuffer = null; + this.currentParser = null; + this.network = network === 'testnet' ? networks.testnet: networks.livenet; + this.magic = this.network.magic.toString('hex'); } BlockExtractor.prototype.currentFile = function() { - var self = this; - - return self.files[self.currentFileIndex]; + return this.files[this.currentFileIndex]; }; BlockExtractor.prototype.nextFile = function() { - var self = this; - - if (self.currentFileIndex < 0) return false; + if (this.currentFileIndex < 0) return false; var ret = true; - self.isCurrentRead = false; - self.currentBuffer = null; - self.currentParser = null; + this.isCurrentRead = false; + this.currentBuffer = null; + this.currentParser = null; - if (self.currentFileIndex < self.nfiles - 1) { - self.currentFileIndex++; + if (this.currentFileIndex < this.nfiles - 1) { + this.currentFileIndex++; } else { - self.currentFileIndex=-1; + this.currentFileIndex=-1; ret = false; } return ret; }; BlockExtractor.prototype.readCurrentFileSync = function() { - var self = this; + if (this.currentFileIndex < 0 || this.isCurrentRead) return; - if (self.currentFileIndex < 0 || self.isCurrentRead) return; + this.isCurrentRead = true; - - self.isCurrentRead = true; - - var fname = self.currentFile(); + var fname = this.currentFile(); if (!fname) return; @@ -81,80 +72,70 @@ BlockExtractor.prototype.readCurrentFileSync = function() { fs.readSync(fd, buffer, 0, size, 0); - self.currentBuffer = buffer; - self.currentParser = new Parser(buffer); + this.currentBuffer = buffer; + this.currentParser = new Parser(buffer); }; -BlockExtractor.prototype.getNextBlock = function(cb) { - var self = this; +BlockExtractor.prototype._getMagic = function() { + if (!this.currentParser) + return null; + + var byte0 = this.currentParser ? this.currentParser.buffer(1).toString('hex') : null; + + + + // Grab 3 bytes from block without removing them + var p = this.currentParser.pos; + var bytes123 = this.currentParser.subject.toString('hex',p,p+3); + var magic = byte0 + bytes123; + + if (magic !=='00000000' && magic !== this.magic) { + if(this.errorCount++ > 4) + throw new Error('CRITICAL ERROR: Magic number mismatch: ' + + magic + '!=' + this.magic); + magic=null; + } + + if (magic==='00000000') + magic =null; + + return magic; +}; + +BlockExtractor.prototype.getNextBlock = function(cb) { var b; var magic; - async.series([ - function (a_cb) { + var isFinished = 0; - async.whilst( - function() { - return (!magic || magic === '00000000'); - }, - function(w_cb) { - magic = null; + while(!magic && !isFinished) { + this.readCurrentFileSync(); + magic= this._getMagic(); - self.readCurrentFileSync(); - if (self.currentFileIndex < 0) return cb(); + if (!this.currentParser || this.currentParser.eof() ) { - var byte0 = self.currentParser ? self.currentParser.buffer(1).toString('hex') : null; + if (this.nextFile()) { + console.log('Moving forward to file:' + this.currentFile() ); + magic = null; + } else { + console.log('Finished all files'); + isFinished = 1; + } + } + } + if (isFinished) + return cb(); - if (byte0) { - // Grab 3 bytes from block without removing them - var p = self.currentParser.pos; - var bytes123 = self.currentParser.subject.toString('hex',p,p+3); - magic = byte0 + bytes123; + // Remove 3 bytes from magic and spacer + this.currentParser.buffer(3+4); - if (magic !=='00000000' && magic !== self.magic) { - - if (self.errorCount++ > 4) - return cb(new Error('CRITICAL ERROR: Magic number mismatch: ' + - magic + '!=' + self.magic)); - - magic=null; - } - } - - if (!self.currentParser || self.currentParser.eof() ) { - if (self.nextFile()) - console.log('Moving forward to file:' + self.currentFile() ); - else - console.log('Finished all files'); - - magic = null; - return w_cb(); - } - else { - return w_cb(); - } - }, a_cb); - }, - function (a_cb) { - if (!magic) return a_cb(); - // Remove 3 bytes from magic and spacer - self.currentParser.buffer(3+4); - return a_cb(); - }, - function (a_cb) { - if (!magic) return a_cb(); - - b = new Block(); - b.parse(self.currentParser); - b.getHash(); - self.errorCount=0; - return a_cb(); - }, - ], function(err) { - return cb(err,b); - }); + b = new Block(); + b.parse(this.currentParser); + b.getHash(); + this.errorCount=0; + return cb(null,b); }; module.exports = require('soop')(BlockExtractor); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 8fb8905f..d6de02da 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -15,6 +15,10 @@ var bitcoreUtil = bitcore.util; var logger = require('./logger').logger; var info = logger.info; var error = logger.error; +var PERCENTAGE_TO_START_FROM_RPC = 1.1; + +// TODO TODO TODO +//var PERCENTAGE_TO_START_FROM_RPC = 0.98; // var Deserialize = require('bitcore/Deserialize'); var BAD_GEN_ERROR = 'Bad genesis block. Network mismatch between Insight and bitcoind? Insight is configured for:'; @@ -244,7 +248,7 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { var self = this; if ( opts.forceRPC || !config.bitcoind.dataDir || - self.height > self.blockChainHeight * 0.9) return next(); + self.height > self.blockChainHeight * PERCENTAGE_TO_START_FROM_RPC) return next(); try { @@ -279,6 +283,9 @@ HistoricSync.prototype.prepareFileSync = function(opts, next) { }); }); }, function(err){ + console.log('\tFOUND Starting Block!'); + + // TODO SET HEIGHT return next(err); }); }); @@ -394,6 +401,7 @@ HistoricSync.prototype.start = function(opts, next) { function (w_cb) { self.getFn(function(err,blockInfo) { if (err) return w_cb(self.setError(err)); + if (blockInfo && blockInfo.hash && (!opts.stopAt || opts.stopAt !== blockInfo.hash)) { self.sync.storeTipBlock(blockInfo, self.allowReorgs, function(err, height) { if (err) return w_cb(self.setError(err)); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 66762a25..22374319 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -3,13 +3,17 @@ var imports = require('soop').imports(); + // to show tx outs var OUTS_PREFIX = 'txo-'; //txo-- => [addr, btc_sat] var SPENT_PREFIX = 'txs-'; //txs---- = ts // to sum up addr balance (only outs, spents are gotten later) -var ADDR_PREFIX = 'txa-'; //txa--- - // => + btc_sat:ts [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs] +var ADDR_PREFIX = 'txa2-'; //txa---- +// tsr = 1e13-js_timestamp +// => + btc_sat [:isConfirmed:[scriptPubKey|isSpendConfirmed:SpentTxid:SpentVout:SpentTs] +// |balance:txApperances + // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; @@ -17,6 +21,7 @@ var CONCURRENCY = 10; var DEFAULT_SAFE_CONFIRMATIONS = 6; var MAX_OPEN_FILES = 500; +var END_OF_WORLD_TS = 1e13; // var CONFIRMATION_NR_TO_NOT_CHECK = 10; //Spend /** * Module dependencies. @@ -189,29 +194,19 @@ TransactionDb.prototype._fillOutpoints = function(txInfo, cb) { return c_in(); // error not scalated } - txInfo.firstSeenTs = ret.spentTs; + txInfo.firstSeenTs = ret.ts; i.unconfirmedInput = i.unconfirmedInput; i.addr = ret.addr; i.valueSat = ret.valueSat; i.value = ret.valueSat / util.COIN; valueIn += i.valueSat; -/* -* If confirmed by bitcoind, we could not check for double spents -* but we prefer to keep the flag of double spent attempt -* - if (txInfo.confirmations - && txInfo.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) - return c_in(); -isspent -*/ - // Double spent? if (ret.multipleSpentAttempt || !ret.spentTxId || - (ret.spentTxId && ret.spentTxId !== txInfo.txid) + (ret.spentTxId && ret.spentTxId !== info.txid) ) { if (ret.multipleSpentAttempts) { ret.multipleSpentAttempts.forEach(function(mul) { - if (mul.spentTxId !== txInfo.txid) { + if (mul.spentTxId !== info.txid) { i.doubleSpentTxID = ret.spentTxId; i.doubleSpentIndex = ret.spentIndex; } @@ -275,39 +270,27 @@ TransactionDb.prototype.fromIdWithInfo = function(txid, cb) { }); }; +// Gets address info from an outpoint TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { var self = this; var k = OUTS_PREFIX + txid + '-' + n; db.get(k, function(err, val) { + var ret; + if (!val || (err && err.notFound)) { - return cb(null, { - unconfirmedInput: 1 - }); + err=null; + ret= { unconfirmedInput: 1 }; } - - var a = val.split(':'); - var ret = { - addr: a[0], - valueSat: parseInt(a[1]), - }; - - // spent? - var k = SPENT_PREFIX + txid + '-' + n + '-'; - db.createReadStream({ - start: k, - end: k + '~' - }) - .on('data', function(data) { - var k = data.key.split('-'); - self._addSpentInfo(ret, k[3], k[4], data.value); - }) - .on('error', function(error) { - return cb(error); - }) - .on('end', function() { - return cb(null, ret); - }); + else { + var a = val.split(':'); + ret = { + addr: a[0], + valueSat: parseInt(a[1]), +// ts: parseInt(a[2]), // TODO + }; + } + return cb(err, ret); }); }; @@ -324,7 +307,7 @@ TransactionDb.prototype.deleteCacheForAddress = function(addr,cb) { dbScript.push({ type: 'put', key: data.key, - value: v.slice(0,2).join(':'), + value: v[0], }); }) .on('error', function(err) { @@ -362,7 +345,7 @@ TransactionDb.prototype.cacheConfirmations = function(txouts,cb) { //console.log('[TransactionDb.js.352:infoToCache:]',infoToCache); //TODO if (infoToCache.length){ - infoToCache.unshift(txout.value_sat,txout.ts); + infoToCache.unshift(txout.value_sat); //console.log('[BlockDb.js.373:txs:]' ,txout.key, infoToCache.join(':')); //TODO dbScript.push({ type: 'put', @@ -409,50 +392,51 @@ TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) { TransactionDb.prototype._parseAddrData = function(data) { var k = data.key.split('-'); var v = data.value.split(':'); -// console.log('[TransactionDb.js.410]',v); //TODO +// console.log('[TransactionDb.js.375]',data.key,data.value); //TODO var item = { key: data.key, - txid: k[2], - index: parseInt(k[3]), + ts: parseInt(k[2]), + txid: k[3], + index: parseInt(k[4]), value_sat: parseInt(v[0]), - ts: parseInt(v[1]), }; // Cache: - // v[2]== isConfirmedCached - // v[3]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs - // v[4]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey - if (v[2]){ + // v[1]== isConfirmedCached + // v[2]=== '1' -> is SpendCached -> [4]=spendTxId [5]=spentIndex [6]=spendTs + // v[3]!== '1' -> is ScriptPubkey -> [[3] = scriptPubkey + if (v[1]){ item.isConfirmed = 1; item.isConfirmedCached = 1; // console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO // Sent, confirmed - if (v[3] === '1'){ + if (v[2] === '1'){ // console.log('[TransactionDb.js.356] CACHE HIT SPENT:', item.key); //TODO item.spentIsConfirmed = 1; item.spentIsConfirmedCached = 1; - item.spentTxId = v[4]; - item.spentIndex = parseInt(v[5]); - item.spentTs = parseInt(v[6]); + item.spentTxId = v[3]; + item.spentIndex = parseInt(v[4]); + item.spentTs = parseInt(v[5]); } // Scriptpubkey cached - else if (v[3]) { + else if (v[2]) { // console.log('[TransactionDb.js.356] CACHE HIT SCRIPTPUBKEY:', item.key); //TODO - item.scriptPubKey = v[3]; + item.scriptPubKey = v[2]; item.scriptPubKeyCached = 1; } } return item; }; -TransactionDb.prototype.fromAddr = function(addr, cb) { +TransactionDb.prototype.fromAddr = function(addr, cb, txLimit) { var self = this; var k = ADDR_PREFIX + addr + '-'; var ret = []; db.createReadStream({ start: k, - end: k + '~' + end: k + '~', + limit: txLimit>0 ? txLimit: -1, // -1 means not limit }) .on('data', function(data) { ret.push(self._parseAddrData(data)); @@ -597,14 +581,15 @@ TransactionDb.prototype._addScript = function(tx, relatedAddrs) { if (relatedAddrs) relatedAddrs[addr]=1; var k = OUTS_PREFIX + txid + '-' + o.n; + var tsr = END_OF_WORLD_TS - ts; dbScript.push({ type: 'put', key: k, value: addr + ':' + sat, },{ type: 'put', - key: ADDR_PREFIX + addr + '-' + txid + '-' + o.n, - value: sat + ':' + ts, + key: ADDR_PREFIX + addr + '-' + tsr + '-'+ txid + '-' + o.n, + value: sat, }); } } @@ -673,12 +658,55 @@ TransactionDb.prototype.getPoolInfo = function(txid, cb) { TransactionDb.prototype.checkVersion02 = function(cb) { - var k = 'txb-f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba-00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; - db.get(k, function(err, val) { + var k = 'txa-'; + var isV2=1; + db.createReadStream({ + start: k, + end: k + '~', + limit: 1, + }) + .on('data', function(data) { + isV2=0; + }) + .on('end', function (){ + return cb(isV2); + }); +}; - return cb(!val); - }); +TransactionDb.prototype.migrateV02 = function(cb) { + var k = 'txa-'; + var dbScript = []; + var c=0; + var c2=0; + var N=50000; + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + dbScript.push({ + type: 'put', + key: ADDR_PREFIX + k[1] + '-' + (END_OF_WORLD_TS - parseInt(v[1])) + '-' + k[3] + '-' + [4], + value: v[0], + }); + if (c++>N) { + console.log('\t%dM txs processed', ((c2+=N)/1e6).toFixed(3)); //TODO + db.batch(dbScript,function () { + c=0; + dbScript=[]; + }); + } + }) + .on('error', function(err) { + return cb(err); + }) + .on('end', function (){ + return cb(); + }); }; + module.exports = require('soop')(TransactionDb); diff --git a/test/integration/addr.js b/test/integration/addr.js index 8e7ee396..70d80fd5 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -19,7 +19,14 @@ describe('Address balances', function() { before(function(c) { txDb = TransactionDb; - return c(); + + var l =addrValid.length; + var i =0; + addrValid.forEach(function(v) { + TransactionDb.deleteCacheForAddress(v.addr, function() { + if (++i===l) return c(); + }); + }); }); addrValid.forEach(function(v) { @@ -47,7 +54,7 @@ describe('Address balances', function() { if (v.transactions) { v.transactions.forEach(function(tx) { - assert(a.transactions.indexOf(tx) > -1, 'have tx ' + tx); + a.transactions.should.include(tx); }); } done(); @@ -59,7 +66,7 @@ describe('Address balances', function() { a.update(function(err) { if (err) done(err); v.addr.should.equal(a.addrStr); - a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); + a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat'); if (v.txApperances) a.txApperances.should.equal(v.txApperances, 'txApperances'); @@ -68,7 +75,7 @@ describe('Address balances', function() { if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); done(); - },{noTxList:1}); + },{txLimit:0}); }); } }); diff --git a/util/updateToV0.2.js b/util/upgradeV0.2js similarity index 81% rename from util/updateToV0.2.js rename to util/upgradeV0.2js index 2e79ff39..abdf99d8 100755 --- a/util/updateToV0.2.js +++ b/util/upgradeV0.2js @@ -22,6 +22,10 @@ async.series([ return c(err); }); }, + function(c){ + console.log('[1/3] Migrating txs ... (this will take some minutes...)'); //TODO + txDb.migrateV02(c); + }, function(c){ var script=[]; async.whilst( @@ -36,7 +40,7 @@ async.series([ hash = val; if (hash) height++; if (!(height%1000) || !hash) { - console.log('*update 1/2\t%d blocks processed', height); + console.log('[2/3] migrating blocks \t%d blocks processed', height); bDb._runScript(script, function(err) { script=[]; return w_cb(err); @@ -47,7 +51,7 @@ async.series([ }, c); }, function(c){ - console.log('Migrating txs... (this will take some minutes...)'); //TODO + console.log('[3/3] Migrating txs... (this will take some minutes...)'); //TODO bDb.migrateV02(c); }, function(c){