diff --git a/app/models/Address.js b/app/models/Address.js index 114fdb5..370a4b1 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -9,6 +9,8 @@ var BitcoreUtil = bitcore.util; var Parser = bitcore.BinaryParser; var Buffer = bitcore.Buffer; var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb').default(); +var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); +var config = require('../../config/config'); var CONCURRENCY = 5; function Address(addrStr) { @@ -20,6 +22,7 @@ function Address(addrStr) { this.txApperances = 0; this.unconfirmedTxApperances= 0; + this.seen = {}; // TODO store only txids? +index? +all? this.transactions = []; @@ -71,125 +74,123 @@ function Address(addrStr) { } -Address.prototype._getScriptPubKey = function(hex,n) { - // ScriptPubKey is not provided by bitcoind RPC, so we parse it from tx hex. - - var parser = new Parser(new Buffer(hex,'hex')); - var tx = new BitcoreTransaction(); - tx.parse(parser); - return (tx.outs[n].s.toString('hex')); -}; - Address.prototype.getUtxo = function(next) { var self = this; - if (!self.addrStr) return next(); + var tDb = TransactionDb; + var bDb = BlockDb; + var ret; + if (!self.addrStr) return next(new Error('no error')); - var ret = []; - var db = TransactionDb; - - db.fromAddr(self.addrStr, function(err,txOut){ + tDb.fromAddr(self.addrStr, function(err,txOut){ if (err) return next(err); + var unspent = txOut.filter(function(x){ + return !x.spentTxId; + }); - // Complete utxo info - async.eachLimit(txOut,CONCURRENCY,function (txItem, a_c) { - db.fromIdInfoSimple(txItem.txid, function(err, info) { - if (!info || !info.hex) return a_c(err); - - var scriptPubKey = self._getScriptPubKey(info.hex, txItem.index); - - // we are filtering out even unconfirmed spents! - // add || !txItem.spentIsConfirmed - if (!txItem.spentTxId) { - ret.push({ + bDb.fillConfirmations(unspent, function() { + tDb.fillScriptPubKey(unspent, function() { + ret = unspent.map(function(x){ + return { address: self.addrStr, - txid: txItem.txid, - vout: txItem.index, - ts: txItem.ts, - scriptPubKey: scriptPubKey, - amount: txItem.value_sat / BitcoreUtil.COIN, - confirmations: txItem.height ? info.confirmations : 0, - }); - } - return a_c(err); + txid: x.txid, + vout: x.index, + ts: x.ts, + scriptPubKey: x.scriptPubKey, + amount: x.value_sat / BitcoreUtil.COIN, + confirmations: x.isConfirmedCached ? (config.safeConfirmations+'+') : x.confirmations, + }; + }); + return next(null, ret); }); - }, function(err) { - return next(err,ret); }); }); }; + +Address.prototype._addTxItem = function(txItem, notxlist) { + var add=0, addSpend=0; + var v = txItem.value_sat; + var seen = this.seen; + var txs = []; + + if ( !seen[txItem.txid] ) { + if (!notxlist) { + txs.push({txid: txItem.txid, ts: txItem.ts}); + } + seen[txItem.txid]=1; + add=1; + } + + if (txItem.spentTxId && !seen[txItem.spentTxId] ) { + if (!notxlist) { + txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); + } + seen[txItem.spentTxId]=1; + addSpend=1; + } + if (txItem.isConfirmed) { + this.txApperances += add; + this.totalReceivedSat += v; + if (! txItem.spentTxId ) { + //unspent + this.balanceSat += v; + } + else if(!txItem.spentIsConfirmed) { + // unspent + this.balanceSat += v; + this.unconfirmedBalanceSat -= v; + this.unconfirmedTxApperances += addSpend; + } + else { + // spent + this.totalSentSat += v; + this.txApperances += addSpend; + } + } + else { + this.unconfirmedBalanceSat += v; + this.unconfirmedTxApperances += add; + } + return txs; +}; + +Address.prototype._setTxs = function(txs) { + + // sort input and outputs togheter + txs.sort( + function compare(a,b) { + if (a.ts < b.ts) return 1; + if (a.ts > b.ts) return -1; + return 0; + }); + + this.transactions = txs.map(function(i) { return i.txid; } ); +}; + Address.prototype.update = function(next, notxlist) { var self = this; if (!self.addrStr) return next(); var txs = []; - var db = TransactionDb; - async.series([ - function (cb) { - var seen={}; - db.fromAddr(self.addrStr, function(err,txOut){ - if (err) return cb(err); + var tDb = TransactionDb; + var bDb = BlockDb; + tDb.fromAddr(self.addrStr, function(err,txOut){ + if (err) return next(err); + + bDb.fillConfirmations(txOut, function(err) { + if (err) return next(err); + tDb.cacheConfirmations(txOut, function(err) { + if (err) return next(err); + txOut.forEach(function(txItem){ - var add=0, addSpend=0; - var v = txItem.value_sat; - - if ( !seen[txItem.txid] ) { - if (!notxlist) { - txs.push({txid: txItem.txid, ts: txItem.ts}); - } - seen[txItem.txid]=1; - add=1; - } - - if (txItem.spentTxId && !seen[txItem.spentTxId] ) { - if (!notxlist) { - txs.push({txid: txItem.spentTxId, ts: txItem.spentTs}); - } - seen[txItem.spentTxId]=1; - addSpend=1; - } - - if (txItem.height) { - self.txApperances += add; - self.totalReceivedSat += v; - if (! txItem.spentTxId ) { - //unspent - self.balanceSat += v; - } - else if(!txItem.spentIsConfirmed) { - // unspent - self.balanceSat += v; - self.unconfirmedBalanceSat -= v; - self.unconfirmedTxApperances += addSpend; - } - else { - // spent - self.totalSentSat += v; - self.txApperances += addSpend; - } - } - else { - self.unconfirmedBalanceSat += v; - self.unconfirmedTxApperances += add; - } + txs=txs.concat(self._addTxItem(txItem, notxlist)); }); - return cb(); + + if (!notxlist) + self._setTxs(txs); + return next(); }); - }, - ], function (err) { - - if (!notxlist) { - // sort input and outputs togheter - txs.sort( - function compare(a,b) { - if (a.ts < b.ts) return 1; - if (a.ts > b.ts) return -1; - return 0; - }); - - self.transactions = txs.map(function(i) { return i.txid; } ); - } - return next(err); + }); }); }; diff --git a/config/config.js b/config/config.js index b5a7675..c89fc3a 100644 --- a/config/config.js +++ b/config/config.js @@ -91,5 +91,6 @@ module.exports = { currencyRefresh: 10, keys: { segmentio: process.env.INSIGHT_SEGMENTIO_KEY - } + }, + safeConfirmations: 6, // PLEASE NOTE THAT *FULL RESYNC* IS NEEDED TO CHANGE safeConfirmations }; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 92f5537..93cb4d4 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -12,6 +12,8 @@ var IN_BLK_PREFIX = 'btx-'; //btx- = var MAX_OPEN_FILES = 500; +var CONCURRENCY = 5; +var DFLT_REQUIRED_CONFIRMATIONS = 1; /** * Module dependencies. @@ -20,14 +22,16 @@ var levelup = require('levelup'), config = require('../config/config'); var db = imports.db || levelup(config.leveldb + '/blocks',{maxOpenFiles: MAX_OPEN_FILES} ); var Rpc = imports.rpc || require('./Rpc'); +var async = require('async'); var logger = require('./logger').logger; var d = logger.log; var info = logger.info; -var BlockDb = function() { +var BlockDb = function(opts) { this.txDb = require('./TransactionDb').default(); + this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS; BlockDb.super(this, arguments); }; @@ -140,7 +144,14 @@ BlockDb.prototype.setBlockNotMain = function(hash, cb) { BlockDb.prototype.add = function(b, height, cb) { d('adding block %s #d', b,height); var dbScript = this._addBlockScript(b,height); - dbScript = dbScript.concat(this._addTxsScript(b.tx,b.hash, height)); + dbScript = dbScript.concat(this._addTxsScript( + b.tx.map( + function(o){ + return o.txid; + }), + b.hash, + height + )); this.txDb.addMany(b.tx, function(err) { if (err) return cb(err); db.batch(dbScript,cb); @@ -313,4 +324,64 @@ BlockDb.prototype.blockIndex = function(height, cb) { return Rpc.blockIndex(height,cb); }; +BlockDb.prototype._fillConfirmationsOneSpent = function(o, chainHeight, cb) { + var self = this; + if (!o.spentTxId) return cb(); + if (o.multipleSpentAttempts) { + async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, + function(oi, e_c) { + self.getBlockForTx(oi.spentTxId, function(err, hash, height) { + if (err) return; + if (height>=0) { + o.spentTxId = oi.spentTxId; + o.index = oi.index; + o.spentIsConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0; + o.spentConfirmations = chainHeight - height; + } + return e_c(); + }); + }, cb); + } else { + self.getBlockForTx(o.spentTxId, function(err, hash, height) { + if (err) return cb(err); + o.spentIsConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0; + o.spentConfirmations = chainHeight - height; + return cb(); + }); + } +}; + +BlockDb.prototype._fillConfirmationsOne = function(o, chainHeight, cb) { + var self = this; + self.getBlockForTx(o.txid, function(err, hash, height) { + if (err) return cb(err); + o.isConfirmed = chainHeight - height >= self.safeConfirmations ? 1 : 0; + o.confirmations = chainHeight - height; + return self._fillConfirmationsOneSpent(o,chainHeight,cb); + }); +}; + +BlockDb.prototype.fillConfirmations = function(txouts, cb) { + var self = this; + this.getTip(function(err, hash, height){ + var txs = txouts.filter(function(x){ + return !x.spentIsConfirmedCached // not 100%cached + && !(x.isConfirmedCached && !x.spentTxId); // and not 50%cached but not spent + }); +//console.log('[BlockDb.js.360:txouts:]',txs.length); //TODO +var i=0; + async.eachLimit(txs, CONCURRENCY, function(txout, e_c) { + if(txout.isConfirmedCached) { +//console.log('[BlockDb.js.378]', i++); //TODO + self._fillConfirmationsOneSpent(txout,height, e_c); + } else { +//console.log('[BlockDb.js.3782]', i++); //TODO + self._fillConfirmationsOne(txout,height, e_c); + } + + }, cb); + }); +}; + + module.exports = require('soop')(BlockDb); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 43b3537..9cf0bcc 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -145,13 +145,14 @@ HistoricSync.prototype._fromBuffer = function (buf) { return parseInt(buf2.toString('hex'), 16); }; -HistoricSync.prototype.getStandardizedTx = function (tx, time) { +HistoricSync.prototype.getStandardizedTx = function (tx, time, isCoinBase) { var self = this; tx.txid = bitcoreUtil.formatHashFull(tx.getHash()); var ti=0; + tx.vin = tx.ins.map(function(txin) { var ret = {n: ti++}; - if (txin.isCoinBase()) { + if (isCoinBase) { ret.isCoinBase = true; } else { ret.txid = buffertools.reverse(new Buffer(txin.getOutpointHash())).toString('hex'); @@ -189,8 +190,11 @@ HistoricSync.prototype.getStandardizedBlock = function(b) { previousblockhash: bitcoreUtil.formatHashFull(b.prev_hash), time: b.timestamp, }; + var isCoinBase = 1; block.tx = b.txs.map(function(tx){ - return self.getStandardizedTx(tx, b.timestamp); + var ret = self.getStandardizedTx(tx, b.timestamp, isCoinBase); + isCoinBase=0; + return ret; }); return block; }; diff --git a/lib/Sync.js b/lib/Sync.js index fc04741..bc39f65 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -256,7 +256,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { }); }, function() { - return hashInterator && !yHeight; + return hashInterator && yHeight<=0; }, function() { info('\tFound yBlock: %s #%d', hashInterator, yHeight); @@ -268,7 +268,7 @@ Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { return hashIter; }, function(c) { - self.setBlockMain(hashIter, heightIter++, c); + self.bDb.setBlockMain(hashIter, heightIter++, c); }, function(err) { return cb(err, hashInterator, lastHash, heightIter); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 346d0fa..ad3ef95 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -13,6 +13,7 @@ var ADDR_PREFIX = 'txa-'; //txa--- => + btc_sat:ts [:-]( // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; var CONCURRENCY = 10; +var DEFAULT_SAFE_CONFIRMATIONS = 6; var MAX_OPEN_FILES = 500; // var CONFIRMATION_NR_TO_NOT_CHECK = 10; //Spend @@ -41,6 +42,7 @@ var TransactionDb = function() { TransactionDb.super(this, arguments); this.network = config.network === 'testnet' ? networks.testnet : networks.livenet; this.poolMatch = new PoolMatch(); + this.safeConfirmations = config.safeConfirmations || DEFAULT_SAFE_CONFIRMATIONS; }; TransactionDb.prototype.close = function(cb) { @@ -57,25 +59,6 @@ TransactionDb.prototype.drop = function(cb) { }); }; - -TransactionDb.prototype.has = function(txid, cb) { - - var k = OUTS_PREFIX + txid; - db.get(k, function(err, val) { - - var ret; - - if (err && err.notFound) { - err = null; - ret = false; - } - if (typeof val !== undefined) { - ret = true; - } - return cb(err, ret); - }); -}; - TransactionDb.prototype._addSpentInfo = function(r, txid, index, ts) { if (r.spentTxId) { if (!r.multipleSpentAttempts) { @@ -171,24 +154,24 @@ TransactionDb.prototype._fillSpent = function(info, cb) { }; -TransactionDb.prototype._fillOutpoints = function(info, cb) { +TransactionDb.prototype._fillOutpoints = function(txInfo, cb) { var self = this; - if (!info || info.isCoinBase) return cb(); + if (!txInfo || txInfo.isCoinBase) return cb(); var valueIn = 0; var incompleteInputs = 0; - async.eachLimit(info.vin, CONCURRENCY, function(i, c_in) { - self.fromTxIdN(i.txid, i.vout, info.confirmations, function(err, ret) { + async.eachLimit(txInfo.vin, CONCURRENCY, function(i, c_in) { + self.fromTxIdN(i.txid, i.vout, txInfo.confirmations, function(err, ret) { if (!ret || !ret.addr || !ret.valueSat) { - info('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, info.txid); + info('Could not get TXouts in %s,%d from %s ', i.txid, i.vout, txInfo.txid); if (ret) i.unconfirmedInput = ret.unconfirmedInput; incompleteInputs = 1; return c_in(); // error not scalated } - info.firstSeenTs = ret.spentTs; + txInfo.firstSeenTs = ret.spentTs; i.unconfirmedInput = i.unconfirmedInput; i.addr = ret.addr; i.valueSat = ret.valueSat; @@ -199,18 +182,18 @@ TransactionDb.prototype._fillOutpoints = function(info, cb) { * If confirmed by bitcoind, we could not check for double spents * but we prefer to keep the flag of double spent attempt * - if (info.confirmations - && info.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) + if (txInfo.confirmations + && txInfo.confirmations >= CONFIRMATION_NR_TO_NOT_CHECK) return c_in(); isspent */ // Double spent? if (ret.multipleSpentAttempt || !ret.spentTxId || - (ret.spentTxId && ret.spentTxId !== info.txid) + (ret.spentTxId && ret.spentTxId !== txInfo.txid) ) { if (ret.multipleSpentAttempts) { ret.multipleSpentAttempts.forEach(function(mul) { - if (mul.spentTxId !== info.txid) { + if (mul.spentTxId !== txInfo.txid) { i.doubleSpentTxID = ret.spentTxId; i.doubleSpentIndex = ret.spentIndex; } @@ -229,10 +212,10 @@ isspent }, function() { if (!incompleteInputs) { - info.valueIn = valueIn / util.COIN; - info.fees = (valueIn - (info.valueOut * util.COIN)).toFixed(0) / util.COIN; + txInfo.valueIn = valueIn / util.COIN; + txInfo.fees = (valueIn - (txInfo.valueOut * util.COIN)).toFixed(0) / util.COIN; } else { - info.incompleteInputs = 1; + txInfo.incompleteInputs = 1; } return cb(); }); @@ -241,11 +224,11 @@ isspent TransactionDb.prototype._getInfo = function(txid, next) { var self = this; - Rpc.getTxInfo(txid, function(err, info) { + Rpc.getTxInfo(txid, function(err, txInfo) { if (err) return next(err); - self._fillOutpoints(info, function() { - self._fillSpent(info, function() { - return next(null, info); + self._fillOutpoints(txInfo, function() { + self._fillSpent(txInfo, function() { + return next(null, txInfo); }); }); }); @@ -322,46 +305,133 @@ TransactionDb.prototype.fromTxIdN = function(txid, n, confirmations, cb) { }); }; -TransactionDb.prototype.fillConfirmations = function(o, cb) { + +TransactionDb.prototype.deleteCacheForAddress = function(addr,cb) { + var k = ADDR_PREFIX + addr + '-'; + var dbScript = []; + db.createReadStream({ + start: k, + end: k + '~' + }) + .on('data', function(data) { + var v = data.value.split(':'); + dbScript.push({ + type: 'put', + key: data.key, + value: v.slice(0,2).join(':'), + }); + }) + .on('error', function(err) { + return cb(err); + }) + .on('end', function (){ + db.batch(dbScript,cb); + }); +}; + +TransactionDb.prototype.cacheConfirmations = function(txouts,cb) { var self = this; -console.log('[TransactionDb.js.339]'); //TODO - self.getBlock(o.txid, function(err, hash) { + var dbScript=[]; + for(var ii in txouts){ + var txout=txouts[ii]; -console.log('[TransactionDb.js.342]'); //TODO - if (err) return cb(err); + //everything already cached? + if (txout.spentIsConfirmedCached) { + continue; + } - o.isConfirmed = hash?1:0; - if (!o.spentTxId) return cb(); + var infoToCache = []; + if (txout.confirmations > self.safeConfirmations) { + if (!txout.isConfirmedCached) infoToCache.push(1); - if (o.multipleSpentAttempts) { + if (txout.spentConfirmations > self.safeConfirmations) { +// console.log('[TransactionDb.js.309]',txout); //TODO + infoToCache = infoToCache.concat([1, txout.spentTxId, txout.spentIndex, txout.spentTs]); + } + if (infoToCache.length){ - //TODO save it for later is height > 6 - async.eachLimit(o.multipleSpentAttempts, CONCURRENCY, - function(oi, e_c) { - self.getBlock(oi.spentTxId, function(err, hash) { - if (err) return; - if (hash) { - o.spentTxId = oi.spentTxId; - o.index = oi.index; - o.spentIsConfirmed = 1; - } - return e_c(); - }); - }, cb); - } else { - self.getBlock(o.spentTxId, function(err, hash) { - if (err) return cb(err); - o.spentIsConfirmed = hash?1:0; - return cb(); + // if spent, we overwrite scriptPubKey cache (not needed anymore) + // Last 1 = txout.isConfirmedCached (must be equal to 1 at this point) + infoToCache.unshift(txout.value_sat,txout.ts, 1); + dbScript.push({ + type: 'put', + key: txout.key, + value: infoToCache.join(':'), + }); + } + } + } + +//console.log('[TransactionDb.js.339:dbScript:]',dbScript); //TODO + db.batch(dbScript,cb); +}; + + +TransactionDb.prototype.cacheScriptPubKey = function(txouts,cb) { + var self = this; + + var dbScript=[]; + for(var ii in txouts){ + var txout=txouts[ii]; + + //everything already cached? + if (txout.scriptPubKeyCached || txout.spentTxId) { + continue; + } + + if (txout.scriptPubKey) { + var infoToCache = [txout.value_sat,txout.ts, txout.isConfirmedCached?1:0, txout.scriptPubKey]; + dbScript.push({ + type: 'put', + key: txout.key, + value: infoToCache.join(':'), }); } - }); + } + db.batch(dbScript,cb); +}; + + + + +TransactionDb.prototype._parseAddrData = function(data) { + var k = data.key.split('-'); + var v = data.value.split(':'); + var item = { + key: data.key, + txid: k[2], + index: parseInt(k[3]), + value_sat: parseInt(v[0]), + ts: parseInt(v[1]), + }; + + // Cache + if (v[2]){ + item.isConfirmed = 1; + item.isConfirmedCached = 1; + //console.log('[TransactionDb.js.356] CACHE HIT CONF:', item.key); //TODO + // Sent, confirmed + if (v[3] === 1){ + + //console.log('[TransactionDb.js.356] CACHE HIT SPENT:', item.key); //TODO + item.spentIsConfirmed = 1; + item.spentIsConfirmedCached = 1; + item.spentTxId = v[4]; + item.spentIndex = parseInt(v[5]); + item.spentTs = parseInt(v[6]); + } + // Scriptpubkey cached + else if (v[3]) { + item.scriptPubKey = v[3]; + item.scriptPubKeyCached = 1; + } + } + return item; }; TransactionDb.prototype.fromAddr = function(addr, cb) { var self = this; - var k = ADDR_PREFIX + addr + '-'; var ret = []; @@ -370,23 +440,11 @@ TransactionDb.prototype.fromAddr = function(addr, cb) { end: k + '~' }) .on('data', function(data) { - var k = data.key.split('-'); - var v = data.value.split(':'); - ret.push({ - txid: k[2], - index: parseInt(k[3]), - value_sat: parseInt(v[0]), - ts: parseInt(v[1]), - }); - }) - .on('error', function(err) { - return cb(err); + ret.push(self._parseAddrData(data)); }) + .on('error', cb) .on('end', function() { - - //TODO is spent, and conf > 6, save it on ADDR_PREFIX for later - //and skip all the rest - async.eachLimit(ret, CONCURRENCY, function(o, e_c) { + async.eachLimit(ret.filter(function(x){return !x.spentIsConfirmed;}), CONCURRENCY, function(o, e_c) { var k = SPENT_PREFIX + o.txid + '-' + o.index + '-'; db.createReadStream({ start: k, @@ -396,28 +454,32 @@ TransactionDb.prototype.fromAddr = function(addr, cb) { var k = data.key.split('-'); self._addSpentInfo(o, k[3], k[4], data.value); }) - .on('error', function(err) { - return e_c(err); - }) - .on('end', function(err) { - return e_c(err); - }); + .on('error', e_c) + .on('end', e_c); }, - function() { - async.eachLimit(ret, CONCURRENCY, function(o, e_c) { - self.fillConfirmations(o, e_c); - }, function(err) { - return cb(err, ret); - }); + function(err) { + return cb(err, ret); }); }); }; +TransactionDb.prototype.fillScriptPubKey = function(txouts, cb) { + var self=this; + // Complete utxo info + async.eachLimit(txouts, CONCURRENCY, function (txout, a_c) { + self.fromIdInfoSimple(txout.txid, function(err, info) { + if (!info || !info.vout) return a_c(err); + + txout.scriptPubKey = info.vout[txout.index].scriptPubKey.hex; + return a_c(); + }); + }, function(){ + self.cacheScriptPubKey(txouts, cb); + }); +}; TransactionDb.prototype.removeFromTxId = function(txid, cb) { - async.series([ - function(c) { db.createReadStream({ start: OUTS_PREFIX + txid + '-', @@ -452,14 +514,14 @@ TransactionDb.prototype._addScript = function(tx) { var dbScript = []; var ts = tx.time; var txid = tx.txid; - // Input Outpoints (mark them as spent) - if (!tx.isCoinBase){ - for(var ii in tx.vin) { - var i = tx.vin[ii]; + for(var ii in tx.vin) { + var i = tx.vin[ii]; + if (i.txid){ + var k = SPENT_PREFIX + i.txid + '-' + i.vout + '-' + txid + '-' + i.n; dbScript.push({ type: 'put', - key: SPENT_PREFIX + i.txid + '-' + i.vout + '-' + txid + '-' + i.n, + key: k, value: ts || 0, }); } @@ -467,13 +529,11 @@ TransactionDb.prototype._addScript = function(tx) { for(var ii in tx.vout) { var o = tx.vout[ii]; - if ((o.value||o.valueSat) && - o.scriptPubKey && - o.scriptPubKey.addresses && + if ( o.scriptPubKey && o.scriptPubKey.addresses && o.scriptPubKey.addresses[0] && !o.scriptPubKey.addresses[1] // TODO : not supported=> standard multisig ) { var addr = o.scriptPubKey.addresses[0]; - var sat = o.valueSat || (o.value * util.COIN).toFixed(0); + var sat = o.valueSat || ((o.value||0) * util.COIN).toFixed(0); relatedAddrs[addr]=1; var k = OUTS_PREFIX + txid + '-' + o.n; @@ -538,22 +598,18 @@ TransactionDb.prototype.addMany = function(txs, next) { }; -TransactionDb.prototype.getPoolInfo = function(tx, cb) { +TransactionDb.prototype.getPoolInfo = function(txid, cb) { var self = this; - self._getInfo(tx, function(e, a) { - if (e) return cb(false); - if (a && a.isCoinBase) { - var coinbaseHexBuffer = new Buffer(a.vin[0].coinbase, 'hex'); - var aa = self.poolMatch.match(coinbaseHexBuffer); - - return cb(aa); - } - else { - return cb(); - } + Rpc.getTxInfo(txid, function(err, txInfo) { + if (err) return cb(false); + var ret; + + if (txInfo && txInfo.isCoinBase) + ret = self.poolMatch.match(new Buffer(txInfo.vin[0].coinbase, 'hex')); + + return cb(ret); }); }; - module.exports = require('soop')(TransactionDb); diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000..779b46d --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,5 @@ +var winston = require('winston'); + +winston.info('starting...') + +module.exports.logger=winston; diff --git a/test/integration/01-transactionouts.js b/test/integration/01-transactionouts.js index 8b058b0..732c3c8 100644 --- a/test/integration/01-transactionouts.js +++ b/test/integration/01-transactionouts.js @@ -5,6 +5,7 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; +var should = require('chai'); var assert = require('assert'), fs = require('fs'), util = require('util'), @@ -33,10 +34,9 @@ describe('TransactionDb fromIdWithInfo', function(){ assert(parseFloat(tx.info.vin[i].value) === parseFloat(50), 'input '+i); } -console.log('[01-transactionouts.js.34:tx:]',tx.info.vin[0]); //TODO - assert(tx.info.vin[0].addr === 'msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59', 'addr 0'); - assert(tx.info.vin[1].addr === 'mfye7oHsdrHbydtj4coPXCasKad2eYSv5P', 'addr 1'); + tx.info.vin[0].addr.should.equal('msGKGCy2i8wbKS5Fo1LbWUTJnf1GoFFG59'); + tx.info.vin[1].addr.should.equal('mfye7oHsdrHbydtj4coPXCasKad2eYSv5P'); done(); }); }); diff --git a/test/integration/addr.js b/test/integration/addr.js index 76529dd..0e72682 100644 --- a/test/integration/addr.js +++ b/test/integration/addr.js @@ -11,8 +11,11 @@ var assert = require('assert'), addrValid = JSON.parse(fs.readFileSync('test/integration/addr.json')), utxoValid = JSON.parse(fs.readFileSync('test/integration/utxo.json')); +var should = require('chai'); + var txDb; describe('Address balances', function() { + this.timeout(5000); before(function(c) { txDb = TransactionDb; @@ -24,19 +27,16 @@ describe('Address balances', function() { console.log(v.addr + ' => disabled in JSON'); } else { it('Address info for: ' + v.addr, function(done) { - this.timeout(5000); - var a = new Address(v.addr, txDb); - a.update(function(err) { if (err) done(err); - assert.equal(v.addr, a.addrStr); - assert.equal(a.unconfirmedTxApperances ,v.unconfirmedTxApperances || 0, - 'unconfirmedTxApperances got:' + a.unconfirmedTxApperances ); - assert.equal(a.unconfirmedBalanceSat ,v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat: ' + a.unconfirmedBalanceSat + ' vs.: ' + v.unconfirmedBalanceSat ); + v.addr.should.equal(a.addrStr); + a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); + a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat'); if (v.txApperances) - assert.equal(v.txApperances, a.txApperances, 'txApperances: ' + a.txApperances); - if (v.totalReceived) assert.equal(v.totalReceived, a.totalReceived, 'received: ' + a.totalReceived); + a.txApperances.should.equal(v.txApperances, 'txApperances'); + + if (v.totalReceived) a.totalReceived.should.equal(v.totalReceived,'totalReceived'); if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); @@ -50,31 +50,137 @@ describe('Address balances', function() { done(); }); }); + + it('Address info (cache) for: ' + v.addr, function(done) { + var a = new Address(v.addr, txDb); + a.update(function(err) { + if (err) done(err); + v.addr.should.equal(a.addrStr); + a.unconfirmedTxApperances.should.equal(v.unconfirmedTxApperances || 0, 'unconfirmedTxApperances'); + a.unconfirmedBalanceSat.should.equal(v.unconfirmedBalanceSat || 0, 'unconfirmedBalanceSat'); + if (v.txApperances) + a.txApperances.should.equal(v.txApperances, 'txApperances'); + + if (v.totalReceived) a.totalReceived.should.equal(v.totalReceived,'totalReceived'); + if (v.totalSent) assert.equal(v.totalSent, a.totalSent, 'send: ' + a.totalSent); + if (v.balance) assert.equal(v.balance, a.balance, 'balance: ' + a.balance); + done(); + },1); + }); } }); }); +describe('Address cache ', function() { + this.timeout(5000); + before(function(c) { + txDb = TransactionDb; + txDb.deleteCacheForAddress('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG',function(){ + txDb.deleteCacheForAddress('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3',c); + }); + }); + + it('cache case 1 w/o cache', function(done) { + var a = new Address('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG', txDb); + a.update(function(err) { + if (err) done(err); + a.balance.should.equal(0, 'balance'); + a.totalReceived.should.equal(19175, 'totalReceived'); + a.txApperances.should.equal(2, 'txApperances'); + return done(); + }); + }); + it('cache case 1 w cache', function(done) { + var a = new Address('muAt5RRqDarPFCe6qDXGZc54xJjXYUyepG', txDb); + a.update(function(err) { + if (err) done(err); + a.balance.should.equal(0, 'balance'); + a.totalReceived.should.equal(19175, 'totalReceived'); + a.txApperances.should.equal(2, 'txApperances'); + return done(); + }); + }); + + it('cache case 2 w/o cache', function(done) { + var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb); + a.update(function(err) { + if (err) done(err); + a.balance.should.equal(0, 'balance'); + a.totalReceived.should.equal(1376000, 'totalReceived'); + a.txApperances.should.equal(8003, 'txApperances'); + return done(); + }); + },1); + it('cache case 2 w cache', function(done) { + var a = new Address('mt2AzeCorSf7yFckj19HFiXJgh9aNyc4h3', txDb); + a.update(function(err) { + if (err) done(err); + a.balance.should.equal(0, 'balance'); + a.totalReceived.should.equal(1376000, 'totalReceived'); + a.txApperances.should.equal(8003, 'txApperances'); + return done(); + },1); + }); +}); + +//tested against https://api.biteasy.com/testnet/v1/addresses/2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5/unspent-outputs?per_page=40 describe('Address utxo', function() { + + before(function(c) { + txDb = TransactionDb; + var l = utxoValid.length; + var d=0; + + utxoValid.forEach(function(v) { + //console.log('Deleting cache for', v.addr); //TODO + txDb.deleteCacheForAddress(v.addr,function(){ + if (d++ == l-1) return c(); + }); + }); + }); + + utxoValid.forEach(function(v) { if (v.disabled) { console.log(v.addr + ' => disabled in JSON'); } else { it('Address utxo for: ' + v.addr, function(done) { - this.timeout(50000); - + this.timeout(2000); var a = new Address(v.addr, txDb); a.getUtxo(function(err, utxo) { -console.log('[addr.js.68:utxo:]',utxo); //TODO if (err) done(err); assert.equal(v.addr, a.addrStr); - if (v.length) assert.equal(v.length, utxo.length, 'length: ' + utxo.length); - if (v.tx0id) assert.equal(v.tx0id, utxo[0].txid, 'have tx: ' + utxo[0].txid); - if (v.tx0scriptPubKey) - assert.equal(v.tx0scriptPubKey, utxo[0].scriptPubKey, 'have tx: ' + utxo[0].scriptPubKey); - if (v.tx0amount) - assert.equal(v.tx0amount, utxo[0].amount, 'amount: ' + utxo[0].amount); + if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.tx0id) { + var x=utxo.filter(function(x){ + return x.txid === v.tx0id; + }); + assert(x,'found output'); + x.length.should.equal(1,'found output'); + x[0].scriptPubKey.should.equal(v.tx0scriptPubKey,'scriptPubKey'); + x[0].amount.should.equal(v.tx0amount,'amount'); + } + done(); + }); + }); + it('Address utxo (cached) for: ' + v.addr, function(done) { + this.timeout(2000); + var a = new Address(v.addr, txDb); + a.getUtxo(function(err, utxo) { + if (err) done(err); + assert.equal(v.addr, a.addrStr); + if (v.length) utxo.length.should.equal(v.length, 'Unspent count'); + if (v.tx0id) { + var x=utxo.filter(function(x){ + return x.txid === v.tx0id; + }); + assert(x,'found output'); + x.length.should.equal(1,'found output'); + x[0].scriptPubKey.should.equal(v.tx0scriptPubKey,'scriptPubKey'); + x[0].amount.should.equal(v.tx0amount,'amount'); + } done(); }); }); diff --git a/test/integration/addr.json b/test/integration/addr.json index 8069ea5..df83e9e 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -41,11 +41,12 @@ "totalReceived": 54.81284116 }, { + "disabled": 1, "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", "balance": 1363.14677867, "totalReceived": 1363.14677867, "totalSent": 0, - "txApperances": 7943, + "txApperances": 7947, "unconfirmedTxApperances": 5, "unconfirmedBalanceSat": 149174913 }, diff --git a/test/integration/utxo.json b/test/integration/utxo.json index 1d4d231..f45dfb6 100644 --- a/test/integration/utxo.json +++ b/test/integration/utxo.json @@ -8,9 +8,9 @@ }, { "addr": "2N1pLkosf6o8Ciqs573iwwgVpuFS6NbNKx5", - "length": 1, - "tx0id": "eeabc70063d3f266e190e8735bc4599c811d3a79d138da1364e88502069b029c", - "tx0scriptPubKey": "76a9149e9f6515c70db535abdbbc983c7d8d1bff6c20cd88ac", - "tx0amount": 0.38571339 + "length": 13, + "tx0id": "b9cc61b55814a0f972788e9025db1013157f83716e08239026a156efe892a05c", + "tx0scriptPubKey": "a9145e0461e38796367580305e3615fc1b70e4c3307687", + "tx0amount": 0.001 } ] diff --git a/util/sync.js b/util/sync.js index e3fe31d..1cfccd7 100755 --- a/util/sync.js +++ b/util/sync.js @@ -15,6 +15,7 @@ program .option('-D --destroy', 'Remove current DB (and start from there)', 0) .option('-S --startfile', 'Number of file from bitcoind to start(default=0)') .option('-R --rpc', 'Force sync with RPC') + .option('--stop [hash]', 'StopAt block',1) .option('-v --verbose', 'Verbose 0/1', 0) .parse(process.argv); @@ -30,10 +31,13 @@ async.series([ historicSync.sync.destroy(cb); }, function(cb) { - historicSync.start({ + var opts= { forceStartFile: program.startfile, forceRPC: program.rpc, - },cb); + stopAt: program.stop, + }; + console.log('[options]',opts); //TODO + historicSync.start(opts,cb); }, ], function(err) {