From 23960f0c373757a64efe5504913990c103ed28e7 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 1 Feb 2014 13:39:29 -0300 Subject: [PATCH] files sync workgin --- app/models/Block.js | 10 ++-- app/models/TransactionOut.js | 100 +++++++++++++++++++++++++---------- lib/HistoricSync.js | 77 +++++++++++++++++++++++---- 3 files changed, 146 insertions(+), 41 deletions(-) diff --git a/app/models/Block.js b/app/models/Block.js index 93016699..8a01ca57 100644 --- a/app/models/Block.js +++ b/app/models/Block.js @@ -86,16 +86,18 @@ BlockSchema.statics.customCreate = function(block, cb) { newBlock.time = block.time ? block.time : Math.round(new Date().getTime() / 1000); newBlock.hashStr = block.hash; + newBlock.isOrphan = block.isOrphan; newBlock.nextBlockHashStr = block.nextBlockHash; var insertedTxs, updateAddrs; async.series([ function(a_cb) { - TransactionOut.createFromTxs(block.tx, function(err, inInsertedTxs, inUpdateAddrs) { - insertedTxs = inInsertedTxs; - updateAddrs = inUpdateAddrs; - return a_cb(err); + TransactionOut.createFromTxs(block.tx, block.isOrphan, + function(err, inInsertedTxs, inUpdateAddrs) { + insertedTxs = inInsertedTxs; + updateAddrs = inUpdateAddrs; + return a_cb(err); }); }, function(a_cb) { newBlock.save(function(err) { diff --git a/app/models/TransactionOut.js b/app/models/TransactionOut.js index 8247c5e2..a0fec41d 100644 --- a/app/models/TransactionOut.js +++ b/app/models/TransactionOut.js @@ -24,13 +24,16 @@ var TransactionOutSchema = new Schema({ index: true, }, value_sat: Number, + fromOrphan: Boolean, spendTxIdBuf: Buffer, spendIndex: Number, + spendFromOrphan: Boolean, }); // Compound index + TransactionOutSchema.index({txidBuf: 1, index: 1}, {unique: true, sparse: true}); TransactionOutSchema.index({spendTxIdBuf: 1, spendIndex: 1}, {unique: true, sparse: true}); @@ -91,15 +94,52 @@ TransactionOutSchema.statics.removeFromTxId = function(txid, cb) { -TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { +TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, fromOrphan, cb) { var Self = this; var addrs = []; var is_new = true; + if (txInfo.hash) { + + // adapt bitcore TX object to bitcoind JSON response + txInfo.txid = txInfo.hash; + + var count = 0; + txInfo.vin = txInfo.in.map(function (txin) { + var i = {}; + + if (txin.coinbase) { + txInfo.isCoinBase = true; + } + else { + i.txid= txin.prev_out.hash; + i.vout= txin.prev_out.n; + }; + i.n = count++; + return i; + }); + + + count = 0; + txInfo.vout = txInfo.out.map(function (txout) { + var o = {}; + + o.value = txout.value; + o.n = count++; + + if (txout.addrStr){ + o.scriptPubKey = {}; + o.scriptPubKey.addresses = [txout.addrStr]; + } + return o; + }); + + } var bTxId = new Buffer(txInfo.txid,'hex'); + async.series([ // Input Outpoints (mark them as spended) function(p_c) { @@ -114,6 +154,7 @@ TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { spendTxIdBuf: bTxId, spendIndex: i.n, }; + if (fromOrphan) data.spendFromOrphan = true; Self.update({txidBuf: b, index: i.vout}, data, {upsert: true}, next_out); }, function (err) { @@ -136,10 +177,10 @@ TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { ! o.scriptPubKey.addresses[1] // TODO : not supported ){ - // This is only to broadcast - if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { - addrs.push(o.scriptPubKey.addresses[0]); - } + // This is only to broadcast (WIP) +// if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { +// addrs.push(o.scriptPubKey.addresses[0]); +// } var data = { txidBuf: bTxId, @@ -148,11 +189,12 @@ TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { value_sat : o.value * util.COIN, addr : o.scriptPubKey.addresses[0], }; + if (fromOrphan) data.fromOrphan = true; Self.update({txidBuf: bTxId, index: o.n}, data, {upsert: true}, next_out); } else { - console.log ('WARN in TX: %s could not parse OUTPUT %d', txInfo.txid, o.n); - return next_out(); + console.log ('WARN in TX: %s could not parse OUTPUT %d', txInfo.txid, o.n); + return next_out(); } }, function (err) { @@ -174,50 +216,54 @@ TransactionOutSchema.statics.storeTransactionOuts = function(txInfo, cb) { // txs can be a [hashes] or [txObjects] -TransactionOutSchema.statics.createFromTxs = function(txs, next) { - +TransactionOutSchema.statics.createFromTxs = function(txs, fromOrphan, next) { var Self = this; + + if (typeof fromOrphan === 'function') { + next = fromOrphan; + fromOrphan = false; + } + if (!txs) return next(); var inserted_txs = []; var updated_addrs = {}; - async.forEachLimit(txs, CONCURRENCY, function(txid, cb, was_new) { + async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { var txInfo; + async.series([ function(a_cb) { + if (typeof t !== 'string') { + txInfo = t; + return a_cb(); + } + // Is it from genesis block? (testnet==livenet) // TODO: parse it from networks.genesisTX? - if (txid === genesisTXID) return a_cb(); - TransactionRpc.getRpcInfo(txid, function(err, inInfo) { + if (t === genesisTXID) return a_cb(); + + TransactionRpc.getRpcInfo(t, function(err, inInfo) { txInfo =inInfo; return a_cb(err); }); }, function(a_cb) { - if (txid === genesisTXID) return a_cb(); + if (!txInfo) return a_cb(); - Self.storeTransactionOuts(txInfo, function(err, addrs) { + Self.storeTransactionOuts(txInfo, fromOrphan, function(err, addrs) { if (err) return a_cb(err); - - if (was_new) { - inserted_txs.push(txid); - addrs.each(function(a) { - if ( !updated_addrs[a]) updated_addrs[a] = []; - updated_addrs[a].push(txid); - }); - } return a_cb(); }); }], function(err) { - return cb(err); - }); - }, - function(err) { - return next(err, inserted_txs, updated_addrs); + return each_cb(err); }); + }, + function(err) { + return next(err, inserted_txs, updated_addrs); + }); }; diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index f1aec64a..74e85014 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -7,6 +7,9 @@ require('classtool'); function spec() { var util = require('util'); var RpcClient = require('bitcore/RpcClient').class(); + var bitutil = require('bitcore/util/util'); + var Address = require('bitcore/Address').class(); + var Script = require('bitcore/Script').class(); var networks = require('bitcore/networks'); var async = require('async'); var config = require('../config/config'); @@ -96,7 +99,7 @@ function spec() { }; }; - HistoricSync.prototype.showProgress = function() { + HistoricSync.prototype.showProgress = function(height) { var self = this; if (self.error) { @@ -106,12 +109,7 @@ function spec() { self.syncPercentage = parseFloat(100 * (self.syncedBlocks + self.skippedBlocks) / self.blockChainHeight).toFixed(3); if (self.syncPercentage > 100) self.syncPercentage = 100; - p(util.format('status: [%d%%] skipped: %d', self.syncPercentage, self.skippedBlocks)); -//TODO -if (self.syncPercentage>5) { - process.exit(1); -} - // + p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks, height)); } if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); @@ -222,6 +220,46 @@ if (self.syncPercentage>5) { }; + // TODO. replace with + // Script.prototype.getAddrStrs if that one get merged in bitcore + HistoricSync.prototype.getAddrStr = function(s) { + var self = this; + + var addrStrs = []; + var type = s.classify(); + var addr; + + switch(type) { + case Script.TX_PUBKEY: + var chunk = s.captureOne(); + addr = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); + addrStrs = [ addr.toString() ]; + break; + case Script.TX_PUBKEYHASH: + addr = new Address(self.network.addressPubkey, s.captureOne()); + addrStrs = [ addr.toString() ]; + break; + case Script.TX_SCRIPTHASH: + addr = new Address(self.network.addressScript, s.captureOne()); + addrStrs = [ addr.toString() ]; + break; + case Script.TX_MULTISIG: + var addrs = []; + var chunks = s.capture(); + + chunks.forEach(function(chunk) { + var a = new Address(self.network.addressPubkey, bitutil.sha256ripe160(chunk)); + addrStrs.push(a.toString()); + }); + break; + case Script.TX_UNKNOWN: + break; + } + + return addrStrs; + }; + + HistoricSync.prototype.getBlockFromFile = function(height, scanOpts, cb) { var self = this; @@ -237,14 +275,13 @@ if (self.syncPercentage>5) { if (err) return cb(err); nextHash = res.result; -//console.log('[HistoricSync.js.235:nextHash:]',nextHash); //TODO return c(); }); }, //show some (inacurate) status function(c) { if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { - self.showProgress(); + self.showProgress(height); } return c(); @@ -254,7 +291,27 @@ if (self.syncPercentage>5) { self.blockExtractor.getNextBlock(function(err, b) { if (err || ! b) return c(err); - blockInfo = b.getStandardizedObject(b.txs); + blockInfo = b.getStandardizedObject(b.txs, self.network); + + var ti=0; + // Get TX Address + b.txs.forEach(function(t) { + var objTx = blockInfo.tx[ti++]; + var to=0; + t.outs.forEach( function(o) { + + + var s = new Script(o.s); + var addrs = self.getAddrStr(s); + + // support only p2pubkey p2pubkeyhash and p2sh + if (addrs.length === 1) { + objTx.out[to].addrStr = addrs[0]; + } + to++; + }); + }); + return c(); }); },