From e93befe6f7f1bd2ff67da5ce24bbf72adf38eaff Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Fri, 10 Jan 2014 21:42:39 -0300 Subject: [PATCH] sync with TX items!!! --- app/models/Address.js | 63 ++++------------ app/models/Transaction.js | 38 ++++++---- app/models/TransactionItem.js | 35 +++++++++ lib/Sync.js | 136 ++++++++++++++++++++++++++++++---- 4 files changed, 195 insertions(+), 77 deletions(-) create mode 100644 app/models/TransactionItem.js diff --git a/app/models/Address.js b/app/models/Address.js index 68a2979..f4dad6b 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -4,13 +4,11 @@ * Module dependencies. */ var mongoose = require('mongoose'), - Schema = mongoose.Schema, - RpcClient = require('bitcore/RpcClient').class(), - config = require('../../config/config') + Schema = mongoose.Schema ; /** - * Block Schema + * Addr Schema */ var AddressSchema = new Schema({ @@ -22,26 +20,17 @@ var AddressSchema = new Schema({ index: true, unique: true, }, - balance: Number, - totalReceived: Number, - totalSent: Number, - inTransactions: [String], + inputs: [{ + type: mongoose.Schema.Types.ObjectId, + ref: 'TransactionItem' //Edit: I'd put the schema. Silly me. + }], + output: [{ + type: mongoose.Schema.Types.ObjectId, + ref: 'TransactionItem' //Edit: I'd put the schema. Silly me. + }], }); -/** - * Validations - */ - -/* -AddressSchema.path('title').validate(function(title) { - return title.length; -},'Title cannot be blank'); -*/ - -/** - * Statics - */ AddressSchema.statics.load = function(id, cb) { this.findOne({ @@ -58,37 +47,13 @@ AddressSchema.statics.fromAddr = function(hash, cb) { AddressSchema.statics.fromAddrWithInfo = function(hash, cb) { - this.fromHash(hash, function(err, block) { + this.fromHash(hash, function(err, addr) { if (err) return cb(err); - if (!block) { return cb(new Error('Block not found')); } - - block.getInfo(function(err) { return cb(err,block); } ); + if (!addr) { return cb(new Error('Addr not found')); } +// TODO +// addr.getInfo(function(err) { return cb(err,addr); } ); }); }; - -// TODO: Can we store the rpc instance in the Block object? -AddressSchema.methods.getInfo = function (next) { - - var that = this; - var rpc = new RpcClient(config.bitcoind); - - rpc.getBlock(this.hash, function(err, blockInfo) { - if (err) return next(err); - - /* - * Not sure this is the right way to do it. - * Any other way to lazy load a property in a mongoose object? - */ - - that.info = blockInfo.result; - - //console.log("THAT", that); - return next(null, that.info); - }); -}; - - - module.exports = mongoose.model('Address', AddressSchema); diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 7f6014e..b813187 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -27,6 +27,14 @@ var TransactionSchema = new Schema({ index: true, unique: true, }, + processed: { + type: Boolean, + default: false, + }, + orphaned: { + type: Boolean, + default: false, + }, }); /** @@ -154,22 +162,26 @@ TransactionSchema.methods.queryInfo = function (next) { } else { tx.ins.forEach(function(i) { + + if (i.value) { + that.info.vin[c].value = util.formatValue(i.value); + var n = util.valueToBigInt(i.value).toNumber(); + valueIn = valueIn.add( n ); - that.info.vin[c].value = util.formatValue(i.value); - var n = util.valueToBigInt(i.value).toNumber(); - valueIn = valueIn.add( n ); + var scriptSig = i.getScript(); + var pubKey = scriptSig.simpleInPubKey(); - var scriptSig = i.getScript(); - var pubKey = scriptSig.simpleInPubKey(); - - // We check for pubKey in case a broken / strange TX. - if (pubKey) { - var pubKeyHash = util.sha256ripe160(pubKey); - var addr = new Address(network.addressPubkey, pubKeyHash); - var addrStr = addr.toString(); - that.info.vin[c].addr = addrStr; + // We check for pubKey in case a broken / strange TX. + if (pubKey) { + var pubKeyHash = util.sha256ripe160(pubKey); + var addr = new Address(network.addressPubkey, pubKeyHash); + var addrStr = addr.toString(); + that.info.vin[c].addr = addrStr; + } + } + else { + console.log("TX could not be parsed: %s,%d",txInfo.result.txid, c); } - c++; }); } diff --git a/app/models/TransactionItem.js b/app/models/TransactionItem.js new file mode 100644 index 0000000..db06927 --- /dev/null +++ b/app/models/TransactionItem.js @@ -0,0 +1,35 @@ +'use strict'; + +/** + * Module dependencies. + */ +var mongoose = require('mongoose'), + Schema = mongoose.Schema; + +var TransactionItemSchema = new Schema({ + txid: String, + index: Number, + addr: { + type: String, + index: true, + }, + // >0 is Input <0 is Output + value: Number, +}); + + + +TransactionItemSchema.statics.load = function(id, cb) { + this.findOne({ + _id: id + }).exec(cb); +}; + + +TransactionItemSchema.statics.fromAddr = function(addr, cb) { + this.find({ + addr: addr, + }).exec(cb); +}; + +module.exports = mongoose.model('TransactionItem', TransactionItemSchema); diff --git a/lib/Sync.js b/lib/Sync.js index 0fcdc8f..1dd4a47 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -2,21 +2,17 @@ require('classtool'); -/* We dont sync any contents from TXs, only their IDs are stored */ - -var isSyncTxEnabled = 0; function spec() { - var mongoose = require('mongoose'); - var util = require('util'); - - var RpcClient = require('bitcore/RpcClient').class(); - var networks = require('bitcore/networks'); - var async = require('async'); - - var config = require('../config/config'); - var Block = require('../app/models/Block'); - var Transaction = require('../app/models/Transaction'); + var mongoose = require('mongoose'); + var util = require('util'); + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); + var config = require('../config/config'); + var Block = require('../app/models/Block'); + var Transaction = require('../app/models/Transaction'); + var TransactionItem = require('../app/models/TransactionItem'); function Sync(config) { this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet; @@ -151,6 +147,105 @@ function spec() { }); }; + + Sync.prototype.processTXs = function(reindex, cb) { + + var that = this; + + console.log('Syncing TXs...'); + + var filter = reindex ? {} : { processed: false } ; + + Transaction.find(filter, + function(err, txs) { + if (err) return cb(err); + + var read = 0, + pull = 0, + write = 0, + total = txs.length; + + console.log('\tneed to pull %d txs', total); + + if (!total) return cb(); + + async.each(txs, function(tx, next) { + if (read++ % 1000 === 0) progress_bar('read', read, total); + + if (!tx.txid) { + console.log('NO TXID skipping...', tx); + return next(); + } + + + // This will trigger an RPC call + Transaction.fromIdWithInfo( tx.txid, function(err,t) { + if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total); + + if (!err && t) { + var index = 0; + + async.each(t.info.vin, function(i, next_in) { + + /* + * TODO Support multisigs??? + * how?? + */ + + if (i.addr && i.value) { + TransactionItem.create({ + txid : t.txid, + value : -1 * i.value, + addr : i.addr, + index : i.n, + }, next_in); + } + else { + if ( !i.coinbase ) + console.log ("TX: %s seems to be multisig IN. Skipping... ", t.txid); + return next_in(); + } + }, + function (err) { + if (err) console.log (err); + index = 0; + async.each(t.info.vout, function(o, next_out) { + + /* + * TODO Support multisigs + */ + if (o.value && o.scriptPubKey + && o.scriptPubKey.addresses + && o.scriptPubKey.addresses[0] + ) { + TransactionItem.create({ + txid : t.txid, + value : o.value, + addr : o.scriptPubKey.addresses[0], + index : o.n, + }, next_out); + } + else { + console.log ("TX: %s,%d seems to be multisig OUT. Skipping... ", t.txid, o.n); + return next_out(); + } + }, + function (err) { + if (err) console.log (err); + if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total); + return next(); + }); + }); + } + else return next(); + }); + }, + function(err) { + return cb(err); + }); + }); + }; + Sync.prototype.init = function(opts) { if (!(opts && opts.skip_db_connection)) { mongoose.connect(config.db); @@ -191,13 +286,24 @@ function spec() { } }, function(cb) { - if (isSyncTxEnabled && ! opts.skip_txs) { + if (! opts.skip_txs) { + that.processTXs(opts.reindex, cb); + } + else { + return cb(); + } + } +/* We dont sync any contents from TXs, only their IDs are stored + function(cb) { + if (! opts.skip_txs) { that.syncTXs(opts.reindex, cb); } else { return cb(); } - }], function(err) { + } +*/ + ], function(err) { return next(err); }); });