Merge branch 'feature/TX-API' of github.com:matiu/mystery

Gruntfile.js

@@ -39,6 +39,13 @@ module.exports = function(grunt) {
         }
       }
     },
+    mochaTest: {
+      options: {
+        reporter: 'spec',
+      },
+      src: ['test/*.js']
+    },
+
     nodemon: {
       dev: {
         options: {
@@ -61,13 +68,6 @@ module.exports = function(grunt) {
         logConcurrentOutput: true
       }
     },
-    mochaTest: {
-      options: {
-        reporter: 'spec',
-        require: 'server.js'
-      },
-      src: ['test/*.js']
-    },
     env: {
       test: {
         NODE_ENV: 'test'
@@ -87,7 +87,7 @@ module.exports = function(grunt) {
   grunt.option('force', true);
 
   //Default task(s).
-  grunt.registerTask('default', ['jshint', 'concurrent']);
+  grunt.registerTask('default', ['jshint','concurrent']);
   
   //Test task.
   grunt.registerTask('test', ['env:test', 'mochaTest']);

README.md

@@ -58,12 +58,21 @@ $ npm install -g bower
     
   Run sync from mystery repository:
     $ utils/sync.js
+
+check utils/sync.js --help for options.
+
   
 ### Blocks
 ```
   /block/[:hash]
   /block/00000000a967199a2fad0877433c93df785a8d8ce062e5f9b451cd1397bdbf62
 ```
+### Transactions 
+```
+  /tx/[:txid]
+  /tx/525de308971eabd941b139f46c7198b5af9479325c2395db7f2fb5ae8562556c
+```
+
 
 
 

Sync.js

@@ -0,0 +1,219 @@
+require('classtool');
+
+function spec(b) {
+  var mongoose  = require('mongoose');
+  var util      = require('util');
+
+  var RpcClient = require('bitcore/RpcClient').class();
+  var networks  = require('bitcore/networks');
+  var async     = require('async');
+
+  var config    = require('./config/config');
+  var Block     = require('./app/models/Block');
+  var Transaction=require('./app/models/Transaction');
+
+  function Sync(config) {
+    this.network = config.networkName == 'testnet' ? networks.testnet : networks.livenet;
+  }
+
+  var progress_bar = function(string, current, total) {
+      console.log( util.format("\t%s %d/%d [%d%%]", 
+        string, current, total, parseInt(100 * current/total))
+      );
+  }
+
+  Sync.prototype.getNextBlock = function (blockHash,cb) {
+    var that = this;
+
+    if ( !blockHash ) {
+      return cb();
+    }
+
+    this.rpc.getBlock(blockHash, function(err, blockInfo) {
+      if (err) return cb(err); 
+
+      if ( ! ( blockInfo.result.height % 1000) ) {
+        var h = blockInfo.result.height,
+            d = blockInfo.result.confirmations;
+        progress_bar('height', h, h + d);    
+      }
+
+      Block.create( blockInfo.result, function(err, inBlock) {
+
+        // E11000 => already exists
+        if (err && ! err.toString().match(/E11000/)) {
+          return cb(err);
+        }
+
+        if (inBlock) {
+          inBlock.explodeTransactions(function (err) {
+            return that.getNextBlock(blockInfo.result.nextblockhash, cb);
+          });
+        }
+        else  
+          return that.getNextBlock(blockInfo.result.nextblockhash, cb);
+      });
+    });
+  }
+
+  Sync.prototype.syncBlocks = function (reindex, cb)  {
+
+    var that        = this;
+    var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex');
+
+    console.log("Syncing Blocks...");
+    if (reindex) 
+      return this.getNextBlock(genesisHash, cb);
+
+
+    Block.findOne({}, {}, { sort: { 'confirmations' : 1 } }, function(err, block) {
+      if (err) return cb(err);
+
+      var nextHash = 
+        block && block.hash 
+        ? block.hash
+        : genesisHash
+        ;
+
+      
+      console.log('\tStarting at hash: ' + nextHash);
+      return that.getNextBlock(nextHash, cb);
+    });
+  }
+
+
+  Sync.prototype.syncTXs = function (reindex, cb)  {
+
+    var that        = this;
+
+    console.log("Syncing TXs...");
+    if (reindex) {
+      // TODO?
+    }
+      
+
+    Transaction.find({blockhash: null}, function(err, txs) {
+      if (err) return cb(err);
+
+      var read = 0;
+      var pull = 0;
+      var write = 0;
+      var total = txs.length;
+      console.log("\tneed to pull %d txs", total);
+
+      if (!total) return cb();
+
+      async.each(txs, 
+        function(tx, next){
+          if (! tx.txid) {
+            console.log("NO TXID skipping...", tx);
+            return next();
+          }
+
+          if ( ! ( read++ % 1000) ) 
+            progress_bar('read', read, total);
+
+
+          that.rpc.getRawTransaction(tx.txid, 1,  function(err, txInfo) {
+
+            if ( ! ( pull++ % 1000) ) 
+              progress_bar('\tpull', pull, total);
+
+            if (!err && txInfo) {
+              Transaction.update({txid: tx.txid}, txInfo.result, function(err) {
+                if (err) return next(err); 
+
+                if ( ! ( write++ % 1000) ) 
+                  progress_bar('\t\twrite', write, total);
+
+                return next(); 
+              });
+            }
+            else return next(); 
+          });
+        },
+        function(err){
+          if (err) return cb(err);
+          return cb(err);
+        }
+      );
+    });
+  }
+
+  Sync.prototype.start = function (opts, next)  {
+
+
+    mongoose.connect(config.db);
+    var db    = mongoose.connection;
+    this.rpc   = new RpcClient(config.bitcoind);
+    var that  = this;
+
+
+    db.on('error', console.error.bind(console, 'connection error:'));
+
+    db.once('open', function (){
+
+      async.series([ 
+        function(cb){
+          if (opts.destroy) {
+            console.log("Deleting Blocks...");
+            return Block.remove().exec(cb);
+          }
+          return cb();
+        },
+        function(cb){
+          if (opts.destroy) {
+            console.log("Deleting TXs...");
+            return Transaction.remove().exec(cb);
+          }
+          return cb();
+        },
+        function(cb) {
+
+          if (! opts.skip_blocks) {
+            that.syncBlocks(opts.reindex, function(err) {
+              if (err) {
+                return cb(err);
+
+              }
+              console.log("\tBlocks done.");
+
+              return cb();
+            });
+          }
+          else {
+            return cb();
+          }
+        },
+        function(cb) {
+          if (! opts.skip_txs) {
+            that.syncTXs(opts.reindex, function(err) {
+              if (err) {
+                return cb(err);
+
+              }
+              return cb();
+            });
+          }
+          else {
+            return cb();
+          }
+        },
+        function(cb) {
+          db.close();
+          return cb();
+        },
+      ],
+      function(err) {
+        if (err) {
+          db.close();
+          return next(err);
+        }
+        return next();
+      });
+    });
+  }
+  return Sync;
+};
+module.defineClass(spec);
+ 

app/controllers/transactions.js

@@ -0,0 +1,33 @@
+'use strict';
+
+
+var Transaction = require('../models/Transaction');
+//, _ = require('lodash');
+
+
+
+/**
+ * Module dependencies.
+ */
+
+
+/**
+ * Find block by hash ...
+ */
+exports.transaction = function(req, res, next, txid) {
+  Transaction.fromID(txid, function(err, tx) {
+    if (err) return next(err);
+    if (!tx) return next(new Error('Failed to load TX ' + txid));
+    req.transaction = tx;
+    next();
+  });
+};
+
+
+/**
+ * Show block 
+ */
+exports.show = function(req, res) {
+  res.jsonp(req.transaction);
+};
+

app/models/Block.js

@@ -3,9 +3,11 @@
 /**
  * Module dependencies.
  */
-var mongoose = require('mongoose'),
+var mongoose    = require('mongoose'),
-    Schema = mongoose.Schema;
+    Schema      = mongoose.Schema;
 
+var async       = require('async');
+var Transaction = require('./Transaction');
 
 /**
  * Block Schema
@@ -38,6 +40,32 @@ var BlockSchema = new Schema({
   },
 });
 
+BlockSchema.methods.explodeTransactions = function(next) {
+
+  //  console.log('exploding %s', this.hash, typeof this.tx);
+
+  async.forEach( this.tx,
+    function(tx, callback) {
+      // console.log('procesing TX %s', tx);
+      Transaction.create({ txid: tx }, function(err) {
+        if (err && ! err.toString().match(/E11000/)) {
+          return callback();
+        }
+        if (err) {
+
+          return callback(err);
+        }
+        return callback();
+
+      });
+    },
+    function(err) {
+      if (err) return next(err);
+      return next();
+    }
+  );
+};
+
 /**
  * Validations
  */
@@ -65,4 +93,5 @@ BlockSchema.statics.fromHash = function(hash, cb) {
   }).exec(cb);
 };
 
+
 module.exports = mongoose.model('Block', BlockSchema);

app/models/Transaction.js

@@ -0,0 +1,64 @@
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+var mongoose = require('mongoose'),
+    Schema = mongoose.Schema;
+
+
+/**
+ */
+var TransactionSchema = new Schema({
+  txid: {
+    type: String,
+    index: true,
+    unique: true,
+  },
+  version: Number,
+  locktime: Number,
+  vin: {
+    type: Array,
+    default: [],
+  },
+  vout: {
+    type: Array,
+    default: [],
+  },
+  blockhash: {
+    type: String,
+    index: true,
+    default: null,
+  },
+  confirmations: Number,
+  time: Number,
+  blocktime: Number,
+});
+
+/**
+ * Statics
+ */
+
+TransactionSchema.statics.load = function(id, cb) {
+  this.findOne({
+    _id: id
+  }).exec(cb);
+};
+
+
+TransactionSchema.statics.fromID = function(txid, cb) {
+  this.findOne({
+    txid: txid,
+  }).exec(cb);
+};
+
+/*
+ * virtual
+ */
+
+// ugly? new object every call?
+TransactionSchema.virtual('date').get(function () {
+  return new Date(this.time);
+});
+
+module.exports = mongoose.model('Transaction', TransactionSchema);

config/routes.js

@@ -7,11 +7,13 @@ module.exports = function(app) {
   app.get('/', index.render);
 
   //Block routes
-
   var blocks = require('../app/controllers/blocks');
   app.get('/block/:blockHash', blocks.show);
- 
-
   app.param('blockHash', blocks.block);
+
+  var transactions = require('../app/controllers/transactions');
+  app.get('/tx/:txid', transactions.show);
+ 
+  app.param('txid', transactions.transaction);
   
 };

package.json

@@ -35,7 +35,9 @@
     "postinstall": "node node_modules/bower/bin/bower install"
   },
   "dependencies": {
+    "async": "*",
     "classtool": "*",
+    "commander": "*",
     "express": "~3.4.7",
     "jade": "~1.0.2",
     "mongoose": "~3.8.3",

util/sync.js

@@ -1,89 +1,34 @@
 #!/usr/bin/env node
 
 process.env.NODE_ENV = process.env.NODE_ENV || 'development';
+
 require('buffertools').extend();
 
-var util             = require('util');
+var SYNC_VERSION     = '0.1';
-var RpcClient        = require('../node_modules/bitcore/RpcClient').class();
+var program          = require('commander');
-var networks         = require('../node_modules/bitcore/networks');
+var Sync             = require('../Sync').class();
 
-var Block            = require('../app/models/Block');
+program
-var config           = require('../config/config');
+	.version(SYNC_VERSION)
-var mongoose         = require('mongoose');
+	.option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet')
+	.option('-R --reindex', 'Force reindexing', '0')
+	.option('-D --destroy', 'Remove current DB', '0')
+	.option('--skip_blocks', 'Sync blocks')
+	.option('--skip_txs', 'Sync transactions')
+	.parse(process.argv);
 
-var networkName      = process.argv[2] || 'testnet';
+var sync = new Sync({ networkName: program.network });
-var network          = networkName == 'testnet' ? networks.testnet : networks.livenet;
 
+if (program.remove) {
 
-function getNextBlock(blockHash,cb) {
+}
 
-  if ( !blockHash ) {
+sync.start( program, function(err){
-    console.log("done");
+  if (err) {
-    return cb();
+    console.log("CRITICAL ERROR: ", err);
+  }
+  else {
+    console.log('Done!');
   }
-
-  rpc.getBlock(blockHash, function(err, blockInfo) {
-    if (err) {
-      return cb(err); 
-    }
-
-    if ( ! ( blockInfo.result.height % 1000) ) {
-      var h = blockInfo.result.height,
-          d = blockInfo.result.confirmations;
-      console.log( util.format("Height: %d/%d [%d%%]", h, d, 100*h/(h+d)));
-    }
-
-    Block.create( blockInfo.result, function(err, inBlock) {
-
-      // E11000 => already exists
-      if (err && ! err.toString().match(/E11000/)) {
-        return cb(err);
-      }
-
-      return getNextBlock(blockInfo.result.nextblockhash, cb);
-
-    });
-  });
-
-}
-
-function syncBlocks(network, cb) {
-
-  Block.findOne({}, {}, { sort: { 'confirmations' : 1 } }, function(err, block) {
-    if (err) {
-      return cb(err);
-    }
-
-
-
-    var nextHash = 
-      block && block.hash 
-      ? block.hash
-      : network.genesisBlock.hash.reverse().toString('hex')
-      ;
-
-    
-    console.log('Starting at hash: ' + nextHash);
-    getNextBlock(nextHash, cb);
-  });
-}
-
-
-mongoose.connect(config.db);
-
-var db    = mongoose.connection;
-var rpc   = new RpcClient(config.bitcoind);
-
-
-db.on('error', console.error.bind(console, 'connection error:'));
-db.once('open', function callback () {
-  syncBlocks(network, function(err) {
-    if (err) {
-      console.log(err);
-    }
-    mongoose.connection.close();
-  });
 });
 
-
-