diff --git a/.jshintrc b/.jshintrc index bd532c2..3959625 100644 --- a/.jshintrc +++ b/.jshintrc @@ -34,7 +34,7 @@ "inject", "expect" ], - "indent": 2, // Specify indentation spacing + "indent": false, // Specify indentation spacing "devel": true, // Allow development statements e.g. `console.log();`. "noempty": true // Prohibit use of empty blocks. } diff --git a/app/models/Transaction.js b/app/models/Transaction.js index 37d55f3..61c0605 100644 --- a/app/models/Transaction.js +++ b/app/models/Transaction.js @@ -3,6 +3,7 @@ /** * Module dependencies. */ + var mongoose = require('mongoose'), Schema = mongoose.Schema, async = require('async'), @@ -59,31 +60,20 @@ TransactionSchema.statics.fromIdWithInfo = function(txid, cb) { }; TransactionSchema.statics.createFromArray = function(txs, next) { - var that = this; - if (!txs) return next(); -// console.log('exploding ', txs); - async.forEach( txs, function(tx, callback) { - // console.log('procesing TX %s', tx); that.create({ txid: tx }, function(err) { if (err && ! err.toString().match(/E11000/)) { - return callback(); - } - if (err) { - return callback(err); } return callback(); - }); }, function(err) { - if (err) return next(err); - return next(); + return next(err); } ); }; diff --git a/config/env/development.js b/config/env/development.js index e51b30f..f01f9fb 100755 --- a/config/env/development.js +++ b/config/env/development.js @@ -10,7 +10,7 @@ module.exports = { pass: 'real_mystery', protocol: 'http', host: process.env.BITCOIND_HOST || '127.0.0.1', - port: process.env.BITCOIND_PORT || '8332', + port: process.env.BITCOIND_PORT || '18332', }, network: 'testnet', } diff --git a/etc/bitcoind/bitcoin.conf b/etc/bitcoind/bitcoin.conf index 879bede..936a949 100644 --- a/etc/bitcoind/bitcoin.conf +++ b/etc/bitcoind/bitcoin.conf @@ -1,8 +1,6 @@ rpcuser=mystery rpcpassword=real_mystery server=1 -rpcport=8332 -testnet=3 txindex=1 # Allow connections outsite localhost? diff --git a/lib/Sync.js b/lib/Sync.js index 6fb58cd..2408cf3 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -1,226 +1,212 @@ -require('classtool'); +'use strict'; +require('classtool'); /* We dont sync any contents from TXs, only their IDs are stored */ var isSyncTxEnabled = 0; -function spec(b) { - var mongoose = require('mongoose'); - var util = require('util'); +function spec() { + var mongoose = require('mongoose'); + var util = require('util'); - var RpcClient = require('bitcore/RpcClient').class(); - var networks = require('bitcore/networks'); - var async = require('async'); + var RpcClient = require('bitcore/RpcClient').class(); + var networks = require('bitcore/networks'); + var async = require('async'); - var config = require('../config/config'); - var Block = require('../app/models/Block'); - var Transaction=require('../app/models/Transaction'); + var config = require('../config/config'); + var Block = require('../app/models/Block'); + var Transaction = require('../app/models/Transaction'); - function Sync(config) { - this.network = config.networkName == 'testnet' ? networks.testnet : networks.livenet; - } + function Sync(config) { + this.network = config.networkName === 'testnet' ? networks.testnet: networks.livenet; + } - var progress_bar = function(string, current, total) { - console.log( util.format("\t%s %d/%d [%d%%]", - string, current, total, parseInt(100 * current/total)) - ); - } + var progress_bar = function(string, current, total) { + console.log(util.format('\t%s %d/%d [%d%%]', string, current, total, parseInt(100 * current / total))); + }; - Sync.prototype.getNextBlock = function (blockHash,cb) { - var that = this; + Sync.prototype.getNextBlock = function(blockHash, cb) { + var that = this; + if (!blockHash) { + return cb(); + } + this.rpc.getBlock(blockHash, function(err, blockInfo) { + if (err) return cb(err); + if (blockInfo.result.height % 1000 === 0) { + var h = blockInfo.result.height, + d = blockInfo.result.confirmations; + progress_bar('height', h, h + d); + } - if ( !blockHash ) { - return cb(); - } + that.storeBlock(blockInfo.result, function(err) { + if (!err) { + var txs = blockInfo.result.tx; + that.storeTxs(txs, function(err) { + if (!err) { + return that.getNextBlock(blockInfo.result.nextblockhash, cb); + } + }); + } + }); + }); + }; - this.rpc.getBlock(blockHash, function(err, blockInfo) { - if (err) return cb(err); + Sync.prototype.storeBlock = function(block, cb) { + Block.create(block, function(err, inBlock) { + // E11000 => already exists + if (err && ! err.toString().match(/E11000/)) { + return cb(err); + } + cb(); + }); + }; - if ( ! ( blockInfo.result.height % 1000) ) { - var h = blockInfo.result.height, - d = blockInfo.result.confirmations; - progress_bar('height', h, h + d); - } + Sync.prototype.storeTxs = function(txs, cb) { + Transaction.createFromArray(txs, cb); + }; - Block.create( blockInfo.result, function(err, inBlock) { + Sync.prototype.syncBlocks = function(reindex, cb) { + var that = this; + var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex'); - // E11000 => already exists - if (err && ! err.toString().match(/E11000/)) { - return cb(err); - } + console.log('Syncing Blocks... ' + reindex); + if (reindex) { + return this.getNextBlock(genesisHash, cb); + } - if (inBlock) { - Transaction.createFromArray( blockInfo.result.tx,function (err) { - return that.getNextBlock(blockInfo.result.nextblockhash, cb); - }); - } - else - return that.getNextBlock(blockInfo.result.nextblockhash, cb); - }); - }); - } + Block.findOne({}, + {}, + { + sort: { + 'time': - 1 + } + }, + function(err, block) { + if (err) return cb(err); - Sync.prototype.syncBlocks = function (reindex, cb) { + var nextHash = block && block.hash ? block.hash: genesisHash; - var that = this; - var genesisHash = this.network.genesisBlock.hash.reverse().toString('hex'); + console.log('\tStarting at hash: ' + nextHash); + return that.getNextBlock(nextHash, cb); + }); + }; - console.log("Syncing Blocks..."); - if (reindex) - return this.getNextBlock(genesisHash, cb); + // This is not currently used. Transactions are represented by txid only + // in mongodb + Sync.prototype.syncTXs = function(reindex, cb) { + var that = this; - Block.findOne({}, {}, { sort: { 'time' : -1 } }, function(err, block) { - if (err) return cb(err); + console.log('Syncing TXs...'); + if (reindex) { + // TODO? + } - var nextHash = - block && block.hash - ? block.hash - : genesisHash - ; + Transaction.find({ + blockhash: null + }, + function(err, txs) { + if (err) return cb(err); - - console.log('\tStarting at hash: ' + nextHash); - return that.getNextBlock(nextHash, cb); - }); - } + var read = 0; + var pull = 0; + var write = 0; + var total = txs.length; + console.log('\tneed to pull %d txs', total); + if (!total) return cb(); - // This is not currently used. Transactions are represented by txid only - // in mongodb - Sync.prototype.syncTXs = function (reindex, cb) { + async.each(txs, function(tx, next) { + if (!tx.txid) { + console.log('NO TXID skipping...', tx); + return next(); + } - var that = this; + if (read++ % 1000 === 0) progress_bar('read', read, total); - console.log("Syncing TXs..."); - if (reindex) { - // TODO? - } - + that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { - Transaction.find({blockhash: null}, function(err, txs) { - if (err) return cb(err); + if (pull++ % 1000 === 0) progress_bar('\tpull', pull, total); - var read = 0; - var pull = 0; - var write = 0; - var total = txs.length; - console.log("\tneed to pull %d txs", total); + if (!err && txInfo) { + Transaction.update({ + txid: tx.txid + }, + txInfo.result, function(err) { + if (err) return next(err); - if (!total) return cb(); + if (write++ % 1000 === 0) progress_bar('\t\twrite', write, total); - async.each(txs, - function(tx, next){ - if (! tx.txid) { - console.log("NO TXID skipping...", tx); - return next(); - } + return next(); + }); + } + else return next(); + }); + }, + function(err) { + if (err) return cb(err); + return cb(err); + }); + }); + }; - if ( ! ( read++ % 1000) ) - progress_bar('read', read, total); + Sync.prototype.init = function(opts) { + mongoose.connect(config.db); + this.db = mongoose.connection; + this.rpc = new RpcClient(config.bitcoind); - that.rpc.getRawTransaction(tx.txid, 1, function(err, txInfo) { + this.db.on('error', console.error.bind(console, 'connection error:')); - if ( ! ( pull++ % 1000) ) - progress_bar('\tpull', pull, total); + }; - if (!err && txInfo) { - Transaction.update({txid: tx.txid}, txInfo.result, function(err) { - if (err) return next(err); + Sync.prototype.import_history = function(opts, next) { - if ( ! ( write++ % 1000) ) - progress_bar('\t\twrite', write, total); + var that = this; + this.db.once('open', function() { + async.series([ + function(cb) { + if (opts.destroy) { + console.log('Deleting Blocks...'); + that.db.collections.blocks.drop(cb); + } else { + cb(); + } + }, + function(cb) { + if (opts.destroy) { + console.log('Deleting TXs...'); + that.db.collections.transactions.drop(cb); + } else { + cb(); + } + }, + function(cb) { + if (!opts.skip_blocks) { + that.syncBlocks(opts.reindex, cb); + } else { + cb(); + } + }, + function(cb) { + if (isSyncTxEnabled && ! opts.skip_txs) { + that.syncTXs(opts.reindex, cb); + } + else { + return cb(); + } + }], function(err) { + return next(err); + }); + }); + }; - return next(); - }); - } - else return next(); - }); - }, - function(err){ - if (err) return cb(err); - return cb(err); - } - ); - }); - } - - Sync.prototype.start = function (opts, next) { - - - mongoose.connect(config.db); - var db = mongoose.connection; - this.rpc = new RpcClient(config.bitcoind); - var that = this; - - - db.on('error', console.error.bind(console, 'connection error:')); - - db.once('open', function (){ - - async.series([ - function(cb){ - if (opts.destroy) { - console.log("Deleting Blocks..."); - return db.collections['blocks'].drop(cb); - } - return cb(); - }, - function(cb){ - if (opts.destroy) { - console.log("Deleting TXs..."); - return db.collections['transactions'].drop(cb); - } - return cb(); - }, - function(cb) { - - if (! opts.skip_blocks) { - that.syncBlocks(opts.reindex, function(err) { - if (err) { - return cb(err); - - } - console.log("\tBlocks done."); - - return cb(); - }); - } - else { - return cb(); - } - }, - function(cb) { - if ( isSyncTxEnabled && ! opts.skip_txs) { - that.syncTXs(opts.reindex, function(err) { - if (err) { - return cb(err); - - } - return cb(); - }); - } - else { - return cb(); - } - }, - function(cb) { - db.close(); - return cb(); - }, - ], - function(err) { - if (err) { - db.close(); - return next(err); - } - return next(); - }); - }); - } - return Sync; -}; + Sync.prototype.close = function() { + console.log("closing connection"); + this.db.close(); + }; + return Sync; +} module.defineClass(spec); - + diff --git a/p2p.js b/p2p.js index c776deb..8b42c2c 100755 --- a/p2p.js +++ b/p2p.js @@ -1,5 +1,7 @@ 'use strict'; +process.env.NODE_ENV = process.env.NODE_ENV || 'development'; + var fs = require('fs'); var HeaderDB = require('./HeaderDB').class(); var Block = require('bitcore/Block').class(); @@ -7,6 +9,8 @@ var CoinConst = require('bitcore/const'); var coinUtil = require('bitcore/util/util'); var networks = require('bitcore/networks'); var Parser = require('bitcore/util/BinaryParser').class(); +var async = require('async'); +var Sync = require('./lib/Sync').class(); var peerdb_fn = 'peerdb.json'; @@ -14,222 +18,154 @@ var peerdb = undefined; var hdrdb = undefined; var network = networks.testnet; var config = { - network : network.name + network: network.name }; var PeerManager = require('bitcore/PeerManager').createClass({ - config : config + config: config }); var Peer = require('bitcore/Peer').class(); function peerdb_load() { - try { - peerdb = JSON.parse(fs.readFileSync(peerdb_fn)); - } catch (d) { - console.warn('Unable to read peer db', peerdb_fn, 'creating new one.'); - peerdb = [ { - ipv4 : '127.0.0.1', - port : 18333 - }, ]; + try { + peerdb = JSON.parse(fs.readFileSync(peerdb_fn)); + } catch(d) { + console.warn('Unable to read peer db', peerdb_fn, 'creating new one.'); + peerdb = [{ + ipv4: '127.0.0.1', + port: 18333 + }, + ]; - fs.writeFileSync(peerdb_fn, JSON.stringify(peerdb)); - } + fs.writeFileSync(peerdb_fn, JSON.stringify(peerdb)); + } } -function hdrdb_load() -{ - hdrdb = new HeaderDB({network: network}); +function hdrdb_load() { + hdrdb = new HeaderDB({ + network: network + }); } function get_more_headers(info) { - var conn = info.conn; - var loc = hdrdb.locator(); - conn.sendGetHeaders(loc, coinUtil.NULL_HASH); + var conn = info.conn; + var loc = hdrdb.locator(); + conn.sendGetHeaders(loc, coinUtil.NULL_HASH); } function add_header(info, block) { - var hashStr = coinUtil.formatHashFull(block.calcHash()); + var hashStr = coinUtil.formatHashFull(block.calcHash()); - try { - hdrdb.add(block); - } catch (e) { - return; - } + try { + hdrdb.add(block); + } catch(e) { + return; + } } function handle_headers(info) { - console.log('handle headers'); - var headers = info.message.headers; + console.log('handle headers'); + var headers = info.message.headers; - headers.forEach(function(hdr) { - add_header(info, hdr); - }); + headers.forEach(function(hdr) { + add_header(info, hdr); + }); - // We persist the header DB after each batch - //hdrdb.writeFile(hdrdb_fn); - - // Only one request per batch of headers we receive. - get_more_headers(info); + // We persist the header DB after each batch + //hdrdb.writeFile(hdrdb_fn); + // Only one request per batch of headers we receive. + get_more_headers(info); } - function handle_verack(info) { - var inv = { - type : CoinConst.MSG.BLOCK, - hash : network.genesisBlock.hash, - }; - var invs = [ inv ]; - - // Asks for the genesis block - // console.log('p2psync: Asking for the genesis block'); - // info.conn.sendGetData(invs); + var inv = { + type: CoinConst.MSG.BLOCK, + hash: network.genesisBlock.hash, + }; + var invs = [inv]; + // Asks for the genesis block + // console.log('p2psync: Asking for the genesis block'); + // info.conn.sendGetData(invs); } function handle_inv(info) { - console.log('handle inv'); - // TODO: should limit the invs to objects we haven't seen yet - var invs = info.message.invs; - invs.forEach(function(inv) { - console.log('Received inv for a '+CoinConst.MSG.to_str(inv.type)); - } - ); - console.log('requesting getData'); - info.conn.sendGetData(invs); + // TODO: should limit the invs to objects we haven't seen yet + var invs = info.message.invs; + invs.forEach(function(inv) { + console.log('Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); + }); + info.conn.sendGetData(invs); } -function handle_tx(info) { - var tx = info.message.tx.getStandardizedObject(); - console.log('handle tx: '+JSON.stringify(tx)); +var sync = new Sync({ + networkName: networks.testnet +}); +sync.init(); +function handle_tx(info) { + var tx = info.message.tx.getStandardizedObject(); + console.log('Handle tx: ' + tx.hash); + sync.storeTxs([tx.hash], function(err) { + if (err) { + console.log('error in handle TX: ' + err); + } + }); } function handle_block(info) { - console.log('handle block'); - var block = info.message.block; - add_header(info, block); + var block = info.message.block; + var now = Math.round(new Date().getTime() / 1000); + var blockHash = coinUtil.formatHashFull(block.calcHash()); + console.log('Handle block: ' + blockHash); + sync.storeBlock({ + 'hash': blockHash, + 'time': now + }, + function(err) { + if (err) { + console.log('error in handle Block: ' + err); + } else { + var hashes = block.txs.map(function(tx) { + return coinUtil.formatHashFull(tx.hash); + }); + sync.storeTxs(hashes, function() {}); + } + }); + } function handle_connected(data) { - var peerman = data.pm; - var peers_n = peerman.peers.length; - console.log('p2psync: Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's' : '')); + var peerman = data.pm; + var peers_n = peerman.peers.length; + console.log('p2psync: Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); } function p2psync() { - var peerman = new PeerManager(); + var peerman = new PeerManager(); - peerdb.forEach(function(datum) { - var peer = new Peer(datum.ipv4, datum.port); - peerman.addPeer(peer); - }); + peerdb.forEach(function(datum) { + var peer = new Peer(datum.ipv4, datum.port); + peerman.addPeer(peer); + }); - peerman.on('connection', function(conn) { - conn.on('verack', handle_verack); - conn.on('block', handle_block); - conn.on('headers', handle_headers); - conn.on('inv', handle_inv); - conn.on('tx', handle_tx); - }); - peerman.on('connect', handle_connected); + peerman.on('connection', function(conn) { + conn.on('verack', handle_verack); + conn.on('block', handle_block); + conn.on('headers', handle_headers); + conn.on('inv', handle_inv); + conn.on('tx', handle_tx); + }); + peerman.on('connect', handle_connected); - peerman.start(); -} - -function filesync_block_buf(blkdir, fn, buf) { - var parser = new Parser(buf); - var block = new Block(); - block.parse(parser, true); - - var hashStr = coinUtil.formatHashFull(block.calcHash()); - - try { - hdrdb.add(block); - } catch (e) { - var height = hdrdb.size(); - console.log('HeaderDB failed adding block #' + height + ', ' + hashStr); - console.log(' Reason: ' + e); - return; - } - - var height = hdrdb.size() - 1; - if ((height % 1000) == 0) - console.log('HeaderDB added block #' + height + ', ' + hashStr); -} - -function filesync_open_cb(err, fd, blkdir, fn) { - if (err) - throw err; - - var hdrbuf = new Buffer(4 * 2); - while (1) { - // read 2x 32-bit header - var bread = fs.readSync(fd, hdrbuf, 0, 4 * 2, null); - if (bread < (4 * 2)) { - console.log('Short read/EOF, ending scan of ' + fn); - break; - } - - // check magic matches - var magic = hdrbuf.slice(0, 4); - if (magic.toString() != network.magic.toString()) { - console.log('Invalid network magic, ending scan of ' + fn); - break; - } - - // block size - var blkSize = hdrbuf.readUInt32LE(4); - if (blkSize > (1 * 1024 * 1024)) - throw new Error('Invalid block size ' + blkSize); - - // read raw block data - var blkBuf = new Buffer(blkSize); - bread = fs.readSync(fd, blkBuf, 0, blkSize, null); - if (bread != blkSize) - throw new Error('Failed to read block'); - - // process block - filesync_block_buf(blkdir, fn, blkBuf); - } - - fs.closeSync(fd); - - hdrdb.writeFile(hdrdb_fn); - console.log('Wrote header db'); -} - -function filesync_block_file(blkdir, fn) { - console.log('Scanning ' + fn + ' for block data.'); - - var pathname = blkdir + '/' + fn; - fs.open(pathname, 'r', function(err, fd) { - filesync_open_cb(err, fd, blkdir, fn); - }); -} - -function cmd_filesync_rd(err, files, blkdir) { - if (err) - throw err; - - files = files.sort(); - - var scanned = 0; - files.forEach(function(fn) { - var re = /^blk\d+\.dat$/; - if (fn.match(re)) { - filesync_block_file(blkdir, fn); - scanned++; - } - }); - - console.log('Scanned ' + scanned + ' of ' + files.length + ' files in ' - + blkdir); + peerman.start(); } function main() { - peerdb_load(); - hdrdb_load(); + peerdb_load(); + hdrdb_load(); - p2psync(); + p2psync(); } main(); + diff --git a/package.json b/package.json index 33f323d..c9338fc 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,12 @@ { "name": "Matias Alejo Garcia", "email": "ematiu@gmail.com" + }, + { + "name": "Manuel Araoz", + "email": "manuelaraoz@gmail.com" } + ], "bugs": { "url": "https://github.com/bitpay/mystery/issues" diff --git a/test/model/block.js b/test/model/block.js index 1f4a798..f440b20 100644 --- a/test/model/block.js +++ b/test/model/block.js @@ -36,7 +36,6 @@ describe('Block fromHashWithInfo', function(){ }); }); - it('should poll block\'s info from bitcoind', function(done) { var block2 = Block.fromHashWithInfo(TESTING_BLOCK, function(err, b2) { if (err) done(err); diff --git a/util/sync.js b/util/sync.js index c13562a..e26166a 100755 --- a/util/sync.js +++ b/util/sync.js @@ -1,34 +1,44 @@ -#!/usr/bin/env node +# ! /usr/bin / env node + +'use strict'; process.env.NODE_ENV = process.env.NODE_ENV || 'development'; require('buffertools').extend(); -var SYNC_VERSION = '0.1'; -var program = require('commander'); -var Sync = require('../lib/Sync').class(); +var SYNC_VERSION = '0.1'; +var program = require('commander'); +var Sync = require('../lib/Sync').class(); +var async = require('async'); -program - .version(SYNC_VERSION) - .option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet') - .option('-R --reindex', 'Force reindexing', '0') - .option('-D --destroy', 'Remove current DB', '0') - .option('--skip_blocks', 'Sync blocks') - .option('--skip_txs', 'Sync transactions') - .parse(process.argv); +program.version(SYNC_VERSION).option('-N --network [livenet]', 'Set bitcoin network [testnet]', 'testnet').option('-R --reindex', 'Force reindexing', '0').option('-D --destroy', 'Remove current DB', '0').option('--skip_blocks', 'Sync blocks').option('--skip_txs', 'Sync transactions').parse(process.argv); -var sync = new Sync({ networkName: program.network }); +var sync = new Sync({ + networkName: program.network +}); if (program.remove) { } -sync.start( program, function(err){ - if (err) { - console.log("CRITICAL ERROR: ", err); - } - else { - console.log('Done!'); - } -}); +async.series([ +function(cb) { + sync.init(program); + cb(); +}, +function(cb) { + sync.import_history(program, function(err) { + if (err) { + console.log('CRITICAL ERROR: ', err); + } + else { + console.log('Done!'); + } + cb(); + }); +}, +function(cb) { + sync.close(); + cb(); +}]);