Merge branch 'master' into feature/get_async_transactions

This commit is contained in:
Gustavo Cortez 2014-01-14 20:35:33 -03:00
commit c734ec40a3
23 changed files with 323 additions and 202 deletions

3
.gitignore vendored
View File

@ -8,7 +8,7 @@ lib-cov
*.pid
*.gz
*.swp
tags
pids
logs
results
@ -23,7 +23,6 @@ node_modules
peerdb.json
npm-debug.log
node_modules
.nodemonignore
.DS_Store

View File

@ -24,25 +24,39 @@ function spec() {
this.addrStr = addrStr;
} catch(e){
}
Object.defineProperty(this, "totalSent", {
get: function() {
return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN);
},
set: function(i) {
totalSentSat = i * BitcoreUtil.COIN;
},
enumerable: 1,
});
Object.defineProperty(this, "balance", {
get: function() {
return parseFloat(this.balanceSat) / parseFloat(BitcoreUtil.COIN);
},
set: function(i) {
balance = i * BitcoreUtil.COIN;
},
enumerable: 1,
});
Object.defineProperty(this, "totalReceived", {
get: function() {
return parseFloat(this.totalReceivedSat) / parseFloat(BitcoreUtil.COIN);
},
set: function(i) {
totalReceived = i * BitcoreUtil.COIN;
},
enumerable: 1,
});
}
Address.prototype.__defineGetter__('balance', function(){
return parseFloat(this.balanceSat) / parseFloat(BitcoreUtil.COIN);
});
Address.prototype.__defineGetter__('totalReceived', function(){
return parseFloat(this.totalReceivedSat) / parseFloat(BitcoreUtil.COIN);
});
Address.prototype.__defineGetter__('totalSent', function(){
return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN);
});
Address.prototype.update = function(next) {
if (! this.addrStr) {

View File

@ -5,7 +5,10 @@
*/
var mongoose = require('mongoose'),
Schema = mongoose.Schema,
bignum = require('bignum'),
RpcClient = require('bitcore/RpcClient').class(),
util = require('bitcore/util/util'),
BitcoreBlock= require('bitcore/Block').class(),
config = require('../../config/config')
;
@ -26,7 +29,6 @@ var BlockSchema = new Schema({
fromP2P: Boolean,
});
/**
* Validations
*/
@ -82,6 +84,8 @@ BlockSchema.methods.getInfo = function (next) {
that.info = blockInfo.result;
that.info.reward = BitcoreBlock.getBlockValue(that.info.height) / util.COIN ;
//console.log("THAT", that);
return next(null, that.info);
});

View File

@ -10,12 +10,14 @@ var mongoose = require('mongoose'),
RpcClient = require('bitcore/RpcClient').class(),
Transaction = require('bitcore/Transaction').class(),
Address = require('bitcore/Address').class(),
BitcoreBlock= require('bitcore/Block').class(),
networks = require('bitcore/networks'),
util = require('bitcore/util/util'),
bignum = require('bignum'),
config = require('../../config/config'),
TransactionItem = require('./TransactionItem');
var CONCURRENCY = 5;
/**
*/
@ -90,18 +92,22 @@ TransactionSchema.statics.fromIdWithInfo = function(txid, cb) {
TransactionSchema.statics.createFromArray = function(txs, next) {
var that = this;
if (!txs) return next();
var mongo_txs = [];
async.forEach( txs,
function(tx, callback) {
that.create({ txid: tx }, function(err) {
if (err && ! err.toString().match(/E11000/)) {
return callback(err);
function(tx, cb) {
that.create({ txid: tx }, function(err, new_tx) {
if (err) {
if (err.toString().match(/E11000/)) {
return cb();
}
return cb(err);
}
return callback();
mongo_txs.push(new_tx);
return cb();
});
},
function(err) {
return next(err);
return next(err, mongo_txs);
}
);
};
@ -117,7 +123,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, cb) {
i.n = index++;
});
async.each(t.info.vin, function(i, next_in) {
async.forEachLimit(t.info.vin, CONCURRENCY, function(i, next_in) {
if (i.addr && i.value) {
//console.log("Creating IN %s %d", i.addr, i.valueSat);
@ -138,7 +144,7 @@ TransactionSchema.statics.explodeTransactionItems = function(txid, cb) {
},
function (err) {
if (err) console.log (err);
async.each(t.info.vout, function(o, next_out) {
async.forEachLimit(t.info.vout, CONCURRENCY, function(o, next_out) {
/*
* TODO Support multisigs
@ -175,7 +181,7 @@ TransactionSchema.methods.fillInputValues = function (tx, next) {
var network = ( config.network === 'testnet') ? networks.testnet : networks.livenet ;
var that = this;
async.each(tx.ins, function(i, cb) {
async.forEachLimit(tx.ins, CONCURRENCY, function(i, cb) {
var outHash = i.getOutpointHash();
var outIndex = i.getOutpointIndex();
@ -296,9 +302,17 @@ TransactionSchema.methods.queryInfo = function (next) {
that.info.valueIn = valueIn / util.COIN;
that.info.feeds = (valueIn - valueOut) / util.COIN;
}
else {
var reward = BitcoreBlock.getBlockValue(that.info.height) / util.COIN;
that.info.vin[0].reward = reward;
that.info.valueIn = reward;
}
that.info.size = b.length;
return next(err, that.info);
});
});

View File

@ -12,6 +12,7 @@ script(type='text/javascript', src='/lib/angular-resource/angular-resource.js')
script(type='text/javascript', src='/lib/angular-route/angular-route.js')
script(type='text/javascript', src='/lib/qrcode-generator/js/qrcode.js')
script(type='text/javascript', src='/lib/angular-qrcode/qrcode.js')
script(type='text/javascript', src='/lib/angular-animate/angular-animate.js')
//Angular UI
script(type='text/javascript', src='/lib/angular-bootstrap/ui-bootstrap.js')
@ -37,4 +38,5 @@ script(type='text/javascript', src='/js/controllers/header.js')
script(type='text/javascript', src='/js/controllers/blocks.js')
script(type='text/javascript', src='/js/controllers/transactions.js')
script(type='text/javascript', src='/js/controllers/address.js')
script(type='text/javascript', src='/js/controllers/search.js')
script(type='text/javascript', src='/js/init.js')

View File

@ -14,5 +14,4 @@ head
link(rel='stylesheet', href='/css/common.css')
script(src='/socket.io/socket.io.js')
script(src='/lib/jquery/jquery.js')

View File

@ -2,12 +2,24 @@
var Transaction = require('../../models/Transaction');
module.exports = function(app, io) {
// server-side socket behaviour
var io = null;
module.exports.init = function(app, io_ext) {
io = io_ext;
io.set('log level', 1); // reduce logging
io.sockets.on('connection', function(socket) {
Transaction.findOne(function(err, tx) {
socket.emit('tx', tx);
});
});
};
module.exports.broadcast_tx = function(tx) {
io.sockets.emit('tx', tx);
};
module.exports.broadcast_block = function(block) {
io.sockets.emit('block', block);
};

View File

@ -10,6 +10,8 @@
"bootstrap": "3.0.3",
"angular-bootstrap": "0.9.0",
"angular-ui-utils": "0.1.0",
"angular-qrcode": "latest"
"angular-qrcode": "latest",
"angular-animate": "latest"
}
}

View File

@ -13,6 +13,8 @@ function spec() {
var Block = require('../app/models/Block');
var Transaction = require('../app/models/Transaction');
var TransactionItem = require('../app/models/TransactionItem');
var sockets = require('../app/views/sockets/main.js');
var CONCURRENCY = 1;
function Sync(config) {
this.tx_count =0;
@ -56,15 +58,25 @@ function spec() {
};
Sync.prototype.storeBlock = function(block, cb) {
Block.create(block, cb);
var that = this;
Block.create(block, function(err, b){
if (b && that.opts.broadcast_blocks) {
sockets.broadcast_block(b);
}
});
};
Sync.prototype.storeTxs = function(txids, cb) {
var that=this;
Transaction.createFromArray(txids, function(err) {
Transaction.createFromArray(txids, function(err, inserted_txs) {
if (err) return cb(err);
async.forEachLimit(inserted_txs, CONCURRENCY, function(new_tx, next) {
var txid = new_tx.txid;
if (that.opts.broadcast_txs) {
sockets.broadcast_tx(new_tx);
}
async.each(txids, function(txid, next) {
// This will trigger an RPC call
Transaction.explodeTransactionItems( txid, function(err) {
@ -180,7 +192,8 @@ function spec() {
if (!total) return cb();
async.each(txs, function(tx, next) {
async.forEachLimit(txs, CONCURRENCY, function(tx, next) {
if (read++ % 1000 === 0) progress_bar('read', read, total);
if (!tx.txid) {
@ -199,19 +212,35 @@ function spec() {
};
Sync.prototype.init = function(opts) {
if (!(opts && opts.skip_db_connection)) {
mongoose.connect(config.db);
}
this.db = mongoose.connection;
this.rpc = new RpcClient(config.bitcoind);
this.db.on('error', console.error.bind(console, 'connection error:'));
if (!(opts && opts.skip_db_connection)) {
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
}
this.opts = opts;
this.db = mongoose.connection;
this.db.on('error', function(err) {
console.log('connection error:' + err);
moogose.disconnect();
});
this.db.on('disconnect', function(err) {
console.log('disconnect:' + err);
mongoose.connect(config.db, {server: {auto_reconnect: true}} );
});
};
Sync.prototype.import_history = function(opts, next) {
var that = this;
var retry_attemps = 100;
var retry_secs = 2;
this.db.once('open', function() {
async.series([
function(cb) {
@ -240,10 +269,23 @@ function spec() {
},
function(cb) {
function sync() {
that.syncBlocks( function(err) {
if (err && err.message.match(/ECONNREFUSED/) && retry_attemps--){
setTimeout(function() {
console.log("Retrying in %d secs ", retry_secs);
sync();
}, retry_secs * 1000);
}
else
return next(err);
});
}
if (!opts.skip_blocks) {
that.syncBlocks( cb);
} else {
cb();
sync();
}
},
/* Exploding happens on block insertion
@ -267,7 +309,7 @@ function spec() {
}
*/
], function(err) {
return next(err);
return next(err);
});
});
};

131
p2p.js
View File

@ -1,131 +0,0 @@
#!/usr/bin/env node
<<<<<<< HEAD
=======
>>>>>>> 71e1c718ac8f5eb89acedb4f91f2207ec463808b
'use strict';
process.env.NODE_ENV = process.env.NODE_ENV || 'development';
var fs = require('fs');
var HeaderDB = require('./HeaderDB').class();
var Block = require('bitcore/Block').class();
var CoinConst = require('bitcore/const');
var coinUtil = require('bitcore/util/util');
var networks = require('bitcore/networks');
var Parser = require('bitcore/util/BinaryParser').class();
var Sync = require('./lib/Sync').class();
var Peer = require('bitcore/Peer').class();
var peerdb_fn = 'peerdb.json';
var peerdb = undefined;
var PROGRAM_VERSION = '0.1';
var program = require('commander');
program
.version(PROGRAM_VERSION)
.option('-N --network [testnet]', 'Set bitcoin network [testnet]', 'testnet')
.parse(process.argv);
var sync = new Sync({
networkName: program.network
});
sync.init();
var PeerManager = require('bitcore/PeerManager').createClass({
config: {
network: program.network
}
});
function peerdb_load() {
try {
peerdb = JSON.parse(fs.readFileSync(peerdb_fn));
} catch(d) {
console.warn('Unable to read peer db', peerdb_fn, 'creating new one.');
peerdb = [{
ipv4: '127.0.0.1',
port: 18333
},
];
fs.writeFileSync(peerdb_fn, JSON.stringify(peerdb));
}
}
function handle_inv(info) {
// TODO: should limit the invs to objects we haven't seen yet
var invs = info.message.invs;
invs.forEach(function(inv) {
console.log('Handle inv for a ' + CoinConst.MSG.to_str(inv.type));
});
// this is not needed right now, but it's left in case
// we need to store more info in the future
info.conn.sendGetData(invs);
}
function handle_tx(info) {
var tx = info.message.tx.getStandardizedObject();
console.log('Handle tx: ' + tx.hash);
sync.storeTxs([tx.hash], function(err) {
if (err) {
console.log('Error in handle TX: ' + err);
}
});
}
function handle_block(info) {
var block = info.message.block;
var now = Math.round(new Date().getTime() / 1000);
var blockHash = coinUtil.formatHashFull(block.calcHash());
console.log('Handle block: ' + blockHash);
sync.storeBlock({
'hash': blockHash,
'time': now
},
function(err) {
if (err) {
console.log('Error in handle Block: ' + err);
} else {
// if no errors importing block, import the transactions
var hashes = block.txs.map(function(tx) {
return coinUtil.formatHashFull(tx.hash);
});
sync.storeTxs(hashes, function() {});
}
});
}
function handle_connected(data) {
var peerman = data.pm;
var peers_n = peerman.peers.length;
console.log('p2psync: Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': ''));
}
function p2psync() {
var peerman = new PeerManager();
peerdb.forEach(function(datum) {
var peer = new Peer(datum.ipv4, datum.port);
peerman.addPeer(peer);
});
peerman.on('connection', function(conn) {
conn.on('inv', handle_inv);
conn.on('block', handle_block);
conn.on('tx', handle_tx);
});
peerman.on('connect', handle_connected);
peerman.start();
}
function main() {
peerdb_load();
p2psync();
}
main();

View File

@ -48,3 +48,16 @@ body {
font-size: 10px;
}
#search { width: 400px; }
/*Animations*/
.fader.ng-enter {
transition: opacity 1s;
opacity: 0;
}
.fader.ng-enter-active {
opacity: 1;
}

View File

@ -1,9 +1,24 @@
'use strict';
angular.module('mystery', ['ngCookies', 'ngResource', 'ngRoute', 'ui.bootstrap', 'ui.route', 'mystery.system', 'mystery.index', 'mystery.blocks', 'mystery.transactions', 'monospaced.qrcode', 'mystery.address']);
var app = angular.module('mystery',
['ngAnimate',
'ngCookies',
'ngResource',
'ngRoute',
'ui.bootstrap',
'ui.route',
'mystery.system',
'mystery.index',
'mystery.blocks',
'mystery.transactions',
'monospaced.qrcode',
'mystery.address',
'mystery.search'
]);
angular.module('mystery.system', []);
angular.module('mystery.index', []);
angular.module('mystery.blocks', []);
angular.module('mystery.transactions', []);
angular.module('mystery.address', []);
angular.module('mystery.search', []);

View File

@ -1,16 +1,29 @@
'use strict';
angular.module('mystery.system').controller('IndexController', ['$scope', 'Global', 'Index', function($scope, Global, Index) {
var TRANSACTION_DISPLAYED = 5;
var BLOCKS_DISPLAYED = 5;
angular.module('mystery.system').controller('IndexController', ['$scope', 'Global', 'socket', function($scope, Global, socket) {
$scope.global = Global;
$scope.index = Index;
}]);
$(document).ready(function() {
var socket = io.connect('http://localhost');
socket.on('tx', function(data) {
var tx = data;
console.log('Transaction received! '+tx.txid);
console.log('Transaction received! ' + tx);
if ($scope.txs.length === TRANSACTION_DISPLAYED) {
$scope.txs.pop();
}
$scope.txs.unshift(tx);
});
});
socket.on('block', function(data) {
var block = data;
console.log('Block received! ' + block);
if ($scope.blocks.length === BLOCKS_DISPLAYED) {
$scope.blocks.pop();
}
$scope.blocks.unshift(block);
});
$scope.txs = [];
$scope.blocks = [];
}]);

View File

@ -0,0 +1,35 @@
'use strict';
angular.module('mystery.search').controller('SearchController', ['$scope', '$routeParams', '$location', 'Global', 'Block', 'Transaction', 'Address', function ($scope, $routeParams, $location, Global, Block, Transaction, Address) {
$scope.global = Global;
$scope.search = function() {
var q = $scope.q;
var path;
$scope.badQuery = false;
$scope.q = '';
Block.get({
blockHash: q
}, function() {
$location.path('block/' + q);
}, function () { //block not found, search on TX
Transaction.get({
txId: q
}, function() {
$location.path('tx/' + q);
}, function () { //tx not found, search on Address
Address.get({
addrStr: q
}, function() {
$location.path('address/' + q);
}, function () { //address not found, fail :(
$scope.badQuery = true;
$scope.q = q;
});
});
});
};
}]);

View File

@ -3,6 +3,20 @@
angular.module('mystery.address').factory('Address', ['$resource', function($resource) {
return $resource('/api/addr/:addrStr', {
addrStr: '@addStr'
}, {
get: {
method: 'GET',
interceptor: {
response: function (res) {
return res.data;
},
responseError: function (res) {
if (res.status === 404) {
return res;
}
}
}
}
});
}]);

View File

@ -3,6 +3,20 @@
angular.module('mystery.blocks').factory('Block', ['$resource', function($resource) {
return $resource('/api/block/:blockHash', {
blockHash: '@blockHash'
}, {
get: {
method: 'GET',
interceptor: {
response: function (res) {
return res.data;
},
responseError: function (res) {
if (res.status === 404) {
return res;
}
}
}
}
});
}]);

View File

@ -1,5 +1,26 @@
'use strict';
angular.module('mystery.index').factory('Index', ['$resource', function($resource) {
return $resource;
}]);
app.factory('socket', function($rootScope) {
var socket = io.connect();
return {
on: function(eventName, callback) {
socket.on(eventName, function() {
var args = arguments;
$rootScope.$apply(function() {
callback.apply(socket, args);
});
});
},
emit: function(eventName, data, callback) {
socket.emit(eventName, data, function() {
var args = arguments;
$rootScope.$apply(function() {
if (callback) {
callback.apply(socket, args);
}
});
});
}
};
});

View File

@ -3,6 +3,20 @@
angular.module('mystery.transactions').factory('Transaction', ['$resource', function($resource) {
return $resource('/api/tx/:txId', {
txId: '@txId'
}, {
get: {
method: 'GET',
interceptor: {
response: function (res) {
return res.data;
},
responseError: function (res) {
if (res.status === 404) {
return res;
}
}
}
}
});
}]);

View File

@ -11,7 +11,7 @@
<li><a href="#!/blocks-date/{{pagination.next}}">{{pagination.next}} &raquo;</a></li>
</ul>
</div>
<table class="table table-striped">
<table class="table table-striped" ng-show="blocks || blocks.length">
<thead>
<th>Hash</th>
<th>Solved at</th>
@ -23,4 +23,5 @@
</tr>
</tbody>
</table>
<h1 class="text-center text-muted" ng-hide="!blocks || blocks.length">No blocks yet.</h1>
</section>

View File

@ -14,5 +14,13 @@
<a href="#!/{{item.link}}">{{item.title}}</a>
</li>
</ul>
<div ng-controller="SearchController">
<form class="navbar-form navbar-left" role="search" ng-submit="search()">
<div class="form-group" ng-class="{'has-error': badQuery}">
<input id="search" type="text" class="form-control" ng-model="q" placeholder="Search for block, transaction or address">
</div>
<span class="text-danger" ng-show="badQuery">No matching records found!</span>
</form>
</div>
</div>
</div>

View File

@ -1,8 +1,19 @@
<section data-ng-controller="IndexController" data-ng-init="last_blocks()">
<div class="jumbotron">
<h1>Hello, BitPay!</h1>
<p>Start here for blocks information</p>
<p><a href="/#!/blocks" class="btn btn-primary btn-lg">List all blocks</a></p>
<section data-ng-controller="IndexController" data-ng-init="">
<div class="container">
<div class="row">
<div class="col-md-6">
<h3> New transactions </h3>
<div class="alert alert-info fader" ng-repeat='tx in txs'>
<a href="#!/tx/{{tx.txid}}">{{tx.txid}}</a>
</div>
</div>
<div class="col-md-6">
<h3> New blocks </h3>
<div class="alert alert-success fader" ng-repeat='b in blocks'>
<a href="#!/block/{{b.hash}}">{{b.hash}}</a>
</div>
</div>
</div>
</div>
</section>

View File

@ -43,7 +43,9 @@ walk(models_path);
// p2p_sync process
var ps = new PeerSync();
ps.init({
skip_db_connection: true
skip_db_connection: true,
broadcast_txs: true,
broadcast_blocks: true
});
ps.run();
@ -59,12 +61,13 @@ require('./config/routes')(app);
// socket.io
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
require('./app/views/sockets/main.js')(app,io);
require('./app/views/sockets/main.js').init(app,io);
//Start the app by listening on <port>
var port = process.env.PORT || config.port;
server.listen(port);
console.log('Express app started on port ' + port);
server.listen(port, function(){
console.log('Express server listening on port %d in %s mode', server.address().port, process.env.NODE_ENV);
});
//expose app
exports = module.exports = app;

View File

@ -40,6 +40,8 @@ describe('Address balances', function(){
if (v.balance) assert.equal(v.balance, a.balance);
if (v.totalReceived) assert.equal(v.totalReceived, a.totalReceived);
if (v.totalSent) assert.equal(v.totalSent, a.totalSent);
if (v.transactions) {
v.transactions.forEach( function(tx) {