From 94edb188d01392f5cb5723fb05d0b02ae4bb611d Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Thu, 6 Feb 2014 10:20:49 -0300 Subject: [PATCH 01/12] fix standalone p2p --- app/controllers/blocks.js | 7 +-- app/models/Address.js | 30 +++++------ etc/bitcoind/bitcoin-testnet.conf | 4 +- lib/PeerSync.js | 1 - lib/Sync.js | 3 +- lib/TransactionDb.js | 84 +++++++++++++++++++++++++++---- public/js/controllers/blocks.js | 8 +++ public/views/block_list.html | 9 +++- test/integration/addr.json | 5 +- util/p2p.js | 9 +++- 10 files changed, 121 insertions(+), 39 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 468bc13..f66aa35 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -118,9 +118,10 @@ exports.list = function(req, res) { length: allblocks.length, pagination: { next: next, - prev: prev, - current: dateStr, - isToday: isToday + prev: prev, + currentTs: lte-1, + current: dateStr, + isToday: isToday } }); }); diff --git a/app/models/Address.js b/app/models/Address.js index d377cb1..7e44970 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -62,21 +62,23 @@ function spec() { db.fromAddr(self.addrStr, function(err,txOut){ if (err) return cb(err); txOut.forEach(function(txItem){ - var v = txItem.value_sat; - self.totalReceivedSat += v; - self.transactions.push(txItem.txid); - if (! txItem.spendTxId) { - // unspent - self.balanceSat += v; - self.txApperances +=1; - } - else { - // spent - self.totalSentSat += v; - self.transactions.push(txItem.spendTxId); - self.txApperances +=2; - } + if (txItem.isConfirmed) { + var v = txItem.value_sat; + self.totalReceivedSat += v; + self.transactions.push(txItem.txid); + if (! txItem.spendTxId || !txItem.spendIsConfirmed) { + // unspent + self.balanceSat += v; + self.txApperances +=1; + } + else { + // spent + self.totalSentSat += v; + self.transactions.push(txItem.spendTxId); + self.txApperances +=2; + } + }; }); return cb(); }); diff --git a/etc/bitcoind/bitcoin-testnet.conf b/etc/bitcoind/bitcoin-testnet.conf index 01572d2..a9eb96d 100644 --- a/etc/bitcoind/bitcoin-testnet.conf +++ b/etc/bitcoind/bitcoin-testnet.conf @@ -7,8 +7,8 @@ txindex=1 #rpcallowip=192.168.1.* #rpcallowip='192.168.1.*' #rpcallowip=127.0.0.1 +#rpcallowip=* -rpcallowip=* - +port=18333 rpcport=18332 testnet=3 diff --git a/lib/PeerSync.js b/lib/PeerSync.js index ea23f96..3c8525d 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -85,7 +85,6 @@ function spec() { this.sync.storeBlock({ 'hash': blockHash, 'tx': tx_hashes, - // TODO NEXT BLOCK / PREV BLOCK? }, function(err) { if (err) { diff --git a/lib/Sync.js b/lib/Sync.js index d46e818..fbae854 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -125,8 +125,7 @@ console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO Sync.prototype.storeTxs = function(txs, cb) { var self = this; - // TODO - self.txDb.createFromTxs(txs, function(err, inserted_txs, updated_addrs) { + self.txDb.createFromArray(txs, null, function(err, inserted_txs, updated_addrs) { if (err) return cb(err); self._handleBroadcast(null, inserted_txs, updated_addrs); diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 1774bae..0c562b3 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -185,12 +185,30 @@ function spec(b) { }); }; + TransactionDb.prototype.fillConfirmations = function(o, cb) { + var self = this; + + self.isConfirmed(o.txid, function(err,is) { + if (err) return cb(err); + + o.isConfirmed = is; + if (!o.spendTxId) return cb(); + + self.isConfirmed(o.spendTxId, function(err,is) { + if (err) return cb(err); + + o.spendIsConfirmed = is; + return cb(); + }); + }); + }; + TransactionDb.prototype.fromAddr = function(addr, cb) { + var self = this; var k = ADDR_ROOT + addr; var ret=[]; - // db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { var k = data.key.split('-'); @@ -221,7 +239,11 @@ function spec(b) { }); }, function(err) { - return cb(err,ret); + async.each(ret, function(o, e_c){ + self.fillConfirmations(o,e_c); + },function(err) { + return cb(err,ret); + }); }); }); }; @@ -368,6 +390,45 @@ function spec(b) { }); }; + + TransactionDb.prototype.deleteConfirmation = function(txId, blockHash, c) { + if (!blockHash) return c(); + + db.put(ROOT + txId + '-' + blockHash, 0, function(err) { + return c(err); + }); + }; + + + TransactionDb.prototype.addConfirmation = function(txId, blockHash, c) { + if (!blockHash) return c(); + + db.put(ROOT + txId + '-' + blockHash, 1, function(err) { + return c(err); + }); + }; + + + // This slowdown addr balance calculation 100% + TransactionDb.prototype.isConfirmed = function(txId, c) { + var k = ROOT + txId; + var ret = false; + + db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + if (data.value === '1') ret = true; + }) + .on('error', function (err) { + return c(err); + }) + .on('end', function (err) { + + return c(err,ret); + }); + }; + + + // txs can be a [hashes] or [txObjects] TransactionDb.prototype.createFromArray = function(txs, blockHash, next) { var self = this; @@ -388,21 +449,23 @@ function spec(b) { if (!inInfo) return each_cb(err); self.add(inInfo, function(err) { - if (err || !blockHash) return each_cb(err); + if (err) return each_cb(err); - db.put(ROOT + t + '-' + blockHash, 1, function(err) { - return each_cb(err); - }); + // This could mean that the TX was mined since we first received. + if (blockHash && inInfo.blockhash !== blockHash) + console.log('WARN in tx %s: different blockHashses: %s vs %s', + t, blockHash, inInfo.blockhash); + insertedTxs.push(t); + + return self.addConfirmation(t,inInfo.blockhash, each_cb); }); }); } else { self.add(t, function(err) { - if (err) return each_cb(err); + if (err || !blockHash) return each_cb(err); - db.put(ROOT + t.txid + '-' + blockHash, 1, function(err) { - return each_cb(err); - }); + return self.addConfirmation(t,blockHash, each_cb); }); } }, @@ -412,7 +475,6 @@ function spec(b) { }; -// txs can be a [hashes] or [txObjects] TransactionDb.prototype.createFromBlock = function(b, next) { var self = this; if (!b || !b.tx) return next(); diff --git a/public/js/controllers/blocks.js b/public/js/controllers/blocks.js index b93b79c..b4ae5d4 100644 --- a/public/js/controllers/blocks.js +++ b/public/js/controllers/blocks.js @@ -16,6 +16,14 @@ angular.module('insight.blocks').controller('BlocksController', }); } + + $scope.humanSince = function(time) { + var m = moment.unix(time).startOf('day'); + var b = moment().startOf('day'); + return m.max().from(b); + }; + + $scope.list = function() { $scope.loading = true; diff --git a/public/views/block_list.html b/public/views/block_list.html index f826056..9b8f366 100644 --- a/public/views/block_list.html +++ b/public/views/block_list.html @@ -8,10 +8,15 @@

Blocks
mined on:

-

{{pagination.current}}

+

{{pagination.current}} UTC

+

 

+

Today

+

{{humanSince(pagination.currentTs)}} +

 

+
← {{pagination.prev}} - {{pagination.next}} → + {{pagination.next}} →
diff --git a/test/integration/addr.json b/test/integration/addr.json index 090c1c6..b40c03b 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -37,8 +37,9 @@ }, { "addr": "mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ", - "txApperances": 27, - "balance": 5.1 + "txApperances": 28, + "balance": 0, + "totalReceived": 54.81284116 }, { "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", diff --git a/util/p2p.js b/util/p2p.js index 91ead1b..d3bd665 100755 --- a/util/p2p.js +++ b/util/p2p.js @@ -14,7 +14,12 @@ program .parse(process.argv); var ps = new PeerSync(); -ps.init(program); -ps.run(); +ps.init(program, function(err){ + if (err) { + console.log(err); + process.exit(1); + } + ps.run(); +}); From 9ebc70a3e411af7b98b8f765a3b746c0f95632e4 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 8 Feb 2014 10:57:37 -0300 Subject: [PATCH 02/12] reorg case 1) working and tested --- app/models/Address.js | 2 +- dev-util/level.js | 30 +++++ dev-util/tx-level.js | 19 --- lib/BlockDb.js | 97 +++++++++------ lib/HistoricSync.js | 174 ++++++++++++--------------- lib/PeerSync.js | 20 ++-- lib/Sync.js | 235 ++++++++++++++++++++++++++++--------- lib/TransactionDb.js | 131 +++++++++++---------- test/integration/addr.json | 6 +- test/integration/sync.js | 189 ++++++++++++++++++----------- util/sync.js | 1 + 11 files changed, 551 insertions(+), 353 deletions(-) create mode 100755 dev-util/level.js delete mode 100755 dev-util/tx-level.js diff --git a/app/models/Address.js b/app/models/Address.js index 7e44970..657b67e 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -78,7 +78,7 @@ function spec() { self.transactions.push(txItem.spendTxId); self.txApperances +=2; } - }; + } }); return cb(); }); diff --git a/dev-util/level.js b/dev-util/level.js new file mode 100755 index 0000000..f897c2a --- /dev/null +++ b/dev-util/level.js @@ -0,0 +1,30 @@ +#!/usr/bin/env node +'use strict'; + +var config = require('../config/config'), + levelup = require('levelup'); + + + +var s = process.argv[2]; +var isBlock = process.argv[3] === '1'; + + +var dbPath = config.leveldb + (isBlock ? '/blocks' : '/txs'); +console.log('DB: ',dbPath); //TODO + + + +var db = levelup(dbPath ); + + +db.createReadStream({start: s, end: s+'~'}) + .on('data', function (data) { + console.log(data.key + ' => ' + data.value); //TODO + }) + .on('error', function () { + }) + .on('end', function () { + }); + + diff --git a/dev-util/tx-level.js b/dev-util/tx-level.js deleted file mode 100755 index a90cc2e..0000000 --- a/dev-util/tx-level.js +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env node - -var - config = require('../config/config'), - levelup = require('levelup'); - - -db = levelup(config.leveldb + '/txs'); - -var s = 'txouts-addr-mgqvRGJMwR9JU5VhJ3x9uX9MTkzTsmmDgQ'; -db.createReadStream({start: s, end: s+'~'}) - .on('data', function (data) { -console.log('[block-level.js.11:data:]',data); //TODO - if (data==false) c++; - }) - .on('end', function () { - }); - - diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 3e1946b..b253ca8 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -5,8 +5,11 @@ require('classtool'); function spec(b) { - var TIMESTAMP_ROOT = 'b-ts-'; // b-ts- => - var PREV_ROOT = 'b-prev-'; // b-prev- => (0 if orphan) + var TIMESTAMP_PREFIX = 'b-ts-'; // b-ts- => + var PREV_PREFIX = 'b-prev-'; // b-prev- => + var NEXT_PREFIX = 'b-next-'; // b-next- => + var MAIN_PREFIX = 'b-main-'; // b-main- => 1/0 + var TIP = 'b-tip-'; // last block on the chain /** @@ -20,7 +23,6 @@ function spec(b) { var db = b.db || levelup(config.leveldb + '/blocks'); var rpc = b.rpc || new RpcClient(config.bitcoind); - var BlockDb = function() { }; @@ -38,53 +40,71 @@ function spec(b) { }); }; + // adds a block TIP block. Does not update Next pointer in + // the block prev to the new block. + // BlockDb.prototype.add = function(b, cb) { - if (!b.hash) return cb(new Error('no Hash at Block.save')); - - - var time_key = TIMESTAMP_ROOT + + var time_key = TIMESTAMP_PREFIX + ( b.time || Math.round(new Date().getTime() / 1000) ); - db.batch() + return db.batch() .put(time_key, b.hash) - .put(PREV_ROOT + b.hash, b.previousblockhash) + .put(TIP, b.hash) + .put(MAIN_PREFIX + b.hash, 1) + .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(cb); }; - - - BlockDb.prototype.setOrphan = function(hash, cb) { - var k = PREV_ROOT + hash; - - db.get(k, function (err,oldPrevHash) { - if (err || !oldPrevHash) return cb(err); - db.put(PREV_ROOT + hash, 0, function() { - return cb(err, oldPrevHash); - }); - }); - // We keep the block in TIMESTAMP_ROOT - }; - - //mainly for testing - BlockDb.prototype.setPrev = function(hash, prevHash, cb) { - db.put(PREV_ROOT + hash, prevHash, function(err) { - return cb(err); - }); - }; - - //mainly for testing - BlockDb.prototype.getPrev = function(hash, cb) { - db.get(PREV_ROOT + hash, function(err,val) { + BlockDb.prototype.getTip = function(cb) { + db.get(TIP, function(err, val) { return cb(err,val); }); }; + //mainly for testing + BlockDb.prototype.setPrev = function(hash, prevHash, cb) { + db.put(PREV_PREFIX + hash, prevHash, function(err) { + return cb(err); + }); + }; + BlockDb.prototype.getPrev = function(hash, cb) { + db.get(PREV_PREFIX + hash, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); + }; + + BlockDb.prototype.getNext = function(hash, cb) { + db.get(NEXT_PREFIX + hash, function(err,val) { + if (err && err.notFound) { err = null; val = null;} + return cb(err,val); + }); + }; + + BlockDb.prototype.isMain = function(hash, cb) { + db.get(MAIN_PREFIX + hash, function(err, val) { + if (err && err.notFound) { err = null; val = 0;} + return cb(err,parseInt(val)); + }); + }; + + BlockDb.prototype.setMain = function(hash, isMain, cb) { + if (!isMain) console.log('ORPHAN: %s',hash); + db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { + return cb(err); + }); + }; + BlockDb.prototype.setNext = function(hash, nextHash, cb) { + db.put(NEXT_PREFIX + hash, nextHash, function(err) { + return cb(err); + }); + }; BlockDb.prototype.countNotOrphan = function(cb) { var c = 0; console.log('Counting connected blocks. This could take some minutes'); - db.createReadStream({start: PREV_ROOT, end: PREV_ROOT + '~' }) + db.createReadStream({start: MAIN_PREFIX, end: MAIN_PREFIX + '~' }) .on('data', function (data) { if (data.value !== 0) c++; }) @@ -96,8 +116,9 @@ function spec(b) { }); }; + // .has() return true orphans also BlockDb.prototype.has = function(hash, cb) { - var k = PREV_ROOT + hash; + var k = PREV_PREFIX + hash; db.get(k, function (err,val) { var ret; if (err && err.notFound) { @@ -130,14 +151,14 @@ function spec(b) { BlockDb.prototype.getBlocksByDate = function(start_ts, end_ts, limit, cb) { var list = []; db.createReadStream({ - start: TIMESTAMP_ROOT + start_ts, - end: TIMESTAMP_ROOT + end_ts, + start: TIMESTAMP_PREFIX + start_ts, + end: TIMESTAMP_PREFIX + end_ts, fillCache: true, limit: parseInt(limit) // force to int }) .on('data', function (data) { list.push({ - ts: data.key.replace(TIMESTAMP_ROOT, ''), + ts: data.key.replace(TIMESTAMP_PREFIX, ''), hash: data.value, }); }) diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 6b547bc..8534ecf 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -105,6 +105,8 @@ function spec() { HistoricSync.prototype.showProgress = function() { var self = this; + if ( ( self.syncedBlocks + self.skippedBlocks) % self.step !== 1) return; + if (self.error) { p('ERROR: ' + self.error); } @@ -143,18 +145,23 @@ function spec() { }, //show some (inacurate) status function(c) { - if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { - self.showProgress(); - } - + self.showProgress(); return c(); }, function(c) { self.rpc.getBlock(blockHash, function(err, ret) { if (err) return c(err); + if (ret) { + blockInfo = ret.result; + // this is to match block retreived from file + if (blockInfo.hash === self.genesis) + blockInfo.previousblockhash='0000000000000000000000000000000000000000000000000000000000000000'; + } + else { + blockInfo = null; + } - blockInfo = ret ? ret.result : null; return c(); }); }, @@ -162,34 +169,25 @@ function spec() { function(c) { if (existed) return c(); - self.sync.storeBlock(blockInfo, function(err) { + self.sync.storeTipBlock(blockInfo, function(err) { return c(err); }); - }, - /* TODO: Should Start to sync backwards? (this is for partial syncs) - function(c) { - - if (blockInfo.result.prevblockhash != current.blockHash) { - p("reorg?"); - scanOpts.prev = 1; - } - return c(); - } - */ - ], function(err) { + }], function(err) { if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockHash, err, self.syncedBlocks)); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', + blockHash, err, self.syncedBlocks)); return cb(err); } else { self.status = 'syncing'; } - if ( (scanOpts.upToExisting && existed && self.syncedBlocks >= self.blockChainHeight) || + if ( (scanOpts.upToExisting && existed && + self.syncedBlocks >= self.blockChainHeight) || (blockEnd && blockEnd === blockHash)) { self.status = 'finished'; - p('DONE. Found existing block: ', blockHash); + p('DONE. Found block: ', blockHash); self.showProgress(); return cb(err); } @@ -251,91 +249,72 @@ function spec() { return addrStrs; }; - HistoricSync.prototype.getBlockFromFile = function(scanOpts, cb) { + HistoricSync.prototype.getBlockFromFile = function(cb) { + var self = this; + + var blockInfo; + + //get Info + self.blockExtractor.getNextBlock(function(err, b) { + if (err || ! b) return cb(err); + + blockInfo = b.getStandardizedObject(b.txs, self.network); + // blockInfo.curWork = Deserialize.intFromCompact(b.bits); + // We keep the RPC field names + blockInfo.previousblockhash = blockInfo.prev_block; + + var ti=0; + // Get TX Address + b.txs.forEach(function(t) { + + + var objTx = blockInfo.tx[ti++]; + + //add time from block + objTx.time = blockInfo.time; + + var to=0; + t.outs.forEach( function(o) { + + + var s = new Script(o.s); + var addrs = self.getAddrStr(s); + + // support only for p2pubkey p2pubkeyhash and p2sh + if (addrs.length === 1) { + objTx.out[to].addrStr = addrs[0]; + } + to++; + }); + }); + + return cb(err,blockInfo); + }); + }; + + + HistoricSync.prototype.nextBlockFromFile = function(scanOpts, cb) { var self = this; var blockInfo; - var existed; async.series([ - //show some (inacurate) status function(c) { - if ( ( self.syncedBlocks + self.skippedBlocks) % self.step === 1) { - self.showProgress(); - } - + self.showProgress(); return c(); }, - //get Info function(c) { - - self.blockExtractor.getNextBlock(function(err, b) { - if (err || ! b) return c(err); - - blockInfo = b.getStandardizedObject(b.txs, self.network); - // blockInfo.curWork = Deserialize.intFromCompact(b.bits); - // We keep the RPC field names - blockInfo.previousblockhash = blockInfo.prev_block; - - var ti=0; - // Get TX Address - b.txs.forEach(function(t) { - - - var objTx = blockInfo.tx[ti++]; - - //add time from block - objTx.time = blockInfo.time; - - var to=0; - t.outs.forEach( function(o) { - - - var s = new Script(o.s); - var addrs = self.getAddrStr(s); - - // support only p2pubkey p2pubkeyhash and p2sh - if (addrs.length === 1) { - objTx.out[to].addrStr = addrs[0]; - } - to++; - }); - }); - - return c(); - }); - }, - //check prev - function(c) { - if (blockInfo && self.prevHash && blockInfo.previousblockhash !== self.prevHash) { - - console.log('Hole found @%s', blockInfo.hash); - console.log('From: %s To: %s', self.prevHash, blockInfo.previousblockhash); - self.sync.checkOrphan(self.prevHash, blockInfo.previousblockhash, c); - } - else return c(); - }, - //check it - function(c) { - if (!blockInfo) return c(); - - self.sync.bDb.has(blockInfo.hash, function(err, had) { - existed = had; - return c(err); - }); - }, - //store it - function(c) { - if (!blockInfo || existed) return c(); - - self.sync.storeBlock(blockInfo, function(err) { + self.getBlockFromFile(function(err, inBlockInfo) { + blockInfo = inBlockInfo; return c(err); }); }, function(c) { + self.sync.storeTipBlock(blockInfo, function(err) { + return c(err); + }); + }, + function(c) { + // continue if (blockInfo && blockInfo.hash) { - self.prevHash = blockInfo.hash; - if (existed) - self.skippedBlocks++; - else self.syncedBlocks++; } else self.status = 'finished'; @@ -452,12 +431,15 @@ function spec() { p(' scanOpts: ', JSON.stringify(scanOpts)); if (scanOpts.fromFiles) { + self.status = 'syncing'; - self.type = 'from .dat Files'; + self.type = 'from .dat Files'; + + async.whilst(function() { return self.status === 'syncing'; }, function (w_cb) { - self.getBlockFromFile(scanOpts, function(err) { + self.nextBlockFromFile(scanOpts, function(err) { setImmediate(function(){ return w_cb(err); }); diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 3c8525d..4f7a493 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -8,6 +8,7 @@ function spec() { var Sync = require('./Sync').class(); var Peer = require('bitcore/Peer').class(); var config = require('../config/config'); + var networks = require('bitcore/networks'); var peerdb_fn = 'peerdb.json'; @@ -16,18 +17,15 @@ function spec() { PeerSync.prototype.init = function(opts, cb) { if (!opts) opts = {}; - var network = opts && (opts.network || 'testnet'); + var networkName = opts && (opts.network || 'testnet'); + var network = networkName === 'testnet' ? networks.testnet : network.livenet; this.verbose = opts.verbose; this.peerdb = undefined; - this.sync = new Sync({ - networkName: network - }); + this.sync = new Sync(); this.PeerManager = require('bitcore/PeerManager').createClass({ - opts: { network: network - } }); this.peerman = new this.PeerManager(); this.load_peers(); @@ -77,19 +75,23 @@ function spec() { console.log('[p2p_sync] Handle block: ' + blockHash); } - var tx_hashes = block.txs.map(function(tx) { return coinUtil.formatHashFull(tx.hash); }); - this.sync.storeBlock({ + this.sync.storeTipBlock({ 'hash': blockHash, 'tx': tx_hashes, + 'previousblockhash': coinUtil.formatHashFull(block.prev_hash), }, function(err) { if (err) { console.log('[p2p_sync] Error in handle Block: ' + err); } + + // Check for reorgs... + // The previous last block hash + // if different => call }); }; @@ -110,6 +112,8 @@ function spec() { }); this.peerman.on('connection', function(conn) { + +console.log('[PeerSync.js.113]'); //TODO conn.on('inv', self.handle_inv.bind(self)); conn.on('block', self.handle_block.bind(self)); conn.on('tx', self.handle_tx.bind(self)); diff --git a/lib/Sync.js b/lib/Sync.js index fbae854..e923db2 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -11,15 +11,13 @@ function spec() { function Sync() { - this.bDb = new BlockDb(); - this.txDb = new TransactionDb(); } Sync.prototype.init = function(opts, cb) { var self = this; - self.opts = opts; - + this.bDb = new BlockDb(opts); + this.txDb = new TransactionDb(opts); return cb(); }; @@ -39,79 +37,202 @@ function spec() { ], next); }; + /* + * Arrives a NEW block, which is the new TIP + * + * Case 0) Simple case + * A-B-C-D-E(TIP)-NEW + * + * Case 1) + * A-B-C-D-E(TIP) + * \ + * NEW + * + * 1) Declare D-E orphans (and possible invalidate TXs on them) + * + * Case 2) + * A-B-C-D-E(TIP) + * \ + * F-G-NEW + * 1) Set F-G as connected (mark TXs as valid) + * 2) Declare D-E orphans (and possible invalidate TXs on them) + * + * + * Case 3) + * + * A-B-C-D-E(TIP) ... NEW + * + * 1) Get NEW.prev recusively until existing block + * then case 0) / 1) / 2) + * + */ - Sync.prototype.storeBlock = function(block, cb) { + Sync.prototype.storeTipBlock = function(b, cb) { + var self = this; + var oldTip, oldNext, needReorg = true; + var newPrev = b.previousblockhash; + var updatedTxs, updatedAddrs; + + async.series([ + function(c) { + self.txDb.createFromBlock(b, function(err, txs, addrs) { + updatedTxs = txs; + updatedAddrs = addrs; + return c(err); + }); + }, + function(c) { + self.bDb.getTip(function(err, val) { + oldTip = val; + if (typeof oldTip === 'undefined' || newPrev === oldTip) { + needReorg = false; + } + return c(); + }); + }, + function(c) { + if (!needReorg) return c(); + + self.bDb.getNext( newPrev, function(err, val) { + if (err) return c(err); + oldNext = val; + return c(); + }); + }, + function(c) { + self.bDb.add(b, c); + }, + function(c) { + if (!needReorg) return c(); + + console.log('NEW TIP: %s NEED REORG', b.hash, oldTip); + // TODO should modify updatedTxs and addrs. + self.processReorg(oldTip, oldNext, newPrev, cb); + }, + function(c) { + self.bDb.setNext(newPrev, b.hash, function(err) { + return c(err); + }); + }], + function(err) { + self._handleBroadcast(b, updatedTxs, updatedAddrs); + return cb(err); + }); + }; + + + + Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { var self = this; - self.txDb.createFromBlock(block, function(err, insertedTxs, updateAddrs) { - if (err) return cb(err); + var newPrevExisted, orphanizeFrom; - self.bDb.add(block, function(err){ - if (err) return cb(err); - self._handleBroadcast(block, insertedTxs, updateAddrs); - return cb(); - }); + async.series([ + function(c) { + self.bDb.has(newPrev, function(err, ret) { + newPrevExisted = ret; + return c(); + }); + }, + function(c) { + if (newPrevExisted) return c(); + console.log('[BlockDb.js.133] case 3) not implemented yet in reorg'); //TODO + process.exit(1); + }, + function(c) { + self.bDb.isMain(newPrev, function(err,val) { + if (!val) return c(); + // case 1 + orphanizeFrom = oldNext; + return c(err); + }); + }, + function(c) { + if (orphanizeFrom) return c(); + + self.setBranchConnectedBackwards(newPrev, function(err, yHash) { + if (err) return c(err); + self.bDb.getNext(yHash, function(err, yHashNext) { + orphanizeFrom = yHashNext; + return c(err); + }); + }); + }, + function(c) { + if (!orphanizeFrom) return c(); + self.setBranchOrphan(orphanizeFrom, function(err) { + return c(err); + }); + }, + ], + function(err) { + return cb(err); + }); + }; + + Sync.prototype.setBranchOrphan = function(fromHash, cb) { + var self = this, + hashInterator = fromHash; + + async.whilst( + function() { return hashInterator; }, + function(c) { + self.setBlockMain(hashInterator, false, function(err) { + if (err) return cb(err); + self.bDb.getNext(hashInterator, function (err, val) { + hashInterator = val; + return c(err); + }); + }); + }, cb); + }; + + Sync.prototype.setBlockMain = function(hash, isMain, cb) { + var self = this; + + self.bDb.setMain(hash, isMain, function(err) { + if (err) return cb(err); + return self.txDb.handleBlockChange(hash, isMain, cb); }); }; - Sync.prototype.checkOrphan = function(fromBlock, toBlock, c) { - var self = this; + Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { + var self = this, + hashInterator = fromHash, + isMain; - var hash = fromBlock; - - var co = 0; - var limit = 10; - var cont = 1; - - async.whilst( - function () { - if (++co > limit) { -console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO - } - return cont && hash && hash !== toBlock && co < limit; - }, - function (w_c) { - //check with RPC if the block is mainchain - self.bDb.fromHashWithInfo(hash, function (err, info) { - - if (!info) { - console.log('[Sync.js.107:hash:ORPHAN]',hash); //TODO - self.txDb.setOrphan(hash, function(err) { - if (err) return w_c(err); - self.bDb.setOrphan(hash, function(err, prevHash){ - hash = prevHash; - return w_c(err); - }); + async.doWhilst( + function(c) { + self.setConnected(hashInterator, function (err) { + if (err) return c(err); + self.bDb.getPrev(hashInterator, function (err, val) { + if (err) return c(err); + hashInterator = val; + self.bDb.isMain(hashInterator, function (err, val) { + isMain = val; + return c(); }); - } - else { - console.log('[Sync.js.107:hash:NOT ORPHAN]',hash); //TODO - cont = 0; - return w_c(); - } + }); }); }, - function (err) { - return c(err); - } - ); + function() { return hashInterator; }, cb); }; - Sync.prototype._handleBroadcast = function(hash, inserted_txs, updated_addrs) { + Sync.prototype._handleBroadcast = function(hash, updatedTxs, updatedAddrs) { var self = this; if (hash && self.opts.broadcast_blocks) { sockets.broadcast_block({hash: hash}); } - if (inserted_txs && self.opts.broadcast_txs) { - inserted_txs.forEach(function(tx) { + if (updatedTxs && self.opts.broadcast_txs) { + updatedTxs.forEach(function(tx) { sockets.broadcast_tx(tx); }); } - if (updated_addrs && self.opts.broadcast_addresses) { - updated_addrs.forEach(function(addr, txs){ + if (updatedAddrs && self.opts.broadcast_addresses) { + updatedAddrs.forEach(function(addr, txs){ txs.forEach(function(addr, t){ sockets.broadcast_address_tx(addr, {'txid': t}); @@ -120,15 +241,13 @@ console.log('[Sync.js.109] WARN: Reach reog depth limit'); //TODO } }; - - Sync.prototype.storeTxs = function(txs, cb) { var self = this; - self.txDb.createFromArray(txs, null, function(err, inserted_txs, updated_addrs) { + self.txDb.createFromArray(txs, null, function(err, updatedTxs, updatedAddrs) { if (err) return cb(err); - self._handleBroadcast(null, inserted_txs, updated_addrs); + self._handleBroadcast(null, updatedTxs, updatedAddrs); return cb(err); }); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 0c562b3..d2fa9ef 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -5,15 +5,18 @@ require('classtool'); function spec(b) { - // blockHash -> txid mapping (to reorgs) - var ROOT = 'tx-b-'; //tx-b-- => 1/0 (connected or not) + // blockHash -> txid mapping + var IN_BLK_PREFIX = 'tx-b-'; //tx-b-- => 1/0 (connected or not) + + // Only for orphan blocks +// var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1/0 (connected or not) // to show tx outs - var OUTS_ROOT = 'txouts-'; //txouts-- => [addr, btc_sat] + var OUTS_PREFIX = 'txouts-'; //txouts-- => [addr, btc_sat] // to sum up addr balance - var ADDR_ROOT = 'txouts-addr-'; //txouts-addr---- => + btc_sat - var SPEND_ROOT = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] + var ADDR_PREFIX = 'txouts-addr-'; //txouts-addr---- => + btc_sat + var SPEND_PREFIX = 'txouts-spend-';//txouts-spend-- => [txid(in),n(in),ts] // TODO: use bitcore networks module var genesisTXID = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; @@ -51,7 +54,7 @@ function spec(b) { // TransactionDb.prototype.fromTxIdOne = function(txid, cb) { TODO TransactionDb.prototype.has = function(txid, cb) { - var k = OUTS_ROOT + txid; + var k = OUTS_PREFIX + txid; db.get(k, function (err,val) { var ret; @@ -69,7 +72,7 @@ function spec(b) { TransactionDb.prototype.fromTxId = function(txid, cb) { - var k = OUTS_ROOT + txid; + var k = OUTS_PREFIX + txid; var ret=[]; // outs. @@ -87,7 +90,7 @@ function spec(b) { return cb(err); }) .on('end', function () { - var k = SPEND_ROOT + txid; + var k = SPEND_PREFIX + txid; var l = ret.length; db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { @@ -174,7 +177,7 @@ function spec(b) { TransactionDb.prototype.fromTxIdN = function(txid, n, cb) { - var k = OUTS_ROOT + txid + '-' + n; + var k = OUTS_PREFIX + txid + '-' + n; db.get(k, function (err,val) { if (err && err.notFound) { @@ -206,7 +209,7 @@ function spec(b) { TransactionDb.prototype.fromAddr = function(addr, cb) { var self = this; - var k = ADDR_ROOT + addr; + var k = ADDR_PREFIX + addr; var ret=[]; db.createReadStream({start: k, end: k + '~'}) @@ -226,7 +229,7 @@ function spec(b) { .on('end', function () { async.each(ret, function(o, e_c) { - var k = SPEND_ROOT + o.txid + '-' + o.index; + var k = SPEND_PREFIX + o.txid + '-' + o.index; db.get(k, function(err, val) { if (err && err.notFound) err=null; if (err || !val) return e_c(err); @@ -238,7 +241,7 @@ function spec(b) { return e_c(); }); }, - function(err) { + function() { async.each(ret, function(o, e_c){ self.fillConfirmations(o,e_c); },function(err) { @@ -255,16 +258,16 @@ function spec(b) { async.series([ function(c) { db.createReadStream({ - start: OUTS_ROOT + txid, - end: OUTS_ROOT + txid + '~', + start: OUTS_PREFIX + txid, + end: OUTS_PREFIX + txid + '~', }).pipe( db.createWriteStream({type:'del'}) ).on('close', c); }, function(c) { db.createReadStream({ - start: SPEND_ROOT + txid, - end: SPEND_ROOT + txid + '~' + start: SPEND_PREFIX + txid, + end: SPEND_PREFIX + txid + '~' }) .pipe( db.createWriteStream({type:'del'}) @@ -315,7 +318,8 @@ function spec(b) { }; - TransactionDb.prototype.add = function(tx, cb) { + + TransactionDb.prototype.add = function(tx, blockhash, cb) { var self = this; var addrs = []; var is_new = true; @@ -331,7 +335,7 @@ function spec(b) { async.forEachLimit(tx.vin, CONCURRENCY, function(i, next_out) { db.batch() - .put( SPEND_ROOT + i.txid + '-' + i.vout , + .put( SPEND_PREFIX + i.txid + '-' + i.vout , tx.txid + ':' + i.n + ':' + ts) .write(next_out); }, @@ -355,15 +359,15 @@ function spec(b) { ! o.scriptPubKey.addresses[1] // TODO : not supported ){ // This is only to broadcast (WIP) - // if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { - // addrs.push(o.scriptPubKey.addresses[0]); - // } + if (addrs.indexOf(o.scriptPubKey.addresses[0]) === -1) { + addrs.push(o.scriptPubKey.addresses[0]); + } var addr = o.scriptPubKey.addresses[0]; var sat = Math.round(o.value * util.COIN); db.batch() - .put( OUTS_ROOT + tx.txid + '-' + o.n, addr + ':' + sat) - .put( ADDR_ROOT + addr + '-' + ts + '-' + tx.txid + + .put( OUTS_PREFIX + tx.txid + '-' + o.n, addr + ':' + sat) + .put( ADDR_PREFIX + addr + '-' + ts + '-' + tx.txid + '-' + o.n, sat) .write(next_out); @@ -385,33 +389,32 @@ function spec(b) { } return p_c(); }); - }], function(err) { + }, + function (p_c) { + if (!blockhash) return p_c(); + return self.setConfirmation(tx.txid,blockhash, true, p_c); + }, + ], function(err) { return cb(err, addrs, is_new); }); }; - TransactionDb.prototype.deleteConfirmation = function(txId, blockHash, c) { + + TransactionDb.prototype.setConfirmation = function(txId, blockHash, confirmed, c) { if (!blockHash) return c(); - db.put(ROOT + txId + '-' + blockHash, 0, function(err) { - return c(err); - }); + confirmed = confirmed ? 1 : 0; + + db.batch() + .put(IN_BLK_PREFIX + txId + '-' + blockHash, confirmed) + .write(c); }; - TransactionDb.prototype.addConfirmation = function(txId, blockHash, c) { - if (!blockHash) return c(); - - db.put(ROOT + txId + '-' + blockHash, 1, function(err) { - return c(err); - }); - }; - - - // This slowdown addr balance calculation 100% + // This slowdown addr balance calculation by 100% TransactionDb.prototype.isConfirmed = function(txId, c) { - var k = ROOT + txId; + var k = IN_BLK_PREFIX + txId; var ret = false; db.createReadStream({start: k, end: k + '~'}) @@ -422,11 +425,34 @@ function spec(b) { return c(err); }) .on('end', function (err) { - return c(err,ret); }); }; + TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { + var k = IN_BLK_PREFIX; + var toChange = []; + console.log('Searching Txs from block:' + hash); + + // This is slow, but prevent us to create a new block->tx index. + db.createReadStream({start: k, end: k + '~'}) + .on('data', function (data) { + if (data.key.indexOf(hash)>=0) + toChange.push({ + type: 'put', + key: data.key, + value: isMain?1:0, + }); + }) + .on('error', function (err) { + return cb(err); + }) + .on('end', function (err) { + if (err) return cb(err); + console.log('\t%s %d Txs', isMain?'Confirming':'Invalidating',toChange.length); + db.batch(toChange, cb); + }); + }; // txs can be a [hashes] or [txObjects] TransactionDb.prototype.createFromArray = function(txs, blockHash, next) { @@ -435,7 +461,6 @@ function spec(b) { if (!txs) return next(); // TODO - var insertedTxs = []; var updatedAddrs = {}; async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { @@ -448,31 +473,17 @@ function spec(b) { TransactionRpc.getRpcInfo(t, function(err, inInfo) { if (!inInfo) return each_cb(err); - self.add(inInfo, function(err) { - if (err) return each_cb(err); - - // This could mean that the TX was mined since we first received. - if (blockHash && inInfo.blockhash !== blockHash) - console.log('WARN in tx %s: different blockHashses: %s vs %s', - t, blockHash, inInfo.blockhash); - insertedTxs.push(t); - - return self.addConfirmation(t,inInfo.blockhash, each_cb); - }); + return self.add(inInfo, blockHash,each_cb); }); } else { - self.add(t, function(err) { - if (err || !blockHash) return each_cb(err); - - return self.addConfirmation(t,blockHash, each_cb); - }); + return self.add(t, blockHash, each_cb); } }, function(err) { - return next(err, insertedTxs, updatedAddrs); - }); -}; + return next(err, updatedAddrs); + }); + }; TransactionDb.prototype.createFromBlock = function(b, next) { diff --git a/test/integration/addr.json b/test/integration/addr.json index b40c03b..2fcc42f 100644 --- a/test/integration/addr.json +++ b/test/integration/addr.json @@ -43,9 +43,9 @@ }, { "addr": "mzW2hdZN2um7WBvTDerdahKqRgj3md9C29", - "txApperances": 6033, - "balance": 1049.69744101, - "totalReceived": 1049.69744101, + "txApperances": 6046, + "balance": 1149.19744101, + "totalReceived": 1149.19744101, "totalSent": 0 }, { diff --git a/test/integration/sync.js b/test/integration/sync.js index 82b87c4..9433f50 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -7,86 +7,135 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; var assert = require('assert'), async = require('async'), - Sync = require('../../lib/Sync').class(); - - -var b = [ - '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //B#16 - '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743', - '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48', - '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9', - '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //B#20 -]; - -var fix = function(s,cb) { - async.each([1,2,3,4], function(i,c) { - s.bDb.setPrev(b[i],b[i-1], function() { - return c(); - }); - }, cb); -}; - -var test = function(s,cb) { - async.each([2,3,4], function(i,c) { - s.bDb.getPrev(b[i], function(err, p) { - assert.equal(p,0); - return c(); - }); - }, function() { - s.bDb.getPrev(b[1], function(err, p) { - assert.equal(p,b[0]); - return cb(); - }); - }); -}; - - - -var testNo = function(s,cb) { - async.each([2,3,4], function(i,c) { - s.bDb.getPrev(b[i], function(err, p) { - assert.equal(p,b[i-1]); - return c(); - }); - }, function() { - s.bDb.getPrev(b[1], function(err, p) { - assert.equal(p,b[0]); - return cb(); - }); - }); -}; - - + HistoricSync = require('../../lib/HistoricSync').class(); var s; +var b = [ + '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b', //0 B#16 + '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743', //1 + '000000008d55c3e978639f70af1d2bf1fe6f09cb3143e104405a599215c89a48', //2 + '000000009b3bca4909f38313f2746120129cce4a699a1f552390955da470c5a9', //3 + '00000000ede57f31cc598dc241d129ccb4d8168ef112afbdc870dc60a85f5dd3', //4 B#20 +]; +var t = [ + 'd08582d3711f75d085c618874fb0d049ae09d5ec95ec6f5abd289f4b54712c54', // TX from B#16 + '1729001087e0cebea8d14de1653d5cf59628d9746bc1ae65f776f1cbaff7ebad', + 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', + 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', + 'd45f9da73619799e9d7bd03cc290e70875ea4cbad56b8bffa15135fbbb3df9ea', //4 Tx from B20 +]; + +var test = function(cb) { + async.each([2,3,4], function(i,c) { + s.sync.bDb.getPrev(b[i], function(err, p) { + assert.equal(p,b[i-1]); + return c(); + }); + }, function() { + async.each([0,1,2,3,4], function(i,c) { + s.sync.bDb.has(b[i], function(err, p) { + assert(p); + return c(); + }); + }, function() { + async.each([0,1,2,3], function(i,c) { + s.sync.bDb.getNext(b[i], function(err, p) { + assert.equal(p,b[i+1]); + return c(); + }); + }, cb); + }); + }); +}; + describe('Sync checkOrphan', function(){ before(function(done) { - s = new Sync(); - fix(s,done); - }); - - after(function(done) { - - fix(s,function() { - s.close(done); + s = new HistoricSync(); + s.init({}, function(err) { + if (err) return done(err); + s.sync.destroy(done); }); - }); - it('checkOrphan', function(done) { - this.timeout(100000); - - s.bDb.has(b[0], function(err, has) { - assert(has); - s.bDb.has(b[1], function(err, has) { - assert(has); - s.checkOrphan(b[4],b[1], function() { - testNo(s,done); - }); + it('simple RPC forward syncing', function(done) { + s.getPrevNextBlock(s.genesis,b[4], { + next: true, + }, function(err) { + if (err) return done(err); + test(done); }); - }); + }); + + + it('reorg, case 1', function(done) { + var case1 = { + hash: '0000000000000000000000000000000000000000000000000000000000000001', + tx: [ '1000000000000000000000000000000000000000000000000000000000000000' ], + time: 1296690099, + previousblockhash: b[2], + }; + + async.series([ + function (c) { + s.sync.txDb.isConfirmed(t[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case1, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[2], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[3], function(err,is) { + assert(!err); + assert(!is, b[3] + 'should not be on main chain'); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + + ], done ); }); }); + diff --git a/util/sync.js b/util/sync.js index 7a9a094..0906092 100755 --- a/util/sync.js +++ b/util/sync.js @@ -18,6 +18,7 @@ program .option('-R --reverse', 'Sync backwards', 0) .option('-U --uptoexisting', 'Sync only until an existing block is found', 0) .option('-F --fromfiles', 'Sync using bitcoind .dat block files (faster)', 0) + .option('-v --verbose', 'Verbose 0/1', 0) .parse(process.argv); var historicSync = new HistoricSync(); From 97e1903dd16b2f56fa1bf8943456372dfbead3c7 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 8 Feb 2014 16:20:51 -0300 Subject: [PATCH 03/12] complete sync process from files working --- lib/BlockDb.js | 2 +- lib/HistoricSync.js | 56 +++--- lib/Sync.js | 78 +++++--- lib/TransactionDb.js | 2 +- test/integration/sync.js | 384 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 449 insertions(+), 73 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index b253ca8..05c0650 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -90,7 +90,7 @@ function spec(b) { }; BlockDb.prototype.setMain = function(hash, isMain, cb) { - if (!isMain) console.log('ORPHAN: %s',hash); + if (!isMain) console.log('\tNew orphan: %s',hash); db.put(MAIN_PREFIX + hash, isMain?1:0, function(err) { return cb(err); }); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 8534ecf..5b829e8 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -119,6 +119,11 @@ function spec() { if (self.opts.shouldBroadcast) { sockets.broadcastSyncInfo(self.info()); } + +//TODO +// if (self.syncPercentage > 10) { +// process.exit(-1); +// } }; HistoricSync.prototype.getPrevNextBlock = function(blockHash, blockEnd, scanOpts, cb) { @@ -156,7 +161,8 @@ function spec() { blockInfo = ret.result; // this is to match block retreived from file if (blockInfo.hash === self.genesis) - blockInfo.previousblockhash='0000000000000000000000000000000000000000000000000000000000000000'; + blockInfo.previousblockhash = + '0000000000000000000000000000000000000000000000000000000000000000'; } else { blockInfo = null; @@ -295,38 +301,30 @@ function spec() { HistoricSync.prototype.nextBlockFromFile = function(scanOpts, cb) { var self = this; - var blockInfo; - async.series([ - function(c) { - self.showProgress(); - return c(); - }, - function(c) { - self.getBlockFromFile(function(err, inBlockInfo) { - blockInfo = inBlockInfo; - return c(err); - }); - }, - function(c) { - self.sync.storeTipBlock(blockInfo, function(err) { - return c(err); - }); - }, - function(c) { - // continue - if (blockInfo && blockInfo.hash) { - self.syncedBlocks++; - } else - self.status = 'finished'; - return c(); - }, - ], function(err) { + self.showProgress(); + + self.getBlockFromFile(function(err, blockInfo) { if (err) { - self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', + blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + return cb(err); } - return cb(err); + + self.sync.storeTipBlock(blockInfo, function(err) { + if (blockInfo && blockInfo.hash) { + self.syncedBlocks++; + } else + self.status = 'finished'; + + if (err) { + self.setError(util.format('ERROR: @%s: %s [count: syncedBlocks: %d]', + blockInfo ? blockInfo.hash : '-', err, self.syncedBlocks)); + } + return cb(err); + }); }); + }; diff --git a/lib/Sync.js b/lib/Sync.js index e923db2..58671d8 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -62,18 +62,32 @@ function spec() { * * A-B-C-D-E(TIP) ... NEW * - * 1) Get NEW.prev recusively until existing block - * then case 0) / 1) / 2) + * NEW is ignored * */ Sync.prototype.storeTipBlock = function(b, cb) { + if (!b) return cb(); + var self = this; var oldTip, oldNext, needReorg = true; var newPrev = b.previousblockhash; var updatedTxs, updatedAddrs; async.series([ + function(c) { + self.bDb.has(b.hash, function(err, val) { + return c(err || + (val ? new Error('WARN: Ignoring already existing block:' + b.hash) : null )); + }); + }, + function(c) { + self.bDb.has(newPrev, function(err, val) { + if (!val && newPrev.match(/^0+$/)) return c(); + return c(err || + (!val ? new Error('WARN: Ignoring block with non existing prev:' + b.hash) : null )); + }); + }, function(c) { self.txDb.createFromBlock(b, function(err, txs, addrs) { updatedTxs = txs; @@ -105,9 +119,9 @@ function spec() { function(c) { if (!needReorg) return c(); - console.log('NEW TIP: %s NEED REORG', b.hash, oldTip); + console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); // TODO should modify updatedTxs and addrs. - self.processReorg(oldTip, oldNext, newPrev, cb); + self.processReorg(oldTip, oldNext, newPrev, c); }, function(c) { self.bDb.setNext(newPrev, b.hash, function(err) { @@ -115,7 +129,11 @@ function spec() { }); }], function(err) { - self._handleBroadcast(b, updatedTxs, updatedAddrs); + if (!err) self._handleBroadcast(b, updatedTxs, updatedAddrs); + if (err && err.toString().match(/WARN/) ) { + console.log(err); + err=null; + } return cb(err); }); }; @@ -125,23 +143,14 @@ function spec() { Sync.prototype.processReorg = function(oldTip, oldNext, newPrev, cb) { var self = this; - var newPrevExisted, orphanizeFrom; + var orphanizeFrom; async.series([ - function(c) { - self.bDb.has(newPrev, function(err, ret) { - newPrevExisted = ret; - return c(); - }); - }, - function(c) { - if (newPrevExisted) return c(); - console.log('[BlockDb.js.133] case 3) not implemented yet in reorg'); //TODO - process.exit(1); - }, function(c) { self.bDb.isMain(newPrev, function(err,val) { if (!val) return c(); + + console.log('# Reorg Case 1)'); // case 1 orphanizeFrom = oldNext; return c(err); @@ -150,11 +159,14 @@ function spec() { function(c) { if (orphanizeFrom) return c(); - self.setBranchConnectedBackwards(newPrev, function(err, yHash) { + console.log('# Reorg Case 2)'); + self.setBranchConnectedBackwards(newPrev, function(err, yHash, newYHashNext) { if (err) return c(err); self.bDb.getNext(yHash, function(err, yHashNext) { orphanizeFrom = yHashNext; - return c(err); + self.bDb.setNext(yHash, newYHashNext, function(err) { + return c(err); + }); }); }); }, @@ -170,6 +182,14 @@ function spec() { }); }; + Sync.prototype.setBlockMain = function(hash, isMain, cb) { + var self = this; + self.bDb.setMain(hash, isMain, function(err) { + if (err) return cb(err); + return self.txDb.handleBlockChange(hash, isMain, cb); + }); + }; + Sync.prototype.setBranchOrphan = function(fromHash, cb) { var self = this, hashInterator = fromHash; @@ -187,26 +207,19 @@ function spec() { }, cb); }; - Sync.prototype.setBlockMain = function(hash, isMain, cb) { - var self = this; - - self.bDb.setMain(hash, isMain, function(err) { - if (err) return cb(err); - return self.txDb.handleBlockChange(hash, isMain, cb); - }); - }; - Sync.prototype.setBranchConnectedBackwards = function(fromHash, cb) { var self = this, hashInterator = fromHash, + lastHash = fromHash, isMain; async.doWhilst( function(c) { - self.setConnected(hashInterator, function (err) { + self.setBlockMain(hashInterator, true, function (err) { if (err) return c(err); self.bDb.getPrev(hashInterator, function (err, val) { if (err) return c(err); + lastHash = hashInterator; hashInterator = val; self.bDb.isMain(hashInterator, function (err, val) { isMain = val; @@ -215,7 +228,12 @@ function spec() { }); }); }, - function() { return hashInterator; }, cb); + function() { return hashInterator && !isMain; }, + function(err) { + console.log('\tFound yBlock:', hashInterator); + return cb(err, hashInterator, lastHash); + } + ); }; Sync.prototype._handleBroadcast = function(hash, updatedTxs, updatedAddrs) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index d2fa9ef..734f846 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -432,7 +432,7 @@ function spec(b) { TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { var k = IN_BLK_PREFIX; var toChange = []; - console.log('Searching Txs from block:' + hash); + console.log('\tSearching Txs from block:' + hash); // This is slow, but prevent us to create a new block->tx index. db.createReadStream({start: k, end: k + '~'}) diff --git a/test/integration/sync.js b/test/integration/sync.js index 9433f50..9d4e47c 100644 --- a/test/integration/sync.js +++ b/test/integration/sync.js @@ -1,6 +1,5 @@ #!/usr/bin/env node 'use strict'; - process.env.NODE_ENV = process.env.NODE_ENV || 'development'; @@ -20,9 +19,9 @@ var b = [ ]; var t = [ 'd08582d3711f75d085c618874fb0d049ae09d5ec95ec6f5abd289f4b54712c54', // TX from B#16 - '1729001087e0cebea8d14de1653d5cf59628d9746bc1ae65f776f1cbaff7ebad', - 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', - 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', + '1729001087e0cebea8d14de1653d5cf59628d9746bc1ae65f776f1cbaff7ebad', //1 + 'cf53d7ccd83a099acfbc319ee10c1e3b10e3d42ba675b569fdd6b69cb8d2db4e', //2 + '73a4988adf462b6540cfa59097804174b298cfa439f73c1a072c2c6fbdbe57c7', //3 'd45f9da73619799e9d7bd03cc290e70875ea4cbad56b8bffa15135fbbb3df9ea', //4 Tx from B20 ]; @@ -49,7 +48,34 @@ var test = function(cb) { }); }; -describe('Sync checkOrphan', function(){ +/* + * TEST CASES + * + * Blocks: 0-1-2-3-4 + * case 1) + * 0-1-2-3-4 + * \ + * C1* + * + * case 2) + * 0-1-2---3-4 + * \ \ + * C1 C2* + * + * case 2b) + * 0-1-2---3-4 + * \ \ + * C1 C2-C2b(TX=C1.TX)* + * case 2c) + * 0-1-2---3-4 + * \ \ + * C1 C2-C2b(TX=C1.TX) + * \ + * C2c(TX=C2.TX)* + * + */ + +describe('Sync Reorgs', function(){ before(function(done) { s = new HistoricSync(); @@ -68,15 +94,15 @@ describe('Sync checkOrphan', function(){ }); }); + var case1 = { + hash: '0000000000000000000000000000000000000000000000000000000000000001', + tx: [ 'f0596531810160d090813673b4a397f4617aab44eb26c7f06c8a766eac984b91' ], + time: 1296690099, + previousblockhash: b[2], + }; + it('reorg, case 1', function(done) { - var case1 = { - hash: '0000000000000000000000000000000000000000000000000000000000000001', - tx: [ '1000000000000000000000000000000000000000000000000000000000000000' ], - time: 1296690099, - previousblockhash: b[2], - }; - async.series([ function (c) { s.sync.txDb.isConfirmed(t[0], function(err,is) { @@ -85,6 +111,13 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, function (c) { s.sync.txDb.isConfirmed(t[4], function(err,is) { assert(!err); @@ -126,6 +159,13 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, function (c) { s.sync.txDb.isConfirmed(t[4], function(err,is) { assert(!err); @@ -133,6 +173,326 @@ describe('Sync checkOrphan', function(){ return c(); }); }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + ], done ); + }); + + it('reorg, case 1 (repeat)', function(done) { + s.sync.storeTipBlock(case1, function(err) { + assert(!err, 'shouldnt return error' + err); + return done(); + }); + }); + + var case2 = { + hash: '0000000000000000000000000000000000000000000000000000000000000002', + tx: [ '99bb359a4b12a588fcb9e59e5e8d92d593ce7a56d2ba42085fe86d9a0b4fde15' ], + time: 1296690099, + previousblockhash: b[3], + }; + + + it('reorg, case 2', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(t[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[3], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(b[4], function(err,is) { + assert(!err); + assert(!is, b[3] + 'should not be on main chain'); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case1.hash, function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case2.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is, 'transaction t[3] should be valid:' + t[3]); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.bDb.getNext(b[2], function(err, val) { + assert(!err); + assert.equal(val,b[3]); + return c(); + }); + }, + + + + ], done ); + }); + + + var case2b = { + hash: '0000000000000000000000000000000000000000000000000000000000000003', + tx: case1.tx, + time: 1296690099, + previousblockhash: case2.hash, + }; + + it('reorg, case 2b', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(case2b.tx[0], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2b, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(is, 'transaction t[3] should be valid:' + t[3]); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2b.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + ], done ); + }); + + + + var case2c = { + hash: '0000000000000000000000000000000000000000000000000000000000000004', + tx: case2.tx, + time: 1296690099, + previousblockhash: case1.hash, + }; + + it('reorg, case 2c', function(done) { + async.series([ + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.isMain(case1.hash, function(err,is) { + assert(!err); + assert(!is, 'case1 block shouldnt be main:' + case1.hash); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); //It was there before (from case2) + return c(); + }); + }, + function (c) { + s.sync.storeTipBlock(case2c, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case1.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case2c.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is, 'TX t[3]: shouldnt be confirmed:' + t[3] +':'+ is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + + ], done ); + }); + + var case3 = { + hash: '0000000000000000000000000000000000000000000000000000000000000005', + tx: case2.tx, + time: 1296690099, + previousblockhash: '666', + }; + + it('reorg, case 3)', function(done) { + async.series([ + function (c) { + s.sync.storeTipBlock(case3, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + + //shoudnt change anything + function (c) { + s.sync.txDb.isConfirmed(case1.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case1.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(case2c.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2c.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[3], function(err,is) { + assert(!err); + assert(!is, 'TX t[3]: shouldnt be confirmed:' + t[3] +':'+ is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(t[4], function(err,is) { + assert(!err); + assert(!is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(case2.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, ], done ); }); From b5ad5643b1f54985907b67198e0084b5de1cb053 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 8 Feb 2014 20:09:54 -0300 Subject: [PATCH 04/12] sync working embedded in http server --- insight.js | 31 +++++++++------- lib/HistoricSync.js | 6 +-- lib/PeerSync.js | 19 ++++------ lib/Sync.js | 24 +++++++----- lib/TransactionDb.js | 7 ++-- test/integration/{sync.js => 99-sync.js} | 47 ++++++++++++++++++++++++ test/integration/block.js | 19 +--------- util/p2p.js | 1 + 8 files changed, 95 insertions(+), 59 deletions(-) rename test/integration/{sync.js => 99-sync.js} (91%) diff --git a/insight.js b/insight.js index b330942..0b811f7 100644 --- a/insight.js +++ b/insight.js @@ -40,6 +40,20 @@ var walk = function(path) { walk(models_path); +/** + * p2pSync process + */ +if (!config.disableP2pSync) { + var ps = new PeerSync(); + ps.init({ + broadcast_txs: true, + broadcast_address_tx: true, + broadcast_blocks: true, + }, function() { + ps.run(); + }); +} + /** * historic_sync process */ @@ -60,25 +74,16 @@ if (!config.disableHistoricSync) { historicSync.smartImport({}, function(err){ var txt = 'ended.'; if (err) txt = 'ABORTED with error: ' + err.message; + else + ps.allowReorgs = true; + console.log('[historic_sync] ' + txt, historicSync.info()); }); } }); } -/** - * p2pSync process - */ -if (!config.disableP2pSync) { - var ps = new PeerSync(); - ps.init({ - broadcast_txs: true, - broadcast_address_tx: true, - broadcast_blocks: true, - }, function() { - ps.run(); - }); -} + //express settings require('./config/express')(expressApp, historicSync); diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 5b829e8..6e477e3 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -161,8 +161,7 @@ function spec() { blockInfo = ret.result; // this is to match block retreived from file if (blockInfo.hash === self.genesis) - blockInfo.previousblockhash = - '0000000000000000000000000000000000000000000000000000000000000000'; + blockInfo.previousblockhash = self.network.genesisBlock.prev_hash.toString('hex'); } else { blockInfo = null; @@ -175,7 +174,8 @@ function spec() { function(c) { if (existed) return c(); - self.sync.storeTipBlock(blockInfo, function(err) { + // When storing files from RPC recusively, reorgs are disabled + self.sync.storeTipBlock(blockInfo, false, function(err) { return c(err); }); }], function(err) { diff --git a/lib/PeerSync.js b/lib/PeerSync.js index 4f7a493..cb09b37 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -23,6 +23,7 @@ function spec() { this.verbose = opts.verbose; this.peerdb = undefined; this.sync = new Sync(); + this.allowReorgs = false; this.PeerManager = require('bitcore/PeerManager').createClass({ network: network @@ -60,38 +61,34 @@ function spec() { if (this.verbose) { console.log('[p2p_sync] Handle tx: ' + tx.hash); } + this.sync.storeTxs([tx.hash], function(err) { if (err) { - console.log('[PeerSync.js.71:err:]',err); //TODO console.log('[p2p_sync] Error in handle TX: ' + JSON.stringify(err)); } }); }; PeerSync.prototype.handle_block = function(info) { + var self = this; var block = info.message.block; var blockHash = coinUtil.formatHashFull(block.calcHash()); - if (this.verbose) { - console.log('[p2p_sync] Handle block: ' + blockHash); - } + + console.log('[p2p_sync] Handle block: %s (allowReorgs: %s)', blockHash, self.allowReorgs); var tx_hashes = block.txs.map(function(tx) { return coinUtil.formatHashFull(tx.hash); }); + // Reorgs enabled. this.sync.storeTipBlock({ 'hash': blockHash, 'tx': tx_hashes, 'previousblockhash': coinUtil.formatHashFull(block.prev_hash), - }, - function(err) { + }, self.allowReorgs, function(err) { if (err) { console.log('[p2p_sync] Error in handle Block: ' + err); } - - // Check for reorgs... - // The previous last block hash - // if different => call }); }; @@ -112,8 +109,6 @@ function spec() { }); this.peerman.on('connection', function(conn) { - -console.log('[PeerSync.js.113]'); //TODO conn.on('inv', self.handle_inv.bind(self)); conn.on('block', self.handle_block.bind(self)); conn.on('tx', self.handle_tx.bind(self)); diff --git a/lib/Sync.js b/lib/Sync.js index 58671d8..a90623f 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -66,11 +66,16 @@ function spec() { * */ - Sync.prototype.storeTipBlock = function(b, cb) { + Sync.prototype.storeTipBlock = function(b, allowReorgs, cb) { + + if (typeof allowReorgs === 'function') { + cb = allowReorgs; + allowReorgs = true; + } if (!b) return cb(); var self = this; - var oldTip, oldNext, needReorg = true; + var oldTip, oldNext, needReorg = false; var newPrev = b.previousblockhash; var updatedTxs, updatedAddrs; @@ -82,6 +87,8 @@ function spec() { }); }, function(c) { + if (!allowReorgs) return c(); + self.bDb.has(newPrev, function(err, val) { if (!val && newPrev.match(/^0+$/)) return c(); return c(err || @@ -89,18 +96,17 @@ function spec() { }); }, function(c) { - self.txDb.createFromBlock(b, function(err, txs, addrs) { - updatedTxs = txs; + self.txDb.createFromBlock(b, function(err, addrs) { + updatedTxs = b.tx; updatedAddrs = addrs; return c(err); }); }, function(c) { + if (!allowReorgs) return c(); self.bDb.getTip(function(err, val) { oldTip = val; - if (typeof oldTip === 'undefined' || newPrev === oldTip) { - needReorg = false; - } + if (oldTip && newPrev !== oldTip) needReorg = true; return c(); }); }, @@ -262,10 +268,10 @@ function spec() { Sync.prototype.storeTxs = function(txs, cb) { var self = this; - self.txDb.createFromArray(txs, null, function(err, updatedTxs, updatedAddrs) { + self.txDb.createFromArray(txs, null, function(err, updatedAddrs) { if (err) return cb(err); - self._handleBroadcast(null, updatedTxs, updatedAddrs); + self._handleBroadcast(null, txs, updatedAddrs); return cb(err); }); }; diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 734f846..9e5da43 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -183,7 +183,7 @@ function spec(b) { if (err && err.notFound) { err = null; } - var a = val.split(':'); + var a = val?val.split(':'):[null,null]; return cb(err, a[0], parseInt(a[1])); }); }; @@ -460,8 +460,7 @@ function spec(b) { if (!txs) return next(); - // TODO - var updatedAddrs = {}; + var updatedAddrs = []; // TODO async.forEachLimit(txs, CONCURRENCY, function(t, each_cb) { if (typeof t === 'string') { @@ -473,7 +472,7 @@ function spec(b) { TransactionRpc.getRpcInfo(t, function(err, inInfo) { if (!inInfo) return each_cb(err); - return self.add(inInfo, blockHash,each_cb); + return self.add(inInfo, blockHash, each_cb); }); } else { diff --git a/test/integration/sync.js b/test/integration/99-sync.js similarity index 91% rename from test/integration/sync.js rename to test/integration/99-sync.js index 9d4e47c..23ae44d 100644 --- a/test/integration/sync.js +++ b/test/integration/99-sync.js @@ -496,6 +496,53 @@ describe('Sync Reorgs', function(){ ], done ); }); + + var p2p = { + hash: '0000000000000000000000000000000000000000000000000000000000000006', + tx: ['f6c2901f39fd07f2f2e503183d76f73ecc1aee9ac9216fde58e867bc29ce674e'], + time: 1296690099, + previousblockhash: '111', + }; + + it('p2p, no reorg allowed', function(done) { + async.series([ + function (c) { + s.sync.storeTipBlock(p2p, false, function(err) { + assert(!err, 'shouldnt return error' + err); + return c(); + }); + }, + function (c) { + s.sync.bDb.has(p2p.hash, function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.txDb.isConfirmed(p2p.tx[0], function(err,is) { + assert(!err); + assert(is); + return c(); + }); + }, + function (c) { + s.sync.bDb.getNext(p2p.hash, function(err,v) { + assert(!err); + assert.equal(v,p2p.nextblockhash); + return c(); + }); + }, + function (c) { + s.sync.bDb.getNext(p2p.previousblockhash, function(err,v) { + assert(!err); + assert.equal(v,p2p.hash); + return c(); + }); + }, + + ], done ); + }); }); diff --git a/test/integration/block.js b/test/integration/block.js index e61655b..1a4ed86 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -8,7 +8,7 @@ var TESTING_BLOCK = '000000000185678d3d7ecc9962c96418174431f93fe20bf216d55652724 var assert = require('assert'), - config = require('../../config/config'), +// config = require('../../config/config'), BlockDb = require('../../lib/BlockDb').class(); var bDb; @@ -36,22 +36,5 @@ describe('BlockDb fromHashWithInfo', function(){ done(); }); }); - it('setOrphan', function(done) { - var b16 = '00000000c4cbd75af741f3a2b2ff72d9ed4d83a048462c1efe331be31ccf006b'; - var b17 = '00000000fe198cce4c8abf9dca0fee1182cb130df966cc428ad2a230df8da743'; - - bDb.has(b17, function(err, has) { - assert(has); - bDb.setOrphan(b17, function(err, oldPrev) { - assert.equal(oldPrev, b16); - bDb.setPrev(b17, b16, function(err, oldPrev) { - bDb.getPrev(b17, function(err, p) { - assert.equal(p, b16); - done(); - }); - }); - }); - }); - }); }); diff --git a/util/p2p.js b/util/p2p.js index d3bd665..6954dde 100755 --- a/util/p2p.js +++ b/util/p2p.js @@ -11,6 +11,7 @@ var program = require('commander'); program .version(PROGRAM_VERSION) .option('-N --network [testnet]', 'Set bitcoin network [testnet]', 'testnet') + .option('-V --verbose', 'Verbose', 1) .parse(process.argv); var ps = new PeerSync(); From 165b8f53cab512f87922779759e58a3ae1866ceb Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sat, 8 Feb 2014 22:36:03 -0300 Subject: [PATCH 05/12] add block reward to block page, remove it from coinbase --- lib/BlockDb.js | 4 ++-- lib/TransactionRpc.js | 4 ---- public/views/block.html | 5 +++++ public/views/transaction/tx.html | 1 - 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 05c0650..97e2a32 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -139,8 +139,8 @@ function spec(b) { if (err && err.code === -5) return cb(); if (err) return cb(err); - info.reward = BitcoreBlock.getBlockValue(info.height) / util.COIN ; - + if (info.result.height) + info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / util.COIN ; return cb(null, { hash: hash, info: info.result, diff --git a/lib/TransactionRpc.js b/lib/TransactionRpc.js index cd439ec..0eaef27 100644 --- a/lib/TransactionRpc.js +++ b/lib/TransactionRpc.js @@ -24,10 +24,6 @@ function spec(b) { // Inputs if (tx.isCoinBase()) { info.isCoinBase = true; - - var reward = BitcoreBlock.getBlockValue(info.height) / util.COIN; - info.vin[0].reward = reward; - info.valueIn = reward; } var n =0; diff --git a/public/views/block.html b/public/views/block.html index 07662f5..09646fd 100644 --- a/public/views/block.html +++ b/public/views/block.html @@ -47,6 +47,11 @@ Height {{block.height}} + + Block Reward + {{$root.currency.getConvertion(block.reward)}} + + Timestamp {{block.time * 1000 | date:'medium'}} diff --git a/public/views/transaction/tx.html b/public/views/transaction/tx.html index e5eddc7..4a64860 100644 --- a/public/views/transaction/tx.html +++ b/public/views/transaction/tx.html @@ -14,7 +14,6 @@
-
{{$root.currency.getConvertion(vin.reward)}}
No Inputs (Newly Generated Coins)
From 2eacaf3bf9bec69786c11e2be5ef756380203365 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 9 Feb 2014 09:34:47 -0300 Subject: [PATCH 06/12] add index to handle txs in orphan blocks faster --- app/models/Address.js | 5 +++-- lib/TransactionDb.js | 12 +++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/Address.js b/app/models/Address.js index 657b67e..cdb820b 100644 --- a/app/models/Address.js +++ b/app/models/Address.js @@ -21,8 +21,7 @@ function spec() { var a = new BitcoreAddress(addrStr); a.validate(); this.addrStr = addrStr; - - + Object.defineProperty(this, 'totalSent', { get: function() { return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN); @@ -56,6 +55,8 @@ function spec() { Address.prototype.update = function(next) { var self = this; + if (!self.addrStr) return next(); + var db = new TransactionDb(); async.series([ function (cb) { diff --git a/lib/TransactionDb.js b/lib/TransactionDb.js index 9e5da43..3b4ee25 100644 --- a/lib/TransactionDb.js +++ b/lib/TransactionDb.js @@ -9,7 +9,7 @@ function spec(b) { var IN_BLK_PREFIX = 'tx-b-'; //tx-b-- => 1/0 (connected or not) // Only for orphan blocks -// var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1/0 (connected or not) + var FROM_BLK_PREFIX = 'tx-'; //tx-- => 1 // to show tx outs var OUTS_PREFIX = 'txouts-'; //txouts-- => [addr, btc_sat] @@ -408,6 +408,7 @@ function spec(b) { db.batch() .put(IN_BLK_PREFIX + txId + '-' + blockHash, confirmed) + .put(FROM_BLK_PREFIX + blockHash + '-' + txId, 1) .write(c); }; @@ -430,19 +431,20 @@ function spec(b) { }; TransactionDb.prototype.handleBlockChange = function(hash, isMain, cb) { - var k = IN_BLK_PREFIX; var toChange = []; console.log('\tSearching Txs from block:' + hash); + var k = FROM_BLK_PREFIX + hash; + var k2 = IN_BLK_PREFIX; // This is slow, but prevent us to create a new block->tx index. db.createReadStream({start: k, end: k + '~'}) .on('data', function (data) { - if (data.key.indexOf(hash)>=0) + var ks = data.key.split('-'); toChange.push({ + key: k2 + ks[2] + '-' + ks[1], type: 'put', - key: data.key, value: isMain?1:0, - }); + }); }) .on('error', function (err) { return cb(err); From c6d137f856c438d0251bc955e5e7ad702dbf7fad Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Sun, 9 Feb 2014 15:59:28 -0300 Subject: [PATCH 07/12] fixed: blocklist by date with limit. when arrive new block, reload last 5 blocks on homepage. The same for new txs. --- app/controllers/blocks.js | 8 ++--- lib/BlockDb.js | 7 ++--- public/js/controllers/index.js | 54 +++++++++++----------------------- test/integration/blocklist.js | 7 ++--- 4 files changed, 27 insertions(+), 49 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index f66aa35..4eae98e 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -72,7 +72,6 @@ console.log('[blocks.js.60]: could not get %s from RPC. Orphan? Error?', blockha * List of blocks by date */ exports.list = function(req, res) { - var limit = req.query.limit || -1; var isToday = false; //helper to convert timestamps to yyyy-mm-dd format @@ -103,14 +102,15 @@ exports.list = function(req, res) { var prev = formatTimestamp(new Date((gte - 86400) * 1000)); var next = formatTimestamp(new Date(lte * 1000)); - bdb.getBlocksByDate(gte, lte, limit, function(err, blocks) { + bdb.getBlocksByDate(gte, lte, function(err, blocks) { if (err) { res.status(500).send(err); } else { var blockshashList = []; - for(var i=0;i= parseInt(TRANSACTION_DISPLAYED, 10)) { - $scope.txs = $scope.txs.splice(0, TRANSACTION_DISPLAYED); - } - }); + _getTransactions(); }); socket.on('block', function(block) { - var blockHash = block.hash.toString(); - console.log('Block received! ' + JSON.stringify(block)); - if (parseInt($scope.blocks.length, 10) > parseInt(BLOCKS_DISPLAYED, 10) - 1) { - $scope.blocks.pop(); - } - - _getBlock(blockHash); + var blockHash = block.hash.hash.toString(); + console.log('Block received! ' + JSON.stringify(blockHash)); + _getBlocks(); }); $scope.humanSince = function(time) { @@ -57,18 +47,8 @@ angular.module('insight.system').controller('IndexController', }; $scope.index = function() { - Blocks.get({ - limit: BLOCKS_DISPLAYED - }, function(res) { - $scope.blocks = res.blocks; - $scope.blocksLength = res.lenght; - }); - - Transactions.get({ - limit: TRANSACTION_DISPLAYED - }, function(res) { - $scope.txs = res.txs; - }); + _getBlocks(); + _getTransactions(); }; $scope.txs = []; diff --git a/test/integration/blocklist.js b/test/integration/blocklist.js index d1ed33d..628ef0d 100644 --- a/test/integration/blocklist.js +++ b/test/integration/blocklist.js @@ -3,11 +3,10 @@ process.env.NODE_ENV = process.env.NODE_ENV || 'development'; -var TESTING_BLOCK0 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'; -var TESTING_BLOCK1 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; +var TESTING_BLOCK0 = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; +var TESTING_BLOCK1 = '000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943'; var START_TS = 1; var END_TS = '1296688928~'; // 2/2/2011 23:23PM -var LIMIT = 2; var assert = require('assert'), BlockDb = require('../../lib/BlockDb').class(); @@ -24,7 +23,7 @@ describe('BlockDb getBlocksByDate', function(){ it('Get Hash by Date', function(done) { - bDb.getBlocksByDate(START_TS, END_TS, LIMIT, function(err, list) { + bDb.getBlocksByDate(START_TS, END_TS, function(err, list) { if (err) done(err); assert(list, 'returns list'); assert.equal(list.length,2, 'list has 2 items'); From 34f2e933b6741aaa8fffc8acd394b9ae94836068 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Sun, 9 Feb 2014 16:26:41 -0300 Subject: [PATCH 08/12] fix update of blockheight on navbar --- public/js/controllers/header.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/js/controllers/header.js b/public/js/controllers/header.js index 7147272..5b4d3a9 100755 --- a/public/js/controllers/header.js +++ b/public/js/controllers/header.js @@ -33,7 +33,7 @@ angular.module('insight.system').controller('HeaderController', }; socket.on('block', function(block) { - var blockHash = block.hash.toString(); + var blockHash = block.hash.hash.toString(); console.log('Updated Blocks Height!'); _getBlock(blockHash); }); From 1c32a43854263b7f6e659574fe220f9f1929bd82 Mon Sep 17 00:00:00 2001 From: "Ryan X. Charles" Date: Sun, 9 Feb 2014 15:59:35 -0500 Subject: [PATCH 09/12] remove lorem ipsum content and add real about content --- public/views/index.html | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/public/views/index.html b/public/views/index.html index cba1d2c..a2f48b4 100644 --- a/public/views/index.html +++ b/public/views/index.html @@ -31,9 +31,7 @@

About

-

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Quas, sint, neque harum libero eos maiores rerum rem fuga quae architecto ea incidunt dolore optio ullam sit placeat vero perferendis beatae?

-

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Blanditiis, unde quidem commodi dolor asperiores ullam molestias sit a sapiente ipsa!

-

Lorem ipsum dolor sit amet, consectetur adipisicing elit. Deserunt tempora fugiat dolorem cupiditate perspiciatis praesentium.

+

Insight is a bitcoin blockchain API for writing web wallets and other apps that need more advanced blockchain queries than provided by bitcoind RPC. Check out the source code.

@@ -58,33 +56,6 @@ -

Other Bitcoin Links

-
From 87c10a122bef4e155962d6d83b2b89baab337c29 Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 9 Feb 2014 19:33:39 -0300 Subject: [PATCH 10/12] fix broadcast --- app/controllers/socket.js | 6 ++-- app/models/Status.js | 51 ++++++++++++++++++-------------- insight.js | 10 +++---- lib/BlockDb.js | 11 +++++-- lib/HistoricSync.js | 3 +- lib/PeerSync.js | 33 +++++++-------------- lib/Sync.js | 39 +++++++++++++----------- public/js/controllers/address.js | 6 ++-- public/js/controllers/header.js | 2 +- public/js/controllers/index.js | 5 ++-- public/views/status.html | 8 ++++- 11 files changed, 91 insertions(+), 83 deletions(-) diff --git a/app/controllers/socket.js b/app/controllers/socket.js index 6f37f5f..ec9ba60 100644 --- a/app/controllers/socket.js +++ b/app/controllers/socket.js @@ -14,15 +14,15 @@ module.exports.init = function(app, io_ext) { }); }; -module.exports.broadcast_tx = function(tx) { +module.exports.broadcastTx = function(tx) { if (ios) ios.sockets.in('inv').emit('tx', tx); }; -module.exports.broadcast_block = function(block) { +module.exports.broadcastBlock = function(block) { if (ios) ios.sockets.in('inv').emit('block', block); }; -module.exports.broadcast_address_tx = function(address, tx) { +module.exports.broadcastAddressTx = function(address, tx) { if (ios) ios.sockets.in(address).emit(address, tx); }; diff --git a/app/models/Status.js b/app/models/Status.js index 517b51c..106c309 100644 --- a/app/models/Status.js +++ b/app/models/Status.js @@ -5,10 +5,12 @@ require('classtool'); function spec() { var async = require('async'); var RpcClient = require('bitcore/RpcClient').class(); + var BlockDb = require('../../lib/BlockDb').class(); var config = require('../../config/config'); var rpc = new RpcClient(config.bitcoind); function Status() { + this.bDb = new BlockDb(); } Status.prototype.getInfo = function(next) { @@ -21,7 +23,7 @@ function spec() { that.info = info.result; return cb(); }); - } + }, ], function (err) { return next(err); }); @@ -69,7 +71,8 @@ function spec() { that.bestblockhash = bbh.result; return cb(); }); - } + }, + ], function (err) { return next(err); }); @@ -77,27 +80,29 @@ function spec() { Status.prototype.getLastBlockHash = function(next) { var that = this; - - async.waterfall( - [ - function(callback){ - rpc.getBlockCount(function(err, bc){ - if (err) return callback(err); - callback(null, bc.result); - }); - }, - function(bc, callback){ - rpc.getBlockHash(bc, function(err, bh){ - if (err) return callback(err); - callback(null, bh.result); - }); - } - ], - function (err, result) { - that.lastblockhash = result; - return next(); - } - ); + that.bDb.getTip(function(err,tip) { + that.syncTipHash = tip; + async.waterfall( + [ + function(callback){ + rpc.getBlockCount(function(err, bc){ + if (err) return callback(err); + callback(null, bc.result); + }); + }, + function(bc, callback){ + rpc.getBlockHash(bc, function(err, bh){ + if (err) return callback(err); + callback(null, bh.result); + }); + } + ], + function (err, result) { + that.lastblockhash = result; + return next(); + } + ); + }); }; return Status; diff --git a/insight.js b/insight.js index 0b811f7..b37690b 100644 --- a/insight.js +++ b/insight.js @@ -40,15 +40,16 @@ var walk = function(path) { walk(models_path); +var syncOpts = { +}; + /** * p2pSync process */ if (!config.disableP2pSync) { var ps = new PeerSync(); ps.init({ - broadcast_txs: true, - broadcast_address_tx: true, - broadcast_blocks: true, + shouldBroadcast: true, }, function() { ps.run(); }); @@ -63,8 +64,7 @@ if (!config.disableHistoricSync) { historicSync = new HistoricSync(); historicSync.init({ - shouldBroadcast: true, - networkName: config.network + shouldBroadcastSync: true, }, function(err) { if (err) { var txt = 'ABORTED with error: ' + err.message; diff --git a/lib/BlockDb.js b/lib/BlockDb.js index 97e2a32..c83cd4b 100644 --- a/lib/BlockDb.js +++ b/lib/BlockDb.js @@ -40,8 +40,8 @@ function spec(b) { }); }; - // adds a block TIP block. Does not update Next pointer in - // the block prev to the new block. + // adds a block. Does not update Next pointer in + // the block prev to the new block, nor TIP pointer // BlockDb.prototype.add = function(b, cb) { var time_key = TIMESTAMP_PREFIX + @@ -49,7 +49,6 @@ function spec(b) { return db.batch() .put(time_key, b.hash) - .put(TIP, b.hash) .put(MAIN_PREFIX + b.hash, 1) .put(PREV_PREFIX + b.hash, b.previousblockhash) .write(cb); @@ -61,6 +60,12 @@ function spec(b) { }); }; + BlockDb.prototype.setTip = function(hash, cb) { + db.put(TIP, hash, function(err) { + return cb(err); + }); + }; + //mainly for testing BlockDb.prototype.setPrev = function(hash, prevHash, cb) { db.put(PREV_PREFIX + hash, prevHash, function(err) { diff --git a/lib/HistoricSync.js b/lib/HistoricSync.js index 6e477e3..800587f 100644 --- a/lib/HistoricSync.js +++ b/lib/HistoricSync.js @@ -97,6 +97,7 @@ function spec() { skippedBlocks: this.skippedBlocks, syncedBlocks: this.syncedBlocks, orphanBlocks: this.orphanBlocks, + syncTipHash: this.sync.tip, error: this.error, type: this.type, }; @@ -116,7 +117,7 @@ function spec() { p(util.format('status: [%d%%] skipped: %d ', self.syncPercentage, self.skippedBlocks)); } - if (self.opts.shouldBroadcast) { + if (self.opts.shouldBroadcastSync) { sockets.broadcastSyncInfo(self.info()); } diff --git a/lib/PeerSync.js b/lib/PeerSync.js index cb09b37..dab7094 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -17,16 +17,12 @@ function spec() { PeerSync.prototype.init = function(opts, cb) { if (!opts) opts = {}; - var networkName = opts && (opts.network || 'testnet'); - var network = networkName === 'testnet' ? networks.testnet : network.livenet; - - this.verbose = opts.verbose; this.peerdb = undefined; - this.sync = new Sync(); this.allowReorgs = false; + this.sync = new Sync(); this.PeerManager = require('bitcore/PeerManager').createClass({ - network: network + network: (config.network === 'testnet' ? networks.testnet : networks.livenet) }); this.peerman = new this.PeerManager(); this.load_peers(); @@ -44,23 +40,18 @@ function spec() { fs.writeFileSync(peerdb_fn, JSON.stringify(this.peerdb)); }; - PeerSync.prototype.handle_inv = function(info) { - var self = this; + PeerSync.prototype.handleInv = function(info) { var invs = info.message.invs; invs.forEach(function(inv) { - if (self.verbose) { - console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); - } + console.log('[p2p_sync] Handle inv for a ' + CoinConst.MSG.to_str(inv.type)); }); // TODO: should limit the invs to objects we haven't seen yet info.conn.sendGetData(invs); }; - PeerSync.prototype.handle_tx = function(info) { + PeerSync.prototype.handleTx = function(info) { var tx = info.message.tx.getStandardizedObject(); - if (this.verbose) { - console.log('[p2p_sync] Handle tx: ' + tx.hash); - } + console.log('[p2p_sync] Handle tx: ' + tx.hash); this.sync.storeTxs([tx.hash], function(err) { if (err) { @@ -69,7 +60,7 @@ function spec() { }); }; - PeerSync.prototype.handle_block = function(info) { + PeerSync.prototype.handleBlock = function(info) { var self = this; var block = info.message.block; var blockHash = coinUtil.formatHashFull(block.calcHash()); @@ -95,9 +86,7 @@ function spec() { PeerSync.prototype.handle_connected = function(data) { var peerman = data.pm; var peers_n = peerman.peers.length; - if (this.verbose) { - console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); - } + console.log('[p2p_sync] Connected to ' + peers_n + ' peer' + (peers_n !== 1 ? 's': '')); }; PeerSync.prototype.run = function() { @@ -109,9 +98,9 @@ function spec() { }); this.peerman.on('connection', function(conn) { - conn.on('inv', self.handle_inv.bind(self)); - conn.on('block', self.handle_block.bind(self)); - conn.on('tx', self.handle_tx.bind(self)); + conn.on('inv', self.handleInv.bind(self)); + conn.on('block', self.handleBlock.bind(self)); + conn.on('tx', self.handleTx.bind(self)); }); this.peerman.on('connect', self.handle_connected.bind(self)); diff --git a/lib/Sync.js b/lib/Sync.js index a90623f..5e4fbbc 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -124,14 +124,16 @@ function spec() { }, function(c) { if (!needReorg) return c(); - console.log('NEW TIP: %s NEED REORG (old tip: %s)', b.hash, oldTip); // TODO should modify updatedTxs and addrs. self.processReorg(oldTip, oldNext, newPrev, c); }, function(c) { - self.bDb.setNext(newPrev, b.hash, function(err) { - return c(err); + self.bDb.setTip(b.hash, function(err) { + if (err) return c(err); + self.bDb.setNext(newPrev, b.hash, function(err) { + return c(err); + }); }); }], function(err) { @@ -245,23 +247,24 @@ function spec() { Sync.prototype._handleBroadcast = function(hash, updatedTxs, updatedAddrs) { var self = this; - if (hash && self.opts.broadcast_blocks) { - sockets.broadcast_block({hash: hash}); - } - - if (updatedTxs && self.opts.broadcast_txs) { - updatedTxs.forEach(function(tx) { - sockets.broadcast_tx(tx); - }); - } - - if (updatedAddrs && self.opts.broadcast_addresses) { - updatedAddrs.forEach(function(addr, txs){ - txs.forEach(function(addr, t){ - sockets.broadcast_address_tx(addr, {'txid': t}); + if (self.opts.shouldBroadcast) { + if (hash) { + sockets.broadcastBlock(hash); + } + if (updatedTxs) { + updatedTxs.forEach(function(tx) { + sockets.broadcastTx(tx); }); - }); + } + + if (updatedAddrs ) { + updatedAddrs.forEach(function(addr, txs){ + txs.forEach(function(addr, t){ + sockets.broadcastAddressTx(addr, t); + }); + }); + } } }; diff --git a/public/js/controllers/address.js b/public/js/controllers/address.js index 4ae129b..89780ee 100644 --- a/public/js/controllers/address.js +++ b/public/js/controllers/address.js @@ -31,10 +31,8 @@ function($scope, $rootScope, $routeParams, $location, Global, Address, getSocket var socket = getSocket($scope); socket.emit('subscribe', $routeParams.addrStr); socket.on($routeParams.addrStr, function(tx) { - console.log('atx ' + tx.txid); - var beep = new Audio('/sound/transaction.mp3'); - beep.play(); - $rootScope.$broadcast('tx', tx.txid); + console.log('AddressTx event received ' + tx); + $rootScope.$broadcast('tx', tx); }); $scope.params = $routeParams; diff --git a/public/js/controllers/header.js b/public/js/controllers/header.js index 7147272..3837bbc 100755 --- a/public/js/controllers/header.js +++ b/public/js/controllers/header.js @@ -33,7 +33,7 @@ angular.module('insight.system').controller('HeaderController', }; socket.on('block', function(block) { - var blockHash = block.hash.toString(); + var blockHash = block.toString(); console.log('Updated Blocks Height!'); _getBlock(blockHash); }); diff --git a/public/js/controllers/index.js b/public/js/controllers/index.js index 71e432a..687c491 100755 --- a/public/js/controllers/index.js +++ b/public/js/controllers/index.js @@ -32,7 +32,7 @@ angular.module('insight.system').controller('IndexController', socket.on('tx', function(tx) { console.log('Transaction received! ' + JSON.stringify(tx)); - var txStr = tx.txid.toString(); + var txStr = tx.toString(); _getTransaction(txStr, function(res) { $scope.txs.unshift(res); if (parseInt($scope.txs.length, 10) >= parseInt(TRANSACTION_DISPLAYED, 10)) { @@ -42,8 +42,9 @@ angular.module('insight.system').controller('IndexController', }); socket.on('block', function(block) { - var blockHash = block.hash.toString(); console.log('Block received! ' + JSON.stringify(block)); + + var blockHash = block.toString(); if (parseInt($scope.blocks.length, 10) > parseInt(BLOCKS_DISPLAYED, 10) - 1) { $scope.blocks.pop(); } diff --git a/public/views/status.html b/public/views/status.html index 0cf5060..7f7f4e2 100644 --- a/public/views/status.html +++ b/public/views/status.html @@ -38,6 +38,7 @@ Skipped Blocks (previously synced) {{sync.skippedBlocks}} + @@ -80,9 +81,14 @@ - + + + + + +
Last Block HashLast Block Hash (Bitcoind) {{lastblockhash}}
Current Blockchain Tip(Insight){{syncTipHash}}
From d5020b9b559c487e841d26145ab779d473779e7f Mon Sep 17 00:00:00 2001 From: Matias Alejo Garcia Date: Sun, 9 Feb 2014 19:46:40 -0300 Subject: [PATCH 11/12] fix json for blocks --- lib/PeerSync.js | 1 - lib/Sync.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/PeerSync.js b/lib/PeerSync.js index dab7094..091b50c 100644 --- a/lib/PeerSync.js +++ b/lib/PeerSync.js @@ -71,7 +71,6 @@ function spec() { return coinUtil.formatHashFull(tx.hash); }); - // Reorgs enabled. this.sync.storeTipBlock({ 'hash': blockHash, 'tx': tx_hashes, diff --git a/lib/Sync.js b/lib/Sync.js index 5e4fbbc..6718953 100644 --- a/lib/Sync.js +++ b/lib/Sync.js @@ -137,7 +137,7 @@ function spec() { }); }], function(err) { - if (!err) self._handleBroadcast(b, updatedTxs, updatedAddrs); + if (!err) self._handleBroadcast(b.hash, updatedTxs, updatedAddrs); if (err && err.toString().match(/WARN/) ) { console.log(err); err=null; From db8b6e2728150672f7e7e02272998c3215e652b7 Mon Sep 17 00:00:00 2001 From: Gustavo Cortez Date: Sun, 9 Feb 2014 21:11:20 -0300 Subject: [PATCH 12/12] when arrive tx, push on list. fixed streamed blocks on homepage. --- app/controllers/blocks.js | 3 +++ insight.js | 2 +- lib/TransactionRpc.js | 4 ++-- public/js/controllers/header.js | 2 +- public/js/controllers/index.js | 24 +++++++++++++++--------- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/app/controllers/blocks.js b/app/controllers/blocks.js index 4eae98e..a2b6ca7 100644 --- a/app/controllers/blocks.js +++ b/app/controllers/blocks.js @@ -109,6 +109,9 @@ exports.list = function(req, res) { else { var blockshashList = []; var limit = parseInt(req.query.limit || blocks.length); + if (blocks.length < limit) { + limit = blocks.length; + } for(var i=0;i= parseInt(TRANSACTION_DISPLAYED, 10)) { + $scope.txs = $scope.txs.splice(0, TRANSACTION_DISPLAYED); + } + }); }); socket.on('block', function(block) { - var blockHash = block.hash.hash.toString(); + var blockHash = block.hash.toString(); console.log('Block received! ' + JSON.stringify(blockHash)); _getBlocks(); }); @@ -48,7 +55,6 @@ angular.module('insight.system').controller('IndexController', $scope.index = function() { _getBlocks(); - _getTransactions(); }; $scope.txs = [];