From f2fccdd14f7094a1253d30f7496bbcea40cc70d4 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 19 Mar 2015 13:18:31 -0300 Subject: [PATCH 01/42] sync first attempt --- lib/services/block.js | 26 +++++++++++++++++++++----- package.json | 3 ++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/services/block.js b/lib/services/block.js index 712e6f37..d95edd7c 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -206,9 +206,27 @@ BlockService.prototype.getLatest = function() { return self.getBlock(blockHash); - }).catch(blockNotFound); + }).catch(function(err) { + if (err instanceof LevelUp.errors.NotFoundError) { + return null; + } + throw err; + }); }; + +/** + * Save a new block + * + * @param {bitcore.Block} block + * @return {Promise} a promise of the same block, for chaining + */ +BlockService.prototype.save = function(block) { + // TODO: unconfirm previous tip, confirm new tip. + return this._confirmBlock(block); +}; + + /** * Set a block as the current tip of the blockchain * @@ -250,11 +268,9 @@ BlockService.prototype._confirmBlock = function(block) { }).then(function() { - return self.database.batchAsync(ops) + return self.database.batchAsync(ops); - }).then(function() { - return self.unlock(); - }); + }).then(this.unlock.bind(this)); }; BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { diff --git a/package.json b/package.json index ecce833d..cb4592e0 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,8 @@ "glob": "*", "js-yaml": "^3.2.7", "level-lock": "^1.0.1", - "levelup": "~0.19.0", + "leveldown": "^1.0.0", + "levelup": "^0.19.0", "moment": "~2.5.0", "morgan": "^1.5.1", "request": "^2.48.0", From 2468cedde42dc04981f249892da9d09a6f63321d Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 19 Mar 2015 14:39:34 -0300 Subject: [PATCH 02/42] first prototype architecture --- config/default.yml | 2 +- index.js | 2 ++ lib/networkmonitor.js | 12 ++++++++++-- lib/node.js | 35 +++++++++++++++++++++++++++++++++++ lib/services/block.js | 9 +++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/config/default.yml b/config/default.yml index 5b0d42ac..d97beb8c 100644 --- a/config/default.yml +++ b/config/default.yml @@ -1,6 +1,6 @@ BitcoreNode: + network: testnet NetworkMonitor: - network: livenet host: localhost port: 8333 Reporter: simple # none, simple, matrix diff --git a/index.js b/index.js index aae7b25e..30884880 100644 --- a/index.js +++ b/index.js @@ -2,9 +2,11 @@ var BitcoreNode = require('./lib/node'); var reporters = require('./lib/reporters'); +var bitcore = require('bitcore'); if (require.main === module) { var config = require('config'); + bitcore.Networks.defaultNetwork = bitcore.Networks.get(config.network); var node = BitcoreNode.create(config.get('BitcoreNode')); node.start(); node.on('error', function(err) { diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 84e48dfc..bdb7812f 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -21,10 +21,9 @@ util.inherits(NetworkMonitor, EventEmitter); NetworkMonitor.create = function(eventBus, opts) { opts = opts || {}; - var network = Networks.get(opts.network) || Networks.defaultNetwork; var host = opts.host || 'localhost'; var port = opts.port || Networks.defaultNetwork.port; - var peer = new Peer(host, port, network); + var peer = new Peer(host, port, Networks.defaultNetwork); return new NetworkMonitor(eventBus, peer); }; @@ -47,12 +46,21 @@ NetworkMonitor.prototype.setupPeer = function(peer) { peer.on('error', function(err) { self.emit('error', err); }); + peer.on('disconnect', function() { + self.emit('disconnect'); + }); }; NetworkMonitor.prototype.start = function() { this.peer.connect(); }; +NetworkMonitor.prototype.stop = function() { + this.peer.disconnect(); +}; +NetworkMonitor.prototype.syncFrom = function(start) { + this.peer.sendMessage(messages.GetBlocks([start])); +}; module.exports = NetworkMonitor; diff --git a/lib/node.js b/lib/node.js index 6e67fc83..e6580483 100644 --- a/lib/node.js +++ b/lib/node.js @@ -4,10 +4,13 @@ var util = require('util'); var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); +var p2p = require('bitcore-p2p'); +var messages = new p2p.Messages(); var $ = bitcore.util.preconditions; var NetworkMonitor = require('./networkmonitor'); var EventBus = require('./eventbus'); +var BlockService = require('./services/block.js'); var BitcoreNode = function(bus, nm) { $.checkArgument(bus); @@ -16,12 +19,20 @@ var BitcoreNode = function(bus, nm) { this.bus = bus; this.nm = nm; + this.bs = new BlockService(); + + this.bus.register(bitcore.Block, this.bs.onBlock.bind(this.bs)); + this.bus.onAny(function(value) { self.emit(this.event, value); }); this.nm.on('error', function(err) { self.emit('error', err); }); + this.nm.on('disconnect', function() { + console.log('network monitor disconnected'); + }); + }; util.inherits(BitcoreNode, EventEmitter); @@ -33,7 +44,31 @@ BitcoreNode.create = function(opts) { }; BitcoreNode.prototype.start = function() { + this.sync(); this.nm.start(); }; +BitcoreNode.prototype.sync = function() { + var genesis = bitcore.Networks.defaultNetwork.genesis; + console.log(bitcore.Networks.defaultNetwork.name); + console.log(genesis); + var self = this; + this.nm.on('ready', function() { + console.log('ready'); + self.bs.getLatest().then(function(latest) { + var start = genesis; + if (latest) { + start = latest.rawHash; + } + console.log('Starting sync from', start); + self.nm.syncFrom(start); + }) + .catch(function(err) { + self.nm.disconnect(); + throw err; + }); + }); +}; + + module.exports = BitcoreNode; diff --git a/lib/services/block.js b/lib/services/block.js index d95edd7c..2f163986 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -214,6 +214,15 @@ BlockService.prototype.getLatest = function() { }); }; +/** + * Handle a block from the network + * + * @param {bitcore.Block} block + * @return a list of events back to the event bus + */ +BlockService.prototype.onBlock = function(block) { + console.log('block', block); +}; /** * Save a new block From cac5264b644e7abe000661f68c5203472a0ad2f8 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 19 Mar 2015 17:41:41 -0300 Subject: [PATCH 03/42] moving forward with new sync --- config/default.yml | 4 +-- index.js | 3 +- lib/networkmonitor.js | 20 ++++++++++-- lib/services/block.js | 76 ++++++++++++++++++++++++++----------------- 4 files changed, 68 insertions(+), 35 deletions(-) diff --git a/config/default.yml b/config/default.yml index d97beb8c..2eb74ad5 100644 --- a/config/default.yml +++ b/config/default.yml @@ -3,11 +3,11 @@ BitcoreNode: NetworkMonitor: host: localhost port: 8333 -Reporter: simple # none, simple, matrix +Reporter: none # none, simple, matrix LevelUp: ./db RPC: user: username pass: password protocol: http host: 127.0.0.1 - port: 8332 + port: 18332 diff --git a/index.js b/index.js index 30884880..1f05bdae 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,8 @@ var BitcoreNode = require('./lib/node'); var reporters = require('./lib/reporters'); var bitcore = require('bitcore'); +BitcoreNode.errors = require('./lib/errors'); + if (require.main === module) { var config = require('config'); bitcore.Networks.defaultNetwork = bitcore.Networks.get(config.network); @@ -27,6 +29,5 @@ if (require.main === module) { } -BitcoreNode.errors = require('./lib/errors'); module.exports = BitcoreNode; diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index bdb7812f..ccbf1956 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -6,6 +6,7 @@ var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); var Networks = bitcore.Networks; var $ = bitcore.util.preconditions; +var _ = bitcore.deps._; var p2p = require('bitcore-p2p'); var Peer = p2p.Peer; var messages = new p2p.Messages(); @@ -38,13 +39,22 @@ NetworkMonitor.prototype.setupPeer = function(peer) { peer.sendMessage(messages.GetData(m.inventory)); }); peer.on('tx', function(m) { - self.bus.process(m.transaction); + self.bus.process(m.transaction) + .catch(function(err) { + self.stop(err); + }); + //.catch(self.stop.bind(self)); }); peer.on('block', function(m) { - self.bus.process(m.block); + self.bus.process(m.block) + .catch(function(err) { + self.stop(err); + }); + //.catch(self.stop.bind(self)); }); peer.on('error', function(err) { self.emit('error', err); + self.stop(err); }); peer.on('disconnect', function() { self.emit('disconnect'); @@ -55,11 +65,15 @@ NetworkMonitor.prototype.setupPeer = function(peer) { NetworkMonitor.prototype.start = function() { this.peer.connect(); }; -NetworkMonitor.prototype.stop = function() { +NetworkMonitor.prototype.stop = function(reason) { this.peer.disconnect(); + if (reason) { + throw reason; + } }; NetworkMonitor.prototype.syncFrom = function(start) { + $.checkArgument(_.isString(start), 'start must be a block hash string'); this.peer.sendMessage(messages.GetBlocks([start])); }; diff --git a/lib/services/block.js b/lib/services/block.js index 2f163986..bc6dabc6 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -6,9 +6,10 @@ var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); var TransactionService = require('./transaction'); var bitcore = require('bitcore'); +var Transaction = bitcore.Transaction; var config = require('config'); -var BitcoreNode = require('../../'); +var errors = require('../errors'); var $ = bitcore.util.preconditions; var JSUtil = bitcore.util.js; @@ -36,11 +37,11 @@ var helper = function(index) { }; var Index = { - timestamp: 'bts-', // bts- -> hash for the block that was mined at this TS - prev: 'prev-', // prev- -> parent hash - next: 'nxt-', // nxt- -> hash for the next block in the main chain that is a child - height: 'bh-', // bh- -> height (-1 means disconnected) - tip: 'tip' // tip -> { hash: hex, height: int }, the latest tip + timestamp: 'bts-', // bts- -> hash for the block that was mined at this TS + prev: 'prev-', // prev- -> parent hash + next: 'nxt-', // nxt- -> hash for the next block in the main chain that is a child + height: 'bh-', // bh- -> height (-1 means disconnected) + tip: 'tip' // tip -> { hash: hex, height: int }, the latest tip }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -51,7 +52,7 @@ _.extend(Index, { } }); -function BlockService (opts) { +function BlockService(opts) { opts = _.extend({}, opts); this.database = opts.database || Promise.promisifyAll(new LevelUp(config.get('LevelUp'))); this.rpc = opts.rpc || Promise.promisifyAll(new RPC(config.get('RPC'))); @@ -93,10 +94,10 @@ BlockService.prototype.unlock = function() { * merkle root hash * @return {bitcore.Block} */ -BlockService.blockRPCtoBitcore = function(blockData, transactions) { - $.checkArgument(_.all(transactions, function(transaction) { - return transaction instanceof bitcore.Transaction; - }), 'All transactions must be instances of bitcore.Transaction'); +BlockService.blockRPCtoBitcore = function(blockData) { + console.log(blockData); + $.checkArgument(blockData, 'blockData is required'); + $.checkArgument(blockData.previousblockhash, 'blockData.previousblockhash is required'); var block = new bitcore.Block({ header: new bitcore.BlockHeader({ version: blockData.version, @@ -107,13 +108,12 @@ BlockService.blockRPCtoBitcore = function(blockData, transactions) { time: blockData.time, nonce: blockData.nonce, bits: new bitcore.deps.bnjs( - new bitcore.deps.Buffer(blockData.bits, 'hex') + new bitcore.deps.Buffer(blockData.bits, 'hex') ), merkleRoot: bitcore.util.buffer.reverse( new bitcore.deps.Buffer(blockData.merkleroot, 'hex') ) }), - transactions: transactions }); block.height = blockData.height; return block; @@ -126,8 +126,7 @@ BlockService.blockRPCtoBitcore = function(blockData, transactions) { * @return {Promise} a promise that will always be rejected */ var blockNotFound = function(err) { - console.log(err, err.stack); - return Promise.reject(new BitcoreNode.errors.Blocks.NotFound()); + throw new errors.Blocks.NotFound(err); }; /** @@ -136,7 +135,7 @@ var blockNotFound = function(err) { * @param {string} blockHash the hash of the block to be fetched * @return {Promise} */ -BlockService.prototype.getBlock = function(blockHash) { +BlockService.prototype.getBlock = function(blockHash, opts) { $.checkArgument( JSUtil.isHexa(blockHash) || bitcore.util.buffer.isBuffer(blockHash), 'Block hash must be a buffer or hexa' @@ -144,27 +143,33 @@ BlockService.prototype.getBlock = function(blockHash) { if (bitcore.util.buffer.isBuffer(blockHash)) { blockHash = bitcore.util.buffer.reverse(blockHash).toString('hex'); } + opts = opts || {}; var blockData; var self = this; return Promise.try(function() { + return self.rpc.getBlockAsync(blockHash); + }) + .catch(blockNotFound) + .then(function(block) { - return self.rpc.getBlockAsync(blockHash); + blockData = block.result; - }).then(function(block) { + if (opts.withoutTransactions) { + return []; + } - blockData = block.result; - return Promise.all(blockData.tx.map(function(txId) { - return self.transactionService.getTransaction(txId); - })); + return Promise.all(blockData.tx.map(function(txId) { + return self.transactionService.getTransaction(txId); + })); - }).then(function(transactions) { + }).then(function(transactions) { - blockData.transactions = transactions; - return BlockService.blockRPCtoBitcore(blockData); + blockData.transactions = transactions; + return BlockService.blockRPCtoBitcore(blockData); - }).catch(blockNotFound); + }); }; /** @@ -182,11 +187,13 @@ BlockService.prototype.getBlockByHeight = function(height) { return self.rpc.getBlockHash(height); - }).then(function(blockHash) { + }) + .catch(blockNotFound) + .then(function(blockHash) { return self.getBlock(blockHash); - }).catch(blockNotFound); + }); }; /** @@ -222,6 +229,15 @@ BlockService.prototype.getLatest = function() { */ BlockService.prototype.onBlock = function(block) { console.log('block', block); + var events = []; + return this.save(block) + .then(function(block) { + return undefined; + block.transactions.forEach(function(tx) { + events.push(tx); + }); + return events; + }); }; /** @@ -256,7 +272,9 @@ BlockService.prototype._confirmBlock = function(block) { }).then(function() { if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { - return self.getBlock(block.header.prevHash); + return self.getBlock(block.header.prevHash, { + withoutTransactions: true + }); } else { return GENESISPARENT; } From e39fb861acd18b7b1d05b1992bf646c51c16c0c6 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 20 Mar 2015 12:16:12 -0300 Subject: [PATCH 04/42] advance --- config/default.yml | 2 +- index.js | 2 + lib/Rpc.js | 119 ------------------------------------------ lib/services/block.js | 3 +- 4 files changed, 4 insertions(+), 122 deletions(-) delete mode 100644 lib/Rpc.js diff --git a/config/default.yml b/config/default.yml index 2eb74ad5..dac9711e 100644 --- a/config/default.yml +++ b/config/default.yml @@ -10,4 +10,4 @@ RPC: pass: password protocol: http host: 127.0.0.1 - port: 18332 + port: 8322 diff --git a/index.js b/index.js index 1f05bdae..c7958dd4 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,8 @@ var BitcoreNode = require('./lib/node'); var reporters = require('./lib/reporters'); var bitcore = require('bitcore'); +var Promise = require('bluebird'); +Promise.longStackTraces(); BitcoreNode.errors = require('./lib/errors'); diff --git a/lib/Rpc.js b/lib/Rpc.js deleted file mode 100644 index f81f3721..00000000 --- a/lib/Rpc.js +++ /dev/null @@ -1,119 +0,0 @@ -'use strict'; - -var imports = require('soop').imports(); - -var bitcore = require('bitcore'), - RpcClient = bitcore.RpcClient, - BitcoreBlock = bitcore.Block, - util = require('util'), - config = require('../config/config'); - -var bitcoreRpc = imports.bitcoreRpc || new RpcClient(config.bitcoind); - -function Rpc() { -} - -Rpc._parseTxResult = function(info) { - var b = new Buffer(info.hex,'hex'); - - // remove fields we dont need, to speed and adapt the information - delete info.hex; - - // Inputs => add index + coinBase flag - var n =0; - info.vin.forEach(function(i) { - i.n = n++; - if (i.coinbase) info.isCoinBase = true; - }); - - // Outputs => add total - var valueOutSat = 0; - info.vout.forEach( function(o) { - o.value = o.value.toFixed(8); - valueOutSat += o.value * bitcore.util.COIN; - }); - info.valueOut = valueOutSat.toFixed(0) / bitcore.util.COIN; - info.size = b.length; - - return info; -}; - - -Rpc.errMsg = function(err) { - var e = err; - e.message += util.format(' [Host: %s:%d User:%s Using password:%s]', - bitcoreRpc.host, - bitcoreRpc.port, - bitcoreRpc.user, - bitcoreRpc.pass?'yes':'no' - ); - return e; -}; - -Rpc.getTxInfo = function(txid, doNotParse, cb) { - var self = this; - - if (typeof doNotParse === 'function') { - cb = doNotParse; - doNotParse = false; - } - - bitcoreRpc.getRawTransaction(txid, 1, function(err, txInfo) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(self.errMsg(err)); - - var info = doNotParse ? txInfo.result : self._parseTxResult(txInfo.result); - return cb(null,info); - }); -}; - - -Rpc.blockIndex = function(height, cb) { - var self = this; - - bitcoreRpc.getBlockHash(height, function(err, bh){ - if (err) return cb(self.errMsg(err)); - cb(null, { blockHash: bh.result }); - }); -}; - -Rpc.getBlock = function(hash, cb) { - var self = this; - - bitcoreRpc.getBlock(hash, function(err,info) { - // Not found? - if (err && err.code === -5) return cb(); - if (err) return cb(self.errMsg(err)); - - - if (info.result.height) - info.result.reward = BitcoreBlock.getBlockValue(info.result.height) / bitcore.util.COIN ; - - return cb(err,info.result); - }); -}; - -Rpc.sendRawTransaction = function(rawtx, cb) { - bitcoreRpc.sendRawTransaction(rawtx, function(err, txid) { - if (err) return cb(err); - - return cb(err, txid.result); - }); -}; - -Rpc.verifyMessage = function(address, signature, message, cb) { - var self = this; - bitcoreRpc.verifyMessage(address, signature, message, function(err, message) { - if (err && (err.code === -3 || err.code === -5)) - return cb(err); // -3 = invalid address, -5 = malformed base64 / etc. - if (err) - return cb(self.errMsg(err)); - - return cb(err, message.result); - }); -}; - -module.exports = require('soop')(Rpc); - - diff --git a/lib/services/block.js b/lib/services/block.js index bc6dabc6..661c0159 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -97,7 +97,6 @@ BlockService.prototype.unlock = function() { BlockService.blockRPCtoBitcore = function(blockData) { console.log(blockData); $.checkArgument(blockData, 'blockData is required'); - $.checkArgument(blockData.previousblockhash, 'blockData.previousblockhash is required'); var block = new bitcore.Block({ header: new bitcore.BlockHeader({ version: blockData.version, @@ -296,7 +295,7 @@ BlockService.prototype._confirmBlock = function(block) { }).then(function() { return self.database.batchAsync(ops); - + }).then(this.unlock.bind(this)); }; From 76af2d13146a400e344860ad7f73093927bccef3 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 20 Mar 2015 13:32:11 -0300 Subject: [PATCH 05/42] remove logs --- lib/node.js | 2 -- lib/services/block.js | 10 ++++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/node.js b/lib/node.js index e6580483..92dd8bf7 100644 --- a/lib/node.js +++ b/lib/node.js @@ -50,8 +50,6 @@ BitcoreNode.prototype.start = function() { BitcoreNode.prototype.sync = function() { var genesis = bitcore.Networks.defaultNetwork.genesis; - console.log(bitcore.Networks.defaultNetwork.name); - console.log(genesis); var self = this; this.nm.on('ready', function() { console.log('ready'); diff --git a/lib/services/block.js b/lib/services/block.js index 661c0159..7f5b4335 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -89,7 +89,7 @@ BlockService.prototype.unlock = function() { * @param {Number} blockData.time a 32 bit number with the timestamp when this block was created * @param {Number} blockData.nonce a 32 bit number with a random number * @param {string} blockData.bits a 32 bit "varint" encoded number with the length of the block - * @param {string} blockData.merkleRoot an hex string of length 64 with the hash of the block + * @param {string} blockData.merkleroot an hex string of length 64 with the hash of the block * @param {Array} transactions an array of bitcore.Transaction objects, in the order that forms the * merkle root hash * @return {bitcore.Block} @@ -113,6 +113,7 @@ BlockService.blockRPCtoBitcore = function(blockData) { new bitcore.deps.Buffer(blockData.merkleroot, 'hex') ) }), + transactions: blockData.transaction }); block.height = blockData.height; return block; @@ -212,11 +213,8 @@ BlockService.prototype.getLatest = function() { return self.getBlock(blockHash); - }).catch(function(err) { - if (err instanceof LevelUp.errors.NotFoundError) { - return null; - } - throw err; + }).catch(LevelUp.errors.NotFoundError, function() { + return null; }); }; From 9f91bbfbc949c565576d853b45d779cc63fc3011 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 20 Mar 2015 15:19:25 -0300 Subject: [PATCH 06/42] merge with master fixes --- config/default.yml | 2 +- lib/networkmonitor.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/default.yml b/config/default.yml index dac9711e..9b5e636c 100644 --- a/config/default.yml +++ b/config/default.yml @@ -6,7 +6,7 @@ BitcoreNode: Reporter: none # none, simple, matrix LevelUp: ./db RPC: - user: username + user: user pass: password protocol: http host: 127.0.0.1 diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index ccbf1956..97dad3d7 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -66,6 +66,7 @@ NetworkMonitor.prototype.start = function() { this.peer.connect(); }; NetworkMonitor.prototype.stop = function(reason) { + console.trace(); this.peer.disconnect(); if (reason) { throw reason; From 8c3a9e95cceb07b57a1df16d32e5972792c4ed1d Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 12:20:22 -0300 Subject: [PATCH 07/42] Update bitcore-p2p to latest minor --- config/default.yml | 4 ++-- index.js | 5 ++--- lib/networkmonitor.js | 6 +++++- package.json | 5 ++--- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/config/default.yml b/config/default.yml index 9b5e636c..c2bebc07 100644 --- a/config/default.yml +++ b/config/default.yml @@ -1,5 +1,5 @@ BitcoreNode: - network: testnet + network: livenet NetworkMonitor: host: localhost port: 8333 @@ -10,4 +10,4 @@ RPC: pass: password protocol: http host: 127.0.0.1 - port: 8322 + port: 8332 diff --git a/index.js b/index.js index c7958dd4..9b3e729b 100644 --- a/index.js +++ b/index.js @@ -10,7 +10,8 @@ BitcoreNode.errors = require('./lib/errors'); if (require.main === module) { var config = require('config'); - bitcore.Networks.defaultNetwork = bitcore.Networks.get(config.network); + bitcore.Networks.defaultNetwork = bitcore.Networks.get(config.get('BitcoreNode').network); + var node = BitcoreNode.create(config.get('BitcoreNode')); node.start(); node.on('error', function(err) { @@ -30,6 +31,4 @@ if (require.main === module) { node.on('Transaction', reporter); } - - module.exports = BitcoreNode; diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 97dad3d7..210c6452 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -24,7 +24,11 @@ NetworkMonitor.create = function(eventBus, opts) { opts = opts || {}; var host = opts.host || 'localhost'; var port = opts.port || Networks.defaultNetwork.port; - var peer = new Peer(host, port, Networks.defaultNetwork); + var peer = new Peer({ + host: host, + port: port, + network: Networks.defaultNetwork + }); return new NetworkMonitor(eventBus, peer); }; diff --git a/package.json b/package.json index cb4592e0..bf3093e7 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "async": "0.9.0", "bitcoind-rpc": "^0.2.1", "bitcore": "bitpay/bitcore", - "bitcore-p2p": "bitpay/bitcore-p2p", + "bitcore-p2p": "^0.11.0", "bluebird": "^2.9.12", "body-parser": "^1.12.0", "bufferput": "bitpay/node-bufferput", @@ -59,8 +59,7 @@ "express": "4.11.1", "glob": "*", "js-yaml": "^3.2.7", - "level-lock": "^1.0.1", - "leveldown": "^1.0.0", + "leveldown": "^0.10.4", "levelup": "^0.19.0", "moment": "~2.5.0", "morgan": "^1.5.1", From 5fc678022594d10ecb719b51508b5113b93a02c2 Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 16:18:12 -0300 Subject: [PATCH 08/42] Drop old http --- .gitignore | 2 + config/default.yml | 3 + http/controllers/addresses.js | 206 ------------------------------ http/controllers/blocks.js | 176 -------------------------- http/controllers/common.js | 16 --- http/controllers/currency.js | 60 --------- http/controllers/index.js | 26 ---- http/controllers/messages.js | 27 ---- http/controllers/status.js | 62 --------- http/controllers/transactions.js | 166 ------------------------ http/express.js | 71 ----------- http/headers.js | 14 -- http/models/Address.js | 211 ------------------------------- http/models/Status.js | 105 --------------- http/routes.js | 65 ---------- 15 files changed, 5 insertions(+), 1205 deletions(-) delete mode 100644 http/controllers/addresses.js delete mode 100644 http/controllers/blocks.js delete mode 100644 http/controllers/common.js delete mode 100644 http/controllers/currency.js delete mode 100644 http/controllers/index.js delete mode 100644 http/controllers/messages.js delete mode 100644 http/controllers/status.js delete mode 100644 http/controllers/transactions.js delete mode 100644 http/express.js delete mode 100644 http/headers.js delete mode 100644 http/models/Address.js delete mode 100644 http/models/Status.js delete mode 100644 http/routes.js diff --git a/.gitignore b/.gitignore index 15093bfa..04327abc 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ db/testnet/blocks README.html public + +blocks diff --git a/config/default.yml b/config/default.yml index c2bebc07..fb60e8e2 100644 --- a/config/default.yml +++ b/config/default.yml @@ -4,6 +4,9 @@ BitcoreNode: host: localhost port: 8333 Reporter: none # none, simple, matrix +BitcoreHTTP: + host: localhost + port: 8080 LevelUp: ./db RPC: user: user diff --git a/http/controllers/addresses.js b/http/controllers/addresses.js deleted file mode 100644 index 30cfca3c..00000000 --- a/http/controllers/addresses.js +++ /dev/null @@ -1,206 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ - -var _ = require('lodash'); -var Address = require('../models/Address'); -var common = require('./common'); -var async = require('async'); - -var tDb = require('../../lib/TransactionDb').default(); - -var getAddr = function(req, res, next) { - var a; - try { - var addr = req.param('addr'); - a = new Address(addr); - } catch (e) { - common.handleErrors({ - message: 'Invalid address:' + e.message, - code: 1 - }, res, next); - return null; - } - return a; -}; - -var getAddrs = function(req, res, next) { - var as = []; - try { - var addrStrs = req.param('addrs'); - var s = addrStrs.split(','); - if (s.length === 0) return as; - for (var i = 0; i < s.length; i++) { - var a = new Address(s[i]); - as.push(a); - } - } catch (e) { - common.handleErrors({ - message: 'Invalid address:' + e.message, - code: 1 - }, res, next); - return null; - } - return as; -}; - -exports.show = function(req, res, next) { - var a = getAddr(req, res, next); - - if (a) { - a.update(function(err) { - if (err) { - return common.handleErrors(err, res); - } else { - return res.jsonp(a.getObj()); - } - }, {txLimit: req.query.noTxList?0:-1, ignoreCache: req.param('noCache')}); - } -}; - - - -exports.utxo = function(req, res, next) { - var a = getAddr(req, res, next); - if (a) { - a.update(function(err) { - if (err) - return common.handleErrors(err, res); - else { - return res.jsonp(a.unspent); - } - }, {onlyUnspent:1, ignoreCache: req.param('noCache')}); - } -}; - -exports.multiutxo = function(req, res, next) { - var as = getAddrs(req, res, next); - if (as) { - var utxos = []; - async.each(as, function(a, callback) { - a.update(function(err) { - if (err) callback(err); - utxos = utxos.concat(a.unspent); - callback(); - }, {onlyUnspent:1, ignoreCache: req.param('noCache')}); - }, function(err) { // finished callback - if (err) return common.handleErrors(err, res); - res.jsonp(utxos); - }); - } -}; - -exports.multitxs = function(req, res, next) { - - function processTxs(txs, from, to, cb) { - txs = _.uniq(_.flatten(txs), 'txid'); - var nbTxs = txs.length; - var paginated = !_.isUndefined(from) || !_.isUndefined(to); - - if (paginated) { - txs.sort(function(a, b) { - return (b.ts || b.ts) - (a.ts || a.ts); - }); - var start = Math.max(from || 0, 0); - var end = Math.min(to || txs.length, txs.length); - txs = txs.slice(start, end); - } - - var txIndex = {}; - _.each(txs, function (tx) { txIndex[tx.txid] = tx; }); - - async.each(txs, function (tx, callback) { - tDb.fromIdWithInfo(tx.txid, function(err, tx) { - if (err) console.log(err); - if (tx && tx.info) { - txIndex[tx.txid].info = tx.info; - } - callback(); - }); - }, function (err) { - if (err) return cb(err); - - var transactions = _.pluck(txs, 'info'); - if (paginated) { - transactions = { - totalItems: nbTxs, - from: +from, - to: +to, - items: transactions, - }; - } - return cb(null, transactions); - }); - }; - - var from = req.param('from'); - var to = req.param('to'); - - var as = getAddrs(req, res, next); - if (as) { - var txs = []; - async.eachLimit(as, 10, function(a, callback) { - a.update(function(err) { - if (err) callback(err); - txs.push(a.transactions); - callback(); - }, {ignoreCache: req.param('noCache'), includeTxInfo: true}); - }, function(err) { // finished callback - if (err) return common.handleErrors(err, res); - processTxs(txs, from, to, function (err, transactions) { - if (err) return common.handleErrors(err, res); - res.jsonp(transactions); - }); - }); - } -}; - -exports.balance = function(req, res, next) { - var a = getAddr(req, res, next); - if (a) - a.update(function(err) { - if (err) { - return common.handleErrors(err, res); - } else { - return res.jsonp(a.balanceSat); - } - }, {ignoreCache: req.param('noCache')}); -}; - -exports.totalReceived = function(req, res, next) { - var a = getAddr(req, res, next); - if (a) - a.update(function(err) { - if (err) { - return common.handleErrors(err, res); - } else { - return res.jsonp(a.totalReceivedSat); - } - }, {ignoreCache: req.param('noCache')}); -}; - -exports.totalSent = function(req, res, next) { - var a = getAddr(req, res, next); - if (a) - a.update(function(err) { - if (err) { - return common.handleErrors(err, res); - } else { - return res.jsonp(a.totalSentSat); - } - }, {ignoreCache: req.param('noCache')}); -}; - -exports.unconfirmedBalance = function(req, res, next) { - var a = getAddr(req, res, next); - if (a) - a.update(function(err) { - if (err) { - return common.handleErrors(err, res); - } else { - return res.jsonp(a.unconfirmedBalanceSat); - } - }, {ignoreCache: req.param('noCache')}); -}; diff --git a/http/controllers/blocks.js b/http/controllers/blocks.js deleted file mode 100644 index 2de390d4..00000000 --- a/http/controllers/blocks.js +++ /dev/null @@ -1,176 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var common = require('./common'), - async = require('async'), - BlockDb = require('../../lib/BlockDb'), - TransactionDb = require('../../lib/TransactionDb'); - -var bdb = new BlockDb(); -var tdb = new TransactionDb(); - -/** - * Find block by hash ... - */ -exports.block = function(req, res, next, hash) { - bdb.fromHashWithInfo(hash, function(err, block) { - if (err || !block) - return common.handleErrors(err, res, next); - else { - tdb.getPoolInfo(block.info.tx[0], function(info) { - block.info.poolInfo = info; - req.block = block.info; - return next(); - }); - } - }); -}; - - -/** - * Show block - */ -exports.show = function(req, res) { - if (req.block) { - res.jsonp(req.block); - } -}; - -/** - * Show block by Height - */ -exports.blockindex = function(req, res, next, height) { - bdb.blockIndex(height, function(err, hashStr) { - if (err) { - console.log(err); - res.status(400).send('Bad Request'); // TODO - } else { - res.jsonp(hashStr); - } - }); -}; - -var getBlock = function(blockhash, cb) { - bdb.fromHashWithInfo(blockhash, function(err, block) { - if (err) { - console.log(err); - return cb(err); - } - - // TODO - if (!block.info) { - console.log('Could not get %s from RPC. Orphan? Error?', blockhash); //TODO - // Probably orphan - block.info = { - hash: blockhash, - isOrphan: 1, - }; - } - - tdb.getPoolInfo(block.info.tx[0], function(info) { - block.info.poolInfo = info; - return cb(err, block.info); - }); - - }); -}; - -/** - * List of blocks by date - */ - -var DFLT_LIMIT=200; - // in testnet, this number is much bigger, we dont support - // exploring blocks by date. - -exports.list = function(req, res) { - var isToday = false; - - //helper to convert timestamps to yyyy-mm-dd format - var formatTimestamp = function(date) { - var yyyy = date.getUTCFullYear().toString(); - var mm = (date.getUTCMonth() + 1).toString(); // getMonth() is zero-based - var dd = date.getUTCDate().toString(); - - return yyyy + '-' + (mm[1] ? mm : '0' + mm[0]) + '-' + (dd[1] ? dd : '0' + dd[0]); //padding - }; - - var dateStr; - var todayStr = formatTimestamp(new Date()); - - if (req.query.blockDate) { - // TODO: Validate format yyyy-mm-dd - dateStr = req.query.blockDate; - isToday = dateStr === todayStr; - } else { - dateStr = todayStr; - isToday = true; - } - var gte = Math.round((new Date(dateStr)).getTime() / 1000); - - //pagination - var lte = parseInt(req.query.startTimestamp) || gte + 86400; - var prev = formatTimestamp(new Date((gte - 86400) * 1000)); - var next = lte ? formatTimestamp(new Date(lte * 1000)) :null; - var limit = parseInt(req.query.limit || DFLT_LIMIT) + 1; - var more; - - bdb.getBlocksByDate(gte, lte, limit, function(err, blockList) { - - if (err) { - res.status(500).send(err); - } else { - var l = blockList.length; - - if (l===limit) { - more = true; - blockList.pop; - } - - var moreTs=lte; - async.mapSeries(blockList, - function(b, cb) { - getBlock(b.hash, function(err, info) { - if (err) { - console.log(err); - return cb(err); - } - if (b.ts < moreTs) moreTs = b.ts; - return cb(err, { - height: info.height, - size: info.size, - hash: b.hash, - time: b.ts || info.time, - txlength: info.tx.length, - poolInfo: info.poolInfo - }); - }); - }, function(err, allblocks) { - - // sort blocks by height - allblocks.sort( - function compare(a,b) { - if (a.height < b.height) return 1; - if (a.height > b.height) return -1; - return 0; - }); - - res.jsonp({ - blocks: allblocks, - length: allblocks.length, - pagination: { - next: next, - prev: prev, - currentTs: lte - 1, - current: dateStr, - isToday: isToday, - more: more, - moreTs: moreTs, - } - }); - }); - } - }); -}; diff --git a/http/controllers/common.js b/http/controllers/common.js deleted file mode 100644 index b44756bf..00000000 --- a/http/controllers/common.js +++ /dev/null @@ -1,16 +0,0 @@ -'use strict'; - - -exports.handleErrors = function (err, res) { - if (err) { - if (err.code) { - res.status(400).send(err.message + '. Code:' + err.code); - } - else { - res.status(503).send(err.message); - } - } - else { - res.status(404).send('Not found'); - } -}; diff --git a/http/controllers/currency.js b/http/controllers/currency.js deleted file mode 100644 index 244ef327..00000000 --- a/http/controllers/currency.js +++ /dev/null @@ -1,60 +0,0 @@ -'use strict'; - -var config = require('../../config/config'); - -// Set the initial vars -var timestamp = +new Date(), - delay = config.currencyRefresh * 60000, - bitstampRate = 0; - -exports.index = function(req, res) { - - var _xhr = function() { - if (typeof XMLHttpRequest !== 'undefined' && XMLHttpRequest !== null) { - return new XMLHttpRequest(); - } else if (typeof require !== 'undefined' && require !== null) { - var XMLhttprequest = require('xmlhttprequest').XMLHttpRequest; - return new XMLhttprequest(); - } - }; - - var _request = function(url, cb) { - var request; - request = _xhr(); - request.open('GET', url, true); - request.onreadystatechange = function() { - if (request.readyState === 4) { - if (request.status === 200) { - return cb(false, request.responseText); - } - - return cb(true, { - status: request.status, - message: 'Request error' - }); - } - }; - - return request.send(null); - }; - - // Init - var currentTime = +new Date(); - if (bitstampRate === 0 || currentTime >= (timestamp + delay)) { - timestamp = currentTime; - - _request('https://www.bitstamp.net/api/ticker/', function(err, data) { - if (!err) bitstampRate = parseFloat(JSON.parse(data).last); - - res.jsonp({ - status: 200, - data: { bitstamp: bitstampRate } - }); - }); - } else { - res.jsonp({ - status: 200, - data: { bitstamp: bitstampRate } - }); - } -}; diff --git a/http/controllers/index.js b/http/controllers/index.js deleted file mode 100644 index d5f51bb3..00000000 --- a/http/controllers/index.js +++ /dev/null @@ -1,26 +0,0 @@ -'use strict'; - -var config = require('../../config/config'); - -var _getVersion = function() { - var pjson = require('../../package.json'); - return pjson.version; -}; - -exports.render = function(req, res) { - - if (config.publicPath) { - return res.sendfile(config.publicPath + '/index.html'); - } - else { - var version = _getVersion(); - res.send('bitcore-node API v' + version); - } -}; - -exports.version = function(req, res) { - var version = _getVersion(); - res.json({ - version: version - }); -}; diff --git a/http/controllers/messages.js b/http/controllers/messages.js deleted file mode 100644 index 5c2fbfc6..00000000 --- a/http/controllers/messages.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var common = require('./common'); -var Rpc = require('../../lib/Rpc'); - - -exports.verify = function(req, res) { - var address = req.param('address'), - signature = req.param('signature'), - message = req.param('message'); - - if(typeof(address) == 'undefined' - || typeof(signature) == 'undefined' - || typeof(message) == 'undefined') { - return common.handleErrors({ - message: 'Missing parameters (expected "address", "signature" and "message")', - code: 1 - }, res); - } - - Rpc.verifyMessage(address, signature, message, function(err, result) { - if (err) { - return common.handleErrors(err, res); - } - res.json({'result' : result}); - }); -}; diff --git a/http/controllers/status.js b/http/controllers/status.js deleted file mode 100644 index 322135fb..00000000 --- a/http/controllers/status.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ - -var Status = require('../models/Status'), - common = require('./common'); - -/** - * Status - */ -exports.show = function(req, res) { - - if (! req.query.q) { - res.status(400).send('Bad Request'); - } - else { - var option = req.query.q; - var statusObject = new Status(); - - var returnJsonp = function (err) { - if (err || ! statusObject) - return common.handleErrors(err, res); - else { - res.jsonp(statusObject); - } - }; - - switch(option) { - case 'getInfo': - statusObject.getInfo(returnJsonp); - break; - case 'getDifficulty': - statusObject.getDifficulty(returnJsonp); - break; - case 'getTxOutSetInfo': - statusObject.getTxOutSetInfo(returnJsonp); - break; - case 'getLastBlockHash': - statusObject.getLastBlockHash(returnJsonp); - break; - case 'getBestBlockHash': - statusObject.getBestBlockHash(returnJsonp); - break; - default: - res.status(400).send('Bad Request'); - } - } -}; - -exports.sync = function(req, res) { - if (req.historicSync) - res.jsonp(req.historicSync.info()); -}; - -exports.peer = function(req, res) { - if (req.peerSync) { - var info = req.peerSync.info(); - res.jsonp(info); - } -}; diff --git a/http/controllers/transactions.js b/http/controllers/transactions.js deleted file mode 100644 index ed01c329..00000000 --- a/http/controllers/transactions.js +++ /dev/null @@ -1,166 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var Address = require('../models/Address'); -var async = require('async'); -var common = require('./common'); -var util = require('util'); - -var Rpc = require('../../lib/Rpc'); - -var tDb = require('../../lib/TransactionDb').default(); -var bdb = require('../../lib/BlockDb').default(); - -exports.send = function(req, res) { - Rpc.sendRawTransaction(req.body.rawtx, function(err, txid) { - if (err) { - var message; - if(err.code == -25) { - message = util.format( - 'Generic error %s (code %s)', - err.message, err.code); - } else if(err.code == -26) { - message = util.format( - 'Transaction rejected by network (code %s). Reason: %s', - err.code, err.message); - } else { - message = util.format('%s (code %s)', err.message, err.code); - } - return res.status(400).send(message); - } - res.json({'txid' : txid}); - }); -}; - - -/** - * Find transaction by hash ... - */ -exports.transaction = function(req, res, next, txid) { - - tDb.fromIdWithInfo(txid, function(err, tx) { - if (err || ! tx) - return common.handleErrors(err, res); - else { - req.transaction = tx.info; - return next(); - } - }); -}; - - -/** - * Show transaction - */ -exports.show = function(req, res) { - - if (req.transaction) { - res.jsonp(req.transaction); - } -}; - - -var getTransaction = function(txid, cb) { - - tDb.fromIdWithInfo(txid, function(err, tx) { - if (err) console.log(err); - - if (!tx || !tx.info) { - console.log('[transactions.js.48]:: TXid %s not found in RPC. CHECK THIS.', txid); - return ({ txid: txid }); - } - - return cb(null, tx.info); - }); -}; - - -/** - * List of transaction - */ -exports.list = function(req, res, next) { - var bId = req.query.block; - var addrStr = req.query.address; - var page = req.query.pageNum; - var pageLength = 10; - var pagesTotal = 1; - var txLength; - var txs; - - if (bId) { - bdb.fromHashWithInfo(bId, function(err, block) { - if (err) { - console.log(err); - return res.status(500).send('Internal Server Error'); - } - - if (! block) { - return res.status(404).send('Not found'); - } - - txLength = block.info.tx.length; - - if (page) { - var spliceInit = page * pageLength; - txs = block.info.tx.splice(spliceInit, pageLength); - pagesTotal = Math.ceil(txLength / pageLength); - } - else { - txs = block.info.tx; - } - - async.mapSeries(txs, getTransaction, function(err, results) { - if (err) { - console.log(err); - res.status(404).send('TX not found'); - } - - res.jsonp({ - pagesTotal: pagesTotal, - txs: results - }); - }); - }); - } - else if (addrStr) { - var a = new Address(addrStr); - - a.update(function(err) { - if (err && !a.totalReceivedSat) { - console.log(err); - res.status(404).send('Invalid address'); - return next(); - } - - txLength = a.transactions.length; - - if (page) { - var spliceInit = page * pageLength; - txs = a.transactions.splice(spliceInit, pageLength); - pagesTotal = Math.ceil(txLength / pageLength); - } - else { - txs = a.transactions; - } - - async.mapSeries(txs, getTransaction, function(err, results) { - if (err) { - console.log(err); - res.status(404).send('TX not found'); - } - - res.jsonp({ - pagesTotal: pagesTotal, - txs: results - }); - }); - }); - } - else { - res.jsonp({ - txs: [] - }); - } -}; diff --git a/http/express.js b/http/express.js deleted file mode 100644 index 8de62c81..00000000 --- a/http/express.js +++ /dev/null @@ -1,71 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var express = require('express'); -var config = require('./config'); -var path = require('path'); -var logger = require('../lib/logger').logger; - -module.exports = function(app, historicSync, peerSync) { - - - //custom middleware - var setHistoric = function(req, res, next) { - req.historicSync = historicSync; - next(); - }; - - var setPeer = function(req, res, next) { - req.peerSync = peerSync; - next(); - }; - - app.set('showStackError', true); - app.set('json spaces', 0); - - app.enable('jsonp callback'); - app.use(config.apiPrefix + '/sync', setHistoric); - app.use(config.apiPrefix + '/peer', setPeer); - app.use(express.logger('dev')); - app.use(express.json()); - app.use(express.urlencoded()); - app.use(express.methodOverride()); - app.use(express.compress()); - - if (config.enableEmailstore) { - var allowCopayCrossDomain = function(req, res, next) { - if ('OPTIONS' == req.method) { - res.send(200); - res.end(); - return; - } - next(); - } - app.use(allowCopayCrossDomain); - } - - if (config.publicPath) { - var staticPath = path.normalize(config.rootPath + '/../' + config.publicPath); - //IMPORTANT: for html5mode, this line must to be before app.router - app.use(express.static(staticPath)); - } - - app.use(function(req, res, next) { - app.locals.config = config; - next(); - }); - - //routes should be at the last - app.use(app.router); - - //Assume 404 since no middleware responded - app.use(function(req, res) { - res.status(404).jsonp({ - status: 404, - url: req.originalUrl, - error: 'Not found' - }); - }); -}; diff --git a/http/headers.js b/http/headers.js deleted file mode 100644 index 8cd9e73c..00000000 --- a/http/headers.js +++ /dev/null @@ -1,14 +0,0 @@ -'use strict'; - -var logger = require('../lib/logger').logger; - -module.exports = function(app) { - - app.use(function(req, res, next) { - res.setHeader('Access-Control-Allow-Origin', '*'); - res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, DELETE'); - res.setHeader('Access-Control-Allow-Headers', 'X-Requested-With,Content-Type,Authorization'); - res.setHeader('Access-Control-Expose-Headers', 'X-Email-Needs-Validation,X-Quota-Per-Item,X-Quota-Items-Limit,X-RateLimit-Limit,X-RateLimit-Remaining'); - next(); - }); -}; diff --git a/http/models/Address.js b/http/models/Address.js deleted file mode 100644 index 1cfe7e76..00000000 --- a/http/models/Address.js +++ /dev/null @@ -1,211 +0,0 @@ -'use strict'; - -var imports = require('soop').imports(); -var async = require('async'); -var bitcore = require('bitcore'); -var BitcoreAddress = bitcore.Address; -var BitcoreTransaction = bitcore.Transaction; -var BitcoreUtil = bitcore.util; -var Parser = bitcore.BinaryParser; -var Buffer = bitcore.Buffer; -var TransactionDb = imports.TransactionDb || require('../../lib/TransactionDb').default(); -var BlockDb = imports.BlockDb || require('../../lib/BlockDb').default(); -var config = require('../../config/config'); -var CONCURRENCY = 5; - -function Address(addrStr) { - this.balanceSat = 0; - this.totalReceivedSat = 0; - this.totalSentSat = 0; - - this.unconfirmedBalanceSat = 0; - - this.txApperances = 0; - this.unconfirmedTxApperances= 0; - this.seen = {}; - - // TODO store only txids? +index? +all? - this.transactions = []; - this.unspent = []; - - var a = new BitcoreAddress(addrStr); - a.validate(); - this.addrStr = addrStr; - - Object.defineProperty(this, 'totalSent', { - get: function() { - return parseFloat(this.totalSentSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.totalSentSat = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); - - Object.defineProperty(this, 'balance', { - get: function() { - return parseFloat(this.balanceSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.balance = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); - - Object.defineProperty(this, 'totalReceived', { - get: function() { - return parseFloat(this.totalReceivedSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.totalReceived = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); - - - Object.defineProperty(this, 'unconfirmedBalance', { - get: function() { - return parseFloat(this.unconfirmedBalanceSat) / parseFloat(BitcoreUtil.COIN); - }, - set: function(i) { - this.unconfirmedBalanceSat = i * BitcoreUtil.COIN; - }, - enumerable: 1, - }); - -} - -Address.prototype.getObj = function() { - // Normalize json address - return { - 'addrStr': this.addrStr, - 'balance': this.balance, - 'balanceSat': this.balanceSat, - 'totalReceived': this.totalReceived, - 'totalReceivedSat': this.totalReceivedSat, - 'totalSent': this.totalSent, - 'totalSentSat': this.totalSentSat, - 'unconfirmedBalance': this.unconfirmedBalance, - 'unconfirmedBalanceSat': this.unconfirmedBalanceSat, - 'unconfirmedTxApperances': this.unconfirmedTxApperances, - 'txApperances': this.txApperances, - 'transactions': this.transactions - }; -}; - -Address.prototype._addTxItem = function(txItem, txList, includeInfo) { - function addTx(data) { - if (!txList) return; - if (includeInfo) { - txList.push(data); - } else { - txList.push(data.txid); - } - }; - - var add=0, addSpend=0; - var v = txItem.value_sat; - var seen = this.seen; - - // Founding tx - if (!seen[txItem.txid]) { - seen[txItem.txid] = 1; - add = 1; - - addTx({ txid: txItem.txid, ts: txItem.ts }); - } - - // Spent tx - if (txItem.spentTxId && !seen[txItem.spentTxId] ) { - addTx({ txid: txItem.spentTxId, ts: txItem.spentTs }); - seen[txItem.spentTxId]=1; - addSpend=1; - } - if (txItem.isConfirmed) { - this.txApperances += add; - this.totalReceivedSat += v; - if (! txItem.spentTxId ) { - //unspent - this.balanceSat += v; - } - else if(!txItem.spentIsConfirmed) { - // unspent - this.balanceSat += v; - this.unconfirmedBalanceSat -= v; - this.unconfirmedTxApperances += addSpend; - } - else { - // spent - this.totalSentSat += v; - this.txApperances += addSpend; - } - } - else { - this.unconfirmedBalanceSat += v; - this.unconfirmedTxApperances += add; - } -}; - -// opts are -// .onlyUnspent -// .txLimit (=0 -> no txs, => -1 no limit) -// .includeTxInfo -// -Address.prototype.update = function(next, opts) { - var self = this; - if (!self.addrStr) return next(); - opts = opts || {}; - - if (! ('ignoreCache' in opts) ) - opts.ignoreCache = config.ignoreCache; - - // should collect txList from address? - var txList = opts.txLimit === 0 ? null: []; - - var tDb = TransactionDb; - var bDb = BlockDb; - tDb.fromAddr(self.addrStr, opts, function(err,txOut){ - if (err) return next(err); - - bDb.fillConfirmations(txOut, function(err) { - if (err) return next(err); - - tDb.cacheConfirmations(txOut, function(err) { -// console.log('[Address.js.161:txOut:]',txOut); //TODO - if (err) return next(err); - if (opts.onlyUnspent) { - txOut = txOut.filter(function(x){ - return !x.spentTxId; - }); - tDb.fillScriptPubKey(txOut, function() { - self.unspent = txOut.map(function(x){ - return { - address: self.addrStr, - txid: x.txid, - vout: x.index, - ts: x.ts, - scriptPubKey: x.scriptPubKey, - amount: x.value_sat / BitcoreUtil.COIN, - confirmations: x.isConfirmedCached ? (config.safeConfirmations) : x.confirmations, - confirmationsFromCache: !!x.isConfirmedCached, - }; - }); - return next(); - }); - } - else { - txOut.forEach(function(txItem){ - self._addTxItem(txItem, txList, opts.includeTxInfo); - }); - if (txList) - self.transactions = txList; - - return next(); - } - }); - }); - }); -}; - -module.exports = require('soop')(Address); - diff --git a/http/models/Status.js b/http/models/Status.js deleted file mode 100644 index ffb744ad..00000000 --- a/http/models/Status.js +++ /dev/null @@ -1,105 +0,0 @@ -'use strict'; -//var imports = require('soop').imports(); - -var async = require('async'); -var bitcore = require('bitcore'); -var RpcClient = bitcore.RpcClient; -var config = require('../../config/config'); -var rpc = new RpcClient(config.bitcoind); -var bDb = require('../../lib/BlockDb').default(); - -function Status() {} - -Status.prototype.getInfo = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getInfo(function(err, info){ - if (err) return cb(err); - - that.info = info.result; - return cb(); - }); - }, - ], function (err) { - return next(err); - }); -}; - -Status.prototype.getDifficulty = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getDifficulty(function(err, df){ - if (err) return cb(err); - - that.difficulty = df.result; - return cb(); - }); - } - ], function (err) { - return next(err); - }); -}; - -Status.prototype.getTxOutSetInfo = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getTxOutSetInfo(function(err, txout){ - if (err) return cb(err); - - that.txoutsetinfo = txout.result; - return cb(); - }); - } - ], function (err) { - return next(err); - }); -}; - -Status.prototype.getBestBlockHash = function(next) { - var that = this; - async.series([ - function (cb) { - rpc.getBestBlockHash(function(err, bbh){ - if (err) return cb(err); - - that.bestblockhash = bbh.result; - return cb(); - }); - }, - - ], function (err) { - return next(err); - }); -}; - -Status.prototype.getLastBlockHash = function(next) { - var that = this; - bDb.getTip(function(err,tip) { - that.syncTipHash = tip; - async.waterfall( - [ - function(callback){ - rpc.getBlockCount(function(err, bc){ - if (err) return callback(err); - callback(null, bc.result); - }); - }, - function(bc, callback){ - rpc.getBlockHash(bc, function(err, bh){ - if (err) return callback(err); - callback(null, bh.result); - }); - } - ], - function (err, result) { - that.lastblockhash = result; - return next(); - } - ); - }); -}; - -module.exports = require('soop')(Status); diff --git a/http/routes.js b/http/routes.js deleted file mode 100644 index eeedf7de..00000000 --- a/http/routes.js +++ /dev/null @@ -1,65 +0,0 @@ -'use strict'; - -/** - * Module dependencies. - */ -var config = require('./config'); - -module.exports = function(app) { - - var apiPrefix = config.apiPrefix; - - //Block routes - var blocks = require('../app/controllers/blocks'); - app.get(apiPrefix + '/blocks', blocks.list); - - - app.get(apiPrefix + '/block/:blockHash', blocks.show); - app.param('blockHash', blocks.block); - - app.get(apiPrefix + '/block-index/:height', blocks.blockindex); - app.param('height', blocks.blockindex); - - // Transaction routes - var transactions = require('../app/controllers/transactions'); - app.get(apiPrefix + '/tx/:txid', transactions.show); - app.param('txid', transactions.transaction); - app.get(apiPrefix + '/txs', transactions.list); - app.post(apiPrefix + '/tx/send', transactions.send); - - // Address routes - var addresses = require('../app/controllers/addresses'); - app.get(apiPrefix + '/addr/:addr', addresses.show); - app.get(apiPrefix + '/addr/:addr/utxo', addresses.utxo); - app.get(apiPrefix + '/addrs/:addrs/utxo', addresses.multiutxo); - app.post(apiPrefix + '/addrs/utxo', addresses.multiutxo); - app.get(apiPrefix + '/addrs/:addrs/txs', addresses.multitxs); - app.post(apiPrefix + '/addrs/txs', addresses.multitxs); - - // Address property routes - app.get(apiPrefix + '/addr/:addr/balance', addresses.balance); - app.get(apiPrefix + '/addr/:addr/totalReceived', addresses.totalReceived); - app.get(apiPrefix + '/addr/:addr/totalSent', addresses.totalSent); - app.get(apiPrefix + '/addr/:addr/unconfirmedBalance', addresses.unconfirmedBalance); - - // Status route - var st = require('../app/controllers/status'); - app.get(apiPrefix + '/status', st.show); - - app.get(apiPrefix + '/sync', st.sync); - app.get(apiPrefix + '/peer', st.peer); - - // Currency - var currency = require('../app/controllers/currency'); - app.get(apiPrefix + '/currency', currency.index); - - // Address routes - var messages = require('../app/controllers/messages'); - app.get(apiPrefix + '/messages/verify', messages.verify); - app.post(apiPrefix + '/messages/verify', messages.verify); - - //Home route - var index = require('../app/controllers/index'); - app.get(apiPrefix + '/version', index.version); - app.get('*', index.render); -}; From 12f101451c85b59ac77ab387ef8805c788d6ed1d Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 16:18:57 -0300 Subject: [PATCH 09/42] Add db to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 04327abc..83b58634 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ db/blocks/* db/blocks db/testnet/blocks/* db/testnet/blocks +db/* README.html public From a5a8230166a221fc6751b9daa9c3139bdf6a48eb Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 16:47:42 -0300 Subject: [PATCH 10/42] Get rid of short names --- lib/node.js | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/node.js b/lib/node.js index 92dd8bf7..f9e7d8c9 100644 --- a/lib/node.js +++ b/lib/node.js @@ -12,24 +12,24 @@ var NetworkMonitor = require('./networkmonitor'); var EventBus = require('./eventbus'); var BlockService = require('./services/block.js'); -var BitcoreNode = function(bus, nm) { +var BitcoreNode = function(bus, networkMonitor) { $.checkArgument(bus); - $.checkArgument(nm); + $.checkArgument(networkMonitor); var self = this; this.bus = bus; - this.nm = nm; + this.networkMonitor = networkMonitor; - this.bs = new BlockService(); + this.blockService = new BlockService(); - this.bus.register(bitcore.Block, this.bs.onBlock.bind(this.bs)); + this.bus.register(bitcore.Block, this.blockService.onBlock.bind(this.blockService)); this.bus.onAny(function(value) { self.emit(this.event, value); }); - this.nm.on('error', function(err) { + this.networkMonitor.on('error', function(err) { self.emit('error', err); }); - this.nm.on('disconnect', function() { + this.networkMonitor.on('disconnect', function() { console.log('network monitor disconnected'); }); @@ -39,30 +39,30 @@ util.inherits(BitcoreNode, EventEmitter); BitcoreNode.create = function(opts) { opts = opts || {}; var bus = new EventBus(); - var nm = NetworkMonitor.create(bus, opts.NetworkMonitor); - return new BitcoreNode(bus, nm); + var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); + return new BitcoreNode(bus, networkMonitor); }; BitcoreNode.prototype.start = function() { this.sync(); - this.nm.start(); + this.networkMonitor.start(); }; BitcoreNode.prototype.sync = function() { var genesis = bitcore.Networks.defaultNetwork.genesis; var self = this; - this.nm.on('ready', function() { + this.networkMonitor.on('ready', function() { console.log('ready'); - self.bs.getLatest().then(function(latest) { + self.blockService.getLatest().then(function(latest) { var start = genesis; if (latest) { start = latest.rawHash; } console.log('Starting sync from', start); - self.nm.syncFrom(start); + self.networkMonitor.syncFrom(start); }) .catch(function(err) { - self.nm.disconnect(); + self.networkMonitor.disconnect(); throw err; }); }); From 541a06f5c3a31564fa30127b1d4f87fba3847eb7 Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 17:34:29 -0300 Subject: [PATCH 11/42] Fix minor issues with rpc --- api/controllers/blocks.js | 4 ++-- lib/services/block.js | 10 ++++------ package.json | 3 +-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/api/controllers/blocks.js b/api/controllers/blocks.js index c7478c90..f8dd5661 100644 --- a/api/controllers/blocks.js +++ b/api/controllers/blocks.js @@ -38,7 +38,7 @@ Blocks.blockHashParam = function(req, res, next, blockHash) { */ Blocks.heightParam = function(req, res, next, height) { height = parseInt(height); - node.getBlock(height) + node.blockService.getBlockByHeight(height) .then(function(block) { req.block = block; }) @@ -83,7 +83,7 @@ Blocks.list = function(req, res) { }; Blocks.getLatest = function(req, res) { - node.getLatestBlock() + node.blockService.getLatest() .then(function(block) { req.block = block; Blocks.get(req, res); diff --git a/lib/services/block.js b/lib/services/block.js index 7f5b4335..41e3d73a 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -1,7 +1,6 @@ 'use strict'; var LevelUp = require('levelup'); -var LevelLock = require('level-lock'); var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); var TransactionService = require('./transaction'); @@ -95,7 +94,6 @@ BlockService.prototype.unlock = function() { * @return {bitcore.Block} */ BlockService.blockRPCtoBitcore = function(blockData) { - console.log(blockData); $.checkArgument(blockData, 'blockData is required'); var block = new bitcore.Block({ header: new bitcore.BlockHeader({ @@ -113,7 +111,7 @@ BlockService.blockRPCtoBitcore = function(blockData) { new bitcore.deps.Buffer(blockData.merkleroot, 'hex') ) }), - transactions: blockData.transaction + transactions: blockData.transactions }); block.height = blockData.height; return block; @@ -185,12 +183,13 @@ BlockService.prototype.getBlockByHeight = function(height) { return Promise.try(function() { - return self.rpc.getBlockHash(height); + return self.rpc.getBlockHashAsync(height); }) .catch(blockNotFound) - .then(function(blockHash) { + .then(function(result) { + var blockHash = result.result; return self.getBlock(blockHash); }); @@ -225,7 +224,6 @@ BlockService.prototype.getLatest = function() { * @return a list of events back to the event bus */ BlockService.prototype.onBlock = function(block) { - console.log('block', block); var events = []; return this.save(block) .then(function(block) { diff --git a/package.json b/package.json index bf3093e7..9eec33e2 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,7 @@ "express": "4.11.1", "glob": "*", "js-yaml": "^3.2.7", - "leveldown": "^0.10.4", - "levelup": "^0.19.0", + "leveldown": "^1.0.1", "moment": "~2.5.0", "morgan": "^1.5.1", "request": "^2.48.0", From de62831851a58601fcdf6fe32502efcb2e0e8c2b Mon Sep 17 00:00:00 2001 From: eordano Date: Wed, 25 Mar 2015 17:52:18 -0300 Subject: [PATCH 12/42] Improve initialization of a simple node --- api/controllers/transactions.js | 2 +- lib/node.js | 38 +++++++++++++++++++++++++++++---- lib/services/address.js | 1 + 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/api/controllers/transactions.js b/api/controllers/transactions.js index 388f297a..2dd73599 100644 --- a/api/controllers/transactions.js +++ b/api/controllers/transactions.js @@ -25,7 +25,7 @@ Transactions.setNode = function(aNode) { * Finds a transaction by its hash */ Transactions.txHashParam = function(req, res, next, txHash) { - node.getTransaction(txHash) + node.transactionService.getTransaction(txHash) .then(function(tx) { req.tx = tx; }) diff --git a/lib/node.js b/lib/node.js index f9e7d8c9..36c8f309 100644 --- a/lib/node.js +++ b/lib/node.js @@ -4,22 +4,31 @@ var util = require('util'); var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); +var config = require('config'); var p2p = require('bitcore-p2p'); var messages = new p2p.Messages(); var $ = bitcore.util.preconditions; +var Promise = require('bluebird'); +var RPC = require('bitcoind-rpc'); var NetworkMonitor = require('./networkmonitor'); var EventBus = require('./eventbus'); -var BlockService = require('./services/block.js'); -var BitcoreNode = function(bus, networkMonitor) { +var LevelUp = require('levelup'); +var BlockService = require('./services/block'); +var TransactionService = require('./services/transaction'); +var AddressService = require('./services/address'); + +var BitcoreNode = function(bus, networkMonitor, blockService, transactionService, addressService) { $.checkArgument(bus); $.checkArgument(networkMonitor); var self = this; this.bus = bus; this.networkMonitor = networkMonitor; - this.blockService = new BlockService(); + this.addressService = addressService; + this.transactionService = transactionService; + this.blockService = blockService; this.bus.register(bitcore.Block, this.blockService.onBlock.bind(this.blockService)); @@ -38,9 +47,30 @@ util.inherits(BitcoreNode, EventEmitter); BitcoreNode.create = function(opts) { opts = opts || {}; + var bus = new EventBus(); + var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); - return new BitcoreNode(bus, networkMonitor); + + var database = new LevelUp(opts.LevelUp || config.get('LevelUp')); + var rpc = Promise.promisifyAll(new RPC(config.get('RPC'))); + + var transactionService = new TransactionService({ + rpc: rpc, + database: database + }); + var blockService = new BlockService({ + rpc: rpc, + database: database, + transactionService: transactionService + }); + var addressService = new AddressService({ + rpc: rpc, + database: database, + transactionService: transactionService, + blockService: blockService + }); + return new BitcoreNode(bus, networkMonitor, blockService, transactionService, addressService); }; BitcoreNode.prototype.start = function() { diff --git a/lib/services/address.js b/lib/services/address.js index 47c37896..76e6092e 100644 --- a/lib/services/address.js +++ b/lib/services/address.js @@ -3,6 +3,7 @@ var Promise = require('bluebird'); var bitcore = require('bitcore'); var TransactionService = require('./transaction'); +var RPC = require('bitcoind-rpc'); var _ = bitcore.deps._; var NULLTXHASH = bitcore.util.buffer.emptyBuffer(32).toString('hex'); From 8121f52c38c7a03157323c5f755177b1273b05e3 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 26 Mar 2015 12:57:45 -0300 Subject: [PATCH 13/42] WIP --- lib/networkmonitor.js | 18 ++++++------- lib/node.js | 11 ++++---- lib/services/block.js | 61 +++++++++++++++++++++++-------------------- package.json | 1 + 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 210c6452..2d161614 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -45,20 +45,18 @@ NetworkMonitor.prototype.setupPeer = function(peer) { peer.on('tx', function(m) { self.bus.process(m.transaction) .catch(function(err) { - self.stop(err); + self.abort(err); }); - //.catch(self.stop.bind(self)); }); peer.on('block', function(m) { self.bus.process(m.block) .catch(function(err) { - self.stop(err); + self.abort(err); }); - //.catch(self.stop.bind(self)); }); peer.on('error', function(err) { self.emit('error', err); - self.stop(err); + self.abort(err); }); peer.on('disconnect', function() { self.emit('disconnect'); @@ -70,11 +68,13 @@ NetworkMonitor.prototype.start = function() { this.peer.connect(); }; NetworkMonitor.prototype.stop = function(reason) { - console.trace(); this.peer.disconnect(); - if (reason) { - throw reason; - } + console.log('Stopping network, reason:', reason); +}; +NetworkMonitor.prototype.abort = function(reason) { + this.peer.disconnect(); + console.log('Unexpected error, aborting:', reason); + process.exit(1); }; NetworkMonitor.prototype.syncFrom = function(start) { diff --git a/lib/node.js b/lib/node.js index 36c8f309..37ee068f 100644 --- a/lib/node.js +++ b/lib/node.js @@ -52,7 +52,7 @@ BitcoreNode.create = function(opts) { var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); - var database = new LevelUp(opts.LevelUp || config.get('LevelUp')); + var database = Promise.promisifyAll(new LevelUp(opts.LevelUp || config.get('LevelUp'))); var rpc = Promise.promisifyAll(new RPC(config.get('RPC'))); var transactionService = new TransactionService({ @@ -79,20 +79,21 @@ BitcoreNode.prototype.start = function() { }; BitcoreNode.prototype.sync = function() { - var genesis = bitcore.Networks.defaultNetwork.genesis; + var genesis = bitcore.Block.fromBuffer(bitcore.Networks.defaultNetwork.genesis); var self = this; this.networkMonitor.on('ready', function() { console.log('ready'); self.blockService.getLatest().then(function(latest) { - var start = genesis; + var start = genesis.hash; + console.log('latest', latest); if (latest) { - start = latest.rawHash; + start = latest.hash; } console.log('Starting sync from', start); self.networkMonitor.syncFrom(start); }) .catch(function(err) { - self.networkMonitor.disconnect(); + self.networkMonitor.stop(); throw err; }); }); diff --git a/lib/services/block.js b/lib/services/block.js index 41e3d73a..90e8d8c9 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -183,16 +183,16 @@ BlockService.prototype.getBlockByHeight = function(height) { return Promise.try(function() { - return self.rpc.getBlockHashAsync(height); + return self.rpc.getBlockHashAsync(height); - }) - .catch(blockNotFound) - .then(function(result) { + }) + .catch(blockNotFound) + .then(function(result) { - var blockHash = result.result; - return self.getBlock(blockHash); + var blockHash = result.result; + return self.getBlock(blockHash); - }); + }); }; /** @@ -227,7 +227,7 @@ BlockService.prototype.onBlock = function(block) { var events = []; return this.save(block) .then(function(block) { - return undefined; + console.log('block', block.id, 'saved'); block.transactions.forEach(function(tx) { events.push(tx); }); @@ -261,38 +261,41 @@ BlockService.prototype._confirmBlock = function(block) { var ops = []; return this.writeLock().then(function() { + return self._setNextBlock(ops, block.header.prevHash, block); - return self._setNextBlock(ops, block.header.prevHash, block); + }).then(function() { - }).then(function() { + if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { + return self.getBlock(block.header.prevHash, { + withoutTransactions: true + }); + } else { + return GENESISPARENT; + } - if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { - return self.getBlock(block.header.prevHash, { - withoutTransactions: true - }); - } else { - return GENESISPARENT; - } + }).then(function(parent) { - }).then(function(parent) { + return self._setBlockHeight(ops, block, parent.height + 1); - return self._setBlockHeight(ops, block, parent.height + 1); + }).then(function() { - }).then(function() { + return self._setBlockByTs(ops, block); - return self._setBlockByTs(ops, block); + }).then(function() { - }).then(function() { + return Promise.all(block.transactions.map(function(transaction) { + return self.transactionService._confirmTransaction(ops, block, transaction); + })); - return Promise.all(block.transactions.map(function(transaction) { - return self.transactionService._confirmTransaction(ops, block, transaction); - })); + }).then(function() { - }).then(function() { + return self.database.batchAsync(ops); - return self.database.batchAsync(ops); - - }).then(this.unlock.bind(this)); + }) + .then(this.unlock.bind(this)) + .then(function() { + return block; + }); }; BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { diff --git a/package.json b/package.json index 9eec33e2..8799a9b2 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "glob": "*", "js-yaml": "^3.2.7", "leveldown": "^1.0.1", + "levelup": "git://github.com/rvagg/node-levelup", "moment": "~2.5.0", "morgan": "^1.5.1", "request": "^2.48.0", From c58d4156557f0d6789b7cd1702789fb47dcadfe3 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 31 Mar 2015 18:46:25 -0300 Subject: [PATCH 14/42] fix mutex race condition --- config/default.yml | 2 +- lib/networkmonitor.js | 1 + lib/node.js | 5 ++++- lib/services/block.js | 35 ++++++++++++++++++++++++++--------- package.json | 5 +++-- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/config/default.yml b/config/default.yml index fb60e8e2..48ddb954 100644 --- a/config/default.yml +++ b/config/default.yml @@ -1,4 +1,5 @@ BitcoreNode: + LevelUp: ./db network: livenet NetworkMonitor: host: localhost @@ -7,7 +8,6 @@ Reporter: none # none, simple, matrix BitcoreHTTP: host: localhost port: 8080 -LevelUp: ./db RPC: user: user pass: password diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 2d161614..85a5acd5 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -49,6 +49,7 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }); }); peer.on('block', function(m) { + console.log('block', m.block.id); self.bus.process(m.block) .catch(function(err) { self.abort(err); diff --git a/lib/node.js b/lib/node.js index 37ee068f..30ba9549 100644 --- a/lib/node.js +++ b/lib/node.js @@ -52,7 +52,10 @@ BitcoreNode.create = function(opts) { var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); - var database = Promise.promisifyAll(new LevelUp(opts.LevelUp || config.get('LevelUp'))); + console.log(opts.LevelUp); + var database = Promise.promisifyAll( + new LevelUp(opts.LevelUp || config.get('LevelUp')) + ); var rpc = Promise.promisifyAll(new RPC(config.get('RPC'))); var transactionService = new TransactionService({ diff --git a/lib/services/block.js b/lib/services/block.js index 90e8d8c9..b49e5b09 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -63,13 +63,19 @@ function BlockService(opts) { BlockService.prototype.writeLock = function() { var self = this; - return new Promise(function(resolve, reject) { - if (self.lock) { - return reject(); - } else { + return new Promise(function(resolve) { + console.log('getting lock'); + var checkLock = function() { + if (self.lock) { + setImmediate(checkLock); + return; + } self.lock = true; - return resolve(); - } + console.log('lock acquired'); + resolve(); + + }; + checkLock(); }); }; @@ -260,40 +266,51 @@ BlockService.prototype._confirmBlock = function(block) { var ops = []; - return this.writeLock().then(function() { + return this.writeLock() + .then(function() { + console.log(1); return self._setNextBlock(ops, block.header.prevHash, block); }).then(function() { + console.log(2); if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { + console.log(2.1); return self.getBlock(block.header.prevHash, { withoutTransactions: true }); } else { + console.log(2.2); return GENESISPARENT; } }).then(function(parent) { + console.log(3); return self._setBlockHeight(ops, block, parent.height + 1); }).then(function() { + console.log(4); return self._setBlockByTs(ops, block); }).then(function() { + console.log(5); return Promise.all(block.transactions.map(function(transaction) { return self.transactionService._confirmTransaction(ops, block, transaction); })); }).then(function() { - - return self.database.batchAsync(ops); + console.log(6); + var p = self.database.batchAsync(ops); + console.log(6.5); + return p; }) .then(this.unlock.bind(this)) .then(function() { + console.log(7); return block; }); }; diff --git a/package.json b/package.json index 8799a9b2..a5cfec7d 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,9 @@ "express": "4.11.1", "glob": "*", "js-yaml": "^3.2.7", - "leveldown": "^1.0.1", - "levelup": "git://github.com/rvagg/node-levelup", + "leveldown": "~1.0.0", + "levelup": "^0.19.0", + "memdown": "^1.0.0", "moment": "~2.5.0", "morgan": "^1.5.1", "request": "^2.48.0", From 9a41658f18e6b5984bbb1286ae1ef1c9cb7a9d9c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 1 Apr 2015 15:39:28 -0300 Subject: [PATCH 15/42] make events be processed sequencially --- lib/eventbus.js | 16 ++++++++++++-- lib/networkmonitor.js | 4 +--- lib/services/block.js | 49 ++++++++++++------------------------------- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/lib/eventbus.js b/lib/eventbus.js index 2859d9a8..90730674 100644 --- a/lib/eventbus.js +++ b/lib/eventbus.js @@ -37,13 +37,25 @@ EventBus.prototype.process = function(e) { ); }); }; - var eventsEmitted = processEvent(e) + + var whenPreviousFinishes = Promise.resolve(); + if (this.previous && !this.previous.isFulfilled()) { + whenPreviousFinishes = this.previous; + } + + var current = whenPreviousFinishes + .then(function() { + return processEvent(e); + }) .then(function() { done.forEach(function(event) { self.emit(event.name || event.constructor.name, event); }); }); - return eventsEmitted; + current.e = e; + this.previous = current; + return current; + }; diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 85a5acd5..e0a1c2ba 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -49,7 +49,6 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }); }); peer.on('block', function(m) { - console.log('block', m.block.id); self.bus.process(m.block) .catch(function(err) { self.abort(err); @@ -74,8 +73,7 @@ NetworkMonitor.prototype.stop = function(reason) { }; NetworkMonitor.prototype.abort = function(reason) { this.peer.disconnect(); - console.log('Unexpected error, aborting:', reason); - process.exit(1); + throw reason; }; NetworkMonitor.prototype.syncFrom = function(start) { diff --git a/lib/services/block.js b/lib/services/block.js index b49e5b09..39468652 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -14,7 +14,6 @@ var $ = bitcore.util.preconditions; var JSUtil = bitcore.util.js; var _ = bitcore.deps._; -var LOCK = 'lock-'; var NULLBLOCKHASH = bitcore.util.buffer.emptyBuffer(32).toString('hex'); var GENESISPARENT = { height: -1, @@ -61,27 +60,6 @@ function BlockService(opts) { }); } -BlockService.prototype.writeLock = function() { - var self = this; - return new Promise(function(resolve) { - console.log('getting lock'); - var checkLock = function() { - if (self.lock) { - setImmediate(checkLock); - return; - } - self.lock = true; - console.log('lock acquired'); - resolve(); - - }; - checkLock(); - }); -}; - -BlockService.prototype.unlock = function() { - this.lock = false; -}; /** * Transforms data as received from an RPC result structure for `getblock`, @@ -233,7 +211,7 @@ BlockService.prototype.onBlock = function(block) { var events = []; return this.save(block) .then(function(block) { - console.log('block', block.id, 'saved'); + console.log('block', block.id, 'saved with height', block.height); block.transactions.forEach(function(tx) { events.push(tx); }); @@ -266,51 +244,49 @@ BlockService.prototype._confirmBlock = function(block) { var ops = []; - return this.writeLock() - .then(function() { - console.log(1); + return Promise.try(function() { + //console.log(1); return self._setNextBlock(ops, block.header.prevHash, block); }).then(function() { - console.log(2); + //console.log(2); if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { - console.log(2.1); + //console.log(2.1); return self.getBlock(block.header.prevHash, { withoutTransactions: true }); } else { - console.log(2.2); + //console.log(2.2); return GENESISPARENT; } }).then(function(parent) { - console.log(3); + //console.log(3); return self._setBlockHeight(ops, block, parent.height + 1); }).then(function() { - console.log(4); + //console.log(4); return self._setBlockByTs(ops, block); }).then(function() { - console.log(5); + //console.log(5); return Promise.all(block.transactions.map(function(transaction) { return self.transactionService._confirmTransaction(ops, block, transaction); })); }).then(function() { - console.log(6); + //console.log(6); var p = self.database.batchAsync(ops); - console.log(6.5); + //console.log(6.5); return p; }) - .then(this.unlock.bind(this)) .then(function() { - console.log(7); + //console.log(7); return block; }); }; @@ -334,6 +310,7 @@ BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { }; BlockService.prototype._setBlockHeight = function(ops, block, height) { + block.height = height; return Promise.try(function() { ops.push({ type: 'put', From abac6a9ed3df392751e4507638f0fa8dbabf97f0 Mon Sep 17 00:00:00 2001 From: eordano Date: Mon, 6 Apr 2015 12:50:08 -0300 Subject: [PATCH 16/42] Dont require patched version of bitcore --- lib/data/genesis.js | 13 +++++++++++++ lib/node.js | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 lib/data/genesis.js diff --git a/lib/data/genesis.js b/lib/data/genesis.js new file mode 100644 index 00000000..46bf2f59 --- /dev/null +++ b/lib/data/genesis.js @@ -0,0 +1,13 @@ +module.exports = { + livenet: new Buffer('010000000000000000000000000000000000000000000000000000000000' + + '0000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a5132' + + '3a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c01010000000100000000' + + '00000000000000000000000000000000000000000000000000000000ffff' + + 'ffff4d04ffff001d0104455468652054696d65732030332f4a616e2f3230' + + '3039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f' + + '6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01' + + '000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a6' + + '7962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b' + + '8d578a4c702b6bf11d5fac00000000', 'hex'), + testnet: new Buffer('') +} diff --git a/lib/node.js b/lib/node.js index 30ba9549..d504af47 100644 --- a/lib/node.js +++ b/lib/node.js @@ -19,6 +19,8 @@ var BlockService = require('./services/block'); var TransactionService = require('./services/transaction'); var AddressService = require('./services/address'); +var genesisBlocks = require('./data/genesis'); + var BitcoreNode = function(bus, networkMonitor, blockService, transactionService, addressService) { $.checkArgument(bus); $.checkArgument(networkMonitor); @@ -82,7 +84,7 @@ BitcoreNode.prototype.start = function() { }; BitcoreNode.prototype.sync = function() { - var genesis = bitcore.Block.fromBuffer(bitcore.Networks.defaultNetwork.genesis); + var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); var self = this; this.networkMonitor.on('ready', function() { console.log('ready'); From 13b4f41f80189bff6983a6b4a7d37bc4406e5615 Mon Sep 17 00:00:00 2001 From: eordano Date: Mon, 6 Apr 2015 15:14:12 -0300 Subject: [PATCH 17/42] Change name to sync on networkmonitor --- lib/networkmonitor.js | 10 +++++----- lib/node.js | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index e0a1c2ba..fb3435c3 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -64,6 +64,11 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }; +NetworkMonitor.prototype.requestBlocks = function(start) { + $.checkArgument(_.isString(start), 'start must be a block hash string'); + this.peer.sendMessage(messages.GetBlocks([start])); +}; + NetworkMonitor.prototype.start = function() { this.peer.connect(); }; @@ -76,9 +81,4 @@ NetworkMonitor.prototype.abort = function(reason) { throw reason; }; -NetworkMonitor.prototype.syncFrom = function(start) { - $.checkArgument(_.isString(start), 'start must be a block hash string'); - this.peer.sendMessage(messages.GetBlocks([start])); -}; - module.exports = NetworkMonitor; diff --git a/lib/node.js b/lib/node.js index d504af47..bd6b9f54 100644 --- a/lib/node.js +++ b/lib/node.js @@ -28,6 +28,8 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.bus = bus; this.networkMonitor = networkMonitor; + this.tip = null; + this.addressService = addressService; this.transactionService = transactionService; this.blockService = blockService; @@ -95,7 +97,8 @@ BitcoreNode.prototype.sync = function() { start = latest.hash; } console.log('Starting sync from', start); - self.networkMonitor.syncFrom(start); + self.tip = latest; + self.networkMonitor.requestBlocks(start); }) .catch(function(err) { self.networkMonitor.stop(); From 9c7fa5cfaf8ed77aac132c1aedc606c7795541c6 Mon Sep 17 00:00:00 2001 From: eordano Date: Mon, 6 Apr 2015 15:44:31 -0300 Subject: [PATCH 18/42] Add blockchain --- lib/blockchain.js | 45 +++++++++++++++++++++++++++++++++++++++++++ lib/node.js | 17 ++++++++-------- lib/services/block.js | 25 +++++++++++++++++++++++- 3 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 lib/blockchain.js diff --git a/lib/blockchain.js b/lib/blockchain.js new file mode 100644 index 00000000..49cbafc8 --- /dev/null +++ b/lib/blockchain.js @@ -0,0 +1,45 @@ +'use strict'; + +var bitcore = require('bitcore'); +var $ = bitcore.util.preconditions; +var _ = bitcore.deps._; + +function BlockChain() { + this.tip = null; + this.headers = {}; + this.next = {}; + this.prev = {}; +} + +BlockChain.fromObject = function(obj) { + var blockchain = new BlockChain(); + blockchain.tip = obj.tip; + blockchain.headers = obj.headers; + blockchain.next = obj.next; + blockchain.prev = obj.prev; + return blockchain; +}; + +BlockChain.prototype.setTip = function(block) { + $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); + this.tip = block.hash; + this.headers[block.hash] = block.header; + var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); + this.next[prevHash] = block.hash; + this.prev[block.hash] = prevHash; +}; + +BlockChain.prototype.toObject = function() { + return { + tip: this.tip, + headers: _.map(this.headers, function(header) { return header.toObject(); }), + next: this.next, + prev: this.prev + }; +}; + +BlockChain.prototype.toJSON = function() { + return JSON.stringify(this.toObject()); +}; + +module.exports = BlockChain; diff --git a/lib/node.js b/lib/node.js index bd6b9f54..49a6a1a3 100644 --- a/lib/node.js +++ b/lib/node.js @@ -19,6 +19,7 @@ var BlockService = require('./services/block'); var TransactionService = require('./services/transaction'); var AddressService = require('./services/address'); +var BlockChain = require('./blockchain'); var genesisBlocks = require('./data/genesis'); var BitcoreNode = function(bus, networkMonitor, blockService, transactionService, addressService) { @@ -89,16 +90,14 @@ BitcoreNode.prototype.sync = function() { var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); var self = this; this.networkMonitor.on('ready', function() { - console.log('ready'); - self.blockService.getLatest().then(function(latest) { - var start = genesis.hash; - console.log('latest', latest); - if (latest) { - start = latest.hash; + self.blockService.getBlockchain().then(function(blockchain) { + if (blockchain) { + self.blockchain = blockchain; + } else { + self.blockchain = new BlockChain(); + self.blockchain.setTip(genesis); } - console.log('Starting sync from', start); - self.tip = latest; - self.networkMonitor.requestBlocks(start); + self.networkMonitor.requestBlocks(self.blockchain.tip); }) .catch(function(err) { self.networkMonitor.stop(); diff --git a/lib/services/block.js b/lib/services/block.js index 39468652..3c614035 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -39,7 +39,8 @@ var Index = { prev: 'prev-', // prev- -> parent hash next: 'nxt-', // nxt- -> hash for the next block in the main chain that is a child height: 'bh-', // bh- -> height (-1 means disconnected) - tip: 'tip' // tip -> { hash: hex, height: int }, the latest tip + tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip + blockchain: 'chain' }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -381,4 +382,26 @@ BlockService.prototype.getBlockForTransaction = function(transaction) { }); }; +BlockService.prototype.getBlockchain = function() { + var self = this; + return new Promise(function(resolve, reject) { + self.database.getAsync(Index.blockchain).then(function(blockchain) { + blockchain = BlockChain.fromObject(JSON.parse(blockchain)); + return resolve(blockchain); + }).catch(function(arg) { + if (arg.message.indexOf('Key not found in database') !== -1) { + return resolve(); + } + return reject(arg); + }); + }); +}; + +BlockService.prototype.saveBlockchain = function(blockchain) { + return this.database.putAsync({ + key: Index.blockchain, + value: blockchain.toJSON() + }); +}; + module.exports = BlockService; From 1907e5f92ba2af1fc7e4688634d24e8d7b57cdea Mon Sep 17 00:00:00 2001 From: eordano Date: Tue, 7 Apr 2015 15:13:21 -0300 Subject: [PATCH 19/42] Refactor to blockchain --- lib/blockchain.js | 119 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 49cbafc8..50b46275 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -6,7 +6,11 @@ var _ = bitcore.deps._; function BlockChain() { this.tip = null; - this.headers = {}; + this.work = { + '0000000000000000000000000000000000000000000000000000000000000000': 0 + }; + this.height = {}; + this.hashByHeight = {}; this.next = {}; this.prev = {}; } @@ -14,26 +18,127 @@ function BlockChain() { BlockChain.fromObject = function(obj) { var blockchain = new BlockChain(); blockchain.tip = obj.tip; - blockchain.headers = obj.headers; + blockchain.work = obj.work; + blockchain.hashByHeight = obj.hashByHeight; + blockchain.height = obj.height; blockchain.next = obj.next; blockchain.prev = obj.prev; return blockchain; }; -BlockChain.prototype.setTip = function(block) { +var getWork = function(bits) { + var bytes = ((bits >>> 24) & 0xff) >>> 0; + return ((bits & 0xffffff) << (8 * (bytes - 3))) >>> 0; +}; + +BlockChain.prototype.addData = function(block) { $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); - this.tip = block.hash; - this.headers[block.hash] = block.header; + var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); - this.next[prevHash] = block.hash; + + this.work[block.hash] = this.work[prevHash].work + getWork(block.header.bits); this.prev[block.hash] = prevHash; }; +BlockChain.prototype.proposeNewBlock = function(block) { + $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); + var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); + + if (!this.work[prevHash]) { + throw new Error('No previous data to estimate work'); + } + this.addData(block); + + if (this.work[block.hash] > this.work[this.tip]) { + + var toUnconfirm = []; + var toConfirm = []; + var commonAncestor; + + var pointer = block.hash; + while (!this.height[pointer]) { + toConfirm.push(pointer); + pointer = this.prev[pointer]; + } + commonAncestor = pointer; + + pointer = this.tip; + while (pointer != commonAncestor) { + toUnconfirm.push(pointer); + pointer = this.prev[pointer]; + } + + toConfirm.reverse(); + + var self = this; + toUnconfirm.map(function(hash) { + self.unconfirm(hash); + }); + toConfirm.map(function(hash) { + self.confirm(hash); + }); + return { + unconfirmed: toUnconfirm, + confirmed: toConfirm + }; + } + return { + unconfirmed: [], + confirmed: [] + }; +}; + +BlockChain.prototype.confirm = function(hash) { + var prevHash = this.prev[hash]; + $.checkState(prevHash === this.tip); + + this.tip = hash; + var height = this.height[prevHash] + 1; + this.next[prevHash] = hash; + this.hashByHeight[height] = hash; + this.height[hash] = height; +}; + +BlockChain.prototype.unconfirm = function(hash) { + var prevHash = this.prev[hash]; + $.checkState(hash === this.tip); + + this.tip = prevHash; + var height = this.height[hash]; + this.next[prevHash] = undefined; + this.hashByHeight[height] = undefined; + this.height[hash] = undefined; +}; + +BlockChain.prototype.getBlockLocator = function() { + $.checkState(this.tip); + $.checkState(this.height[this.tip]); + + var result = []; + var currentHeight = this.height[this.tip]; + var exponentialBackOff = 1; + for (var i = 0; i < 10; i++) { + result.push(this.hashByHeight[currentHeight--]); + } + while (currentHeight > 0) { + result.push(this.hashByHeight[currentHeight]); + currentHeight -= exponentialBackOff; + exponentialBackOff *= 2; + } + return result; +}; + +BlockChain.prototype.hasData = function(hash) { + return !!this.prev[hash]; +}; + BlockChain.prototype.toObject = function() { return { tip: this.tip, - headers: _.map(this.headers, function(header) { return header.toObject(); }), + work: this.work, next: this.next, + hashByHeight: this.hashByHeight, + height: this.height, prev: this.prev }; }; From 7d2b759f2be92620f7e559e9ebbdfeb5b89a6a72 Mon Sep 17 00:00:00 2001 From: eordano Date: Tue, 7 Apr 2015 15:28:12 -0300 Subject: [PATCH 20/42] Prune --- lib/blockchain.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 50b46275..6498491b 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -63,7 +63,7 @@ BlockChain.prototype.proposeNewBlock = function(block) { commonAncestor = pointer; pointer = this.tip; - while (pointer != commonAncestor) { + while (pointer !== commonAncestor) { toUnconfirm.push(pointer); pointer = this.prev[pointer]; } @@ -105,9 +105,9 @@ BlockChain.prototype.unconfirm = function(hash) { this.tip = prevHash; var height = this.height[hash]; - this.next[prevHash] = undefined; - this.hashByHeight[height] = undefined; - this.height[hash] = undefined; + delete this.next[prevHash]; + delete this.hashByHeight[height]; + delete this.height[hash]; }; BlockChain.prototype.getBlockLocator = function() { @@ -132,6 +132,16 @@ BlockChain.prototype.hasData = function(hash) { return !!this.prev[hash]; }; +BlockChain.prototype.prune = function() { + var self = this; + _.each(this.prev, function(key, value) { + if (!self.height[key]) { + delete this.prev[key]; + delete this.work[key]; + } + }); +}; + BlockChain.prototype.toObject = function() { return { tip: this.tip, From 1cd51ef944d8ec0d81985c2b3d650c68712a40d4 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 7 Apr 2015 15:50:13 -0300 Subject: [PATCH 21/42] handle SIGINT --- index.js | 4 ++++ lib/node.js | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/index.js b/index.js index 9b3e729b..8ffbaab9 100644 --- a/index.js +++ b/index.js @@ -21,6 +21,10 @@ if (require.main === module) { console.log('Error: ', err); } }); + process.on('SIGINT', function() { + node.stop(); + process.exit(); + }); var reporterName = config.get('Reporter'); var reporter = reporters[reporterName]; diff --git a/lib/node.js b/lib/node.js index 49a6a1a3..3fc762eb 100644 --- a/lib/node.js +++ b/lib/node.js @@ -86,6 +86,11 @@ BitcoreNode.prototype.start = function() { this.networkMonitor.start(); }; +BitcoreNode.prototype.stop = function() { + this.networkMonitor.stop(); +}; + + BitcoreNode.prototype.sync = function() { var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); var self = this; From acfbef8a333f68ba89200f0fb920a530e017a466 Mon Sep 17 00:00:00 2001 From: eordano Date: Tue, 7 Apr 2015 16:08:46 -0300 Subject: [PATCH 22/42] Reorgs and sync --- lib/node.js | 57 ++++++++++++++++++++++++++++++------------- lib/services/block.js | 5 ++-- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/lib/node.js b/lib/node.js index 3fc762eb..b89ef2d8 100644 --- a/lib/node.js +++ b/lib/node.js @@ -35,7 +35,29 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.transactionService = transactionService; this.blockService = blockService; - this.bus.register(bitcore.Block, this.blockService.onBlock.bind(this.blockService)); + this.blockCache = {}; + + this.bus.register(bitcore.Block, function(block) { + + var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); + self.blockCache[block.hash] = block; + + if (!self.blockchain.hasData(prevHash)) { + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + return; + } + + var blockchainChanges = self.blockchain.proposeNewBlock(block); + Promise.series(blockchainChanges.unconfirmed.map(function(hash) { + return self.blockService.unconfirm(self.blockCache[hash]); + })) + .then(Promise.series(blockchainChanges.confirmed.map(function(hash) { + return self.blockService.confirm(self.blockCache[hash]); + }))) + .then(function() { + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + }); + }); this.bus.onAny(function(value) { self.emit(this.event, value); @@ -82,8 +104,18 @@ BitcoreNode.create = function(opts) { }; BitcoreNode.prototype.start = function() { - this.sync(); - this.networkMonitor.start(); + var self = this; + this.blockService.getBlockchain().then(function(blockchain) { + if (!blockchain) { + self.blockchain = new BlockChain(); + self.blockchain.setTip(genesis); + } + self.sync(); + self.networkMonitor.start(); + }); + this.networkMonitor.on('stop', function() { + self.blockService.saveBlockchain(self.blockchain); + }); }; BitcoreNode.prototype.stop = function() { @@ -92,24 +124,15 @@ BitcoreNode.prototype.stop = function() { BitcoreNode.prototype.sync = function() { - var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); var self = this; this.networkMonitor.on('ready', function() { self.blockService.getBlockchain().then(function(blockchain) { - if (blockchain) { - self.blockchain = blockchain; - } else { - self.blockchain = new BlockChain(); - self.blockchain.setTip(genesis); - } - self.networkMonitor.requestBlocks(self.blockchain.tip); - }) - .catch(function(err) { - self.networkMonitor.stop(); - throw err; - }); + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + }).catch(function(err) { + self.networkMonitor.stop(); + throw err; + }); }); }; - module.exports = BitcoreNode; diff --git a/lib/services/block.js b/lib/services/block.js index 3c614035..9a6a0680 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -226,9 +226,8 @@ BlockService.prototype.onBlock = function(block) { * @param {bitcore.Block} block * @return {Promise} a promise of the same block, for chaining */ -BlockService.prototype.save = function(block) { +BlockService.prototype.unconfirm = function(block) { // TODO: unconfirm previous tip, confirm new tip. - return this._confirmBlock(block); }; @@ -238,7 +237,7 @@ BlockService.prototype.save = function(block) { * @param {bitcore.Block} block * @return {Promise} a promise of the same block, for chaining */ -BlockService.prototype._confirmBlock = function(block) { +BlockService.prototype.confirm = function(block) { $.checkArgument(block instanceof bitcore.Block); var self = this; From 1d7ff62a9ea139bd17bb23689e252638ecc8c468 Mon Sep 17 00:00:00 2001 From: eordano Date: Tue, 7 Apr 2015 16:18:08 -0300 Subject: [PATCH 23/42] Fix to locator --- lib/blockchain.js | 16 +++++++++++----- lib/networkmonitor.js | 5 +++-- lib/node.js | 11 ++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 6498491b..7cadd778 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -5,12 +5,16 @@ var $ = bitcore.util.preconditions; var _ = bitcore.deps._; function BlockChain() { - this.tip = null; + this.tip = '0000000000000000000000000000000000000000000000000000000000000000'; this.work = { '0000000000000000000000000000000000000000000000000000000000000000': 0 }; - this.height = {}; - this.hashByHeight = {}; + this.height = { + '0000000000000000000000000000000000000000000000000000000000000000': -1 + }; + this.hashByHeight = { + '-1': '0000000000000000000000000000000000000000000000000000000000000000' + }; this.next = {}; this.prev = {}; } @@ -44,7 +48,7 @@ BlockChain.prototype.proposeNewBlock = function(block) { $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); - if (!this.work[prevHash]) { + if (_.isUndefined(this.work[prevHash])) { throw new Error('No previous data to estimate work'); } this.addData(block); @@ -118,7 +122,9 @@ BlockChain.prototype.getBlockLocator = function() { var currentHeight = this.height[this.tip]; var exponentialBackOff = 1; for (var i = 0; i < 10; i++) { - result.push(this.hashByHeight[currentHeight--]); + if (currentHeight >= 0) { + result.push(this.hashByHeight[currentHeight--]); + } } while (currentHeight > 0) { result.push(this.hashByHeight[currentHeight]); diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index fb3435c3..56acfb0a 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -65,8 +65,9 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }; NetworkMonitor.prototype.requestBlocks = function(start) { - $.checkArgument(_.isString(start), 'start must be a block hash string'); - this.peer.sendMessage(messages.GetBlocks([start])); + $.checkArgument(_.isArray(start) || + _.isString(start), 'start must be a block hash string or array'); + this.peer.sendMessage(messages.GetBlocks(_.isArray(start) ? start : [start])); }; NetworkMonitor.prototype.start = function() { diff --git a/lib/node.js b/lib/node.js index b89ef2d8..be57b068 100644 --- a/lib/node.js +++ b/lib/node.js @@ -48,12 +48,12 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService } var blockchainChanges = self.blockchain.proposeNewBlock(block); - Promise.series(blockchainChanges.unconfirmed.map(function(hash) { + Promise.each(blockchainChanges.unconfirmed, function(hash) { return self.blockService.unconfirm(self.blockCache[hash]); - })) - .then(Promise.series(blockchainChanges.confirmed.map(function(hash) { + }) + .then(Promise.each(blockchainChanges.confirmed, function(hash) { return self.blockService.confirm(self.blockCache[hash]); - }))) + })) .then(function() { self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); }); @@ -105,10 +105,11 @@ BitcoreNode.create = function(opts) { BitcoreNode.prototype.start = function() { var self = this; + var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); this.blockService.getBlockchain().then(function(blockchain) { if (!blockchain) { self.blockchain = new BlockChain(); - self.blockchain.setTip(genesis); + self.blockchain.proposeNewBlock(genesis); } self.sync(); self.networkMonitor.start(); From b3ad83b9eee90fd4afdb116adca5c18c9cf576d2 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 7 Apr 2015 16:34:54 -0300 Subject: [PATCH 24/42] fix work bug --- lib/blockchain.js | 2 +- lib/node.js | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 7cadd778..33dae810 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -40,7 +40,7 @@ BlockChain.prototype.addData = function(block) { var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); - this.work[block.hash] = this.work[prevHash].work + getWork(block.header.bits); + this.work[block.hash] = this.work[prevHash] + getWork(block.header.bits); this.prev[block.hash] = prevHash; }; diff --git a/lib/node.js b/lib/node.js index be57b068..61f5b2d9 100644 --- a/lib/node.js +++ b/lib/node.js @@ -39,6 +39,7 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.bus.register(bitcore.Block, function(block) { + console.log('Block', block.id); var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); self.blockCache[block.hash] = block; @@ -48,15 +49,21 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService } var blockchainChanges = self.blockchain.proposeNewBlock(block); + console.log('changes', blockchainChanges); Promise.each(blockchainChanges.unconfirmed, function(hash) { - return self.blockService.unconfirm(self.blockCache[hash]); - }) - .then(Promise.each(blockchainChanges.confirmed, function(hash) { - return self.blockService.confirm(self.blockCache[hash]); - })) - .then(function() { - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); - }); + return self.blockService.unconfirm(self.blockCache[hash]); + }) + .then(function() { + return Promise.all(blockchainChanges.confirmed.map(function(hash) { + return self.blockService.confirm(self.blockCache[hash]); + })); + }) + .then(function() { + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + }) + .catch(function(error) { + self.stop(error); + }); }); this.bus.onAny(function(value) { @@ -79,7 +86,6 @@ BitcoreNode.create = function(opts) { var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); - console.log(opts.LevelUp); var database = Promise.promisifyAll( new LevelUp(opts.LevelUp || config.get('LevelUp')) ); @@ -119,8 +125,8 @@ BitcoreNode.prototype.start = function() { }); }; -BitcoreNode.prototype.stop = function() { - this.networkMonitor.stop(); +BitcoreNode.prototype.stop = function(reason) { + this.networkMonitor.stop(reason); }; From b8a08d05a89cce946297718209f1b82b2eda5a0a Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Tue, 7 Apr 2015 17:29:17 -0300 Subject: [PATCH 25/42] towards sync --- lib/blockchain.js | 6 +++--- lib/networkmonitor.js | 3 ++- lib/node.js | 25 ++++++++++++++++++++++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 33dae810..f313b7cc 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -60,7 +60,7 @@ BlockChain.prototype.proposeNewBlock = function(block) { var commonAncestor; var pointer = block.hash; - while (!this.height[pointer]) { + while (_.isUndefined(this.height[pointer])) { toConfirm.push(pointer); pointer = this.prev[pointer]; } @@ -116,7 +116,7 @@ BlockChain.prototype.unconfirm = function(hash) { BlockChain.prototype.getBlockLocator = function() { $.checkState(this.tip); - $.checkState(this.height[this.tip]); + $.checkState(!_.isUndefined(this.height[this.tip])); var result = []; var currentHeight = this.height[this.tip]; @@ -135,7 +135,7 @@ BlockChain.prototype.getBlockLocator = function() { }; BlockChain.prototype.hasData = function(hash) { - return !!this.prev[hash]; + return !_.isUndefined(this.work[hash]); }; BlockChain.prototype.prune = function() { diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 56acfb0a..5c5f7312 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -39,6 +39,7 @@ NetworkMonitor.prototype.setupPeer = function(peer) { self.emit('ready'); }); peer.on('inv', function(m) { + self.emit('inv', m.inventory); // TODO only ask for data if tx or block is unknown peer.sendMessage(messages.GetData(m.inventory)); }); @@ -66,7 +67,7 @@ NetworkMonitor.prototype.setupPeer = function(peer) { NetworkMonitor.prototype.requestBlocks = function(start) { $.checkArgument(_.isArray(start) || - _.isString(start), 'start must be a block hash string or array'); + _.isString(start), 'start must be a block hash string or array'); this.peer.sendMessage(messages.GetBlocks(_.isArray(start) ? start : [start])); }; diff --git a/lib/node.js b/lib/node.js index 61f5b2d9..a4f020b9 100644 --- a/lib/node.js +++ b/lib/node.js @@ -4,6 +4,7 @@ var util = require('util'); var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); +var _ = bitcore.deps._; var config = require('config'); var p2p = require('bitcore-p2p'); var messages = new p2p.Messages(); @@ -36,12 +37,27 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.blockService = blockService; this.blockCache = {}; + this.inventory = {}; // blockHash -> bool (has data) + + + this.networkMonitor.on('inv', function(inventory) { + _.each(inventory, function(info) { + var hash = bitcore.util.buffer.reverse(info.hash).toString('hex'); + $.checkState(_.isUndefined(self.inventory[hash])); + if (info.type === 2) { // TODO: use static field from bitcore-p2p + self.inventory[hash] = false; + } + }); + }); this.bus.register(bitcore.Block, function(block) { console.log('Block', block.id); var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); self.blockCache[block.hash] = block; + self.inventory[block.hash] = true; + console.log('prevHash', prevHash); + console.log('height', self.blockchain.height[self.blockchain.tip]); if (!self.blockchain.hasData(prevHash)) { self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); @@ -49,7 +65,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService } var blockchainChanges = self.blockchain.proposeNewBlock(block); - console.log('changes', blockchainChanges); Promise.each(blockchainChanges.unconfirmed, function(hash) { return self.blockService.unconfirm(self.blockCache[hash]); }) @@ -59,7 +74,11 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService })); }) .then(function() { - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + if (_.size(self.inventory) && _.all(_.values(self.inventory))) { + self.inventory = {}; + console.log('requesting ...', self.blockchain.getBlockLocator().length); + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + } }) .catch(function(error) { self.stop(error); @@ -115,7 +134,7 @@ BitcoreNode.prototype.start = function() { this.blockService.getBlockchain().then(function(blockchain) { if (!blockchain) { self.blockchain = new BlockChain(); - self.blockchain.proposeNewBlock(genesis); + self.bus.process(genesis); } self.sync(); self.networkMonitor.start(); From 9f290649e6ca816be101a973b9c4a1af62a4ac97 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 8 Apr 2015 16:04:16 -0300 Subject: [PATCH 26/42] p2p sync algorithm working --- lib/networkmonitor.js | 9 +++++---- lib/node.js | 36 ++++++++++++++++-------------------- lib/services/block.js | 16 +--------------- lib/services/transaction.js | 2 ++ 4 files changed, 24 insertions(+), 39 deletions(-) diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 5c5f7312..7fbd83c4 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -65,10 +65,11 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }; -NetworkMonitor.prototype.requestBlocks = function(start) { - $.checkArgument(_.isArray(start) || - _.isString(start), 'start must be a block hash string or array'); - this.peer.sendMessage(messages.GetBlocks(_.isArray(start) ? start : [start])); +NetworkMonitor.prototype.requestBlocks = function(locator) { + $.checkArgument(_.isArray(locator) && _.isString(locator[0]), 'start must be a block hash string array'); + this.peer.sendMessage(messages.GetBlocks({ + starts: locator + })); }; NetworkMonitor.prototype.start = function() { diff --git a/lib/node.js b/lib/node.js index a4f020b9..78a01eee 100644 --- a/lib/node.js +++ b/lib/node.js @@ -6,8 +6,6 @@ var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); var _ = bitcore.deps._; var config = require('config'); -var p2p = require('bitcore-p2p'); -var messages = new p2p.Messages(); var $ = bitcore.util.preconditions; var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); @@ -39,30 +37,27 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.blockCache = {}; this.inventory = {}; // blockHash -> bool (has data) - + /* this.networkMonitor.on('inv', function(inventory) { _.each(inventory, function(info) { var hash = bitcore.util.buffer.reverse(info.hash).toString('hex'); - $.checkState(_.isUndefined(self.inventory[hash])); - if (info.type === 2) { // TODO: use static field from bitcore-p2p + $.checkState(_.isUndefined(self.inventory[hash]), hash); + if (self.inventory[hash] && info.type === 2) { // TODO: use static field from bitcore-p2p self.inventory[hash] = false; } }); }); - + */ this.bus.register(bitcore.Block, function(block) { - console.log('Block', block.id); var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); self.blockCache[block.hash] = block; self.inventory[block.hash] = true; - console.log('prevHash', prevHash); - console.log('height', self.blockchain.height[self.blockchain.tip]); - if (!self.blockchain.hasData(prevHash)) { - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + self.requestFromTip(); return; } + console.log('block', block.id, 'height', self.blockchain.height[self.blockchain.tip] + 1); var blockchainChanges = self.blockchain.proposeNewBlock(block); Promise.each(blockchainChanges.unconfirmed, function(hash) { @@ -70,14 +65,15 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService }) .then(function() { return Promise.all(blockchainChanges.confirmed.map(function(hash) { + self.blockCache[hash].height = self.blockchain.height[hash]; return self.blockService.confirm(self.blockCache[hash]); })); }) .then(function() { - if (_.size(self.inventory) && _.all(_.values(self.inventory))) { + // TODO: include this + if (false && _.size(self.inventory) && _.all(_.values(self.inventory))) { self.inventory = {}; - console.log('requesting ...', self.blockchain.getBlockLocator().length); - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + self.requestFromTip(); } }) .catch(function(error) { @@ -98,6 +94,11 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService }; util.inherits(BitcoreNode, EventEmitter); +BitcoreNode.prototype.requestFromTip = function() { + var locator = this.blockchain.getBlockLocator(); + this.networkMonitor.requestBlocks(locator); +}; + BitcoreNode.create = function(opts) { opts = opts || {}; @@ -152,12 +153,7 @@ BitcoreNode.prototype.stop = function(reason) { BitcoreNode.prototype.sync = function() { var self = this; this.networkMonitor.on('ready', function() { - self.blockService.getBlockchain().then(function(blockchain) { - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); - }).catch(function(err) { - self.networkMonitor.stop(); - throw err; - }); + self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); }); }; diff --git a/lib/services/block.js b/lib/services/block.js index 9a6a0680..38d833e9 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -249,22 +249,9 @@ BlockService.prototype.confirm = function(block) { return self._setNextBlock(ops, block.header.prevHash, block); }).then(function() { - //console.log(2); - - if (block.header.prevHash.toString('hex') !== NULLBLOCKHASH) { - //console.log(2.1); - return self.getBlock(block.header.prevHash, { - withoutTransactions: true - }); - } else { - //console.log(2.2); - return GENESISPARENT; - } - - }).then(function(parent) { //console.log(3); - return self._setBlockHeight(ops, block, parent.height + 1); + return self._setBlockHeight(ops, block, block.height); }).then(function() { //console.log(4); @@ -310,7 +297,6 @@ BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { }; BlockService.prototype._setBlockHeight = function(ops, block, height) { - block.height = height; return Promise.try(function() { ops.push({ type: 'put', diff --git a/lib/services/transaction.js b/lib/services/transaction.js index 375b5d85..22c2ea96 100644 --- a/lib/services/transaction.js +++ b/lib/services/transaction.js @@ -179,6 +179,7 @@ TransactionService.prototype._getAddressForInput = function(input) { hash, bitcore.Networks.defaultNetwork, bitcore.Address.PayToPublicKeyHash ); } else if (script.isPublicKeyIn()) { + /* return self.getTransaction(input.prevTxId.toString('hex')).then(function(transaction) { var outputScript = transaction.outputs[input.outputIndex].script; if (outputScript.isPublicKeyOut()) { @@ -189,6 +190,7 @@ TransactionService.prototype._getAddressForInput = function(input) { } return; }); + */ } else { return new bitcore.Script(script.chunks[script.chunks.length - 1]).toAddress(); } From fe038ac5f8a52d6fea1adbd78ad17a12a9b33e94 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 9 Apr 2015 13:17:19 -0300 Subject: [PATCH 27/42] faster sync via p2p --- lib/eventbus.js | 1 - lib/networkmonitor.js | 8 ++++++-- lib/node.js | 3 ++- lib/services/block.js | 31 +++++++++++++++++++++++-------- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/eventbus.js b/lib/eventbus.js index 90730674..e67c09e3 100644 --- a/lib/eventbus.js +++ b/lib/eventbus.js @@ -52,7 +52,6 @@ EventBus.prototype.process = function(e) { self.emit(event.name || event.constructor.name, event); }); }); - current.e = e; this.previous = current; return current; diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index 7fbd83c4..fb500299 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -66,19 +66,23 @@ NetworkMonitor.prototype.setupPeer = function(peer) { }; NetworkMonitor.prototype.requestBlocks = function(locator) { - $.checkArgument(_.isArray(locator) && _.isString(locator[0]), 'start must be a block hash string array'); + $.checkArgument(_.isArray(locator) && + _.isUndefined(locator[0]) || + _.isString(locator[0]), 'start must be a block hash string array'); this.peer.sendMessage(messages.GetBlocks({ - starts: locator + starts: locator, })); }; NetworkMonitor.prototype.start = function() { this.peer.connect(); }; + NetworkMonitor.prototype.stop = function(reason) { this.peer.disconnect(); console.log('Stopping network, reason:', reason); }; + NetworkMonitor.prototype.abort = function(reason) { this.peer.disconnect(); throw reason; diff --git a/lib/node.js b/lib/node.js index 78a01eee..c8c78ca0 100644 --- a/lib/node.js +++ b/lib/node.js @@ -96,6 +96,7 @@ util.inherits(BitcoreNode, EventEmitter); BitcoreNode.prototype.requestFromTip = function() { var locator = this.blockchain.getBlockLocator(); + console.log('requesting blocks, locator size:', locator.length); this.networkMonitor.requestBlocks(locator); }; @@ -153,7 +154,7 @@ BitcoreNode.prototype.stop = function(reason) { BitcoreNode.prototype.sync = function() { var self = this; this.networkMonitor.on('ready', function() { - self.networkMonitor.requestBlocks(self.blockchain.getBlockLocator()); + self.requestFromTip(); }); }; diff --git a/lib/services/block.js b/lib/services/block.js index 38d833e9..eb4fb229 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -244,31 +244,35 @@ BlockService.prototype.confirm = function(block) { var ops = []; + //console.log(0); return Promise.try(function() { //console.log(1); return self._setNextBlock(ops, block.header.prevHash, block); - }).then(function() { + }) + .then(function() { //console.log(3); return self._setBlockHeight(ops, block, block.height); - }).then(function() { + }) + .then(function() { //console.log(4); return self._setBlockByTs(ops, block); - }).then(function() { + }) + .then(function() { //console.log(5); return Promise.all(block.transactions.map(function(transaction) { return self.transactionService._confirmTransaction(ops, block, transaction); })); - }).then(function() { + }) + .then(function() { //console.log(6); var p = self.database.batchAsync(ops); - //console.log(6.5); return p; }) @@ -308,15 +312,22 @@ BlockService.prototype._setBlockHeight = function(ops, block, height) { }; BlockService.prototype._setBlockByTs = function(ops, block) { + + // TODO: uncomment this + /* var self = this; - var key = Index.timestamp + block.time; + var key = Index.timestamp + block.header.time; + console.log('key', key); return Promise.try(function() { + console.log('a'); return self.database.getAsync(key); - }).then(function(result) { + }) + .then(function(result) { + console.log('b'); if (result === block.hash) { return Promise.resolve(); } else { @@ -324,7 +335,10 @@ BlockService.prototype._setBlockByTs = function(ops, block) { throw new Error('Found blocks that have same timestamp'); } - }).error(function(err) { + }) + .error(function(err) { + console.log('err', err); + // TODO: Check if err is not found return ops.push({ type: 'put', @@ -332,6 +346,7 @@ BlockService.prototype._setBlockByTs = function(ops, block) { value: block.hash }); }); + */ }; /** From 2da7e9c23908d53001d0a720f2ea73168a4fedfd Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 9 Apr 2015 17:05:29 -0300 Subject: [PATCH 28/42] fix promise chaining --- lib/eventbus.js | 2 ++ lib/networkmonitor.js | 1 + lib/node.js | 11 ++++++---- lib/services/block.js | 50 +++++++++++++++---------------------------- 4 files changed, 27 insertions(+), 37 deletions(-) diff --git a/lib/eventbus.js b/lib/eventbus.js index e67c09e3..750c41b2 100644 --- a/lib/eventbus.js +++ b/lib/eventbus.js @@ -40,11 +40,13 @@ EventBus.prototype.process = function(e) { var whenPreviousFinishes = Promise.resolve(); if (this.previous && !this.previous.isFulfilled()) { + //console.log('setting new task with other running, lets queue'); whenPreviousFinishes = this.previous; } var current = whenPreviousFinishes .then(function() { + //console.log('ok, lets go with the new block'); return processEvent(e); }) .then(function() { diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index fb500299..bfc25a32 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -71,6 +71,7 @@ NetworkMonitor.prototype.requestBlocks = function(locator) { _.isString(locator[0]), 'start must be a block hash string array'); this.peer.sendMessage(messages.GetBlocks({ starts: locator, + //stop: '000000002c05cc2e78923c34df87fd108b22221ac6076c18f3ade378a4d915e9' // TODO: remove this!!! })); }; diff --git a/lib/node.js b/lib/node.js index c8c78ca0..fafd0fb2 100644 --- a/lib/node.js +++ b/lib/node.js @@ -57,15 +57,18 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService self.requestFromTip(); return; } - console.log('block', block.id, 'height', self.blockchain.height[self.blockchain.tip] + 1); - var blockchainChanges = self.blockchain.proposeNewBlock(block); - Promise.each(blockchainChanges.unconfirmed, function(hash) { + var height = self.blockchain.height[block.id]; + if (height % 100 === 0) { + console.log('block', block.id, 'height', height); + } + block.height = height; + + return Promise.each(blockchainChanges.unconfirmed, function(hash) { return self.blockService.unconfirm(self.blockCache[hash]); }) .then(function() { return Promise.all(blockchainChanges.confirmed.map(function(hash) { - self.blockCache[hash].height = self.blockchain.height[hash]; return self.blockService.confirm(self.blockCache[hash]); })); }) diff --git a/lib/services/block.js b/lib/services/block.js index eb4fb229..9cc5beb3 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -247,24 +247,15 @@ BlockService.prototype.confirm = function(block) { //console.log(0); return Promise.try(function() { //console.log(1); - return self._setNextBlock(ops, block.header.prevHash, block); + self._setNextBlock(ops, block.header.prevHash, block); - }) - .then(function() { //console.log(3); + self._setBlockHeight(ops, block, block.height); - return self._setBlockHeight(ops, block, block.height); - - }) - .then(function() { //console.log(4); + self._setBlockByTs(ops, block); - return self._setBlockByTs(ops, block); - - }) - .then(function() { //console.log(5); - return Promise.all(block.transactions.map(function(transaction) { return self.transactionService._confirmTransaction(ops, block, transaction); })); @@ -272,9 +263,7 @@ BlockService.prototype.confirm = function(block) { }) .then(function() { //console.log(6); - var p = self.database.batchAsync(ops); - return p; - + return self.database.batchAsync(ops); }) .then(function() { //console.log(7); @@ -286,28 +275,23 @@ BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { if (bitcore.util.buffer.isBuffer(prevBlockHash)) { prevBlockHash = bitcore.util.buffer.reverse(prevBlockHash).toString('hex'); } - return Promise.try(function() { - ops.push({ - type: 'put', - key: Index.getNextBlock(prevBlockHash), - value: block.hash - }); - ops.push({ - type: 'put', - key: Index.getPreviousBlock(block.hash), - value: prevBlockHash.toString('hex') - }); + ops.push({ + type: 'put', + key: Index.getNextBlock(prevBlockHash), + value: block.hash + }); + ops.push({ + type: 'put', + key: Index.getPreviousBlock(block.hash), + value: prevBlockHash.toString('hex') }); }; BlockService.prototype._setBlockHeight = function(ops, block, height) { - return Promise.try(function() { - ops.push({ - type: 'put', - key: Index.getBlockHeight(block), - value: height - }); - return ops; + ops.push({ + type: 'put', + key: Index.getBlockHeight(block), + value: height }); }; From 035bd6f351e7b4e87dca4cb47c0829380ae107ff Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 9 Apr 2015 17:35:21 -0300 Subject: [PATCH 29/42] fix some tests --- lib/networkmonitor.js | 4 +++- test/networkmonitor.js | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index bfc25a32..9e87523c 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -86,7 +86,9 @@ NetworkMonitor.prototype.stop = function(reason) { NetworkMonitor.prototype.abort = function(reason) { this.peer.disconnect(); - throw reason; + if (reason) { + throw reason; + } }; module.exports = NetworkMonitor; diff --git a/test/networkmonitor.js b/test/networkmonitor.js index 732f8310..40c9d560 100644 --- a/test/networkmonitor.js +++ b/test/networkmonitor.js @@ -38,6 +38,9 @@ describe('NetworkMonitor', function() { block: mockBlock }); }; + peerMock.disconnect = function() { + + }; }); it('instantiates correctly from constructor', function() { @@ -62,7 +65,10 @@ describe('NetworkMonitor', function() { it('broadcasts errors in underlying peer', function(cb) { var nm = new NetworkMonitor(busMock, peerMock); - nm.on('error', cb); + nm.on('error', function() { + console.log('under'); + cb(); + }); nm.start(); peerMock.emit('error'); }); From db17accf9897dc69a6b8a58d83d3736b241c945f Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 9 Apr 2015 18:19:55 -0300 Subject: [PATCH 30/42] fix some more tests --- lib/node.js | 18 ++++++++++-------- test/node.js | 14 ++++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/node.js b/lib/node.js index fafd0fb2..248317cf 100644 --- a/lib/node.js +++ b/lib/node.js @@ -22,8 +22,11 @@ var BlockChain = require('./blockchain'); var genesisBlocks = require('./data/genesis'); var BitcoreNode = function(bus, networkMonitor, blockService, transactionService, addressService) { - $.checkArgument(bus); - $.checkArgument(networkMonitor); + $.checkArgument(bus, 'bus is required'); + $.checkArgument(networkMonitor, 'networkMonitor is required'); + $.checkArgument(blockService, 'blockService is required'); + $.checkArgument(transactionService, 'transactionService is required'); + $.checkArgument(addressService, 'addressService is required'); var self = this; this.bus = bus; this.networkMonitor = networkMonitor; @@ -97,12 +100,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService }; util.inherits(BitcoreNode, EventEmitter); -BitcoreNode.prototype.requestFromTip = function() { - var locator = this.blockchain.getBlockLocator(); - console.log('requesting blocks, locator size:', locator.length); - this.networkMonitor.requestBlocks(locator); -}; - BitcoreNode.create = function(opts) { opts = opts || {}; @@ -153,6 +150,11 @@ BitcoreNode.prototype.stop = function(reason) { this.networkMonitor.stop(reason); }; +BitcoreNode.prototype.requestFromTip = function() { + var locator = this.blockchain.getBlockLocator(); + console.log('requesting blocks, locator size:', locator.length); + this.networkMonitor.requestBlocks(locator); +}; BitcoreNode.prototype.sync = function() { var self = this; diff --git a/test/node.js b/test/node.js index fe9ca6e8..dc4a525b 100644 --- a/test/node.js +++ b/test/node.js @@ -13,16 +13,20 @@ Promise.longStackTraces(); describe('BitcoreNode', function() { // mocks - var busMock, nmMock; + var node, busMock, nmMock, bsMock, tsMock, asMock; beforeEach(function() { busMock = new EventBus(); nmMock = new EventEmitter(); nmMock.start = function() {}; + bsMock = {}; + tsMock = {}; + asMock = {}; + node = new BitcoreNode(busMock, nmMock, bsMock, tsMock, asMock); }); describe('instantiates', function() { it('from constructor', function() { - var node = new BitcoreNode(busMock, nmMock); - should.exist(node); + var n = new BitcoreNode(busMock, nmMock, bsMock, tsMock, asMock); + should.exist(n); }); it('from create', function() { @@ -32,17 +36,15 @@ describe('BitcoreNode', function() { }); it('starts', function() { - var node = new BitcoreNode(busMock, nmMock); + node.start(); node.start.bind(node).should.not.throw(); }); it('broadcasts errors from network monitor', function(cb) { - var node = new BitcoreNode(busMock, nmMock); node.on('error', cb); nmMock.emit('error'); }); it('exposes all events from the event bus', function(cb) { - var node = new BitcoreNode(busMock, nmMock); node.on('foo', cb); busMock.emit('foo'); }); From 7102e63933a24afd708e8c40c65b21457c6518b5 Mon Sep 17 00:00:00 2001 From: eordano Date: Thu, 9 Apr 2015 18:53:03 -0300 Subject: [PATCH 31/42] Restore blockchain from database --- lib/blockchain.js | 18 +++--- lib/node.js | 11 ++-- lib/services/block.js | 126 ++++++++++++++++++++++++++++++++---------- 3 files changed, 111 insertions(+), 44 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index f313b7cc..4bd822dc 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -4,21 +4,19 @@ var bitcore = require('bitcore'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; +var NULL = '0000000000000000000000000000000000000000000000000000000000000000'; + function BlockChain() { - this.tip = '0000000000000000000000000000000000000000000000000000000000000000'; - this.work = { - '0000000000000000000000000000000000000000000000000000000000000000': 0 - }; - this.height = { - '0000000000000000000000000000000000000000000000000000000000000000': -1 - }; - this.hashByHeight = { - '-1': '0000000000000000000000000000000000000000000000000000000000000000' - }; + this.tip = NULL; + this.work = { NULL: 0 }; + this.height = { NULL: -1 }; + this.hashByHeight = { '-1': NULL }; this.next = {}; this.prev = {}; } +BlockChain.NULL = NULL; + BlockChain.fromObject = function(obj) { var blockchain = new BlockChain(); blockchain.tip = obj.tip; diff --git a/lib/node.js b/lib/node.js index 248317cf..75deaed6 100644 --- a/lib/node.js +++ b/lib/node.js @@ -61,11 +61,14 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService return; } var blockchainChanges = self.blockchain.proposeNewBlock(block); - var height = self.blockchain.height[block.id]; - if (height % 100 === 0) { - console.log('block', block.id, 'height', height); + + // Annotate block with extra data from the chain + block.height = self.blockchain.height[block.id]; + block.work = self.blockchain.work[block.id]; + + if (block.height % 100 === 0) { + console.log('block', block.id, 'height', block.height); } - block.height = height; return Promise.each(blockchainChanges.unconfirmed, function(hash) { return self.blockService.unconfirm(self.blockCache[hash]); diff --git a/lib/services/block.js b/lib/services/block.js index 9cc5beb3..f0480b8f 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -9,6 +9,7 @@ var Transaction = bitcore.Transaction; var config = require('config'); var errors = require('../errors'); +var BlockChain = require('../blockchain'); var $ = bitcore.util.preconditions; var JSUtil = bitcore.util.js; @@ -40,7 +41,7 @@ var Index = { next: 'nxt-', // nxt- -> hash for the next block in the main chain that is a child height: 'bh-', // bh- -> height (-1 means disconnected) tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip - blockchain: 'chain' + work: 'wk-' // wk- -> amount of work for block }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -220,29 +221,20 @@ BlockService.prototype.onBlock = function(block) { }); }; -/** - * Save a new block - * - * @param {bitcore.Block} block - * @return {Promise} a promise of the same block, for chaining - */ -BlockService.prototype.unconfirm = function(block) { - // TODO: unconfirm previous tip, confirm new tip. -}; - /** * Set a block as the current tip of the blockchain * * @param {bitcore.Block} block + * @param {Array=} ops * @return {Promise} a promise of the same block, for chaining */ -BlockService.prototype.confirm = function(block) { +BlockService.prototype.confirm = function(block, ops) { $.checkArgument(block instanceof bitcore.Block); var self = this; - var ops = []; + ops = ops || []; //console.log(0); return Promise.try(function() { @@ -250,7 +242,10 @@ BlockService.prototype.confirm = function(block) { self._setNextBlock(ops, block.header.prevHash, block); //console.log(3); - self._setBlockHeight(ops, block, block.height); + self._setBlockHeight(ops, block); + + //console.log(3); + self._setBlockWork(ops, block); //console.log(4); self._setBlockByTs(ops, block); @@ -287,11 +282,19 @@ BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { }); }; -BlockService.prototype._setBlockHeight = function(ops, block, height) { +BlockService.prototype._setBlockHeight = function(ops, block) { ops.push({ type: 'put', key: Index.getBlockHeight(block), - value: height + value: block.height + }); +}; + +BlockService.prototype._setBlockWork = function(ops, block) { + ops.push({ + type: 'put', + key: Index.getBlockWork(block), + value: block.work }); }; @@ -333,6 +336,36 @@ BlockService.prototype._setBlockByTs = function(ops, block) { */ }; +/** + * Unconfirm a block + * + * @param {bitcore.Block} block + * @param {Array=} ops + * @return {Promise} a promise of the same block, for chaining + */ +BlockService.prototype.unconfirm = function(block, ops) { + + ops = ops || []; + + return Promise.try(function() { + + self._removeNextBlock(ops, block.header.prevHash, block); + + self._unsetBlockHeight(ops, block, block.height); + + self._dropBlockByTs(ops, block); + + return Promise.all(block.transactions.map(function(transaction) { + return self.transactionService._unconfirmTransaction(ops, block, transaction); + })); + + }).then(function() { + + return self.database.batchAsync(ops); + + }); +}; + /** * Retrieve the block hash that forms part of the current main chain that confirmed a given * transaction. @@ -368,24 +401,57 @@ BlockService.prototype.getBlockForTransaction = function(transaction) { BlockService.prototype.getBlockchain = function() { var self = this; + return new Promise(function(resolve, reject) { - self.database.getAsync(Index.blockchain).then(function(blockchain) { - blockchain = BlockChain.fromObject(JSON.parse(blockchain)); - return resolve(blockchain); - }).catch(function(arg) { - if (arg.message.indexOf('Key not found in database') !== -1) { - return resolve(); + + var blockchain = new BlockChain(); + + var fetchBlock = function(blockHash) { + return Promise.all([ + function() { + return self.database.getAsync(Index.getPreviousBlock(blockHash)).then(function(prevHash) { + blockchain.prev[blockHash] = prevHash; + blockchain.next[prevHash] = blockHash; + }); + }, + function() { + return self.database.getAsync(Index.getBlockHeight(blockHash)).then(function(height) { + blockchain.height[blockHash] = height; + blockchain.hashByHeight[height] = blockHash; + }); + }, + function() { + return self.database.getAsync(Index.getWork(blockHash)).then(function(work) { + blockchain.work[blockHash] = work; + }); + } + ]).then(function() { + return blockHash; + }); + }; + + var fetchUnlessGenesis = function(blockHash) { + return fetchBlock(blockHash).then(function() { + if (blockchain.prev[blockHash] === BlockChain.NULL) { + return; + } else { + return fetchUnlessGenesis(blockchain.prev[blockHash]); + } + }); + }; + + return self.database.getAsync(Index.tip) + .catch(function(err) { + if (err.notFound) { + return undefined; } - return reject(arg); + throw err; + }) + .then(function(tip) { + blockchain.tip = tip; + return fetchUnlessGenesis(tip); }); }); }; -BlockService.prototype.saveBlockchain = function(blockchain) { - return this.database.putAsync({ - key: Index.blockchain, - value: blockchain.toJSON() - }); -}; - module.exports = BlockService; From 12d5c1cffd265b10a2397105a35fb4af67467b6e Mon Sep 17 00:00:00 2001 From: eordano Date: Thu, 9 Apr 2015 19:07:48 -0300 Subject: [PATCH 32/42] Blockchain retrieval and sync --- lib/blockchain.js | 6 ++- lib/node.js | 6 ++- lib/services/block.js | 99 +++++++++++++++++++++++-------------------- 3 files changed, 63 insertions(+), 48 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 4bd822dc..c40de043 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -8,8 +8,10 @@ var NULL = '0000000000000000000000000000000000000000000000000000000000000000'; function BlockChain() { this.tip = NULL; - this.work = { NULL: 0 }; - this.height = { NULL: -1 }; + this.work = {}; + this.work[NULL] = 0; + this.height = {}; + this.height[NULL] = -1; this.hashByHeight = { '-1': NULL }; this.next = {}; this.prev = {}; diff --git a/lib/node.js b/lib/node.js index 75deaed6..ac644419 100644 --- a/lib/node.js +++ b/lib/node.js @@ -86,7 +86,7 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService } }) .catch(function(error) { - self.stop(error); + self.abort(error); }); }); @@ -136,10 +136,14 @@ BitcoreNode.create = function(opts) { BitcoreNode.prototype.start = function() { var self = this; var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); + this.blockService.getBlockchain().then(function(blockchain) { if (!blockchain) { + console.log('nothing'); self.blockchain = new BlockChain(); self.bus.process(genesis); + } else { + self.blockchain = blockchain; } self.sync(); self.networkMonitor.start(); diff --git a/lib/services/block.js b/lib/services/block.js index f0480b8f..6906a20f 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -47,6 +47,7 @@ _.extend(Index, { getNextBlock: helper(Index.next), getPreviousBlock: helper(Index.prev), getBlockHeight: helper(Index.height), + getBlockWork: helper(Index.work), getBlockByTs: function(block) { return Index.timestamp + block.header.time; } @@ -250,6 +251,8 @@ BlockService.prototype.confirm = function(block, ops) { //console.log(4); self._setBlockByTs(ops, block); + self._setTip(ops, block); + //console.log(5); return Promise.all(block.transactions.map(function(transaction) { return self.transactionService._confirmTransaction(ops, block, transaction); @@ -290,6 +293,14 @@ BlockService.prototype._setBlockHeight = function(ops, block) { }); }; +BlockService.prototype._setTip = function(ops, block) { + ops.push({ + type: 'put', + key: Index.tip, + value: block.hash + }); +}; + BlockService.prototype._setBlockWork = function(ops, block) { ops.push({ type: 'put', @@ -402,54 +413,52 @@ BlockService.prototype.getBlockForTransaction = function(transaction) { BlockService.prototype.getBlockchain = function() { var self = this; - return new Promise(function(resolve, reject) { + var blockchain = new BlockChain(); - var blockchain = new BlockChain(); + var fetchBlock = function(blockHash) { + return Promise.all([ + self.database.getAsync(Index.getPreviousBlock(blockHash)).then(function(prevHash) { + blockchain.prev[blockHash] = prevHash; + blockchain.next[prevHash] = blockHash; + }), + self.database.getAsync(Index.getBlockHeight(blockHash)).then(function(height) { + blockchain.height[blockHash] = +height; + blockchain.hashByHeight[height] = blockHash; + }), + self.database.getAsync(Index.getBlockWork(blockHash)).then(function(work) { + blockchain.work[blockHash] = work; + }) + ]).then(function() { + return blockHash; + }); + }; - var fetchBlock = function(blockHash) { - return Promise.all([ - function() { - return self.database.getAsync(Index.getPreviousBlock(blockHash)).then(function(prevHash) { - blockchain.prev[blockHash] = prevHash; - blockchain.next[prevHash] = blockHash; - }); - }, - function() { - return self.database.getAsync(Index.getBlockHeight(blockHash)).then(function(height) { - blockchain.height[blockHash] = height; - blockchain.hashByHeight[height] = blockHash; - }); - }, - function() { - return self.database.getAsync(Index.getWork(blockHash)).then(function(work) { - blockchain.work[blockHash] = work; - }); - } - ]).then(function() { - return blockHash; - }); - }; - - var fetchUnlessGenesis = function(blockHash) { - return fetchBlock(blockHash).then(function() { - if (blockchain.prev[blockHash] === BlockChain.NULL) { - return; - } else { - return fetchUnlessGenesis(blockchain.prev[blockHash]); - } - }); - }; - - return self.database.getAsync(Index.tip) - .catch(function(err) { - if (err.notFound) { - return undefined; + var fetchUnlessGenesis = function(blockHash) { + return fetchBlock(blockHash).then(function() { + if (blockchain.prev[blockHash] === BlockChain.NULL) { + return; + } else { + return fetchUnlessGenesis(blockchain.prev[blockHash]); } - throw err; - }) - .then(function(tip) { - blockchain.tip = tip; - return fetchUnlessGenesis(tip); + }); + }; + + return self.database.getAsync(Index.tip) + .catch(function(err) { + if (err.notFound) { + return undefined; + } + throw err; + }) + .then(function(tip) { + if (!tip) { + console.log('No tip found'); + return; + } + console.log('Tip is', tip); + blockchain.tip = tip; + return fetchUnlessGenesis(tip).then(function() { + return blockchain; }); }); }; From 16194e5b3fd3682263e5df05c1215e842c1c80a8 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Thu, 9 Apr 2015 19:42:15 -0300 Subject: [PATCH 33/42] delete old blocks from cache --- lib/node.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/node.js b/lib/node.js index ac644419..eee7c8ac 100644 --- a/lib/node.js +++ b/lib/node.js @@ -68,6 +68,7 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService if (block.height % 100 === 0) { console.log('block', block.id, 'height', block.height); + console.log('cache size', Object.keys(self.blockCache).length); } return Promise.each(blockchainChanges.unconfirmed, function(hash) { @@ -78,6 +79,13 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService return self.blockService.confirm(self.blockCache[hash]); })); }) + .then(function() { + var deleteHeight = block.height - 100; + if (deleteHeight > 0) { + var deleteHash = self.blockchain.hashByHeight[deleteHeight]; + delete self.blockCache[deleteHash]; + } + }) .then(function() { // TODO: include this if (false && _.size(self.inventory) && _.all(_.values(self.inventory))) { @@ -86,7 +94,7 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService } }) .catch(function(error) { - self.abort(error); + self.stop(error); }); }); @@ -154,7 +162,7 @@ BitcoreNode.prototype.start = function() { }; BitcoreNode.prototype.stop = function(reason) { - this.networkMonitor.stop(reason); + this.networkMonitor.abort(reason); }; BitcoreNode.prototype.requestFromTip = function() { From 5afcce87daa0587ec55217705176294479fce09d Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 10 Apr 2015 12:26:35 -0300 Subject: [PATCH 34/42] update some more tests --- test/node.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/node.js b/test/node.js index dc4a525b..63c7301a 100644 --- a/test/node.js +++ b/test/node.js @@ -13,12 +13,16 @@ Promise.longStackTraces(); describe('BitcoreNode', function() { // mocks - var node, busMock, nmMock, bsMock, tsMock, asMock; + var node, busMock, nmMock, bsMock, tsMock, asMock, chainMock; beforeEach(function() { busMock = new EventBus(); nmMock = new EventEmitter(); nmMock.start = function() {}; + chainMock = {}; bsMock = {}; + bsMock.getBlockchain = function() { + return Promise.resolve(chainMock); + }; tsMock = {}; asMock = {}; node = new BitcoreNode(busMock, nmMock, bsMock, tsMock, asMock); From 177cf9ac250948caa90cc1e0babd84a1908b68e5 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 10 Apr 2015 12:47:15 -0300 Subject: [PATCH 35/42] fix more tests --- .gitignore | 2 +- lib/node.js | 6 ++---- test/node.js | 22 +++++++++++++++++++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 83b58634..a477a2de 100644 --- a/.gitignore +++ b/.gitignore @@ -24,7 +24,6 @@ report *~ .idea .project -peerdb.json npm-debug.log .nodemonignore @@ -39,6 +38,7 @@ db/blocks db/testnet/blocks/* db/testnet/blocks db/* +db-test/ README.html public diff --git a/lib/node.js b/lib/node.js index eee7c8ac..ab855519 100644 --- a/lib/node.js +++ b/lib/node.js @@ -5,7 +5,6 @@ var EventEmitter = require('eventemitter2').EventEmitter2; var bitcore = require('bitcore'); var _ = bitcore.deps._; -var config = require('config'); var $ = bitcore.util.preconditions; var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); @@ -68,7 +67,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService if (block.height % 100 === 0) { console.log('block', block.id, 'height', block.height); - console.log('cache size', Object.keys(self.blockCache).length); } return Promise.each(blockchainChanges.unconfirmed, function(hash) { @@ -119,9 +117,9 @@ BitcoreNode.create = function(opts) { var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); var database = Promise.promisifyAll( - new LevelUp(opts.LevelUp || config.get('LevelUp')) + new LevelUp(opts.LevelUp) ); - var rpc = Promise.promisifyAll(new RPC(config.get('RPC'))); + var rpc = Promise.promisifyAll(new RPC(opts.RPC)); var transactionService = new TransactionService({ rpc: rpc, diff --git a/test/node.js b/test/node.js index 63c7301a..929e8a99 100644 --- a/test/node.js +++ b/test/node.js @@ -34,7 +34,27 @@ describe('BitcoreNode', function() { }); it('from create', function() { - var node = BitcoreNode.create(); + var opts = { + LevelUp: './db-test', + network: 'testnet', + NetworkMonitor: { + host: 'localhost', + port: 8555, + }, + Reporter: 'none', + BitcoreHTTP: { + host: 'somehost', + port: 9090, + }, + RPC: { + user: 'test-user', + pass: 'test-password', + protocol: 'http', + host: '8.8.8.8', + port: 8552, + } + }; + var node = BitcoreNode.create(opts); should.exist(node); }); }); From 81a8b58c9a2f77316cd509d21720f9e2049b6416 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 10 Apr 2015 12:59:36 -0300 Subject: [PATCH 36/42] fixing more tests --- test/services/block.js | 88 +++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 36 deletions(-) diff --git a/test/services/block.js b/test/services/block.js index f3706012..76eb5c9b 100644 --- a/test/services/block.js +++ b/test/services/block.js @@ -28,7 +28,7 @@ describe('BlockService', function() { describe('getBlock', function() { var mockRpc, transactionMock, database, blockService; - + beforeEach(function() { database = sinon.mock(); mockRpc = sinon.mock(); @@ -43,7 +43,7 @@ describe('BlockService', function() { height: 2, version: 1, merkleroot: '9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5', - tx: [ '9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5' ], + tx: ['9b0fc92260312ce44e74ef369f5c66bbb85848f2eddd5a7a1cde251e54ccfdd5'], time: 1231469744, nonce: 1639830024, bits: '1d00ffff', @@ -63,7 +63,7 @@ describe('BlockService', function() { transactionService: transactionMock, database: database }); - }); + }); it('retrieves correctly a block, uses RPC', function(callback) { @@ -90,7 +90,7 @@ describe('BlockService', function() { var genesisBlock = require('../data/genesis'); var block169 = require('../data/169'); var block170 = require('../data/170'); - + beforeEach(function() { database = sinon.mock(); mockRpc = sinon.mock(); @@ -102,52 +102,66 @@ describe('BlockService', function() { database: database }); blockService.writeLock = sinon.mock(); - }); + }); it('makes the expected calls when confirming the genesis block', function(callback) { database.batchAsync = function(ops) { - ops.should.deep.equal([ - { type: 'put', - key: 'nxt-0000000000000000000000000000000000000000000000000000000000000000', - value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' }, - { type: 'put', - key: 'prev-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - value: '0000000000000000000000000000000000000000000000000000000000000000' }, - { type: 'put', - key: 'bh-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - value: 0 }, - { type: 'put', - key: 'bts-1231006505', - value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' } - ]); + var expectedOps = [{ + type: 'put', + key: 'nxt-0000000000000000000000000000000000000000000000000000000000000000', + value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + }, { + type: 'put', + key: 'prev-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + value: '0000000000000000000000000000000000000000000000000000000000000000' + }, { + type: 'put', + key: 'bh-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + value: 0 + }, { + type: 'put', + key: 'bts-1231006505', + value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + }, { + type: 'put', + key: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + }]; + console.log(ops); + console.log(expectedOps); + ops.should.deep.equal(expectedOps); return thenCaller; }; blockService.unlock = callback; blockService.writeLock.onFirstCall().returns(thenCaller); blockService.getBlock = sinon.mock(); database.getAsync = function() { - return Promise.reject({notFound: true}); + return Promise.reject({ + notFound: true + }); }; transactionMock._confirmTransaction = sinon.mock(); - blockService._confirmBlock(genesisBlock); + blockService.confirm(genesisBlock); }); it('makes the expected calls when confirming the block #170', function(callback) { database.batchAsync = function(ops) { - ops.should.deep.equal([ - { type: 'put', - key: 'nxt-000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55', - value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee' }, - { type: 'put', - key: 'prev-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', - value: '000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55' }, - { type: 'put', - key: 'bh-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', - value: 170 }, - { type: 'put', - key: 'bts-1231731025', - value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee' } - ]); + ops.should.deep.equal([{ + type: 'put', + key: 'nxt-000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55', + value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee' + }, { + type: 'put', + key: 'prev-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', + value: '000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55' + }, { + type: 'put', + key: 'bh-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', + value: 170 + }, { + type: 'put', + key: 'bts-1231731025', + value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee' + }]); return thenCaller; }; blockService.unlock = callback; @@ -156,7 +170,9 @@ describe('BlockService', function() { return Promise.resolve(block169); }; database.getAsync = function() { - return Promise.reject({notFound: true}); + return Promise.reject({ + notFound: true + }); }; transactionMock._confirmTransaction = sinon.spy(); blockService._confirmBlock(block170); From adaa55f36899580de6beae129cef6451a4fd7027 Mon Sep 17 00:00:00 2001 From: eordano Date: Fri, 10 Apr 2015 18:02:51 -0300 Subject: [PATCH 37/42] Tests fixed --- lib/services/block.js | 29 ++++++++++++++++++++- lib/services/transaction.js | 14 +++++++++++ test/services/block.js | 49 ++++++++++++++++-------------------- test/services/transaction.js | 6 +++-- 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/lib/services/block.js b/lib/services/block.js index 6906a20f..e360ef1b 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -281,7 +281,7 @@ BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { ops.push({ type: 'put', key: Index.getPreviousBlock(block.hash), - value: prevBlockHash.toString('hex') + value: prevBlockHash }); }; @@ -377,6 +377,33 @@ BlockService.prototype.unconfirm = function(block, ops) { }); }; +BlockService.prototype._removeNextBlock = function(ops, prevHash, block) { + + if (bitcore.util.buffer.isBuffer(prevBlockHash)) { + prevBlockHash = bitcore.util.buffer.reverse(prevBlockHash).toString('hex'); + } + + ops.push({ + type: 'del', + key: Index.getNextBlock(prevBlockHash) + }); + ops.push({ + type: 'del', + key: Index.getPreviousBlock(block.hash) + }); +}; + +BlockService.prototype._unsetBlockHeight = function(ops, block, height) { + ops.push({ + type: 'del', + key: Index.getBlockHeight(block) + }); +}; + +BlockService.prototype._dropBlockByTs = function(ops, block) { + // TODO +}; + /** * Retrieve the block hash that forms part of the current main chain that confirmed a given * transaction. diff --git a/lib/services/transaction.js b/lib/services/transaction.js index 22c2ea96..a704bc12 100644 --- a/lib/services/transaction.js +++ b/lib/services/transaction.js @@ -210,4 +210,18 @@ TransactionService.prototype._confirmTransaction = function(ops, block, transact )); }; +TransactionService.prototype._unconfirmTransaction = function(ops, block, transaction) { + var self = this; + ops.push({ + type: 'del', + key: Index.getBlockForTransaction(transaction), + value: block.id + }); + return Promise.all( + _.map(transaction.outputs, self._unconfirmOutput(ops, block, transaction)) + .concat( + _.map(transaction.inputs, self._unconfirmInput(ops, block, transaction)) + )); +}; + module.exports = TransactionService; diff --git a/test/services/block.js b/test/services/block.js index 76eb5c9b..b204d0cb 100644 --- a/test/services/block.js +++ b/test/services/block.js @@ -87,9 +87,18 @@ describe('BlockService', function() { return arg(); } }; + var work = 1000; + var work169 = 169; + var work170 = 170; var genesisBlock = require('../data/genesis'); + genesisBlock.work = work; + genesisBlock.height = 1; var block169 = require('../data/169'); + block169.work = work169; + block169.height = 169; var block170 = require('../data/170'); + block170.work = work170; + block170.height = 170; beforeEach(function() { database = sinon.mock(); @@ -120,24 +129,15 @@ describe('BlockService', function() { value: 0 }, { type: 'put', - key: 'bts-1231006505', - value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + key: 'wk-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + value: work }, { type: 'put', - key: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + key: 'tip', + value: genesisBlock.id }]; - console.log(ops); - console.log(expectedOps); ops.should.deep.equal(expectedOps); - return thenCaller; - }; - blockService.unlock = callback; - blockService.writeLock.onFirstCall().returns(thenCaller); - blockService.getBlock = sinon.mock(); - database.getAsync = function() { - return Promise.reject({ - notFound: true - }); + return callback(); }; transactionMock._confirmTransaction = sinon.mock(); blockService.confirm(genesisBlock); @@ -159,23 +159,18 @@ describe('BlockService', function() { value: 170 }, { type: 'put', - key: 'bts-1231731025', - value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee' + key: 'wk-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', + value: work170 + }, { + type: 'put', + key: 'tip', + value: block170.id }]); - return thenCaller; + return callback(); }; - blockService.unlock = callback; blockService.writeLock.onFirstCall().returns(thenCaller); - blockService.getBlock = function() { - return Promise.resolve(block169); - }; - database.getAsync = function() { - return Promise.reject({ - notFound: true - }); - }; transactionMock._confirmTransaction = sinon.spy(); - blockService._confirmBlock(block170); + blockService.confirm(block170); }); }); }); diff --git a/test/services/transaction.js b/test/services/transaction.js index 07561780..ac07a991 100644 --- a/test/services/transaction.js +++ b/test/services/transaction.js @@ -147,7 +147,9 @@ describe('TransactionService', function() { sequenceNumber: 4294967295, script: '71 0x304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901', heightConfirmed: 170 } }, - { type: 'put', + ]); + /* TODO: This should work if address spent is accepted for public key. Add test for P2PKH if not accepted + * { type: 'put', key: 'txas-12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S-f4184fc596403b9d638783cf57adfe4c75c605f6356fbc91338530e9831e9e16-0', value: { heightSpent: 170, @@ -156,7 +158,7 @@ describe('TransactionService', function() { spendInput: { prevTxId: '0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9', outputIndex: 0, sequenceNumber: 4294967295, - script: '71 0x304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901' }}}]); + script: '71 0x304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901' }}}]);*/ callback(); }); }); From 6691f0b4b5b6ba2c8dcf917e865df1c2eaef95f8 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Mon, 13 Apr 2015 17:31:00 -0300 Subject: [PATCH 38/42] more tests working --- api/controllers/blocks.js | 2 +- api/test/http.js | 5 ++++- api/test/v1/blocks.js | 22 ++++++++++++---------- lib/node.js | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/api/controllers/blocks.js b/api/controllers/blocks.js index f8dd5661..01aab381 100644 --- a/api/controllers/blocks.js +++ b/api/controllers/blocks.js @@ -23,7 +23,7 @@ Blocks.setNode = function(aNode) { * Finds a block by its hash */ Blocks.blockHashParam = function(req, res, next, blockHash) { - node.getBlock(blockHash) + node.blockService.getBlock(blockHash) .then(function(block) { req.block = block; }) diff --git a/api/test/http.js b/api/test/http.js index e4d2951d..1ecb38b2 100644 --- a/api/test/http.js +++ b/api/test/http.js @@ -11,6 +11,9 @@ describe('BitcoreHTTP', function() { // mocks var opts = { + BitcoreNode: { + LevelUp: './test-db' + }, port: 1234 }; var nodeMock; @@ -23,7 +26,7 @@ describe('BitcoreHTTP', function() { should.exist(http); }); it('from create', function() { - var http = new BitcoreHTTP.create(); + var http = new BitcoreHTTP.create(opts); should.exist(http); }); }); diff --git a/api/test/v1/blocks.js b/api/test/v1/blocks.js index b25acfa7..5cf29082 100644 --- a/api/test/v1/blocks.js +++ b/api/test/v1/blocks.js @@ -25,26 +25,28 @@ describe('BitcoreHTTP v1 blocks routes', function() { return mockBlocks[hash]; }; var last3 = _.keys(mockBlocks).splice(-3).map(blockForHash); - var some2 = _.keys(mockBlocks).splice(2,2).map(blockForHash); + var some2 = _.keys(mockBlocks).splice(2, 2).map(blockForHash); var nodeMock, app, agent; var blockList = _.values(mockBlocks); beforeEach(function() { nodeMock = new EventEmitter(); - nodeMock.getBlock = function(blockHash) { - var block; - if (typeof blockHash === 'number') { - var height = blockHash; - block = mockBlocks[_.keys(mockBlocks)[height - 100000]]; - } else { - block = mockBlocks[blockHash]; - } + nodeMock.blockService = {}; + nodeMock.blockService.resolveBlock = function(block, blockHash) { if (_.isUndefined(block)) { return Promise.reject(new BitcoreNode.errors.Blocks.NotFound(blockHash)); } return Promise.resolve(block); + }; + nodeMock.blockService.getBlockByHeight = function(height) { + var block = mockBlocks[_.keys(mockBlocks)[height - 100000]]; + return this.resolveBlock(block, height); + }; + nodeMock.blockService.getBlock = function(blockHash) { + var block = mockBlocks[blockHash]; + return this.resolveBlock(block, blockHash); }; - nodeMock.getLatestBlock = function() { + nodeMock.blockService.getLatest = function() { return Promise.resolve(lastBlock); }; nodeMock.listBlocks = function(from, to, offset, limit) { diff --git a/lib/node.js b/lib/node.js index ab855519..125cce59 100644 --- a/lib/node.js +++ b/lib/node.js @@ -117,7 +117,7 @@ BitcoreNode.create = function(opts) { var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); var database = Promise.promisifyAll( - new LevelUp(opts.LevelUp) + new LevelUp(opts.LevelUp || './db') ); var rpc = Promise.promisifyAll(new RPC(opts.RPC)); From 80279a0c6e6e4bfbb1c1cf7dc456751511acab83 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Mon, 13 Apr 2015 17:53:17 -0300 Subject: [PATCH 39/42] improve logging, via time --- lib/node.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/node.js b/lib/node.js index 125cce59..a3200fda 100644 --- a/lib/node.js +++ b/lib/node.js @@ -50,6 +50,17 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService }); }); */ + + setInterval(function() { + if (!self.blockchain) { + // not ready yet + return; + } + var tipHash = self.blockchain.tip; + var block = self.blockCache[tipHash]; + console.log('block', block.id, 'height', block.height); + }, 5 * 1000); + this.bus.register(bitcore.Block, function(block) { var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); @@ -65,10 +76,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService block.height = self.blockchain.height[block.id]; block.work = self.blockchain.work[block.id]; - if (block.height % 100 === 0) { - console.log('block', block.id, 'height', block.height); - } - return Promise.each(blockchainChanges.unconfirmed, function(hash) { return self.blockService.unconfirm(self.blockCache[hash]); }) From 9d4d06dc6c0d9e988cf062943ec2ff2818fd0009 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 15 Apr 2015 11:21:44 -0300 Subject: [PATCH 40/42] fix all tests with sync working --- api/test/v1/transactions.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/test/v1/transactions.js b/api/test/v1/transactions.js index 287573f3..bb30b3b0 100644 --- a/api/test/v1/transactions.js +++ b/api/test/v1/transactions.js @@ -23,7 +23,8 @@ describe('BitcoreHTTP v1 transactions routes', function() { var nodeMock, app, agent; beforeEach(function() { nodeMock = new EventEmitter(); - nodeMock.getTransaction = function(txHash) { + nodeMock.transactionService = {}; + nodeMock.transactionService.getTransaction = function(txHash) { var tx = mockTransactions[txHash]; if (_.isUndefined(tx)) { return Promise.reject(new BitcoreNode.errors.Transactions.NotFound(txHash)); From 5a63b58737fdf29e41fb23e5a62a78018be2493a Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 15 Apr 2015 12:49:41 -0300 Subject: [PATCH 41/42] trying to fix travis error --- lib/node.js | 78 +++++++++++++++++++++++----------------------------- test/node.js | 24 ++++------------ 2 files changed, 41 insertions(+), 61 deletions(-) diff --git a/lib/node.js b/lib/node.js index a3200fda..984ab2b6 100644 --- a/lib/node.js +++ b/lib/node.js @@ -26,7 +26,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService $.checkArgument(blockService, 'blockService is required'); $.checkArgument(transactionService, 'transactionService is required'); $.checkArgument(addressService, 'addressService is required'); - var self = this; this.bus = bus; this.networkMonitor = networkMonitor; @@ -38,18 +37,43 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.blockCache = {}; this.inventory = {}; // blockHash -> bool (has data) + this.initialize(); +}; +util.inherits(BitcoreNode, EventEmitter); - /* - this.networkMonitor.on('inv', function(inventory) { - _.each(inventory, function(info) { - var hash = bitcore.util.buffer.reverse(info.hash).toString('hex'); - $.checkState(_.isUndefined(self.inventory[hash]), hash); - if (self.inventory[hash] && info.type === 2) { // TODO: use static field from bitcore-p2p - self.inventory[hash] = false; - } - }); +BitcoreNode.create = function(opts) { + opts = opts || {}; + + var bus = new EventBus(); + + var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); + + var database = opts.database || Promise.promisifyAll( + new LevelUp(opts.LevelUp || './db') + ); + var rpc = opts.rpc || Promise.promisifyAll(new RPC(opts.RPC)); + + var transactionService = opts.transactionService || new TransactionService({ + rpc: rpc, + database: database }); - */ + var blockService = opts.blockService || new BlockService({ + rpc: rpc, + database: database, + transactionService: transactionService + }); + var addressService = opts.addressService || new AddressService({ + rpc: rpc, + database: database, + transactionService: transactionService, + blockService: blockService + }); + return new BitcoreNode(bus, networkMonitor, blockService, transactionService, addressService); +}; + + +BitcoreNode.prototype.initialize = function() { + var self = this; setInterval(function() { if (!self.blockchain) { @@ -112,38 +136,6 @@ var BitcoreNode = function(bus, networkMonitor, blockService, transactionService this.networkMonitor.on('disconnect', function() { console.log('network monitor disconnected'); }); - -}; -util.inherits(BitcoreNode, EventEmitter); - -BitcoreNode.create = function(opts) { - opts = opts || {}; - - var bus = new EventBus(); - - var networkMonitor = NetworkMonitor.create(bus, opts.NetworkMonitor); - - var database = Promise.promisifyAll( - new LevelUp(opts.LevelUp || './db') - ); - var rpc = Promise.promisifyAll(new RPC(opts.RPC)); - - var transactionService = new TransactionService({ - rpc: rpc, - database: database - }); - var blockService = new BlockService({ - rpc: rpc, - database: database, - transactionService: transactionService - }); - var addressService = new AddressService({ - rpc: rpc, - database: database, - transactionService: transactionService, - blockService: blockService - }); - return new BitcoreNode(bus, networkMonitor, blockService, transactionService, addressService); }; BitcoreNode.prototype.start = function() { diff --git a/test/node.js b/test/node.js index 929e8a99..8170c1f7 100644 --- a/test/node.js +++ b/test/node.js @@ -34,25 +34,13 @@ describe('BitcoreNode', function() { }); it('from create', function() { + var dbMock = {}; + var rpcMock = {}; var opts = { - LevelUp: './db-test', - network: 'testnet', - NetworkMonitor: { - host: 'localhost', - port: 8555, - }, - Reporter: 'none', - BitcoreHTTP: { - host: 'somehost', - port: 9090, - }, - RPC: { - user: 'test-user', - pass: 'test-password', - protocol: 'http', - host: '8.8.8.8', - port: 8552, - } + database: dbMock, + rpc: rpcMock, + blockService: bsMock, + transactionService: tsMock }; var node = BitcoreNode.create(opts); should.exist(node); From b5db2bd8590f75bbe844f578224cc3ac9985c785 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Wed, 15 Apr 2015 12:56:55 -0300 Subject: [PATCH 42/42] try to fix travis again --- api/test/http.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/test/http.js b/api/test/http.js index 1ecb38b2..2bcac25c 100644 --- a/api/test/http.js +++ b/api/test/http.js @@ -12,7 +12,7 @@ describe('BitcoreHTTP', function() { // mocks var opts = { BitcoreNode: { - LevelUp: './test-db' + database: {} }, port: 1234 };