From 1986cb40ef76ec4e068c9ca772fc9ab73a0dbda5 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 24 Apr 2015 10:45:53 -0300 Subject: [PATCH 1/3] moving towards removing memory leak --- lib/blockchain.js | 6 +-- lib/networkmonitor.js | 1 + lib/node.js | 14 +++++-- lib/services/block.js | 96 +++++++++++++++++++++++-------------------- 4 files changed, 65 insertions(+), 52 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index c40de043..58a356e5 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -140,10 +140,10 @@ BlockChain.prototype.hasData = function(hash) { BlockChain.prototype.prune = function() { var self = this; - _.each(this.prev, function(key, value) { + _.each(this.prev, function(key) { if (!self.height[key]) { - delete this.prev[key]; - delete this.work[key]; + delete self.prev[key]; + delete self.work[key]; } }); }; diff --git a/lib/networkmonitor.js b/lib/networkmonitor.js index fd82714b..6d792b9c 100644 --- a/lib/networkmonitor.js +++ b/lib/networkmonitor.js @@ -78,6 +78,7 @@ NetworkMonitor.prototype.requestBlocks = function(locator) { }; NetworkMonitor.prototype.start = function() { + console.log('starting network monitor'); this.peer.connect(); }; diff --git a/lib/node.js b/lib/node.js index e1c6f8e3..0b181455 100644 --- a/lib/node.js +++ b/lib/node.js @@ -79,15 +79,22 @@ BitcoreNode.prototype.initialize = function() { var prevHeight = 0; var statTimer = 5 * 1000; setInterval(function() { + console.log(process.memoryUsage().heapTotal / 1024 / 1024, 'mb used'); if (!self.blockchain) { // not ready yet + console.log('No blockchain yet'); return; } var tipHash = self.blockchain.tip; var block = self.blockCache[tipHash]; + if (_.isUndefined(block)) { + console.log('No blocks yet'); + return; + } var delta = block.height - prevHeight; prevHeight = block.height; console.log(block.id, block.height, 'vel', delta * 1000 / statTimer, 'b/s'); + console.log('cache', Object.keys(self.blockCache).length); }, statTimer); this.bus.register(bitcore.Block, function(block) { @@ -149,7 +156,9 @@ BitcoreNode.prototype.start = function() { var self = this; var genesis = bitcore.Block.fromBuffer(genesisBlocks[bitcore.Networks.defaultNetwork.name]); + console.log('getting blockchain'); this.blockService.getBlockchain().then(function(blockchain) { + console.log('got blockchain', _.isUndefined(blockchain)); if (!blockchain) { self.blockchain = new BlockChain(); self.bus.process(genesis); @@ -159,9 +168,6 @@ BitcoreNode.prototype.start = function() { self.sync(); self.networkMonitor.start(); }); - this.networkMonitor.on('stop', function() { - self.blockService.saveBlockchain(self.blockchain); - }); }; BitcoreNode.prototype.stop = function(reason) { @@ -170,7 +176,7 @@ BitcoreNode.prototype.stop = function(reason) { BitcoreNode.prototype.requestFromTip = function() { var locator = this.blockchain.getBlockLocator(); - console.log('requesting blocks, locator size:', locator.length); + //console.log('requesting blocks, locator size:', locator.length); this.networkMonitor.requestBlocks(locator); }; diff --git a/lib/services/block.js b/lib/services/block.js index 0353685b..fdaa5941 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -182,27 +182,26 @@ BlockService.prototype.getBlockByHeight = function(height) { }); }; +BlockService.prototype._getLatestHash = function() { + var self = this; + return Promise.try(function() { + return self.database.getAsync(Index.tip); + }).catch(LevelUp.errors.NotFoundError, function() { + return null; + }); +}; /** * Fetch the block that is currently the tip of the blockchain * * @return {Promise} */ BlockService.prototype.getLatest = function() { - var self = this; - - return Promise.try(function() { - - return self.database.getAsync(Index.tip); - - }).then(function(blockHash) { - + return this._getLatestHash().then(function(blockHash) { + if (!blockHash) { + return null; + } return self.getBlock(blockHash); - - }).catch(LevelUp.errors.NotFoundError, function() { - - return null; - }); }; @@ -444,42 +443,48 @@ BlockService.prototype.getBlockchain = function() { var self = this; var blockchain = new BlockChain(); + console.log(process.memoryUsage().heapTotal / 1024 / 1024, 'mb used'); + var amount = 0; + var limit = 100; var fetchBlock = function(blockHash) { - return Promise.all([ - self.database.getAsync(Index.getPreviousBlock(blockHash)).then(function(prevHash) { + //console.log(process.memoryUsage().heapTotal / 1024 / 1024); + if (blockHash === BlockChain.NULL) { + console.log(process.memoryUsage().heapTotal / 1024 / 1024, 'mb used'); + return; + } + amount += 1; + var prevKey = Index.getPreviousBlock(blockHash); + var heightKey = Index.getBlockHeight(blockHash); + var workKey = Index.getBlockWork(blockHash); + + return self.database.getAsync(prevKey) + .then(function(prevHash) { blockchain.prev[blockHash] = prevHash; blockchain.next[prevHash] = blockHash; - }), - self.database.getAsync(Index.getBlockHeight(blockHash)).then(function(height) { - blockchain.height[blockHash] = +height; - blockchain.hashByHeight[height] = blockHash; - }), - self.database.getAsync(Index.getBlockWork(blockHash)).then(function(work) { - blockchain.work[blockHash] = work; }) - ]).then(function() { - return blockHash; - }); + .then(function() { + return self.database.getAsync(heightKey) + .then(function(height) { + blockchain.height[blockHash] = +height; + blockchain.hashByHeight[height] = blockHash; + }); + }) + .then(function() { + if (amount >= limit) { + return; + } + return self.database.getAsync(workKey) + .then(function(work) { + blockchain.work[blockHash] = work; + }); + }) + .then(function() { + return fetchBlock(blockchain.prev[blockHash]); + }); }; - var fetchUnlessGenesis = function(blockHash) { - return fetchBlock(blockHash).then(function() { - if (blockchain.prev[blockHash] === BlockChain.NULL) { - return; - } else { - return fetchUnlessGenesis(blockchain.prev[blockHash]); - } - }); - }; - - return self.database.getAsync(Index.tip) - .catch(function(err) { - if (err.notFound) { - return undefined; - } - throw err; - }) + return self._getLatestHash() .then(function(tip) { if (!tip) { console.log('No tip found'); @@ -487,9 +492,10 @@ BlockService.prototype.getBlockchain = function() { } console.log('Tip is', tip); blockchain.tip = tip; - return fetchUnlessGenesis(tip).then(function() { - return blockchain; - }); + return fetchBlock(tip) + .then(function() { + return blockchain; + }); }); }; From 8a529476377265966c57b83c08674e9d4ff6c90c Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 24 Apr 2015 13:14:59 -0300 Subject: [PATCH 2/3] memory leak fixed :D --- lib/blockchain.js | 137 +++++++++++++++++++++++------------------- lib/node.js | 3 +- lib/services/block.js | 85 ++++++++++++-------------- 3 files changed, 113 insertions(+), 112 deletions(-) diff --git a/lib/blockchain.js b/lib/blockchain.js index 58a356e5..4de9ae4d 100644 --- a/lib/blockchain.js +++ b/lib/blockchain.js @@ -3,6 +3,7 @@ var bitcore = require('bitcore'); var $ = bitcore.util.preconditions; var _ = bitcore.deps._; +var BufferUtil = bitcore.util.buffer; var NULL = '0000000000000000000000000000000000000000000000000000000000000000'; @@ -12,7 +13,9 @@ function BlockChain() { this.work[NULL] = 0; this.height = {}; this.height[NULL] = -1; - this.hashByHeight = { '-1': NULL }; + this.hashByHeight = { + '-1': NULL + }; this.next = {}; this.prev = {}; } @@ -35,66 +38,74 @@ var getWork = function(bits) { return ((bits & 0xffffff) << (8 * (bytes - 3))) >>> 0; }; -BlockChain.prototype.addData = function(block) { - $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); +BlockChain.prototype.addData = function(header) { + $.checkArgument(header instanceof bitcore.Block.BlockHeader, 'Argument is not a BlockHeader instance'); - var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); + var prevHash = BufferUtil.reverse(header.prevHash).toString('hex'); + var hash = header.hash; - this.work[block.hash] = this.work[prevHash] + getWork(block.header.bits); - this.prev[block.hash] = prevHash; + this.work[hash] = this.work[prevHash] + getWork(header.bits); + this.prev[hash] = prevHash; }; -BlockChain.prototype.proposeNewBlock = function(block) { - $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); - var prevHash = bitcore.util.buffer.reverse(block.header.prevHash).toString('hex'); +BlockChain.prototype._appendNewBlock = function(hash) { + var toUnconfirm = []; + var toConfirm = []; + var self = this; - if (_.isUndefined(this.work[prevHash])) { - throw new Error('No previous data to estimate work'); + var pointer = hash; + while (_.isUndefined(this.height[pointer])) { + toConfirm.push(pointer); + pointer = this.prev[pointer]; } - this.addData(block); + var commonAncestor = pointer; - if (this.work[block.hash] > this.work[this.tip]) { + pointer = this.tip; + while (pointer !== commonAncestor) { + toUnconfirm.push(pointer); + pointer = this.prev[pointer]; + } - var toUnconfirm = []; - var toConfirm = []; - var commonAncestor; + toConfirm.reverse(); + toUnconfirm.map(function(hash) { + self.unconfirm(hash); + }); + toConfirm.map(function(hash) { + self.confirm(hash); + }); + return { + unconfirmed: toUnconfirm, + confirmed: toConfirm + }; +}; - var pointer = block.hash; - while (_.isUndefined(this.height[pointer])) { - toConfirm.push(pointer); - pointer = this.prev[pointer]; - } - commonAncestor = pointer; +BlockChain.prototype.proposeNewHeader = function(header) { + $.checkArgument(header instanceof bitcore.Block.BlockHeader, 'Argument is not a BlockHeader instance'); + var prevHash = BufferUtil.reverse(header.prevHash).toString('hex'); + var hash = header.hash; - pointer = this.tip; - while (pointer !== commonAncestor) { - toUnconfirm.push(pointer); - pointer = this.prev[pointer]; - } - - toConfirm.reverse(); - - var self = this; - toUnconfirm.map(function(hash) { - self.unconfirm(hash); - }); - toConfirm.map(function(hash) { - self.confirm(hash); - }); - return { - unconfirmed: toUnconfirm, - confirmed: toConfirm - }; + $.checkState(this.hasData(prevHash), 'No previous data to estimate work'); + this.addData(header); + var work = this.work[hash]; + var tipWork = this.work[this.tip]; + $.checkState(!_.isUndefined(work), 'No work found for ' + hash); + $.checkState(!_.isUndefined(tipWork), 'No work found for tip ' + this.tip); + if (work > tipWork) { + return this._appendNewBlock(hash); } return { unconfirmed: [], confirmed: [] }; }; +BlockChain.prototype.proposeNewBlock = function(block) { + $.checkArgument(block instanceof bitcore.Block, 'Argument is not a Block instance'); + return this.proposeNewHeader(block.header); +}; BlockChain.prototype.confirm = function(hash) { var prevHash = this.prev[hash]; - $.checkState(prevHash === this.tip); + $.checkState(prevHash === this.tip, 'Attempting to confirm a non-contiguous block.'); this.tip = hash; var height = this.height[prevHash] + 1; @@ -105,7 +116,7 @@ BlockChain.prototype.confirm = function(hash) { BlockChain.prototype.unconfirm = function(hash) { var prevHash = this.prev[hash]; - $.checkState(hash === this.tip); + $.checkState(hash === this.tip, 'Attempting to unconfirm a non-tip block'); this.tip = prevHash; var height = this.height[hash]; @@ -114,26 +125,6 @@ BlockChain.prototype.unconfirm = function(hash) { delete this.height[hash]; }; -BlockChain.prototype.getBlockLocator = function() { - $.checkState(this.tip); - $.checkState(!_.isUndefined(this.height[this.tip])); - - var result = []; - var currentHeight = this.height[this.tip]; - var exponentialBackOff = 1; - for (var i = 0; i < 10; i++) { - if (currentHeight >= 0) { - result.push(this.hashByHeight[currentHeight--]); - } - } - while (currentHeight > 0) { - result.push(this.hashByHeight[currentHeight]); - currentHeight -= exponentialBackOff; - exponentialBackOff *= 2; - } - return result; -}; - BlockChain.prototype.hasData = function(hash) { return !_.isUndefined(this.work[hash]); }; @@ -163,4 +154,24 @@ BlockChain.prototype.toJSON = function() { return JSON.stringify(this.toObject()); }; +BlockChain.prototype.getBlockLocator = function() { + $.checkState(this.tip); + $.checkState(!_.isUndefined(this.height[this.tip])); + + var result = []; + var currentHeight = this.height[this.tip]; + var exponentialBackOff = 1; + for (var i = 0; i < 10; i++) { + if (currentHeight >= 0) { + result.push(this.hashByHeight[currentHeight--]); + } + } + while (currentHeight > 0) { + result.push(this.hashByHeight[currentHeight]); + currentHeight -= exponentialBackOff; + exponentialBackOff *= 2; + } + return result; +}; + module.exports = BlockChain; diff --git a/lib/node.js b/lib/node.js index 0b181455..f692c875 100644 --- a/lib/node.js +++ b/lib/node.js @@ -94,7 +94,6 @@ BitcoreNode.prototype.initialize = function() { var delta = block.height - prevHeight; prevHeight = block.height; console.log(block.id, block.height, 'vel', delta * 1000 / statTimer, 'b/s'); - console.log('cache', Object.keys(self.blockCache).length); }, statTimer); this.bus.register(bitcore.Block, function(block) { @@ -158,7 +157,7 @@ BitcoreNode.prototype.start = function() { console.log('getting blockchain'); this.blockService.getBlockchain().then(function(blockchain) { - console.log('got blockchain', _.isUndefined(blockchain)); + console.log('got blockchain', !!blockchain); if (!blockchain) { self.blockchain = new BlockChain(); self.bus.process(genesis); diff --git a/lib/services/block.js b/lib/services/block.js index fdaa5941..ea57238e 100644 --- a/lib/services/block.js +++ b/lib/services/block.js @@ -1,12 +1,13 @@ 'use strict'; +var config = require('config'); var LevelUp = require('levelup'); var Promise = require('bluebird'); var RPC = require('bitcoind-rpc'); var TransactionService = require('./transaction'); var bitcore = require('bitcore'); var Transaction = bitcore.Transaction; -var config = require('config'); +var BufferUtil = bitcore.util.buffer; var errors = require('../errors'); var BlockChain = require('../blockchain'); @@ -41,7 +42,8 @@ var Index = { next: 'nxt-', // nxt- -> hash for the next block in the main chain that is a child height: 'bh-', // bh- -> height (-1 means disconnected) tip: 'tip', // tip -> { hash: hex, height: int }, the latest tip - work: 'wk-' // wk- -> amount of work for block + work: 'wk-', // wk- -> amount of work for block + header: 'header-' // header- -> JSON for block header }; _.extend(Index, { getNextBlock: helper(Index.next), @@ -50,7 +52,8 @@ _.extend(Index, { getBlockWork: helper(Index.work), getBlockByTs: function(block) { return Index.timestamp + block.header.time; - } + }, + getBlockHeader: helper(Index.header), }); function BlockService(opts) { @@ -240,6 +243,8 @@ BlockService.prototype.confirm = function(block, ops) { //console.log(0); return Promise.try(function() { + //console.log(0.5); + self._setHeader(ops, block); //console.log(1); self._setNextBlock(ops, block.header.prevHash, block); @@ -271,6 +276,14 @@ BlockService.prototype.confirm = function(block, ops) { }); }; +BlockService.prototype._setHeader = function(ops, block) { + ops.push({ + type: 'put', + key: Index.getBlockHeader(block.hash), + value: block.header.toJSON(), + }); +}; + BlockService.prototype._setNextBlock = function(ops, prevBlockHash, block) { if (bitcore.util.buffer.isBuffer(prevBlockHash)) { prevBlockHash = bitcore.util.buffer.reverse(prevBlockHash).toString('hex'); @@ -357,15 +370,13 @@ BlockService.prototype._setBlockByTs = function(ops, block) { * @return {Promise} a promise of the same block, for chaining */ BlockService.prototype.unconfirm = function(block, ops) { - ops = ops || []; + var self = this; return Promise.try(function() { self._removeNextBlock(ops, block.header.prevHash, block); - self._unsetBlockHeight(ops, block, block.height); - self._dropBlockByTs(ops, block); return Promise.all(block.transactions.map(function(transaction) { @@ -373,21 +384,19 @@ BlockService.prototype.unconfirm = function(block, ops) { })); }).then(function() { - return self.database.batchAsync(ops); - }); }; BlockService.prototype._removeNextBlock = function(ops, prevHash, block) { - if (bitcore.util.buffer.isBuffer(prevBlockHash)) { - prevBlockHash = bitcore.util.buffer.reverse(prevBlockHash).toString('hex'); + if (bitcore.util.buffer.isBuffer(prevHash)) { + prevHash = bitcore.util.buffer.reverse(prevHash).toString('hex'); } ops.push({ type: 'del', - key: Index.getNextBlock(prevBlockHash) + key: Index.getNextBlock(prevHash) }); ops.push({ type: 'del', @@ -395,7 +404,7 @@ BlockService.prototype._removeNextBlock = function(ops, prevHash, block) { }); }; -BlockService.prototype._unsetBlockHeight = function(ops, block, height) { +BlockService.prototype._unsetBlockHeight = function(ops, block) { ops.push({ type: 'del', key: Index.getBlockHeight(block) @@ -443,44 +452,23 @@ BlockService.prototype.getBlockchain = function() { var self = this; var blockchain = new BlockChain(); - console.log(process.memoryUsage().heapTotal / 1024 / 1024, 'mb used'); + var headers = []; - var amount = 0; - var limit = 100; - var fetchBlock = function(blockHash) { - //console.log(process.memoryUsage().heapTotal / 1024 / 1024); + console.log('Fetching headers from db...'); + var fetchHeader = function(blockHash) { if (blockHash === BlockChain.NULL) { + console.log('All headers fetched, total =', headers.length); console.log(process.memoryUsage().heapTotal / 1024 / 1024, 'mb used'); return; } - amount += 1; - var prevKey = Index.getPreviousBlock(blockHash); - var heightKey = Index.getBlockHeight(blockHash); - var workKey = Index.getBlockWork(blockHash); - - return self.database.getAsync(prevKey) - .then(function(prevHash) { - blockchain.prev[blockHash] = prevHash; - blockchain.next[prevHash] = blockHash; + var headerKey = Index.getBlockHeader(blockHash); + return self.database.getAsync(headerKey) + .then(function(json) { + return bitcore.Block.BlockHeader.fromJSON(json); }) - .then(function() { - return self.database.getAsync(heightKey) - .then(function(height) { - blockchain.height[blockHash] = +height; - blockchain.hashByHeight[height] = blockHash; - }); - }) - .then(function() { - if (amount >= limit) { - return; - } - return self.database.getAsync(workKey) - .then(function(work) { - blockchain.work[blockHash] = work; - }); - }) - .then(function() { - return fetchBlock(blockchain.prev[blockHash]); + .then(function(header) { + headers.push(header); + return fetchHeader(BufferUtil.reverse(header.prevHash).toString('hex')); }); }; @@ -488,12 +476,15 @@ BlockService.prototype.getBlockchain = function() { .then(function(tip) { if (!tip) { console.log('No tip found'); - return; + return null; } console.log('Tip is', tip); - blockchain.tip = tip; - return fetchBlock(tip) + return fetchHeader(tip) .then(function() { + while (headers.length !== 0) { + var header = headers.pop(); + blockchain.proposeNewHeader(header); + } return blockchain; }); }); From 5865dc39b36ecac93b1e6c39607806d646fd09e8 Mon Sep 17 00:00:00 2001 From: Manuel Araoz Date: Fri, 24 Apr 2015 15:28:19 -0300 Subject: [PATCH 3/3] fix tests --- config/livenet.yml | 16 ++++++++++++++++ test/services/block.js | 8 ++++++++ 2 files changed, 24 insertions(+) create mode 100644 config/livenet.yml diff --git a/config/livenet.yml b/config/livenet.yml new file mode 100644 index 00000000..48ddb954 --- /dev/null +++ b/config/livenet.yml @@ -0,0 +1,16 @@ +BitcoreNode: + LevelUp: ./db + network: livenet + NetworkMonitor: + host: localhost + port: 8333 +Reporter: none # none, simple, matrix +BitcoreHTTP: + host: localhost + port: 8080 +RPC: + user: user + pass: password + protocol: http + host: 127.0.0.1 + port: 8332 diff --git a/test/services/block.js b/test/services/block.js index b204d0cb..a329113b 100644 --- a/test/services/block.js +++ b/test/services/block.js @@ -116,6 +116,10 @@ describe('BlockService', function() { it('makes the expected calls when confirming the genesis block', function(callback) { database.batchAsync = function(ops) { var expectedOps = [{ + type: 'put', + key: 'header-000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + value: '{"version":1,"prevHash":"0000000000000000000000000000000000000000000000000000000000000000","merkleRoot":"3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a","time":1231006505,"bits":486604799,"nonce":2083236893}' + }, { type: 'put', key: 'nxt-0000000000000000000000000000000000000000000000000000000000000000', value: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' @@ -146,6 +150,10 @@ describe('BlockService', function() { it('makes the expected calls when confirming the block #170', function(callback) { database.batchAsync = function(ops) { ops.should.deep.equal([{ + type: 'put', + key: 'header-00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee', + value: '{"version":1,"prevHash":"55bd840a78798ad0da853f68974f3d183e2bd1db6a842c1feecf222a00000000","merkleRoot":"ff104ccb05421ab93e63f8c3ce5c2c2e9dbb37de2764b3a3175c8166562cac7d","time":1231731025,"bits":486604799,"nonce":1889418792}' + }, { type: 'put', key: 'nxt-000000002a22cfee1f2c846adbd12b3e183d4f97683f85dad08a79780a84bd55', value: '00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee'