From 3fa1340ef3f659df765e5b90ea1c5f96557c353b Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Tue, 29 Sep 2015 15:20:05 -0400 Subject: [PATCH 1/2] save tip as part of block db operations --- lib/services/db.js | 149 +++++++++++++++++++-------------------------- 1 file changed, 61 insertions(+), 88 deletions(-) diff --git a/lib/services/db.js b/lib/services/db.js index 38e7340b..b4e493b6 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -62,7 +62,8 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; DB.PREFIXES = { - BLOCKS: new Buffer('01', 'hex') + BLOCKS: new Buffer('01', 'hex'), + TIP: new Buffer('04', 'hex') }; /** @@ -109,45 +110,14 @@ DB.prototype.start = function(callback) { }); }); - // Does our database already have a tip? - self.getMetadata(function(err, metadata) { + self.loadTip(function(err) { if(err) { return callback(err); - } else if(!metadata || !metadata.tip) { - self.tip = self.genesis; - self.tip.__height = 0; - self.connectBlock(self.genesis, function(err) { - if(err) { - return callback(err); - } - - self.emit('addblock', self.genesis); - self.saveMetadata(); - self.sync(); - self.emit('ready'); - setImmediate(callback); - - }); - } else { - self.getBlock(metadata.tip, function(err, tip) { - if(err) { - log.warn( - 'Database is in an inconsistent state, a reindex is needed. Could not get current tip:', - metadata.tip - ); - return callback(err); - } - self.tip = tip; - var blockIndex = self.node.services.bitcoind.getBlockIndex(self.tip.hash); - if (!blockIndex) { - return callback(new Error('Could not get height for tip.')); - } - self.tip.__height = blockIndex.height; - self.sync(); - self.emit('ready'); - setImmediate(callback); - }); } + + self.sync(); + self.emit('ready'); + setImmediate(callback); }); }; @@ -220,6 +190,52 @@ DB.prototype.getAPIMethods = function() { return methods; }; +DB.prototype.loadTip = function(callback) { + var self = this; + + var options = { + keyEncoding: 'binary', + valueEncoding: 'binary' + }; + + self.store.get(DB.PREFIXES.TIP, options, function(err, tipData) { + if(err && err instanceof levelup.errors.NotFoundError) { + self.tip = self.genesis; + self.tip.__height = 0; + self.connectBlock(self.genesis, function(err) { + if(err) { + return callback(err); + } + + self.emit('addblock', self.genesis); + callback(); + }); + return; + } else if(err) { + return callback(err); + } + + var hash = tipData.toString('hex'); + + self.getBlock(hash, function(err, tip) { + if(err) { + log.warn('Database is in an inconsistent state, a reindex is needed. Could not get current tip:', + hash + ); + return callback(err); + } + + self.tip = tip; + var blockIndex = self.node.services.bitcoind.getBlockIndex(self.tip.hash); + if(!blockIndex) { + return callback(new Error('Could not get height for tip.')); + } + self.tip.__height = blockIndex.height; + callback(); + }); + }); +}; + /** * Will get a block from bitcoind and give a Bitcore Block * @param {String|Number} hash - A block hash or block height @@ -404,54 +420,6 @@ DB.prototype.getPrevHash = function(blockHash, callback) { }); }; -/** - * Saves metadata to the database - * @param {Function} callback - A function that accepts: Error - */ -DB.prototype.saveMetadata = function(callback) { - var self = this; - - function defaultCallback(err) { - if (err) { - self.emit('error', err); - } - } - - callback = callback || defaultCallback; - - var metadata = { - tip: self.tip ? self.tip.hash : null - }; - - this.store.put('metadata', JSON.stringify(metadata), {}, callback); - -}; - -/** - * Retrieves metadata from the database - * @param {Function} callback - A function that accepts: Error and Object - */ -DB.prototype.getMetadata = function(callback) { - var self = this; - - self.store.get('metadata', {}, function(err, data) { - if (err instanceof levelup.errors.NotFoundError) { - return callback(null, {}); - } else if (err) { - return callback(err); - } - - var metadata; - try { - metadata = JSON.parse(data); - } catch(e) { - return callback(new Error('Could not parse metadata')); - } - - callback(null, metadata); - }); -}; - /** * Connects a block to the database and add indexes * @param {Block} block - The bitcore block @@ -506,6 +474,14 @@ DB.prototype.runAllBlockHandlers = function(block, add, callback) { this.subscriptions.block[i].emit('db/block', block.hash); } + // Update tip + var tipHash = add ? new Buffer(block.hash, 'hex') : BufferUtil.reverse(block.header.prevHash); + operations.push({ + type: 'put', + key: DB.PREFIXES.TIP, + value: tipHash + }); + // Update block index operations.push({ type: add ? 'put' : 'del', @@ -668,7 +644,6 @@ DB.prototype.syncRewind = function(block, done) { // Set the new tip previousTip.__height = self.tip.__height - 1; self.tip = previousTip; - self.saveMetadata(); self.emit('removeblock', tip); removeDone(); }); @@ -725,8 +700,6 @@ DB.prototype.sync = function() { return done(err); } self.tip = block; - log.debug('Saving metadata'); - self.saveMetadata(); log.debug('Chain added block to main chain'); self.emit('addblock', block); setImmediate(done); From 26b27b292e5a8297f639572c1349267aff4ad8c6 Mon Sep 17 00:00:00 2001 From: Patrick Nagurny Date: Tue, 29 Sep 2015 16:03:56 -0400 Subject: [PATCH 2/2] update tests --- test/services/db.unit.js | 326 +++++++++++++-------------------------- 1 file changed, 104 insertions(+), 222 deletions(-) diff --git a/test/services/db.unit.js b/test/services/db.unit.js index b4b296bd..1a3bf7d2 100644 --- a/test/services/db.unit.js +++ b/test/services/db.unit.js @@ -38,6 +38,9 @@ describe('DB Service', function() { store: memdown }; + var genesisBuffer = new Buffer('0100000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000bac8b0fa927c0ac8234287e33c5f74d38d354820e24756ad709d7038fc5f31f020e7494dffff001d03e4b6720101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0420e7494d017f062f503253482fffffffff0100f2052a010000002321021aeaf2f8638a129a3156fbe7e5ef635226b0bafd495ff03afe2c843d7e3a4b51ac00000000', 'hex'); + + describe('#_setDataPath', function() { it('should set the database path', function() { var config = { @@ -104,7 +107,6 @@ describe('DB Service', function() { describe('#start', function() { var TestDB; - var genesisBuffer; before(function() { TestDB = proxyquire('../../lib/services/db', { @@ -113,7 +115,6 @@ describe('DB Service', function() { }, levelup: sinon.stub() }); - genesisBuffer = new Buffer('0100000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea330900000000bac8b0fa927c0ac8234287e33c5f74d38d354820e24756ad709d7038fc5f31f020e7494dffff001d03e4b6720101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0420e7494d017f062f503253482fffffffff0100f2052a010000002321021aeaf2f8638a129a3156fbe7e5ef635226b0bafd495ff03afe2c843d7e3a4b51ac00000000', 'hex'); }); it('should emit ready', function(done) { @@ -124,9 +125,8 @@ describe('DB Service', function() { on: sinon.spy(), genesisBuffer: genesisBuffer }; - db.getMetadata = sinon.stub().callsArg(0); + db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); - db.saveMetadata = sinon.stub(); db.sync = sinon.stub(); var readyFired = false; db.on('ready', function() { @@ -138,111 +138,13 @@ describe('DB Service', function() { }); }); - it('genesis block if no metadata is found in the db', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub() - } - } - }; - var db = new TestDB({node: node}); - db.getMetadata = sinon.stub().callsArgWith(0, null, null); - db.connectBlock = sinon.stub().callsArg(1); - db.saveMetadata = sinon.stub(); - db.sync = sinon.stub(); - db.start(function() { - should.exist(db.tip); - db.tip.hash.should.equal('00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'); - done(); - }); - }); - - it('metadata from the database if it exists', function(done) { - var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - getBlockIndex: sinon.stub().returns({tip:tipHash}), - on: sinon.stub() - } - } - }; - var tip = Block.fromBuffer(genesisBuffer); - var db = new TestDB({node: node}); - db.getMetadata = sinon.stub().callsArgWith(0, null, { - tip: tipHash, - tipHeight: 0 - }); - db.getBlock = sinon.stub().callsArgWith(1, null, tip); - db.saveMetadata = sinon.stub(); - db.sync = sinon.stub(); - db.start(function() { - should.exist(db.tip); - db.tip.hash.should.equal(tipHash); - done(); - }); - }); - - it('emit error from getMetadata', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub() - } - } - }; - var db = new TestDB({node: node}); - db.getMetadata = sinon.stub().callsArgWith(0, new Error('test')); - db.start(function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - - it('emit error from getBlock', function(done) { - var node = { - network: Networks.testnet, - datadir: 'testdir', - services: { - bitcoind: { - genesisBuffer: genesisBuffer, - on: sinon.stub() - } - } - }; - var db = new TestDB({node: node}); - var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; - db.getMetadata = sinon.stub().callsArgWith(0, null, { - tip: tipHash, - tipHeigt: 0 - }); - db.getBlock = sinon.stub().callsArgWith(1, new Error('test')); - db.start(function(err) { - should.exist(err); - err.message.should.equal('test'); - done(); - }); - }); - it('will call sync when there is a new tip', function(done) { var db = new TestDB(baseConfig); db.node.services = {}; db.node.services.bitcoind = new EventEmitter(); db.node.services.bitcoind.genesisBuffer = genesisBuffer; - db.getMetadata = sinon.stub().callsArg(0); + db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); - db.saveMetadata = sinon.stub(); db.sync = sinon.stub(); db.start(function() { db.sync = function() { @@ -258,9 +160,8 @@ describe('DB Service', function() { db.node.services.bitcoind = new EventEmitter(); db.node.services.bitcoind.syncPercentage = sinon.spy(); db.node.services.bitcoind.genesisBuffer = genesisBuffer; - db.getMetadata = sinon.stub().callsArg(0); + db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); - db.saveMetadata = sinon.stub(); db.node.stopping = true; db.sync = sinon.stub(); db.start(function() { @@ -339,6 +240,98 @@ describe('DB Service', function() { }); }); + describe('#loadTip', function() { + it('genesis block if no metadata is found in the db', function(done) { + var db = new DB(baseConfig); + db.genesis = Block.fromBuffer(genesisBuffer); + db.store = { + get: sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()) + }; + db.connectBlock = sinon.stub().callsArg(1); + db.sync = sinon.stub(); + db.loadTip(function() { + should.exist(db.tip); + db.tip.hash.should.equal('00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'); + done(); + }); + }); + + it('tip from the database if it exists', function(done) { + var node = { + network: Networks.testnet, + datadir: 'testdir', + services: { + bitcoind: { + genesisBuffer: genesisBuffer, + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({height: 1}) + } + } + }; + var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; + var tip = Block.fromBuffer(genesisBuffer); + var db = new DB({node: node}); + db.store = { + get: sinon.stub().callsArgWith(2, null, new Buffer(tipHash, 'hex')) + }; + db.getBlock = sinon.stub().callsArgWith(1, null, tip); + db.sync = sinon.stub(); + db.loadTip(function() { + should.exist(db.tip); + db.tip.hash.should.equal(tipHash); + db.tip.__height.should.equal(1); + done(); + }); + }); + + it('give error if levelup error', function(done) { + var node = { + network: Networks.testnet, + datadir: 'testdir', + services: { + bitcoind: { + genesisBuffer: genesisBuffer, + on: sinon.stub() + } + } + }; + var db = new DB({node: node}); + db.store = { + get: sinon.stub().callsArgWith(2, new Error('test')) + }; + db.loadTip(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + + it('give error from getBlock', function(done) { + var node = { + network: Networks.testnet, + datadir: 'testdir', + services: { + bitcoind: { + genesisBuffer: genesisBuffer, + on: sinon.stub(), + getBlockIndex: sinon.stub().returns({height: 1}) + } + } + }; + var db = new DB({node: node}); + var tipHash = '00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206'; + db.store = { + get: sinon.stub().callsArgWith(2, null, new Buffer(tipHash, 'hex')) + }; + db.getBlock = sinon.stub().callsArgWith(1, new Error('test')); + db.loadTip(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + }); + describe('#getBlock', function() { var db = new DB(baseConfig); var blockBuffer = new Buffer(blockData, 'hex'); @@ -581,119 +574,6 @@ describe('DB Service', function() { }); }); - describe('#saveMetadata', function() { - it('will emit an error with default callback', function(done) { - var db = new DB(baseConfig); - db.cache = { - hashes: {}, - chainHashes: {} - }; - db.tip = { - hash: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - __height: 0 - }; - db.store = { - put: sinon.stub().callsArgWith(3, new Error('test')) - }; - db.on('error', function(err) { - err.message.should.equal('test'); - done(); - }); - db.saveMetadata(); - }); - it('will give an error with callback', function(done) { - var db = new DB(baseConfig); - db.cache = { - hashes: {}, - chainHashes: {} - }; - db.tip = { - hash: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - __height: 0 - }; - db.store = { - put: sinon.stub().callsArgWith(3, new Error('test')) - }; - db.saveMetadata(function(err) { - err.message.should.equal('test'); - done(); - }); - }); - it('will call store with the correct arguments', function(done) { - var db = new DB(baseConfig); - db.cache = { - hashes: {}, - chainHashes: {} - }; - db.tip = { - hash: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - __height: 0 - }; - db.store = { - put: function(key, value, options, callback) { - key.should.equal('metadata'); - JSON.parse(value).should.deep.equal({ - tip: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' - }); - options.should.deep.equal({}); - callback.should.be.a('function'); - done(); - } - }; - db.saveMetadata(); - }); - }); - - describe('#getMetadata', function() { - it('will get metadata', function() { - var db = new DB(baseConfig); - var json = JSON.stringify({ - tip: '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - tipHeight: 101, - cache: { - hashes: {}, - chainHashes: {} - } - }); - db.store = {}; - db.store.get = sinon.stub().callsArgWith(2, null, json); - db.getMetadata(function(err, data) { - data.tip.should.equal('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'); - data.tipHeight.should.equal(101); - data.cache.should.deep.equal({ - hashes: {}, - chainHashes: {} - }); - }); - }); - it('will handle a notfound error from leveldb', function() { - var db = new DB(baseConfig); - db.store = {}; - var error = new levelup.errors.NotFoundError(); - db.store.get = sinon.stub().callsArgWith(2, error); - db.getMetadata(function(err, data) { - should.not.exist(err); - data.should.deep.equal({}); - }); - }); - it('will handle error from leveldb', function() { - var db = new DB(baseConfig); - db.store = {}; - db.store.get = sinon.stub().callsArgWith(2, new Error('test')); - db.getMetadata(function(err) { - err.message.should.equal('test'); - }); - }); - it('give an error when parsing invalid json', function() { - var db = new DB(baseConfig); - db.store = {}; - db.store.get = sinon.stub().callsArgWith(2, null, '{notvalid@json}'); - db.getMetadata(function(err) { - err.message.should.equal('Could not parse metadata'); - }); - }); - }); - describe('#connectBlock', function() { it('should remove block from mempool and call blockHandler with true', function(done) { var db = new DB(baseConfig); @@ -749,12 +629,17 @@ describe('DB Service', function() { it('should call blockHandler in all services and perform operations', function(done) { db.runAllBlockHandlers(block, true, function(err) { should.not.exist(err); + var tipOp = { + type: 'put', + key: DB.PREFIXES.TIP, + value: new Buffer('00000000000000000d0aaf93e464ddeb503655a0750f8b9c6eed0bdf0ccfc863', 'hex') + } var blockOp = { type: 'put', key: db._encodeBlockIndexKey(1441906365), value: db._encodeBlockIndexValue('00000000000000000d0aaf93e464ddeb503655a0750f8b9c6eed0bdf0ccfc863') }; - db.store.batch.args[0][0].should.deep.equal([blockOp, 'op1', 'op2', 'op3', 'op4', 'op5']); + db.store.batch.args[0][0].should.deep.equal([tipOp, blockOp, 'op1', 'op2', 'op3', 'op4', 'op5']); done(); }); }); @@ -904,7 +789,6 @@ describe('DB Service', function() { prevHash: hexlebuf(chainHashes[chainHashes.length - 1]) } }; - db.saveMetadata = sinon.stub(); db.emit = sinon.stub(); db.getBlock = function(hash, callback) { setImmediate(function() { @@ -964,7 +848,6 @@ describe('DB Service', function() { __height: 0, hash: lebufhex(block.header.prevHash) }; - db.saveMetadata = sinon.stub(); db.emit = sinon.stub(); db.cache = { hashes: {} @@ -1009,7 +892,6 @@ describe('DB Service', function() { __height: 0, hash: block.prevHash }; - db.saveMetadata = sinon.stub(); db.emit = sinon.stub(); db.cache = { hashes: {}