From 98bd8ee56022188f84eaad9f0b816cd77967c53f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Jan 2016 11:14:04 -0500 Subject: [PATCH 1/2] DB Service: Include a version number for upgrading purposes --- lib/services/db.js | 69 ++++++++++++++++++-- test/services/db.unit.js | 135 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 5 deletions(-) diff --git a/lib/services/db.js b/lib/services/db.js index c3c3dfd4..6cb183d1 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -38,6 +38,10 @@ function DB(options) { Service.call(this, options); + // Used to keep track of the version of the indexes + // to determine during an upgrade if a reindex is required + this.version = 2; + this.tip = null; this.genesis = null; @@ -66,6 +70,7 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; DB.PREFIXES = { + VERSION: new Buffer('ff', 'hex'), BLOCKS: new Buffer('01', 'hex'), TIP: new Buffer('04', 'hex') }; @@ -90,6 +95,48 @@ DB.prototype._setDataPath = function() { } }; +DB.prototype._checkVersion = function(callback) { + var self = this; + var options = { + keyEncoding: 'binary', + valueEncoding: 'binary' + }; + self.store.get(DB.PREFIXES.TIP, options, function(err) { + if (err instanceof levelup.errors.NotFoundError) { + // The database is brand new and doesn't have a tip stored + // we can skip version checking + return callback(); + } else if (err) { + return callback(err); + } + self.store.get(DB.PREFIXES.VERSION, options, function(err, buffer) { + var version; + if (err instanceof levelup.errors.NotFoundError) { + // The initial version (1) of the database didn't store the version number + version = 1; + } else if (err) { + return callback(err); + } else { + version = buffer.readUInt32BE(); + } + if (self.version !== version) { + return callback(new Error( + 'The version of the database "' + version + '" does not match the expected version "' + + self.version + '". A reindex (can take several hours) is required or to switch ' + + 'versions of software to match.' + )); + } + callback(); + }); + }); +}; + +DB.prototype._setVersion = function(callback) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(this.version); + this.store.put(DB.PREFIXES.VERSION, versionBuffer, callback); +}; + /** * Called by Node to start the service. * @param {Function} callback @@ -116,14 +163,26 @@ DB.prototype.start = function(callback) { }); }); - self.loadTip(function(err) { - if(err) { + async.series([ + function(next) { + self._checkVersion(next); + }, + function(next) { + self._setVersion(next); + } + ], function(err) { + if (err) { return callback(err); } + self.loadTip(function(err) { + if (err) { + return callback(err); + } - self.sync(); - self.emit('ready'); - setImmediate(callback); + self.sync(); + self.emit('ready'); + setImmediate(callback); + }); }); }; diff --git a/test/services/db.unit.js b/test/services/db.unit.js index bf4f0c4e..5b75ee2e 100644 --- a/test/services/db.unit.js +++ b/test/services/db.unit.js @@ -104,6 +104,135 @@ describe('DB Service', function() { }); }); + describe('#_checkVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will handle an error while retrieving the tip', function() { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new Error('test')); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will handle an error while retrieving the version', function() { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new Error('test')); + } + callCount++; + setImmediate(callback); + }); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will NOT check the version if a tip is not found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); + db._checkVersion(done); + }); + it('will NOT give an error if the versions match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 2; + db._checkVersion(done); + }); + it('will give an error if the versions do NOT match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 3; + db._checkVersion(function(err) { + should.exist(err); + err.message.should.match(/^The version of the database/); + done(); + }); + }); + it('will default to version 1 if the version is NOT found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new levelup.errors.NotFoundError()); + } + callCount++; + setImmediate(callback); + }); + db.version = 1; + db._checkVersion(done); + }); + }); + + describe('#_setVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will give an error from the store', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, new Error('test')); + db._setVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will set the version', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, null); + db.version = 5; + db._setVersion(function(err) { + if (err) { + return done(err); + } + db.store.put.args[0][0].should.deep.equal(new Buffer('ff', 'hex')); + db.store.put.args[0][1].should.deep.equal(new Buffer('00000005', 'hex')); + done(); + }); + }); + }); + describe('#start', function() { var TestDB; @@ -126,6 +255,8 @@ describe('DB Service', function() { }; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); var readyFired = false; db.on('ready', function() { @@ -144,6 +275,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); db.start(function() { db.sync = function() { @@ -161,6 +294,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.node.stopping = true; db.sync = sinon.stub(); db.start(function() { From 995b4b57d4aed791b9c023ab37a42d2f078a740d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 28 Jan 2016 13:47:26 -0500 Subject: [PATCH 2/2] DB: Include docs on how to recreate the database --- docs/services/db.md | 9 +++++++++ lib/services/db.js | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/services/db.md b/docs/services/db.md index 57439247..73a6cbb2 100644 --- a/docs/services/db.md +++ b/docs/services/db.md @@ -1,6 +1,15 @@ # Database Service This service synchronizes a leveldb database with the [Bitcoin Service](bitcoind.md) block chain by connecting and disconnecting blocks to build new indexes that can be queried. Other services can extend the data that is indexed by implementing a `blockHandler` method, similar to the built-in [Address Service](address.md). +## How to Reindex + +If you need to be able to recreate the database from historical transactions in blocks: +- Shutdown your node +- Remove the `bitcore-node.db` directory in the data directory (e.g. `~/.bitcore/bitcore-node.db`) +- Start your node again + +The database will then ask bitcoind for all the blocks again and recreate the database. This is sometimes required during upgrading as the format of the keys and values has changed. For "livenet" this can take half a day or more, for "testnet" this can take around an hour. + ## Adding Indexes For a service to include additional block data, it can implement a `blockHandler` method that will be run to when there are new blocks added or removed. diff --git a/lib/services/db.js b/lib/services/db.js index 6cb183d1..22cbadf7 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -120,10 +120,12 @@ DB.prototype._checkVersion = function(callback) { version = buffer.readUInt32BE(); } if (self.version !== version) { + var helpUrl = 'https://github.com/bitpay/bitcore-node/blob/master/docs/services/db.md#how-to-reindex'; return callback(new Error( 'The version of the database "' + version + '" does not match the expected version "' + - self.version + '". A reindex (can take several hours) is required or to switch ' + - 'versions of software to match.' + self.version + '". A recreation of "' + self.dataPath + '" (can take several hours) is ' + + 'required or to switch versions of software to match. Please see ' + helpUrl + + ' for more information.' )); } callback();