diff --git a/docs/services/db.md b/docs/services/db.md index 57439247..73a6cbb2 100644 --- a/docs/services/db.md +++ b/docs/services/db.md @@ -1,6 +1,15 @@ # Database Service This service synchronizes a leveldb database with the [Bitcoin Service](bitcoind.md) block chain by connecting and disconnecting blocks to build new indexes that can be queried. Other services can extend the data that is indexed by implementing a `blockHandler` method, similar to the built-in [Address Service](address.md). +## How to Reindex + +If you need to be able to recreate the database from historical transactions in blocks: +- Shutdown your node +- Remove the `bitcore-node.db` directory in the data directory (e.g. `~/.bitcore/bitcore-node.db`) +- Start your node again + +The database will then ask bitcoind for all the blocks again and recreate the database. This is sometimes required during upgrading as the format of the keys and values has changed. For "livenet" this can take half a day or more, for "testnet" this can take around an hour. + ## Adding Indexes For a service to include additional block data, it can implement a `blockHandler` method that will be run to when there are new blocks added or removed. diff --git a/lib/services/db.js b/lib/services/db.js index c3c3dfd4..22cbadf7 100644 --- a/lib/services/db.js +++ b/lib/services/db.js @@ -38,6 +38,10 @@ function DB(options) { Service.call(this, options); + // Used to keep track of the version of the indexes + // to determine during an upgrade if a reindex is required + this.version = 2; + this.tip = null; this.genesis = null; @@ -66,6 +70,7 @@ util.inherits(DB, Service); DB.dependencies = ['bitcoind']; DB.PREFIXES = { + VERSION: new Buffer('ff', 'hex'), BLOCKS: new Buffer('01', 'hex'), TIP: new Buffer('04', 'hex') }; @@ -90,6 +95,50 @@ DB.prototype._setDataPath = function() { } }; +DB.prototype._checkVersion = function(callback) { + var self = this; + var options = { + keyEncoding: 'binary', + valueEncoding: 'binary' + }; + self.store.get(DB.PREFIXES.TIP, options, function(err) { + if (err instanceof levelup.errors.NotFoundError) { + // The database is brand new and doesn't have a tip stored + // we can skip version checking + return callback(); + } else if (err) { + return callback(err); + } + self.store.get(DB.PREFIXES.VERSION, options, function(err, buffer) { + var version; + if (err instanceof levelup.errors.NotFoundError) { + // The initial version (1) of the database didn't store the version number + version = 1; + } else if (err) { + return callback(err); + } else { + version = buffer.readUInt32BE(); + } + if (self.version !== version) { + var helpUrl = 'https://github.com/bitpay/bitcore-node/blob/master/docs/services/db.md#how-to-reindex'; + return callback(new Error( + 'The version of the database "' + version + '" does not match the expected version "' + + self.version + '". A recreation of "' + self.dataPath + '" (can take several hours) is ' + + 'required or to switch versions of software to match. Please see ' + helpUrl + + ' for more information.' + )); + } + callback(); + }); + }); +}; + +DB.prototype._setVersion = function(callback) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(this.version); + this.store.put(DB.PREFIXES.VERSION, versionBuffer, callback); +}; + /** * Called by Node to start the service. * @param {Function} callback @@ -116,14 +165,26 @@ DB.prototype.start = function(callback) { }); }); - self.loadTip(function(err) { - if(err) { + async.series([ + function(next) { + self._checkVersion(next); + }, + function(next) { + self._setVersion(next); + } + ], function(err) { + if (err) { return callback(err); } + self.loadTip(function(err) { + if (err) { + return callback(err); + } - self.sync(); - self.emit('ready'); - setImmediate(callback); + self.sync(); + self.emit('ready'); + setImmediate(callback); + }); }); }; diff --git a/test/services/db.unit.js b/test/services/db.unit.js index bf4f0c4e..5b75ee2e 100644 --- a/test/services/db.unit.js +++ b/test/services/db.unit.js @@ -104,6 +104,135 @@ describe('DB Service', function() { }); }); + describe('#_checkVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will handle an error while retrieving the tip', function() { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new Error('test')); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will handle an error while retrieving the version', function() { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new Error('test')); + } + callCount++; + setImmediate(callback); + }); + db._checkVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + }); + }); + it('will NOT check the version if a tip is not found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = sinon.stub().callsArgWith(2, new levelup.errors.NotFoundError()); + db._checkVersion(done); + }); + it('will NOT give an error if the versions match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 2; + db._checkVersion(done); + }); + it('will give an error if the versions do NOT match', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + var versionBuffer = new Buffer(new Array(4)); + versionBuffer.writeUInt32BE(2); + return callback(null, versionBuffer); + } + callCount++; + setImmediate(callback); + }); + db.version = 3; + db._checkVersion(function(err) { + should.exist(err); + err.message.should.match(/^The version of the database/); + done(); + }); + }); + it('will default to version 1 if the version is NOT found', function(done) { + var db = new DB(config); + db.store = {}; + db.store.get = function() {}; + var callCount = 0; + sinon.stub(db.store, 'get', function(key, options, callback) { + if (callCount === 1) { + return callback(new levelup.errors.NotFoundError()); + } + callCount++; + setImmediate(callback); + }); + db.version = 1; + db._checkVersion(done); + }); + }); + + describe('#_setVersion', function() { + var config = { + node: { + network: Networks.get('testnet'), + datadir: 'testdir' + }, + store: memdown + }; + it('will give an error from the store', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, new Error('test')); + db._setVersion(function(err) { + should.exist(err); + err.message.should.equal('test'); + done(); + }); + }); + it('will set the version', function(done) { + var db = new DB(config); + db.store = {}; + db.store.put = sinon.stub().callsArgWith(2, null); + db.version = 5; + db._setVersion(function(err) { + if (err) { + return done(err); + } + db.store.put.args[0][0].should.deep.equal(new Buffer('ff', 'hex')); + db.store.put.args[0][1].should.deep.equal(new Buffer('00000005', 'hex')); + done(); + }); + }); + }); + describe('#start', function() { var TestDB; @@ -126,6 +255,8 @@ describe('DB Service', function() { }; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); var readyFired = false; db.on('ready', function() { @@ -144,6 +275,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.sync = sinon.stub(); db.start(function() { db.sync = function() { @@ -161,6 +294,8 @@ describe('DB Service', function() { db.node.services.bitcoind.genesisBuffer = genesisBuffer; db.loadTip = sinon.stub().callsArg(0); db.connectBlock = sinon.stub().callsArg(1); + db._checkVersion = sinon.stub().callsArg(0); + db._setVersion = sinon.stub().callsArg(0); db.node.stopping = true; db.sync = sinon.stub(); db.start(function() {