bitcoind: handle unexpected process exits

lib/services/bitcoind.js

@@ -52,6 +52,9 @@ function Bitcoin(options) {
   this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY;
   this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT;
 
+  // spawn restart setting
+  this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME;
+
   // try all interval
   this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL;
   this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL;
@@ -70,6 +73,7 @@ Bitcoin.dependencies = [];
 Bitcoin.DEFAULT_MAX_HISTORY = 10;
 Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000;
 Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000;
+Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000;
 Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000;
 Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000;
 Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000;
@@ -613,6 +617,39 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) {
   });
 };
 
+Bitcoin.prototype._stopSpawnedBitcoin = function(callback) {
+  var spawnOptions = this.options.spawn;
+  var pidPath = spawnOptions.datadir + '/bitcoind.pid';
+
+  function stopProcess() {
+    fs.readFile(pidPath, 'utf8', function(err, pid) {
+      if (err && err.code === 'ENOENT') {
+        // pid file doesn't exist we can continue
+        return callback(null);
+      } else if (err) {
+        return callback(err);
+      }
+      pid = parseInt(pid);
+      log.warn('Stopping existing spawned bitcoin process with pid: ' + pid);
+      try {
+        process.kill(pid, 'SIGINT');
+      } catch(err) {
+        if (err && err.code === 'ESRCH') {
+          log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid);
+          return callback(null);
+        } else if(err) {
+          return callback(err);
+        }
+      }
+      setTimeout(function() {
+        stopProcess();
+      }, 10000);
+    });
+  }
+
+  stopProcess();
+};
+
 Bitcoin.prototype._spawnChildProcess = function(callback) {
   var self = this;
 
@@ -634,43 +671,68 @@ Bitcoin.prototype._spawnChildProcess = function(callback) {
   if (self._getNetworkOption()) {
     options.push(self._getNetworkOption());
   }
-  self.spawn.process = spawn(this.spawn.exec, options, {stdio: 'inherit'});
 
-  self.spawn.process.on('error', function(err) {
+  self._stopSpawnedBitcoin(function(err) {
-    log.error(err);
-  });
-
-  async.retry({times: 60, interval: self.startRetryInterval}, function(done) {
-    if (self.node.stopping) {
-      return done(new Error('Stopping while trying to connect to bitcoind.'));
-    }
-
-    node.client = new BitcoinRPC({
-      protocol: 'http',
-      host: '127.0.0.1',
-      port: self.spawn.config.rpcport,
-      user: self.spawn.config.rpcuser,
-      pass: self.spawn.config.rpcpassword
-    });
-
-    self._loadTipFromNode(node, done);
-
-  }, function(err) {
     if (err) {
       return callback(err);
     }
 
-    self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx);
+    log.info('Starting bitcoin process');
+    self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'});
 
-    self._checkReindex(node, function(err) {
+    self.spawn.process.on('error', function(err) {
+      self.emit('error', err);
+    });
+
+    self.spawn.process.once('exit', function(code) {
+      if (!self.node.stopping) {
+        log.warn('Bitcoin process unexpectedly exited with code:', code);
+        log.warn('Restarting bitcoin child process in ' + self.spawnRestartTime + 'ms');
+        setTimeout(function() {
+          self._spawnChildProcess(function(err) {
+            if (err) {
+              return self.emit('error', err);
+            }
+            log.warn('Bitcoin process restarted');
+          });
+        }, self.spawnRestartTime);
+      }
+    });
+
+    async.retry({times: 60, interval: self.startRetryInterval}, function(done) {
+      if (self.node.stopping) {
+        return done(new Error('Stopping while trying to connect to bitcoind.'));
+      }
+
+      node.client = new BitcoinRPC({
+        protocol: 'http',
+        host: '127.0.0.1',
+        port: self.spawn.config.rpcport,
+        user: self.spawn.config.rpcuser,
+        pass: self.spawn.config.rpcpassword
+      });
+
+      self._loadTipFromNode(node, done);
+
+    }, function(err) {
       if (err) {
         return callback(err);
       }
-      self._checkSyncedAndSubscribeZmqEvents(node);
+
-      callback(null, node);
+      self._initZmqSubSocket(node, self.spawn.config.zmqpubrawtx);
+
+      self._checkReindex(node, function(err) {
+        if (err) {
+          return callback(err);
+        }
+        self._checkSyncedAndSubscribeZmqEvents(node);
+        callback(null, node);
+      });
+
     });
 
   });
+
 };
 
 Bitcoin.prototype._connectProcess = function(config, callback) {