bitcoind: handle unexpected process exits

lib/services/bitcoind.js

@@ -52,6 +52,9 @@ function Bitcoin(options) {
   this.maxAddressesQuery = options.maxAddressesQuery || Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY;
   this.shutdownTimeout = options.shutdownTimeout || Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT;
 
+  // spawn restart setting
+  this.spawnRestartTime = options.spawnRestartTime || Bitcoin.DEFAULT_SPAWN_RESTART_TIME;
+
   // try all interval
   this.tryAllInterval = options.tryAllInterval || Bitcoin.DEFAULT_TRY_ALL_INTERVAL;
   this.startRetryInterval = options.startRetryInterval || Bitcoin.DEFAULT_START_RETRY_INTERVAL;
@@ -70,6 +73,7 @@ Bitcoin.dependencies = [];
 Bitcoin.DEFAULT_MAX_HISTORY = 10;
 Bitcoin.DEFAULT_SHUTDOWN_TIMEOUT = 15000;
 Bitcoin.DEFAULT_MAX_ADDRESSES_QUERY = 10000;
+Bitcoin.DEFAULT_SPAWN_RESTART_TIME = 5000;
 Bitcoin.DEFAULT_TRY_ALL_INTERVAL = 1000;
 Bitcoin.DEFAULT_REINDEX_INTERVAL = 10000;
 Bitcoin.DEFAULT_START_RETRY_INTERVAL = 5000;
@@ -613,6 +617,39 @@ Bitcoin.prototype._loadTipFromNode = function(node, callback) {
   });
 };
 
+Bitcoin.prototype._stopSpawnedBitcoin = function(callback) {
+  var spawnOptions = this.options.spawn;
+  var pidPath = spawnOptions.datadir + '/bitcoind.pid';
+
+  function stopProcess() {
+    fs.readFile(pidPath, 'utf8', function(err, pid) {
+      if (err && err.code === 'ENOENT') {
+        // pid file doesn't exist we can continue
+        return callback(null);
+      } else if (err) {
+        return callback(err);
+      }
+      pid = parseInt(pid);
+      log.warn('Stopping existing spawned bitcoin process with pid: ' + pid);
+      try {
+        process.kill(pid, 'SIGINT');
+      } catch(err) {
+        if (err && err.code === 'ESRCH') {
+          log.warn('Unclean bitcoin process shutdown, process not found with pid: ' + pid);
+          return callback(null);
+        } else if(err) {
+          return callback(err);
+        }
+      }
+      setTimeout(function() {
+        stopProcess();
+      }, 10000);
+    });
+  }
+
+  stopProcess();
+};
+
 Bitcoin.prototype._spawnChildProcess = function(callback) {
   var self = this;
 
@@ -634,10 +671,32 @@ Bitcoin.prototype._spawnChildProcess = function(callback) {
   if (self._getNetworkOption()) {
     options.push(self._getNetworkOption());
   }
-  self.spawn.process = spawn(this.spawn.exec, options, {stdio: 'inherit'});
+
+  self._stopSpawnedBitcoin(function(err) {
+    if (err) {
+      return callback(err);
+    }
+
+    log.info('Starting bitcoin process');
+    self.spawn.process = spawn(self.spawn.exec, options, {stdio: 'inherit'});
 
     self.spawn.process.on('error', function(err) {
-    log.error(err);
+      self.emit('error', err);
+    });
+
+    self.spawn.process.once('exit', function(code) {
+      if (!self.node.stopping) {
+        log.warn('Bitcoin process unexpectedly exited with code:', code);
+        log.warn('Restarting bitcoin child process in ' + self.spawnRestartTime + 'ms');
+        setTimeout(function() {
+          self._spawnChildProcess(function(err) {
+            if (err) {
+              return self.emit('error', err);
+            }
+            log.warn('Bitcoin process restarted');
+          });
+        }, self.spawnRestartTime);
+      }
     });
 
     async.retry({times: 60, interval: self.startRetryInterval}, function(done) {
@@ -671,6 +730,9 @@ Bitcoin.prototype._spawnChildProcess = function(callback) {
       });
 
     });
+
+  });
+
 };
 
 Bitcoin.prototype._connectProcess = function(config, callback) {