[IMPROVED] Re-factored monitoring criticals/errors

* [ADDED] new monitoring method : endCronjob
* [IMPROVED] Use newly added error codes
* [ADDED] mail notifications, enabled by default
* [ADDED] cron disable on fatal errors with exit code != 0
* [ADDED] Command line swtich: -f = Force running crons even if disabled
* [ADDED] Disabled status in monitoring site

This will improve error handling in our cronjobs. Fatal errors now
require manual intervention by explicityly running crons with the force
option (`-f`). Until they are forced to run, crons will stay disabled.

Fixes #773 once merged

cronjobs/archive_cleanup.php

@@ -28,10 +28,7 @@ require_once('shared.inc.php');
 // If we don't keep archives, delete some now to release disk space
 if (!$share->purgeArchive()) {
   $log->logError("Failed to delete archived shares, not critical but should be checked!");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0008', 1, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Failed to delete archived shares");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-  exit(1);
 }
 
 // Cron cleanup and monitoring

cronjobs/auto_payout.php

@@ -27,19 +27,12 @@ require_once('shared.inc.php');
 
 if ($setting->getValue('disable_ap') == 1) {
   $log->logInfo(" auto payout disabled via admin panel");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+  $monitoring->endCronjob($cron_name, 'E0009', 0, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Auto-Payout disabled");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-  $monitoring->setStatus($cron_name . "_endtime", "date", time());
-  exit(0);
 }
 
 if ($bitcoin->can_connect() !== true) {
   $log->logFatal(" unable to connect to RPC server, exiting");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+  $monitoring->endCronjob($cron_name, 'E0006', 1, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Unable to connect to RPC server");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-  exit(1);
 }
 
 // Fetch all users with setup AP

cronjobs/blockupdate.php

@@ -27,10 +27,7 @@ require_once('shared.inc.php');
 
 if ( $bitcoin->can_connect() !== true ) {
   $log->logFatal("Failed to connect to RPC server\n");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0006', 1, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Unable to connect to RPC server");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-  exit(1);
 }
 
 // Fetch all unconfirmed blocks

cronjobs/cron_end.inc.php

@@ -19,10 +19,7 @@ limitations under the License.
  */
 
 // Monitoring cleanup and status update
-$monitoring->setStatus($cron_name . "_message", "message", "OK");
+$monitoring->endCronjob($cron_name, 'OK', 0, false, false);
-$monitoring->setStatus($cron_name . "_status", "okerror", 0);
 $monitoring->setStatus($cron_name . "_runtime", "time", microtime(true) - $cron_start[$cron_name]);
 $monitoring->setStatus($cron_name . "_endtime", "date", time());
-// Mark cron as running for monitoring
-$monitoring->setStatus($cron_name . '_active', "yesno", 0);
 ?>

cronjobs/findblock.php

@@ -35,10 +35,7 @@ if ( $bitcoin->can_connect() === true ){
   $aTransactions = $bitcoin->query('listsinceblock', $strLastBlockHash);
 } else {
   $log->logFatal('Unable to conenct to RPC server backend');
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0006', 1, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Unable to connect to RPC server");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-  exit(1);
 }
 
 // Nothing to do so bail out
@@ -97,43 +94,28 @@ if (empty($aAllBlocks)) {
           if ( !$aShareError = $share->getShareById($aBlockError['share_id']) || !$aShareCurrent = $share->getShareById($iCurrentUpstreamId)) {
             // We were not able to fetch all shares that were causing this detection to trigger, bail out
             $log->logFatal('E0002: Failed to fetch both offending shares ' . $iCurrentUpstreamId . ' and ' . $iPreviousShareId . '. Block height: ' . $aBlock['height']);
-            $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+            $monitoring->endCronjob($cron_name, 'E0002', 1, true);
-            $monitoring->setStatus($cron_name . "_message", "message", "E0002: Upstream shares not found");
-            $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-            exit(1);
           }
           // Shares seem to be out of order, so lets change them
           if ( !$share->updateShareById($iCurrentUpstreamId, $aShareError) || !$share->updateShareById($iPreviousShareId, $aShareCurrent)) {
             // We couldn't update one of the shares! That might mean they have been deleted already
             $log->logFatal('E0003: Failed to change shares order!');
-            $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+            $monitoring->endCronjob($cron_name, 'E0003', 1, true);
-            $monitoring->setStatus($cron_name . "_message", "message", "E0003: Failed share update");
-            $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-            exit(1);
           }
           // Reset our offending block so the next run re-checks the shares
           if (!$block->setShareId($aBlockError['id'], NULL) && !$block->setFinder($aBlockError['id'], NULL) || !$block->setShares($aBlockError['id'], NULL)) {
             $log->logFatal('E0004: Failed to reset previous block: ' . $aBlockError['height']);
             $log->logError('Failed to reset block in database: ' . $aBlockError['height']);
-            $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+            $monitoring->endCronjob($cron_name, 'E0004', 1, true);
-            $monitoring->setStatus($cron_name . "_message", "message", "E0004: Failed to reset block");
-            $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-            exit(1);
           }
-          $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+          $monitoring->endCronjob($cron_name, 'E0007', 0, true);
-          $monitoring->setStatus($cron_name . "_message", "message", "Out of Order Share detected, autofixed");
-          $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-          exit(0);
         } else {
           $iRoundShares = $share->getRoundShares($iPreviousShareId, $iCurrentUpstreamId);
           $iAccountId = $user->getUserId($share->getUpstreamFinder());
         }
       } else {
         $log->logFatal('E0005: Unable to fetch blocks upstream share, aborted:' . $share->getError());
-        $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+        $monitoring->endCronjob($cron_name, 'E0005', 1, true);
-        $monitoring->setStatus($cron_name . "_message", "message", "Unable to fetch blocks " . $aBlock['height'] . " upstream share: " . $share->getError());
-        $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-        exit(1);
       }
 
       $log->logInfo(

cronjobs/manual_payout.php

@@ -26,20 +26,13 @@ chdir(dirname(__FILE__));
 require_once('shared.inc.php');
 
 if ($setting->getValue('disable_mp') == 1) {
-  $log->logInfo(" auto payout disabled via admin panel");
+  $log->logInfo(" manual payout disabled via admin panel");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0009', 0, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Auto-Payout disabled");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-  $monitoring->setStatus($cron_name . "_endtime", "date", time());
-  exit(0);
 }
 
 if ($bitcoin->can_connect() !== true) {
   $log->logFatal(" unable to connect to RPC server, exiting");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0006', 1, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Unable to connect to RPC server");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-  exit(1);
 }
 
 // Fetch outstanding payout requests
@@ -53,6 +46,12 @@ if (count($aPayouts) > 0) {
     $aData['coin_address'] = $user->getCoinAddress($aData['account_id']);
     $aData['username'] = $user->getUserName($aData['account_id']);
     if ($dBalance > $config['txfee']) {
+      // To ensure we don't run this transaction again, lets mark it completed
+      if (!$oPayout->setProcessed($aData['id'])) {
+        $log->logFatal('unable to mark transactions ' . $aData['id'] . ' as processed.');
+        $monitoring->endCronjob($cron_name, 'E0010', 1, true);
+      }
+
       $log->logInfo("\t" . $aData['account_id'] . "\t\t" . $aData['username'] . "\t" . $dBalance . "\t\t" . $aData['coin_address']);
       try {
         $aStatus = $bitcoin->validateaddress($aData['coin_address']);
@@ -70,14 +69,6 @@ if (count($aPayouts) > 0) {
         $log->logError('Failed to send requested balance to coin address, please check payout process');
         continue;
       }
-      // To ensure we don't run this transaction again, lets mark it completed
-      if (!$oPayout->setProcessed($aData['id'])) {
-        $log->logFatal('unable to mark transactions ' . $aData['id'] . ' as processed.');
-        $monitoring->setStatus($cron_name . "_active", "yesno", 0);
-        $monitoring->setStatus($cron_name . "_message", "message", "Unable set payout as processed");
-        $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-        exit(1);
-      }
 
       if ($transaction->addTransaction($aData['account_id'], $dBalance - $config['txfee'], 'Debit_MP', NULL, $aData['coin_address']) && $transaction->addTransaction($aData['account_id'], $config['txfee'], 'TXFee', NULL, $aData['coin_address'])) {
         // Mark all older transactions as archived

cronjobs/notifications.php

@@ -26,10 +26,7 @@ chdir(dirname(__FILE__));
 require_once('shared.inc.php');
 
 if ($setting->getValue('disable_notifications') == 1) {
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+  $monitoring->endCronjob($cron_name, 'E0009', 0, true);
-  $monitoring->setStatus($cron_name . "_message", "message", "Cron disabled by admin");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 0);
-  exit(0);
 }
 
 $log->logDebug("  IDLE Worker Notifications ...");

cronjobs/pplns_payout.php

@@ -35,10 +35,7 @@ if ($config['payout_system'] != 'pplns') {
 $aAllBlocks = $block->getAllUnaccounted('ASC');
 if (empty($aAllBlocks)) {
   $log->logDebug("No new unaccounted blocks found");
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+  $monitoring->endCronjob($cron_name, 'E0011', 0, true, false);
-  $monitoring->setStatus($cron_name . "_message", "message", "No new unaccounted blocks");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 0); 
-  exit(0);
 }
 
 $count = 0;
@@ -64,10 +61,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     $iCurrentUpstreamId = $aBlock['share_id'];
     if (!is_numeric($iCurrentUpstreamId)) {
       $log->logFatal("Block " . $aBlock['height'] . " has no share_id associated with it, not going to continue");
-      $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+      $monitoring->endCronjob($cron_name, 'E0012', 1, true);
-      $monitoring->setStatus($cron_name . "_message", "message", "Block " . $aBlock['height'] . " has no share_id associated with it");
-      $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-      exit(1);
     }
     $iRoundShares = $share->getRoundShares($iPreviousShareId, $aBlock['share_id']);
     $iNewRoundShares = 0;
@@ -83,10 +77,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
       $aAccountShares = $share->getSharesForAccounts($iMinimumShareId - 1, $aBlock['share_id']);
       if (empty($aAccountShares)) {
         $log->logFatal("No shares found for this block, aborted! Block Height : " . $aBlock['height']);
-        $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+        $monitoring->endCronjob($cron_name, 'E0013', 1, true);
-        $monitoring->setStatus($cron_name . "_message", "message", "No shares found for this block: " . $aBlock['height']);
-        $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-        exit(1);
       }
       foreach($aAccountShares as $key => $aData) {
         $iNewRoundShares += $aData['valid'];
@@ -100,10 +91,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
       $aAccountShares = $aRoundAccountShares;
       if (empty($aAccountShares)) {
         $log->logFatal("No shares found for this block, aborted! Block height: " . $aBlock['height']);
-        $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+        $monitoring->endCronjob($cron_name, 'E0013', 1, true);
-        $monitoring->setStatus($cron_name . "_message", "message", "No shares found for this block: " . $aBlock['height']);
-        $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-        exit(1);
       }
 
       // Grab only the most recent shares from Archive that fill the missing shares
@@ -141,7 +129,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
                }
              }
           $aAccountShares = $aSharesData;
-          }  
+          }
         }
         // We tried to fill up to PPLNS target, now we need to check the actual shares to properly payout users
         foreach($aAccountShares as $key => $aData) {
@@ -225,15 +213,12 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     // Delete all accounted shares
     if (!$share->deleteAccountedShares($iCurrentUpstreamId, $iPreviousShareId)) {
       $log->logFatal("Failed to delete accounted shares from $iPreviousShareId to $iCurrentUpstreamId, aborting!");
-      exit(1);
+      $monitoring->endCronjob($cron_name, 'E0016', 1, true);
     }
     // Mark this block as accounted for
     if (!$block->setAccounted($aBlock['id'])) {
       $log->logFatal("Failed to mark block as accounted! Aborting!");
-      $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+      $monitoring->endCronjob($cron_name, 'E0014', 1, true);
-      $monitoring->setStatus($cron_name . "_message", "message", "Failed to mark block " . $aBlock['height'] . " as accounted");
-      $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-      exit(1);
     }
   } else {
     $aMailData = array(
@@ -247,10 +232,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     if (!$mail->sendMail('notifications/error', $aMailData))
       $log->logError("    Failed sending notifications: " . $notification->getError() . "\n");
     $log->logFatal('Potential double payout detected. Aborted.');
-    $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+    $monitoring->endCronjob($cron_name, 'E0015', 1, true);
-    $monitoring->setStatus($cron_name . "_message", "message", "Block height for block too low! Potential double payout detected.");
-    $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-    exit(1);
   }
 }
 

cronjobs/pps_payout.php

@@ -121,7 +121,10 @@ $setting->setValue('pps_last_share_id', $iLastShareId);
 
 // Fetch all unaccounted blocks
 $aAllBlocks = $block->getAllUnaccounted('ASC');
-if (empty($aAllBlocks)) $log->logDebug("No new unaccounted blocks found");
+if (empty($aAllBlocks)) {
+  $log->logDebug("No new unaccounted blocks found");
+  // No monitoring event here, not fatal for PPS
+}
 
 // Go through blocks and archive/delete shares that have been accounted for
 foreach ($aAllBlocks as $iIndex => $aBlock) {

cronjobs/proportional_payout.php

@@ -35,10 +35,7 @@ if ($config['payout_system'] != 'prop') {
 $aAllBlocks = $block->getAllUnaccounted('ASC');
 if (empty($aAllBlocks)) {
   $log->logDebug('No new unaccounted blocks found in database');
-  $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+  $monitoring->endCronjob($cron_name, 'E0011', 0, true, false);
-  $monitoring->setStatus($cron_name . "_message", "message", "No new unaccounted blocks");
-  $monitoring->setStatus($cron_name . "_status", "okerror", 0);
-  exit(0);
 }
 
 $count = 0;
@@ -60,10 +57,7 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
 
     if (empty($aAccountShares)) {
       $log->logFatal('No shares found for this block, aborted: ' . $aBlock['height']);
-      $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+      $monitoring->endCronjob($cron_name, 'E0013', 1, true);
-      $monitoring->setStatus($cron_name . "_message", "message", "No shares found for this block, aborted: " . $aBlock['height']);
-      $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-      exit(1);
     }
 
     // Loop through all accounts that have found shares for this round
@@ -115,18 +109,12 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     // Delete all accounted shares
     if (!$share->deleteAccountedShares($iCurrentUpstreamId, $iPreviousShareId)) {
       $log->logFatal('Failed to delete accounted shares from ' . $iPreviousShareId . ' to ' . $iCurrentUpstreamId . ', aborted');
-      $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+      $monitoring->endCronjob($cron_name, 'E0016', 1, true);
-      $monitoring->setStatus($cron_name . "_message", "message", "Failed to delete accounted shares from " . $iPreviousShareId . " to " . $iCurrentUpstreamId);
-      $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-      exit(1);
     }
     // Mark this block as accounted for
     if (!$block->setAccounted($aBlock['id'])) {
       $log->logFatal('Failed to mark block as accounted! Aborted.');
-      $monitoring->setStatus($cron_name . "_active", "yesno", 0); 
+      $monitoring->endCronjob($cron_name, 'E0014', 1, true);
-      $monitoring->setStatus($cron_name . "_message", "message", "Failed to mark block " . $aBlock['height'] . " as accounted");
-      $monitoring->setStatus($cron_name . "_status", "okerror", 1); 
-      exit(1);
     }
   } else {
     $log->logFatal('Possible double payout detected. Aborted.');
@@ -140,10 +128,8 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     );
     if (!$mail->sendMail('notifications/error', $aMailData))
       $log->logError("    Failed sending notifications: " . $notification->getError() . "\n");
-    $monitoring->setStatus($cron_name . "_active", "yesno", 0);
+    $log->logFatal('Potential double payout detected. Aborted.');
-    $monitoring->setStatus($cron_name . "_message", "message", 'Possible double payout detected. Aborted.');
+    $monitoring->endCronjob($cron_name, 'E0015', 1, true);
-    $monitoring->setStatus($cron_name . "_status", "okerror", 1);
-    exit(1);
   }
 }
 

cronjobs/run-crons.sh

@@ -29,17 +29,32 @@ SUBFOLDER=""
 #                                                              #
 ################################################################
 
+# Mac OS detection
+OS=`uname`
+
+
+case "$OS" in
+  Darwin) READLINK=$( which greadlink ) ;;
+  *) READLINK=$( which readlink ) ;;
+esac
+
+if [[ ! -x $READLINK ]]; then
+  echo "readlink not found, please install first";
+  exit 1;
+fi
+
 # My own name
 ME=$( basename $0 )
 
 # Overwrite some settings via command line arguments
-while getopts "hvp:d:" opt; do
+while getopts "hfvp:d:" opt; do
   case "$opt" in
     h|\?)
       echo "Usage: $0 [-v] [-p PHP_BINARY] [-d SUBFOLDER]";
       exit 0
       ;;
     v) VERBOSE=1 ;;
+    f) PHP_OPTS="$PHP_OPTS -f";;
     p) PHP_BIN=$OPTARG ;;
     d) SUBFOLDER=$OPTARG ;;
     :)
@@ -52,7 +67,7 @@ done
 # Path to PID file, needs to be writable by user running this
 PIDFILE="${BASEPATH}/${SUBFOLDER}/${ME}.pid"
 # Clean PIDFILE path
-PIDFILE=$(readlink -m "$PIDFILE")
+PIDFILE=$($READLINK -m "$PIDFILE")
 
 # Create folders recursively if necessary
 if ! $(mkdir -p $( dirname $PIDFILE)); then
@@ -62,7 +77,7 @@ fi
 
 # Find scripts path
 if [[ -L $0 ]]; then
-  CRONHOME=$( dirname $( readlink $0 ) )
+  CRONHOME=$( dirname $( $READLINK $0 ) )
 else
   CRONHOME=$( dirname $0 )
 fi
@@ -104,7 +119,7 @@ echo $PID > $PIDFILE
 
 for cron in $CRONS; do
   [[ $VERBOSE == 1 ]] && echo "Running $cron, check logfile for details"
-  $PHP_BIN $cron
+  $PHP_BIN $cron $PHP_OPTS
 done
 
 # Remove pidfile

cronjobs/shared.inc.php

@@ -41,6 +41,16 @@ require_once(BASEPATH . 'include/config/global.inc.php');
 // We include all needed files here, even though our templates could load them themself
 require_once(INCLUDE_DIR . '/autoloader.inc.php');
 
+// Command line switches
+array_shift($argv);
+foreach ($argv as $option) {
+  switch ($option) {
+  case '-f':
+    $monitoring->setStatus($cron_name . "_disabled", "yesno", 0);
+    break;
+  }
+}
+
 // Load 3rd party logging library for running crons
 $log = new KLogger ( 'logs/' . $cron_name . '.txt' , KLogger::INFO );
 $log->LogDebug('Starting ' . $cron_name);
@@ -48,8 +58,13 @@ $log->LogDebug('Starting ' . $cron_name);
 // Load the start time for later runtime calculations for monitoring
 $cron_start[$cron_name] = microtime(true);
 
+// Check if our cron is activated
+if ($monitoring->isDisabled($cron_name)) {
+  $log->logFatal('Cronjob is currently disabled due to errors, use -f option to force running cron.');
+  $monitoring->endCronjob($cron_name, 'E0018', 1, true, true);
+}
+
 // Mark cron as running for monitoring
 $log->logDebug('Marking cronjob as running for monitoring');
-$monitoring->setStatus($cron_name . '_active', 'yesno', 1);
 $monitoring->setStatus($cron_name . '_starttime', 'date', time());
 ?>

cronjobs/tickerupdate.php

@@ -29,7 +29,7 @@ require_once('shared.inc.php');
 require_once(CLASS_DIR . '/tools.class.php');
 
 if ($price = $tools->getPrice()) {
-  $log->logInfo("Price update: found $price as price");
+  $log->logDebug("Price update: found $price as price");
   if (!$setting->setValue('price', $price))
     $log->logError("unable to update value in settings table");
 } else {
@@ -40,11 +40,11 @@ if ($setting->getValue('monitoring_uptimerobot_private_key')) {
   $monitoring->setTools($tools);
   if (!$monitoring->storeUptimeRobotStatus()) {
     $log->logError('Failed to update Uptime Robot Status: ' . $monitoring->getError());
+    $monitoring->endCronjob($cron_name, 'E0017', 1, true);
   }
 } else {
   $log->logDebug('Skipped Uptime Robot API update, missing private key');
 }
 
-
 require_once('cron_end.inc.php');
 ?>

public/include/autoloader.inc.php

@@ -11,6 +11,7 @@ require_once(CLASS_DIR . '/debug.class.php');
 require_once(INCLUDE_DIR . '/lib/KLogger.php');
 require_once(INCLUDE_DIR . '/database.inc.php');
 require_once(INCLUDE_DIR . '/config/memcache_keys.inc.php');
+require_once(INCLUDE_DIR . '/config/error_codes.inc.php');
 
 // We need to load these two first
 require_once(CLASS_DIR . '/base.class.php');

public/include/classes/base.class.php

@@ -32,6 +32,9 @@ class Base {
   public function setConfig($config) {
     $this->config = $config;
   }
+  public function setErrorCodes($aErrorCodes) {
+    $this->aErrorCodes = $aErrorCodes;
+  }
   public function setToken($token) {
     $this->token = $token;
   }
@@ -59,7 +62,11 @@ class Base {
   public function getError() {
     return $this->sError;
   }
-
+  public function getErrorMsg($errCode) {
+    if (!is_array($this->aErrorCodes)) return 'Error codes not loaded';
+    if (!array_key_exists($errCode, $this->aErrorCodes)) return 'Unknown Error: ' . $errCode;
+    return $this->aErrorCodes[$errCode];
+  }
   protected function getAllAssoc($value, $field='id', $type='i') {
     $this->debug->append("STA " . __METHOD__, 4);
     $stmt = $this->mysqli->prepare("SELECT * FROM $this->table WHERE $field = ? LIMIT 1");

public/include/classes/monitoring.class.php

@@ -24,8 +24,13 @@ class Monitoring extends Base {
         $aMonitor['api_key'] = $temp[0];
         $aMonitor['monitor_id'] = $temp[1];
         $target = '/getMonitors?apiKey=' . $aMonitor['api_key'] . '&monitors=' . $aMonitor['monitor_id'] . '&format=json&noJsonCallback=1&customUptimeRatio=1-7-30&logs=1';
-        if (!$aMonitorStatus = $this->tools->getApi($url, $target)) {
+        $aMonitorStatus = $this->tools->getApi($url, $target);
-          $this->setErrorMessage('Failed to run API call: ' . $this->tools->getError());
+        if (!$aMonitorStatus || @$aMonitorStatus['stat'] == 'fail') {
+          if (is_array($aMonitorStatus) && array_key_exists('message', @$aMonitorStatus)) {
+            $this->setErrorMessage('Failed to run API call: ' . $aMonitorStatus['message']);
+          } else {
+            $this->setErrorMessage('Failed to run API call: ' . $this->tools->getError());
+          }
           return false;
         }
         $aMonitorStatus['monitors']['monitor'][0]['customuptimeratio'] = explode('-', $aMonitorStatus['monitors']['monitor'][0]['customuptimeratio']);
@@ -50,6 +55,16 @@ class Monitoring extends Base {
     return false;
   }
 
+  /**
+   * Check that our cron is currently activated
+   * @param name string Cronjob name
+   * @return bool true or false
+   **/
+  public function isDisabled($name) {
+    $aStatus = $this->getStatus($name . '_disabled');
+    return $aStatus['value'];
+  }
+
   /**
    * Fetch a value from our table
    * @param name string Setting name
@@ -83,10 +98,41 @@ class Monitoring extends Base {
     $this->debug->append("Failed to set $name to $value");
     return false;
   }
+
+  /**
+   * End cronjob with an error message
+   * @param cron_name string Cronjob Name
+   * @param msgCode string Message code as stored in error_codes array
+   * @param exitCode int Exit code to pass on to exit function and monitor report
+   * @param fatal boolean Should we exit out entirely
+   * @return none
+   **/
+  public function endCronjob($cron_name, $msgCode, $exitCode=0, $fatal=false, $mail=true) {
+    $this->setStatus($cron_name . "_active", "yesno", 0);
+    $this->setStatus($cron_name . "_message", "message", $this->getErrorMsg($msgCode));
+    $this->setStatus($cron_name . "_status", "okerror", $exitCode);
+    $this->setStatus($cron_name . "_endtime", "date", time());
+    if ($mail) {
+      $aMailData = array(
+        'email' => $this->setting->getValue('system_error_email'),
+        'subject' => 'Cronjob Failure',
+        'Error Code' => $msgCode,
+        'Error Message' => $this->getErrorMsg($msgCode)
+      );
+      if (!$this->mail->sendMail('notifications/error', $aMailData))
+        $this->setErrorMessage('Failed to send mail notification');
+    }
+    if ($fatal) {
+      if ($exitCode != 0) $this->setStatus($cron_name . "_disabled", "yesno", 1);
+      exit($exitCode);
+    }
+  }
 }
 
 $monitoring = new Monitoring();
+$monitoring->setErrorCodes($aErrorCodes);
 $monitoring->setConfig($config);
 $monitoring->setDebug($debug);
+$monitoring->setMail($mail);
 $monitoring->setMysql($mysqli);
 $monitoring->setSetting($setting);

public/include/pages/admin/monitoring.inc.php

@@ -28,6 +28,7 @@ case 'prop':
 // Data array for template
 foreach ($aCrons as $strCron) {
   $aCronStatus[$strCron] = array(
+    'disabled' => $monitoring->getStatus($strCron . '_disabled'),
     'exit' => $monitoring->getStatus($strCron . '_status'),
     'active' => $monitoring->getStatus($strCron . '_active'),
     'runtime' => $monitoring->getStatus($strCron . '_runtime'),

public/templates/mmcFE/admin/monitoring/default.tpl

@@ -1,6 +1,7 @@
 {include file="global/block_header.tpl" BLOCK_HEADER="Monitoring"}
     <table width="88%">
       <thead>
+        <th>Disabled</th>
         <th>Cronjob</th>
         <th>Exit Code</th>
         <th>Active</th>

public/templates/mpos/admin/monitoring/default.tpl

@@ -3,6 +3,7 @@
     <table class="tablesorter" cellspacing="0">
       <thead>
         <th>Cronjob</th>
+        <th align="center">Disabled</th>
         <th align="center">Exit Code</th>
         <th align="center">Active</th>
         <th align="center">Runtime</th>