Latest version for PPLNS payouts

* Rebased with next to add logging functions
* Added block method to fetch specific block
* Modified getArchive shares method to be more constistent
* Added new global configuration
* Adjusted smarty globals with new configuration
* More verbose pplns cron with logging
* Re-target the round in case of PPLNS not being matched by archive

Fixes #143

cronjobs/pplns_payout.php

@@ -24,62 +24,90 @@ require_once('shared.inc.php');
 
 // Check if we are set as the payout system
 if ($config['payout_system'] != 'pplns') {
-  verbose("Please activate this cron in configuration via payout_system = pplns\n");
+  $log->logInfo("Please activate this cron in configuration via payout_system = pplns");
   exit(0);
 }
 
 // Fetch all unaccounted blocks
 $aAllBlocks = $block->getAllUnaccounted('ASC');
 if (empty($aAllBlocks)) {
-  verbose("No new unaccounted blocks found\n");
+  $log->logDebug("No new unaccounted blocks found");
   exit(0);
 }
 
-// We support some dynamic share targets but fall back to our fixed value
-if ($config['pplns']['shares']['type'] == 'blockavg' && $block->getBlockCount() > 0) {
-  $pplns_target = round($block->getAvgBlockShares($config['pplns']['type']['blockavg']['blockcount']));
-} else {
-  $pplns_target = $config['pplns']['shares']['default'] ;
-}
-
 $count = 0;
 foreach ($aAllBlocks as $iIndex => $aBlock) {
+  // We support some dynamic share targets but fall back to our fixed value
+  // Re-calculate after each run due to re-targets in this loop
+  if ($config['pplns']['shares']['type'] == 'blockavg' && $block->getBlockCount() > 0) {
+    $pplns_target = round($block->getAvgBlockShares($config['pplns']['blockavg']['blockcount']));
+  } else {
+    $pplns_target = $config['pplns']['shares']['default'] ;
+  }
+
   if (!$aBlock['accounted']) {
     $iPreviousShareId = @$aAllBlocks[$iIndex - 1]['share_id'] ? $aAllBlocks[$iIndex - 1]['share_id'] : 0;
     $iCurrentUpstreamId = $aBlock['share_id'];
-    if (!is_number($iCurrentUpstreamId)) die("Block " . $aBlock['height'] . " has no share_id associated with it, not going to continue\n");
+    if (!is_numeric($iCurrentUpstreamId)) {
+      $log->logFatal("Block " . $aBlock['height'] . " has no share_id associated with it, not going to continue");
+      exit(1);
+    }
     $iRoundShares = $share->getRoundShares($iPreviousShareId, $aBlock['share_id']);
+    $iNewRoundShares = 0;
     $config['reward_type'] == 'block' ? $dReward = $aBlock['amount'] : $dReward = $config['reward'];
     $aRoundAccountShares = $share->getSharesForAccounts($iPreviousShareId, $aBlock['share_id']);
 
     if ($iRoundShares >= $pplns_target) {
-      verbose("Matching or exceeding PPLNS target of $pplns_target\n");
+      $log->logDebug("Matching or exceeding PPLNS target of $pplns_target with $iRoundShares");
       $aAccountShares = $share->getSharesForAccounts($aBlock['share_id'] - $pplns_target + 1, $aBlock['share_id']);
+      if (empty($aAccountShares)) {
+        $log->logFatal("No shares found for this block, aborted!");
+        exit(1);
+      }
     } else {
-      verbose("Not able to match PPLNS target of $pplns_target\n");
+      $log->logDebug("Not able to match PPLNS target of $pplns_target with $iRoundShares");
       // We need to fill up with archived shares
       // Grab the full current round shares since we didn't match target
       $aAccountShares = $aRoundAccountShares;
+      if (empty($aAccountShares)) {
+        $log->logFatal("No shares found for this block, aborted!");
+        exit(1);
+      }
+
       // Grab only the most recent shares from Archive that fill the missing shares
-      $aArchiveShares = $share->getArchiveShares($share->getMaxArchiveShareId() - ($pplns_target- $iRoundShares) + 1, $share->getMaxArchiveShareId());
+      $log->logInfo('Fetching ' . ($pplns_target - $iRoundShares) . ' additional shares from archive');
-      // Add archived shares to users current shares, if we have any in archive
+      if (!$aArchiveShares = $share->getArchiveShares($pplns_target - $iRoundShares)) {
-      if (is_array($aArchiveShares)) {
+        $log->logError('Failed to fetch shares from archive, setting target to round total');
-        foreach($aAccountShares as $key => $aData) {
+        $pplns_target = $iRoundShares;
-          if (array_key_exists($aData['username'], $aArchiveShares)) {
+      } else {
-            $aAccountShares[$key]['valid'] += $aArchiveShares[$aData['username']]['valid'];
+        // Add archived shares to users current shares, if we have any in archive
-            $aAccountShares[$key]['invalid'] += $aArchiveShares[$aData['username']]['invalid'];
+        if (is_array($aArchiveShares)) {
+          $log->logDebug('Found shares in archive to match PPLNS target, calculating per-user shares');
+          foreach($aAccountShares as $key => $aData) {
+            if (array_key_exists($aData['username'], $aArchiveShares)) {
+              $log->logDebug('Found user ' . $aData['username'] . ' in archived shares');
+              $log->logDebug('  valid   : ' . $aAccountShares[$key]['valid'] . ' + ' . $aArchiveShares[$aData['username']]['valid'] . ' = ' . ($aAccountShares[$key]['valid'] + $aArchiveShares[$aData['username']]['valid']) );
+              $log->logDebug('  invalid : ' . $aAccountShares[$key]['invalid'] . ' + ' . $aArchiveShares[$aData['username']]['invalid'] . ' = ' . ($aAccountShares[$key]['invalid'] + $aArchiveShares[$aData['username']]['invalid']) );
+              $aAccountShares[$key]['valid'] += $aArchiveShares[$aData['username']]['valid'];
+              $aAccountShares[$key]['invalid'] += $aArchiveShares[$aData['username']]['invalid'];
+            }
           }
         }
+        // We tried to fill up to PPLNS target, now we need to check the actual shares to properly payout users
+        foreach($aAccountShares as $key => $aData) {
+          $iNewRoundShares += $aData['valid'];
+        }
       }
     }
-    if (empty($aAccountShares)) {
+
-      verbose("\nNo shares found for this block\n\n");
+    // We filled from archive but still are not able to match PPLNS target, re-adjust
-      sleep(2);
+    if ($iRoundShares < $iNewRoundShares) {
-      continue;
+      $log->logInfo('Adjusting round target to ' . $iNewRoundShares);
+      $iRoundShares = $iNewRoundShares;
     }
 
     // Table header for account shares
-    verbose("ID\tUsername\tValid\tInvalid\tPercentage\tPayout\t\tDonation\tFee\t\tStatus\n");
+    $log->logInfo("ID\tUsername\tValid\tInvalid\tPercentage\tPayout\t\tDonation\tFee");
 
     // Loop through all accounts that have found shares for this round
     foreach ($aAccountShares as $key => $aData) {
@@ -96,48 +124,50 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
       $aData['donation'] = number_format(round($user->getDonatePercent($user->getUserId($aData['username'])) / 100 * ( $aData['payout'] - $aData['fee']), 8), 8);
 
       // Verbose output of this users calculations
-      verbose($aData['id'] . "\t" .
+      $log->logInfo($aData['id'] . "\t" .
-           $aData['username'] . "\t" .
+        $aData['username'] . "\t" .
-           $aData['valid'] . "\t" .
+        $aData['valid'] . "\t" .
-           $aData['invalid'] . "\t" .
+        $aData['invalid'] . "\t" .
-           $aData['percentage'] . "\t" .
+        $aData['percentage'] . "\t" .
-           $aData['payout'] . "\t" .
+        $aData['payout'] . "\t" .
-           $aData['donation'] . "\t" .
+        $aData['donation'] . "\t" .
-           $aData['fee'] . "\t");
+        $aData['fee']);
 
-      $strStatus = "OK";
       // Add full round share statistics, not just PPLNS
       foreach ($aRoundAccountShares as $key => $aRoundData) {
         if ($aRoundData['username'] == $aData['username'])
           if (!$statistics->updateShareStatistics($aRoundData, $aBlock['id']))
-            $strStatus = "Stats Failed";
+            $log->logError('Failed to update share statistics for ' . $aData['username']);
       }
       // Add new credit transaction
       if (!$transaction->addTransaction($aData['id'], $aData['payout'], 'Credit', $aBlock['id']))
-        $strStatus = "Transaction Failed";
+        $log->logFatal('Failed to insert new Credit transaction to database for ' . $aData['username']);
       // Add new fee debit for this block
       if ($aData['fee'] > 0 && $config['fees'] > 0)
         if (!$transaction->addTransaction($aData['id'], $aData['fee'], 'Fee', $aBlock['id']))
-          $strStatus = "Fee Failed";
+          $log->logFatal('Failed to insert new Fee transaction to database for ' . $aData['username']);
       // Add new donation debit
       if ($aData['donation'] > 0)
         if (!$transaction->addTransaction($aData['id'], $aData['donation'], 'Donation', $aBlock['id']))
-          $strStatus = "Donation Failed";
+          $log->logFatal('Failed to insert new Donation transaction to database for ' . $aData['username']);
-      verbose("\t$strStatus\n");
     }
 
     // Move counted shares to archive before this blockhash upstream share
-    $share->moveArchive($iCurrentUpstreamId, $aBlock['id'], $iPreviousShareId);
+    if (!$share->moveArchive($iCurrentUpstreamId, $aBlock['id'], $iPreviousShareId))
+      $log->logError('Failed to copy shares to archive table');
     // Delete all accounted shares
     if (!$share->deleteAccountedShares($iCurrentUpstreamId, $iPreviousShareId)) {
-      verbose("\nERROR : Failed to delete accounted shares from $iPreviousShareId to $iCurrentUpstreamId, aborting!\n");
+      $log->logFatal("Failed to delete accounted shares from $iPreviousShareId to $iCurrentUpstreamId, aborting!");
       exit(1);
     }
+    // If we don't keep archives, delete some now to release disk space
+    if (!$share->purgeArchive()) {
+      $log->logError("Failed to delete archived shares, not critical but should be checked!");
+    }
     // Mark this block as accounted for
     if (!$block->setAccounted($aBlock['id'])) {
-      verbose("\nERROR : Failed to mark block as accounted! Aborting!\n");
+      $log->logFatal("Failed to mark block as accounted! Aborting!");
+      exit(1);
     }
-
-    verbose("------------------------------------------------------------------------\n\n");
   }
 }

cronjobs/proportional_payout.php

@@ -97,7 +97,8 @@ foreach ($aAllBlocks as $iIndex => $aBlock) {
     }
 
     // Move counted shares to archive before this blockhash upstream share
-    if ($config['archive_shares']) $share->moveArchive($iCurrentUpstreamId, $aBlock['id'], $iPreviousShareId);
+    if (!$share->moveArchive($iCurrentUpstreamId, $aBlock['id'], $iPreviousShareId))
+      $log->logError('Failed to copy shares to archive');
     // Delete all accounted shares
     if (!$share->deleteAccountedShares($iCurrentUpstreamId, $iPreviousShareId)) {
       $log->logFatal('Failed to delete accounted shares from ' . $iPreviousShareId . ' to ' . $iCurrentUpstreamId . ', aborted');

public/include/classes/block.class.php

@@ -43,6 +43,18 @@ class Block {
     return false;
   }
 
+  /**
+   * Get a specific block, by block height
+   * @param height int Block Height
+   * @return data array Block information from DB
+   **/
+  public function getBlock($height) {
+    $stmt = $this->mysqli->prepare("SELECT * FROM $this->table WHERE height = ? LIMIT 1");
+    if ($this->checkStmt($stmt) && $stmt->bind_param('i', $height) && $stmt->execute() && $result = $stmt->get_result())
+      return $result->fetch_assoc();
+    return false;
+  }
+
   /**
    * Get our last, highest share ID inserted for a block
    * @param none

public/include/classes/share.class.php

@@ -13,10 +13,12 @@ class Share {
   // This defines each share
   public $rem_host, $username, $our_result, $upstream_result, $reason, $solution, $time;
 
-  public function __construct($debug, $mysqli, $user) {
+  public function __construct($debug, $mysqli, $user, $block, $config) {
     $this->debug = $debug;
     $this->mysqli = $mysqli;
     $this->user = $user;
+    $this->config = $config;
+    $this->block = $block;
     $this->debug->append("Instantiated Share class", 2);
   }
 
@@ -126,28 +128,55 @@ class Share {
    * param right int Right/highest share ID
    * return array data Returns an array with usernames as keys for easy access
    **/
-  function getArchiveShares($left, $right) {
+  function getArchiveShares($iCount) {
+    $iMinId = $this->getMaxArchiveShareId() - $iCount;
+    $iMaxId = $this->getMaxArchiveShareId();
     $stmt = $this->mysqli->prepare("
       SELECT
         a.id,
-        SUBSTRING_INDEX( s.username , '.', 1 ) as username,
+        SUBSTRING_INDEX( s.username , '.', 1 ) as account,
-        SUM(IF(our_result='Y', 1, 0)) AS valid,
+        IFNULL(SUM(IF(our_result='Y', 1, 0)), 0) AS valid,
-        SUM(IF(our_result='N', 1, 0)) AS invalid
+        IFNULL(SUM(IF(our_result='N', 1, 0)), 0) AS invalid
       FROM $this->tableArchive AS s
       LEFT JOIN " . $this->user->getTableName() . " AS a
       ON a.username = SUBSTRING_INDEX( s.username , '.', 1 )
-      WHERE s.id BETWEEN ? AND ?
+      WHERE s.share_id > ? AND s.share_id <= ?
-      GROUP BY username DESC
+      GROUP BY account DESC");
-    ");
+    if ($this->checkStmt($stmt) && $stmt->bind_param("ii", $iMinId, $iMaxId) && $stmt->execute() && $result = $stmt->get_result()) {
-    if ($this->checkStmt($stmt) && $stmt->bind_param("ii", $left, $right) && $stmt->execute() && $result = $stmt->get_result()) {
       $aData = NULL;
       while ($row = $result->fetch_assoc()) {
-        $aData[$row['username']] = $row;
+        $aData[$row['account']] = $row;
       }
       if (is_array($aData)) return $aData;
     }
     return false;
   }
+
+  /**
+   * We keep shares only up to a certain point
+   * This can be configured by the user.
+   * @return return bool true or false
+   **/
+  public function purgeArchive() {
+    if ($this->config['payout_system'] == 'pplns') {
+      // Fetch our last block so we can go back configured rounds
+      $aLastBlock = $this->block->getLast();
+      // Fetch the block we need to find the share_id
+      $aBlock = $this->block->getBlock($aLastBlock['height'] - $this->config['archive']['maxrounds']);
+      // Now that we know our block, remove those shares
+      $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE block_id < ? AND time < DATE_SUB(now(), INTERVAL ? MINUTE)");
+      if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $aBlock['id'], $config['archive']['maxage']) && $stmt->execute())
+        return true;
+    } else {
+      // We are not running pplns, so we just need to keep shares of the past <interval> minutes
+      $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE time < DATE_SUB(now(), INTERVAL ? MINUTE)");
+      if ($this->checkStmt($stmt) && $stmt->bind_param('i', $config['archive']['maxage']) && $stmt->execute())
+      return true;
+    }
+    // Catchall
+    return false;
+  }
+
   /**
    * Move accounted shares to archive table, this step is optional
    * @param previous_upstream int Previous found share accepted by upstream to limit results
@@ -289,4 +318,4 @@ class Share {
   }
 }
 
-$share = new Share($debug, $mysqli, $user);
+$share = new Share($debug, $mysqli, $user, $block, $config);

public/include/config/global.inc.dist.php

@@ -169,8 +169,35 @@ $config['block_bonus'] = 0;
 **/
 $config['payout_system'] = 'prop';
 
-// For debugging purposes you can archive shares in the archive_shares table, default: true
+/**
-$config['archive_shares'] = true;
+ * Archiving configuration for debugging
+ *
+ * Explanation:
+ *   By default, we don't need to archive for a long time. PPLNS and Hashrate
+ *   calculations rely on this archive, but all shares past a certain point can
+ *   safely be deleted.
+ *
+ *   To ensure we have enough shares on stack for PPLNS, this
+ *   is set to the past 10 rounds. Even with lucky ones in between those should
+ *   fit the PPLNS target. On top of that, even if we have more than 10 rounds,
+ *   we still keep the last maxage shares to ensure we can calculate hashrates.
+ *   Both conditions need to be met in order for shares to be purged from archive.
+ *
+ *   Proportional mode will only keep the past 24 hours. These are required for
+ *   hashrate calculations to work past a round, hence 24 hours was selected as
+ *   the default. You may want to increase the time for debugging, then add any
+ *   integer reflecting minutes of shares to keep.
+ *
+ * Availabe Options:
+ *   maxrounds  :  PPLNS, keep shares for maxrounds
+ *   maxage     :  PROP and PPLNS, delete shares older than maxage minutes
+ *
+ * Default:
+ *   maxrounds  =  10
+ *   maxage     =  60 * 60 * 24   (24h)
+ **/
+$config['archive']['maxrounds'] = 10; 
+$config['archive']['maxage'] = 60 * 60 * 24; 
 
 // URL prefix for block searches, used for block links, default: `http://explorer.litecoin.net/search?q=`
 // If empty, the block link to the block information page will be removed

public/include/smarty_globals.inc.php

@@ -84,7 +84,7 @@ if (@$_SESSION['USERDATA']['id']) {
 
   switch ($config['payout_system']) {
   case 'pplns':
-    if ($iAvgBlockShares = round($block->getAvgBlockShares($config['pplns']['type']['blockavg']['blockcount']))) {
+    if ($iAvgBlockShares = round($block->getAvgBlockShares($config['pplns']['blockavg']['blockcount']))) {
       $aGlobal['pplns']['target'] = $iAvgBlockShares;
     } else {
       $aGlobal['pplns']['target'] = $config['pplns']['shares']['default'];