Merge pull request #890 from TheSerapher/issue-886

[IMPROVED] Use limits for DELETE statements

public/include/classes/share.class.php

@@ -171,22 +171,41 @@ class Share Extends Base {
    * @return return bool true or false
    **/
   public function purgeArchive() {
+    // Fallbacks if unset
+    if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 25000;
+    if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 1;
+
     if ($this->config['payout_system'] == 'pplns') {
       // Fetch our last block so we can go back configured rounds
       $aLastBlock = $this->block->getLast();
       // Fetch the block we need to find the share_id
       $aBlock = $this->block->getBlock($aLastBlock['height'] - $this->config['archive']['maxrounds']);
       // Now that we know our block, remove those shares
-      $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE block_id < ? AND time < DATE_SUB(now(), INTERVAL ? MINUTE)");
+      $affected = 1;
-      if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $aBlock['id'], $this->config['archive']['maxage']) && $stmt->execute())
+      while ($affected > 0) {
-        return true;
+        // Sleep first to allow any IO to cleanup
+        sleep($this->config['purge']['sleep']);
+        $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE block_id < ? AND time < DATE_SUB(now(), INTERVAL ? MINUTE) LIMIT " . $this->config['purge']['shares']);
+        if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $aBlock['id'], $this->config['archive']['maxage']) && $stmt->execute()) {
+          $affected = $stmt->affected_rows;
+        } else {
+          return $this->sqlError();
+        }
+      }
     } else {
-      // We are not running pplns, so we just need to keep shares of the past <interval> minutes
+      $affected = 1;
-      $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE time < DATE_SUB(now(), INTERVAL ? MINUTE)");
+      while ($affected > 0) {
-      if ($this->checkStmt($stmt) && $stmt->bind_param('i', $this->config['archive']['maxage']) && $stmt->execute())
+        // Sleep first to allow any IO to cleanup
-      return true;
+        sleep($this->config['purge']['sleep']);
+        $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE time < DATE_SUB(now(), INTERVAL ? MINUTE) LIMIT " . $this->config['purge']['shares']);
+        if ($this->checkStmt($stmt) && $stmt->bind_param('i', $this->config['archive']['maxage']) && $stmt->execute()) {
+          $affected = $stmt->affected_rows;
+        } else {
+          return $this->sqlError();
+        }
+      }
     }
-    return $this->sqlError();
+    return true;
   }
 
   /**
@@ -197,11 +216,23 @@ class Share Extends Base {
    * @return bool
    **/
   public function moveArchive($current_upstream, $block_id, $previous_upstream=0) {
-    $archive_stmt = $this->mysqli->prepare("
+    if ($this->config['payout_system'] != 'pplns') {
-      INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty)
+      // We don't need archived shares that much, so only archive as much as configured
-        SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty
+      $sql = "
-        FROM $this->table
+        INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty)
-        WHERE id > ? AND id <= ?");
+          SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty
+          FROM $this->table
+          WHERE id > ? AND id <= ?
+            AND time >= DATE_SUB(now(), INTERVAL " . $this->config['archive']['maxage'] . " MINUTE)";
+    } else {
+      // PPLNS needs archived shares for later rounds, so we have to copy them all
+      $sql = "
+        INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty)
+          SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty
+          FROM $this->table
+          WHERE id > ? AND id <= ?";
+    }
+    $archive_stmt = $this->mysqli->prepare($sql);
     if ($this->checkStmt($archive_stmt) && $archive_stmt->bind_param('iii', $block_id, $previous_upstream, $current_upstream) && $archive_stmt->execute())
       return true;
     return $this->sqlError();
@@ -214,11 +245,25 @@ class Share Extends Base {
    * @return bool true or false
    **/
   public function deleteAccountedShares($current_upstream, $previous_upstream=0) {
-    $stmt = $this->mysqli->prepare("DELETE FROM $this->table WHERE id > ? AND id <= ?");
+    // Fallbacks if unset
-    if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $previous_upstream, $current_upstream) && $stmt->execute())
+    if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 25000;
-      return true;
+    if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 1;
-    return $this->sqlError();
+
+    $affected = 1;
+    while ($affected > 0) {
+      // Sleep first to allow any IO to cleanup
+      sleep($this->config['purge']['sleep']);
+      $stmt = $this->mysqli->prepare("DELETE FROM $this->table WHERE id > ? AND id <= ? LIMIT " . $this->config['purge']['shares']);
+      $start = microtime(true);
+      if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $previous_upstream, $current_upstream) && $stmt->execute()) {
+        $affected = $stmt->affected_rows;
+      } else {
+        return $this->sqlError();
+      }
+    }
+    return true;
   }
+
   /**
    * Set/get last found share accepted by upstream: id and accounts
    **/

public/include/config/global.inc.dist.php

@@ -215,6 +215,35 @@ $config['block_bonus'] = 0;
 **/
 $config['payout_system'] = 'prop';
 
+/**
+ * Round purging
+ *
+ * Explanation:
+ *   As soon as a round is finished, shares of that rate are archived (see below)
+ *   and deleted from the `shares` table. Due to a large amount of shares in a
+ *   single round, this can take a very long time. To reduce server load and allow
+ *   other systems to access the DB during this high-load time, the DELETE
+ *   calls are being limited to a number of rows. Then the process sleeps and
+ *   continues to delete shares until all shares have been purged.
+ *
+ *   You can adjust some purging settings here in order to improve your overall
+ *   site performance during round ends. Keep in mind that drecreasing shares/time
+ *   will make the cron run longer but at least keeps your site active. Vice versa
+ *   higher numbers allow for a faster deletion but might affect the live site.
+ *
+ *   This system is also used when purging archived shares.
+ *
+ * Available Options:
+ *   sleep   :  Time to sleep between delete calls
+ *   shares  :  How many shares to delete at one time
+ *
+ * Default:
+ *   sleep   :  5 seconds
+ *   shares  :  500000
+ **/
+$config['purge']['sleep'] = 1;
+$config['purge']['shares'] = 25000;
+
 /**
  * Archiving configuration for debugging
  *