From ab2326a557cfd10d59970c2879ca6b6c01d93549 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Tue, 3 Dec 2013 08:09:08 +0100 Subject: [PATCH 1/2] [IMPROVED] Use limits for DELETE statements This will address #886. Long rounds can cause a system to become very unresponsive due to high SQL/IO load when doing cleanups of shares and archived tables. * Run DELETE from shares with LIMIT * Run DELETE from shares_archive with LIMIT * Configure DELETE behaviour via config file * Only archive shares that are really required (PROP, PPS) Should greatly improve round ends on PROP and PPS after large rounds, also improves PPLNS though archving will still take some time unless we limit the share amount artificially. Shares could be needed though, so we don't. --- public/include/classes/share.class.php | 78 ++++++++++++++++++----- public/include/config/global.inc.dist.php | 29 +++++++++ 2 files changed, 90 insertions(+), 17 deletions(-) diff --git a/public/include/classes/share.class.php b/public/include/classes/share.class.php index 30ab344c..6ee1c30c 100644 --- a/public/include/classes/share.class.php +++ b/public/include/classes/share.class.php @@ -171,22 +171,41 @@ class Share Extends Base { * @return return bool true or false **/ public function purgeArchive() { + // Fallbacks if unset + if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 500000; + if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 5; + if ($this->config['payout_system'] == 'pplns') { // Fetch our last block so we can go back configured rounds $aLastBlock = $this->block->getLast(); // Fetch the block we need to find the share_id $aBlock = $this->block->getBlock($aLastBlock['height'] - $this->config['archive']['maxrounds']); // Now that we know our block, remove those shares - $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE block_id < ? AND time < DATE_SUB(now(), INTERVAL ? MINUTE)"); - if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $aBlock['id'], $this->config['archive']['maxage']) && $stmt->execute()) - return true; + $affected = 1; + while ($affected > 0) { + // Sleep first to allow any IO to cleanup + sleep($this->config['purge']['sleep']); + $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE block_id < ? AND time < DATE_SUB(now(), INTERVAL ? MINUTE) LIMIT " . $this->config['purge']['shares']); + if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $aBlock['id'], $this->config['archive']['maxage']) && $stmt->execute()) { + $affected = $stmt->affected_rows; + } else { + return $this->sqlError(); + } + } } else { - // We are not running pplns, so we just need to keep shares of the past minutes - $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE time < DATE_SUB(now(), INTERVAL ? MINUTE)"); - if ($this->checkStmt($stmt) && $stmt->bind_param('i', $this->config['archive']['maxage']) && $stmt->execute()) - return true; + $affected = 1; + while ($affected > 0) { + // Sleep first to allow any IO to cleanup + sleep($this->config['purge']['sleep']); + $stmt = $this->mysqli->prepare("DELETE FROM $this->tableArchive WHERE time < DATE_SUB(now(), INTERVAL ? MINUTE) LIMIT " . $this->config['purge']['shares']); + if ($this->checkStmt($stmt) && $stmt->bind_param('i', $this->config['archive']['maxage']) && $stmt->execute()) { + $affected = $stmt->affected_rows; + } else { + return $this->sqlError(); + } + } } - return $this->sqlError(); + return true; } /** @@ -197,11 +216,23 @@ class Share Extends Base { * @return bool **/ public function moveArchive($current_upstream, $block_id, $previous_upstream=0) { - $archive_stmt = $this->mysqli->prepare(" - INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty) - SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty - FROM $this->table - WHERE id > ? AND id <= ?"); + if ($this->config['payout_system'] != 'pplns') { + // We don't need archived shares that much, so only archive as much as configured + $sql = " + INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty) + SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty + FROM $this->table + WHERE id > ? AND id <= ? + AND time >= DATE_SUB(now(), INTERVAL " . $this->config['archive']['maxage'] . " MINUTE)"; + } else { + // PPLNS needs archived shares for later rounds, so we have to copy them all + $sql = " + INSERT INTO $this->tableArchive (share_id, username, our_result, upstream_result, block_id, time, difficulty) + SELECT id, username, our_result, upstream_result, ?, time, IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty + FROM $this->table + WHERE id > ? AND id <= ?"; + } + $archive_stmt = $this->mysqli->prepare($sql); if ($this->checkStmt($archive_stmt) && $archive_stmt->bind_param('iii', $block_id, $previous_upstream, $current_upstream) && $archive_stmt->execute()) return true; return $this->sqlError(); @@ -214,11 +245,24 @@ class Share Extends Base { * @return bool true or false **/ public function deleteAccountedShares($current_upstream, $previous_upstream=0) { - $stmt = $this->mysqli->prepare("DELETE FROM $this->table WHERE id > ? AND id <= ?"); - if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $previous_upstream, $current_upstream) && $stmt->execute()) - return true; - return $this->sqlError(); + // Fallbacks if unset + if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 500000; + if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 5; + + $affected = 1; + while ($affected > 0) { + // Sleep first to allow any IO to cleanup + sleep($this->config['purge']['sleep']); + $stmt = $this->mysqli->prepare("DELETE FROM $this->table WHERE id > ? AND id <= ? LIMIT " . $this->config['purge']['shares']); + if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $previous_upstream, $current_upstream) && $stmt->execute()) { + $affected = $stmt->affected_rows; + } else { + return $this->sqlError(); + } + } + return true; } + /** * Set/get last found share accepted by upstream: id and accounts **/ diff --git a/public/include/config/global.inc.dist.php b/public/include/config/global.inc.dist.php index 2d1ba2a6..73028fc9 100644 --- a/public/include/config/global.inc.dist.php +++ b/public/include/config/global.inc.dist.php @@ -215,6 +215,35 @@ $config['block_bonus'] = 0; **/ $config['payout_system'] = 'prop'; +/** + * Round purging + * + * Explanation: + * As soon as a round is finished, shares of that rate are archived (see below) + * and deleted from the `shares` table. Due to a large amount of shares in a + * single round, this can take a very long time. To reduce server load and allow + * other systems to access the DB during this high-load time, the DELETE + * calls are being limited to a number of rows. Then the process sleeps and + * continues to delete shares until all shares have been purged. + * + * You can adjust some purging settings here in order to improve your overall + * site performance during round ends. Keep in mind that drecreasing shares/time + * will make the cron run longer but at least keeps your site active. Vice versa + * higher numbers allow for a faster deletion but might affect the live site. + * + * This system is also used when purging archived shares. + * + * Available Options: + * sleep : Time to sleep between delete calls + * shares : How many shares to delete at one time + * + * Default: + * sleep : 5 seconds + * shares : 500000 + **/ +$config['purge']['sleep'] = 5; +$config['purge']['shares'] = 500000; + /** * Archiving configuration for debugging * From 847c1283b6dced37895f80f352599e66b8727c2e Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Tue, 3 Dec 2013 16:21:59 +0100 Subject: [PATCH 2/2] [CHANGE] Adjusted purge defaults --- public/include/classes/share.class.php | 9 +++++---- public/include/config/global.inc.dist.php | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/public/include/classes/share.class.php b/public/include/classes/share.class.php index 6ee1c30c..5af52897 100644 --- a/public/include/classes/share.class.php +++ b/public/include/classes/share.class.php @@ -172,8 +172,8 @@ class Share Extends Base { **/ public function purgeArchive() { // Fallbacks if unset - if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 500000; - if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 5; + if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 25000; + if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 1; if ($this->config['payout_system'] == 'pplns') { // Fetch our last block so we can go back configured rounds @@ -246,14 +246,15 @@ class Share Extends Base { **/ public function deleteAccountedShares($current_upstream, $previous_upstream=0) { // Fallbacks if unset - if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 500000; - if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 5; + if (!isset($this->config['purge']['shares'])) $this->config['purge']['shares'] = 25000; + if (!isset($this->config['purge']['sleep'])) $this->config['purge']['sleep'] = 1; $affected = 1; while ($affected > 0) { // Sleep first to allow any IO to cleanup sleep($this->config['purge']['sleep']); $stmt = $this->mysqli->prepare("DELETE FROM $this->table WHERE id > ? AND id <= ? LIMIT " . $this->config['purge']['shares']); + $start = microtime(true); if ($this->checkStmt($stmt) && $stmt->bind_param('ii', $previous_upstream, $current_upstream) && $stmt->execute()) { $affected = $stmt->affected_rows; } else { diff --git a/public/include/config/global.inc.dist.php b/public/include/config/global.inc.dist.php index 73028fc9..780ec62d 100644 --- a/public/include/config/global.inc.dist.php +++ b/public/include/config/global.inc.dist.php @@ -241,8 +241,8 @@ $config['payout_system'] = 'prop'; * sleep : 5 seconds * shares : 500000 **/ -$config['purge']['sleep'] = 5; -$config['purge']['shares'] = 500000; +$config['purge']['sleep'] = 1; +$config['purge']['shares'] = 25000; /** * Archiving configuration for debugging