From cf49db4535cc5875da97ecb6145d5bafb78fd457 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:05:27 +0100 Subject: [PATCH 1/7] [IMPROVED] Cronbased global Hash-/Sharerate cache * [ADDED] New statistic method to fetch all user mining stats * [ADDED] New global cache to getUserHash/Sharerate calls * [ADDED] New memcache key for new global cache Addresses #1471 and may fix it already if no other changes are required. --- cronjobs/statistics.php | 8 ++- public/include/classes/statistics.class.php | 68 +++++++++++++++++++++ public/include/config/memcache_keys.inc.php | 1 + 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/cronjobs/statistics.php b/cronjobs/statistics.php index 9228095b..20746c6e 100755 --- a/cronjobs/statistics.php +++ b/cronjobs/statistics.php @@ -27,10 +27,16 @@ require_once('shared.inc.php'); // Per user share statistics based on all shares submitted $start = microtime(true); -if ( ! $aAllUserShares = $statistics->getAllUserShares() ) +if ( ! $statistics->getAllUserShares() ) $log->logError('getAllUserShares update failed'); $log->logInfo("getAllUserShares " . number_format(microtime(true) - $start, 2) . " seconds"); +// Get all user hashrate statistics for caching +$start = microtime(true); +if ( ! $statistics->getAllUserMiningStats() ) + $log->logError('getAllUserMiningStats update failed'); +$log->logInfo("getAllUserMiningStats " . number_format(microtime(true) - $start, 2) . " seconds"); + $start = microtime(true); if (!$statistics->getTopContributors('hashes')) $log->logError("getTopContributors hashes update failed"); diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index 4b369931..1dac7361 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -448,6 +448,53 @@ class Statistics extends Base { return $this->sqlError(); } + /** + * Fetch all user hashrates based on shares and archived shares + * @return data integer Current Hashrate in khash/s + **/ + public function getAllUserMiningStats($interval=600) { + $this->debug->append("STA " . __METHOD__, 4); + $stmt = $this->mysqli->prepare(" + SELECT + a.id AS id, + a.username AS account, + IFNULL(ROUND(SUM(t1.difficulty) * POW(2, 16) / ? / 1000, 2), 0) AS hashrate, + ROUND(COUNT(t1.id) / ?, 2) AS sharerate, + IFNULL(AVG(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS avgsharediff + FROM ( + SELECT + id, + IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + username + FROM shares + WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' + UNION + SELECT + share_id, + IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + username + FROM shares_archive + WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' + ) AS t1 + LEFT JOIN " . $this->user->getTableName() . " AS a + ON SUBSTRING_INDEX( t1.username, '.', 1 ) = a.username + WHERE a.id IS NOT NULL + GROUP BY account + ORDER BY hashrate DESC + "); + if ($this->checkStmt($stmt) && $stmt->bind_param("iiii", $interval, $interval, $interval, $interval) && $stmt->execute() && $result = $stmt->get_result() ) { + $aData = array(); + while ($row = $result->fetch_assoc()) { + $aData['data'][$row['id']] = $row; + } + var_dump($aData); + return $this->memcache->setCache(STATISTICS_ALL_USER_HASHRATES, $aData); + } else { + echo $this->mysqli->error; + return $this->sqlError(); + } + } + /** * Fetch total user hashrate based on shares and archived shares * @param account_id integer User ID @@ -455,6 +502,13 @@ class Statistics extends Base { **/ public function getUserHashrate($account_id, $interval=600) { $this->debug->append("STA " . __METHOD__, 4); + // Dual-caching, try statistics cron first, then fallback to local, then fallbock to SQL + if ($data = $this->memcache->get(STATISTICS_ALL_USER_HASHRATES)) { + if (array_key_exists($account_id, $data['data'])) + return $data['data'][$account_id]['hashrate']; + // We have no cached value, we return defaults + return 0; + } if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT @@ -510,6 +564,13 @@ class Statistics extends Base { **/ public function getUserShareDifficulty($account_id, $interval=600) { $this->debug->append("STA " . __METHOD__, 4); + // Dual-caching, try statistics cron first, then fallback to local, then fallbock to SQL + if ($data = $this->memcache->get(STATISTICS_ALL_USER_HASHRATES)) { + if (array_key_exists($account_id, $data['data'])) + return $data['data'][$account_id]['avgsharediff']; + // We have no cached value, we return defaults + return 0; + } if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT @@ -532,6 +593,13 @@ class Statistics extends Base { **/ public function getUserSharerate($account_id, $interval=600) { $this->debug->append("STA " . __METHOD__, 4); + // Dual-caching, try statistics cron first, then fallback to local, then fallbock to SQL + if ($data = $this->memcache->get(STATISTICS_ALL_USER_HASHRATES)) { + if (array_key_exists($account_id, $data['data'])) + return $data['data'][$account_id]['sharerate']; + // We have no cached value, we return defaults + return 0; + } if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data; $stmt = $this->mysqli->prepare(" SELECT diff --git a/public/include/config/memcache_keys.inc.php b/public/include/config/memcache_keys.inc.php index f689a868..f9405b0c 100644 --- a/public/include/config/memcache_keys.inc.php +++ b/public/include/config/memcache_keys.inc.php @@ -4,5 +4,6 @@ if (!defined('SECURITY')) die('Hacking attempt'); define('STATISTICS_ALL_USER_SHARES', 'STATISTICS_ALL_USER_SHARES'); +define('STATISTICS_ALL_USER_HASHRATES', 'STATISTICS_ALL_USER_HASHRATES'); define('STATISTICS_ROUND_SHARES', 'STATISTICS_ROUND_SHARES'); ?> From 0fb543c3ed066a90c79ada0d10dfad96515718de Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:17:24 +0100 Subject: [PATCH 2/7] [FIX] Honor target_bits for hashrate --- public/include/classes/statistics.class.php | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index 1dac7361..efefa949 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -458,21 +458,21 @@ class Statistics extends Base { SELECT a.id AS id, a.username AS account, - IFNULL(ROUND(SUM(t1.difficulty) * POW(2, 16) / ? / 1000, 2), 0) AS hashrate, + IFNULL(ROUND(SUM(t1.difficulty) * POW(2, " . $this->config['target_bits'] . ") / ? / 1000, 2), 0) AS hashrate, ROUND(COUNT(t1.id) / ?, 2) AS sharerate, IFNULL(AVG(IF(difficulty=0, pow(2, (" . $this->config['difficulty'] . " - 16)), difficulty)), 0) AS avgsharediff FROM ( SELECT - id, - IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, - username + id, + IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + username FROM shares WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' UNION SELECT - share_id, - IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, - username + share_id, + IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + username FROM shares_archive WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' ) AS t1 @@ -490,7 +490,6 @@ class Statistics extends Base { var_dump($aData); return $this->memcache->setCache(STATISTICS_ALL_USER_HASHRATES, $aData); } else { - echo $this->mysqli->error; return $this->sqlError(); } } From b905089a01267f966c560edb83a1ee7e5c6d1662 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:18:09 +0100 Subject: [PATCH 3/7] [FIX] Removed debug output --- public/include/classes/statistics.class.php | 1 - 1 file changed, 1 deletion(-) diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index efefa949..b5fce2bb 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -487,7 +487,6 @@ class Statistics extends Base { while ($row = $result->fetch_assoc()) { $aData['data'][$row['id']] = $row; } - var_dump($aData); return $this->memcache->setCache(STATISTICS_ALL_USER_HASHRATES, $aData); } else { return $this->sqlError(); From d4db477c2dcfc03b2c0d77167e58f5229318a645 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:22:00 +0100 Subject: [PATCH 4/7] [FIX] Also honor diff for share difficulties if unset --- public/include/classes/statistics.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index b5fce2bb..2e759959 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -464,14 +464,14 @@ class Statistics extends Base { FROM ( SELECT id, - IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty username FROM shares WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' UNION SELECT share_id, - IFNULL(IF(difficulty=0, pow(2, (20 - 16)), difficulty), 0) AS difficulty, + IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty username FROM shares_archive WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' From fbea3341212b18b9f4b547a9cfdc435691d8eff3 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:22:40 +0100 Subject: [PATCH 5/7] [REMOVED] Unused stats update --- cronjobs/statistics.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cronjobs/statistics.php b/cronjobs/statistics.php index 20746c6e..0149afc0 100755 --- a/cronjobs/statistics.php +++ b/cronjobs/statistics.php @@ -47,13 +47,5 @@ if (!$statistics->getCurrentHashrate()) $log->logError("getCurrentHashrate update failed"); $log->logInfo("getCurrentHashrate " . number_format(microtime(true) - $start, 2) . " seconds"); -/* -// Admin specific statistics, we cache the global query due to slowness -$start = microtime(true); -if (!$statistics->getAllUserStats('%')) - $log->logError("getAllUserStats update failed"); -$log->logInfo("getAllUserStats " . number_format(microtime(true) - $start, 2) . " seconds"); -*/ - require_once('cron_end.inc.php'); ?> From 8a983835c6eab919527c02921b4149854bf189bc Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:25:55 +0100 Subject: [PATCH 6/7] [FIX] Whoopsie SQL --- public/include/classes/statistics.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index 2e759959..8687d421 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -464,14 +464,14 @@ class Statistics extends Base { FROM ( SELECT id, - IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty + IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty, username FROM shares WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' UNION SELECT share_id, - IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty + IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty, username FROM shares_archive WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' From 5b7cf6ab93e8f281d9b9880ab6a48640d85fffd0 Mon Sep 17 00:00:00 2001 From: Sebastian Grewe Date: Sun, 19 Jan 2014 17:28:34 +0100 Subject: [PATCH 7/7] [FIX] SQL again, sigh --- public/include/classes/statistics.class.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/include/classes/statistics.class.php b/public/include/classes/statistics.class.php index 8687d421..327614d9 100644 --- a/public/include/classes/statistics.class.php +++ b/public/include/classes/statistics.class.php @@ -464,14 +464,14 @@ class Statistics extends Base { FROM ( SELECT id, - IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty, + IF(difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty, username FROM shares WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y' UNION SELECT share_id, - IF(s.difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), s.difficulty) AS difficulty, + IF(difficulty = 0, POW(2, (" . $this->config['difficulty'] . " - 16)), difficulty) AS difficulty, username FROM shares_archive WHERE time > DATE_SUB(now(), INTERVAL ? SECOND) AND our_result = 'Y'