RanchiMall/php-mpos
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IMPROVEMENT] Adding incremental round/user shares

Browse Source 
This will address #510. It needs thorough testing and is a WiP but
is a start to improve cron runtime and DB loads.
This commit is contained in:
Sebastian Grewe 2013-09-18 09:45:52 +02:00
parent 49b6ebe62f
commit a0fa71b264
6 changed files with 95 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cronjobs/statistics.php
View File

@ -22,26 +22,28 @@ limitations under the License.
// Include all settings and classes
require_once('shared.inc.php');
// Fetch all cachable values but disable fetching from cache
$statistics->setGetCache(false);
// Since fetching from cache is disabled, overwrite our stats
$start = microtime(true);
if (!$statistics->getRoundShares())
$log->logError("getRoundShares update failed");
$log->logInfo("getRoundShares update " . number_format(microtime(true) - $start, 2) . " seconds");
$start = microtime(true);
if (!$statistics->getTopContributors('shares'))
$log->logError("getTopContributors shares update failed");
$log->logInfo("getTopContributors shares " . number_format(microtime(true) - $start, 2) . " seconds");
$start = microtime(true);
if (!$statistics->getTopContributors('hashes'))
$log->logError("getTopContributors hashes update failed");
$log->logInfo("getTopContributors hashes " . number_format(microtime(true) - $start, 2) . " seconds");
$start = microtime(true);
if (!$statistics->getCurrentHashrate())
$log->logError("getCurrentHashrate update failed");
$log->logInfo("getCurrentHashrate " . number_format(microtime(true) - $start, 2) . " seconds");
// Per user share statistics based on all shares submitted
$start = microtime(true);
if ( ! $aAllUserShares = $statistics->getAllUserShares() )
$log->logError('getAllUserShares update failed');
$log->logInfo("getAllUserShares " . number_format(microtime(true) - $start, 2) . " seconds");
/*
// Admin specific statistics, we cache the global query due to slowness
$start = microtime(true);
@ -50,13 +52,5 @@ if (!$statistics->getAllUserStats('%'))
$log->logInfo("getAllUserStats " . number_format(microtime(true) - $start, 2) . " seconds");
*/
// Per user share statistics based on all shares submitted
$start = microtime(true);
$aUserShares = $statistics->getAllUserShares();
$log->logInfo("getAllUserShares " . number_format(microtime(true) - $start, 2) . " seconds");
foreach ($aUserShares as $aShares) {
$memcache->setCache('getUserShares'. $aShares['id'], $aShares);
}
require_once('cron_end.inc.php');
?>

10
public/include/autoloader.inc.php
View File

@ -2,11 +2,9 @@
// Default classes
require_once(CLASS_DIR . '/debug.class.php');
require_once(CLASS_DIR . '/bitcoin.class.php');
require_once(CLASS_DIR . '/statscache.class.php');
require_once(CLASS_DIR . '/bitcoinwrapper.class.php');
require_once(INCLUDE_DIR . '/lib/KLogger.php');
require_once(INCLUDE_DIR . '/database.inc.php');
require_once(INCLUDE_DIR . '/config/memcache_keys.inc.php');
// We need to load these two first
require_once(CLASS_DIR . '/base.class.php');
@ -35,6 +33,12 @@ require_once(CLASS_DIR . '/tokentype.class.php');
require_once(CLASS_DIR . '/token.class.php');
require_once(CLASS_DIR . '/payout.class.php');
require_once(CLASS_DIR . '/block.class.php');
// We require the block class to properly grab the round ID
require_once(CLASS_DIR . '/statscache.class.php');
require_once(CLASS_DIR . '/bitcoin.class.php');
require_once(CLASS_DIR . '/bitcoinwrapper.class.php');
require_once(CLASS_DIR . '/monitoring.class.php');
require_once(CLASS_DIR . '/user.class.php');
require_once(CLASS_DIR . '/invitation.class.php');

10
public/include/classes/share.class.php
View File

@ -111,6 +111,16 @@ class Share {
return false;
}
/**
* Fetch the highest available share ID
**/
function getMaxShareId() {
$stmt = $this->mysqli->prepare("SELECT MAX(id) AS id FROM $this->table");
if ($this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result())
return $result->fetch_object()->id;
return false;
}
/**
* Fetch the highest available share ID from archive
**/

52
public/include/classes/statistics.class.php
View File

@ -146,7 +146,20 @@ class Statistics {
**/
public function getRoundShares() {
$this->debug->append("STA " . __METHOD__, 4);
if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data;
// Try the statistics cron cache, then function cache, then fallback to SQL
if ($data = $this->memcache->get(STATISTICS_ALL_USER_SHARES)) {
$this->debug->append("Found data in statistics cache", 2);
$total = array('valid' => 0, 'invalid' => 0);
foreach ($data['data'] as $aUser) {
$total['valid'] += $aUser['valid'];
$total['invalid'] += $aUser['invalid'];
}
return $total;
}
if ($data = $this->memcache->get(STATISTICS_ROUND_SHARES)) {
$this->debug->append("Found data in local cache", 2);
return $data;
}
$stmt = $this->mysqli->prepare("
SELECT
IFNULL(SUM(IF(our_result='Y', 1, 0)), 0) AS valid,
@ -154,7 +167,7 @@ class Statistics {
FROM " . $this->share->getTableName() . "
WHERE UNIX_TIMESTAMP(time) >IFNULL((SELECT MAX(time) FROM " . $this->block->getTableName() . "),0)");
if ( $this->checkStmt($stmt) && $stmt->execute() && $result = $stmt->get_result() )
return $this->memcache->setCache(__FUNCTION__, $result->fetch_assoc());
return $this->memcache->setCache(STATISTICS_ROUND_SHARES, $result->fetch_assoc());
// Catchall
$this->debug->append("Failed to fetch round shares: " . $this->mysqli->error);
return false;
@ -168,7 +181,10 @@ class Statistics {
**/
public function getAllUserShares() {
$this->debug->append("STA " . __METHOD__, 4);
if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__)) return $data;
if (! $data = $this->memcache->get(STATISTICS_ALL_USER_SHARES)) {
$data['share_id'] = 0;
$data['data'] = array();
}
$stmt = $this->mysqli->prepare("
SELECT
IFNULL(SUM(IF(our_result='Y', 1, 0)), 0) AS valid,
@ -178,10 +194,26 @@ class Statistics {
FROM " . $this->share->getTableName() . " AS s,
" . $this->user->getTableName() . " AS u
WHERE u.username = SUBSTRING_INDEX( s.username, '.', 1 )
AND UNIX_TIMESTAMP(s.time) >IFNULL((SELECT MAX(b.time) FROM " . $this->block->getTableName() . " AS b),0)
GROUP BY u.id");
if ($stmt && $stmt->execute() && $result = $stmt->get_result())
return $this->memcache->setCache(__FUNCTION__, $result->fetch_all(MYSQLI_ASSOC));
AND UNIX_TIMESTAMP(s.time) > IFNULL(
(
SELECT MAX(b.time)
FROM " . $this->block->getTableName() . " AS b
) ,0 )
AND s.id > ?
GROUP BY u.id");
if ($stmt && $stmt->bind_param('i', $data['share_id']) && $stmt->execute() && $result = $stmt->get_result()) {
$data_new = array();
while ($row = $result->fetch_assoc()) {
if (! array_key_exists($row['id'], $data['data'])) {
$data['data'][$row['id']] = $row;
} else {
$data['data'][$row['id']]['valid'] += $row['valid'];
$data['data'][$row['id']]['invalid'] += $row['invalid'];
}
}
$data['share_id'] = $this->share->getMaxShareId();
return $this->memcache->setCache(STATISTICS_ALL_USER_SHARES, $data);
}
// Catchall
$this->debug->append("Unable to fetch all users round shares: " . $this->mysqli->error);
return false;
@ -194,7 +226,9 @@ class Statistics {
**/
public function getUserShares($account_id) {
$this->debug->append("STA " . __METHOD__, 4);
if ($this->getGetCache() && $data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data;
// Dual-caching, try statistics cron first, then fallback to local, then fallbock to SQL
if ($data = $this->memcache->get(STATISTICS_ALL_USER_SHARES)) return $data['data'][$account_id];
if ($data = $this->memcache->get(__FUNCTION__ . $account_id)) return $data;
$stmt = $this->mysqli->prepare("
SELECT
IFNULL(SUM(IF(our_result='Y', 1, 0)), 0) AS valid,
@ -343,7 +377,7 @@ class Statistics {
ORDER BY shares DESC
LIMIT ?");
if ($this->checkStmt($stmt) && $stmt->bind_param("i", $limit) && $stmt->execute() && $result = $stmt->get_result())
return $this->memcache->setCache(__FUNCTION__ . $type . $limit, $result->fetch_all(MYSQLI_ASSOC));
$this->memcache->set(__FUNCTION__ . $type . $limit, $result->fetch_all(MYSQLI_ASSOC));
$this->debug->append("Fetching shares failed: ");
return false;
break;

28
public/include/classes/statscache.class.php
View File

@ -10,8 +10,8 @@ if (!defined('SECURITY'))
* Also sets a default time if no time is passed to it to enforce caching
**/
class StatsCache {
private $cache;
private $cache, $round;
public function __construct($config, $debug) {
$this->config = $config;
$this->debug = $debug;
@ -22,6 +22,13 @@ class StatsCache {
}
}
public function setRound($round_id) {
$this->round = $round_id;
}
public function getRound() {
return $this->round;
}
/**
* Wrapper around memcache->set
* Do not store values if memcache is disabled
@ -30,8 +37,8 @@ class StatsCache {
if (! $this->config['memcache']['enabled']) return false;
if (empty($expiration))
$expiration = $this->config['memcache']['expiration'] + rand( -$this->config['memcache']['splay'], $this->config['memcache']['splay']);
$this->debug->append("Storing " . $this->config['memcache']['keyprefix'] . "$key with expiration $expiration", 3);
return $this->cache->set($this->config['memcache']['keyprefix'] . $key, $value, $expiration);
$this->debug->append("Storing " . $this->getRound() . '_' . $this->config['memcache']['keyprefix'] . "$key with expiration $expiration", 3);
return $this->cache->set($this->getRound() . '_' . $this->config['memcache']['keyprefix'] . $key, $value, $expiration);
}
/**
@ -40,8 +47,8 @@ class StatsCache {
**/
public function get($key, $cache_cb = NULL, &$cas_token = NULL) {
if (! $this->config['memcache']['enabled']) return false;
$this->debug->append("Trying to fetch key " . $this->config['memcache']['keyprefix'] . "$key from cache", 3);
if ($data = $this->cache->get($this->config['memcache']['keyprefix'].$key)) {
$this->debug->append("Trying to fetch key " . $this->getRound() . '_' . $this->config['memcache']['keyprefix'] . "$key from cache", 3);
if ($data = $this->cache->get($this->getRound() . '_' . $this->config['memcache']['keyprefix'].$key)) {
$this->debug->append("Found key in cache", 3);
return $data;
} else {
@ -60,7 +67,7 @@ class StatsCache {
if ($this->config['memcache']['enabled']) $this->set($key, $data, $expiration);
return $data;
}
/**
* This method is invoked if the called method was not realised in this class
**/
@ -73,3 +80,10 @@ class StatsCache {
$memcache = new StatsCache($config, $debug);
$memcache->addServer($config['memcache']['host'], $config['memcache']['port']);
// Now we can set our additional key prefix
if ($aTmpBlock = $block->getLast()) {
$iRoundId = $aTmpBlock['id'];
} else {
$iRoundId = 0;
}
$memcache->setRound($iRoundId);

5
public/include/config/memcache_keys.inc.php Normal file
View File

@ -0,0 +1,5 @@
<?php
define('STATISTICS_ALL_USER_SHARES', 'STATISTICS_ALL_USER_SHARES');
define('STATISTICS_ROUND_SHARES', 'STATISTICS_ROUND_SHARES');
?>