From 9fe598d774c122849a6e8570a78fa82335d888cd Mon Sep 17 00:00:00 2001 From: milux Date: Wed, 12 Aug 2020 21:23:47 +0200 Subject: [PATCH] Increased graph data processing speed (#944) --- composer.json | 1 + src/psm/Util/Server/HistoryGraph.php | 166 ++++++++++-------- .../default/module/server/history.tpl.html | 6 +- 3 files changed, 95 insertions(+), 78 deletions(-) diff --git a/composer.json b/composer.json index 88c4658d..0498c033 100644 --- a/composer.json +++ b/composer.json @@ -6,6 +6,7 @@ "require": { "php": "^5.5.9|>=7.0.8", "ext-curl": "*", + "ext-json": "*", "ext-pdo": "*", "ext-xml": "*", "phpmailer/phpmailer": ">=6.0.6 ~6.0", diff --git a/src/psm/Util/Server/HistoryGraph.php b/src/psm/Util/Server/HistoryGraph.php index 3990f993..11425e8b 100644 --- a/src/psm/Util/Server/HistoryGraph.php +++ b/src/psm/Util/Server/HistoryGraph.php @@ -29,7 +29,10 @@ namespace psm\Util\Server; +use DateTime; use psm\Service\Database; +use Twig\Error\Error; +use Twig_Environment; /** * History util, create HTML for server graphs @@ -39,17 +42,17 @@ class HistoryGraph /** * Database service - * @var \psm\Service\Database $db; + * @var Database $db; */ protected $db; /** * Twig environment - * @var \Twig_Environment $twig + * @var Twig_Environment $twig */ protected $twig; - public function __construct(Database $db, \Twig_Environment $twig) + public function __construct(Database $db, Twig_Environment $twig) { $this->db = $db; $this->twig = $twig; @@ -57,7 +60,9 @@ class HistoryGraph /** * Prepare the HTML for the graph - * @return string + * @param string $server_id ID of server to fetch data for + * @return string Created HTML + * @throws Error On twig error */ public function createHTML($server_id) { @@ -65,9 +70,9 @@ class HistoryGraph $archive = new ArchiveManager($this->db); $archive->archive($server_id); - $now = new \DateTime(); - $last_week = new \DateTime('-1 week 0:0:0'); - $last_year = new \DateTime('-1 year -1 week 0:0:0'); + $now = new DateTime(); + $last_week = new DateTime('-1 week 0:0:0'); + $last_year = new DateTime('-1 year -1 week 0:0:0'); $graphs = array( 0 => $this->generateGraphUptime($server_id, $last_week, $now), @@ -101,8 +106,8 @@ class HistoryGraph /** * Generate data for uptime graph * @param int $server_id - * @param \DateTime $start_time Lowest DateTime of the graph - * @param \DateTime $end_time Highest DateTime of the graph + * @param DateTime $start_time Lowest DateTime of the graph + * @param DateTime $end_time Highest DateTime of the graph * @return array */ public function generateGraphUptime($server_id, $start_time, $end_time) @@ -112,9 +117,9 @@ class HistoryGraph 'latency' => array(), ); - $hour = new \DateTime('-1 hour'); - $day = new \DateTime('-1 day'); - $week = new \DateTime('-1 week'); + $hour = new DateTime('-1 hour'); + $day = new DateTime('-1 day'); + $week = new DateTime('-1 week'); $records = $this->getRecords('uptime', $server_id, $start_time, $end_time); @@ -148,8 +153,8 @@ class HistoryGraph /** * Generate data for history graph * @param int $server_id - * @param \DateTime $start_time Lowest DateTime of the graph - * @param \DateTime $end_time Highest DateTime of the graph + * @param DateTime $start_time Lowest DateTime of the graph + * @param DateTime $end_time Highest DateTime of the graph * @return array */ public function generateGraphHistory($server_id, $start_time, $end_time) @@ -160,9 +165,9 @@ class HistoryGraph 'latency_max' => array(), ); - $week = new \DateTime('-2 week 0:0:0'); - $month = new \DateTime('-1 month -1 week 0:0:0'); - $year = new \DateTime('-1 year -1 week 0:0:0'); + $week = new DateTime('-2 week 0:0:0'); + $month = new DateTime('-1 month -1 week 0:0:0'); + $year = new DateTime('-1 year -1 week 0:0:0'); $records = $this->getRecords('history', $server_id, $year, $end_time); @@ -197,8 +202,8 @@ class HistoryGraph * Get all uptime/history records for a server * @param string $type * @param int $server_id - * @param \DateTime $start_time Lowest DateTime of the graph - * @param \DateTime $end_time Highest DateTime of the graph + * @param DateTime $start_time Lowest DateTime of the graph + * @param DateTime $end_time Highest DateTime of the graph * @return array */ protected function getRecords($type, $server_id, $start_time, $end_time) @@ -207,17 +212,19 @@ class HistoryGraph return array(); } - $records = $this->db->execute( - "SELECT * + /** @noinspection SqlNoDataSourceInspection */ + /** @noinspection SqlResolve */ + /** @noinspection PhpUndefinedConstantInspection */ + return $this->db->execute( + "SELECT *, UNIX_TIMESTAMP(CONVERT_TZ(`date`, '+00:00', @@session.time_zone)) AS date_ts FROM `" . PSM_DB_PREFIX . "servers_$type` - WHERE `server_id` = :server_id AND `date` BETWEEN :start_time AND :end_time ORDER BY `date` ASC", + WHERE `server_id` = :server_id AND `date` BETWEEN :start_time AND :end_time ORDER BY `date`", array( 'server_id' => $server_id, 'start_time' => $start_time->format('Y-m-d H:i:s'), 'end_time' => $end_time->format('Y-m-d H:i:s'), ) ); - return $records; } /** @@ -225,12 +232,9 @@ class HistoryGraph * @param array $records All uptime records to parse, MUST BE SORTED BY DATE IN ASCENDING ORDER * @param array $lines Array with keys as line ids to prepare (key must be available in uptime records) * @param string $latency_avg_key which key from uptime records to use for calculating averages - * @param \DateTime $start_time Lowest DateTime of the graph - * @param \DateTime $end_time Highest DateTime of the graph + * @param DateTime $start_time Lowest DateTime of the graph + * @param DateTime $end_time Highest DateTime of the graph * @param boolean $add_uptime Add uptime calculation? - * @param array $prev Previous result - * @param int $downtime Total downtime - * @param int $prev_downtime Timestamp from last offline record. 0 when last record is uptime * @return array */ protected function generateGraphLines( @@ -241,79 +245,90 @@ class HistoryGraph $end_time, $add_uptime = false ) { - $now = new \DateTime(); + $now = new DateTime(); $data = array(); // PLEASE NOTE: all times are in microseconds! because of javascript. $latency_avg = 0; + /** @var array $prev Previous record */ $prev = reset($records); + // Timestamp from last offline record. 0 when last record is up. $prev_downtime = 0; + // Total downtime $downtime = 0; + // The keys of the lines iterated + $line_keys = array_keys($lines); + // Determine whether to process data for the short history graph + $is_short_graph = count($line_keys) === 1 && $line_keys[0] === 'latency'; + // get highest latency record for offline height - $n = count($records); - $highest_latency = 0.0000; - for ($i = 0; $i < $n; $i++) { - if (!array_key_exists('latency', $records[$i])) { - continue; + $highest_latency = 0.0; + if ($is_short_graph) { + foreach ($records as $record) { + $latency = (float) $record['latency']; + if ($latency > $highest_latency) { + $highest_latency = $latency; + } } - // Update if latency is higher - $highest_latency = $highest_latency < floatval($records[$i]['latency']) ? - floatval($records[$i]['latency']) : $highest_latency; + // to ms + $highest_latency = round($highest_latency * 1000); } - // to ms - $highest_latency = round($highest_latency * 1000); // Create the list of points and server down zones foreach ($records as $record) { - $time = strtotime($record['date']); // use the first line to calculate average latency $latency_avg += (float) $record[$latency_avg_key]; - foreach ($lines as $key => $value) { - // add the value for each of the different lines - if (isset($record[$key])) { - if (isset($record['status'])) { - // down - if ($record['status'] == 0) { - $lines['online'][] = $prev['status'] - // Previous datapoint was online - ? '{ x: ' . ($time * 1000) . ', y: ' . $prev['latency'] * 1000 . '}' - // Previous datapoint was offline - : '{ x: ' . ($time * 1000) . ', y: null}'; - // new outage start - $lines['offline'][] = '{ x: ' . ($time * 1000) . ', y:' . $highest_latency . '}'; + if ($is_short_graph) { + $time = (int) $record['date_ts']; + // Timestamp in milliseconds + $time_ms = $time * 1000; + if (!$record['status']) { + // down + $lines['online'][] = $prev['status'] + // Previous datapoint was online + ? ['x' => $time_ms, 'y' => round($prev['latency'] * 1000, 3)] + // Previous datapoint was offline + : ['x' => $time_ms, 'y' => null]; + // new outage start + $lines['offline'][] = ['x' => $time_ms, 'y' => $highest_latency]; - $prev_downtime != 0 ?: $prev_downtime = $time; - } else { - // up - // outage ends - $lines['offline'][] = $prev['status'] - // Previous datapoint was online - ? '{ x: ' . ($time * 1000) . ', y:null}' - // Previous datapoint was offline - : '{ x: ' . ($time * 1000) . ', y:' . $highest_latency . '}'; - $lines['online'][] = '{ x: ' . ($time * 1000) . ', y: ' . - round((float) $record[$key] * 1000, 2) . '}'; - - $prev_downtime == 0 ?: $downtime += ($time - $prev_downtime); - $prev_downtime = 0; - } - } else { - $lines[$key][] = '{ x: \'' . $record['date'] . '\', y: ' . $record[$key] * 1000 . '}'; + if ($prev_downtime === 0) { + $prev_downtime = $time; } - $prev = $record; + } else { + // up + // outage ends + $lines['offline'][] = $prev['status'] + // Previous datapoint was online + ? ['x' => $time_ms, 'y' => null] + // Previous datapoint was offline + : ['x' => $time_ms, 'y' => $highest_latency]; + $lines['online'][] = ['x' => $time_ms, 'y' => round($record['latency'] * 1000, 3)]; + + if ($prev_downtime !== 0) { + $downtime += ($time - $prev_downtime); + } + $prev_downtime = 0; + } + } else { + foreach ($line_keys as $key) { + // add the value for each of the different lines + $lines[$key][] = ['x' => $record['date'], 'y' => $record[$key] * 1000]; } } + $prev = $record; } // Was down before. // Record the first and last date as a string in the down array $prev_downtime == 0 ?: $downtime += ($now->getTimestamp() - $prev_downtime); if ($add_uptime) { - $prev['status'] ?: $lines['offline'][] = - '{ x: ' . ($now->getTimestamp() * 1000) . ', y:' . $highest_latency . '}'; + if (!$prev['status']) { + $lines['offline'][] = ['x' => $now->getTimestamp() * 1000, 'y' => $highest_latency]; + } $data['uptime'] = 100 - ($downtime / ($end_time->getTimestamp() - $start_time->getTimestamp())); } @@ -322,11 +337,12 @@ class HistoryGraph if (empty($line_value)) { continue; } - $lines_merged[$line_key]['value'] = implode(', ', $line_value); + $lines_merged[$line_key]['value'] = json_encode($line_value); $lines_merged[$line_key]['name'] = psm_get_lang('servers', $line_key); } - $data['latency_avg'] = count($records) > 0 ? ($latency_avg / count($records)) : 0; + $n_records = count($records); + $data['latency_avg'] = $n_records > 0 ? ($latency_avg / $n_records) : 0; $data['lines'] = sizeof($lines_merged) ? $lines_merged : ''; $data['end_timestamp'] = number_format($end_time->getTimestamp(), 0, '', '') * 1000; $data['start_timestamp'] = number_format($start_time->getTimestamp(), 0, '', '') * 1000; diff --git a/src/templates/default/module/server/history.tpl.html b/src/templates/default/module/server/history.tpl.html index 163c01d9..7c08a2b3 100644 --- a/src/templates/default/module/server/history.tpl.html +++ b/src/templates/default/module/server/history.tpl.html @@ -32,7 +32,7 @@ data: { datasets: [ { - data: [{{ graph.lines.offline.value }}], + data: {{ graph.lines.offline.value|raw }}, label: '{{ graph.lines.offline.name }}', backgroundColor: '#dc3545', borderColor: '#dc3545', @@ -43,7 +43,7 @@ spanGaps: false, }, { - data: [{{ graph.lines.online.value }}], + data: {{ graph.lines.online.value|raw }}, label: '{{graph.lines.online.name }}', fill: false, spanGaps: false, @@ -120,7 +120,7 @@ datasets: [ {% for key,line in graph.lines %} { - data: [{{ line.value|raw }}], + data: {{ line.value|raw }}, label: '{{ line.name }}', backgroundColor: colors['{{key}}'], borderColor: colors['{{key}}'],