From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5924D7445B for ; Wed, 1 Jun 2022 10:13:12 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id C7A5B1A4C2 for ; Wed, 1 Jun 2022 10:13:11 +0200 (CEST) Received: from bastionodiso.odiso.net (bastionodiso.odiso.net [185.151.191.93]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 0BD101A0F6 for ; Wed, 1 Jun 2022 10:13:02 +0200 (CEST) Received: from kvmformation3.odiso.net (formationkvm3.odiso.net [10.3.94.12]) by bastionodiso.odiso.net (Postfix) with ESMTP id 58B3E34B03; Wed, 1 Jun 2022 10:12:55 +0200 (CEST) Received: by kvmformation3.odiso.net (Postfix, from userid 0) id 578F8176723; Wed, 1 Jun 2022 10:12:55 +0200 (CEST) From: Alexandre Derumier To: pve-devel@lists.proxmox.com Cc: t.lamprecht@proxmox.com, Alexandre Derumier Date: Wed, 1 Jun 2022 10:12:51 +0200 Message-Id: <20220601081253.2542697-9-aderumier@odiso.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220601081253.2542697-1-aderumier@odiso.com> References: <20220601081253.2542697-1-aderumier@odiso.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.121 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% HEADER_FROM_DIFFERENT_DOMAINS 0.249 From and EnvelopeFrom 2nd level mail domains are different KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.001 Envelope sender has no MX or A DNS records SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH pve-manager 3/4] pvestatd: qemu/lxc/node : add hostcpu/hostmem average stats X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 01 Jun 2022 08:13:12 -0000 we aggregate each last X second stats to 1min average we aggragate each last 5 1min average to 5min average. vm avgstats are resetted when vm is stopped or removed. Signed-off-by: Alexandre Derumier --- PVE/Service/pvestatd.pm | 94 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm index dd578d6b..a33b21cd 100755 --- a/PVE/Service/pvestatd.pm +++ b/PVE/Service/pvestatd.pm @@ -81,6 +81,7 @@ my $next_flag_update_time; my $failed_flag_update_delay_sec = 120; my $balancer_stats = {}; my $last_balancer_broadcast_time = 0; +my $avgstats = {}; sub update_supported_cpuflags { my $kvm_version = PVE::QemuServer::kvm_user_version(); @@ -177,6 +178,11 @@ sub update_node_status { $node_metric->{cpustat}->{cpus} = $maxcpu; compute_pressure($ctime, $node_metric, 'node', $nodename); + my $avg_metrics = { + cpu => $stat->{cpu}, + mem => $meminfo->{memused}, + }; + compute_avg_metrics($ctime, $avg_metrics, 'node', $nodename); my $transactions = PVE::ExtMetric::transactions_start($status_cfg); PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime); @@ -233,6 +239,11 @@ sub update_qemu_status { $d->{netin}, $d->{netout}, $d->{diskread}, $d->{diskwrite}]); compute_pressure($ctime, $d, 'qemu', $vmid); + my $avg_metrics = { + cpu => $d->{hostcpu}, + mem => $d->{hostmem}, + }; + compute_avg_metrics($ctime, $avg_metrics, 'qemu', $vmid); } else { $data = $generate_rrd_string->( [0, $d->{name}, $status, $template, $ctime, $d->{cpus}, undef, @@ -242,7 +253,7 @@ sub update_qemu_status { PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename); } - + delete_old_qemu_avgstats($vmstatus); PVE::ExtMetric::transactions_finish($transactions); } @@ -443,6 +454,11 @@ sub update_lxc_status { $d->{netin}, $d->{netout}, $d->{diskread}, $d->{diskwrite}]); compute_pressure($ctime, $d, 'lxc', $vmid); + my $avg_metrics = { + cpu => $d->{cpu}, + mem => $d->{mem}, + }; + compute_avg_metrics($ctime, $avg_metrics, 'lxc', $vmid); } else { $data = $generate_rrd_string->( [0, $d->{name}, $d->{status}, $template, $ctime, $d->{cpus}, undef, @@ -452,6 +468,7 @@ sub update_lxc_status { PVE::ExtMetric::update_all($transactions, 'lxc', $vmid, $d, $ctime, $nodename); } + delete_old_lxc_avgstats($vmstatus); PVE::ExtMetric::transactions_finish($transactions); } @@ -541,6 +558,81 @@ sub compute_pressure { delete $d->{pressure}; } +sub compute_avg_metrics { + my ($ctime, $avg_metrics, $objectype, $id) = @_; + + foreach my $metric (keys %$avg_metrics) { + my $value = $avg_metrics->{$metric}; + next if !defined($value); + + my $stats = $avgstats->{$objectype}->{$id}->{$metric} || {}; + $stats->{series}->{60}->{$ctime} = $value; + $stats->{avg60} = 0 if !defined($stats->{avg60}); + $stats->{avg300} = 0 if !defined($stats->{avg300}); + $stats->{last_compute_time} = $ctime if !defined($stats->{last_compute_time}); + + #compute avg each minute + + if($stats->{last_compute_time} && $ctime >= $stats->{last_compute_time} + 60) { + $stats->{avg60} = compute_avg($stats->{series}->{60}, $ctime, 60); + $stats->{series}->{300}->{$ctime} = $stats->{avg60}; + $stats->{avg300} = compute_avg($stats->{series}->{300}, $ctime, 300); + $stats->{last_compute_time} = $ctime; + } + + $balancer_stats->{$objectype}->{$id}->{$metric}->{avg60} = $stats->{avg60}; + $balancer_stats->{$objectype}->{$id}->{$metric}->{avg300} = $stats->{avg300}; + $avgstats->{$objectype}->{$id}->{$metric} = $stats; + } +} + +sub compute_avg { + my ($series, $ctime, $delta) = @_; + + my $total = 0; + my $count = 0; + my $to_delete; + + foreach my $time (keys %$series) { + if ($ctime - $delta >= $time) { + $to_delete->{$time} = 1; + next; + } + $count++; + $total += $series->{$time}; + } + + delete %$series{keys %{$to_delete}}; + + my $avg = $total / $count if $count > 0; + return $avg; +} + +sub delete_old_qemu_avgstats { + my ($vmstatus) = @_; + + my $stats = $avgstats->{'qemu'}; + + my $to_delete; + #delete stats for removed vm, or stopped vm + foreach my $vmid (keys %$stats) { + $to_delete->{$vmid} = 1 if !defined($vmstatus->{$vmid}) || !$vmstatus->{$vmid}->{pid}; + } + delete %$stats{keys %{$to_delete}}; +} + +sub delete_old_lxc_avgstats { + my ($vmstatus) = @_; + + my $stats = $avgstats->{'lxc'}; + + my $to_delete; + #delete stats for removed ct, or stopped ct + foreach my $vmid (keys %$stats) { + $to_delete->{$vmid} = 1 if !defined($vmstatus->{$vmid}) || $vmstatus->{$vmid}->{status} ne 'running'; + } + delete %$stats{keys %{$to_delete}}; +} my $broadcast_version_info_done = 0; my sub broadcast_version_info : prototype() { if (!$broadcast_version_info_done) { -- 2.30.2