[pve-devel] [PATCH pve-manager 3/4] pvestatd: qemu/lxc/node : add hostcpu/hostmem average stats

[Alexandre Derumier <aderumier@odiso.com>]

we aggregate each last X second stats to 1min average
we aggragate each last 5 1min average to 5min average.

vm avgstats are resetted when vm is stopped or removed.

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/Service/pvestatd.pm | 94 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 93 insertions(+), 1 deletion(-)

diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index dd578d6b..a33b21cd 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -81,6 +81,7 @@ my $next_flag_update_time;
 my $failed_flag_update_delay_sec = 120;
 my $balancer_stats = {};
 my $last_balancer_broadcast_time = 0;
+my $avgstats = {};
 
 sub update_supported_cpuflags {
     my $kvm_version = PVE::QemuServer::kvm_user_version();
@@ -177,6 +178,11 @@ sub update_node_status {
     $node_metric->{cpustat}->{cpus} = $maxcpu;
 
     compute_pressure($ctime, $node_metric, 'node', $nodename);
+    my $avg_metrics = {
+	cpu => $stat->{cpu},
+        mem => $meminfo->{memused},
+    };
+    compute_avg_metrics($ctime, $avg_metrics, 'node', $nodename);
 
     my $transactions = PVE::ExtMetric::transactions_start($status_cfg);
     PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime);
@@ -233,6 +239,11 @@ sub update_qemu_status {
 		 $d->{netin}, $d->{netout}, $d->{diskread}, $d->{diskwrite}]);
 
 	    compute_pressure($ctime, $d, 'qemu', $vmid);
+	    my $avg_metrics = {
+		cpu => $d->{hostcpu},
+		mem => $d->{hostmem},
+	    };
+	    compute_avg_metrics($ctime, $avg_metrics, 'qemu', $vmid);
 	} else {
 	    $data = $generate_rrd_string->(
 		[0, $d->{name}, $status, $template, $ctime, $d->{cpus}, undef,
@@ -242,7 +253,7 @@ sub update_qemu_status {
 
 	PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename);
     }
-
+    delete_old_qemu_avgstats($vmstatus);
     PVE::ExtMetric::transactions_finish($transactions);
 }
 
@@ -443,6 +454,11 @@ sub update_lxc_status {
 		 $d->{netin}, $d->{netout},
 		 $d->{diskread}, $d->{diskwrite}]);
 	    compute_pressure($ctime, $d, 'lxc', $vmid);
+	    my $avg_metrics = {
+		cpu => $d->{cpu},
+		mem => $d->{mem},
+	    };
+	    compute_avg_metrics($ctime, $avg_metrics, 'lxc', $vmid);
 	} else {
 	    $data = $generate_rrd_string->(
 		[0, $d->{name}, $d->{status}, $template, $ctime, $d->{cpus}, undef,
@@ -452,6 +468,7 @@ sub update_lxc_status {
 
 	PVE::ExtMetric::update_all($transactions, 'lxc', $vmid, $d, $ctime, $nodename);
     }
+    delete_old_lxc_avgstats($vmstatus);
     PVE::ExtMetric::transactions_finish($transactions);
 }
 
@@ -541,6 +558,81 @@ sub compute_pressure {
     delete $d->{pressure};
 }
 
+sub compute_avg_metrics {
+   my ($ctime, $avg_metrics, $objectype, $id) = @_;
+
+    foreach my $metric (keys %$avg_metrics) {
+	my $value = $avg_metrics->{$metric};
+	next if !defined($value);
+
+        my $stats = $avgstats->{$objectype}->{$id}->{$metric} || {};
+	$stats->{series}->{60}->{$ctime} = $value;
+	$stats->{avg60} = 0 if !defined($stats->{avg60});
+	$stats->{avg300} = 0 if !defined($stats->{avg300});
+	$stats->{last_compute_time} = $ctime if !defined($stats->{last_compute_time});
+
+	#compute avg each minute
+
+	if($stats->{last_compute_time} && $ctime >= $stats->{last_compute_time} + 60) {
+	    $stats->{avg60} = compute_avg($stats->{series}->{60}, $ctime, 60);
+	    $stats->{series}->{300}->{$ctime} = $stats->{avg60};
+	    $stats->{avg300} = compute_avg($stats->{series}->{300}, $ctime, 300);
+	    $stats->{last_compute_time} = $ctime;
+	}
+
+	$balancer_stats->{$objectype}->{$id}->{$metric}->{avg60} = $stats->{avg60};
+	$balancer_stats->{$objectype}->{$id}->{$metric}->{avg300} = $stats->{avg300};
+	$avgstats->{$objectype}->{$id}->{$metric} = $stats;
+    }
+}
+
+sub compute_avg {
+   my ($series, $ctime, $delta) = @_;
+
+    my $total = 0;
+    my $count = 0;
+    my $to_delete;
+
+    foreach my $time (keys %$series) {
+	if ($ctime - $delta >= $time) {
+	    $to_delete->{$time} = 1;
+	    next;
+	}
+	$count++;
+	$total += $series->{$time};
+    }
+
+    delete %$series{keys %{$to_delete}};
+
+    my $avg = $total / $count if $count > 0;
+    return $avg;
+}
+
+sub delete_old_qemu_avgstats {
+    my ($vmstatus) = @_;
+
+    my $stats = $avgstats->{'qemu'};
+
+    my $to_delete;
+    #delete stats for removed vm, or stopped vm
+    foreach my $vmid (keys %$stats) {
+	$to_delete->{$vmid} = 1 if !defined($vmstatus->{$vmid}) || !$vmstatus->{$vmid}->{pid};
+    }
+    delete %$stats{keys %{$to_delete}};
+}
+
+sub delete_old_lxc_avgstats {
+    my ($vmstatus) = @_;
+
+    my $stats = $avgstats->{'lxc'};
+
+    my $to_delete;
+    #delete stats for removed ct, or stopped ct
+    foreach my $vmid (keys %$stats) {
+	$to_delete->{$vmid} = 1 if !defined($vmstatus->{$vmid}) || $vmstatus->{$vmid}->{status} ne 'running';
+    }
+    delete %$stats{keys %{$to_delete}};
+}
 my $broadcast_version_info_done = 0;
 my sub broadcast_version_info : prototype() {
     if (!$broadcast_version_info_done) {
-- 
2.30.2