[pve-devel] [PATCH-SERIES cluster/common/qemu-server/lxc/manager] add new metric stats in single rrd

[pve-devel] [PATCH-SERIES cluster/common/qemu-server/lxc/manager] add new metric stats in single rrd

[Alexandre Derumier]

Hi,

I'm still working on vm balancing/scheduling, and need some new metrics.

This patch-series add new metrics stats in rrd

- cpu,mem,io pressure for qemu/lxc/host
- hostcpu/hostmem cgroup for qemu
- ksm

as we discussed last year, theses new metrics are pushed in single rrd files,
like proxmox backup server.

The paths are :

pve2-metrics/vms/<vmid>/<metric>
pve2-metrics/nodes/<node>/<metric>
pve2-metrics/storages/<node>/<storage>/<metric>



pve-cluster:

Alexandre Derumier (1):
  add pve2-metrics rrd (single metrics)

 data/src/status.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)


pve-common:

Alexandre Derumier (2):
  cgroup: get_pressure_stat: use controllers in get_path
  cgroup: get_pressure_stat: add cpu full pressure

 src/PVE/CGroup.pm | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

qemu-server:

Alexandre Derumier (3):
  vmstatus: add hostcpu value
  vmstatus: add hostmem value
  vmstatus: add pressure stats

 PVE/QemuServer.pm | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

pve-container:

Alexandre Derumier (1):
  vmstatus: add pressure stats

 src/PVE/LXC.pm | 2 ++
 1 file changed, 2 insertions(+)

pve-manager:

Alexandre Derumier (3):
  pvestatd: qemu/lxc/host : broadcast rrd pressure metrics
  pvestatd: qemu: broadcast rrd hostcpu && hostmem metrics
  pvestatd: host: broadcast rrd ksm metric

 PVE/Service/pvestatd.pm | 53 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)

-- 
2.30.2

[pve-devel] [PATCH pve-cluster 1/1] add pve2-metrics rrd (single metrics)

[Alexandre Derumier]

This create 1 single rrd for each metric

allowed paths:

pve2-metrics/vms/<vmid>/<metric>
pve2-metrics/nodes/<node>/<metric>
pve2-metrics/storages/<node>/<storage>/<metric>

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 data/src/status.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/data/src/status.c b/data/src/status.c
index 9bceaeb..75ab89e 100644
--- a/data/src/status.c
+++ b/data/src/status.c
@@ -1097,6 +1097,23 @@ static const char *rrd_def_storage[] = {
 	NULL,
 };
 
+static const char *rrd_def_metric[] = {
+	"DS:metric:GAUGE:120:0:U",
+
+	"RRA:AVERAGE:0.5:1:70", // 1 min avg - one hour
+	"RRA:AVERAGE:0.5:30:70", // 30 min avg - one day
+	"RRA:AVERAGE:0.5:180:70", // 3 hour avg - one week
+	"RRA:AVERAGE:0.5:720:70", // 12 hour avg - one month
+	"RRA:AVERAGE:0.5:10080:70", // 7 day avg - ony year
+
+	"RRA:MAX:0.5:1:70", // 1 min max - one hour
+	"RRA:MAX:0.5:30:70", // 30 min max - one day
+	"RRA:MAX:0.5:180:70",  // 3 hour max - one week
+	"RRA:MAX:0.5:720:70", // 12 hour max - one month
+	"RRA:MAX:0.5:10080:70", // 7 day max - ony year
+	NULL,
+};
+
 #define RRDDIR "/var/lib/rrdcached/db"
 
 static void
@@ -1173,6 +1190,40 @@ update_rrd_data(
 			create_rrd_file(filename, argcount, rrd_def_node);
 		}
 
+	} else if ((strncmp(key, "pve2-metrics/", 13) == 0)) {
+		const char *path = key + 13;
+		char **pa = g_strsplit(path, "/", -1);
+
+		if (!pa[0] || !pa[1] || !pa[2] || strlen(pa[1]) < 1 || strlen(pa[2]) < 1) {
+			goto keyerror;
+		} else {
+			if (strcmp(pa[0], "vms") == 0 || strcmp(pa[0], "nodes") == 0) {
+				if (pa[3]) {
+					goto keyerror;
+				}
+			} else if (strcmp(pa[0], "storages") == 0) {
+				if (pa[4] || !pa[3] || strlen(pa[3]) < 1) {
+					goto keyerror;
+				}
+			} else {
+				goto keyerror;
+			}
+		}
+		g_strfreev(pa);
+
+
+		filename = g_strdup_printf(RRDDIR "/%s", key);
+
+		if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
+
+			char *dir = g_path_get_dirname(filename);
+			g_mkdir_with_parents(dir, 0755);
+			g_free(dir);
+
+			int argcount = sizeof(rrd_def_metric)/sizeof(void*) - 1;
+			create_rrd_file(filename, argcount, rrd_def_metric);
+		}
+
 	} else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
 		   (strncmp(key, "pve2.3-vm/", 10) == 0)) {
 		const char *vmid;
-- 
2.30.2

[pve-devel] [PATCH pve-common 1/2] cgroup: get_pressure_stat: use controllers in get_path

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 src/PVE/CGroup.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/PVE/CGroup.pm b/src/PVE/CGroup.pm
index 44b3297..d3873fd 100644
--- a/src/PVE/CGroup.pm
+++ b/src/PVE/CGroup.pm
@@ -380,7 +380,8 @@ sub get_pressure_stat {
 	},
     };
 
-    my ($path, $version) = $self->get_path(undef, 1);
+    my ($path, $version) = $self->get_any_path(1, 'cpu', 'memory', 'io');
+
     if (!defined($path)) {
 	return $res; # container or VM most likely isn't running, retrun zero stats
     } elsif ($version == 1) {
-- 
2.30.2

[pve-devel] [PATCH pve-manager 1/3] pvestatd: qemu/lxc/host : broadcast rrd pressure metrics

[Alexandre Derumier]

only "some" values for now, not sure we need full values

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/Service/pvestatd.pm | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index b1e71ec8..832d9dc5 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -132,6 +132,7 @@ sub update_node_status {
     my $stat = PVE::ProcFSTools::read_proc_stat();
     my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
     my $maxcpu = $cpuinfo->{cpus};
+    my $pressure = PVE::ProcFSTools::read_pressure();
 
     update_supported_cpuflags();
 
@@ -168,10 +169,13 @@ sub update_node_status {
 	memory => $meminfo,
 	blockstat => $dinfo,
 	nics => $netdev,
+	pressure => $pressure,
     };
     $node_metric->{cpustat}->@{qw(avg1 avg5 avg15)} = ($avg1, $avg5, $avg15);
     $node_metric->{cpustat}->{cpus} = $maxcpu;
 
+    broadcast_rrd_pressure($ctime, $node_metric, "pve2-metrics/nodes/$nodename");
+
     my $transactions = PVE::ExtMetric::transactions_start($status_cfg);
     PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime);
     PVE::ExtMetric::transactions_finish($transactions);
@@ -232,12 +236,41 @@ sub update_qemu_status {
 	}
 	PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 
+	broadcast_rrd_pressure($ctime, $d, "pve2-metrics/vms/$vmid");
+
 	PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename);
     }
 
     PVE::ExtMetric::transactions_finish($transactions);
 }
 
+sub broadcast_rrd_pressure {
+    my ($ctime, $d, $path) = @_;
+
+    return if !defined($d->{pressure});
+
+    my $pressure = $d->{pressure};
+
+    foreach my $type (keys %{$pressure}) {
+	my $pressuretype = $pressure->{$type};
+
+	foreach my $kind (keys %{$pressuretype}) {
+	    next if $kind ne 'some';
+	    my $pressurekind = $pressuretype->{$kind};
+
+	    foreach my $avg (keys %{$pressurekind}) {
+		next if $avg eq 'total';
+		my $value = $pressurekind->{$avg};
+		my $metric = "pressure_".$type."_".$kind."_".$avg;
+		my $data = $generate_rrd_string->([$ctime, $value]);
+		PVE::Cluster::broadcast_rrd("$path/$metric", $data);
+		$d->{$metric} = $value;
+	    }
+	}
+    }
+    delete $d->{pressure};
+}
+
 sub remove_stale_lxc_consoles {
 
     my $vmstatus = PVE::LXC::vmstatus();
@@ -441,6 +474,8 @@ sub update_lxc_status {
 	}
 	PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 
+	broadcast_rrd_pressure($ctime, $d, "pve2-metrics/vms/$vmid");
+
 	PVE::ExtMetric::update_all($transactions, 'lxc', $vmid, $d, $ctime, $nodename);
     }
     PVE::ExtMetric::transactions_finish($transactions);
-- 
2.30.2

[pve-devel] [PATCH qemu-server 1/3] vmstatus: add hostcpu value

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/QemuServer.pm | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index e9aa248..9441cf2 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -2922,8 +2922,11 @@ sub vmstatus {
 
 	my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
 	next if !$pstat; # not running
+	my $cgroups = PVE::QemuServer::CGroup->new($vmid);
+	my $hostcpustat = $cgroups->get_cpu_stat();
 
 	my $used = $pstat->{utime} + $pstat->{stime};
+	my $hostused = $hostcpustat->{utime} + $hostcpustat->{stime};
 
 	$d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
 
@@ -2937,6 +2940,9 @@ sub vmstatus {
 		time => $ctime,
 		used => $used,
 		cpu => 0,
+		hostused => $hostused,
+		hostcpu => 0,
+
 	    };
 	    next;
 	}
@@ -2945,15 +2951,20 @@ sub vmstatus {
 
 	if ($dtime > 1000) {
 	    my $dutime = $used -  $old->{used};
+	    my $dhostutime = $hostused -  $old->{hostused};
 
 	    $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
+	    $d->{hostcpu} = (($dhostutime/$dtime)* $cpucount) / $d->{cpus};
 	    $last_proc_pid_stat->{$pid} = {
 		time => $ctime,
 		used => $used,
 		cpu => $d->{cpu},
+		hostused => $hostused,
+		hostcpu => $d->{hostcpu},
 	    };
 	} else {
 	    $d->{cpu} = $old->{cpu};
+	    $d->{hostcpu} = $old->{hostcpu};
 	}
     }
 
-- 
2.30.2

[pve-devel] [PATCH pve-container 1/1] vmstatus: add pressure stats

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 src/PVE/LXC.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/PVE/LXC.pm b/src/PVE/LXC.pm
index fe63087..af47ff9 100644
--- a/src/PVE/LXC.pm
+++ b/src/PVE/LXC.pm
@@ -299,6 +299,8 @@ sub vmstatus {
 	} else {
 	    $d->{cpu} = 0;
 	}
+
+	$d->{pressure} = $cgroups->get_pressure_stat();
     }
 
     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
-- 
2.30.2

[pve-devel] [PATCH pve-common 2/2] cgroup: get_pressure_stat: add cpu full pressure

[Alexandre Derumier]

available since kernel 5.13
https://lore.kernel.org/all/20210303034659.91735-2-zhouchengming@bytedance.com/T/#u

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 src/PVE/CGroup.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/PVE/CGroup.pm b/src/PVE/CGroup.pm
index d3873fd..bc5b8c8 100644
--- a/src/PVE/CGroup.pm
+++ b/src/PVE/CGroup.pm
@@ -368,7 +368,8 @@ sub get_pressure_stat {
 
     my $res = {
 	cpu => {
-	    some => { avg10 => 0, avg60 => 0, avg300 => 0 }
+	    some => { avg10 => 0, avg60 => 0, avg300 => 0 },
+	    full => { avg10 => 0, avg60 => 0, avg300 => 0 }
 	},
 	memory => {
 	    some => { avg10 => 0, avg60 => 0, avg300 => 0 },
-- 
2.30.2

[pve-devel] [PATCH pve-manager 2/3] pvestatd: qemu: broadcast rrd hostcpu && hostmem metrics

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/Service/pvestatd.pm | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index 832d9dc5..7ed12504 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -236,6 +236,9 @@ sub update_qemu_status {
 	}
 	PVE::Cluster::broadcast_rrd("pve2.3-vm/$vmid", $data);
 
+	my $single_metrics = ['hostcpu', 'hostmem'];
+	broadcast_rrd_metrics($ctime, $d, $single_metrics, "pve2-metrics/vms/$vmid");
+
 	broadcast_rrd_pressure($ctime, $d, "pve2-metrics/vms/$vmid");
 
 	PVE::ExtMetric::update_all($transactions, 'qemu', $vmid, $d, $ctime, $nodename);
@@ -271,6 +274,17 @@ sub broadcast_rrd_pressure {
     delete $d->{pressure};
 }
 
+sub broadcast_rrd_metrics {
+    my ($ctime, $d, $single_metrics, $path) = @_;
+
+    foreach my $metric (@$single_metrics) {
+	my $value = $d->{$metric};
+	next if !defined($value);
+	my $data = $generate_rrd_string->([$ctime, $value]);
+	PVE::Cluster::broadcast_rrd("$path/$metric", $data);
+    }
+}
+
 sub remove_stale_lxc_consoles {
 
     my $vmstatus = PVE::LXC::vmstatus();
-- 
2.30.2

[pve-devel] [PATCH qemu-server 2/3] vmstatus: add hostmem value

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/QemuServer.pm | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 9441cf2..4fc183e 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -2933,6 +2933,11 @@ sub vmstatus {
 	if ($pstat->{vsize}) {
 	    $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
 	}
+	if (defined(my $hostmemstat = $cgroups->get_memory_stat())) {
+	    $d->{hostmem} = $hostmemstat->{mem};
+	} else {
+	    $d->{hostmem} = 0;
+	}
 
 	my $old = $last_proc_pid_stat->{$pid};
 	if (!$old) {
-- 
2.30.2

[pve-devel] [PATCH pve-manager 3/3] pvestatd: host: broadcast rrd ksm metric

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/Service/pvestatd.pm | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index 7ed12504..1e7400e0 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -170,12 +170,16 @@ sub update_node_status {
 	blockstat => $dinfo,
 	nics => $netdev,
 	pressure => $pressure,
+	ksm => $meminfo->{memshared},
     };
     $node_metric->{cpustat}->@{qw(avg1 avg5 avg15)} = ($avg1, $avg5, $avg15);
     $node_metric->{cpustat}->{cpus} = $maxcpu;
 
     broadcast_rrd_pressure($ctime, $node_metric, "pve2-metrics/nodes/$nodename");
 
+    my $single_metrics = ['ksm'];
+    broadcast_rrd_metrics($ctime, $node_metric, $single_metrics, "pve2-metrics/nodes/$nodename");
+
     my $transactions = PVE::ExtMetric::transactions_start($status_cfg);
     PVE::ExtMetric::update_all($transactions, 'node', $nodename, $node_metric, $ctime);
     PVE::ExtMetric::transactions_finish($transactions);
-- 
2.30.2

[pve-devel] [PATCH qemu-server 3/3] vmstatus: add pressure stats

[Alexandre Derumier]

Signed-off-by: Alexandre Derumier <aderumier@odiso.com>
---
 PVE/QemuServer.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 4fc183e..09f3a0c 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -2971,6 +2971,8 @@ sub vmstatus {
 	    $d->{cpu} = $old->{cpu};
 	    $d->{hostcpu} = $old->{hostcpu};
 	}
+
+	$d->{pressure} = $cgroups->get_pressure_stat();
     }
 
     return $res if !$full;
-- 
2.30.2

Re: [pve-devel] [PATCH-SERIES cluster/common/qemu-server/lxc/manager] add new metric stats in single rrd

[Thomas Lamprecht]

On 25/05/2022 08:52, Alexandre Derumier wrote:
> Hi,
> 
> I'm still working on vm balancing/scheduling, and need some new metrics.
> 

but you don't actually need the metrics themselves but only the computed
moving avg result and such of them? Because if that's the case we could just
keep the last X values we require for the moving window cached locally in
pvestatd and use those to broadcast the calculated ones use for DRS once or
twice every minute or so via the per-node kv status in the pvestatd?

Could mean less data an no new RRD format/paths (iow. no touching of
pve-cluster at all). What do you think?

Re: [pve-devel] [PATCH-SERIES cluster/common/qemu-server/lxc/manager] add new metric stats in single rrd

[DERUMIER, Alexandre]

Hi Thomas,

for pressure, I don't really need rrd or history, as I can already
broadcast last 10s,60s, 300s.
so could simply broadcast them in kvstore.

This was the original patch in 2020
https://lists.proxmox.com/pipermail/pve-devel/2020-October/045493.html

but Dietmar asked about to put them in rrd ;)

So, as you want, just tell me :)

I don't care about rrd for my need. 

Anyway, For vm balancing, I was planning to compute average metric
instead parsing at lot of rrd.


for ksm, I really need current value

for qemu hostcpu/hostmem, last (1-5) minutes average could be enough,
so I could compute from pvestatd too.



Alexandre


Le mercredi 25 mai 2022 à 17:03 +0200, Thomas Lamprecht a écrit :
> On 25/05/2022 08:52, Alexandre Derumier wrote:
> > Hi,
> > 
> > I'm still working on vm balancing/scheduling, and need some new
> > metrics.
> > 
> 
> but you don't actually need the metrics themselves but only the
> computed
> moving avg result and such of them? Because if that's the case we
> could just
> keep the last X values we require for the moving window cached
> locally in
> pvestatd and use those to broadcast the calculated ones use for DRS
> once or
> twice every minute or so via the per-node kv status in the pvestatd?
> 
> Could mean less data an no new RRD format/paths (iow. no touching of
> pve-cluster at all). What do you think?
> 
> 

[pve-devel] [PATCH qemu-server 0/3] vmstatus: add pressure + hostcpu/hostmem

[Alexandre Derumier]

Need pve-common patches serie.


Alexandre Derumier (3):
  vmstatus: add hostcpu value
  vmstatus: add hostmem value
  vmstatus: add pressure stats

 PVE/QemuServer.pm | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

-- 
2.20.1

[pve-devel] [PATCH qemu-server 1/3] vmstatus: add hostcpu value

[Alexandre Derumier]

---
 PVE/QemuServer.pm | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index af74d64..0c2b1db 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -2647,8 +2647,11 @@ sub vmstatus {
 
 	my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
 	next if !$pstat; # not running
+	my $cgroups = PVE::QemuServer::CGroup->new($vmid);
+	my $hostcpustat = $cgroups->get_cpu_stat();
 
 	my $used = $pstat->{utime} + $pstat->{stime};
+	my $hostused = $hostcpustat->{utime} + $hostcpustat->{stime};
 
 	$d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
 
@@ -2662,6 +2665,9 @@ sub vmstatus {
 		time => $ctime,
 		used => $used,
 		cpu => 0,
+		hostused => $hostused,
+		hostcpu => 0,
+
 	    };
 	    next;
 	}
@@ -2670,15 +2676,20 @@ sub vmstatus {
 
 	if ($dtime > 1000) {
 	    my $dutime = $used -  $old->{used};
+	    my $dhostutime = $hostused -  $old->{hostused};
 
 	    $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
+	    $d->{hostcpu} = (($dhostutime/$dtime)* $cpucount) / $d->{cpus};
 	    $last_proc_pid_stat->{$pid} = {
 		time => $ctime,
 		used => $used,
 		cpu => $d->{cpu},
+		hostused => $hostused,
+		hostcpu => $d->{hostcpu},
 	    };
 	} else {
 	    $d->{cpu} = $old->{cpu};
+	    $d->{hostcpu} = $old->{hostcpu};
 	}
     }
 
-- 
2.20.1