[pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types

[pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types

[Folke Gleumes]

The goal of this patchseries is to make it more intuitive to understand
the memory usage of an instance by splitting the memory usage graph
into 'Used', 'Buffer/Cache', 'ZFS Arc' and 'Free' instead of 'Total'
and 'RAM usage', displayed as different colored segments in a stacking
graph that adds up to the total memory. Previously, the zfs arc counted
towards the used memory, which could give a skewed perspective, since
there is no visual difference between almost all memory being used with
next to no arc or half of the memory used with a big arc.

Since there has already been a heated discussion on Bugzilla [0] on what
to display, I thought I would post this as an RFC before I finish up the
patchseries.

Another question would be if other places that display the memory usage
should also be split, i.e. the progress bar and gauge chart in the
cluster overview. Previously they included the zfs arc, with this series
the arc is not counted as used memory, since it technically can be freed
(with some caveats).

Unfortunately, there is a visual inconsistency with the way that missing
data is displayed in the new graph, since it seems impossible to display
missing data as a gap when using an area graph, which is possible and
the default in the line graph that is used throughout the rest of the
ui.

Note that applying this patch will currently reset your metrics, so
you'll have to wait a bit to see results.

[0] https://bugzilla.proxmox.com/show_bug.cgi?id=1454

cluster: Folke Gleumes (1):
  rrd: add free, buffer/cache and arc size to memory statistics

 src/pmxcfs/status.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

common: Folke Gleumes (1):
  add more detailed statistics to memory report

 src/PVE/ProcFSTools.pm | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

widget-toolkit: Folke Gleumes (1):
  add option to display rrd graph as stacking

 src/panel/RRDChart.js | 59 +++++++++++++++++++++++++++++++++----------
 1 file changed, 46 insertions(+), 13 deletions(-)

manager: Folke Gleumes (1):
  split ram usage into usage types

 PVE/API2/Nodes.pm            |  6 +++---
 PVE/API2Tools.pm             |  2 +-
 PVE/Service/pvestatd.pm      | 15 +++++++++++----
 www/manager6/node/Summary.js |  6 ++++--
 4 files changed, 19 insertions(+), 10 deletions(-)

-- 
2.39.2

[pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics

[Folke Gleumes]

adding values to the rrd format break compatability with the old file.
Therfore the filename/path had to be changed as well.

Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
 src/pmxcfs/status.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/pmxcfs/status.c b/src/pmxcfs/status.c
index dc44464..8834644 100644
--- a/src/pmxcfs/status.c
+++ b/src/pmxcfs/status.c
@@ -1130,7 +1130,10 @@ static const char *rrd_def_node[] = {
 	"DS:cpu:GAUGE:120:0:U",
 	"DS:iowait:GAUGE:120:0:U",
 	"DS:memtotal:GAUGE:120:0:U",
+	"DS:memfree:GAUGE:120:0:U",
 	"DS:memused:GAUGE:120:0:U",
+	"DS:membuffercache:GAUGE:120:0:U",
+	"DS:arcsize:GAUGE:120:0:U",
 	"DS:swaptotal:GAUGE:120:0:U",
 	"DS:swapused:GAUGE:120:0:U",
 	"DS:roottotal:GAUGE:120:0:U",
@@ -1252,7 +1255,7 @@ update_rrd_data(
 
 	int skip = 0;
 
-	if (strncmp(key, "pve2-node/", 10) == 0) {
+	if (strncmp(key, "pve8-node/", 10) == 0) {
 		const char *node = key + 10;
 
 		skip = 2;
@@ -1267,12 +1270,12 @@ update_rrd_data(
 
 		if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
 
-			mkdir(RRDDIR "/pve2-node", 0755);
+			mkdir(RRDDIR "/pve8-node", 0755);
 			int argcount = sizeof(rrd_def_node)/sizeof(void*) - 1;
 			create_rrd_file(filename, argcount, rrd_def_node);
 		}
 
-	} else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
+	} else if ((strncmp(key, "pve8-vm/", 8) == 0) ||
 		   (strncmp(key, "pve2.3-vm/", 10) == 0)) {
 		const char *vmid;
 
-- 
2.39.2

Re: [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics

[DERUMIER, Alexandre]

Hi,

if we are going to bump the rrd version,

I think it could be great to add pressure stats too.

I have send patches 2 year ago, it can be really usefull to implement
vm balancing/scheduling/node eviction  or even simply to diplay graph
to have more information about load.


https://lists.proxmox.com/pipermail/pve-devel/2022-May/053068.html


This patch was with pressure stats in separate rrd, but if you are
bumping the main rrd version, it could be directly integrated.

I can rework the patch if you want.



-------- Message initial --------
De: Folke Gleumes <f.gleumes@proxmox.com>
Répondre à: Proxmox VE development discussion <pve-
devel@lists.proxmox.com>
À: pve-devel@lists.proxmox.com
Objet: [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and
arc size to memory statistics
Date: 11/12/2023 15:47:18

adding values to the rrd format break compatability with the old file.
Therfore the filename/path had to be changed as well.

Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
 src/pmxcfs/status.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/pmxcfs/status.c b/src/pmxcfs/status.c
index dc44464..8834644 100644
--- a/src/pmxcfs/status.c
+++ b/src/pmxcfs/status.c
@@ -1130,7 +1130,10 @@ static const char *rrd_def_node[] = {
 	"DS:cpu:GAUGE:120:0:U",
 	"DS:iowait:GAUGE:120:0:U",
 	"DS:memtotal:GAUGE:120:0:U",
+	"DS:memfree:GAUGE:120:0:U",
 	"DS:memused:GAUGE:120:0:U",
+	"DS:membuffercache:GAUGE:120:0:U",
+	"DS:arcsize:GAUGE:120:0:U",
 	"DS:swaptotal:GAUGE:120:0:U",
 	"DS:swapused:GAUGE:120:0:U",
 	"DS:roottotal:GAUGE:120:0:U",
@@ -1252,7 +1255,7 @@ update_rrd_data(
 
 	int skip = 0;
 
-	if (strncmp(key, "pve2-node/", 10) == 0) {
+	if (strncmp(key, "pve8-node/", 10) == 0) {
 		const char *node = key + 10;
 
 		skip = 2;
@@ -1267,12 +1270,12 @@ update_rrd_data(
 
 		if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
 
-			mkdir(RRDDIR "/pve2-node", 0755);
+			mkdir(RRDDIR "/pve8-node", 0755);
 			int argcount =
sizeof(rrd_def_node)/sizeof(void*) - 1;
 			create_rrd_file(filename, argcount,
rrd_def_node);
 		}
 
-	} else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
+	} else if ((strncmp(key, "pve8-vm/", 8) == 0) ||
 		   (strncmp(key, "pve2.3-vm/", 10) == 0)) {
 		const char *vmid;
 

[pve-devel] [RFC common 2/4] add more detailed statistics to memory report

[Folke Gleumes]

Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
 src/PVE/ProcFSTools.pm | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/PVE/ProcFSTools.pm b/src/PVE/ProcFSTools.pm
index 3826fcc..5f5d768 100644
--- a/src/PVE/ProcFSTools.pm
+++ b/src/PVE/ProcFSTools.pm
@@ -279,7 +279,10 @@ sub read_meminfo {
     my $res = {
 	memtotal => 0,
 	memfree => 0,
+	memavailable => 0,
 	memused => 0,
+	membuffers => 0,
+	memcached => 0,
 	memshared => 0,
 	swaptotal => 0,
 	swapfree => 0,
@@ -299,7 +302,13 @@ sub read_meminfo {
     close($fh);
 
     $res->{memtotal} = $d->{memtotal};
-    $res->{memfree} =  $d->{memfree} + $d->{buffers} + $d->{cached};
+    $res->{memfree} = $d->{memfree};
+    $res->{membuffers} = $d->{buffers};
+    $res->{memcached} = $d->{cached};
+    $res->{memshared} = $d->{memshared};
+
+    # memfree is unused memory, memused is everything that is in use
+    # be it by caches, buffers or applications
     $res->{memused} = $res->{memtotal} - $res->{memfree};
 
     $res->{swaptotal} = $d->{swaptotal};
-- 
2.39.2

[pve-devel] [RFC widget-toolkit 3/4] add option to display rrd graph as stacking

[Folke Gleumes]

Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
 src/panel/RRDChart.js | 59 +++++++++++++++++++++++++++++++++----------
 1 file changed, 46 insertions(+), 13 deletions(-)

diff --git a/src/panel/RRDChart.js b/src/panel/RRDChart.js
index dc5940c..983437e 100644
--- a/src/panel/RRDChart.js
+++ b/src/panel/RRDChart.js
@@ -118,9 +118,14 @@ Ext.define('Proxmox.widget.RRDChart', {
 	    if (view.fieldTitles && view.fieldTitles[view.fields.indexOf(item.field)]) {
 		prefix = view.fieldTitles[view.fields.indexOf(item.field)];
 	    }
-	    let v = this.convertToUnits(record.get(item.field));
-	    let t = new Date(record.get('time'));
-	    tooltip.setHtml(`${prefix}: ${v}${suffix}<br>${t}`);
+	    let value = record.get(item.field);
+	    if (value !== null) {
+		let v = this.convertToUnits(record.get(item.field));
+		let t = new Date(record.get('time'));
+		tooltip.setHtml(`${prefix}: ${v}${suffix}<br>${t}`);
+	    } else {
+		tooltip.setHtml('No Data');
+	    }
 	},
 
 	onAfterAnimation: function(chart, eopts) {
@@ -256,18 +261,13 @@ Ext.define('Proxmox.widget.RRDChart', {
 	}
 
 	// add a series for each field we get
-	me.fields.forEach(function(item, index) {
-	    let title = item;
-	    if (me.fieldTitles && me.fieldTitles[index]) {
-		title = me.fieldTitles[index];
-	    }
+	if (me.stacking) {
 	    me.addSeries(Ext.apply(
 		{
-		    type: 'line',
+		    type: 'area',
 		    xField: 'time',
-		    yField: item,
-		    title: title,
-		    fill: true,
+		    yField: me.fields,
+		    title: me.fieldTitles,
 		    style: {
 			lineWidth: 1.5,
 			opacity: 0.60,
@@ -287,7 +287,40 @@ Ext.define('Proxmox.widget.RRDChart', {
 		},
 		me.seriesConfig,
 	    ));
-	});
+	} else {
+	    me.fields.forEach(function(item, index) {
+		let title = item;
+		if (me.fieldTitles && me.fieldTitles[index]) {
+		    title = me.fieldTitles[index];
+		}
+		me.addSeries(Ext.apply(
+		    {
+			type: 'line',
+			xField: 'time',
+			yField: item,
+			title: title,
+			fill: true,
+			style: {
+			    lineWidth: 1.5,
+			    opacity: 0.60,
+			},
+			marker: {
+			    opacity: 0,
+			    scaling: 0.01,
+			},
+			highlightCfg: {
+			    opacity: 1,
+			    scaling: 1.5,
+			},
+			tooltip: {
+			    trackMouse: true,
+			    renderer: 'onSeriesTooltipRender',
+			},
+		    },
+		    me.seriesConfig,
+		));
+	    });
+	}
 
 	// enable animation after the store is loaded
 	me.store.onAfter('load', function() {
-- 
2.39.2

[pve-devel] [RFC manager 4/4] split ram usage into usage types

[Folke Gleumes]

Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
 PVE/API2/Nodes.pm            |  6 +++---
 PVE/API2Tools.pm             |  2 +-
 PVE/Service/pvestatd.pm      | 15 +++++++++++----
 www/manager6/node/Summary.js |  6 ++++--
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 3619190d..7dd8e5c2 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -430,7 +430,7 @@ __PACKAGE__->register_method({
 	$res->{memory} = {
 	    free => $meminfo->{memfree},
 	    total => $meminfo->{memtotal},
-	    used => $meminfo->{memused},
+	    used => $meminfo->{memused} - ($meminfo->{memcached} + $meminfo->{membuffers} + $meminfo->{arcsize}),
 	};
 
 	$res->{ksm} = {
@@ -715,7 +715,7 @@ __PACKAGE__->register_method({
 	my ($param) = @_;
 
 	return PVE::RRD::create_rrd_graph(
-	    "pve2-node/$param->{node}", $param->{timeframe},
+	    "pve8-node/$param->{node}", $param->{timeframe},
 	    $param->{ds}, $param->{cf});
 
     }});
@@ -757,7 +757,7 @@ __PACKAGE__->register_method({
 	my ($param) = @_;
 
 	return PVE::RRD::create_rrd_data(
-	    "pve2-node/$param->{node}", $param->{timeframe}, $param->{cf});
+	    "pve8-node/$param->{node}", $param->{timeframe}, $param->{cf});
     }});
 
 __PACKAGE__->register_method({
diff --git a/PVE/API2Tools.pm b/PVE/API2Tools.pm
index a56eb732..f5ddff6e 100644
--- a/PVE/API2Tools.pm
+++ b/PVE/API2Tools.pm
@@ -49,7 +49,7 @@ sub extract_node_stats {
 	status => 'unknown',
     };
 
-    if (my $d = $rrd->{"pve2-node/$node"}) {
+    if (my $d = $rrd->{"pve8-node/$node"}) {
 
 	if (!$members || # no cluster
 	    ($members->{$node} && $members->{$node}->{online})) {
diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index 2515120c..3fed4247 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -29,6 +29,7 @@ use PVE::AccessControl;
 use PVE::Ceph::Services;
 use PVE::Ceph::Tools;
 use PVE::pvecfg;
+use Data::Dumper;
 
 use PVE::ExtMetric;
 use PVE::Status::Plugin;
@@ -177,13 +178,19 @@ sub update_node_status {
 
     my $ctime = time();
 
+    my $memused = $meminfo->{memtotal} - $meminfo->{memfree};
+    my $mem_non_cache = $memused - ($meminfo->{memcached} + $meminfo->{membuffers} + $meminfo->{arcsize});
+
     my $data = $generate_rrd_string->(
 	[$uptime, $sublevel, $ctime, $avg1, $maxcpu, $stat->{cpu}, $stat->{wait},
-	 $meminfo->{memtotal}, $meminfo->{memused},
-	 $meminfo->{swaptotal}, $meminfo->{swapused},
-	 $dinfo->{blocks}, $dused, $netin, $netout]
+	 $meminfo->{memtotal}, $meminfo->{memfree},
+	 $mem_non_cache,
+	 $meminfo->{membuffers} + $meminfo->{memcached},
+	 $meminfo->{arcsize}, $meminfo->{swaptotal},
+	 $meminfo->{swapused}, $dinfo->{blocks},
+	 $dused, $netin, $netout]
     );
-    PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data);
+    PVE::Cluster::broadcast_rrd("pve8-node/$nodename", $data);
 
     my $node_metric = {
 	uptime => $uptime,
diff --git a/www/manager6/node/Summary.js b/www/manager6/node/Summary.js
index c2dca0df..d7bb8815 100644
--- a/www/manager6/node/Summary.js
+++ b/www/manager6/node/Summary.js
@@ -174,11 +174,13 @@ Ext.define('PVE.node.Summary', {
 			{
 			    xtype: 'proxmoxRRDChart',
 			    title: gettext('Memory usage'),
-			    fields: ['memtotal', 'memused'],
-			    fieldTitles: [gettext('Total'), gettext('RAM usage')],
+			    fields: ['memused', 'membuffercache', 'arcsize', 'memfree'],
+			    fieldTitles: [gettext('Used'), gettext('Buffer/Cache'), 'ZFS Arc', gettext('Free')],
 			    unit: 'bytes',
 			    powerOfTwo: true,
 			    store: rrdstore,
+			    colors: ['#115fa6', '#a61187', '#ff8809', '#94ae0a'],
+			    stacking: true,
 			},
 			{
 			    xtype: 'proxmoxRRDChart',
-- 
2.39.2