* [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics
2023-12-11 14:47 [pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types Folke Gleumes
@ 2023-12-11 14:47 ` Folke Gleumes
2023-12-12 7:49 ` DERUMIER, Alexandre
2023-12-11 14:47 ` [pve-devel] [RFC common 2/4] add more detailed statistics to memory report Folke Gleumes
` (2 subsequent siblings)
3 siblings, 1 reply; 6+ messages in thread
From: Folke Gleumes @ 2023-12-11 14:47 UTC (permalink / raw)
To: pve-devel
adding values to the rrd format break compatability with the old file.
Therfore the filename/path had to be changed as well.
Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
src/pmxcfs/status.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/pmxcfs/status.c b/src/pmxcfs/status.c
index dc44464..8834644 100644
--- a/src/pmxcfs/status.c
+++ b/src/pmxcfs/status.c
@@ -1130,7 +1130,10 @@ static const char *rrd_def_node[] = {
"DS:cpu:GAUGE:120:0:U",
"DS:iowait:GAUGE:120:0:U",
"DS:memtotal:GAUGE:120:0:U",
+ "DS:memfree:GAUGE:120:0:U",
"DS:memused:GAUGE:120:0:U",
+ "DS:membuffercache:GAUGE:120:0:U",
+ "DS:arcsize:GAUGE:120:0:U",
"DS:swaptotal:GAUGE:120:0:U",
"DS:swapused:GAUGE:120:0:U",
"DS:roottotal:GAUGE:120:0:U",
@@ -1252,7 +1255,7 @@ update_rrd_data(
int skip = 0;
- if (strncmp(key, "pve2-node/", 10) == 0) {
+ if (strncmp(key, "pve8-node/", 10) == 0) {
const char *node = key + 10;
skip = 2;
@@ -1267,12 +1270,12 @@ update_rrd_data(
if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
- mkdir(RRDDIR "/pve2-node", 0755);
+ mkdir(RRDDIR "/pve8-node", 0755);
int argcount = sizeof(rrd_def_node)/sizeof(void*) - 1;
create_rrd_file(filename, argcount, rrd_def_node);
}
- } else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
+ } else if ((strncmp(key, "pve8-vm/", 8) == 0) ||
(strncmp(key, "pve2.3-vm/", 10) == 0)) {
const char *vmid;
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics
2023-12-11 14:47 ` [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics Folke Gleumes
@ 2023-12-12 7:49 ` DERUMIER, Alexandre
0 siblings, 0 replies; 6+ messages in thread
From: DERUMIER, Alexandre @ 2023-12-12 7:49 UTC (permalink / raw)
To: pve-devel
Hi,
if we are going to bump the rrd version,
I think it could be great to add pressure stats too.
I have send patches 2 year ago, it can be really usefull to implement
vm balancing/scheduling/node eviction or even simply to diplay graph
to have more information about load.
https://lists.proxmox.com/pipermail/pve-devel/2022-May/053068.html
This patch was with pressure stats in separate rrd, but if you are
bumping the main rrd version, it could be directly integrated.
I can rework the patch if you want.
-------- Message initial --------
De: Folke Gleumes <f.gleumes@proxmox.com>
Répondre à: Proxmox VE development discussion <pve-
devel@lists.proxmox.com>
À: pve-devel@lists.proxmox.com
Objet: [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and
arc size to memory statistics
Date: 11/12/2023 15:47:18
adding values to the rrd format break compatability with the old file.
Therfore the filename/path had to be changed as well.
Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
src/pmxcfs/status.c | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/pmxcfs/status.c b/src/pmxcfs/status.c
index dc44464..8834644 100644
--- a/src/pmxcfs/status.c
+++ b/src/pmxcfs/status.c
@@ -1130,7 +1130,10 @@ static const char *rrd_def_node[] = {
"DS:cpu:GAUGE:120:0:U",
"DS:iowait:GAUGE:120:0:U",
"DS:memtotal:GAUGE:120:0:U",
+ "DS:memfree:GAUGE:120:0:U",
"DS:memused:GAUGE:120:0:U",
+ "DS:membuffercache:GAUGE:120:0:U",
+ "DS:arcsize:GAUGE:120:0:U",
"DS:swaptotal:GAUGE:120:0:U",
"DS:swapused:GAUGE:120:0:U",
"DS:roottotal:GAUGE:120:0:U",
@@ -1252,7 +1255,7 @@ update_rrd_data(
int skip = 0;
- if (strncmp(key, "pve2-node/", 10) == 0) {
+ if (strncmp(key, "pve8-node/", 10) == 0) {
const char *node = key + 10;
skip = 2;
@@ -1267,12 +1270,12 @@ update_rrd_data(
if (!g_file_test(filename, G_FILE_TEST_EXISTS)) {
- mkdir(RRDDIR "/pve2-node", 0755);
+ mkdir(RRDDIR "/pve8-node", 0755);
int argcount =
sizeof(rrd_def_node)/sizeof(void*) - 1;
create_rrd_file(filename, argcount,
rrd_def_node);
}
- } else if ((strncmp(key, "pve2-vm/", 8) == 0) ||
+ } else if ((strncmp(key, "pve8-vm/", 8) == 0) ||
(strncmp(key, "pve2.3-vm/", 10) == 0)) {
const char *vmid;
^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [RFC common 2/4] add more detailed statistics to memory report
2023-12-11 14:47 [pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics Folke Gleumes
@ 2023-12-11 14:47 ` Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC widget-toolkit 3/4] add option to display rrd graph as stacking Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC manager 4/4] split ram usage into usage types Folke Gleumes
3 siblings, 0 replies; 6+ messages in thread
From: Folke Gleumes @ 2023-12-11 14:47 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
src/PVE/ProcFSTools.pm | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/src/PVE/ProcFSTools.pm b/src/PVE/ProcFSTools.pm
index 3826fcc..5f5d768 100644
--- a/src/PVE/ProcFSTools.pm
+++ b/src/PVE/ProcFSTools.pm
@@ -279,7 +279,10 @@ sub read_meminfo {
my $res = {
memtotal => 0,
memfree => 0,
+ memavailable => 0,
memused => 0,
+ membuffers => 0,
+ memcached => 0,
memshared => 0,
swaptotal => 0,
swapfree => 0,
@@ -299,7 +302,13 @@ sub read_meminfo {
close($fh);
$res->{memtotal} = $d->{memtotal};
- $res->{memfree} = $d->{memfree} + $d->{buffers} + $d->{cached};
+ $res->{memfree} = $d->{memfree};
+ $res->{membuffers} = $d->{buffers};
+ $res->{memcached} = $d->{cached};
+ $res->{memshared} = $d->{memshared};
+
+ # memfree is unused memory, memused is everything that is in use
+ # be it by caches, buffers or applications
$res->{memused} = $res->{memtotal} - $res->{memfree};
$res->{swaptotal} = $d->{swaptotal};
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [RFC widget-toolkit 3/4] add option to display rrd graph as stacking
2023-12-11 14:47 [pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC cluster 1/4] rrd: add free, buffer/cache and arc size to memory statistics Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC common 2/4] add more detailed statistics to memory report Folke Gleumes
@ 2023-12-11 14:47 ` Folke Gleumes
2023-12-11 14:47 ` [pve-devel] [RFC manager 4/4] split ram usage into usage types Folke Gleumes
3 siblings, 0 replies; 6+ messages in thread
From: Folke Gleumes @ 2023-12-11 14:47 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
src/panel/RRDChart.js | 59 +++++++++++++++++++++++++++++++++----------
1 file changed, 46 insertions(+), 13 deletions(-)
diff --git a/src/panel/RRDChart.js b/src/panel/RRDChart.js
index dc5940c..983437e 100644
--- a/src/panel/RRDChart.js
+++ b/src/panel/RRDChart.js
@@ -118,9 +118,14 @@ Ext.define('Proxmox.widget.RRDChart', {
if (view.fieldTitles && view.fieldTitles[view.fields.indexOf(item.field)]) {
prefix = view.fieldTitles[view.fields.indexOf(item.field)];
}
- let v = this.convertToUnits(record.get(item.field));
- let t = new Date(record.get('time'));
- tooltip.setHtml(`${prefix}: ${v}${suffix}<br>${t}`);
+ let value = record.get(item.field);
+ if (value !== null) {
+ let v = this.convertToUnits(record.get(item.field));
+ let t = new Date(record.get('time'));
+ tooltip.setHtml(`${prefix}: ${v}${suffix}<br>${t}`);
+ } else {
+ tooltip.setHtml('No Data');
+ }
},
onAfterAnimation: function(chart, eopts) {
@@ -256,18 +261,13 @@ Ext.define('Proxmox.widget.RRDChart', {
}
// add a series for each field we get
- me.fields.forEach(function(item, index) {
- let title = item;
- if (me.fieldTitles && me.fieldTitles[index]) {
- title = me.fieldTitles[index];
- }
+ if (me.stacking) {
me.addSeries(Ext.apply(
{
- type: 'line',
+ type: 'area',
xField: 'time',
- yField: item,
- title: title,
- fill: true,
+ yField: me.fields,
+ title: me.fieldTitles,
style: {
lineWidth: 1.5,
opacity: 0.60,
@@ -287,7 +287,40 @@ Ext.define('Proxmox.widget.RRDChart', {
},
me.seriesConfig,
));
- });
+ } else {
+ me.fields.forEach(function(item, index) {
+ let title = item;
+ if (me.fieldTitles && me.fieldTitles[index]) {
+ title = me.fieldTitles[index];
+ }
+ me.addSeries(Ext.apply(
+ {
+ type: 'line',
+ xField: 'time',
+ yField: item,
+ title: title,
+ fill: true,
+ style: {
+ lineWidth: 1.5,
+ opacity: 0.60,
+ },
+ marker: {
+ opacity: 0,
+ scaling: 0.01,
+ },
+ highlightCfg: {
+ opacity: 1,
+ scaling: 1.5,
+ },
+ tooltip: {
+ trackMouse: true,
+ renderer: 'onSeriesTooltipRender',
+ },
+ },
+ me.seriesConfig,
+ ));
+ });
+ }
// enable animation after the store is loaded
me.store.onAfter('load', function() {
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [RFC manager 4/4] split ram usage into usage types
2023-12-11 14:47 [pve-devel] [RFC cluster/common/widget-toolkit/manager 0/4] split node memory graph into usage types Folke Gleumes
` (2 preceding siblings ...)
2023-12-11 14:47 ` [pve-devel] [RFC widget-toolkit 3/4] add option to display rrd graph as stacking Folke Gleumes
@ 2023-12-11 14:47 ` Folke Gleumes
3 siblings, 0 replies; 6+ messages in thread
From: Folke Gleumes @ 2023-12-11 14:47 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Folke Gleumes <f.gleumes@proxmox.com>
---
PVE/API2/Nodes.pm | 6 +++---
PVE/API2Tools.pm | 2 +-
PVE/Service/pvestatd.pm | 15 +++++++++++----
www/manager6/node/Summary.js | 6 ++++--
4 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 3619190d..7dd8e5c2 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -430,7 +430,7 @@ __PACKAGE__->register_method({
$res->{memory} = {
free => $meminfo->{memfree},
total => $meminfo->{memtotal},
- used => $meminfo->{memused},
+ used => $meminfo->{memused} - ($meminfo->{memcached} + $meminfo->{membuffers} + $meminfo->{arcsize}),
};
$res->{ksm} = {
@@ -715,7 +715,7 @@ __PACKAGE__->register_method({
my ($param) = @_;
return PVE::RRD::create_rrd_graph(
- "pve2-node/$param->{node}", $param->{timeframe},
+ "pve8-node/$param->{node}", $param->{timeframe},
$param->{ds}, $param->{cf});
}});
@@ -757,7 +757,7 @@ __PACKAGE__->register_method({
my ($param) = @_;
return PVE::RRD::create_rrd_data(
- "pve2-node/$param->{node}", $param->{timeframe}, $param->{cf});
+ "pve8-node/$param->{node}", $param->{timeframe}, $param->{cf});
}});
__PACKAGE__->register_method({
diff --git a/PVE/API2Tools.pm b/PVE/API2Tools.pm
index a56eb732..f5ddff6e 100644
--- a/PVE/API2Tools.pm
+++ b/PVE/API2Tools.pm
@@ -49,7 +49,7 @@ sub extract_node_stats {
status => 'unknown',
};
- if (my $d = $rrd->{"pve2-node/$node"}) {
+ if (my $d = $rrd->{"pve8-node/$node"}) {
if (!$members || # no cluster
($members->{$node} && $members->{$node}->{online})) {
diff --git a/PVE/Service/pvestatd.pm b/PVE/Service/pvestatd.pm
index 2515120c..3fed4247 100755
--- a/PVE/Service/pvestatd.pm
+++ b/PVE/Service/pvestatd.pm
@@ -29,6 +29,7 @@ use PVE::AccessControl;
use PVE::Ceph::Services;
use PVE::Ceph::Tools;
use PVE::pvecfg;
+use Data::Dumper;
use PVE::ExtMetric;
use PVE::Status::Plugin;
@@ -177,13 +178,19 @@ sub update_node_status {
my $ctime = time();
+ my $memused = $meminfo->{memtotal} - $meminfo->{memfree};
+ my $mem_non_cache = $memused - ($meminfo->{memcached} + $meminfo->{membuffers} + $meminfo->{arcsize});
+
my $data = $generate_rrd_string->(
[$uptime, $sublevel, $ctime, $avg1, $maxcpu, $stat->{cpu}, $stat->{wait},
- $meminfo->{memtotal}, $meminfo->{memused},
- $meminfo->{swaptotal}, $meminfo->{swapused},
- $dinfo->{blocks}, $dused, $netin, $netout]
+ $meminfo->{memtotal}, $meminfo->{memfree},
+ $mem_non_cache,
+ $meminfo->{membuffers} + $meminfo->{memcached},
+ $meminfo->{arcsize}, $meminfo->{swaptotal},
+ $meminfo->{swapused}, $dinfo->{blocks},
+ $dused, $netin, $netout]
);
- PVE::Cluster::broadcast_rrd("pve2-node/$nodename", $data);
+ PVE::Cluster::broadcast_rrd("pve8-node/$nodename", $data);
my $node_metric = {
uptime => $uptime,
diff --git a/www/manager6/node/Summary.js b/www/manager6/node/Summary.js
index c2dca0df..d7bb8815 100644
--- a/www/manager6/node/Summary.js
+++ b/www/manager6/node/Summary.js
@@ -174,11 +174,13 @@ Ext.define('PVE.node.Summary', {
{
xtype: 'proxmoxRRDChart',
title: gettext('Memory usage'),
- fields: ['memtotal', 'memused'],
- fieldTitles: [gettext('Total'), gettext('RAM usage')],
+ fields: ['memused', 'membuffercache', 'arcsize', 'memfree'],
+ fieldTitles: [gettext('Used'), gettext('Buffer/Cache'), 'ZFS Arc', gettext('Free')],
unit: 'bytes',
powerOfTwo: true,
store: rrdstore,
+ colors: ['#115fa6', '#a61187', '#ff8809', '#94ae0a'],
+ stacking: true,
},
{
xtype: 'proxmoxRRDChart',
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread