From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 0BF411FF0C0 for ; Wed, 18 Nov 2026 12:08:20 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id DBB3921774; Wed, 18 Nov 2026 12:07:32 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH ha-manager 05/15] env: pve2: delegate cluster stat gathering to scheduling library Date: Wed, 18 Nov 2026 12:05:12 +0100 Message-ID: <20261118110705.595989-6-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118110705.595989-1-t.lamprecht@proxmox.com> References: <20261118110705.595989-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795000030535 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.826 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: FY3QR6QYOWB6SNV6IMJQHEZ5XPTGXLBO X-Message-ID-Hash: FY3QR6QYOWB6SNV6IMJQHEZ5XPTGXLBO X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The cluster node and guest stat gathering now lives in the PVE::ResourceScheduling::Stats module of libpve-guest-common-perl. Consume it here instead of keeping a second copy, so the HA manager and the resource scheduling API share a single implementation and cannot drift apart. The library's dynamic guest stats carry the host memory accounting from fix #7974, and the stats the HA manager sees stay the same. The library has no dependency on the HA resource plugins, so build the static per-guest-type usage resolvers from those plugins and pass them in. As the library reports undecodable static node info as a plain warning, log that at error level like before. Signed-off-by: Thomas Lamprecht --- NOTE: the new libpve-guest-common-perl dependency needs a version requirement for the first release with PVE::ResourceScheduling. debian/control | 2 + src/PVE/HA/Env/PVE2.pm | 155 ++++++++--------------------------------- 2 files changed, 32 insertions(+), 125 deletions(-) diff --git a/debian/control b/debian/control index 107e367..2577d43 100644 --- a/debian/control +++ b/debian/control @@ -8,6 +8,7 @@ Build-Depends: debhelper-compat (= 13), libpve-access-control, libpve-cluster-perl, libpve-common-perl, + libpve-guest-common-perl, libpve-notify-perl, libpve-rs-perl (>= 0.14.0), libtest-mockmodule-perl , @@ -23,6 +24,7 @@ Architecture: any Depends: libjson-perl, libpve-cluster-perl (>= 9.1.5), libpve-common-perl, + libpve-guest-common-perl, libpve-notify-perl (>= 8.1.0), libpve-rs-perl (>= 0.14.0), pve-cluster (>= 3.0-17), diff --git a/src/PVE/HA/Env/PVE2.pm b/src/PVE/HA/Env/PVE2.pm index 5f195da..de56e20 100644 --- a/src/PVE/HA/Env/PVE2.pm +++ b/src/PVE/HA/Env/PVE2.pm @@ -14,6 +14,7 @@ use PVE::DataCenterConfig; use PVE::INotify; use PVE::RPCEnvironment; use PVE::Notify; +use PVE::ResourceScheduling::Stats; use PVE::HA::Tools ':exit_codes'; use PVE::HA::Env; @@ -36,30 +37,19 @@ PVE::HA::Rules::ResourceAffinity->register(); PVE::HA::Rules->init(property_isolation => 1); +# The stats library has no dependency on the HA resource plugins, so build the per-guest-type static +# usage resolvers from those plugins and pass them in. +my sub static_stats_resolvers { + return { + map { + my $plugin = PVE::HA::Resources->lookup($_); + ($_ => sub { $plugin->get_static_stats_from_config($_[0]) }); + } qw(vm ct) + }; +} + my $lockdir = "/etc/pve/priv/lock"; -# rrd entry indices for VM and CT guests -# taken from PVE::Service::pvestatd::update_{lxc,qemu}_status() -use constant { - RRD_VM_INDEX_STATUS => 2, - RRD_VM_INDEX_MAXCPU => 5, - RRD_VM_INDEX_CPU => 6, - RRD_VM_INDEX_MAXMEM => 7, - RRD_VM_INDEX_MEM => 8, - RRD_VM_INDEX_MEMHOST => 15, -}; - -# rrd entry indices for PVE nodes -# taken from PVE::Service::pvestatd::update_node_status() -use constant { - RRD_NODE_INDEX_MAXCPU => 4, - RRD_NODE_INDEX_CPU => 5, - RRD_NODE_INDEX_MAXMEM => 7, - RRD_NODE_INDEX_MEM => 8, -}; - -my $HOSTNAME_RE = qr/(?:[a-zA-Z0-9](?:[a-zA-Z0-9\-]{,61}?[a-zA-Z0-9])?)/; - sub new { my ($this, $nodename) = @_; @@ -529,133 +519,48 @@ sub get_datacenter_settings { }; } -my sub get_cluster_service_stats { - my $vmlist = PVE::Cluster::get_vmlist(); - my $idlist = $vmlist->{ids} // {}; - - my $rrd = PVE::Cluster::rrd_dump(); - - my $stats = {}; - for my $id (keys %$idlist) { - my $type = eval { PVE::HA::Tools::get_ha_resource_type($idlist->{$id}->{type}) }; - next if $@; # silently ignore unknown pve types - - my $sid = "$type:$id"; - my $nodename = $idlist->{$id}->{node}; - - my $rrdentry = $rrd->{"pve-vm-9.0/$id"} // []; - # can be any QMP RunState, but 'running' is the only active VM state - my $status = $rrdentry->[RRD_VM_INDEX_STATUS] // "stopped"; - my $state = $status eq "running" ? "started" : "stopped"; - - $stats->{$sid} = { - id => $id, - node => $nodename, - state => $state, - running => $state eq 'started', - type => $type, - usage => {}, - }; - } - - return $stats; -} - sub get_basic_service_stats { my ($self) = @_; - return get_cluster_service_stats(); + return PVE::ResourceScheduling::Stats::get_cluster_guest_stats(); } sub get_static_service_stats { my ($self) = @_; - my $properties = ['cores', 'cpulimit', 'memory', 'sockets', 'vcpus']; - my $stats = get_cluster_service_stats(); - my $confs = PVE::Cluster::get_guest_config_properties($properties); - - for my $sid (keys %$stats) { - my ($id, $type) = $stats->{$sid}->@{qw(id type)}; - - my $conf = $confs->{$id} // {}; - my $plugin = PVE::HA::Resources->lookup($type); - - $stats->{$sid}->{usage} = $plugin->get_static_stats_from_config($conf); - } - - return $stats; + return PVE::ResourceScheduling::Stats::get_static_guest_stats(static_stats_resolvers()); } sub get_dynamic_service_stats { my ($self) = @_; - my $rrd = PVE::Cluster::rrd_dump(); - - my $stats = get_cluster_service_stats(); - for my $sid (keys %$stats) { - my $id = $stats->{$sid}->{id}; - my $rrdentry = $rrd->{"pve-vm-9.0/$id"} // []; - - # NOTE the guests' broadcasted vmstatus() caps maxcpu at the node's maxcpu - my $maxcpu = ($rrdentry->[RRD_VM_INDEX_MAXCPU] || 0.0) + 0.0; - - # designated fallback for containers - my $mem = $rrdentry->[RRD_VM_INDEX_MEMHOST] // $rrdentry->[RRD_VM_INDEX_MEM]; - - $stats->{$sid}->{usage} = { - maxcpu => $maxcpu, - cpu => (($rrdentry->[RRD_VM_INDEX_CPU] || 0.0) + 0.0) * $maxcpu, - maxmem => int($rrdentry->[RRD_VM_INDEX_MAXMEM] || 0), - mem => int($mem || 0), - }; - } - - return $stats; + return PVE::ResourceScheduling::Stats::get_dynamic_guest_stats(); } sub get_static_node_stats { my ($self) = @_; - my $stats = PVE::Cluster::get_node_kv('static-info'); - for my $node (keys $stats->%*) { - $stats->{$node} = eval { - my $node_stats = decode_json($stats->{$node}); + # The library reports undecodable static node info with warn(). Log that at 'err' level, as an + # online node without static info makes the CRM fall back to basic scheduling. + my $previous = $SIG{__WARN__}; + local $SIG{__WARN__} = sub { + my ($msg) = @_; + if ($msg =~ m/^unable to decode static node info /) { + $self->log('err', $msg); + } elsif (ref($previous) eq 'CODE') { + $previous->($msg); + } else { + warn $msg; + } + }; - return { - maxcpu => $node_stats->{cpus}, - maxmem => $node_stats->{memory}, - }; - }; - $self->log('err', "unable to decode static node info for '$node' - $@") if $@; - } - - return $stats; + return PVE::ResourceScheduling::Stats::get_static_node_stats(); } sub get_dynamic_node_stats { my ($self) = @_; - my $rrd = PVE::Cluster::rrd_dump(); - - my $stats = {}; - for my $key (keys %$rrd) { - my ($nodename) = $key =~ m/^pve-node-9.0\/($HOSTNAME_RE)$/; - - next if !$nodename; - - my $rrdentry = $rrd->{$key} // []; - - my $maxcpu = int($rrdentry->[RRD_NODE_INDEX_MAXCPU] || 0); - - $stats->{$nodename} = { - maxcpu => $maxcpu, - cpu => (($rrdentry->[RRD_NODE_INDEX_CPU] || 0.0) + 0.0) * $maxcpu, - maxmem => int($rrdentry->[RRD_NODE_INDEX_MAXMEM] || 0), - mem => int($rrdentry->[RRD_NODE_INDEX_MEM] || 0), - }; - } - - return $stats; + return PVE::ResourceScheduling::Stats::get_dynamic_node_stats(); } sub get_node_version { -- 2.47.3