From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id F1A1F96C1B for ; Tue, 16 Apr 2024 14:21:06 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BB1DC1A5D9 for ; Tue, 16 Apr 2024 14:21:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 16 Apr 2024 14:21:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id D3FCD44FFA for ; Tue, 16 Apr 2024 14:21:05 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Tue, 16 Apr 2024 14:20:44 +0200 Message-Id: <20240416122054.733817-10-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240416122054.733817-1-f.gruenbichler@proxmox.com> References: <20240416122054.733817-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.056 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v2 guest-common 1/1] helpers: add pool limit/usage helpers X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 16 Apr 2024 12:21:07 -0000 one for combining the per-node broadcasted values, one for checking a pool's limit, and one specific helper for checking guest-related actions such as starting a VM. Signed-off-by: Fabian Grünbichler --- Notes: v2: - style - introduce new helper for mapping limit key to usage hash - introduce new helper for default usage hash - avoid hard-coding cpu/mem and run/config where sensible src/PVE/GuestHelpers.pm | 183 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/src/PVE/GuestHelpers.pm b/src/PVE/GuestHelpers.pm index 961a7b8..e52eaf0 100644 --- a/src/PVE/GuestHelpers.pm +++ b/src/PVE/GuestHelpers.pm @@ -416,4 +416,187 @@ sub check_vnet_access { if !($tag || $trunks); } +sub pool_limit_to_usage { + my ($limit_key) = @_; + + my ($resource, $kind) = split(/-/, $limit_key, 2); + + return ($resource, $kind, $kind eq 'run' ? 1 : 0); +} + +sub pool_default_usage { + my $default = {}; + + for my $limit (keys $PVE::AccessControl::pool_limits_desc->%*) { + my ($resource, $kind) = pool_limit_to_usage($limit); + $default->{$resource}->{$kind} = 0; + } + + return $default; +} + +# combines the broadcasted pool usage information to get per-pool stats +# +# $pools parsed pool info from user.cfg +# $usage broadcasted KV hash +# $pool filter for specific pool +# $skip skip a certain guest to ignore its current usage +# +# returns usage hash: +# pool -> cpu/mem/.. -> run/config -> $usage +sub get_pool_usage { + my ($pools, $usage, $pool, $skip) = @_; + + my $res = {}; + my $included_guests = {}; + for my $node (keys $usage->%*) { + my $node_usage = JSON::decode_json($usage->{$node} // ''); + + # long IDs first, so we can add children to their parents right away + for my $poolid (sort {$b cmp $a} keys $pools->%*) { + if ( + defined($pool) + && !($pool eq $poolid || $poolid =~ m!^$pool/! || $pool =~ m!^$poolid/!) + ) { + next; + } + + my $d = $res->{$poolid} //= pool_default_usage(); + + my $pool_usage = $node_usage->{data}->{$poolid} // {}; + for my $vmid (keys $pool_usage->%*) { + # only include once in case of migration between broadcast + next if $included_guests->{$vmid}; + next if $skip && $skip->{$vmid}; + $included_guests->{$vmid} = 1; + + my $vm_data = $pool_usage->{$vmid}; + for my $key (keys $vm_data->%*) { + next if $key eq 'running'; + $d->{$key}->{run} += $vm_data->{$key}->{run} if $vm_data->{running}; + $d->{$key}->{config} += $vm_data->{$key}->{config}; + } + } + + if (my $parent = $pools->{$poolid}->{parent}) { + $res->{$parent} //= pool_default_usage(); + for my $key (keys $d->%*) { + for my $kind (keys $d->{$key}->%*) { + $res->{$parent}->{$key}->{$kind} = $d->{$key}->{$kind}; + } + } + } + } + } + + return $res; +} + +# checks whether a pool is (or would be) over its resource limits +# +# $changes is for checking limits for config/state changes like VM starts, if +# set, only the limits with changes are checked (see check_guest_pool_limit) +# +# return value indicates whether any limit was overstepped or not (if $noerr is set) +sub check_pool_limits { + my ($usage, $limits, $noerr, $changes) = @_; + + my $over = {}; + my $only_changed = defined($changes); + + my $check_limit = sub { + my ($key, $running, $limit, $change) = @_; + + return if $only_changed && $change == 0; + + my $kind = $running ? 'run' : 'config'; + + my $value = $usage->{$key}->{$kind}; + $value = int($value); + $value += $change; + $value = $value / (1024*1024) if $key eq 'mem'; + if ($limit < $value) { + $over->{$key}->{$kind}->{change} = $change if $change; + $over->{$key}->{$kind}->{over} = 1; + } + }; + + my $get_change = sub { + my ($key, $running) = @_; + + return 0 if !defined($changes); + + my $check_running = defined($changes->{running}) && $changes->{running} ? 1 : 0; + + if ($running == $check_running) { + return $changes->{$key} // 0; + } else { + return 0; + } + }; + + while (my ($key, $limit) = each $limits->%*) { + my ($resource, $kind, $running) = pool_limit_to_usage($key); + my $change = $get_change->($resource, $running); + $check_limit->($resource, $running, $limit, $change); + } + + if (!$noerr) { + my $msg = ''; + for my $key (keys $over->%*) { + for my $kind (keys $over->{$key}->%*) { + my $value = $usage->{$key}->{$kind}; + $value = $value / (1024*1024) if $key eq 'mem'; + my $change = $over->{$key}->{$kind}->{change}; + if ($change) { + $change = $change / (1024*1024) if $key eq 'mem'; + $value = "$value + $change" if $change; + } + my $limit = $limits->{"$key-$kind"}; + $msg .= "($kind) $key: $value over $limit, "; + } + } + if ($msg) { + $msg =~ s/, $//; + die "pool limits exhausted: $msg\n"; + } + } + + return $over->%* ? 1 : 0; +} + +# checks whether the given changes for a certain guest would overstep a pool limit +# +# $changes is an optional hash containing +# - absolute: flag whether changes are relative or absolute +# - running: flag whether the config or running limits should be checked +# - cpu: change value for cpu limit +# - mem: change value for mem limit +# all elements are optional +# +# if no $changes is provided, the limits are checked against the current usage +# +# $poolid allows overriding the guest's pool membership, for example in case it +# is not yet properly set when creating the guest +sub check_guest_pool_limit { + my ($vmid, $changes, $poolid) = @_; + + my $user_cfg = PVE::Cluster::cfs_read_file("user.cfg"); + + $poolid = $user_cfg->{vms}->{$vmid} if !defined($poolid); + if ($poolid) { + my $pool = $user_cfg->{pools}->{$poolid}; + + my $limits = $pool->{limits}; + return if !$limits; + + my $skip = {}; + $skip->{$vmid} = 1 if $changes && $changes->{absolute}; + my $usage = PVE::Cluster::get_node_kv('pool-usage'); + + $usage = get_pool_usage($user_cfg->{pools}, $usage, $poolid, $skip); + check_pool_limits($usage->{$poolid}, $limits, 0, $changes); + } +} + 1; -- 2.39.2