From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH guest-common 1/1] helpers: add pool limit/usage helpers
Date: Wed, 10 Apr 2024 15:13:06 +0200 [thread overview]
Message-ID: <20240410131316.1208679-10-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20240410131316.1208679-1-f.gruenbichler@proxmox.com>
one for combining the per-node broadcasted values, one for checking a pool's
limit, and one specific helper for checking guest-related actions such as
starting a VM.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/PVE/GuestHelpers.pm | 190 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 190 insertions(+)
diff --git a/src/PVE/GuestHelpers.pm b/src/PVE/GuestHelpers.pm
index 961a7b8..36b44bb 100644
--- a/src/PVE/GuestHelpers.pm
+++ b/src/PVE/GuestHelpers.pm
@@ -416,4 +416,194 @@ sub check_vnet_access {
if !($tag || $trunks);
}
+# combines the broadcasted pool usage information to get per-pool stats
+#
+# $pools parsed pool info from user.cfg
+# $usage broadcasted KV hash
+# $pool filter for specific pool
+# $skip skip a certain guest to ignore its current usage
+#
+# returns usage hash:
+# pool -> cpu/mem/.. -> run/config -> $usage
+sub get_pool_usage {
+ my ($pools, $usage, $pool, $skip) = @_;
+
+ my $res = {};
+ my $included_guests = {};
+ for my $node (keys $usage->%*) {
+ my $node_usage = JSON::decode_json($usage->{$node} // '');
+
+ # long IDs first, so we can add children to their parents right away
+ for my $poolid (sort {$b cmp $a} keys $pools->%*) {
+ next if defined($pool) && !($pool eq $poolid || $poolid =~ m!^$pool/! || $pool =~ m!^$poolid/!);
+
+ my $d = $res->{$poolid} //= {
+ cpu => {
+ run => 0,
+ config => 0,
+ },
+ mem => {
+ run => 0,
+ config => 0,
+ },
+ };
+
+ my $pool_usage = $node_usage->{data}->{$poolid} // {};
+ for my $vmid (keys $pool_usage->%*) {
+ # only include once in case of migration between broadcast
+ next if $included_guests->{$vmid};
+ next if $skip && $skip->{$vmid};
+ $included_guests->{$vmid} = 1;
+
+ my $vm_data = $pool_usage->{$vmid};
+ for my $key (keys $vm_data->%*) {
+ next if $key eq 'running';
+ $d->{$key}->{run} += $vm_data->{$key}->{run} if $vm_data->{running};
+ $d->{$key}->{config} += $vm_data->{$key}->{config};
+ }
+ }
+
+ if (my $parent = $pools->{$poolid}->{parent}) {
+ $res->{$parent} //= {
+ cpu => {
+ run=> 0,
+ config => 0,
+ },
+ mem => {
+ run => 0,
+ config => 0,
+ },
+ };
+ $res->{$parent}->{cpu}->{run} += $d->{cpu}->{run};
+ $res->{$parent}->{mem}->{run} += $d->{mem}->{run};
+ $res->{$parent}->{cpu}->{config} += $d->{cpu}->{config};
+ $res->{$parent}->{mem}->{config} += $d->{mem}->{config};
+ }
+ }
+ }
+
+ return $res;
+}
+
+# checks whether a pool is (or would be) over its resource limits
+#
+# $changes is for checking limits for config/state changes like VM starts, if
+# set, only the limits with changes are checked (see check_guest_pool_limit)
+#
+# return value indicates whether any limit was overstepped or not (if $noerr is set)
+sub check_pool_limits {
+ my ($usage, $limits, $noerr, $changes) = @_;
+
+ my $over = {};
+ my $only_changed = defined($changes);
+
+ my $check_limit = sub {
+ my ($key, $running, $limit, $change) = @_;
+
+ return if $only_changed && $change == 0;
+
+ my $kind = $running ? 'run' : 'config';
+
+ my $value = $usage->{$key}->{$kind};
+ $value = int($value);
+ $value += $change;
+ $value = $value / (1024*1024) if $key eq 'mem';
+ if ($limit < $value) {
+ $over->{$key}->{$kind}->{change} = $change if $change;
+ $over->{$key}->{$kind}->{over} = 1;
+ }
+ };
+
+ my $get_change = sub {
+ my ($key, $running) = @_;
+
+ return 0 if !defined($changes);
+
+ my $check_running = defined($changes->{running}) && $changes->{running} ? 1 : 0;
+
+ if ($running == $check_running) {
+ return $changes->{$key} // 0;
+ } else {
+ return 0;
+ }
+ };
+
+ if (my $limit = $limits->{"mem-run"}) {
+ my $change = $get_change->('mem', 1);
+ $check_limit->('mem', 1, $limit, $change);
+ }
+
+ if (my $limit = $limits->{"mem-config"}) {
+ my $change = $get_change->('mem', 0);
+ $check_limit->('mem', 0, $limit, $change);
+ }
+
+ if (my $limit = $limits->{"cpu-run"}) {
+ my $change = $get_change->('cpu', 1);
+ $check_limit->('cpu', 1, $limit, $change);
+ }
+
+ if (my $limit = $limits->{"cpu-config"}) {
+ my $change = $get_change->('cpu', 0);
+ $check_limit->('cpu', 0, $limit, $change);
+ }
+
+ if (!$noerr) {
+ my $msg = '';
+ for my $key (keys $over->%*) {
+ for my $kind (keys $over->{$key}->%*) {
+ my $value = $usage->{$key}->{$kind};
+ $value = $value / (1024*1024) if $key eq 'mem';
+ my $change = $over->{$key}->{$kind}->{change};
+ if ($change) {
+ $change = $change / (1024*1024) if $key eq 'mem';
+ $value = "$value + $change" if $change;
+ }
+ my $limit = $limits->{"$key-$kind"};
+ $msg .= "($kind) $key: $value over $limit, ";
+ }
+ }
+ if ($msg) {
+ $msg =~ s/, $//;
+ die "pool limits exhausted: $msg\n";
+ }
+ }
+
+ return $over->%* ? 1 : 0;
+}
+
+# checks whether the given changes for a certain guest would overstep a pool limit
+#
+# $changes is an optional hash containing
+# - absolute: flag whether changes are relative or absolute
+# - running: flag whether the config or running limits should be checked
+# - cpu: change value for cpu limit
+# - mem: change value for mem limit
+# all elements are optional
+#
+# if no $changes is provided, the limits are checked against the current usage
+#
+# $poolid allows overriding the guest's pool membership, for example in case it
+# is not yet properly set when creating the guest
+sub check_guest_pool_limit {
+ my ($vmid, $changes, $poolid) = @_;
+
+ my $user_cfg = PVE::Cluster::cfs_read_file("user.cfg");
+
+ $poolid = $user_cfg->{vms}->{$vmid} if !defined($poolid);
+ if ($poolid) {
+ my $pool = $user_cfg->{pools}->{$poolid};
+
+ my $limits = $pool->{limits};
+ return if !$limits;
+
+ my $skip = {};
+ $skip->{$vmid} = 1 if $changes && $changes->{absolute};
+ my $usage = PVE::Cluster::get_node_kv('pool-usage');
+
+ $usage = PVE::GuestHelpers::get_pool_usage($user_cfg->{pools}, $usage, $poolid, $skip);
+ check_pool_limits($usage->{$poolid}, $limits, 0, $changes);
+ }
+}
+
1;
--
2.39.2
next prev parent reply other threads:[~2024-04-10 13:13 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-04-10 13:12 [pve-devel] [RFC qemu-server/pve-container/.. 0/19] pool resource limits Fabian Grünbichler
2024-04-10 13:12 ` [pve-devel] [PATCH access-control 1/1] pools: define " Fabian Grünbichler
2024-04-10 13:12 ` [pve-devel] [PATCH container 1/7] config: add pool usage helper Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 2/7] status: add pool usage fields Fabian Grünbichler
2024-04-11 9:28 ` Wolfgang Bumiller
2024-04-15 9:32 ` Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 3/7] create/restore/clone: handle pool limits Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 4/7] start: " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 5/7] hotplug: " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 6/7] rollback: " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH container 7/7] update: " Fabian Grünbichler
2024-04-11 7:23 ` Fabian Grünbichler
2024-04-11 10:03 ` Wolfgang Bumiller
2024-04-15 9:35 ` Fabian Grünbichler
2024-04-10 13:13 ` Fabian Grünbichler [this message]
2024-04-11 9:17 ` [pve-devel] [PATCH guest-common 1/1] helpers: add pool limit/usage helpers Wolfgang Bumiller
2024-04-15 9:38 ` Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH manager 1/4] api: pools: add limits management Fabian Grünbichler
2024-04-11 9:24 ` Wolfgang Bumiller
2024-04-10 13:13 ` [pve-devel] [PATCH manager 2/4] pvestatd: collect and broadcast pool usage Fabian Grünbichler
2024-04-11 9:32 ` Wolfgang Bumiller
2024-04-15 12:36 ` Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH manager 3/4] api: return pool usage when queried Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH manager 4/4] ui: add pool limits and usage Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 1/6] config: add pool usage helper Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 2/6] vmstatus: add usage values for pool limits Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 3/6] create/restore/clone: handle " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 4/6] update/hotplug: " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 5/6] start: " Fabian Grünbichler
2024-04-10 13:13 ` [pve-devel] [PATCH qemu-server 6/6] rollback: " Fabian Grünbichler
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240410131316.1208679-10-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox