[PATCH manager 17/19] api: backup info: add endpoints for cluster-wide backup slots

[Thomas Lamprecht <t.lamprecht@proxmox.com>]

List the guest backups that hold or wait for cluster-wide backup
slots, to see where the slots went when guest backups wait.

Slots of an offline node stay taken, as its guest backups might still
be running. Allow releasing them once it is clear that these backups
stopped, for example because the node is powered off. Otherwise the
node releases them itself when it is back online.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 PVE/API2/Cluster/BackupInfo.pm | 106 ++++++++++++++++++++++++++++++++-
 PVE/VZDump/ClusterSlots.pm     |  14 +++++
 2 files changed, 118 insertions(+), 2 deletions(-)

diff --git a/PVE/API2/Cluster/BackupInfo.pm b/PVE/API2/Cluster/BackupInfo.pm
index 4ef59ac3d..d5e14a3c9 100644
--- a/PVE/API2/Cluster/BackupInfo.pm
+++ b/PVE/API2/Cluster/BackupInfo.pm
@@ -9,10 +9,11 @@ use PVE::Tools qw(extract_param);
 use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file);
 use PVE::RESTHandler;
 use PVE::RPCEnvironment;
-use PVE::JSONSchema;
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::Storage;
 use PVE::Exception qw(raise_param_exc);
 use PVE::VZDump;
+use PVE::VZDump::ClusterSlots;
 use PVE::VZDump::Common;
 
 use base qw(PVE::RESTHandler);
@@ -57,11 +58,112 @@ __PACKAGE__->register_method({
     },
     code => sub {
         return [
-            { subdir => 'not-backed-up' },
+            { subdir => 'not-backed-up' }, { subdir => 'slots' },
         ];
     },
 });
 
+__PACKAGE__->register_method({
+    name => 'get_backup_slots',
+    path => 'slots',
+    method => 'GET',
+    protected => 1,
+    description => "Shows guest backups that hold or wait for cluster-wide backup slots.",
+    permissions => {
+        description => "Only guests with 'VM.Audit' permissions are listed.",
+        check => ['perm', '/', ['Sys.Audit']],
+    },
+    parameters => {
+        additionalProperties => 0,
+        properties => {},
+    },
+    returns => {
+        type => 'array',
+        items => {
+            type => 'object',
+            properties => {
+                node => get_standard_option('pve-node'),
+                vmid => get_standard_option('pve-vmid'),
+                upid => {
+                    type => 'string',
+                    description => 'Backup task the guest backup belongs to.',
+                },
+                state => {
+                    type => 'string',
+                    enum => ['running', 'waiting'],
+                    description =>
+                        'Whether the guest backup holds its slots or waits for them.',
+                },
+                since => {
+                    type => 'integer',
+                    description =>
+                        'When the guest backup got its slots or started waiting for them.',
+                },
+                storages => {
+                    type => 'array',
+                    items => get_standard_option('pve-storage-id'),
+                    description =>
+                        'Shared storages with a backup concurrency limit that the guest'
+                        . ' backup uses.',
+                },
+            },
+        },
+    },
+    code => sub {
+        my $rpcenv = PVE::RPCEnvironment::get();
+        my $user = $rpcenv->get_user();
+
+        my $state = PVE::VZDump::ClusterSlots::read_state();
+
+        my $result = [];
+        for my $kind (qw(holders waiters)) {
+            for my $entry (values $state->{$kind}->%*) {
+                next if !$rpcenv->check($user, "/vms/$entry->{vmid}", ['VM.Audit'], 1);
+
+                push $result->@*,
+                    {
+                        node => $entry->{node},
+                        vmid => int($entry->{vmid}),
+                        upid => $entry->{upid},
+                        state => $kind eq 'holders' ? 'running' : 'waiting',
+                        since => int($kind eq 'holders' ? $entry->{granted} : $entry->{queued}),
+                        storages => [map { m!^storage/(.+)$! ? $1 : () } $entry->{resources}->@*],
+                    };
+            }
+        }
+
+        return $result;
+    },
+});
+
+__PACKAGE__->register_method({
+    name => 'release_backup_slots',
+    path => 'slots/{node}',
+    method => 'DELETE',
+    protected => 1,
+    description =>
+        "Release the cluster-wide backup slots of an offline node. They stay taken because its"
+        . " guest backups might still be running, so only release them once you are sure that"
+        . " these stopped, for example because the node is powered off.",
+    permissions => {
+        check => ['perm', '/', ['Sys.Modify']],
+    },
+    parameters => {
+        additionalProperties => 0,
+        properties => {
+            node => get_standard_option('pve-node'),
+        },
+    },
+    returns => { type => 'null' },
+    code => sub {
+        my ($param) = @_;
+
+        PVE::VZDump::ClusterSlots::release_node($param->{node});
+
+        return;
+    },
+});
+
 __PACKAGE__->register_method({
     name => 'get_guests_not_in_backup',
     path => 'not-backed-up',
diff --git a/PVE/VZDump/ClusterSlots.pm b/PVE/VZDump/ClusterSlots.pm
index f4cac71c7..962b270a0 100644
--- a/PVE/VZDump/ClusterSlots.pm
+++ b/PVE/VZDump/ClusterSlots.pm
@@ -282,4 +282,18 @@ sub cleanup_local_entries {
     });
 }
 
+sub release_node {
+    my ($node) = @_;
+
+    update_state(sub {
+        my ($state) = @_;
+
+        # checked under the lock, so the node cannot rejoin and take new slots meanwhile
+        die "cannot release backup slots of node '$node' - it is online\n"
+            if get_members()->{$node}->{online};
+
+        remove_entries($state, sub { $_[0]->{node} eq $node });
+    });
+}
+
 1;
-- 
2.47.3