From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 510021FF0B3 for ; Wed, 11 Nov 2026 10:35:18 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 5A50B216E3; Wed, 11 Nov 2026 10:34:49 +0100 (CET) From: Jakob Klocker To: pve-devel@lists.proxmox.com Subject: [PATCH pve-manager v2 2/4] fix #6464: vzdump: add retry strategies for failed guest backups Date: Wed, 11 Nov 2026 10:34:05 +0100 Message-ID: <20261111093407.109872-3-j.klocker@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261111093407.109872-1-j.klocker@proxmox.com> References: <20261111093407.109872-1-j.klocker@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794389681922 X-SPAM-LEVEL: Spam detection results: 0 AWL 1.326 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: PT5SELS6WDNB7FUOCDYCUK3OPJEXGYP2 X-Message-ID-Hash: PT5SELS6WDNB7FUOCDYCUK3OPJEXGYP2 X-MailFrom: j.klocker@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add the `error-behavior` and `max-retries` options to automatically retry failed guest backups, which helps with transient failures. Supported retry strategies: - `ignore`: Preserves current behavior (no retries, default). - `retry-immediately`: Retries the guest inline after failure. - `retry`: Defers failed guests until all other guests have been processed, then retries them in the order they failed. A guest is retried at most `max-retries` times. Between attempts, the delay starts at 30 seconds and doubles with each retry, capped at 30 minutes as suggested on bugzilla [0]. If a guest still fails after the last retry, it is counted as an error and the job finishes with errors. If a retry succeeds, the guest is reported as successful in the notifications and no error is counted. The task log is opened in append mode for retries, so the log contains the complete transcript of all attempts. [0]: https://bugzilla.proxmox.com/show_bug.cgi?id=6464#c5 Link: https://bugzilla.proxmox.com/show_bug.cgi?id=6464 Signed-off-by: Jakob Klocker --- PVE/VZDump.pm | 131 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 2 deletions(-) diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm index 56d2aea3..e4aa303d 100644 --- a/PVE/VZDump.pm +++ b/PVE/VZDump.pm @@ -10,6 +10,7 @@ use File::Path; use IO::File; use IO::Select; use IPC::Open3; +use List::Util qw(min); use POSIX qw(strftime); use Time::Local; @@ -1048,7 +1049,8 @@ sub exec_backup_task { -d $task->{tmpdir} || die "unable to create temporary directory '$task->{tmpdir}'"; - $logfd = IO::File->new(">$tmplog") + my $logmode = $task->{retries} ? '>>' : '>'; + $logfd = IO::File->new("${logmode}${tmplog}") || die "unable to create log file '$tmplog'"; $task->{dumpdir} = $opts->{dumpdir}; @@ -1435,14 +1437,139 @@ sub exec_backup { my $starttime = time(); my $errcount = 0; + + my $error_behavior = $opts->{'error-behavior'} // 'ignore'; + my $max_retries = $opts->{'max-retries'} // 5; + my $min_between_retry = 30; + my $max_wait = 60 * 30; + + my $get_backoff = sub { + my ($retry) = @_; + return min($min_between_retry * 2**($retry - 1), $max_wait); + }; + eval { $self->{'backup-provider'}->job_init($starttime) if $self->{'backup-provider'}; $self->run_hook_script('job-start', undef, $job_start_fd); + my $failed_tasks = {}; + foreach my $task (@$tasklist) { + my $vmid = $task->{vmid}; + my $initial_task = {%$task}; + + $task->{retries} = 0; $self->exec_backup_task($task); - $errcount += 1 if $task->{state} ne 'ok'; + + next if $task->{state} eq 'ok'; + + if ($error_behavior eq 'retry-immediately') { + for my $retry (1 .. $max_retries) { + my $backoff = $get_backoff->($retry); + + debugmsg( + 'info', + "Retrying backup for guest $vmid (attempt $retry/$max_retries) after ${backoff}s delay...", + undef, + 1, + ); + sleep($backoff); + + %$task = (%$initial_task, retries => $retry); + $self->exec_backup_task($task); + + last if $task->{state} eq 'ok'; + } + + } elsif ($error_behavior eq 'retry') { + $failed_tasks->{$vmid} = { + task => $task, + initial_task => $initial_task, + fail_time => time(), + retries => 0, + }; + next; + } + + if ($task->{state} ne 'ok') { + debugmsg('err', "Guest $vmid failed after $max_retries retries.", undef, 1) + if $error_behavior eq 'retry-immediately'; + $errcount++; + } + } + + if ($error_behavior eq 'retry') { + while (keys %$failed_tasks > 0) { + my $now = time(); + my $min_sleep = $max_wait; + my $ready_to_run = []; + + my @sorted_vmids = sort { + $failed_tasks->{$a}->{fail_time} <=> $failed_tasks->{$b}->{fail_time} + || $a <=> $b + } keys %$failed_tasks; + + foreach my $vmid (@sorted_vmids) { + my $fail_info = $failed_tasks->{$vmid}; + + if ($fail_info->{retries} >= $max_retries) { + debugmsg( + 'err', + "Guest $vmid failed after $max_retries retries.", + undef, + 1, + ); + $errcount++; + delete $failed_tasks->{$vmid}; + next; + } + + my $delay = $get_backoff->($fail_info->{retries} + 1); + my $target_time = $fail_info->{fail_time} + $delay; + my $wait_time = $target_time - $now; + + if ($wait_time > 0) { + $min_sleep = $wait_time if $wait_time < $min_sleep; + } else { + push @$ready_to_run, $vmid; + } + } + + if (scalar(@$ready_to_run) == 0 && keys %$failed_tasks > 0) { + debugmsg( + 'info', + "Waiting $min_sleep seconds before next backup retry...", + undef, + 1, + ); + sleep($min_sleep); + next; + } + + foreach my $vmid (@$ready_to_run) { + my $fail_info = $failed_tasks->{$vmid}; + + $fail_info->{retries}++; + debugmsg( + 'info', + "Retrying backup for guest $vmid (attempt $fail_info->{retries}/$max_retries)", + undef, + 1, + ); + + my $task = $fail_info->{task}; + %$task = + (%{ $fail_info->{initial_task} }, retries => $fail_info->{retries}); + $self->exec_backup_task($task); + + if ($task->{state} eq 'ok') { + delete $failed_tasks->{$vmid}; + } else { + $fail_info->{fail_time} = time(); + } + } + } } $self->{'backup-provider'}->job_cleanup() if $self->{'backup-provider'}; -- 2.47.3