From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 9B3501FF0BC for ; Mon, 16 Nov 2026 11:15:03 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 110CE21808; Mon, 16 Nov 2026 11:14:26 +0100 (CET) From: Jakob Klocker To: pve-devel@lists.proxmox.com Subject: [PATCH pve-manager v3 2/4] fix #6464: vzdump: add retry strategies for failed guest backups Date: Mon, 16 Nov 2026 11:13:42 +0100 Message-ID: <20261116101344.140523-3-j.klocker@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116101344.140523-1-j.klocker@proxmox.com> References: <20261116101344.140523-1-j.klocker@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794824059628 X-SPAM-LEVEL: Spam detection results: 0 AWL 1.262 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: W26KTFTFYTZF4FFKXUA6UCFGMAKUYA6S X-Message-ID-Hash: W26KTFTFYTZF4FFKXUA6UCFGMAKUYA6S X-MailFrom: j.klocker@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add the `error-behavior` and `max-retries` options to automatically retry failed guest backups, which helps with transient failures. Supported retry strategies: - `ignore`: Preserves current behavior (no retries, default). - `retry-immediately`: Retries the guest inline after failure. - `retry`: Defers failed guests until all other guests have been processed, then retries them in the order they failed. A guest is retried at most `max-retries` times. Between attempts, the delay starts at 30 seconds and doubles with each retry, capped at 30 minutes as suggested on bugzilla [0]. If a guest still fails after the last retry, it is counted as an error and the job finishes with errors. If a retry succeeds, the task finishes with a warning and the notification is sent with severity `warning`, showing the number of retries in the subject and guest table. Guests whose archive was already created (e.g. failing backup-end hook) are not retried, to avoid duplicate archives and repeated pruning. Retries are not possible with --stdout, as a retry would corrupt the stream. Setting `error-behavior` to a retry strategy together with --stdout is rejected, and a retry strategy from /etc/vzdump.conf is ignored with a warning. The task log is kept across retries and contains all attempts. [0]: https://bugzilla.proxmox.com/show_bug.cgi?id=6464#c5 Link: https://bugzilla.proxmox.com/show_bug.cgi?id=6464 Signed-off-by: Jakob Klocker --- PVE/VZDump.pm | 219 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 215 insertions(+), 4 deletions(-) diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm index d2861f9e..d0f98a13 100644 --- a/PVE/VZDump.pm +++ b/PVE/VZDump.pm @@ -10,6 +10,7 @@ use File::Path; use IO::File; use IO::Select; use IPC::Open3; +use List::Util qw(min); use POSIX qw(strftime); use Time::Local; @@ -20,6 +21,7 @@ use PVE::HA::Config; use PVE::HA::Env::PVE2; use PVE::JSONSchema qw(get_standard_option); use PVE::Notify; +use PVE::RESTEnvironment qw(log_warn); use PVE::RPCEnvironment; use PVE::Storage; use PVE::VZDump::Common; @@ -395,6 +397,11 @@ my sub read_backup_task_logs { return $task_logs; } +my sub retries_text { + my ($count) = @_; + return $count == 1 ? '1 retry' : "$count retries"; +} + my sub build_guest_table { my ($task_list) = @_; @@ -436,11 +443,18 @@ my sub build_guest_table { my $successful = $task->{state} eq 'ok'; my $size = $successful ? $task->{size} : 0; my $filename = $successful ? $task->{target} : undef; + + my $status = $task->{state}; + my $retries = $task->{retries}; + if ($successful && $retries) { + $status .= " (after " . retries_text($retries) . ")"; + } + push @{ $table->{data} }, { "vmid" => int($task->{vmid}), "name" => $task->{hostname}, - "status" => $task->{state}, + "status" => $status, "time" => int($task->{backuptime}), "size" => int($size), "filename" => $filename, @@ -502,8 +516,11 @@ sub send_notification { my ($error_count, $total_size) = aggregate_task_statistics($tasklist); my $failed = ($error_count || $err); + my $retried_count = grep { $_->{state} eq 'ok' && $_->{retries} } @$tasklist; my $status_text = $failed ? 'backup failed' : 'backup successful'; + $status_text .= " ($retried_count guest(s) needed retries)" + if !$failed && $retried_count; if ($err) { if ($err =~ /\n/) { @@ -540,7 +557,7 @@ sub send_notification { # Add backup-job metadata field in case this is a backup job. $fields->{'job-id'} = $job_id if $job_id; - my $severity = $failed ? "error" : "info"; + my $severity = $failed ? "error" : $retried_count ? "warning" : "info"; my $email_configured = $mailto && scalar(@$mailto); if (($mode eq 'auto' && $email_configured) || $mode eq 'legacy-sendmail') { @@ -1048,7 +1065,9 @@ sub exec_backup_task { -d $task->{tmpdir} || die "unable to create temporary directory '$task->{tmpdir}'"; - $logfd = IO::File->new(">$tmplog") + # keep the log of earlier attempts of this job + my $logmode = defined($task->{tmplog}) ? '>>' : '>'; + $logfd = IO::File->new("${logmode}${tmplog}") || die "unable to create log file '$tmplog'"; $task->{dumpdir} = $opts->{dumpdir}; @@ -1216,6 +1235,9 @@ sub exec_backup_task { debugmsg('info', "archive file size: $cs", $logfd); } + # The backup exists from here on, later failures must not trigger a retry + $task->{committed} = 1; + # Mark as protected before pruning. if (my $storeid = $opts->{storage}) { my $volname = @@ -1445,14 +1467,193 @@ sub exec_backup { my $starttime = time(); my $errcount = 0; + my $retried_count = 0; + + my $error_behavior = $opts->{'error-behavior'} // 'ignore'; + if ($opts->{stdout} && $error_behavior ne 'ignore') { + log_warn( + "ignoring 'error-behavior' from vzdump.conf - retries are not possible with 'stdout'"); + $error_behavior = 'ignore'; + } + + my $max_retries = $opts->{'max-retries'} // 5; + my $min_between_retry = 30; + my $max_wait = 60 * 30; + + my $get_backoff = sub { + my ($retry) = @_; + return min($min_between_retry * 2**($retry - 1), $max_wait); + }; + + my $log_committed_failure = sub { + my ($vmid) = @_; + debugmsg( + 'err', + "Backup of guest $vmid was created, but a later step failed - not retrying", + undef, + 1, + ); + }; + + my $log_retried_success = sub { + my ($vmid, $retries) = @_; + log_warn("Backup of guest $vmid succeeded after " . retries_text($retries)); + $retried_count++; + }; + eval { $self->{'backup-provider'}->job_init($starttime) if $self->{'backup-provider'}; $self->run_hook_script('job-start', undef, $job_start_fd); + my $failed_tasks = {}; + foreach my $task (@$tasklist) { + my $vmid = $task->{vmid}; + my $initial_task = {%$task}; + + $task->{retries} = 0; $self->exec_backup_task($task); - $errcount += 1 if $task->{state} ne 'ok'; + + next if $task->{state} eq 'ok'; + + if ($error_behavior ne 'ignore' && $task->{committed}) { + $log_committed_failure->($vmid); + $errcount++; + next; + } + + if ($error_behavior eq 'retry-immediately') { + for my $retry (1 .. $max_retries) { + my $backoff = $get_backoff->($retry); + + debugmsg( + 'info', + "Retrying backup for guest $vmid (retry $retry/$max_retries)" + . " after ${backoff}s delay...", + undef, + 1, + ); + sleep($backoff); + + %$task = (%$initial_task, retries => $retry, tmplog => $task->{tmplog}); + $self->exec_backup_task($task); + + if ($task->{state} eq 'ok') { + $log_retried_success->($vmid, $retry); + last; + } + last if $task->{committed}; + } + + } elsif ($error_behavior eq 'retry') { + $failed_tasks->{$vmid} = { + task => $task, + initial_task => $initial_task, + fail_time => time(), + retries => 0, + }; + next; + } + + if ($task->{state} ne 'ok') { + if ($error_behavior eq 'retry-immediately') { + if ($task->{committed}) { + $log_committed_failure->($vmid); + } else { + debugmsg( + 'err', + "Guest $vmid failed after " . retries_text($max_retries), + undef, + 1, + ); + } + } + $errcount++; + } + } + + if ($error_behavior eq 'retry') { + while (%$failed_tasks) { + my $now = time(); + my $min_sleep = $max_wait; + my $ready_to_run = []; + + my @sorted_vmids = sort { + $failed_tasks->{$a}->{fail_time} <=> $failed_tasks->{$b}->{fail_time} + || $a <=> $b + } keys %$failed_tasks; + + for my $vmid (@sorted_vmids) { + my $fail_info = $failed_tasks->{$vmid}; + + if ($fail_info->{retries} >= $max_retries) { + debugmsg( + 'err', + "Guest $vmid failed after " . retries_text($max_retries), + undef, + 1, + ); + $errcount++; + delete $failed_tasks->{$vmid}; + next; + } + + my $delay = $get_backoff->($fail_info->{retries} + 1); + my $target_time = $fail_info->{fail_time} + $delay; + my $wait_time = $target_time - $now; + + if ($wait_time > 0) { + $min_sleep = $wait_time if $wait_time < $min_sleep; + } else { + push @$ready_to_run, $vmid; + } + } + + if (scalar(@$ready_to_run) == 0 && %$failed_tasks) { + debugmsg( + 'info', + "Waiting $min_sleep seconds before next backup retry...", + undef, + 1, + ); + sleep($min_sleep); + next; + } + + for my $vmid (@$ready_to_run) { + my $fail_info = $failed_tasks->{$vmid}; + + $fail_info->{retries}++; + debugmsg( + 'info', + "Retrying backup for guest $vmid" + . " (retry $fail_info->{retries}/$max_retries)", + undef, + 1, + ); + + my $task = $fail_info->{task}; + my $tmplog = $task->{tmplog}; + %$task = ( + %{ $fail_info->{initial_task} }, + retries => $fail_info->{retries}, + tmplog => $tmplog, + ); + $self->exec_backup_task($task); + + if ($task->{state} eq 'ok') { + $log_retried_success->($vmid, $fail_info->{retries}); + delete $failed_tasks->{$vmid}; + } elsif ($task->{committed}) { + $log_committed_failure->($vmid); + $errcount++; + delete $failed_tasks->{$vmid}; + } else { + $fail_info->{fail_time} = time(); + } + } + } } $self->{'backup-provider'}->job_cleanup() if $self->{'backup-provider'}; @@ -1472,6 +1673,13 @@ sub exec_backup { } else { if ($errcount) { debugmsg('info', "Backup job finished with errors", undef, 1); + } elsif ($retried_count) { + debugmsg( + 'info', + "Backup job finished with warnings - $retried_count guest(s) needed retries", + undef, + 1, + ); } else { debugmsg('info', "Backup job finished successfully", undef, 1); } @@ -1547,6 +1755,9 @@ sub verify_vzdump_parameters { raise_param_exc({ pool => "option conflicts with option 'vmid'" }) if $param->{pool} && $param->{vmid}; + raise_param_exc({ 'error-behavior' => "retries are not possible with option 'stdout'" }) + if $param->{stdout} && ($param->{'error-behavior'} // 'ignore') ne 'ignore'; + $parse_prune_backups->($param, 'CLI parameters'); parse_fleecing($param); parse_performance($param); -- 2.47.3