From: Jakob Klocker <j.klocker@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-manager v3 2/4] fix #6464: vzdump: add retry strategies for failed guest backups
Date: Mon, 16 Nov 2026 11:13:42 +0100 [thread overview]
Message-ID: <20261116101344.140523-3-j.klocker@proxmox.com> (raw)
In-Reply-To: <20261116101344.140523-1-j.klocker@proxmox.com>
Add the `error-behavior` and `max-retries` options to automatically
retry failed guest backups, which helps with transient failures.
Supported retry strategies:
- `ignore`: Preserves current behavior (no retries, default).
- `retry-immediately`: Retries the guest inline after failure.
- `retry`: Defers failed guests until all other guests have been
processed, then retries them in the order they failed.
A guest is retried at most `max-retries` times. Between attempts, the
delay starts at 30 seconds and doubles with each retry, capped at 30
minutes as suggested on bugzilla [0]. If a guest still fails after the
last retry, it is counted as an error and the job finishes with errors.
If a retry succeeds, the task finishes with a warning and the
notification is sent with severity `warning`, showing the number of
retries in the subject and guest table.
Guests whose archive was already created (e.g. failing backup-end hook)
are not retried, to avoid duplicate archives and repeated pruning.
Retries are not possible with --stdout, as a retry would corrupt the
stream. Setting `error-behavior` to a retry strategy together with
--stdout is rejected, and a retry strategy from /etc/vzdump.conf is
ignored with a warning.
The task log is kept across retries and contains all attempts.
[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=6464#c5
Link: https://bugzilla.proxmox.com/show_bug.cgi?id=6464
Signed-off-by: Jakob Klocker <j.klocker@proxmox.com>
---
PVE/VZDump.pm | 219 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 215 insertions(+), 4 deletions(-)
diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index d2861f9e..d0f98a13 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -10,6 +10,7 @@ use File::Path;
use IO::File;
use IO::Select;
use IPC::Open3;
+use List::Util qw(min);
use POSIX qw(strftime);
use Time::Local;
@@ -20,6 +21,7 @@ use PVE::HA::Config;
use PVE::HA::Env::PVE2;
use PVE::JSONSchema qw(get_standard_option);
use PVE::Notify;
+use PVE::RESTEnvironment qw(log_warn);
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::VZDump::Common;
@@ -395,6 +397,11 @@ my sub read_backup_task_logs {
return $task_logs;
}
+my sub retries_text {
+ my ($count) = @_;
+ return $count == 1 ? '1 retry' : "$count retries";
+}
+
my sub build_guest_table {
my ($task_list) = @_;
@@ -436,11 +443,18 @@ my sub build_guest_table {
my $successful = $task->{state} eq 'ok';
my $size = $successful ? $task->{size} : 0;
my $filename = $successful ? $task->{target} : undef;
+
+ my $status = $task->{state};
+ my $retries = $task->{retries};
+ if ($successful && $retries) {
+ $status .= " (after " . retries_text($retries) . ")";
+ }
+
push @{ $table->{data} },
{
"vmid" => int($task->{vmid}),
"name" => $task->{hostname},
- "status" => $task->{state},
+ "status" => $status,
"time" => int($task->{backuptime}),
"size" => int($size),
"filename" => $filename,
@@ -502,8 +516,11 @@ sub send_notification {
my ($error_count, $total_size) = aggregate_task_statistics($tasklist);
my $failed = ($error_count || $err);
+ my $retried_count = grep { $_->{state} eq 'ok' && $_->{retries} } @$tasklist;
my $status_text = $failed ? 'backup failed' : 'backup successful';
+ $status_text .= " ($retried_count guest(s) needed retries)"
+ if !$failed && $retried_count;
if ($err) {
if ($err =~ /\n/) {
@@ -540,7 +557,7 @@ sub send_notification {
# Add backup-job metadata field in case this is a backup job.
$fields->{'job-id'} = $job_id if $job_id;
- my $severity = $failed ? "error" : "info";
+ my $severity = $failed ? "error" : $retried_count ? "warning" : "info";
my $email_configured = $mailto && scalar(@$mailto);
if (($mode eq 'auto' && $email_configured) || $mode eq 'legacy-sendmail') {
@@ -1048,7 +1065,9 @@ sub exec_backup_task {
-d $task->{tmpdir}
|| die "unable to create temporary directory '$task->{tmpdir}'";
- $logfd = IO::File->new(">$tmplog")
+ # keep the log of earlier attempts of this job
+ my $logmode = defined($task->{tmplog}) ? '>>' : '>';
+ $logfd = IO::File->new("${logmode}${tmplog}")
|| die "unable to create log file '$tmplog'";
$task->{dumpdir} = $opts->{dumpdir};
@@ -1216,6 +1235,9 @@ sub exec_backup_task {
debugmsg('info', "archive file size: $cs", $logfd);
}
+ # The backup exists from here on, later failures must not trigger a retry
+ $task->{committed} = 1;
+
# Mark as protected before pruning.
if (my $storeid = $opts->{storage}) {
my $volname =
@@ -1445,14 +1467,193 @@ sub exec_backup {
my $starttime = time();
my $errcount = 0;
+ my $retried_count = 0;
+
+ my $error_behavior = $opts->{'error-behavior'} // 'ignore';
+ if ($opts->{stdout} && $error_behavior ne 'ignore') {
+ log_warn(
+ "ignoring 'error-behavior' from vzdump.conf - retries are not possible with 'stdout'");
+ $error_behavior = 'ignore';
+ }
+
+ my $max_retries = $opts->{'max-retries'} // 5;
+ my $min_between_retry = 30;
+ my $max_wait = 60 * 30;
+
+ my $get_backoff = sub {
+ my ($retry) = @_;
+ return min($min_between_retry * 2**($retry - 1), $max_wait);
+ };
+
+ my $log_committed_failure = sub {
+ my ($vmid) = @_;
+ debugmsg(
+ 'err',
+ "Backup of guest $vmid was created, but a later step failed - not retrying",
+ undef,
+ 1,
+ );
+ };
+
+ my $log_retried_success = sub {
+ my ($vmid, $retries) = @_;
+ log_warn("Backup of guest $vmid succeeded after " . retries_text($retries));
+ $retried_count++;
+ };
+
eval {
$self->{'backup-provider'}->job_init($starttime) if $self->{'backup-provider'};
$self->run_hook_script('job-start', undef, $job_start_fd);
+ my $failed_tasks = {};
+
foreach my $task (@$tasklist) {
+ my $vmid = $task->{vmid};
+ my $initial_task = {%$task};
+
+ $task->{retries} = 0;
$self->exec_backup_task($task);
- $errcount += 1 if $task->{state} ne 'ok';
+
+ next if $task->{state} eq 'ok';
+
+ if ($error_behavior ne 'ignore' && $task->{committed}) {
+ $log_committed_failure->($vmid);
+ $errcount++;
+ next;
+ }
+
+ if ($error_behavior eq 'retry-immediately') {
+ for my $retry (1 .. $max_retries) {
+ my $backoff = $get_backoff->($retry);
+
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid (retry $retry/$max_retries)"
+ . " after ${backoff}s delay...",
+ undef,
+ 1,
+ );
+ sleep($backoff);
+
+ %$task = (%$initial_task, retries => $retry, tmplog => $task->{tmplog});
+ $self->exec_backup_task($task);
+
+ if ($task->{state} eq 'ok') {
+ $log_retried_success->($vmid, $retry);
+ last;
+ }
+ last if $task->{committed};
+ }
+
+ } elsif ($error_behavior eq 'retry') {
+ $failed_tasks->{$vmid} = {
+ task => $task,
+ initial_task => $initial_task,
+ fail_time => time(),
+ retries => 0,
+ };
+ next;
+ }
+
+ if ($task->{state} ne 'ok') {
+ if ($error_behavior eq 'retry-immediately') {
+ if ($task->{committed}) {
+ $log_committed_failure->($vmid);
+ } else {
+ debugmsg(
+ 'err',
+ "Guest $vmid failed after " . retries_text($max_retries),
+ undef,
+ 1,
+ );
+ }
+ }
+ $errcount++;
+ }
+ }
+
+ if ($error_behavior eq 'retry') {
+ while (%$failed_tasks) {
+ my $now = time();
+ my $min_sleep = $max_wait;
+ my $ready_to_run = [];
+
+ my @sorted_vmids = sort {
+ $failed_tasks->{$a}->{fail_time} <=> $failed_tasks->{$b}->{fail_time}
+ || $a <=> $b
+ } keys %$failed_tasks;
+
+ for my $vmid (@sorted_vmids) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ if ($fail_info->{retries} >= $max_retries) {
+ debugmsg(
+ 'err',
+ "Guest $vmid failed after " . retries_text($max_retries),
+ undef,
+ 1,
+ );
+ $errcount++;
+ delete $failed_tasks->{$vmid};
+ next;
+ }
+
+ my $delay = $get_backoff->($fail_info->{retries} + 1);
+ my $target_time = $fail_info->{fail_time} + $delay;
+ my $wait_time = $target_time - $now;
+
+ if ($wait_time > 0) {
+ $min_sleep = $wait_time if $wait_time < $min_sleep;
+ } else {
+ push @$ready_to_run, $vmid;
+ }
+ }
+
+ if (scalar(@$ready_to_run) == 0 && %$failed_tasks) {
+ debugmsg(
+ 'info',
+ "Waiting $min_sleep seconds before next backup retry...",
+ undef,
+ 1,
+ );
+ sleep($min_sleep);
+ next;
+ }
+
+ for my $vmid (@$ready_to_run) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ $fail_info->{retries}++;
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid"
+ . " (retry $fail_info->{retries}/$max_retries)",
+ undef,
+ 1,
+ );
+
+ my $task = $fail_info->{task};
+ my $tmplog = $task->{tmplog};
+ %$task = (
+ %{ $fail_info->{initial_task} },
+ retries => $fail_info->{retries},
+ tmplog => $tmplog,
+ );
+ $self->exec_backup_task($task);
+
+ if ($task->{state} eq 'ok') {
+ $log_retried_success->($vmid, $fail_info->{retries});
+ delete $failed_tasks->{$vmid};
+ } elsif ($task->{committed}) {
+ $log_committed_failure->($vmid);
+ $errcount++;
+ delete $failed_tasks->{$vmid};
+ } else {
+ $fail_info->{fail_time} = time();
+ }
+ }
+ }
}
$self->{'backup-provider'}->job_cleanup() if $self->{'backup-provider'};
@@ -1472,6 +1673,13 @@ sub exec_backup {
} else {
if ($errcount) {
debugmsg('info', "Backup job finished with errors", undef, 1);
+ } elsif ($retried_count) {
+ debugmsg(
+ 'info',
+ "Backup job finished with warnings - $retried_count guest(s) needed retries",
+ undef,
+ 1,
+ );
} else {
debugmsg('info', "Backup job finished successfully", undef, 1);
}
@@ -1547,6 +1755,9 @@ sub verify_vzdump_parameters {
raise_param_exc({ pool => "option conflicts with option 'vmid'" })
if $param->{pool} && $param->{vmid};
+ raise_param_exc({ 'error-behavior' => "retries are not possible with option 'stdout'" })
+ if $param->{stdout} && ($param->{'error-behavior'} // 'ignore') ne 'ignore';
+
$parse_prune_backups->($param, 'CLI parameters');
parse_fleecing($param);
parse_performance($param);
--
2.47.3
next prev parent reply other threads:[~2026-11-16 10:15 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 10:13 [PATCH docs/guest-common/manager v3 0/4] fix #6464: backup: add backup retry strategies Jakob Klocker
2026-11-16 10:13 ` [PATCH pve-guest-common v3 1/4] vzdump: add error-behavior and max-retries options Jakob Klocker
2026-11-16 10:13 ` Jakob Klocker [this message]
2026-11-16 10:13 ` [PATCH pve-manager v3 3/4] ui: backup: add error behavior and retry options Jakob Klocker
2026-11-16 10:13 ` [PATCH pve-docs v3 4/4] vzdump: document backup " Jakob Klocker
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116101344.140523-3-j.klocker@proxmox.com \
--to=j.klocker@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox