From: Jakob Klocker <j.klocker@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-manager v3 2/4] fix #6464: vzdump: add retry strategies for failed guest backups
Date: Mon, 16 Nov 2026 11:13:42 +0100 [thread overview]
Message-ID: <20261116101344.140523-3-j.klocker@proxmox.com> (raw)
In-Reply-To: <20261116101344.140523-1-j.klocker@proxmox.com>
Add the `error-behavior` and `max-retries` options to automatically
retry failed guest backups, which helps with transient failures.
Supported retry strategies:
- `ignore`: Preserves current behavior (no retries, default).
- `retry-immediately`: Retries the guest inline after failure.
- `retry`: Defers failed guests until all other guests have been
processed, then retries them in the order they failed.
A guest is retried at most `max-retries` times. Between attempts, the
delay starts at 30 seconds and doubles with each retry, capped at 30
minutes as suggested on bugzilla [0]. If a guest still fails after the
last retry, it is counted as an error and the job finishes with errors.
If a retry succeeds, the task finishes with a warning and the
notification is sent with severity `warning`, showing the number of
retries in the subject and guest table.
Guests whose archive was already created (e.g. failing backup-end hook)
are not retried, to avoid duplicate archives and repeated pruning.
Retries are not possible with --stdout, as a retry would corrupt the
stream. Setting `error-behavior` to a retry strategy together with
--stdout is rejected, and a retry strategy from /etc/vzdump.conf is
ignored with a warning.
The task log is kept across retries and contains all attempts.
[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=6464#c5
Link: https://bugzilla.proxmox.com/show_bug.cgi?id=6464
Signed-off-by: Jakob Klocker <j.klocker@proxmox.com>
---
PVE/VZDump.pm | 219 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 215 insertions(+), 4 deletions(-)
diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index d2861f9e..d0f98a13 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -10,6 +10,7 @@ use File::Path;
use IO::File;
use IO::Select;
use IPC::Open3;
+use List::Util qw(min);
use POSIX qw(strftime);
use Time::Local;
@@ -20,6 +21,7 @@ use PVE::HA::Config;
use PVE::HA::Env::PVE2;
use PVE::JSONSchema qw(get_standard_option);
use PVE::Notify;
+use PVE::RESTEnvironment qw(log_warn);
use PVE::RPCEnvironment;
use PVE::Storage;
use PVE::VZDump::Common;
@@ -395,6 +397,11 @@ my sub read_backup_task_logs {
return $task_logs;
}
+my sub retries_text {
+ my ($count) = @_;
+ return $count == 1 ? '1 retry' : "$count retries";
+}
+
my sub build_guest_table {
my ($task_list) = @_;
@@ -436,11 +443,18 @@ my sub build_guest_table {
my $successful = $task->{state} eq 'ok';
my $size = $successful ? $task->{size} : 0;
my $filename = $successful ? $task->{target} : undef;
+
+ my $status = $task->{state};
+ my $retries = $task->{retries};
+ if ($successful && $retries) {
+ $status .= " (after " . retries_text($retries) . ")";
+ }
+
push @{ $table->{data} },
{
"vmid" => int($task->{vmid}),
"name" => $task->{hostname},
- "status" => $task->{state},
+ "status" => $status,
"time" => int($task->{backuptime}),
"size" => int($size),
"filename" => $filename,
@@ -502,8 +516,11 @@ sub send_notification {
my ($error_count, $total_size) = aggregate_task_statistics($tasklist);
my $failed = ($error_count || $err);
+ my $retried_count = grep { $_->{state} eq 'ok' && $_->{retries} } @$tasklist;
my $status_text = $failed ? 'backup failed' : 'backup successful';
+ $status_text .= " ($retried_count guest(s) needed retries)"
+ if !$failed && $retried_count;
if ($err) {
if ($err =~ /\n/) {
@@ -540,7 +557,7 @@ sub send_notification {
# Add backup-job metadata field in case this is a backup job.
$fields->{'job-id'} = $job_id if $job_id;
- my $severity = $failed ? "error" : "info";
+ my $severity = $failed ? "error" : $retried_count ? "warning" : "info";
my $email_configured = $mailto && scalar(@$mailto);
if (($mode eq 'auto' && $email_configured) || $mode eq 'legacy-sendmail') {
@@ -1048,7 +1065,9 @@ sub exec_backup_task {
-d $task->{tmpdir}
|| die "unable to create temporary directory '$task->{tmpdir}'";
- $logfd = IO::File->new(">$tmplog")
+ # keep the log of earlier attempts of this job
+ my $logmode = defined($task->{tmplog}) ? '>>' : '>';
+ $logfd = IO::File->new("${logmode}${tmplog}")
|| die "unable to create log file '$tmplog'";
$task->{dumpdir} = $opts->{dumpdir};
@@ -1216,6 +1235,9 @@ sub exec_backup_task {
debugmsg('info', "archive file size: $cs", $logfd);
}
+ # The backup exists from here on, later failures must not trigger a retry
+ $task->{committed} = 1;
+
# Mark as protected before pruning.
if (my $storeid = $opts->{storage}) {
my $volname =
@@ -1445,14 +1467,193 @@ sub exec_backup {
my $starttime = time();
my $errcount = 0;
+ my $retried_count = 0;
+
+ my $error_behavior = $opts->{'error-behavior'} // 'ignore';
+ if ($opts->{stdout} && $error_behavior ne 'ignore') {
+ log_warn(
+ "ignoring 'error-behavior' from vzdump.conf - retries are not possible with 'stdout'");
+ $error_behavior = 'ignore';
+ }
+
+ my $max_retries = $opts->{'max-retries'} // 5;
+ my $min_between_retry = 30;
+ my $max_wait = 60 * 30;
+
+ my $get_backoff = sub {
+ my ($retry) = @_;
+ return min($min_between_retry * 2**($retry - 1), $max_wait);
+ };
+
+ my $log_committed_failure = sub {
+ my ($vmid) = @_;
+ debugmsg(
+ 'err',
+ "Backup of guest $vmid was created, but a later step failed - not retrying",
+ undef,
+ 1,
+ );
+ };
+
+ my $log_retried_success = sub {
+ my ($vmid, $retries) = @_;
+ log_warn("Backup of guest $vmid succeeded after " . retries_text($retries));
+ $retried_count++;
+ };
+
eval {
$self->{'backup-provider'}->job_init($starttime) if $self->{'backup-provider'};
$self->run_hook_script('job-start', undef, $job_start_fd);
+ my $failed_tasks = {};
+
foreach my $task (@$tasklist) {
+ my $vmid = $task->{vmid};
+ my $initial_task = {%$task};
+
+ $task->{retries} = 0;
$self->exec_backup_task($task);
- $errcount += 1 if $task->{state} ne 'ok';
+
+ next if $task->{state} eq 'ok';
+
+ if ($error_behavior ne 'ignore' && $task->{committed}) {
+ $log_committed_failure->($vmid);
+ $errcount++;
+ next;
+ }
+
+ if ($error_behavior eq 'retry-immediately') {
+ for my $retry (1 .. $max_retries) {
+ my $backoff = $get_backoff->($retry);
+
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid (retry $retry/$max_retries)"
+ . " after ${backoff}s delay...",
+ undef,
+ 1,
+ );
+ sleep($backoff);
+
+ %$task = (%$initial_task, retries => $retry, tmplog => $task->{tmplog});
+ $self->exec_backup_task($task);
+
+ if ($task->{state} eq 'ok') {
+ $log_retried_success->($vmid, $retry);
+ last;
+ }
+ last if $task->{committed};
+ }
+
+ } elsif ($error_behavior eq 'retry') {
+ $failed_tasks->{$vmid} = {
+ task => $task,
+ initial_task => $initial_task,
+ fail_time => time(),
+ retries => 0,
+ };
+ next;
+ }
+
+ if ($task->{state} ne 'ok') {
+ if ($error_behavior eq 'retry-immediately') {
+ if ($task->{committed}) {
+ $log_committed_failure->($vmid);
+ } else {
+ debugmsg(
+ 'err',
+ "Guest $vmid failed after " . retries_text($max_retries),
+ undef,
+ 1,
+ );
+ }
+ }
+ $errcount++;
+ }
+ }
+
+ if ($error_behavior eq 'retry') {
+ while (%$failed_tasks) {
+ my $now = time();
+ my $min_sleep = $max_wait;
+ my $ready_to_run = [];
+
+ my @sorted_vmids = sort {
+ $failed_tasks->{$a}->{fail_time} <=> $failed_tasks->{$b}->{fail_time}
+ || $a <=> $b
+ } keys %$failed_tasks;
+
+ for my $vmid (@sorted_vmids) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ if ($fail_info->{retries} >= $max_retries) {
+ debugmsg(
+ 'err',
+ "Guest $vmid failed after " . retries_text($max_retries),
+ undef,
+ 1,
+ );
+ $errcount++;
+ delete $failed_tasks->{$vmid};
+ next;
+ }
+
+ my $delay = $get_backoff->($fail_info->{retries} + 1);
+ my $target_time = $fail_info->{fail_time} + $delay;
+ my $wait_time = $target_time - $now;
+
+ if ($wait_time > 0) {
+ $min_sleep = $wait_time if $wait_time < $min_sleep;
+ } else {
+ push @$ready_to_run, $vmid;
+ }
+ }
+
+ if (scalar(@$ready_to_run) == 0 && %$failed_tasks) {
+ debugmsg(
+ 'info',
+ "Waiting $min_sleep seconds before next backup retry...",
+ undef,
+ 1,
+ );
+ sleep($min_sleep);
+ next;
+ }
+
+ for my $vmid (@$ready_to_run) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ $fail_info->{retries}++;
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid"
+ . " (retry $fail_info->{retries}/$max_retries)",
+ undef,
+ 1,
+ );
+
+ my $task = $fail_info->{task};
+ my $tmplog = $task->{tmplog};
+ %$task = (
+ %{ $fail_info->{initial_task} },
+ retries => $fail_info->{retries},
+ tmplog => $tmplog,
+ );
+ $self->exec_backup_task($task);
+
+ if ($task->{state} eq 'ok') {
+ $log_retried_success->($vmid, $fail_info->{retries});
+ delete $failed_tasks->{$vmid};
+ } elsif ($task->{committed}) {
+ $log_committed_failure->($vmid);
+ $errcount++;
+ delete $failed_tasks->{$vmid};
+ } else {
+ $fail_info->{fail_time} = time();
+ }
+ }
+ }
}
$self->{'backup-provider'}->job_cleanup() if $self->{'backup-provider'};
@@ -1472,6 +1673,13 @@ sub exec_backup {
} else {
if ($errcount) {
debugmsg('info', "Backup job finished with errors", undef, 1);
+ } elsif ($retried_count) {
+ debugmsg(
+ 'info',
+ "Backup job finished with warnings - $retried_count guest(s) needed retries",
+ undef,
+ 1,
+ );
} else {
debugmsg('info', "Backup job finished successfully", undef, 1);
}
@@ -1547,6 +1755,9 @@ sub verify_vzdump_parameters {
raise_param_exc({ pool => "option conflicts with option 'vmid'" })
if $param->{pool} && $param->{vmid};
+ raise_param_exc({ 'error-behavior' => "retries are not possible with option 'stdout'" })
+ if $param->{stdout} && ($param->{'error-behavior'} // 'ignore') ne 'ignore';
+
$parse_prune_backups->($param, 'CLI parameters');
parse_fleecing($param);
parse_performance($param);
--
2.47.3
next prev parent reply other threads:[~2026-11-16 10:15 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 10:13 [PATCH docs/guest-common/manager v3 0/4] fix #6464: backup: add backup retry strategies Jakob Klocker
2026-11-16 10:13 ` [PATCH pve-guest-common v3 1/4] vzdump: add error-behavior and max-retries options Jakob Klocker
2026-11-16 10:13 ` Jakob Klocker [this message]
2026-11-16 10:13 ` [PATCH pve-manager v3 3/4] ui: backup: add error behavior and retry options Jakob Klocker
2026-11-16 10:13 ` [PATCH pve-docs v3 4/4] vzdump: document backup " Jakob Klocker
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116101344.140523-3-j.klocker@proxmox.com \
--to=j.klocker@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.