* [PATCH pve-manager 2/3] fix #6464: vzdump: add retry strategies for failed guest backups
2026-11-02 11:58 [PATCH guest-common/manager 0/3] fix #6464: backup: add backup retry strategies Jakob Klocker
2026-11-02 11:58 ` [PATCH pve-guest-common 1/3] vzdump: add error-behavior and max-retries options Jakob Klocker
@ 2026-11-02 11:58 ` Jakob Klocker
2026-11-02 11:58 ` [PATCH pve-manager 3/3] ui: backup: add error behavior and retry options Jakob Klocker
2 siblings, 0 replies; 4+ messages in thread
From: Jakob Klocker @ 2026-11-02 11:58 UTC (permalink / raw)
To: pve-devel
Add the `error-behavior` and `max-retries` options to automatically
retry failed guest backups, which helps with transient failures.
Supported retry strategies:
- `ignore`: Preserves current behavior (no retries, default).
- `retry-immediately`: Retries the guest inline after
failure.
- `retry`: Defers failed guests until all other guests have been
processed, then retries them in the order they failed.
A guest is retried at most `max-retries` times. Between attempts, the
delay starts at 30 seconds and dobules with each retry, capped at 30
minutes as suggested on bugzilla [0]. If a guest still fails after the
last retry, it is counted as an error and the job finishes with errors.
If a retry succeeds, the guest is reported as successful in the
notifications and no error is counted.
The task log is opened in append mode for retries, so the log contains
the complete transcript of all attempts.
[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=6464#c5
Link: https://bugzilla.proxmox.com/show_bug.cgi?id=6464
Signed-off-by: Jakob Klocker <j.klocker@proxmox.com>
---
PVE/VZDump.pm | 131 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 129 insertions(+), 2 deletions(-)
diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index defe9e0a..6199b7ff 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -10,6 +10,7 @@ use File::Path;
use IO::File;
use IO::Select;
use IPC::Open3;
+use List::Util qw(min);
use POSIX qw(strftime);
use Time::Local;
@@ -1047,7 +1048,8 @@ sub exec_backup_task {
-d $task->{tmpdir}
|| die "unable to create temporary directory '$task->{tmpdir}'";
- $logfd = IO::File->new(">$tmplog")
+ my $logmode = $task->{retries} ? '>>' : '>';
+ $logfd = IO::File->new("${logmode}${tmplog}")
|| die "unable to create log file '$tmplog'";
$task->{dumpdir} = $opts->{dumpdir};
@@ -1434,14 +1436,139 @@ sub exec_backup {
my $starttime = time();
my $errcount = 0;
+
+ my $error_behavior = $opts->{'error-behavior'} // 'ignore';
+ my $max_retries = $opts->{'max-retries'} // 5;
+ my $min_between_retry = 30;
+ my $max_wait = 60 * 30;
+
+ my $get_backoff = sub {
+ my ($retry) = @_;
+ return min($min_between_retry * 2**($retry - 1), $max_wait);
+ };
+
eval {
$self->{'backup-provider'}->job_init($starttime) if $self->{'backup-provider'};
$self->run_hook_script('job-start', undef, $job_start_fd);
+ my $failed_tasks = {};
+
foreach my $task (@$tasklist) {
+ my $vmid = $task->{vmid};
+ my $initial_task = {%$task};
+
+ $task->{retries} = 0;
$self->exec_backup_task($task);
- $errcount += 1 if $task->{state} ne 'ok';
+
+ next if $task->{state} eq 'ok';
+
+ if ($error_behavior eq 'retry-immediately') {
+ for my $retry (1 .. $max_retries) {
+ my $backoff = $get_backoff->($retry);
+
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid (attempt $retry/$max_retries) after ${backoff}s delay...",
+ undef,
+ 1,
+ );
+ sleep($backoff);
+
+ %$task = (%$initial_task, retries => $retry);
+ $self->exec_backup_task($task);
+
+ last if $task->{state} eq 'ok';
+ }
+
+ } elsif ($error_behavior eq 'retry') {
+ $failed_tasks->{$vmid} = {
+ task => $task,
+ initial_task => $initial_task,
+ fail_time => time(),
+ retries => 0,
+ };
+ next;
+ }
+
+ if ($task->{state} ne 'ok') {
+ debugmsg('err', "Guest $vmid failed after $max_retries retries.", undef, 1)
+ if $error_behavior eq 'retry-immediately';
+ $errcount++;
+ }
+ }
+
+ if ($error_behavior eq 'retry') {
+ while (keys %$failed_tasks > 0) {
+ my $now = time();
+ my $min_sleep = $max_wait;
+ my $ready_to_run = [];
+
+ my @sorted_vmids = sort {
+ $failed_tasks->{$a}->{fail_time} <=> $failed_tasks->{$b}->{fail_time}
+ || $a <=> $b
+ } keys %$failed_tasks;
+
+ foreach my $vmid (@sorted_vmids) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ if ($fail_info->{retries} >= $max_retries) {
+ debugmsg(
+ 'err',
+ "Guest $vmid failed after $max_retries retries.",
+ undef,
+ 1,
+ );
+ $errcount++;
+ delete $failed_tasks->{$vmid};
+ next;
+ }
+
+ my $delay = $get_backoff->($fail_info->{retries} + 1);
+ my $target_time = $fail_info->{fail_time} + $delay;
+ my $wait_time = $target_time - $now;
+
+ if ($wait_time > 0) {
+ $min_sleep = $wait_time if $wait_time < $min_sleep;
+ } else {
+ push @$ready_to_run, $vmid;
+ }
+ }
+
+ if (scalar(@$ready_to_run) == 0 && keys %$failed_tasks > 0) {
+ debugmsg(
+ 'info',
+ "Waiting $min_sleep seconds before next backup retry...",
+ undef,
+ 1,
+ );
+ sleep($min_sleep);
+ next;
+ }
+
+ foreach my $vmid (@$ready_to_run) {
+ my $fail_info = $failed_tasks->{$vmid};
+
+ $fail_info->{retries}++;
+ debugmsg(
+ 'info',
+ "Retrying backup for guest $vmid (attempt $fail_info->{retries}/$max_retries)",
+ undef,
+ 1,
+ );
+
+ my $task = $fail_info->{task};
+ %$task =
+ (%{ $fail_info->{initial_task} }, retries => $fail_info->{retries});
+ $self->exec_backup_task($task);
+
+ if ($task->{state} eq 'ok') {
+ delete $failed_tasks->{$vmid};
+ } else {
+ $fail_info->{fail_time} = time();
+ }
+ }
+ }
}
$self->{'backup-provider'}->job_cleanup() if $self->{'backup-provider'};
--
2.47.3
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH pve-manager 3/3] ui: backup: add error behavior and retry options
2026-11-02 11:58 [PATCH guest-common/manager 0/3] fix #6464: backup: add backup retry strategies Jakob Klocker
2026-11-02 11:58 ` [PATCH pve-guest-common 1/3] vzdump: add error-behavior and max-retries options Jakob Klocker
2026-11-02 11:58 ` [PATCH pve-manager 2/3] fix #6464: vzdump: add retry strategies for failed guest backups Jakob Klocker
@ 2026-11-02 11:58 ` Jakob Klocker
2 siblings, 0 replies; 4+ messages in thread
From: Jakob Klocker @ 2026-11-02 11:58 UTC (permalink / raw)
To: pve-devel
Expose the `error-behavior` and `max-retries` vzdump parameters in the
scheduled datacenter backup and in the manual backup window.
When creating a new backup job or starting a manual backup, the UI
defaults to `retry` with a maximum of 5 retries.
The manual backup window only offers `retry` and `ignore`, since it
backs up a single guest, where `retry` and `retry-immediately` behave
the same. The width of the scheduled backup job window is increased
from 720 to 750 pixels so the new fields fit without cutting off their
labels.
Signed-off-by: Jakob Klocker <j.klocker@proxmox.com>
---
www/manager6/dc/Backup.js | 2 +-
www/manager6/panel/BackupAdvancedOptions.js | 52 +++++++++++++++++++++
www/manager6/window/Backup.js | 47 ++++++++++++++++++-
3 files changed, 99 insertions(+), 2 deletions(-)
diff --git a/www/manager6/dc/Backup.js b/www/manager6/dc/Backup.js
index d3db9d9c..b640fa1a 100644
--- a/www/manager6/dc/Backup.js
+++ b/www/manager6/dc/Backup.js
@@ -7,7 +7,7 @@ Ext.define('PVE.dc.BackupEdit', {
defaultFocus: undefined,
subject: gettext('Backup Job'),
- width: 720,
+ width: 750,
bodyPadding: 0,
url: '/api2/extjs/cluster/backup',
diff --git a/www/manager6/panel/BackupAdvancedOptions.js b/www/manager6/panel/BackupAdvancedOptions.js
index fe8cc3bf..b06011fe 100644
--- a/www/manager6/panel/BackupAdvancedOptions.js
+++ b/www/manager6/panel/BackupAdvancedOptions.js
@@ -24,6 +24,11 @@ Ext.define('PVE.panel.BackupAdvancedOptions', {
me.lookup('fleecingStorage').setDisabled(!value);
},
+ errorBehaviorChange: function (field, value) {
+ let me = this;
+ me.lookup('maxRetriesField').setDisabled(value === 'ignore');
+ },
+
control: {
'proxmoxcheckbox[reference=fleecingEnabled]': {
change: 'toggleFleecing',
@@ -117,6 +122,53 @@ Ext.define('PVE.panel.BackupAdvancedOptions', {
},
items: [
+ {
+ xtype: 'pveTwoColumnContainer',
+ startColumn: {
+ xtype: 'proxmoxKVComboBox',
+ fieldLabel: gettext('Error Behavior'),
+ name: 'error-behavior',
+ value: 'retry',
+ comboItems: [
+ ['retry', gettext('Retry at end of job')],
+ ['retry-immediately', gettext('Retry immediately')],
+ ['ignore', gettext('Ignore')],
+ ],
+ cbind: {
+ deleteEmpty: '{!isCreate}',
+ },
+ listeners: {
+ change: 'errorBehaviorChange',
+ },
+ },
+ endFlex: 2,
+ endColumn: {
+ xtype: 'displayfield',
+ value: gettext('How to handle a guest backup failure.'),
+ },
+ },
+ {
+ xtype: 'pveTwoColumnContainer',
+ startColumn: {
+ xtype: 'proxmoxintegerfield',
+ fieldLabel: gettext('Max Retries'),
+ name: 'max-retries',
+ minValue: 1,
+ maxValue: 10,
+ reference: 'maxRetriesField',
+ value: 5,
+ allowBlank: true,
+ emptyText: '5',
+ cbind: {
+ deleteEmpty: '{!isCreate}',
+ },
+ },
+ endFlex: 2,
+ endColumn: {
+ xtype: 'displayfield',
+ value: gettext('Maximum number of retries before giving up on a guest.'),
+ },
+ },
{
xtype: 'pveTwoColumnContainer',
startColumn: {
diff --git a/www/manager6/window/Backup.js b/www/manager6/window/Backup.js
index 7c1c54de..e1b7c915 100644
--- a/www/manager6/window/Backup.js
+++ b/www/manager6/window/Backup.js
@@ -54,6 +54,36 @@ Ext.define('PVE.window.Backup', {
},
});
+ let maxRetriesField = Ext.create({
+ xtype: 'proxmoxintegerfield',
+ fieldLabel: gettext('Max Retries'),
+ name: 'max-retries',
+ minValue: 1,
+ maxValue: 10,
+ value: 5,
+ allowBlank: true,
+ emptyText: '5',
+ advanced: true,
+ });
+
+ let errorBehaviorSelector = Ext.create({
+ xtype: 'proxmoxKVComboBox',
+ fieldLabel: gettext('Error Behavior'),
+ name: 'error-behavior',
+ value: 'retry',
+ deleteEmpty: false,
+ advanced: true,
+ comboItems: [
+ ['retry', gettext('Retry')],
+ ['ignore', gettext('Ignore')],
+ ],
+ listeners: {
+ change: function (field, value) {
+ maxRetriesField.setDisabled(value === 'ignore');
+ },
+ },
+ });
+
let pbsChangeDetectionModeSelector = Ext.create({
xtype: 'proxmoxKVComboBox',
flex: 1,
@@ -249,7 +279,14 @@ Ext.define('PVE.window.Backup', {
bodyPadding: 10,
border: false,
column1: [storagesel, modeSelector, protectedCheckbox, pbsChangeDetection],
- column2: [compressionSelector, notificationModeSelector, mailtoField, removeCheckbox],
+ column2: [
+ compressionSelector,
+ notificationModeSelector,
+ mailtoField,
+ removeCheckbox,
+ errorBehaviorSelector,
+ maxRetriesField,
+ ],
columnB: [
{
xtype: 'textareafield',
@@ -341,6 +378,14 @@ Ext.define('PVE.window.Backup', {
);
}
+ if (values['error-behavior']) {
+ params['error-behavior'] = values['error-behavior'];
+ }
+
+ if (values['max-retries'] !== undefined && values['max-retries'] !== '') {
+ params['max-retries'] = values['max-retries'];
+ }
+
Proxmox.Utils.API2Request({
url: '/nodes/' + me.nodename + '/vzdump',
params: params,
--
2.47.3
^ permalink raw reply related [flat|nested] 4+ messages in thread