[pve-devel] [PATCH storage 1/3] prune: introduce keep-all option

[pve-devel] [PATCH storage 1/3] prune: introduce keep-all option

[Fabian Ebner]

useful to have an alternative to the old maxfiles = 0. There has to
be a way for vzdump to distinguish between:
1. use the /etc/vzdump.conf default (when no options are configured for the storage)
2. use no limit (when keep-all=1)

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---

A mutual dependency bump between storage and manager is needed for this and
patch #3.

 PVE/Storage.pm           |  9 ++++++---
 PVE/Storage/PBSPlugin.pm | 15 +++++++++++----
 PVE/Storage/Plugin.pm    | 20 +++++++++++++++++++-
 3 files changed, 36 insertions(+), 8 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index bd6e15e..d613469 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -41,11 +41,11 @@ use PVE::Storage::DRBDPlugin;
 use PVE::Storage::PBSPlugin;
 
 # Storage API version. Increment it on changes in storage API interface.
-use constant APIVER => 8;
+use constant APIVER => 9;
 # Age is the number of versions we're backward compatible with.
 # This is like having 'current=APIVER' and age='APIAGE' in libtool,
 # see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
-use constant APIAGE => 7;
+use constant APIAGE => 8;
 
 # load standard plugins
 PVE::Storage::DirPlugin->register();
@@ -1622,7 +1622,10 @@ my $prune_mark = sub {
 sub prune_mark_backup_group {
     my ($backup_group, $keep) = @_;
 
-    if (!scalar(grep {$_ > 0} values %{$keep})) {
+    my $keep_all = delete $keep->{'keep-all'};
+
+    if ($keep_all || !scalar(grep {$_ > 0} values %{$keep})) {
+	$keep = { 'keep-all' => 1 } if $keep_all;
 	foreach my $prune_entry (@{$backup_group}) {
 	    $prune_entry->{mark} = 'keep';
 	}
diff --git a/PVE/Storage/PBSPlugin.pm b/PVE/Storage/PBSPlugin.pm
index 2e6d3f6..ef9bc79 100644
--- a/PVE/Storage/PBSPlugin.pm
+++ b/PVE/Storage/PBSPlugin.pm
@@ -313,10 +313,17 @@ sub prune_backups {
     }
 
     my @param;
-    foreach my $opt (keys %{$keep}) {
-	next if $keep->{$opt} == 0;
-	push @param, "--$opt";
-	push @param, "$keep->{$opt}";
+
+    my $keep_all = delete $keep->{'keep-all'};
+
+    if (!$keep_all) {
+	foreach my $opt (keys %{$keep}) {
+	    next if $keep->{$opt} == 0;
+	    push @param, "--$opt";
+	    push @param, "$keep->{$opt}";
+	}
+    } else { # no need to pass anything to PBS
+	$keep = { 'keep-all' => 1 };
     }
 
     push @param, '--dry-run' if $dryrun;
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index fe56864..391f441 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -52,6 +52,11 @@ my %prune_option = (
 );
 
 our $prune_backups_format = {
+    'keep-all' => {
+	type => 'boolean',
+	description => 'Keep all backups. Conflicts with the other options when true.',
+	optional => 1,
+    },
     'keep-last' => {
 	%prune_option,
 	description => 'Keep the last <N> backups.',
@@ -82,7 +87,20 @@ our $prune_backups_format = {
 		       'than one backup for a single year, only the latest one is kept.'
     },
 };
-PVE::JSONSchema::register_format('prune-backups', $prune_backups_format);
+PVE::JSONSchema::register_format('prune-backups', $prune_backups_format, \&validate_prune_backups);
+sub validate_prune_backups {
+    my ($prune_backups) = @_;
+
+    my $keep_all = delete $prune_backups->{'keep-all'};
+
+    if (!scalar(grep {$_ > 0} values %{$prune_backups})) {
+	$prune_backups = { 'keep-all' => 1 };
+    } elsif ($keep_all) {
+	die "keep-all cannot be set together with other options.\n";
+    }
+
+    return $prune_backups;
+}
 register_standard_option('prune-backups', {
     description => "The retention options with shorter intervals are processed first " .
 		   "with --keep-last being the very first one. Each option covers a " .
-- 
2.20.1

[pve-devel] [PATCH storage 2/3] convert maxfiles to prune_backups when reading the storage configuration

[Fabian Ebner]

If there are already prune options configured, simply delete the maxfiles
setting. Having set both is invalid from vzdump's perspective anyways, and any
backup job on such a storage failed, meaning a user would've noticed.

If there are no prune options, translate the maxfiles value to keep-last,
except for maxfiles being zero (=unlimited), in which case we use keep-all.

If both are not set, don't set anything, so:
1. Storages don't suddenly have retention options set.
2. People relying on vzdump defaults can still use those.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 PVE/Storage.pm | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index d613469..18357a8 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -129,6 +129,26 @@ sub lock_storage_config {
     }
 }
 
+# FIXME remove maxfiles for PVE 7.0
+my $convert_maxfiles_to_prune_backups = sub {
+    my ($scfg) = @_;
+
+    my $maxfiles = delete $scfg->{maxfiles};
+
+    if (!defined($scfg->{'prune-backups'}) && defined($maxfiles)) {
+	my $prune_backups;
+	if ($maxfiles) {
+	    $prune_backups = { 'keep-last' => $maxfiles };
+	} else { # maxfiles 0 means no limit
+	    $prune_backups = { 'keep-all' => 1 };
+	}
+	$scfg->{'prune-backups'} = PVE::JSONSchema::print_property_string(
+	    $prune_backups,
+	    'prune-backups'
+	);
+    }
+};
+
 sub storage_config {
     my ($cfg, $storeid, $noerr) = @_;
 
@@ -138,6 +158,8 @@ sub storage_config {
 
     die "storage '$storeid' does not exist\n" if (!$noerr && !$scfg);
 
+    $convert_maxfiles_to_prune_backups->($scfg);
+
     return $scfg;
 }
 
-- 
2.20.1

[pve-devel] [PATCH manager 3/3] vzdump: adapt to new keep-all prune option

[Fabian Ebner]

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 PVE/VZDump.pm | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 59062d2b..178f37f5 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -494,11 +494,13 @@ sub new {
 	if ($maxfiles) {
 	    $opts->{'prune-backups'} = { 'keep-last' => $maxfiles };
 	} else {
-	    # maxfiles being zero means keep all, so avoid triggering any remove code path to be safe
-	    $opts->{remove} = 0;
+	    $opts->{'prune-backups'} = { 'keep-all' => 1 };
 	}
     }
 
+    # avoid triggering any remove code path if keep-all is set
+    $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
+
     if ($opts->{tmpdir} && ! -d $opts->{tmpdir}) {
 	$errors .= "\n" if $errors;
 	$errors .= "tmpdir '$opts->{tmpdir}' does not exist";
@@ -735,8 +737,13 @@ sub exec_backup_task {
 	my $prune_options = $opts->{'prune-backups'};
 
 	my $backup_limit = 0;
-	foreach my $keep (values %{$prune_options}) {
-	    $backup_limit += $keep;
+	my $keep_all = delete $prune_options->{'keep-all'};
+	if ($keep_all) {
+	    $prune_options = { 'keep-all' => 1 };
+	} else {
+	    foreach my $keep (values %{$prune_options}) {
+		$backup_limit += $keep;
+	    }
 	}
 
 	if ($backup_limit && !$opts->{remove}) {
-- 
2.20.1

[pve-devel] applied: [PATCH storage 1/3] prune: introduce keep-all option

[Thomas Lamprecht]

On 23.11.20 13:33, Fabian Ebner wrote:
> useful to have an alternative to the old maxfiles = 0. There has to
> be a way for vzdump to distinguish between:
> 1. use the /etc/vzdump.conf default (when no options are configured for the storage)
> 2. use no limit (when keep-all=1)
> 
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
> 
> A mutual dependency bump between storage and manager is needed for this and
> patch #3.
> 
>  PVE/Storage.pm           |  9 ++++++---
>  PVE/Storage/PBSPlugin.pm | 15 +++++++++++----
>  PVE/Storage/Plugin.pm    | 20 +++++++++++++++++++-
>  3 files changed, 36 insertions(+), 8 deletions(-)
> 
>

applied, thanks!

[pve-devel] applied: [PATCH storage 2/3] convert maxfiles to prune_backups when reading the storage configuration

[Thomas Lamprecht]

On 23.11.20 13:33, Fabian Ebner wrote:
> If there are already prune options configured, simply delete the maxfiles
> setting. Having set both is invalid from vzdump's perspective anyways, and any
> backup job on such a storage failed, meaning a user would've noticed.
> 
> If there are no prune options, translate the maxfiles value to keep-last,
> except for maxfiles being zero (=unlimited), in which case we use keep-all.
> 
> If both are not set, don't set anything, so:
> 1. Storages don't suddenly have retention options set.
> 2. People relying on vzdump defaults can still use those.
> 
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>  PVE/Storage.pm | 22 ++++++++++++++++++++++
>  1 file changed, 22 insertions(+)
> 
>

applied, thanks!