[pve-devel] [PATCH manager 1/4] test: add tests for retention parameters for vzdump's new()

[pve-devel] [PATCH manager 1/4] test: add tests for retention parameters for vzdump's new()

[Fabian Ebner]

To get a more complete picture, instead of mocking storage_config,
PVE::Cluster's get_config is mocked. This ensures that the prune-backups
validation and the maxfiles conversion are also called.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 test/Makefile                     |   1 +
 test/vzdump_new_retention_test.pl | 413 ++++++++++++++++++++++++++++++
 2 files changed, 414 insertions(+)
 create mode 100755 test/vzdump_new_retention_test.pl

diff --git a/test/Makefile b/test/Makefile
index 44f3b0d7..5c4478ce 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -23,6 +23,7 @@ mail_test:
 
 vzdump_test:
 	./vzdump_guest_included_test.pl
+	./vzdump_new_retention_test.pl
 
 .PHONY: install
 install:
diff --git a/test/vzdump_new_retention_test.pl b/test/vzdump_new_retention_test.pl
new file mode 100755
index 00000000..87faa899
--- /dev/null
+++ b/test/vzdump_new_retention_test.pl
@@ -0,0 +1,413 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use lib '..';
+
+use Test::More;
+use Test::MockModule;
+
+use PVE::VZDump;
+
+my $vzdump_config;
+my $storage_config;
+
+sub prepare_storage_config {
+    my ($param) = @_;
+
+    $storage_config = "dir: local\n";
+    $storage_config .= "\tcontent backup\n";
+    $storage_config .= "\tpath /var/lib/vz\n";
+
+    foreach my $key (keys %{$param}) {
+	my $value = $param->{$key};
+	$storage_config .= "\t${key} ${value}\n";
+    }
+}
+
+sub prepare_vzdump_config {
+    my ($param) = @_;
+
+    $vzdump_config = "";
+    foreach my $key (keys %{$param}) {
+	my $value = $param->{$key};
+	$vzdump_config .= "${key}: ${value}\n";
+    }
+}
+
+my $pve_vzdump_module = Test::MockModule->new('PVE::VZDump');
+$pve_vzdump_module->mock(
+    mkpath => sub {
+	return;
+    },
+    check_bin => sub {
+	return;
+    },
+);
+
+my $pve_storage_module = Test::MockModule->new('PVE::Storage');
+$pve_storage_module->mock(
+    activate_storage => sub {
+	return;
+    },
+);
+
+my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
+$pve_cluster_module->mock(
+    get_config => sub {
+	my ($filename) = @_;
+
+	die "unexpected filename '$filename'\n" if $filename ne 'storage.cfg';
+	return $storage_config;
+    },
+);
+
+my $pve_tools_module = Test::MockModule->new('PVE::Tools');
+$pve_tools_module->mock(
+    file_get_contents => sub {
+	my ($filename) = @_;
+
+	die "unexpected filename '$filename'\n" if $filename ne '/etc/vzdump.conf';
+	return $vzdump_config;
+    },
+);
+
+my @tested_options = qw(prune-backups remove);
+
+# each test consists of the following:
+# name          - unique name for the test
+# cli_param     - CLI parameters to be passed to new(); vmid and storage are hardcoded
+# storage_param - parameters for the mocked storage configuration
+# vzdump_param  - parameters for the mocked /etc/vzdump.conf
+# expected      - expected options
+my @tests = (
+    {
+	description => 'no params',
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 1,
+	    },
+	    remove => 1,
+	},
+    },
+    # TODO make parse error critical?
+    {
+	description => 'maxfiles vzdump 1',
+	vzdump_param => {
+	    maxfiles => 0,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 1,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'maxfiles vzdump 2',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 7,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'maxfiles storage 1',
+	storage_param => {
+	    maxfiles => 0,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'maxfiles storage 2',
+	storage_param => {
+	    maxfiles => 7,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 7,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'maxfiles CLI 1',
+	cli_param => {
+	    maxfiles => 0,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'maxfiles CLI 2',
+	cli_param => {
+	    maxfiles => 7,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 7,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'prune-backups storage 1',
+	storage_param => {
+	    'prune-backups' => 'keep-last=1,keep-hourly=2,keep-daily=3,' .
+		'keep-weekly=4,keep-monthly=5,keep-yearly=6',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 1,
+		'keep-hourly' => 2,
+		'keep-daily' => 3,
+		'keep-weekly' => 4,
+		'keep-monthly' => 5,
+		'keep-yearly' => 6,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'prune-backups storage 2',
+	storage_param => {
+	    'prune-backups' => 'keep-last=0,keep-hourly=0,keep-daily=0,' .
+		'keep-weekly=0,keep-monthly=0,keep-yearly=0',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'prune-backups storage 3',
+	storage_param => {
+	    'prune-backups' => 'keep-hourly=0,keep-monthly=0,keep-yearly=0',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'both storage 1',
+	storage_param => {
+	    'prune-backups' => 'keep-hourly=1,keep-monthly=2,keep-yearly=3',
+	    maxfiles => 0,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-hourly' => 1,
+		'keep-monthly' => 2,
+		'keep-yearly' => 3,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'prune-backups CLI 1',
+	cli_param => {
+	    'prune-backups' => 'keep-last=1,keep-hourly=2,keep-daily=3,' .
+		'keep-weekly=4,keep-monthly=5,keep-yearly=6',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 1,
+		'keep-hourly' => 2,
+		'keep-daily' => 3,
+		'keep-weekly' => 4,
+		'keep-monthly' => 5,
+		'keep-yearly' => 6,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'prune-backups CLI 2',
+	cli_param => {
+	    'prune-backups' => 'keep-last=0,keep-hourly=0,keep-daily=0,' .
+		'keep-weekly=0,keep-monthly=0,keep-yearly=0',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'prune-backups CLI 3',
+	cli_param => {
+	    'prune-backups' => 'foo=bar',
+	},
+	expected => "format error\n" .
+	    "foo: property is not defined in schema and the schema does not allow additional properties\n",
+    },
+    {
+	description => 'both CLI 1',
+	cli_param => {
+	    'prune-backups' => 'keep-hourly=1,keep-monthly=2,keep-yearly=3',
+	    maxfiles => 4,
+	},
+	expected => "400 Parameter verification failed.\n" .
+	    "prune-backups: option conflicts with option 'maxfiles'\n",
+    },
+    {
+	description => 'mixed 1',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	storage_param => {
+	    'prune-backups' => 'keep-hourly=24',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-hourly' => 24,
+	    },
+	    remove => 1,
+	},
+    },
+    # TODO make parse error critical?
+    {
+	description => 'mixed 2',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	storage_param => {
+	    'prune-backups' => 'keephourly=24',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 7,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'mixed 3',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	cli_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'mixed 4',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	storage_param => {
+	    'prune-backups' => 'keep-all=0,keep-last=10',
+	},
+	cli_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'mixed 5',
+	vzdump_param => {
+	    maxfiles => 7,
+	},
+	storage_param => {
+	    'prune-backups' => 'keep-all=0,keep-last=10',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 10,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'mixed 6',
+	storage_param => {
+	    'prune-backups' => 'keep-last=10',
+	},
+	cli_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'mixed 7',
+	storage_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	cli_param => {
+	    'prune-backups' => 'keep-last=10',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 10,
+	    },
+	    remove => 1,
+	},
+    },
+);
+
+plan tests => scalar @tests;
+
+foreach my $test (@tests) {
+    prepare_storage_config($test->{storage_param});
+    prepare_vzdump_config($test->{vzdump_param});
+
+    $test->{cli_param}->{vmid} = 100;
+    $test->{cli_param}->{storage} = 'local';
+
+    my $got = eval {
+	PVE::VZDump::verify_vzdump_parameters($test->{cli_param}, 1);
+
+	my $vzdump = PVE::VZDump->new('fake cmdline', $test->{cli_param}, undef);
+
+	my $opts = $vzdump->{opts} or die "did not get options\n";
+	die "maxfiles is defined" if defined($opts->{maxfiles});
+
+	my $res = {};
+	foreach my $opt (@tested_options) {
+	    next if !defined($opts->{$opt});
+	    $res->{$opt} = $opts->{$opt};
+	}
+	return $res;
+    };
+    $got = $@ if $@;
+
+    is_deeply($got, $test->{expected}, $test->{description}) || diag(explain($got));
+}
+
+done_testing();
-- 
2.20.1

[pve-devel] [PATCH manager 2/4] vzdump: storage_info: adapt to new maxfiles backend behavior

[Fabian Ebner]

It's automatically converted to prune-backups when using storage_config() now.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 PVE/VZDump.pm | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index c37fa513..94aa76ab 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -86,15 +86,11 @@ sub storage_info {
 
     my $info = {
 	scfg => $scfg,
-	maxfiles => $scfg->{maxfiles},
     };
 
     $info->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'})
 	if defined($scfg->{'prune-backups'});
 
-    die "cannot have 'maxfiles' and 'prune-backups' configured at the same time\n"
-	if defined($info->{'prune-backups'}) && defined($info->{maxfiles});
-
     if ($type eq 'pbs') {
 	$info->{pbs} = 1;
     } else {
-- 
2.20.1

[pve-devel] [PATCH manager 3/4] vzdump: convert maxfiles CLI parameter to prune-backups

[Fabian Ebner]

in preparation to clean up handling in new().

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 PVE/VZDump.pm | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 94aa76ab..8574fc94 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -69,6 +69,29 @@ sub run_command {
     PVE::Tools::run_command($cmdstr, %param, logfunc => $logfunc);
 }
 
+my $parse_prune_backups_maxfiles = sub {
+    my ($param, $kind) = @_;
+
+    my $maxfiles = delete $param->{maxfiles};
+    my $prune_backups = $param->{'prune-backups'};
+
+    warn "both 'maxfiles' and 'prune-backups' defined as ${kind} - ignoring 'maxfiles'\n"
+        if defined($maxfiles) && defined($prune_backups);
+
+    if (defined($prune_backups)) {
+	$param->{'prune-backups'} = PVE::JSONSchema::parse_property_string(
+	    'prune-backups',
+	    $prune_backups
+	);
+    } elsif (defined($maxfiles)) {
+	if ($maxfiles) {
+	    $param->{'prune-backups'} = { 'keep-last' => $maxfiles };
+	} else {
+	    $param->{'prune-backups'} = { 'keep-all' => 1 };
+	}
+    }
+};
+
 sub storage_info {
     my $storage = shift;
 
@@ -1170,8 +1193,7 @@ sub verify_vzdump_parameters {
     raise_param_exc({ 'prune-backups' => "option conflicts with option 'maxfiles'"})
 	if defined($param->{'prune-backups'}) && defined($param->{maxfiles});
 
-    $param->{'prune-backups'} = PVE::JSONSchema::parse_property_string('prune-backups', $param->{'prune-backups'})
-	if defined($param->{'prune-backups'});
+    $parse_prune_backups_maxfiles->($param, 'CLI parameters');
 
     $param->{all} = 1 if (defined($param->{exclude}) && !$param->{pool});
 
-- 
2.20.1

[pve-devel] [PATCH manager 4/4] vzdump: defaults: correctly parse prune-backups and convert maxfiles

[Fabian Ebner]

Also simplify handling in new(), now that we never have maxfiles there anymore.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 PVE/VZDump.pm                     | 20 ++------
 test/vzdump_new_retention_test.pl | 85 +++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+), 15 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 8574fc94..6892918f 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -230,6 +230,8 @@ sub read_vzdump_defaults {
 	$res->{$key} = $defaults->{$key} if !defined($res->{$key});
     }
 
+    $parse_prune_backups_maxfiles->($res, "options in '$fn'");
+
     return $res;
 }
 
@@ -437,7 +439,7 @@ sub new {
     $opts->{remove} = 1 if !defined($opts->{remove});
 
     foreach my $k (keys %$defaults) {
-	next if $k eq 'exclude-path' || $k eq 'maxfiles'; # dealt with separately
+	next if $k eq 'exclude-path' || $k eq 'prune-backups'; # dealt with separately
 	if ($k eq 'dumpdir' || $k eq 'storage') {
 	    $opts->{$k} = $defaults->{$k} if !defined ($opts->{dumpdir}) &&
 		!defined ($opts->{storage});
@@ -494,11 +496,7 @@ sub new {
 	    $opts->{dumpdir} = $info->{dumpdir};
 	    $opts->{scfg} = $info->{scfg};
 	    $opts->{pbs} = $info->{pbs};
-
-	    if (!defined($opts->{'prune-backups'}) && !defined($opts->{maxfiles})) {
-		$opts->{'prune-backups'} = $info->{'prune-backups'};
-		$opts->{maxfiles} = $info->{maxfiles};
-	    }
+	    $opts->{'prune-backups'} //= $info->{'prune-backups'};
 	}
     } elsif ($opts->{dumpdir}) {
 	$errors .= "dumpdir '$opts->{dumpdir}' does not exist"
@@ -507,15 +505,7 @@ sub new {
 	die "internal error";
     }
 
-    if (!defined($opts->{'prune-backups'})) {
-	my $maxfiles = delete $opts->{maxfiles} // $defaults->{maxfiles};
-	$maxfiles = int($maxfiles); # shouldn't be necessary, but be safe
-	if ($maxfiles) {
-	    $opts->{'prune-backups'} = { 'keep-last' => $maxfiles };
-	} else {
-	    $opts->{'prune-backups'} = { 'keep-all' => 1 };
-	}
-    }
+    $opts->{'prune-backups'} //= $defaults->{'prune-backups'};
 
     # avoid triggering any remove code path if keep-all is set
     $opts->{remove} = 0 if $opts->{'prune-backups'}->{'keep-all'};
diff --git a/test/vzdump_new_retention_test.pl b/test/vzdump_new_retention_test.pl
index 87faa899..569419fb 100755
--- a/test/vzdump_new_retention_test.pl
+++ b/test/vzdump_new_retention_test.pl
@@ -164,6 +164,61 @@ my @tests = (
 	    remove => 1,
 	},
     },
+    {
+	description => 'prune-backups vzdump 1',
+	vzdump_param => {
+	    'prune-backups' => 'keep-last=1,keep-hourly=2,keep-daily=3,' .
+		'keep-weekly=4,keep-monthly=5,keep-yearly=6',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 1,
+		'keep-hourly' => 2,
+		'keep-daily' => 3,
+		'keep-weekly' => 4,
+		'keep-monthly' => 5,
+		'keep-yearly' => 6,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'prune-backups vzdump 2',
+	vzdump_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'prune-backups vzdump 3',
+	vzdump_param => {
+	    'prune-backups' => 'keep-hourly=0,keep-monthly=0,keep-yearly=0',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
+    {
+	description => 'both vzdump 1',
+	vzdump_param => {
+	    'prune-backups' => 'keep-all=1',
+	    maxfiles => 7,
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
     {
 	description => 'prune-backups storage 1',
 	storage_param => {
@@ -379,6 +434,36 @@ my @tests = (
 	    remove => 1,
 	},
     },
+    {
+	description => 'mixed 8',
+	storage_param => {
+	    'prune-backups' => 'keep-last=10',
+	},
+	vzdump_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-last' => 10,
+	    },
+	    remove => 1,
+	},
+    },
+    {
+	description => 'mixed 9',
+	vzdump_param => {
+	    'prune-backups' => 'keep-last=10',
+	},
+	cli_param => {
+	    'prune-backups' => 'keep-all=1',
+	},
+	expected => {
+	    'prune-backups' => {
+		'keep-all' => 1,
+	    },
+	    remove => 0,
+	},
+    },
 );
 
 plan tests => scalar @tests;
-- 
2.20.1

[pve-devel] applied-series: [PATCH manager 1/4] test: add tests for retention parameters for vzdump's new()

[Thomas Lamprecht]

On 01.12.20 09:24, Fabian Ebner wrote:
> To get a more complete picture, instead of mocking storage_config,
> PVE::Cluster's get_config is mocked. This ensures that the prune-backups
> validation and the maxfiles conversion are also called.
> 
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>  test/Makefile                     |   1 +
>  test/vzdump_new_retention_test.pl | 413 ++++++++++++++++++++++++++++++
>  2 files changed, 414 insertions(+)
>  create mode 100755 test/vzdump_new_retention_test.pl
> 
>

applied series, thanks!