From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 177F2617A3 for ; Thu, 9 Jul 2020 14:46:03 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 13A43128D9 for ; Thu, 9 Jul 2020 14:46:03 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C051012861 for ; Thu, 9 Jul 2020 14:45:55 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8B3B3444B2 for ; Thu, 9 Jul 2020 14:45:55 +0200 (CEST) From: Fabian Ebner To: pve-devel@lists.proxmox.com Date: Thu, 9 Jul 2020 14:45:42 +0200 Message-Id: <20200709124547.2913-3-f.ebner@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200709124547.2913-1-f.ebner@proxmox.com> References: <20200709124547.2913-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.109 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_SHORT 0.001 Use of a URL Shortener for very short URL RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v4 storage 2/7] Add prune_backups to storage API X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Jul 2020 12:46:03 -0000 Implement it for generic storages supporting backups (i.e. directory-based storages) and add a wrapper for PBS. Signed-off-by: Fabian Ebner --- Changes from v3: * prune_mark: merge loops * allow filtering by type * more error handling * allow passing along a log function * add more tests PVE/Storage.pm | 91 +++++++++- PVE/Storage/PBSPlugin.pm | 68 ++++++++ PVE/Storage/Plugin.pm | 65 +++++++ test/prune_backups_test.pm | 342 +++++++++++++++++++++++++++++++++++++ test/run_plugin_tests.pl | 1 + 5 files changed, 565 insertions(+), 2 deletions(-) create mode 100644 test/prune_backups_test.pm diff --git a/PVE/Storage.pm b/PVE/Storage.pm index edf9a2e..5c44386 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -40,11 +40,11 @@ use PVE::Storage::DRBDPlugin; use PVE::Storage::PBSPlugin; # Storage API version. Icrement it on changes in storage API interface. -use constant APIVER => 5; +use constant APIVER => 6; # Age is the number of versions we're backward compatible with. # This is like having 'current=APIVER' and age='APIAGE' in libtool, # see https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html -use constant APIAGE => 4; +use constant APIAGE => 5; # load standard plugins PVE::Storage::DirPlugin->register(); @@ -1540,6 +1540,93 @@ sub extract_vzdump_config { } } +sub prune_backups { + my ($cfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + my $scfg = storage_config($cfg, $storeid); + die "storage '$storeid' does not support backups\n" if !$scfg->{content}->{backup}; + + if (!defined($keep)) { + die "no prune-backups options configured for storage '$storeid'\n" + if !defined($scfg->{'prune-backups'}); + $keep = PVE::JSONSchema::parse_property_string('prune-backups', $scfg->{'prune-backups'}); + } + + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + return $plugin->prune_backups($scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc); +} + +my $prune_mark = sub { + my ($prune_entries, $keep_count, $id_func) = @_; + + return if !$keep_count; + + my $already_included = {}; + my $newly_included = {}; + + foreach my $prune_entry (@{$prune_entries}) { + my $mark = $prune_entry->{mark}; + my $id = $id_func->($prune_entry->{ctime}); + + next if $already_included->{$id}; + + if (defined($mark)) { + $already_included->{$id} = 1 if $mark eq 'keep'; + next; + } + + if (!$newly_included->{$id}) { + last if scalar(keys %{$newly_included}) >= $keep_count; + $newly_included->{$id} = 1; + $prune_entry->{mark} = 'keep'; + } else { + $prune_entry->{mark} = 'remove'; + } + } +}; + +sub prune_mark_backup_group { + my ($backup_group, $keep) = @_; + + my $prune_list = [ sort { $b->{ctime} <=> $a->{ctime} } @{$backup_group} ]; + + $prune_mark->($prune_list, $keep->{'keep-last'}, sub { + my ($ctime) = @_; + return $ctime; + }); + $prune_mark->($prune_list, $keep->{'keep-hourly'}, sub { + my ($ctime) = @_; + my (undef, undef, $hour, $day, $month, $year) = localtime($ctime); + return "$hour/$day/$month/$year"; + }); + $prune_mark->($prune_list, $keep->{'keep-daily'}, sub { + my ($ctime) = @_; + my (undef, undef, undef, $day, $month, $year) = localtime($ctime); + return "$day/$month/$year"; + }); + $prune_mark->($prune_list, $keep->{'keep-weekly'}, sub { + my ($ctime) = @_; + my ($sec, $min, $hour, $day, $month, $year) = localtime($ctime); + my $iso_week = int(strftime("%V", $sec, $min, $hour, $day, $month - 1, $year - 1900)); + my $iso_week_year = int(strftime("%G", $sec, $min, $hour, $day, $month - 1, $year - 1900)); + return "$iso_week/$iso_week_year"; + }); + $prune_mark->($prune_list, $keep->{'keep-monthly'}, sub { + my ($ctime) = @_; + my (undef, undef, undef, undef, $month, $year) = localtime($ctime); + return "$month/$year"; + }); + $prune_mark->($prune_list, $keep->{'keep-yearly'}, sub { + my ($ctime) = @_; + my $year = (localtime($ctime))[5]; + return "$year"; + }); + + foreach my $prune_entry (@{$prune_list}) { + $prune_entry->{mark} //= 'remove'; + } +} + sub volume_export { my ($cfg, $fh, $volid, $format, $snapshot, $base_snapshot, $with_snapshots) = @_; diff --git a/PVE/Storage/PBSPlugin.pm b/PVE/Storage/PBSPlugin.pm index 87b09f2..12f56a9 100644 --- a/PVE/Storage/PBSPlugin.pm +++ b/PVE/Storage/PBSPlugin.pm @@ -258,6 +258,74 @@ sub extract_vzdump_config { return $config; } +sub prune_backups { + my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + $logfunc //= sub { print "$_[1]\n" }; + + my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']); + + $type = 'vm' if defined($type) && $type eq 'qemu'; + $type = 'ct' if defined($type) && $type eq 'lxc'; + + my $backup_groups = {}; + foreach my $backup (@{$backups}) { + (my $backup_type = $backup->{format}) =~ s/^pbs-//; + + next if defined($type) && $backup_type ne $type; + + my $backup_group = "$backup_type/$backup->{vmid}"; + $backup_groups->{$backup_group} = 1; + } + + my @param; + foreach my $opt (keys %{$keep}) { + push @param, "--$opt"; + push @param, "$keep->{$opt}"; + } + + push @param, '--dry-run' if $dryrun; + + my $prune_list = []; + my $failed; + + foreach my $backup_group (keys %{$backup_groups}) { + $logfunc->('info', "running 'proxmox-backup-client prune' for '$backup_group'") + if !$dryrun; + eval { + my $res = run_client_cmd($scfg, $storeid, 'prune', [ $backup_group, @param ]); + + foreach my $backup (@{$res}) { + die "result from proxmox-backup-client is not as expected\n" + if !defined($backup->{'backup-time'}) + || !defined($backup->{'backup-type'}) + || !defined($backup->{'backup-id'}) + || !defined($backup->{'keep'}); + + my $ctime = $backup->{'backup-time'}; + my $type = $backup->{'backup-type'}; + my $vmid = $backup->{'backup-id'}; + my $volid = print_volid($storeid, $type, $vmid, $ctime); + + push @{$prune_list}, { + ctime => $ctime, + mark => $backup->{keep} ? 'keep' : 'remove', + type => $type eq 'vm' ? 'qemu' : 'lxc', + vmid => $vmid, + volid => $volid, + }; + } + }; + if (my $err = $@) { + $logfunc->('err', "prune '$backup_group': $err\n"); + $failed = 1; + } + } + die "error pruning backups - check log\n" if $failed; + + return $prune_list; +} + my $autogen_encryption_key = sub { my ($scfg, $storeid) = @_; my $encfile = pbs_encryption_key_file_name($scfg, $storeid); diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 6e321ae..8a58ff4 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -1174,6 +1174,71 @@ sub check_connection { return 1; } +sub prune_backups { + my ($class, $scfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; + + $logfunc //= sub { print "$_[1]\n" }; + + my $backups = $class->list_volumes($storeid, $scfg, $vmid, ['backup']); + + my $backup_groups = {}; + my $prune_list = []; + + foreach my $backup (@{$backups}) { + my $volid = $backup->{volid}; + my $backup_vmid = $backup->{vmid}; + my $archive_info = eval { PVE::Storage::archive_info($volid) } // {}; + my $backup_type = $archive_info->{type} // 'unknown'; + + next if defined($type) && $type ne $backup_type; + + my $prune_entry = { + ctime => $backup->{ctime}, + type => $backup_type, + volid => $volid, + }; + + $prune_entry->{vmid} = $backup_vmid if defined($backup_vmid); + + if ($archive_info->{is_std_name}) { + $prune_entry->{ctime} = $archive_info->{ctime}; + my $group = "$backup_type/$backup_vmid"; + push @{$backup_groups->{$group}}, $prune_entry; + } else { + # ignore backups that don't use the standard naming scheme + $prune_entry->{mark} = 'protected'; + } + + push @{$prune_list}, $prune_entry; + } + + foreach my $backup_group (values %{$backup_groups}) { + PVE::Storage::prune_mark_backup_group($backup_group, $keep); + } + + my $failed; + if (!$dryrun) { + foreach my $prune_entry (@{$prune_list}) { + next if $prune_entry->{mark} ne 'remove'; + + my $volid = $prune_entry->{volid}; + $logfunc->('info', "removing backup '$volid'"); + eval { + my (undef, $volname) = parse_volume_id($volid); + my $archive_path = $class->filesystem_path($scfg, $volname); + PVE::Storage::archive_remove($archive_path); + }; + if (my $err = $@) { + $logfunc->('err', "error when removing backup '$volid' - $err\n"); + $failed = 1; + } + } + } + die "error pruning backups - check log\n" if $failed; + + return $prune_list; +} + # Import/Export interface: # Any path based storage is assumed to support 'raw' and 'tar' streams, so # the default implementations will return this if $scfg->{path} is set, diff --git a/test/prune_backups_test.pm b/test/prune_backups_test.pm new file mode 100644 index 0000000..1e6a3d1 --- /dev/null +++ b/test/prune_backups_test.pm @@ -0,0 +1,342 @@ +package PVE::Storage::TestPruneBackups; + +use strict; +use warnings; + +use lib qw(..); + +use PVE::Storage; +use Test::More; +use Test::MockModule; + +my $storeid = 'BackTest123'; +my @vmids = (1234, 9001); + +# only includes the information needed for prune_backups +my $mocked_backups_lists = {}; + +my $basetime = 1577881101; # 2020_01_01-12_18_21 UTC + +foreach my $vmid (@vmids) { + push @{$mocked_backups_lists->{default}}, ( + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2018_05_26-11_18_21.tar.zst", + 'ctime' => $basetime - 585*24*60*60 - 60*60, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_18_21.tar.zst", + 'ctime' => $basetime - 24*60*60 - 60*60, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_19_21.tar.zst", + 'ctime' => $basetime - 24*60*60 - 60*60 + 60, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-11_18_21.tar.zst", + 'ctime' => $basetime - 60*60, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-12_18_21.tar.zst", + 'ctime' => $basetime, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-lxc-$vmid-2020_01_01-12_18_21.tar.zst", + 'ctime' => $basetime, + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-$vmid-renamed.tar.zst", + 'ctime' => 1234, + 'vmid' => $vmid, + }, + ); +} +push @{$mocked_backups_lists->{year1970}}, ( + { + 'volid' => "$storeid:backup/vzdump-lxc-321-1970_01_01-00_01_23.tar.zst", + 'ctime' => 83, + 'vmid' => 321, + }, + { + 'volid' => "$storeid:backup/vzdump-lxc-321-2070_01_01-00_01_00.tar.zst", + 'ctime' => 60*60*24 * (365*100 + 25) + 60, + 'vmid' => 321, + }, +); +push @{$mocked_backups_lists->{novmid}}, ( + { + 'volid' => "$storeid:backup/vzdump-lxc-novmid.tar.gz", + 'ctime' => 1234, + }, +); +my $current_list; +my $mock_plugin = Test::MockModule->new('PVE::Storage::Plugin'); +$mock_plugin->redefine(list_volumes => sub { + my ($class, $storeid, $scfg, $vmid, $content_types) = @_; + + my $list = $mocked_backups_lists->{$current_list}; + + return $list if !defined($vmid); + + return [ grep { $_->{vmid} eq $vmid } @{$list} ]; +}); + +sub generate_expected { + my ($vmids, $type, $marks) = @_; + + my @expected; + foreach my $vmid (@{$vmids}) { + push @expected, ( + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2018_05_26-11_18_21.tar.zst", + 'type' => 'qemu', + 'ctime' => $basetime - 585*24*60*60 - 60*60, + 'mark' => $marks->[0], + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_18_21.tar.zst", + 'type' => 'qemu', + 'ctime' => $basetime - 24*60*60 - 60*60, + 'mark' => $marks->[1], + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2019_12_31-11_19_21.tar.zst", + 'type' => 'qemu', + 'ctime' => $basetime - 24*60*60 - 60*60 + 60, + 'mark' => $marks->[2], + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-11_18_21.tar.zst", + 'type' => 'qemu', + 'ctime' => $basetime - 60*60, + 'mark' => $marks->[3], + 'vmid' => $vmid, + }, + { + 'volid' => "$storeid:backup/vzdump-qemu-$vmid-2020_01_01-12_18_21.tar.zst", + 'type' => 'qemu', + 'ctime' => $basetime, + 'mark' => $marks->[4], + 'vmid' => $vmid, + }, + ) if !defined($type) || $type eq 'qemu'; + push @expected, ( + { + 'volid' => "$storeid:backup/vzdump-lxc-$vmid-2020_01_01-12_18_21.tar.zst", + 'type' => 'lxc', + 'ctime' => $basetime, + 'mark' => $marks->[5], + 'vmid' => $vmid, + }, + ) if !defined($type) || $type eq 'lxc'; + push @expected, ( + { + 'volid' => "$storeid:backup/vzdump-$vmid-renamed.tar.zst", + 'type' => 'unknown', + 'ctime' => 1234, + 'mark' => 'protected', + 'vmid' => $vmid, + }, + ) if !defined($type); + } + return [ sort { $a->{volid} cmp $b->{volid} } @expected ]; +} + +# an array of test cases, each test is comprised of the following keys: +# description => to identify a single test +# vmid => VMID or undef for all +# type => 'qemu' or 'lxc' or undef for all +# keep => options describing what to keep +# list => backups list to use. defaults to 'default' +# expected => what prune_backups should return +# +# most of them are created further below +my $tests = [ + { + description => 'last=3, multiple IDs', + keep => { + 'keep-last' => 3, + }, + expected => generate_expected(\@vmids, undef, ['remove', 'remove', 'keep', 'keep', 'keep', 'keep']), + }, + { + description => 'weekly=2, one ID', + vmid => $vmids[0], + keep => { + 'keep-weekly' => 2, + }, + expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']), + }, + { + description => 'daily=weekly=monthly=1, multiple IDs', + keep => { + 'keep-hourly' => 0, + 'keep-daily' => 1, + 'keep-weekly' => 1, + 'keep-monthly' => 1, + }, + expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'hourly=4, one ID', + vmid => $vmids[0], + keep => { + 'keep-hourly' => 4, + 'keep-daily' => 0, + }, + expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'keep', 'keep', 'keep']), + }, + { + description => 'yearly=2, multiple IDs', + keep => { + 'keep-hourly' => 0, + 'keep-daily' => 0, + 'keep-weekly' => 0, + 'keep-monthly' => 0, + 'keep-yearly' => 2, + }, + expected => generate_expected(\@vmids, undef, ['remove', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'last=2,hourly=2 one ID', + vmid => $vmids[0], + keep => { + 'keep-last' => 2, + 'keep-hourly' => 2, + }, + expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'keep', 'keep', 'keep']), + }, + { + description => 'last=1,monthly=2, multiple IDs', + keep => { + 'keep-last' => 1, + 'keep-monthly' => 2, + }, + expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'monthly=3, one ID', + vmid => $vmids[0], + keep => { + 'keep-monthly' => 3, + }, + expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'last=daily=weekly=1, multiple IDs', + keep => { + 'keep-last' => 1, + 'keep-daily' => 1, + 'keep-weekly' => 1, + }, + expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'daily=2, one ID', + vmid => $vmids[0], + keep => { + 'keep-daily' => 2, + }, + expected => generate_expected([$vmids[0]], undef, ['remove', 'remove', 'keep', 'remove', 'keep', 'keep']), + }, + { + description => 'weekly=monthly=1, multiple IDs', + keep => { + 'keep-weekly' => 1, + 'keep-monthly' => 1, + }, + expected => generate_expected(\@vmids, undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']), + }, + { + description => 'weekly=yearly=1, one ID', + vmid => $vmids[0], + keep => { + 'keep-weekly' => 1, + 'keep-yearly' => 1, + }, + expected => generate_expected([$vmids[0]], undef, ['keep', 'remove', 'remove', 'remove', 'keep', 'keep']), + }, + { + description => 'weekly=yearly=1, one ID, type qemu', + vmid => $vmids[0], + type => 'qemu', + keep => { + 'keep-weekly' => 1, + 'keep-yearly' => 1, + }, + expected => generate_expected([$vmids[0]], 'qemu', ['keep', 'remove', 'remove', 'remove', 'keep', '']), + }, + { + description => 'week=yearly=1, one ID, type lxc', + vmid => $vmids[0], + type => 'lxc', + keep => { + 'keep-last' => 1, + }, + expected => generate_expected([$vmids[0]], 'lxc', ['', '', '', '', '', 'keep']), + }, + { + description => 'yearly=1, year before 2000', + keep => { + 'keep-yearly' => 1, + }, + list => 'year1970', + expected => [ + { + 'volid' => "$storeid:backup/vzdump-lxc-321-1970_01_01-00_01_23.tar.zst", + 'ctime' => 83, + 'mark' => 'remove', + 'type' => 'lxc', + 'vmid' => 321, + }, + { + 'volid' => "$storeid:backup/vzdump-lxc-321-2070_01_01-00_01_00.tar.zst", + 'ctime' => 60*60*24 * (365*100 + 25) + 60, + 'mark' => 'keep', + 'type' => 'lxc', + 'vmid' => 321, + }, + ], + }, + { + description => 'last=1, ne ID, year before 2000', + keep => { + 'keep-last' => 1, + }, + list => 'novmid', + expected => [ + { + 'volid' => "$storeid:backup/vzdump-lxc-novmid.tar.gz", + 'ctime' => 1234, + 'mark' => 'protected', + 'type' => 'lxc', + }, + ], + }, +]; + +plan tests => scalar @$tests; + +for my $tt (@$tests) { + + my $got = eval { + $current_list = $tt->{list} // 'default'; + my $res = PVE::Storage::Plugin->prune_backups($tt->{scfg}, $storeid, $tt->{keep}, $tt->{vmid}, $tt->{type}, 1); + return [ sort { $a->{volid} cmp $b->{volid} } @{$res} ]; + }; + $got = $@ if $@; + + is_deeply($got, $tt->{expected}, $tt->{description}) || diag(explain($got)); +} + +done_testing(); + +1; diff --git a/test/run_plugin_tests.pl b/test/run_plugin_tests.pl index 54322bb..d33429a 100755 --- a/test/run_plugin_tests.pl +++ b/test/run_plugin_tests.pl @@ -16,6 +16,7 @@ my $res = $harness->runtests( "path_to_volume_id_test.pm", "get_subdir_test.pm", "filesystem_path_test.pm", + "prune_backups_test.pm", ); exit -1 if !$res || $res->{failed} || $res->{parse_errors}; -- 2.20.1