From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 4D77D6179C for ; Thu, 9 Jul 2020 14:46:02 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 49D03128D8 for ; Thu, 9 Jul 2020 14:46:02 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 0DC261287C for ; Thu, 9 Jul 2020 14:45:56 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id C59B9444AC for ; Thu, 9 Jul 2020 14:45:55 +0200 (CEST) From: Fabian Ebner To: pve-devel@lists.proxmox.com Date: Thu, 9 Jul 2020 14:45:43 +0200 Message-Id: <20200709124547.2913-4-f.ebner@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200709124547.2913-1-f.ebner@proxmox.com> References: <20200709124547.2913-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.120 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v4 storage 3/7] Add API and pvesm call for prune_backups X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Jul 2020 12:46:02 -0000 For the pvesm call use a wrapper and a --dry-run option to redirect to the correct API call. Signed-off-by: Fabian Ebner --- Changes from v3: * allow filtering by type * pvesm: redirect to correct API call according to --dry-run PVE/API2/Storage/Makefile | 2 +- PVE/API2/Storage/PruneBackups.pm | 164 +++++++++++++++++++++++++++++++ PVE/API2/Storage/Status.pm | 7 ++ PVE/CLI/pvesm.pm | 126 ++++++++++++++++++++++++ 4 files changed, 298 insertions(+), 1 deletion(-) create mode 100644 PVE/API2/Storage/PruneBackups.pm diff --git a/PVE/API2/Storage/Makefile b/PVE/API2/Storage/Makefile index a33525b..3f667e8 100644 --- a/PVE/API2/Storage/Makefile +++ b/PVE/API2/Storage/Makefile @@ -1,5 +1,5 @@ -SOURCES= Content.pm Status.pm Config.pm +SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm .PHONY: install install: diff --git a/PVE/API2/Storage/PruneBackups.pm b/PVE/API2/Storage/PruneBackups.pm new file mode 100644 index 0000000..a84d1c8 --- /dev/null +++ b/PVE/API2/Storage/PruneBackups.pm @@ -0,0 +1,164 @@ +package PVE::API2::Storage::PruneBackups; + +use strict; +use warnings; + +use PVE::Cluster; +use PVE::JSONSchema qw(get_standard_option); +use PVE::RESTHandler; +use PVE::RPCEnvironment; +use PVE::Storage; +use PVE::Tools qw(extract_param); + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'dryrun', + path => '', + method => 'GET', + description => "Get prune information for backups. NOTE: this is only a preview and might not be exactly " . + "what a subsequent prune call does, if the hour changes or if backups are removed/added " . + "in the meantime.", + permissions => { + check => ['perm', '/storage/{storage}', ['Datastore.Audit', 'Datastore.AllocateSpace'], any => 1], + }, + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + storage => get_standard_option('pve-storage-id', { + completion => \&PVE::Storage::complete_storage_enabled, + }), + 'prune-backups' => get_standard_option('prune-backups', { + description => "Use these retention options instead of those from the storage configuration.", + optional => 1, + }), + type => { + description => "Either 'qemu' or 'lxc'. Only consider backups for guests of this type.", + type => 'string', + optional => 1, + enum => ['qemu', 'lxc'], + }, + vmid => get_standard_option('pve-vmid', { + description => "Only consider backups for this guest.", + optional => 1, + completion => \&PVE::Cluster::complete_vmid, + }), + }, + }, + returns => { + type => 'array', + items => { + type => 'object', + properties => { + volid => { + description => "Backup volume ID.", + type => 'string', + }, + 'ctime' => { + description => "Creation time of the backup (seconds since the UNIX epoch).", + type => 'integer', + }, + 'mark' => { + description => "Whether the backup would be kept or removed. For backups that don't " . + "use the standard naming scheme, it's 'protected'.", + type => 'string', + }, + type => { + description => "One of 'qemu', 'lxc', 'openvz' or 'unknown'.", + type => 'string', + }, + 'vmid' => { + description => "The VM the backup belongs to.", + type => 'integer', + optional => 1, + }, + }, + }, + }, + code => sub { + my ($param) = @_; + + my $cfg = PVE::Storage::config(); + + my $vmid = extract_param($param, 'vmid'); + my $type = extract_param($param, 'type'); + my $storeid = extract_param($param, 'storage'); + + my $prune_backups = extract_param($param, 'prune-backups'); + $prune_backups = PVE::JSONSchema::parse_property_string('prune-backups', $prune_backups) + if defined($prune_backups); + + return PVE::Storage::prune_backups($cfg, $storeid, $prune_backups, $vmid, $type, 1); + }}); + +__PACKAGE__->register_method ({ + name => 'delete', + path => '', + method => 'DELETE', + description => "Prune backups. Only those using the standard naming scheme are considered.", + permissions => { + description => "You need the 'Datastore.Allocate' privilege on the storage " . + "(or if a VM ID is specified, 'Datastore.AllocateSpace' and 'VM.Backup' for the VM).", + user => 'all', + }, + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + storage => get_standard_option('pve-storage-id', { + completion => \&PVE::Storage::complete_storage, + }), + 'prune-backups' => get_standard_option('prune-backups', { + description => "Use these retention options instead of those from the storage configuration.", + }), + type => { + description => "Either 'qemu' or 'lxc'. Only consider backups for guests of this type.", + type => 'string', + optional => 1, + enum => ['qemu', 'lxc'], + }, + vmid => get_standard_option('pve-vmid', { + description => "Only prune backups for this VM.", + completion => \&PVE::Cluster::complete_vmid, + optional => 1, + }), + }, + }, + returns => { type => 'string' }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $cfg = PVE::Storage::config(); + + my $vmid = extract_param($param, 'vmid'); + my $type = extract_param($param, 'type'); + my $storeid = extract_param($param, 'storage'); + + my $prune_backups = extract_param($param, 'prune-backups'); + $prune_backups = PVE::JSONSchema::parse_property_string('prune-backups', $prune_backups) + if defined($prune_backups); + + if (defined($vmid)) { + $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); + $rpcenv->check($authuser, "/vms/$vmid", ['VM.Backup']); + } else { + $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.Allocate']); + } + + my $id = (defined($vmid) ? "$vmid@" : '') . $storeid; + my $worker = sub { + PVE::Storage::prune_backups($cfg, $storeid, $prune_backups, $vmid, $type, 0); + }; + + return $rpcenv->fork_worker('prunebackups', $id, $authuser, $worker); + }}); + +1; diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm index d9d9b36..d12643f 100644 --- a/PVE/API2/Storage/Status.pm +++ b/PVE/API2/Storage/Status.pm @@ -11,6 +11,7 @@ use PVE::Cluster; use PVE::RRD; use PVE::Storage; use PVE::API2::Storage::Content; +use PVE::API2::Storage::PruneBackups; use PVE::RESTHandler; use PVE::RPCEnvironment; use PVE::JSONSchema qw(get_standard_option); @@ -18,6 +19,11 @@ use PVE::Exception qw(raise_param_exc); use base qw(PVE::RESTHandler); +__PACKAGE__->register_method ({ + subclass => "PVE::API2::Storage::PruneBackups", + path => '{storage}/prunebackups', +}); + __PACKAGE__->register_method ({ subclass => "PVE::API2::Storage::Content", # set fragment delimiter (no subdirs) - we need that, because volume @@ -214,6 +220,7 @@ __PACKAGE__->register_method ({ { subdir => 'upload' }, { subdir => 'rrd' }, { subdir => 'rrddata' }, + { subdir => 'prunebackups' }, ]; return $res; diff --git a/PVE/CLI/pvesm.pm b/PVE/CLI/pvesm.pm index f223c92..405224c 100755 --- a/PVE/CLI/pvesm.pm +++ b/PVE/CLI/pvesm.pm @@ -12,8 +12,10 @@ use PVE::Cluster; use PVE::INotify; use PVE::RPCEnvironment; use PVE::Storage; +use PVE::Tools qw(extract_param); use PVE::API2::Storage::Config; use PVE::API2::Storage::Content; +use PVE::API2::Storage::PruneBackups; use PVE::API2::Storage::Status; use PVE::JSONSchema qw(get_standard_option); use PVE::PTY; @@ -720,6 +722,99 @@ __PACKAGE__->register_method ({ return PVE::Storage::scan_zfs(); }}); +__PACKAGE__->register_method ({ + name => 'prunebackups', + path => 'prunebackups', + method => 'GET', + description => "Prune backups. This is only a wrapper for the proper API endpoints.", + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => { + 'dry-run' => { + description => "Only show what would be pruned, don't delete anything.", + type => 'boolean', + optional => 1, + }, + node => get_standard_option('pve-node'), + storage => get_standard_option('pve-storage-id', { + completion => \&PVE::Storage::complete_storage_enabled, + }), + 'prune-backups' => get_standard_option('prune-backups', { + description => "Use these retention options instead of those from the storage configuration.", + optional => 1, + }), + type => { + description => "Either 'qemu' or 'lxc'. Only consider backups for guests of this type.", + type => 'string', + optional => 1, + enum => ['qemu', 'lxc'], + }, + vmid => get_standard_option('pve-vmid', { + description => "Only consider backups for this guest.", + optional => 1, + completion => \&PVE::Cluster::complete_vmid, + }), + }, + }, + returns => { + type => 'object', + properties => { + dryrun => { + description => 'If it was a dry run or not. The list will only be defined in that case.', + type => 'boolean', + }, + list => { + type => 'array', + items => { + type => 'object', + properties => { + volid => { + description => "Backup volume ID.", + type => 'string', + }, + 'ctime' => { + description => "Creation time of the backup (seconds since the UNIX epoch).", + type => 'integer', + }, + 'mark' => { + description => "Whether the backup would be kept or removed. For backups that don't " . + "use the standard naming scheme, it's 'protected'.", + type => 'string', + }, + type => { + description => "One of 'qemu', 'lxc', 'openvz' or 'unknown'.", + type => 'string', + }, + 'vmid' => { + description => "The VM the backup belongs to.", + type => 'integer', + optional => 1, + }, + }, + }, + }, + }, + }, + code => sub { + my ($param) = @_; + + my $dryrun = extract_param($param, 'dry-run') ? 1 : 0; + + my $list = []; + if ($dryrun) { + $list = PVE::API2::Storage::PruneBackups->dryrun($param); + } else { + PVE::API2::Storage::PruneBackups->delete($param); + } + + return { + dryrun => $dryrun, + list => $list, + }; + }}); + our $cmddef = { add => [ "PVE::API2::Storage::Config", 'create', ['type', 'storage'] ], set => [ "PVE::API2::Storage::Config", 'update', ['storage'] ], @@ -819,6 +914,37 @@ our $cmddef = { print "APIVER $res->{apiver}\n"; print "APIAGE $res->{apiage}\n"; }], + 'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub { + my $res = shift; + + my ($dryrun, $list) = ($res->{dryrun}, $res->{list}); + + return if !$dryrun; + + print "NOTE: this is only a preview and might not be exactly what a subsequent prune call does,\n" . + "if the hour changes or if backups are removed/added in the meantime.\n\n"; + + my @sorted = sort { + my $vmcmp = PVE::Tools::safe_compare($a->{vmid}, $b->{vmid}, sub { $_[0] <=> $_[1] }); + return $vmcmp if $vmcmp ne 0; + return $a->{ctime} <=> $b->{ctime}; + } @{$list}; + + my $maxlen = 0; + foreach my $backup (@sorted) { + my $volid = $backup->{volid}; + $maxlen = length($volid) if length($volid) > $maxlen; + } + $maxlen+=1; + + printf("%-${maxlen}s %15s %10s\n", 'Backup', 'Backup-ID', 'Prune-Mark'); + foreach my $backup (@sorted) { + my $type = $backup->{type}; + my $vmid = $backup->{vmid}; + my $backup_id = defined($vmid) ? "$type/$vmid" : "$type"; + printf("%-${maxlen}s %15s %10s\n", $backup->{volid}, $backup_id, $backup->{mark}); + } + }], }; 1; -- 2.20.1