From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 0586E1FF391 for ; Wed, 12 Jun 2024 16:46:05 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 3F43D136A5; Wed, 12 Jun 2024 16:46:40 +0200 (CEST) From: Filip Schauer To: pve-devel@lists.proxmox.com Date: Wed, 12 Jun 2024 16:45:49 +0200 Message-Id: <20240612144549.165990-1-f.schauer@proxmox.com> X-Mailer: git-send-email 2.39.2 MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.059 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [content.pm, config.pm, scan.pm, movevolume.pm, prunebackups.pm, storage.pm, filerestore.pm, proxmox.com, status.pm, pvesm.pm] Subject: [pve-devel] [PATCH storage] add API method to move a volume between storages X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox VE development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pve-devel-bounces@lists.proxmox.com Sender: "pve-devel" Add the ability to move a backup, ISO, container template or snippet between storages of a node via an API method. Moving a VMA backup to a Proxmox Backup Server requires the proxmox-vma-to-pbs package to be installed. Currently only VMA backups can be moved to a Proxmox Backup Server and moving backups from a Proxmox Backup Server is not yet supported. The method can be called from the PVE shell with `pvesm move`: # pvesm move pvesm move local:backup/vzdump-lxc-102-2024_05_29-17_05_27.tar.zst pbs Or use curl to call the API method: curl https://$APINODE:8006/api2/json/nodes/$TARGETNODE/storage/$TARGETSTORAGE/move \ --insecure --cookie "$( --- This patch depends on [PATCH backup-qemu/vma-to-pbs 0/2] add support for notes and logs https://lists.proxmox.com/pipermail/pbs-devel/2024-May/009445.html src/PVE/API2/Storage/Makefile | 2 +- src/PVE/API2/Storage/MoveVolume.pm | 61 +++++++++ src/PVE/API2/Storage/Status.pm | 7 ++ src/PVE/CLI/pvesm.pm | 33 +++++ src/PVE/Storage.pm | 195 +++++++++++++++++++++++++---- 5 files changed, 271 insertions(+), 27 deletions(-) create mode 100644 src/PVE/API2/Storage/MoveVolume.pm diff --git a/src/PVE/API2/Storage/Makefile b/src/PVE/API2/Storage/Makefile index 1705080..11f3c95 100644 --- a/src/PVE/API2/Storage/Makefile +++ b/src/PVE/API2/Storage/Makefile @@ -1,5 +1,5 @@ -SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm +SOURCES= Content.pm Status.pm Config.pm PruneBackups.pm Scan.pm FileRestore.pm MoveVolume.pm .PHONY: install install: diff --git a/src/PVE/API2/Storage/MoveVolume.pm b/src/PVE/API2/Storage/MoveVolume.pm new file mode 100644 index 0000000..52447a4 --- /dev/null +++ b/src/PVE/API2/Storage/MoveVolume.pm @@ -0,0 +1,61 @@ +package PVE::API2::Storage::MoveVolume; + +use strict; +use warnings; + +use PVE::JSONSchema qw(get_standard_option); +use PVE::RESTHandler; +use PVE::RPCEnvironment; +use PVE::Storage; + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'move', + path => '', + method => 'POST', + description => "Move a volume from one storage to another", + permissions => { + description => "You need the 'Datastore.Allocate' privilege on the storages.", + user => 'all', + }, + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + 'source-volume' => { + description => "Source volume", + type => 'string', + }, + 'storage' => get_standard_option('pve-storage-id', { + completion => \&PVE::Storage::complete_storage_enabled, + description => 'Target storage', + }), + }, + }, + returns => { type => 'string' }, + code => sub { + my ($param) = @_; + + my $cfg = PVE::Storage::config(); + my $source_volume = $param->{'source-volume'}; + my $target_storeid = $param->{'storage'}; + + my ($source_storeid, undef) = PVE::Storage::parse_volume_id($source_volume, 0); + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + $rpcenv->check($authuser, "/storage/$source_storeid", ["Datastore.Allocate"]); + $rpcenv->check($authuser, "/storage/$target_storeid", ["Datastore.Allocate"]); + + my $worker = sub { + PVE::Storage::volume_move($cfg, $source_volume, $target_storeid); + }; + + return $rpcenv->fork_worker('move_volume', '', $authuser, $worker); + }}); + +1; diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm index dc6cc69..6c816b7 100644 --- a/src/PVE/API2/Storage/Status.pm +++ b/src/PVE/API2/Storage/Status.pm @@ -18,6 +18,7 @@ use PVE::Tools qw(run_command); use PVE::API2::Storage::Content; use PVE::API2::Storage::FileRestore; +use PVE::API2::Storage::MoveVolume; use PVE::API2::Storage::PruneBackups; use PVE::Storage; @@ -28,6 +29,11 @@ __PACKAGE__->register_method ({ path => '{storage}/prunebackups', }); +__PACKAGE__->register_method ({ + subclass => "PVE::API2::Storage::MoveVolume", + path => '{storage}/move', +}); + __PACKAGE__->register_method ({ subclass => "PVE::API2::Storage::Content", # set fragment delimiter (no subdirs) - we need that, because volume @@ -233,6 +239,7 @@ __PACKAGE__->register_method ({ { subdir => 'rrddata' }, { subdir => 'status' }, { subdir => 'upload' }, + { subdir => 'move' }, ]; return $res; diff --git a/src/PVE/CLI/pvesm.pm b/src/PVE/CLI/pvesm.pm index 9b9676b..4c042aa 100755 --- a/src/PVE/CLI/pvesm.pm +++ b/src/PVE/CLI/pvesm.pm @@ -20,6 +20,7 @@ use PVE::Storage; use PVE::Tools qw(extract_param); use PVE::API2::Storage::Config; use PVE::API2::Storage::Content; +use PVE::API2::Storage::MoveVolume; use PVE::API2::Storage::PruneBackups; use PVE::API2::Storage::Scan; use PVE::API2::Storage::Status; @@ -480,6 +481,37 @@ __PACKAGE__->register_method ({ } }); +__PACKAGE__->register_method ({ + name => 'move', + path => 'move', + method => 'POST', + description => "Move a volume from one storage to another", + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + 'source-volume' => { + description => "Source volume", + type => 'string', + }, + 'storage' => get_standard_option('pve-storage-id', { + completion => \&PVE::Storage::complete_storage_enabled, + description => 'Target storage', + }), + }, + }, + returns => { type => 'null' }, + code => sub { + my ($param) = @_; + + PVE::API2::Storage::MoveVolume->move($param); + + return; + }, +}); + __PACKAGE__->register_method ({ name => 'prunebackups', path => 'prunebackups', @@ -690,6 +722,7 @@ our $cmddef = { print "APIVER $res->{apiver}\n"; print "APIAGE $res->{apiage}\n"; }], + 'move' => [ __PACKAGE__, 'move', ['source-volume', 'storage'], { node => $nodename } ], 'prune-backups' => [ __PACKAGE__, 'prunebackups', ['storage'], { node => $nodename }, sub { my $res = shift; diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index f19a115..fe8a5e0 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -9,6 +9,7 @@ use IO::File; use IO::Socket::IP; use IPC::Open3; use File::Basename; +use File::Copy qw(move); use File::Path; use Cwd 'abs_path'; use Socket; @@ -1620,14 +1621,20 @@ sub archive_info { } sub archive_remove { - my ($archive_path) = @_; + my ($archive_path, $ignore_protected) = @_; + + my $protection_path = protection_file_path($archive_path); die "cannot remove protected archive '$archive_path'\n" - if -e protection_file_path($archive_path); + if !$ignore_protected && -e $protection_path; unlink $archive_path or $! == ENOENT or die "removing archive $archive_path failed: $!\n"; archive_auxiliaries_remove($archive_path); + + if (-e $protection_path) { + unlink $protection_path or $! == ENOENT or log_warn("Removing protection file failed: $!"); + } } sub archive_auxiliaries_remove { @@ -1680,6 +1687,45 @@ sub extract_vzdump_config_tar { return wantarray ? ($raw, $file) : $raw; } +sub decompress_archive_into_pipe { + my ($archive, $cmd, $outfunc) = @_; + + my $info = archive_info($archive); + die "archive is not compressed\n" if !$info->{compression}; + my $decompressor = $info->{decompressor}; + my $full_cmd = [ [@$decompressor, $archive], $cmd ]; + + # lzop/zcat exits with 1 when the pipe is closed early, + # detect this and ignore the exit code later + my $broken_pipe; + my $errstring; + my $err = sub { + my $output = shift; + if ( + $output =~ m/lzop: Broken pipe: / + || $output =~ m/gzip: stdout: Broken pipe/ + || $output =~ m/zstd: error 70 : Write error.*Broken pipe/ + ) { + $broken_pipe = 1; + } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { + $errstring = "failed to decompress archive: $output\n"; + } + }; + + my $rc = eval { run_command($full_cmd, outfunc => $outfunc, errfunc => $err, noerr => 1) }; + my $rerr = $@; + + $broken_pipe ||= $rc == 141; # broken pipe from cmd POV + + if (!$errstring && !$broken_pipe && $rc != 0) { + die "$rerr\n" if $rerr; + die "archive decompression failed with exit code $rc\n"; + } + die "$errstring\n" if $errstring; + + return; +} + sub extract_vzdump_config_vma { my ($archive, $comp) = @_; @@ -1691,30 +1737,7 @@ sub extract_vzdump_config_vma { my $decompressor = $info->{decompressor}; if ($comp) { - my $cmd = [ [@$decompressor, $archive], ["vma", "config", "-"] ]; - - # lzop/zcat exits with 1 when the pipe is closed early by vma, detect this and ignore the exit code later - my $broken_pipe; - my $errstring; - my $err = sub { - my $output = shift; - if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/) { - $broken_pipe = 1; - } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { - $errstring = "Failed to extract config from VMA archive: $output\n"; - } - }; - - my $rc = eval { run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1) }; - my $rerr = $@; - - $broken_pipe ||= $rc == 141; # broken pipe from vma POV - - if (!$errstring && !$broken_pipe && $rc != 0) { - die "$rerr\n" if $rerr; - die "config extraction failed with exit code $rc\n"; - } - die "$errstring\n" if $errstring; + decompress_archive_into_pipe($archive, ["vma", "config", "-"], $out); } else { run_command(["vma", "config", $archive], outfunc => $out); } @@ -1753,6 +1776,126 @@ sub extract_vzdump_config { } } +sub volume_move { + my ($cfg, $source_volid, $target_storeid) = @_; + + my ($source_storeid, $source_volname) = parse_volume_id($source_volid, 0); + + die "source and target storage cannot be the same\n" if ($source_storeid eq $target_storeid); + + activate_storage($cfg, $source_storeid); + my $source_scfg = storage_config($cfg, $source_storeid); + my $source_plugin = PVE::Storage::Plugin->lookup($source_scfg->{type}); + my ($vtype) = $source_plugin->parse_volname($source_volname); + + die "source storage '$source_storeid' does not support volumes of type '$vtype'\n" + if !$source_scfg->{content}->{$vtype}; + + activate_storage($cfg, $target_storeid); + my $target_scfg = storage_config($cfg, $target_storeid); + die "target storage '$target_storeid' does not support volumes of type '$vtype'\n" + if !$target_scfg->{content}->{$vtype}; + + if ($vtype eq 'backup' || $vtype eq 'iso' || $vtype eq 'vztmpl' || $vtype eq 'snippets') { + my $target_plugin = PVE::Storage::Plugin->lookup($target_scfg->{type}); + + die "moving a backup from a Proxmox Backup Server is not yet supported\n" + if ($vtype eq 'backup' && $source_scfg->{type} eq 'pbs'); + + my $source_path = $source_plugin->filesystem_path($source_scfg, $source_volname); + die "$source_path does not exist" if (!-e $source_path); + my $source_dirname = dirname($source_path); + + return if $vtype ne 'backup'; + + if ($target_scfg->{type} eq 'pbs') { + my $info = archive_info($source_path); + die "moving non-VMA backups to a Proxmox Backup Server is not yet supported\n" + if ($info->{format} ne 'vma'); + + my $repo = PVE::PBSClient::get_repository($target_scfg); + my $vmid = ($source_plugin->parse_volname($source_volname))[2]; + my $fingerprint = $target_scfg->{fingerprint}; + my $password = PVE::Storage::PBSPlugin::pbs_password_file_name( + $target_scfg, $target_storeid); + my $namespace = $target_scfg->{namespace}; + my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name( + $target_scfg, $target_storeid); + my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name( + $target_scfg, $target_storeid); + + my $comp = $info->{compression}; + my $backup_time = $info->{ctime}; + my $log_file_path = "$source_dirname/$info->{logfilename}"; + my $notes_file_path = "$source_dirname/$info->{notesfilename}"; + + my $vma_to_pbs_cmd = [ + "vma-to-pbs", + "--repository", $repo, + "--vmid", $vmid, + "--fingerprint", $fingerprint, + "--password-file", $password, + "--backup-time", $backup_time, + "--compress", + ]; + + push @$vma_to_pbs_cmd, "--ns", $namespace if $namespace; + push @$vma_to_pbs_cmd, "--log-file", $log_file_path if -e $log_file_path; + push @$vma_to_pbs_cmd, "--notes-file", $notes_file_path if -e $notes_file_path; + push @$vma_to_pbs_cmd, "--encrypt", "--keyfile", $keyfile if -e $keyfile; + push @$vma_to_pbs_cmd, "--master-keyfile", $master_keyfile if -e $master_keyfile; + + if ($comp) { + decompress_archive_into_pipe($source_path, $vma_to_pbs_cmd); + } else { + push @$vma_to_pbs_cmd, $source_path; + run_command($vma_to_pbs_cmd); + } + + my $protection_source_path = protection_file_path($source_path); + + if (-e $protection_source_path) { + my $target_volid = PVE::Storage::PBSPlugin::print_volid( + $target_storeid, 'vm', $vmid, $backup_time); + my (undef, $target_volname) = parse_volume_id($target_volid, 0); + $target_plugin->update_volume_attribute( + $target_scfg, $target_storeid, $target_volname, 'protected', 1); + } + + archive_remove($source_path, 1); + } else { + my $target_path = $target_plugin->filesystem_path($target_scfg, $source_volname); + + move($source_path, $target_path) or die "failed to move $vtype: $!"; + + my $target_dirname = dirname($target_path); + my $info = archive_info($source_path); + + for my $type (qw(log notes)) { + my $filename = $info->{"${type}filename"} or next; + $source_path = "$source_dirname/$filename"; + $target_path = "$target_dirname/$filename"; + move($source_path, $target_path) or die "moving backup $type file failed: $!" + if -e $source_path; + } + + my $protection_source_path = protection_file_path($source_path); + + if (-e $protection_source_path) { + my $protection_target_path = protection_file_path($target_path); + move($protection_source_path, $protection_target_path) + or die "moving backup protection file failed: $!"; + } + } + } elsif ($vtype eq 'images') { + die "use pct move-volume or qm disk move\n"; + } elsif ($vtype eq 'rootdir') { + die "cannot move OpenVZ rootdir\n"; + } + + return; +} + sub prune_backups { my ($cfg, $storeid, $keep, $vmid, $type, $dryrun, $logfunc) = @_; -- 2.39.2 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel