From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <pve-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
	by lore.proxmox.com (Postfix) with ESMTPS id F09241FF16F
	for <inbox@lore.proxmox.com>; Thu, 13 Feb 2025 18:21:06 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
	by firstgate.proxmox.com (Proxmox) with ESMTP id 34430BA89;
	Thu, 13 Feb 2025 18:21:01 +0100 (CET)
Message-ID: <a6d503c5-486f-4c2c-813b-39866a054d13@proxmox.com>
Date: Thu, 13 Feb 2025 18:20:56 +0100
MIME-Version: 1.0
User-Agent: Mozilla Thunderbird
To: Proxmox VE development discussion <pve-devel@lists.proxmox.com>,
 Filip Schauer <f.schauer@proxmox.com>
References: <20250120112842.36450-1-f.schauer@proxmox.com>
 <20250120112842.36450-6-f.schauer@proxmox.com>
Content-Language: en-US
From: Fiona Ebner <f.ebner@proxmox.com>
In-Reply-To: <20250120112842.36450-6-f.schauer@proxmox.com>
X-SPAM-LEVEL: Spam detection results:  0
 AWL -0.047 Adjusted score from AWL reputation of From: address
 BAYES_00                 -1.9 Bayes spam probability is 0 to 1%
 DMARC_MISSING             0.1 Missing DMARC policy
 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
 RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to
 Validity was blocked. See
 https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more
 information.
 RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to
 Validity was blocked. See
 https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more
 information.
 RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to
 Validity was blocked. See
 https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more
 information.
 SPF_HELO_NONE           0.001 SPF: HELO does not publish an SPF Record
 SPF_PASS               -0.001 SPF: sender matches SPF record
Subject: Re: [pve-devel] [PATCH storage v6 5/7] support moving VMA backups
 to PBS
X-BeenThere: pve-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox VE development discussion <pve-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pve-devel>, 
 <mailto:pve-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pve-devel/>
List-Post: <mailto:pve-devel@lists.proxmox.com>
List-Help: <mailto:pve-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel>, 
 <mailto:pve-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox VE development discussion <pve-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pve-devel-bounces@lists.proxmox.com
Sender: "pve-devel" <pve-devel-bounces@lists.proxmox.com>

Am 20.01.25 um 12:28 schrieb Filip Schauer:
> Extend the move API to support moving VMA backups to a Proxmox Backup
> Server.
> 
> Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
> ---
>  debian/control                  |  1 +
>  src/PVE/API2/Storage/Content.pm | 53 +++++++++++++++------------
>  src/PVE/Storage/PBSPlugin.pm    | 65 +++++++++++++++++++++++++++++++++
>  3 files changed, 96 insertions(+), 23 deletions(-)
> 
> diff --git a/debian/control b/debian/control
> index 4e1a046..0883777 100644
> --- a/debian/control
> +++ b/debian/control
> @@ -46,6 +46,7 @@ Depends: bzip2,
>           nfs-common,
>           proxmox-backup-client (>= 2.1.10~),
>           proxmox-backup-file-restore,
> +         proxmox-vma-to-pbs (>= 0.0.2),
>           pve-cluster (>= 5.0-32),
>           smartmontools,
>           smbclient,
> diff --git a/src/PVE/API2/Storage/Content.pm b/src/PVE/API2/Storage/Content.pm
> index 9ee3c51..c0e2a4e 100644
> --- a/src/PVE/API2/Storage/Content.pm
> +++ b/src/PVE/API2/Storage/Content.pm
> @@ -10,6 +10,7 @@ use File::Copy qw(copy move);
>  use PVE::SafeSyslog;
>  use PVE::Cluster;
>  use PVE::Storage;
> +use PVE::Storage::PBSPlugin;
>  use PVE::INotify;
>  use PVE::Exception qw(raise_param_exc);
>  use PVE::RPCEnvironment;
> @@ -557,12 +558,12 @@ __PACKAGE__->register_method ({
>  	my $user = $rpcenv->get_user();
>  
>  	PVE::Storage::check_volume_access($rpcenv, $user, $cfg, undef, $src_volid);
> +	my $src_cfg = PVE::Storage::storage_config($cfg, $src_storeid);
>  
>  	if ($delete) {
>  	    $rpcenv->check($user, "/storage/$src_storeid", ["Datastore.Allocate"]);
>  
>  	    if ($vtype eq 'backup') {
> -		my $src_cfg = PVE::Storage::storage_config($cfg, $src_storeid);
>  		my $src_plugin = PVE::Storage::Plugin->lookup($src_cfg->{type});
>  		my $protected = $src_plugin->get_volume_attribute($src_cfg, $src_storeid, $volname, 'protected');
>  		die "cannot delete protected backup\n" if $protected;
> @@ -577,30 +578,36 @@ __PACKAGE__->register_method ({
>  
>  	my $worker = sub {
>  	    PVE::Storage::storage_check_enabled($cfg, $dst_storeid, $dst_node);
> -	    my $sshinfo;
>  
> -	    if ($src_node eq $dst_node) {
> -		$sshinfo = {
> -		    ip => "localhost",
> -		    name => $dst_node,
> -		};
> +	    my $dst_cfg = PVE::Storage::storage_config($cfg, $dst_storeid);
> +	    if ($vtype eq 'backup' && $dst_cfg->{type} eq 'pbs') {

Maybe check that the source plugin is path-based here to rule out some
other third-party backup storages early.

> +		PVE::Storage::PBSPlugin::vma_to_pbs($src_cfg, $src_volid, $dst_cfg, $dst_storeid);
>  	    } else {
> -		$sshinfo = PVE::SSHInfo::get_ssh_info($dst_node);
> -	    }
> -
> -	    my $opts = { 'target_volname' => $volname };
> -	    PVE::Storage::storage_migrate($cfg, $src_volid, $sshinfo, $dst_storeid, $opts);
> -
> -	    if ($delete) {
> -		my $src_path = PVE::Storage::abs_filesystem_path($cfg, $src_volid);
> -		PVE::Storage::archive_remove($src_path, 1);
> -	    }
> -
> -	    if ($src_node eq $dst_node) {
> -		print "Moved volume '$src_volid' to '$dst_storeid'\n";
> -	    } else {
> -		print "Moved volume '$src_volid' on node '$src_node'"
> -		    ." to '$dst_storeid' on node '$dst_node'\n";
> +		my $sshinfo;
> +
> +		if ($src_node eq $dst_node) {
> +		    $sshinfo = {
> +			ip => "localhost",
> +			name => $dst_node,
> +		    };
> +		} else {
> +		    $sshinfo = PVE::SSHInfo::get_ssh_info($dst_node);
> +		}
> +
> +		my $opts = { 'target_volname' => $volname };
> +		PVE::Storage::storage_migrate($cfg, $src_volid, $sshinfo, $dst_storeid, $opts);
> +
> +		if ($delete) {
> +		    my $src_path = PVE::Storage::abs_filesystem_path($cfg, $src_volid);
> +		    PVE::Storage::archive_remove($src_path, 1);
> +		}
> +
> +		if ($src_node eq $dst_node) {
> +		    print "Moved volume '$src_volid' to '$dst_storeid'\n";
> +		} else {
> +		    print "Moved volume '$src_volid' on node '$src_node'"
> +			." to '$dst_storeid' on node '$dst_node'\n";
> +		}
>  	    }
>  	};
>  
> diff --git a/src/PVE/Storage/PBSPlugin.pm b/src/PVE/Storage/PBSPlugin.pm
> index 0808bcc..4f8a05d 100644
> --- a/src/PVE/Storage/PBSPlugin.pm
> +++ b/src/PVE/Storage/PBSPlugin.pm
> @@ -6,6 +6,7 @@ use strict;
>  use warnings;
>  
>  use Fcntl qw(F_GETFD F_SETFD FD_CLOEXEC);
> +use File::Basename;
>  use IO::File;
>  use JSON;
>  use MIME::Base64 qw(decode_base64);
> @@ -971,4 +972,68 @@ sub volume_has_feature {
>      return undef;
>  }
>  
> +sub vma_to_pbs {
> +    my ($source_scfg, $source_volid, $target_scfg, $target_storeid) = @_;
> +
> +    my $source_plugin = PVE::Storage::Plugin->lookup($source_scfg->{type});

I know I'm being nitpicky, but I'd kinda prefer not having to access the
source plugin here. The required parameters could be passed in.

> +    my $target_plugin = PVE::Storage::Plugin->lookup($target_scfg->{type});
> +    my ($source_storeid, $source_volname) = PVE::Storage::parse_volume_id($source_volid, 0);
> +    my $source_path = $source_plugin->path($source_scfg, $source_volname, $source_storeid);
> +    my $info = PVE::Storage::archive_info($source_path);
> +    die "moving non-VMA backups to a Proxmox Backup Server is not supported\n"
> +	if $info->{format} ne 'vma';
> +
> +    my $repo = PVE::PBSClient::get_repository($target_scfg);
> +    my $vmid = ($source_plugin->parse_volname($source_volname))[2];
> +    my $fingerprint = $target_scfg->{fingerprint};
> +    my $password = PVE::Storage::PBSPlugin::pbs_password_file_name($target_scfg, $target_storeid);
> +    my $namespace = $target_scfg->{namespace};
> +    my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name(
> +	$target_scfg, $target_storeid);
> +    my $master_keyfile = PVE::Storage::PBSPlugin::pbs_master_pubkey_file_name(
> +	$target_scfg, $target_storeid);
> +
> +    my $comp = $info->{compression};
> +    my $backup_time = $info->{ctime};
> +    my $source_dirname = dirname($source_path);
> +    my $log_file_path = "$source_dirname/$info->{logfilename}";
> +    my $notes_file_path = "$source_dirname/$info->{notesfilename}";
> +
> +    my $vma_to_pbs_cmd = [
> +	"vma-to-pbs",
> +	"--repository", $repo,
> +	"--vmid", $vmid,
> +	"--fingerprint", $fingerprint,
> +	"--password-file", $password,
> +	"--backup-time", $backup_time,
> +	"--compress",
> +    ];
> +
> +    push @$vma_to_pbs_cmd, "--ns", $namespace if $namespace;
> +    push @$vma_to_pbs_cmd, "--log-file", $log_file_path if -e $log_file_path;
> +    push @$vma_to_pbs_cmd, "--notes-file", $notes_file_path if -e $notes_file_path;
> +    push @$vma_to_pbs_cmd, "--encrypt", "--keyfile", $keyfile if -e $keyfile;
> +    push @$vma_to_pbs_cmd, "--master-keyfile", $master_keyfile if -e $master_keyfile;
> +
> +    if ($comp) {
> +	PVE::Storage::decompress_archive_into_pipe($source_path, $vma_to_pbs_cmd);
> +    } else {
> +	push @$vma_to_pbs_cmd, $source_path;
> +	run_command($vma_to_pbs_cmd);
> +    }
> +
> +    my $protected = $source_plugin->get_volume_attribute(
> +	$source_scfg, $source_storeid, $source_volname, 'protected');
> +
> +    if ($protected) {
> +	my $target_volid = PVE::Storage::PBSPlugin::print_volid(
> +	    $target_storeid, 'vm', $vmid, $backup_time);
> +	my (undef, $target_volname) = PVE::Storage::parse_volume_id($target_volid, 0);
> +	$target_plugin->update_volume_attribute(
> +	    $target_scfg, $target_storeid, $target_volname, 'protected', 1);
> +    }
> +
> +    return;
> +}
> +
>  1;



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel