From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 734691FF161 for ; Tue, 13 Aug 2024 15:38:11 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 79755351D; Tue, 13 Aug 2024 15:38:26 +0200 (CEST) From: Fiona Ebner To: pve-devel@lists.proxmox.com Date: Tue, 13 Aug 2024 15:28:27 +0200 Message-Id: <20240813132829.117460-24-f.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240813132829.117460-1-f.ebner@proxmox.com> References: <20240813132829.117460-1-f.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.055 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [create.pm] Subject: [pve-devel] [RFC container v2 23/25] backup: implement restore for external providers X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox VE development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pve-devel-bounces@lists.proxmox.com Sender: "pve-devel" First, the provider is asked about what restore mechanism to use. Currently, 'directory' and 'tar' are possible, for restoring either from a directory containing the full filesystem structure (for which rsync is used) or a potentially compressed tar file containing the same. The new functions are copied and adapted from the existing ones for PBS or tar and it might be worth to factor out the common parts. Signed-off-by: Fiona Ebner --- Changes in v2: * Adapt to API changes. src/PVE/LXC/Create.pm | 141 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/src/PVE/LXC/Create.pm b/src/PVE/LXC/Create.pm index 117103c..9d1c337 100644 --- a/src/PVE/LXC/Create.pm +++ b/src/PVE/LXC/Create.pm @@ -25,6 +25,24 @@ sub restore_archive { if ($scfg->{type} eq 'pbs') { return restore_proxmox_backup_archive($storage_cfg, $archive, $rootdir, $conf, $no_unpack_error, $bwlimit); } + my $log_function = sub { + my ($log_level, $message) = @_; + my $prefix = $log_level eq 'err' ? 'ERROR' : uc($log_level); + print "$prefix: $message\n"; + }; + my $backup_provider = + PVE::Storage::new_backup_provider($storage_cfg, $storeid, $log_function); + if ($backup_provider) { + return restore_external_archive( + $backup_provider, + $storeid, + $volname, + $rootdir, + $conf, + $no_unpack_error, + $bwlimit, + ); + } } $archive = PVE::Storage::abs_filesystem_path($storage_cfg, $archive) if $archive ne '-'; @@ -118,6 +136,55 @@ sub restore_tar_archive { die $err if $err && !$no_unpack_error; } +sub restore_external_archive { + my ($backup_provider, $storeid, $volname, $rootdir, $conf, $no_unpack_error, $bwlimit) = @_; + + my ($mechanism, $vmtype) = $backup_provider->restore_get_mechanism($volname, $storeid); + die "cannot restore non-LXC guest of type '$vmtype'\n" if $vmtype ne 'lxc'; + + my $info = $backup_provider->restore_container_init($volname, $storeid, {}); + eval { + if ($mechanism eq 'tar') { + my $tar_path = $info->{'tar-path'} + or die "did not get path to tar file from backup provider\n"; + die "not a regular file '$tar_path'" if !-f $tar_path; + restore_tar_archive($tar_path, $rootdir, $conf, $no_unpack_error, $bwlimit); + } elsif ($mechanism eq 'directory') { + my $directory = $info->{'archive-directory'} + or die "did not get path to archive directory from backup provider\n"; + die "not a directory '$directory'" if !-d $directory; + + my $rsync = ['rsync', '--stats', '-h', '-X', '-A', '--numeric-ids', '-aH', '--delete', + '--no-whole-file', '--sparse', '--one-file-system', '--relative']; + push $rsync->@*, '--bwlimit', $bwlimit if $bwlimit; + push $rsync->@*, "${directory}/./", $rootdir; + + my $transferred = ''; + my $outfunc = sub { + return if $_[0] !~ /^Total transferred file size: (.+)$/; + $transferred = $1; + }; + my $errfunc = sub { log_warn($_[0]); }; + + my $starttime = time(); + PVE::Tools::run_command($rsync, outfunc => $outfunc, errfunc => $errfunc); + my $delay = time () - $starttime; + + print "sync finished - transferred ${transferred} in ${delay}s\n"; + } else { + die "mechanism '$mechanism' requested by backup provider is not supported for LXCs\n"; + } + }; + my $err = $@; + eval { $backup_provider->restore_container_cleanup($volname, $storeid, {}); }; + if (my $cleanup_err = $@) { + die $cleanup_err if !$err; + warn $cleanup_err; + } + die $err if $err; + +} + sub recover_config { my ($storage_cfg, $volid, $vmid) = @_; @@ -126,6 +193,8 @@ sub recover_config { my $scfg = PVE::Storage::storage_check_enabled($storage_cfg, $storeid); if ($scfg->{type} eq 'pbs') { return recover_config_from_proxmox_backup($storage_cfg, $volid, $vmid); + } elsif (PVE::Storage::new_backup_provider($storage_cfg, $storeid, sub {})) { + return recover_config_from_external_backup($storage_cfg, $volid, $vmid); } } @@ -200,6 +269,26 @@ sub recover_config_from_tar { return wantarray ? ($conf, $mp_param) : $conf; } +sub recover_config_from_external_backup { + my ($storage_cfg, $volid, $vmid) = @_; + + $vmid //= 0; + + my $raw = PVE::Storage::extract_vzdump_config($storage_cfg, $volid); + + my $conf = PVE::LXC::Config::parse_pct_config("/lxc/${vmid}.conf" , $raw); + + delete $conf->{snapshots}; + + my $mp_param = {}; + PVE::LXC::Config->foreach_volume($conf, sub { + my ($ms, $mountpoint) = @_; + $mp_param->{$ms} = $conf->{$ms}; + }); + + return wantarray ? ($conf, $mp_param) : $conf; +} + sub restore_configuration { my ($vmid, $storage_cfg, $archive, $rootdir, $conf, $restricted, $unique, $skip_fw) = @_; @@ -209,6 +298,26 @@ sub restore_configuration { if ($scfg->{type} eq 'pbs') { return restore_configuration_from_proxmox_backup($vmid, $storage_cfg, $archive, $rootdir, $conf, $restricted, $unique, $skip_fw); } + my $log_function = sub { + my ($log_level, $message) = @_; + my $prefix = $log_level eq 'err' ? 'ERROR' : uc($log_level); + print "$prefix: $message\n"; + }; + my $backup_provider = + PVE::Storage::new_backup_provider($storage_cfg, $storeid, $log_function); + if ($backup_provider) { + return restore_configuration_from_external_backup( + $backup_provider, + $vmid, + $storage_cfg, + $archive, + $rootdir, + $conf, + $restricted, + $unique, + $skip_fw, + ); + } } restore_configuration_from_etc_vzdump($vmid, $rootdir, $conf, $restricted, $unique, $skip_fw); } @@ -249,6 +358,38 @@ sub restore_configuration_from_proxmox_backup { } } +sub restore_configuration_from_external_backup { + my ($backup_provider, $vmid, $storage_cfg, $archive, $rootdir, $conf, $restricted, $unique, $skip_fw) = @_; + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive); + my $scfg = PVE::Storage::storage_config($storage_cfg, $storeid); + + my ($vtype, $name, undef, undef, undef, undef, $format) = + PVE::Storage::parse_volname($storage_cfg, $archive); + + my $oldconf = recover_config_from_external_backup($storage_cfg, $archive, $vmid); + + sanitize_and_merge_config($conf, $oldconf, $restricted, $unique); + + my $firewall_config = + $backup_provider->restore_get_firewall_config($volname, $storeid); + + if ($firewall_config) { + my $pve_firewall_dir = '/etc/pve/firewall'; + my $pct_fwcfg_target = "${pve_firewall_dir}/${vmid}.fw"; + if ($skip_fw) { + warn "ignoring firewall config from backup archive, lacking API permission to modify firewall.\n"; + warn "old firewall configuration in '$pct_fwcfg_target' left in place!\n" + if -e $pct_fwcfg_target; + } else { + mkdir $pve_firewall_dir; # make sure the directory exists + PVE::Tools::file_set_contents($pct_fwcfg_target, $firewall_config); + } + } + + return; +} + sub sanitize_and_merge_config { my ($conf, $oldconf, $restricted, $unique) = @_; -- 2.39.2 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel