From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 844AB94BDA for ; Thu, 11 Apr 2024 11:33:22 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1FA3B1E0AC for ; Thu, 11 Apr 2024 11:33:22 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 11 Apr 2024 11:33:19 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 2B68944A27 for ; Thu, 11 Apr 2024 11:33:17 +0200 (CEST) From: Fiona Ebner To: pve-devel@lists.proxmox.com Date: Thu, 11 Apr 2024 11:29:37 +0200 Message-Id: <20240411092943.57377-17-f.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240411092943.57377-1-f.ebner@proxmox.com> References: <20240411092943.57377-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.072 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH qemu-server v3 16/22] backup: implement fleecing option X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 11 Apr 2024 09:33:22 -0000 Management for fleecing images is implemented here. If the fleecing option is set, for each disk (except EFI disk and TPM state) a new fleecing image is allocated on the configured fleecing storage (same storage as original disk by default). The disk is attached to QEMU with the 'size' parameter, because the block node in QEMU has to be the exact same size and the newly allocated image might be bigger if the storage has a coarser allocation or rounded up. After backup, the disks are detached and removed from the storage. If the storage supports qcow2, use that as the fleecing image format. This allows saving some space even on storages that do not properly support discard, like, for example, older versions of NFS. Since there can be multiple volumes with the same volume name on different storages, the fleecing image's name cannot be just based on the original volume's name. The schema vm-ID-fleece-N(.FORMAT) with N incrementing for each disk is used. Partially inspired by the existing handling of the TPM state image during backup. Signed-off-by: Fiona Ebner --- PVE/VZDump/QemuServer.pm | 144 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 2 deletions(-) diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm index 51498dbc..8c97ee62 100644 --- a/PVE/VZDump/QemuServer.pm +++ b/PVE/VZDump/QemuServer.pm @@ -26,6 +26,7 @@ use PVE::Format qw(render_duration render_bytes); use PVE::QemuConfig; use PVE::QemuServer; +use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd); @@ -525,6 +526,121 @@ sub get_and_check_pbs_encryption_config { die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!"; } +my sub cleanup_fleecing_images { + my ($self, $disks) = @_; + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; + $self->log('warn', "error removing fleecing image '$volid' - $@") if $@; + } + } +} + +my sub allocate_fleecing_images { + my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_; + + die "internal error - no fleecing storage specified\n" if !$fleecing_storeid; + + # TODO what about potential left-over images from a failed attempt? Just + # auto-remove? While unlikely, could conflict with manually created image from user... + + eval { + my $n = 0; # counter for fleecing image names + + for my $di ($disks->@*) { + next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it + if ($di->{type} eq 'block' || $di->{type} eq 'file') { + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid); + my $name = "vm-$vmid-fleece-$n"; + $name .= ".$format" if $scfg->{path}; + + my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb'); + + $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc( + $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size); + + $n++; + } else { + die "implement me (type '$di->{type}')"; + } + } + }; + if (my $err = $@) { + cleanup_fleecing_images($self, $disks); + die $err; + } +} + +my sub detach_fleecing_images { + my ($disks, $vmid) = @_; + + return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + my $devid = "$di->{qmdevice}-fleecing"; + $devid =~ s/^drive-//; # re-added by qemu_drivedel() + eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) }; + } + } +} + +my sub attach_fleecing_images { + my ($self, $disks, $vmid, $format) = @_; + + # unconditionally try to remove potential left-overs from a previous backup + detach_fleecing_images($disks, $vmid); + + my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ]; + PVE::Storage::activate_volumes($self->{storecfg}, $vollist); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU"); + + my $path = PVE::Storage::path($self->{storecfg}, $volid); + my $devid = "$di->{qmdevice}-fleecing"; + my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap"; + # Specify size explicitly, to make it work if storage backend rounded up size for + # fleecing image when allocating. + $drive .= ",size=$di->{size}" if $format eq 'raw'; + $drive =~ s/\\/\\\\/g; + my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); + die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s; + } + } +} + +my sub check_and_prepare_fleecing { + my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_; + + # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and + # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty, + # so there is no big cost. + + my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template; + + if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) { + $self->log( + 'warn', + "running QEMU version does not support backup fleecing - continuing without", + ); + $use_fleecing = 0; + } + + if ($use_fleecing) { + my ($default_format, $valid_formats) = PVE::Storage::storage_default_format( + $self->{storecfg}, $fleecing_opts->{storage}); + my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw'; + + allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format); + attach_fleecing_images($self, $disks, $vmid, $format); + } + + return $use_fleecing; +} + sub archive_pbs { my ($self, $task, $vmid) = @_; @@ -578,6 +694,7 @@ sub archive_pbs { # get list early so we die on unkown drive types before doing anything my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); @@ -606,6 +723,11 @@ sub archive_pbs { $attach_tpmstate_drive->($self, $task, $vmid); + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $fs_frozen = $self->qga_fs_freeze($task, $vmid); my $params = { @@ -617,6 +739,8 @@ sub archive_pbs { devlist => $devlist, 'config-file' => $conffile, }; + $params->{fleecing} = JSON::true if $use_fleecing; + if (defined(my $ns = $scfg->{namespace})) { $params->{'backup-ns'} = $ns; } @@ -633,7 +757,6 @@ sub archive_pbs { $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile); } - my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); $params->{'use-dirty-bitmap'} = JSON::true if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; @@ -665,6 +788,11 @@ sub archive_pbs { } $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + die $err if $err; } @@ -724,8 +852,10 @@ sub archive_vma { $speed = $opts->{bwlimit}*1024; } + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + my $diskcount = scalar(@{$task->{disks}}); - if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) { + if ($is_template || !$diskcount) { my @pathlist; foreach my $di (@{$task->{disks}}) { if ($di->{type} eq 'block' || $di->{type} eq 'file') { @@ -765,6 +895,7 @@ sub archive_vma { } my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); @@ -784,6 +915,9 @@ sub archive_vma { $attach_tpmstate_drive->($self, $task, $vmid); + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $outfh; if ($opts->{stdout}) { $outfh = $opts->{stdout}; @@ -812,6 +946,7 @@ sub archive_vma { devlist => $devlist }; $params->{'firewall-file'} = $firewall if -e $firewall; + $params->{fleecing} = JSON::true if $use_fleecing; add_backup_performance_options($params, $opts->{performance}, $qemu_support); $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params); @@ -853,6 +988,11 @@ sub archive_vma { $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + if ($err) { if ($cpid) { kill(9, $cpid); -- 2.39.2