From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 26A5290E95 for ; Thu, 25 Jan 2024 15:42:31 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 09B451BC5B for ; Thu, 25 Jan 2024 15:42:00 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 25 Jan 2024 15:41:55 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A6330492A2 for ; Thu, 25 Jan 2024 15:41:55 +0100 (CET) From: Fiona Ebner To: pve-devel@lists.proxmox.com Date: Thu, 25 Jan 2024 15:41:48 +0100 Message-Id: <20240125144149.216064-13-f.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240125144149.216064-1-f.ebner@proxmox.com> References: <20240125144149.216064-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.074 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [RFC qemu-server 12/13] backup: implement fleecing option X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 25 Jan 2024 14:42:31 -0000 Management for fleecing images is implemented here. If the fleecing option is set, for each disk (except EFI disk and TPM state) a new raw fleecing image is allocated on the configured fleecing storage (same storage as original disk by default). The disk is attached to QEMU with the 'size' parameter, because the block node in QEMU has to be the exact same size and the newly allocated image might be bigger if the storage has a coarser allocation or rounded up. After backup, the disks are detached and removed from the storage. Partially inspired by the existing handling of the TPM state image during backup. Signed-off-by: Fiona Ebner --- STILL OPEN (see also the TODOs): * Naming for fleecing images and also filtering/prohibiting them with other operations. Currently using -fleecing suffix but those can conflict with user-created ones. * Should fleecing be used if the VM was started specifically for backup? In theory makes sense, because VM could be resumed during backup, but in most cases it won't. PVE/VZDump/QemuServer.pm | 140 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 138 insertions(+), 2 deletions(-) diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm index 51498dbc..d642ae46 100644 --- a/PVE/VZDump/QemuServer.pm +++ b/PVE/VZDump/QemuServer.pm @@ -26,6 +26,7 @@ use PVE::Format qw(render_duration render_bytes); use PVE::QemuConfig; use PVE::QemuServer; +use PVE::QemuServer::Helpers; use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd); @@ -525,6 +526,117 @@ sub get_and_check_pbs_encryption_config { die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!"; } +my sub cleanup_fleecing_images { + my ($self, $disks) = @_; + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; + $self->log('warn', "error removing fleecing image '$volid' - $@") if $@; + } + } +} + +my sub allocate_fleecing_images { + my ($self, $disks, $vmid, $fleecing_storeid) = @_; + + # TODO what about left-over images from previous attempts? Just + # auto-remove? While unlikely, could conflict with manually created image from user... + + eval { + for my $di ($disks->@*) { + next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it + if ($di->{type} eq 'block' || $di->{type} eq 'file') { + # TODO better named like fleecing-VMID-disk-N (needs storage plugin support...) or + # vm-VMID-fleecing-N? + # TODO filter/disallow these for qm rescan/qm set/etc. like for -state- volumes + my $storage = $fleecing_storeid || $di->{storeid}; + my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storage); + my (undef, $name) = PVE::Storage::parse_volname($self->{storecfg}, $di->{volid}); + $name =~ s/\.(.*?)$//; + # TODO checking for path is only a heuristic... + if ($scfg->{path}) { + $name .= '-fleecing.raw'; + } else { + $name .= '-fleecing'; + } + my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb'); + $di->{'fleece-volid'} = PVE::Storage::vdisk_alloc( + $self->{storecfg}, $storage, $vmid, 'raw', $name, $size); + } else { + die "implement me (type '$di->{type}')"; + } + } + }; + if (my $err = $@) { + cleanup_fleecing_images($self, $disks); + die $err; + } +} + +my sub detach_fleecing_images { + my ($disks, $vmid) = @_; + + return if !PVE::QemuServer::Helpers::vm_running_locally($vmid); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + my $devid = "$di->{qmdevice}-fleecing"; + $devid =~ s/^drive-//; # re-added by qemu_drivedel() + eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) }; + } + } +} + +my sub attach_fleecing_images { + my ($self, $disks, $vmid) = @_; + + # unconditionally try to remove potential left-overs from a previous backup + detach_fleecing_images($disks, $vmid); + + my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ]; + PVE::Storage::activate_volumes($self->{storecfg}, $vollist); + + for my $di ($disks->@*) { + if (my $volid = $di->{'fleece-volid'}) { + $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU"); + + my $path = PVE::Storage::path($self->{storecfg}, $volid); + my $devid = "$di->{qmdevice}-fleecing"; + # Specify size explicitly, to make it work if storage backend rounded up size for + # fleecing image when allocating. + my $drive = "file=$path,if=none,id=$devid,format=raw,discard=unmap,size=$di->{size}"; + $drive =~ s/\\/\\\\/g; + my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\""); + die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s; + } + } +} + +my sub check_and_prepare_fleecing { + my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_; + + # TODO what if VM was started specifically for backup? While VM could be started during + # backup, so fleecing can in theory makes sense then, in most cases it won't... + + my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template; + + if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) { + $self->log( + 'warn', + "running QEMU version does not support backup fleecing - continuing without", + ); + $use_fleecing = 0; + } + + if ($use_fleecing) { + allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}); + attach_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}); + } + + return $use_fleecing; +} + sub archive_pbs { my ($self, $task, $vmid) = @_; @@ -578,6 +690,7 @@ sub archive_pbs { # get list early so we die on unkown drive types before doing anything my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); @@ -606,6 +719,11 @@ sub archive_pbs { $attach_tpmstate_drive->($self, $task, $vmid); + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $fs_frozen = $self->qga_fs_freeze($task, $vmid); my $params = { @@ -617,6 +735,8 @@ sub archive_pbs { devlist => $devlist, 'config-file' => $conffile, }; + $params->{fleecing} = JSON::true if $use_fleecing; + if (defined(my $ns = $scfg->{namespace})) { $params->{'backup-ns'} = $ns; } @@ -633,7 +753,6 @@ sub archive_pbs { $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile); } - my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); $params->{'use-dirty-bitmap'} = JSON::true if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template; @@ -665,6 +784,11 @@ sub archive_pbs { } $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + die $err if $err; } @@ -724,8 +848,10 @@ sub archive_vma { $speed = $opts->{bwlimit}*1024; } + my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}); + my $diskcount = scalar(@{$task->{disks}}); - if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) { + if ($is_template || !$diskcount) { my @pathlist; foreach my $di (@{$task->{disks}}) { if ($di->{type} eq 'block' || $di->{type} eq 'file') { @@ -765,6 +891,7 @@ sub archive_vma { } my $devlist = _get_task_devlist($task); + my $use_fleecing; $self->enforce_vm_running_for_backup($vmid); $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid); @@ -784,6 +911,9 @@ sub archive_vma { $attach_tpmstate_drive->($self, $task, $vmid); + $use_fleecing = check_and_prepare_fleecing( + $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support); + my $outfh; if ($opts->{stdout}) { $outfh = $opts->{stdout}; @@ -812,6 +942,7 @@ sub archive_vma { devlist => $devlist }; $params->{'firewall-file'} = $firewall if -e $firewall; + $params->{fleecing} = JSON::true if $use_fleecing; add_backup_performance_options($params, $opts->{performance}, $qemu_support); $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params); @@ -853,6 +984,11 @@ sub archive_vma { $self->restore_vm_power_state($vmid); + if ($use_fleecing) { + detach_fleecing_images($task->{disks}, $vmid); + cleanup_fleecing_images($self, $task->{disks}); + } + if ($err) { if ($cpid) { kill(9, $cpid); -- 2.39.2