[pve-devel] [PATCH qemu-server v3 16/22] backup: implement fleecing option

[Fiona Ebner <f.ebner@proxmox.com>]

Management for fleecing images is implemented here. If the fleecing
option is set, for each disk (except EFI disk and TPM state) a new
fleecing image is allocated on the configured fleecing storage (same
storage as original disk by default). The disk is attached to QEMU
with the 'size' parameter, because the block node in QEMU has to be
the exact same size and the newly allocated image might be bigger if
the storage has a coarser allocation or rounded up. After backup, the
disks are detached and removed from the storage.

If the storage supports qcow2, use that as the fleecing image format.
This allows saving some space even on storages that do not properly
support discard, like, for example, older versions of NFS.

Since there can be multiple volumes with the same volume name on
different storages, the fleecing image's name cannot be just based on
the original volume's name. The schema vm-ID-fleece-N(.FORMAT) with N
incrementing for each disk is used.

Partially inspired by the existing handling of the TPM state image
during backup.

Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
---
 PVE/VZDump/QemuServer.pm | 144 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 142 insertions(+), 2 deletions(-)

diff --git a/PVE/VZDump/QemuServer.pm b/PVE/VZDump/QemuServer.pm
index 51498dbc..8c97ee62 100644
--- a/PVE/VZDump/QemuServer.pm
+++ b/PVE/VZDump/QemuServer.pm
@@ -26,6 +26,7 @@ use PVE::Format qw(render_duration render_bytes);
 
 use PVE::QemuConfig;
 use PVE::QemuServer;
+use PVE::QemuServer::Helpers;
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Monitor qw(mon_cmd);
 
@@ -525,6 +526,121 @@ sub get_and_check_pbs_encryption_config {
     die "internal error - unhandled case for getting & checking PBS encryption ($keyfile, $master_keyfile)!";
 }
 
+my sub cleanup_fleecing_images {
+    my ($self, $disks) = @_;
+
+    for my $di ($disks->@*) {
+	if (my $volid = $di->{'fleece-volid'}) {
+	    eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
+	    $self->log('warn', "error removing fleecing image '$volid' - $@") if $@;
+	}
+    }
+}
+
+my sub allocate_fleecing_images {
+    my ($self, $disks, $vmid, $fleecing_storeid, $format) = @_;
+
+    die "internal error - no fleecing storage specified\n" if !$fleecing_storeid;
+
+    # TODO what about potential left-over images from a failed attempt? Just
+    # auto-remove? While unlikely, could conflict with manually created image from user...
+
+    eval {
+	my $n = 0; # counter for fleecing image names
+
+	for my $di ($disks->@*) {
+	    next if $di->{virtdev} =~ m/^(?:tpmstate|efidisk)\d$/; # too small to be worth it
+	    if ($di->{type} eq 'block' || $di->{type} eq 'file') {
+		my $scfg = PVE::Storage::storage_config($self->{storecfg}, $fleecing_storeid);
+		my $name = "vm-$vmid-fleece-$n";
+		$name .= ".$format" if $scfg->{path};
+
+		my $size = PVE::Tools::convert_size($di->{size}, 'b' => 'kb');
+
+		$di->{'fleece-volid'} = PVE::Storage::vdisk_alloc(
+		    $self->{storecfg}, $fleecing_storeid, $vmid, $format, $name, $size);
+
+		$n++;
+	    } else {
+		die "implement me (type '$di->{type}')";
+	    }
+	}
+    };
+    if (my $err = $@) {
+	cleanup_fleecing_images($self, $disks);
+	die $err;
+    }
+}
+
+my sub detach_fleecing_images {
+    my ($disks, $vmid) = @_;
+
+    return if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
+
+    for my $di ($disks->@*) {
+	if (my $volid = $di->{'fleece-volid'}) {
+	    my $devid = "$di->{qmdevice}-fleecing";
+	    $devid =~ s/^drive-//; # re-added by qemu_drivedel()
+	    eval { PVE::QemuServer::qemu_drivedel($vmid, $devid) };
+	}
+    }
+}
+
+my sub attach_fleecing_images {
+    my ($self, $disks, $vmid, $format) = @_;
+
+    # unconditionally try to remove potential left-overs from a previous backup
+    detach_fleecing_images($disks, $vmid);
+
+    my $vollist = [ map { $_->{'fleece-volid'} } grep { $_->{'fleece-volid'} } $disks->@* ];
+    PVE::Storage::activate_volumes($self->{storecfg}, $vollist);
+
+    for my $di ($disks->@*) {
+	if (my $volid = $di->{'fleece-volid'}) {
+	    $self->loginfo("$di->{qmdevice}: attaching fleecing image $volid to QEMU");
+
+	    my $path = PVE::Storage::path($self->{storecfg}, $volid);
+	    my $devid = "$di->{qmdevice}-fleecing";
+	    my $drive = "file=$path,if=none,id=$devid,format=$format,discard=unmap";
+	    # Specify size explicitly, to make it work if storage backend rounded up size for
+	    # fleecing image when allocating.
+	    $drive .= ",size=$di->{size}" if $format eq 'raw';
+	    $drive =~ s/\\/\\\\/g;
+	    my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
+	    die "attaching fleecing image $volid failed - $ret\n" if $ret !~ m/OK/s;
+	}
+    }
+}
+
+my sub check_and_prepare_fleecing {
+    my ($self, $vmid, $fleecing_opts, $disks, $is_template, $qemu_support) = @_;
+
+    # Even if the VM was started specifically for fleecing, it's possible that the VM is resumed and
+    # then starts doing IO. For VMs that are not resumed the fleecing images will just stay empty,
+    # so there is no big cost.
+
+    my $use_fleecing = $fleecing_opts && $fleecing_opts->{enabled} && !$is_template;
+
+    if ($use_fleecing && !defined($qemu_support->{'backup-fleecing'})) {
+	$self->log(
+	    'warn',
+	    "running QEMU version does not support backup fleecing - continuing without",
+	);
+	$use_fleecing = 0;
+    }
+
+    if ($use_fleecing) {
+	my ($default_format, $valid_formats) = PVE::Storage::storage_default_format(
+	    $self->{storecfg}, $fleecing_opts->{storage});
+	my $format = scalar(grep { $_ eq 'qcow2' } $valid_formats->@*) ? 'qcow2' : 'raw';
+
+	allocate_fleecing_images($self, $disks, $vmid, $fleecing_opts->{storage}, $format);
+	attach_fleecing_images($self, $disks, $vmid, $format);
+    }
+
+    return $use_fleecing;
+}
+
 sub archive_pbs {
     my ($self, $task, $vmid) = @_;
 
@@ -578,6 +694,7 @@ sub archive_pbs {
 
     # get list early so we die on unkown drive types before doing anything
     my $devlist = _get_task_devlist($task);
+    my $use_fleecing;
 
     $self->enforce_vm_running_for_backup($vmid);
     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
@@ -606,6 +723,11 @@ sub archive_pbs {
 
 	$attach_tpmstate_drive->($self, $task, $vmid);
 
+	my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
+
+	$use_fleecing = check_and_prepare_fleecing(
+	    $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
+
 	my $fs_frozen = $self->qga_fs_freeze($task, $vmid);
 
 	my $params = {
@@ -617,6 +739,8 @@ sub archive_pbs {
 	    devlist => $devlist,
 	    'config-file' => $conffile,
 	};
+	$params->{fleecing} = JSON::true if $use_fleecing;
+
 	if (defined(my $ns = $scfg->{namespace})) {
 	    $params->{'backup-ns'} = $ns;
 	}
@@ -633,7 +757,6 @@ sub archive_pbs {
 	    $params->{"master-keyfile"} = $master_keyfile if defined($master_keyfile);
 	}
 
-	my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
 	$params->{'use-dirty-bitmap'} = JSON::true
 	    if $qemu_support->{'pbs-dirty-bitmap'} && !$is_template;
 
@@ -665,6 +788,11 @@ sub archive_pbs {
     }
     $self->restore_vm_power_state($vmid);
 
+    if ($use_fleecing) {
+	detach_fleecing_images($task->{disks}, $vmid);
+	cleanup_fleecing_images($self, $task->{disks});
+    }
+
     die $err if $err;
 }
 
@@ -724,8 +852,10 @@ sub archive_vma {
 	$speed = $opts->{bwlimit}*1024;
     }
 
+    my $is_template = PVE::QemuConfig->is_template($self->{vmlist}->{$vmid});
+
     my $diskcount = scalar(@{$task->{disks}});
-    if (PVE::QemuConfig->is_template($self->{vmlist}->{$vmid}) || !$diskcount) {
+    if ($is_template || !$diskcount) {
 	my @pathlist;
 	foreach my $di (@{$task->{disks}}) {
 	    if ($di->{type} eq 'block' || $di->{type} eq 'file') {
@@ -765,6 +895,7 @@ sub archive_vma {
     }
 
     my $devlist = _get_task_devlist($task);
+    my $use_fleecing;
 
     $self->enforce_vm_running_for_backup($vmid);
     $self->{qmeventd_fh} = PVE::QemuServer::register_qmeventd_handle($vmid);
@@ -784,6 +915,9 @@ sub archive_vma {
 
 	$attach_tpmstate_drive->($self, $task, $vmid);
 
+	$use_fleecing = check_and_prepare_fleecing(
+	    $self, $vmid, $opts->{fleecing}, $task->{disks}, $is_template, $qemu_support);
+
 	my $outfh;
 	if ($opts->{stdout}) {
 	    $outfh = $opts->{stdout};
@@ -812,6 +946,7 @@ sub archive_vma {
 		devlist => $devlist
 	    };
 	    $params->{'firewall-file'} = $firewall if -e $firewall;
+	    $params->{fleecing} = JSON::true if $use_fleecing;
 	    add_backup_performance_options($params, $opts->{performance}, $qemu_support);
 
 	    $qmpclient->queue_cmd($vmid, $backup_cb, 'backup', %$params);
@@ -853,6 +988,11 @@ sub archive_vma {
 
     $self->restore_vm_power_state($vmid);
 
+    if ($use_fleecing) {
+	detach_fleecing_images($task->{disks}, $vmid);
+	cleanup_fleecing_images($self, $task->{disks});
+    }
+
     if ($err) {
 	if ($cpid) {
 	    kill(9, $cpid);
-- 
2.39.2