[PATCH qemu-server v3 4/6] partially fix #1989: disk: add qcow2 cache options

[Erik Fastermann <e.fastermann@proxmox.com>]

Add multiple options to configure the qcow2 L2/refcount cache. This
can provide significant performance gains in some cases.

For a detailed explanation of the options see the QEMU docs [0].
Additionally the cache size can be configured based on the size of the
disk image automatically.

Both blockdev and the older drive commandline are supported, which makes
the feature accessible to all QEMU machine versions supported by PVE.
Options which only apply to disk creation (cluster_size, refcount_bits)
are not considered in this patch.

[0] https://gitlab.com/qemu-project/qemu/-/blob/master/docs/qcow2-cache.txt

Signed-off-by: Erik Fastermann <e.fastermann@proxmox.com>
---
 src/PVE/QemuServer.pm          |  31 ++++++
 src/PVE/QemuServer/Blockdev.pm |   7 ++
 src/PVE/QemuServer/Drive.pm    | 195 +++++++++++++++++++++++++++++++++
 3 files changed, 233 insertions(+)

diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index d00f4e41..64639e45 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -1295,6 +1295,14 @@ sub print_drive_commandline_full {
         $opts .= ",auto-remove=on";
     }
 
+    my $qcow2_opt_prefix = $live_restore_name ? 'file.' : '';
+    foreach my $option_value (
+        PVE::QemuServer::Drive::qcow2_cache_options($storecfg, $drive, $drive_id, $format)
+    ) {
+        my ($option, $value) = @$option_value;
+        $opts .= ",${qcow2_opt_prefix}$option=$value";
+    }
+
     # my $file_param = $live_restore_name ? "file.file.filename" : "file";
     my $file_param = "file";
     if ($live_restore_name) {
@@ -5081,6 +5089,29 @@ sub vmconfig_update_disk {
                     || safe_string_ne($drive->{ssd}, $old_drive->{ssd})
                     || safe_string_ne($drive->{vendor}, $old_drive->{vendor})
                     || safe_string_ne($drive->{ro}, $old_drive->{ro})
+                    || safe_num_ne(
+                        $drive->{'qcow2-cache-size'}, $old_drive->{'qcow2-cache-size'},
+                    )
+                    || safe_boolean_ne(
+                        $drive->{'qcow2-cache-size-based-on-disk'},
+                        $old_drive->{'qcow2-cache-size-based-on-disk'},
+                    )
+                    || safe_num_ne(
+                        $drive->{'qcow2-l2-cache-size'},
+                        $old_drive->{'qcow2-l2-cache-size'},
+                    )
+                    || safe_num_ne(
+                        $drive->{'qcow2-l2-cache-entry-size'},
+                        $old_drive->{'qcow2-l2-cache-entry-size'},
+                    )
+                    || safe_num_ne(
+                        $drive->{'qcow2-refcount-cache-size'},
+                        $old_drive->{'qcow2-refcount-cache-size'},
+                    )
+                    || safe_num_ne(
+                        $drive->{'qcow2-cache-clean-interval'},
+                        $old_drive->{'qcow2-cache-clean-interval'},
+                    )
                 ) {
                     die "skip\n";
                 }
diff --git a/src/PVE/QemuServer/Blockdev.pm b/src/PVE/QemuServer/Blockdev.pm
index c2989a37..77b003d6 100644
--- a/src/PVE/QemuServer/Blockdev.pm
+++ b/src/PVE/QemuServer/Blockdev.pm
@@ -403,6 +403,13 @@ sub generate_format_blockdev {
         $blockdev->{'discard-no-unref'} = JSON::true if $format eq 'qcow2';
     }
 
+    foreach my $option_value (
+        PVE::QemuServer::Drive::qcow2_cache_options($storecfg, $drive, $drive_id, $format)
+    ) {
+        my ($option, $value) = @$option_value;
+        $blockdev->{$option} = $value;
+    }
+
     return $blockdev;
 }
 
diff --git a/src/PVE/QemuServer/Drive.pm b/src/PVE/QemuServer/Drive.pm
index 9af87ac7..210fd413 100644
--- a/src/PVE/QemuServer/Drive.pm
+++ b/src/PVE/QemuServer/Drive.pm
@@ -4,7 +4,9 @@ use strict;
 use warnings;
 
 use IO::File;
+use JSON qw(decode_json);
 use List::Util qw(first);
+use POSIX qw(ceil);
 use Storable qw(dclone);
 
 use PVE::JSONSchema qw(get_standard_option);
@@ -257,6 +259,63 @@ my %drivedesc_base = (
         optional => 1,
         default => 0,
     },
+    'qcow2-cache-size' => {
+        type => 'integer',
+        minimum => 1,
+        maximum => 10 * 1024,
+        description =>
+            'Cache size for qcow2 disks in MiB, defaulting up to 32 MiB for the L2 cache'
+            . ' and 4 clusters for the refcount cache (256 KiB with the default cluster size),'
+            . ' assuming qcow2-l2-cache-size and qcow2-refcount-cache-size are not set.'
+            . ' Only two of qcow2-cache-size, qcow2-l2-cache-size, and qcow2-refcount-cache-size'
+            . ' can be set simultaneously. This option is incompatible with'
+            . ' qcow2-cache-size-based-on-disk set to true.',
+        optional => 1,
+    },
+    'qcow2-cache-size-based-on-disk' => {
+        type => 'boolean',
+        description => 'Automatically pick a qcow2 cache size based on the disk image size.'
+            . ' With the default 64 KiB cluster size, 16 refcount bits and no extended'
+            . ' L2 entries this uses at most roughly 160 KiB of memory per GiB of disk.'
+            . ' This option is incompatible with qcow2-cache-size, qcow2-l2-cache-size,'
+            . ' or qcow2-refcount-cache-size being set.',
+        optional => 1,
+    },
+    'qcow2-l2-cache-size' => {
+        type => 'integer',
+        minimum => 1,
+        maximum => 10 * 1024,
+        description => 'L2 cache size for qcow2 disks in MiB. If qcow2-cache-size is set,'
+            . ' this value should be smaller. Only two of qcow2-cache-size, qcow2-l2-cache-size,'
+            . ' and qcow2-refcount-cache-size can be set simultaneously.',
+        optional => 1,
+    },
+    'qcow2-l2-cache-entry-size' => {
+        type => 'integer',
+        minimum => 512,
+        maximum => 2 * 1024 * 1024,
+        description => 'L2 cache entry size for qcow2 disks in bytes. Must be a power of two'
+            . ' and less than or equal to the cluster size.',
+        optional => 1,
+    },
+    'qcow2-refcount-cache-size' => {
+        type => 'integer',
+        minimum => 1,
+        maximum => 10 * 1024,
+        description =>
+            'Refcount cache size for qcow2 disks in MiB. If qcow2-cache-size is set,'
+            . ' this value should be smaller. Only two of qcow2-cache-size, qcow2-l2-cache-size,'
+            . ' and qcow2-refcount-cache-size can be set simultaneously.',
+        optional => 1,
+    },
+    'qcow2-cache-clean-interval' => {
+        type => 'integer',
+        minimum => 0,
+        maximum => 2**32 - 1,
+        description => 'Interval in seconds after which unused qcow2 cache entries are'
+            . ' freed. 0 disables cleaning. QEMU defaults to 600 seconds.',
+        optional => 1,
+    },
 );
 
 my %iothread_fmt = (
@@ -927,6 +986,48 @@ my sub parse_drive_append_errors {
             if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
     }
 
+    my $cache_option_count =
+        defined($res->{'qcow2-cache-size'}) +
+        defined($res->{'qcow2-l2-cache-size'}) +
+        defined($res->{'qcow2-refcount-cache-size'});
+
+    if ($cache_option_count > 2) {
+        push @$errors,
+            "at most two of qcow2-cache-size, qcow2-l2-cache-size, qcow2-refcount-cache-size"
+            . " can be set simultaneously";
+    }
+
+    if (
+        defined($res->{'qcow2-l2-cache-entry-size'})
+        && !is_power_of_two($res->{'qcow2-l2-cache-entry-size'})
+    ) {
+        push @$errors, "qcow2-l2-cache-entry-size must be a power of two";
+    }
+
+    if (
+        defined($res->{'qcow2-cache-size-based-on-disk'})
+        && $res->{'qcow2-cache-size-based-on-disk'}
+    ) {
+        if (
+            defined($res->{'qcow2-cache-size'})
+            || defined($res->{'qcow2-l2-cache-size'})
+            || defined($res->{'qcow2-refcount-cache-size'})
+        ) {
+            push @$errors, "qcow2-cache-size-based-on-disk cannot be combined with"
+                . " qcow2-cache-size, qcow2-l2-cache-size or qcow2-refcount-cache-size";
+        }
+    }
+
+    if (defined($res->{'qcow2-cache-size'})) {
+        if (($res->{'qcow2-l2-cache-size'} // 0) >= $res->{'qcow2-cache-size'}) {
+            push @$errors, "qcow2-l2-cache-size must be smaller than qcow2-cache-size";
+        }
+
+        if (($res->{'qcow2-refcount-cache-size'} // 0) >= $res->{'qcow2-cache-size'}) {
+            push @$errors, "qcow2-refcount-cache-size must be smaller than qcow2-cache-size";
+        }
+    }
+
     return if @$errors;
     return $res;
 }
@@ -946,6 +1047,11 @@ sub parse_drive {
     return $res;
 }
 
+sub is_power_of_two {
+    my ($n) = @_;
+    return $n > 0 && (($n & ($n - 1)) == 0);
+}
+
 sub print_drive {
     my ($drive, $with_alloc) = @_;
     my $skip = ['index', 'interface'];
@@ -1268,4 +1374,93 @@ sub drive_qmp_peer {
     return drive_uses_qsd_fuse($storecfg, $drive) ? qsd_qmp_peer($vmid) : vm_qmp_peer($vmid);
 }
 
+my sub qcow2_cache_size_by_disk_size {
+    # The calculation is adapted from here:
+    # https://gitlab.com/qemu-project/qemu/-/blob/master/docs/qcow2-cache.txt
+    # As this combines the refcount and L2 cache sizes, QEMU is free to
+    # choose a different value for each of them, which means the L2 cache
+    # is not necessarily 4 times bigger than the refcount cache with the
+    # default cluster_size, refcount_bits and no extended L2 entries.
+
+    my ($storecfg, $drive) = @_;
+
+    my $disk_path = PVE::Storage::abs_filesystem_path($storecfg, $drive->{file}, 1);
+    my $img_json = PVE::Storage::Common::qemu_img_info($disk_path, 'qcow2', 10);
+    die "failed to query file information with qemu-img\n" if !$img_json;
+    my $img_info = eval { decode_json($img_json) };
+    die "could not parse qemu-img info command output for '$disk_path' - $@\n" if $@;
+
+    my $disk_size = $img_info->{'virtual-size'}
+        or die "unable to determine virtual size of '$disk_path'\n";
+    my $cluster_size = $img_info->{'cluster-size'} // 65536;
+    my $refcount_bits = $img_info->{'format-specific'}->{data}->{'refcount-bits'} // 16;
+    my $has_extended_l2 = $img_info->{'format-specific'}->{data}->{'extended-l2'} // 0;
+
+    my $l2_multiplier = $has_extended_l2 ? 16 : 8;
+    my $l2_cache_size = ($disk_size * $l2_multiplier) / $cluster_size;
+
+    $l2_cache_size = 2 * $cluster_size if $l2_cache_size < (2 * $cluster_size);
+    $l2_cache_size = ceil($l2_cache_size / $cluster_size) * $cluster_size;
+
+    my $refcount_cache_size = ($disk_size * $refcount_bits) / (8 * $cluster_size);
+
+    $refcount_cache_size = 4 * $cluster_size if $refcount_cache_size < (4 * $cluster_size);
+    $refcount_cache_size = ceil($refcount_cache_size / $cluster_size) * $cluster_size;
+
+    return $l2_cache_size + $refcount_cache_size;
+}
+
+sub qcow2_cache_options {
+    my ($storecfg, $drive, $drive_id, $format) = @_;
+
+    if (!defined($format) || $format ne 'qcow2') {
+        for my $qcow2_cache_option (
+            'qcow2-cache-size',
+            'qcow2-cache-size-based-on-disk',
+            'qcow2-l2-cache-size',
+            'qcow2-l2-cache-entry-size',
+            'qcow2-refcount-cache-size',
+            'qcow2-cache-clean-interval',
+        ) {
+            if (defined($drive->{$qcow2_cache_option})) {
+                log_warn("$drive_id: $qcow2_cache_option requires disk format qcow2");
+            }
+        }
+
+        return ();
+    }
+
+    my $options = [];
+
+    if (
+        defined($drive->{'qcow2-cache-size-based-on-disk'})
+        && $drive->{'qcow2-cache-size-based-on-disk'}
+    ) {
+        my $cache_size = qcow2_cache_size_by_disk_size($storecfg, $drive);
+        push @$options, ['cache-size', int($cache_size)];
+    } elsif (defined($drive->{'qcow2-cache-size'})) {
+        push @$options, ['cache-size', int($drive->{'qcow2-cache-size'}) * 1024 * 1024];
+    }
+
+    if (defined($drive->{'qcow2-l2-cache-size'})) {
+        push @$options, ['l2-cache-size', int($drive->{'qcow2-l2-cache-size'}) * 1024 * 1024];
+    }
+
+    if (defined($drive->{'qcow2-l2-cache-entry-size'})) {
+        push @$options, ['l2-cache-entry-size', int($drive->{'qcow2-l2-cache-entry-size'})];
+    }
+
+    if (defined($drive->{'qcow2-refcount-cache-size'})) {
+        push @$options, [
+            'refcount-cache-size', int($drive->{'qcow2-refcount-cache-size'}) * 1024 * 1024,
+        ];
+    }
+
+    if (defined($drive->{'qcow2-cache-clean-interval'})) {
+        push @$options, ['cache-clean-interval', int($drive->{'qcow2-cache-clean-interval'})];
+    }
+
+    return @$options;
+}
+
 1;
-- 
2.47.3