all lists on lists.proxmox.com
 help / color / mirror / Atom feed
* [pve-devel] [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option
       [not found] <20250522135304.2513284-1-alexandre.derumier@groupe-cyllene.com>
@ 2025-05-22 13:53 ` Alexandre Derumier via pve-devel
  2025-05-27  8:49   ` Fiona Ebner
  2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support Alexandre Derumier via pve-devel
  1 sibling, 1 reply; 6+ messages in thread
From: Alexandre Derumier via pve-devel @ 2025-05-22 13:53 UTC (permalink / raw)
  To: pve-devel; +Cc: Alexandre Derumier

[-- Attachment #1: Type: message/rfc822, Size: 7979 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option
Date: Thu, 22 May 2025 15:53:03 +0200
Message-ID: <20250522135304.2513284-2-alexandre.derumier@groupe-cyllene.com>

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage/Common.pm          | 53 ++++++++++++++++++++++++++++++
 src/PVE/Storage/GlusterfsPlugin.pm |  2 +-
 src/PVE/Storage/Plugin.pm          | 47 +-------------------------
 3 files changed, 55 insertions(+), 47 deletions(-)

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index bd9c951..0770d70 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -5,12 +5,26 @@ use warnings;
 
 use PVE::JSONSchema;
 use PVE::Syscall;
+use PVE::Tools qw(run_command);
 
 use constant {
     FALLOC_FL_KEEP_SIZE => 0x01, # see linux/falloc.h
     FALLOC_FL_PUNCH_HOLE => 0x02, # see linux/falloc.h
 };
 
+our $QCOW2_PREALLOCATION = {
+    off => 1,
+    metadata => 1,
+    falloc => 1,
+    full => 1,
+};
+
+our $RAW_PREALLOCATION = {
+    off => 1,
+    falloc => 1,
+    full => 1,
+};
+
 =pod
 
 =head1 NAME
@@ -107,4 +121,43 @@ sub deallocate : prototype($$$) {
     }
 }
 
+
+sub preallocation_cmd_option {
+    my ($scfg, $fmt) = @_;
+
+    my $prealloc = $scfg->{preallocation};
+
+    if ($fmt eq 'qcow2') {
+        $prealloc = $prealloc // 'metadata';
+
+        die "preallocation mode '$prealloc' not supported by format '$fmt'\n"
+            if !$QCOW2_PREALLOCATION->{$prealloc};
+
+        return "preallocation=$prealloc";
+    } elsif ($fmt eq 'raw') {
+        $prealloc = $prealloc // 'off';
+        $prealloc = 'off' if $prealloc eq 'metadata';
+
+        die "preallocation mode '$prealloc' not supported by format '$fmt'\n"
+            if !$RAW_PREALLOCATION->{$prealloc};
+
+        return "preallocation=$prealloc";
+    }
+
+    return;
+}
+
+sub qemu_img_create {
+    my ($scfg, $fmt, $size, $path) = @_;
+
+    my $cmd = ['/usr/bin/qemu-img', 'create'];
+
+    my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
+    push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
+
+    push @$cmd, '-f', $fmt, $path, "${size}K";
+
+    run_command($cmd, errmsg => "unable to create image");
+}
+
 1;
diff --git a/src/PVE/Storage/GlusterfsPlugin.pm b/src/PVE/Storage/GlusterfsPlugin.pm
index 18493cb..b1a541d 100644
--- a/src/PVE/Storage/GlusterfsPlugin.pm
+++ b/src/PVE/Storage/GlusterfsPlugin.pm
@@ -265,7 +265,7 @@ sub alloc_image {
 
     my $cmd = ['/usr/bin/qemu-img', 'create'];
 
-    my $prealloc_opt = PVE::Storage::Plugin::preallocation_cmd_option($scfg, $fmt);
+    my $prealloc_opt = PVE::Storage::Common::preallocation_cmd_option($scfg, $fmt);
     push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
 
     push @$cmd, '-f', $fmt, $volumepath, "${size}K";
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 4e16420..7030e4e 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -47,19 +47,6 @@ our @SHARED_STORAGE = (
     'pbs',
 );
 
-our $QCOW2_PREALLOCATION = {
-    off => 1,
-    metadata => 1,
-    falloc => 1,
-    full => 1,
-};
-
-our $RAW_PREALLOCATION = {
-    off => 1,
-    falloc => 1,
-    full => 1,
-};
-
 our $MAX_VOLUMES_PER_GUEST = 1024;
 
 cfs_register_file ('storage.cfg',
@@ -577,31 +564,6 @@ sub parse_config {
     return $cfg;
 }
 
-sub preallocation_cmd_option {
-    my ($scfg, $fmt) = @_;
-
-    my $prealloc = $scfg->{preallocation};
-
-    if ($fmt eq 'qcow2') {
-	$prealloc = $prealloc // 'metadata';
-
-	die "preallocation mode '$prealloc' not supported by format '$fmt'\n"
-	    if !$QCOW2_PREALLOCATION->{$prealloc};
-
-	return "preallocation=$prealloc";
-    } elsif ($fmt eq 'raw') {
-	$prealloc = $prealloc // 'off';
-	$prealloc = 'off' if $prealloc eq 'metadata';
-
-	die "preallocation mode '$prealloc' not supported by format '$fmt'\n"
-	    if !$RAW_PREALLOCATION->{$prealloc};
-
-	return "preallocation=$prealloc";
-    }
-
-    return;
-}
-
 # Storage implementation
 
 # called during addition of storage (before the new storage config got written)
@@ -926,14 +888,7 @@ sub alloc_image {
 	umask $old_umask;
 	die $err if $err;
     } else {
-	my $cmd = ['/usr/bin/qemu-img', 'create'];
-
-	my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
-	push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
-
-	push @$cmd, '-f', $fmt, $path, "${size}K";
-
-	eval { run_command($cmd, errmsg => "unable to create image"); };
+	eval { PVE::Storage::Common::qemu_img_create($scfg, $fmt, $size, $path) };
 	if ($@) {
 	    unlink $path;
 	    rmdir $imagedir;
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
       [not found] <20250522135304.2513284-1-alexandre.derumier@groupe-cyllene.com>
  2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option Alexandre Derumier via pve-devel
@ 2025-05-22 13:53 ` Alexandre Derumier via pve-devel
  2025-05-27  8:48   ` Fiona Ebner
  1 sibling, 1 reply; 6+ messages in thread
From: Alexandre Derumier via pve-devel @ 2025-05-22 13:53 UTC (permalink / raw)
  To: pve-devel; +Cc: Alexandre Derumier

[-- Attachment #1: Type: message/rfc822, Size: 6951 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
Date: Thu, 22 May 2025 15:53:04 +0200
Message-ID: <20250522135304.2513284-3-alexandre.derumier@groupe-cyllene.com>

and use it for plugin linked clone

This also enable extended_l2=on, as it's mandatory for backing file
preallocation.

Preallocation was missing previously, so it should increase performance
for linked clone now (around x5 in randwrite 4k)

cluster_size is set to 128k, as it reduce qcow2 overhead (reduce disk,
but also memory needed to cache metadatas)

l2_extended is not enabled yet on base image, but it could help too
to reduce overhead without impacting performance

bench on 100G qcow2 file:

fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --iodepth=32 --ioengine=libaio --name=test
fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --iodepth=32 --ioengine=libaio --name=test

base image:

randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 20215
randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 22219
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20217
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 21742
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21599
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 22037

clone image with backing file:

randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 3912
randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 21476
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20563
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 22265
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 18016
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21611

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage/Common.pm | 17 +++++++++++++----
 src/PVE/Storage/Plugin.pm |  5 +----
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index 0770d70..01a995a 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -148,14 +148,23 @@ sub preallocation_cmd_option {
 }
 
 sub qemu_img_create {
-    my ($scfg, $fmt, $size, $path) = @_;
+    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
+
+    die "size can't be specified if backing file is used" if $size && $backing_path;
 
     my $cmd = ['/usr/bin/qemu-img', 'create'];
 
-    my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
-    push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
+    my $options = [];
+
+    if($backing_path) {
+	push @$cmd, '-b', $backing_path, '-F', 'qcow2';
+	push @$options, 'extended_l2=on','cluster_size=128k';
+    };
 
-    push @$cmd, '-f', $fmt, $path, "${size}K";
+    push @$options, preallocation_cmd_option($scfg, $fmt);
+    push @$cmd, '-o', join(',', @$options) if @$options > 0;
+    push @$cmd, '-f', $fmt, $path;
+    push @$cmd, "${size}K" if $size;
 
     run_command($cmd, errmsg => "unable to create image");
 }
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 7030e4e..a3856ca 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -847,10 +847,7 @@ sub clone_image {
     eval {
 	local $CWD = $imagedir;
 
-	my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename",
-		   '-F', $format, '-f', 'qcow2', $path];
-
-	run_command($cmd);
+	PVE::Storage::Common::qemu_img_create($scfg, $format, undef, $path, "../$basevmid/$basename");
     };
     my $err = $@;
 
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
  2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support Alexandre Derumier via pve-devel
@ 2025-05-27  8:48   ` Fiona Ebner
  2025-05-27  8:55     ` DERUMIER, Alexandre via pve-devel
  0 siblings, 1 reply; 6+ messages in thread
From: Fiona Ebner @ 2025-05-27  8:48 UTC (permalink / raw)
  To: Proxmox VE development discussion

Am 22.05.25 um 15:53 schrieb Alexandre Derumier via pve-devel:
> 
> and use it for plugin linked clone
> 
> This also enable extended_l2=on, as it's mandatory for backing file
> preallocation.
> 
> Preallocation was missing previously, so it should increase performance
> for linked clone now (around x5 in randwrite 4k)
> 
> cluster_size is set to 128k, as it reduce qcow2 overhead (reduce disk,
> but also memory needed to cache metadatas)
> 
> l2_extended is not enabled yet on base image, but it could help too
> to reduce overhead without impacting performance
> 
> bench on 100G qcow2 file:
> 
> fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --iodepth=32 --ioengine=libaio --name=test
> fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --iodepth=32 --ioengine=libaio --name=test
> 
> base image:
> 
> randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 20215
> randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 22219
> randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20217
> randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 21742
> randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21599
> randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 22037
> 
> clone image with backing file:
> 
> randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 3912
> randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 21476
> randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20563
> randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 22265
> randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 18016
> randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21611
> 
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>

In general, the approach is fine by me :)

> ---
>  src/PVE/Storage/Common.pm | 17 +++++++++++++----
>  src/PVE/Storage/Plugin.pm |  5 +----
>  2 files changed, 14 insertions(+), 8 deletions(-)
> 
> diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
> index 0770d70..01a995a 100644
> --- a/src/PVE/Storage/Common.pm
> +++ b/src/PVE/Storage/Common.pm
> @@ -148,14 +148,23 @@ sub preallocation_cmd_option {
>  }
>  
>  sub qemu_img_create {
> -    my ($scfg, $fmt, $size, $path) = @_;
> +    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
> +
> +    die "size can't be specified if backing file is used" if $size && $backing_path;

I'd prefer having a second dedicated function
qemu_img_create_qcow2_with_backing() rather than an interface where some
parameters need to be undef depending on how you want to use it. It also
doesn't require a format parameter, because we only allow it for qcow2.

>      my $cmd = ['/usr/bin/qemu-img', 'create'];
>  
> -    my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
> -    push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
> +    my $options = [];
> +
> +    if($backing_path) {
> +	push @$cmd, '-b', $backing_path, '-F', 'qcow2';

You're now using "-F qcow2" and...(continued below)

> +	push @$options, 'extended_l2=on','cluster_size=128k';
> +    };
>  
> -    push @$cmd, '-f', $fmt, $path, "${size}K";
> +    push @$options, preallocation_cmd_option($scfg, $fmt);
> +    push @$cmd, '-o', join(',', @$options) if @$options > 0;
> +    push @$cmd, '-f', $fmt, $path;

..."-f $fmt"...

> +    push @$cmd, "${size}K" if $size;
>  
>      run_command($cmd, errmsg => "unable to create image");
>  }
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 7030e4e..a3856ca 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -847,10 +847,7 @@ sub clone_image {
>      eval {
>  	local $CWD = $imagedir;
>  
> -	my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename",
> -		   '-F', $format, '-f', 'qcow2', $path];

...but the old command was using "-F $format -f qcow2"!

> -
> -	run_command($cmd);
> +	PVE::Storage::Common::qemu_img_create($scfg, $format, undef, $path, "../$basevmid/$basename");
>      };
>      my $err = $@;
>  
> -- 
> 2.39.5
> 
> 



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel


^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [pve-devel] [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option
  2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option Alexandre Derumier via pve-devel
@ 2025-05-27  8:49   ` Fiona Ebner
  0 siblings, 0 replies; 6+ messages in thread
From: Fiona Ebner @ 2025-05-27  8:49 UTC (permalink / raw)
  To: Proxmox VE development discussion

Yes, this is a better place for such helpers :)

Am 22.05.25 um 15:53 schrieb Alexandre Derumier via pve-devel:
> Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
> ---
>  src/PVE/Storage/Common.pm          | 53 ++++++++++++++++++++++++++++++
>  src/PVE/Storage/GlusterfsPlugin.pm |  2 +-
>  src/PVE/Storage/Plugin.pm          | 47 +-------------------------
>  3 files changed, 55 insertions(+), 47 deletions(-)
> 
> diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
> index bd9c951..0770d70 100644
> --- a/src/PVE/Storage/Common.pm
> +++ b/src/PVE/Storage/Common.pm
> @@ -5,12 +5,26 @@ use warnings;
>  
>  use PVE::JSONSchema;
>  use PVE::Syscall;
> +use PVE::Tools qw(run_command);
>  
>  use constant {
>      FALLOC_FL_KEEP_SIZE => 0x01, # see linux/falloc.h
>      FALLOC_FL_PUNCH_HOLE => 0x02, # see linux/falloc.h
>  };
>  
> +our $QCOW2_PREALLOCATION = {

Should be made private with 'my' instead of 'our', as it's not (intended
to be) used anywhere else.

> +    off => 1,
> +    metadata => 1,
> +    falloc => 1,
> +    full => 1,
> +};
> +
> +our $RAW_PREALLOCATION = {

Same.

> +    off => 1,
> +    falloc => 1,
> +    full => 1,
> +};
> +
>  =pod
>  
>  =head1 NAME
> @@ -107,4 +121,43 @@ sub deallocate : prototype($$$) {
>      }
>  }
>  
> +
> +sub preallocation_cmd_option {

All functions added to the common module should have a documentation
with POD, see existing functions in the module.


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel


^ permalink raw reply	[flat|nested] 6+ messages in thread

* Re: [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
  2025-05-27  8:48   ` Fiona Ebner
@ 2025-05-27  8:55     ` DERUMIER, Alexandre via pve-devel
  0 siblings, 0 replies; 6+ messages in thread
From: DERUMIER, Alexandre via pve-devel @ 2025-05-27  8:55 UTC (permalink / raw)
  To: pve-devel, f.ebner; +Cc: DERUMIER, Alexandre

[-- Attachment #1: Type: message/rfc822, Size: 12798 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>, "f.ebner@proxmox.com" <f.ebner@proxmox.com>
Subject: Re: [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
Date: Tue, 27 May 2025 08:55:18 +0000
Message-ID: <8f0a3adfa53e8b131db3594ee1006a84fcd8522c.camel@groupe-cyllene.com>

>>I'd prefer having a second dedicated function
>>qemu_img_create_qcow2_with_backing() rather than an interface where
>>some
>>parameters need to be undef depending on how you want to use it. It
>>also
>>doesn't require a format parameter, because we only allow it for
>>qcow2.

ok !



>  
> - my $cmd = ['/usr/bin/qemu-img', 'create', '-b',
> "../$basevmid/$basename",
> -    '-F', $format, '-f', 'qcow2', $path];

>>...but the old command was using "-F $format -f qcow2"!

ah yes, sorry, thanks !



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

^ permalink raw reply	[flat|nested] 6+ messages in thread

* [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
       [not found] <20250519102310.911326-1-alexandre.derumier@groupe-cyllene.com>
@ 2025-05-19 10:23 ` Alexandre Derumier via pve-devel
  0 siblings, 0 replies; 6+ messages in thread
From: Alexandre Derumier via pve-devel @ 2025-05-19 10:23 UTC (permalink / raw)
  To: pve-devel; +Cc: Alexandre Derumier

[-- Attachment #1: Type: message/rfc822, Size: 6947 bytes --]

From: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support
Date: Mon, 19 May 2025 12:23:10 +0200
Message-ID: <20250519102310.911326-3-alexandre.derumier@groupe-cyllene.com>

and use it for plugin linked clone

This also enable extended_l2=on, as it's mandatory for backing file
preallocation.

Preallocation was missing previously, so it should increase performance
for linked clone now (around x5 in randwrite 4k)

cluster_size is set to 128k, as it reduce qcow2 overhead (reduce disk,
but also memory needed to cache metadatas)

l2_extended is not enabled yet on base image, but it could help too
to reduce overhead without impacting performance

bench on 100G qcow2 file:

fio --filename=/dev/sdb --direct=1 --rw=randwrite --bs=4k --iodepth=32 --ioengine=libaio --name=test
fio --filename=/dev/sdb --direct=1 --rw=randread --bs=4k --iodepth=32 --ioengine=libaio --name=test

base image:

randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 20215
randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 22219
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20217
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 21742
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21599
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 22037

clone image with backing file:

randwrite 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 3912
randread 4k: prealloc=metadata, l2_extended=off, cluster_size=64k: 21476
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 20563
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=64k: 22265
randwrite 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 18016
randread 4k: prealloc=metadata, l2_extended=on, cluster_size=128k: 21611

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/Storage/Common.pm | 17 +++++++++++++----
 src/PVE/Storage/Plugin.pm |  5 +----
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/PVE/Storage/Common.pm b/src/PVE/Storage/Common.pm
index 0770d70..01a995a 100644
--- a/src/PVE/Storage/Common.pm
+++ b/src/PVE/Storage/Common.pm
@@ -148,14 +148,23 @@ sub preallocation_cmd_option {
 }
 
 sub qemu_img_create {
-    my ($scfg, $fmt, $size, $path) = @_;
+    my ($scfg, $fmt, $size, $path, $backing_path) = @_;
+
+    die "size can't be specified if backing file is used" if $size && $backing_path;
 
     my $cmd = ['/usr/bin/qemu-img', 'create'];
 
-    my $prealloc_opt = preallocation_cmd_option($scfg, $fmt);
-    push @$cmd, '-o', $prealloc_opt if defined($prealloc_opt);
+    my $options = [];
+
+    if($backing_path) {
+	push @$cmd, '-b', $backing_path, '-F', 'qcow2';
+	push @$options, 'extended_l2=on','cluster_size=128k';
+    };
 
-    push @$cmd, '-f', $fmt, $path, "${size}K";
+    push @$options, preallocation_cmd_option($scfg, $fmt);
+    push @$cmd, '-o', join(',', @$options) if @$options > 0;
+    push @$cmd, '-f', $fmt, $path;
+    push @$cmd, "${size}K" if $size;
 
     run_command($cmd, errmsg => "unable to create image");
 }
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 7030e4e..a3856ca 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -847,10 +847,7 @@ sub clone_image {
     eval {
 	local $CWD = $imagedir;
 
-	my $cmd = ['/usr/bin/qemu-img', 'create', '-b', "../$basevmid/$basename",
-		   '-F', $format, '-f', 'qcow2', $path];
-
-	run_command($cmd);
+	PVE::Storage::Common::qemu_img_create($scfg, $format, undef, $path, "../$basevmid/$basename");
     };
     my $err = $@;
 
-- 
2.39.5



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

^ permalink raw reply	[flat|nested] 6+ messages in thread

end of thread, other threads:[~2025-05-27  8:55 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
     [not found] <20250522135304.2513284-1-alexandre.derumier@groupe-cyllene.com>
2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 1/2] common: add qemu_img_create an preallocation_cmd_option Alexandre Derumier via pve-devel
2025-05-27  8:49   ` Fiona Ebner
2025-05-22 13:53 ` [pve-devel] [PATCH pve-storage 2/2] common: qemu_img_create: add backing_file support Alexandre Derumier via pve-devel
2025-05-27  8:48   ` Fiona Ebner
2025-05-27  8:55     ` DERUMIER, Alexandre via pve-devel
     [not found] <20250519102310.911326-1-alexandre.derumier@groupe-cyllene.com>
2025-05-19 10:23 ` Alexandre Derumier via pve-devel

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal