From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 30DD71FF0E5 for ; Wed, 29 Jul 2026 11:28:11 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 05843213DC; Wed, 29 Jul 2026 11:28:11 +0200 (CEST) Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=UTF-8 Date: Wed, 29 Jul 2026 11:27:55 +0200 Message-Id: Subject: Re: [PATCH pve-storage v1 27/54] tree-wide: partially replace usages of regexes for 'import' vtype To: "Wolfgang Bumiller" From: "Max R. Carrara" X-Mailer: aerc 0.18.2-0-ge037c095a049 References: <20260422111322.257380-1-m.carrara@proxmox.com> <20260422111322.257380-28-m.carrara@proxmox.com> <7vfd6eqek2iblghphqvkpzgxrbhjd4h35c74zuogg2j2asqi5y@ksqa64odt5g2> In-Reply-To: <7vfd6eqek2iblghphqvkpzgxrbhjd4h35c74zuogg2j2asqi5y@ksqa64odt5g2> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1785317237433 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.027 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_LOW -0.7 Sender listed at https://www.dnswl.org/, low trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: NIDGQ2CIMAIUJ23OG4ZUAS43RRF5ZWZX X-Message-ID-Hash: NIDGQ2CIMAIUJ23OG4ZUAS43RRF5ZWZX X-MailFrom: m.carrara@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: pve-devel@lists.proxmox.com X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On Tue Jun 30, 2026 at 4:28 PM CEST, Wolfgang Bumiller wrote: > On Wed, Apr 22, 2026 at 01:12:53PM +0200, Max R. Carrara wrote: > > Add partial support for the 'import' volume type in > > `PVE::Storage::Common::Parse`. What remains unsupported right now is > > parsing volume names for that type, as those require a little more > > care. > > > > Replace most usages of the `PVE::Storage::IMPORT_EXT_RE_1` regex with > > parsing functions from `::Common::Parse`. > > > > Replace the one remaining spot where we use the > > `PVE::Storage::UPLOAD_IMPORT_EXT_RE_1` regex with a parsing function > > as well. This regex should now be phased out in a future APIVER + > > APIAGE bump, like the others in previous commits. > > > > Note that the `UPLOAD_IMPORT_EXT_RE_1` regex only exists to handle the > > special case of excluding .ovf files from the upload / download_url > > API methods. Instead of adding a separate parser (or a flag etc.) to > > handle this case, just parse the path of the up-/downloaded file > > regardless. Then, raise a parameter exception when the `ovf` file > > extension is encountered. > > > > Also add a bunch of test cases that target the `import` volume type > > for the `list_volumes()` plugin API method. For all of these tests, > > sort the expected and resulting list items by their volume ID in order > > to make the output comparison deterministic. Iterate over the declared > > content / volume types of the mocked storage config instead of using a > > fixed list of vtypes as well. > > > > Signed-off-by: Max R. Carrara > > --- > > src/PVE/API2/Storage/Status.pm | 13 ++- > > src/PVE/Storage.pm | 4 +- > > src/PVE/Storage/Common/Parse.pm | 13 +++ > > src/PVE/Storage/Plugin.pm | 9 +- > > src/test/list_volumes_test.pm | 180 +++++++++++++++++++++++++++++++- > > 5 files changed, 203 insertions(+), 16 deletions(-) > > > > diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Stat= us.pm > > index 03929d26..b578a1ed 100644 > > --- a/src/PVE/API2/Storage/Status.pm > > +++ b/src/PVE/API2/Storage/Status.pm > > @@ -96,13 +96,18 @@ my sub parse_transferred_file_path_extension : prot= otype($$) { > > } > > > > if ($vtype eq 'import') { > > - if ( > > - $path !~ m!${PVE::Storage::SAFE_CHAR_CLASS_RE}+$PVE::Stora= ge::UPLOAD_IMPORT_EXT_RE_1$! > > - ) { > > + my $parts =3D parse_path_as_volname_parts($path, $vtype); > > + > > + if (!defined($parts)) { > > raise_param_exc({ filename =3D> "invalid filename or wrong= extension" }); > > } > > > > - my $ext =3D $1; > > + my $ext =3D $parts->{ext}; > > + > > + if ($ext eq 'ovf') { > > + raise_param_exc({ filename =3D> "wrong file extension" }); > > + } > > + > > return $ext; > > } > > > > diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm > > index 20b7bafa..3a716894 100755 > > --- a/src/PVE/Storage.pm > > +++ b/src/PVE/Storage.pm > > @@ -773,9 +773,7 @@ sub path_to_volume_id { > > } > > > > if ($vtype eq 'import') { > > - return if $filename !~ m!/(${SAFE_CHAR_CLASS_RE}+${IMPORT_= EXT_RE_1})$!; > > - my $name =3D $1; > > - return "$sid:import/$name"; > > + return parse_path_as_volid($sid, $scfg, $path, $vtype); > > } > > > > return; > > diff --git a/src/PVE/Storage/Common/Parse.pm b/src/PVE/Storage/Common/P= arse.pm > > index 33220758..bd7cec13 100644 > > --- a/src/PVE/Storage/Common/Parse.pm > > +++ b/src/PVE/Storage/Common/Parse.pm > > @@ -37,6 +37,8 @@ my @VZTMPL_COMPRESSION_EXTENSIONS =3D ('gz', 'xz', 'z= st', 'bz2'); > > > > my @BACKUP_COMPRESSION_EXTENSIONS =3D ('gz', 'lzo', 'zst', 'bz2'); > > > > +my @IMPORT_EXTENSIONS =3D ('ova', 'ovf', 'qcow2', 'raw', 'vmdk'); > > + > > my sub join_to_re_alternations(@list) { > > return join('|', map { quotemeta } @list); > > } > > @@ -45,6 +47,10 @@ my $RE_VZTMPL_COMPRESSION_EXTENSIONS =3D join_to_re_= alternations(@VZTMPL_COMPRESSI > > > > my $RE_BACKUP_COMPRESSION_EXTENSIONS =3D join_to_re_alternations(@BACK= UP_COMPRESSION_EXTENSIONS); > > > > +my $RE_IMPORT_EXTENSIONS =3D join_to_re_alternations(@IMPORT_EXTENSION= S); > > + > > +my $RE_SAFE_CHAR_CLASS =3D qr/[a-zA-Z0-9\-\.\+\=3D\_]/; > > Btw. why are we copying this here, rather than using the one from > `PVE::Storage`? > Do we intend to relax this here? (Because it's IMO pretty clear that > relaxing the exported one in `PVE::Storage` is generally not an option.) My intention here was to already do some preparation to also get rid of the regex in PVE::Storage and have plugin authors instead *only* use shared parsing functions -- but on second thought, duplicating the regex here isn't really ideal at all. Relaxing it was never the intention. Anyhow, I'll rework this for v2. Thanks for spotting this! > > > + > > my $RE_PARENT_DIR =3D quotemeta('..'); > > my $RE_CONTAINS_PARENT_DIR =3D qr! > > ( ^$RE_PARENT_DIR/ ) # ../ --> Beginning of path > > @@ -112,11 +118,18 @@ my $RE_SNIPPETS_FILE_PATH =3D qr! > > ) > > !xn; > > > > +my $RE_IMPORT_FILE_PATH =3D qr! > > + (? > > + (? ($RE_SAFE_CHAR_CLASS)+ \. (? $RE_IMPORT_EXTENSIO= NS) ) > > + ) > > +!xn; > > + > > my $RE_FILE_PATH_FOR_VTYPE =3D { > > iso =3D> qr/^$RE_ISO_FILE_PATH$/, > > vztmpl =3D> qr/^$RE_VZTMPL_FILE_PATH$/, > > backup =3D> qr/^$RE_BACKUP_FILE_PATH$/, > > snippets =3D> qr/^$RE_SNIPPETS_FILE_PATH$/, > > + import =3D> qr/^$RE_IMPORT_FILE_PATH$/, > > }; > > > > my $RE_VOLNAME_FOR_VTYPE =3D { > > diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm > > index b08e038b..39e5cb0d 100644 > > --- a/src/PVE/Storage/Plugin.pm > > +++ b/src/PVE/Storage/Plugin.pm > > @@ -1768,13 +1768,12 @@ my sub get_subdir_files { > > } > > > > if ($vtype eq 'import') { > > - return > > - if $filename !~ > > - m!/(${PVE::Storage::SAFE_CHAR_CLASS_RE}+$PVE::Storage:= :IMPORT_EXT_RE_1)$!i; > > + my $parts =3D parse_path_as_volid_parts($storeid, $scfg, $= path, $vtype); > > + return if !defined($parts); > > > > return { > > - volid =3D> "$storeid:import/$1", > > - format =3D> "$2", > > + volid =3D> $parts->{volid}, > > + format =3D> $parts->{ext}, > > }; > > } > > > > diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test= .pm > > index 08769027..5cb08880 100644 > > --- a/src/test/list_volumes_test.pm > > +++ b/src/test/list_volumes_test.pm > > @@ -72,6 +72,7 @@ my $scfg =3D { > > 'images' =3D> 1, > > 'snippets' =3D> 1, > > 'backup' =3D> 1, > > + 'import' =3D> 1, > > }, > > }; > > > > @@ -462,6 +463,171 @@ my @tests =3D ( > > ], > > expected =3D> [], # returns empty list > > }, > > + { > > + description =3D> 'VMID: none, valid file names for import', > > + vmid =3D> undef, > > + files =3D> [ > > + "$storage_dir/import/import.ova", > > + "$storage_dir/import/import.ovf", > > + "$storage_dir/import/some-disk.qcow2", > > + "$storage_dir/import/some-disk.vmdk", > > + "$storage_dir/import/some-raw-disk.raw", > > + ], > > + expected =3D> [ > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/import.ova", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ovf', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/import.ovf", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'qcow2', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/some-disk.qcow2", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'vmdk', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/some-disk.vmdk", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'raw', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/some-raw-disk.raw", > > + }, > > + ], > > + }, > > + { > > + description =3D> 'VMID: none, non-matching file paths for impo= rt', > > + vmid =3D> undef, > > + files =3D> [ > > + # Malformed file names > > + "$storage_dir/import/import.ovff", > > + "$storage_dir/import/importova", > > + "$storage_dir/import/import.ov", > > + "$storage_dir/import/diskraw", > > + "$storage_dir/import/diskvmdk", > > + "$storage_dir/import/disk.invalid", > > + "$storage_dir/import/.ova", > > + "$storage_dir/import/.raw", > > + # Trailing whitespace must not be trimmed > > + "$storage_dir/import/import.ova\t", > > + "$storage_dir/import/disk.raw ", > > + # Whitespace in file name > > + "$storage_dir/import/something I want to import.ova", > > + "$storage_dir/import/ .raw", > > + "$storage_dir/import/ disk .vmdk", > > + "$storage_dir/import/disk .qcow2", > > + "$storage_dir/import/ import.ova", > > + # Unsafe characters in file name > > + "$storage_dir/import/linux=F0=9F=90=A7-vm.ova", > > + "$storage_dir/import/=F0=9F=90=AAperl-playground=F0=9F=90= =AA.ova", > > + "$storage_dir/import/fish_<><_<><_<><.ova", > > + $storage_dir . '/import/C:\\\\Windows\\Path.ova', > > + # Content inside .ova files may only be specified as part > > + # of volume names, and may never appear when looked up as > > + # a file path > > + "$storage_dir/import/import.ova/disk.qcow2", > > + "$storage_dir/import/import.ova/disk.raw", > > + "$storage_dir/import/import.ova/disk.vmdk", > > + "$storage_dir/import/import.ova/disk.invalid", > > + ], > > + expected =3D> [], # returns empty list > > + }, > > + { > > + description =3D> 'VMID: none, weird but valid file names for i= mport', > > + vmid =3D> undef, > > + files =3D> [ > > + "$storage_dir/import/import.ova.ova", > > + "$storage_dir/import/import.ova.ova.ova", > > + "$storage_dir/import/import.ova.ova.ova.ova", > > + "$storage_dir/import/ova.ova", > > + "$storage_dir/import/ova.ovf", > > + "$storage_dir/import/ova.vmdk", > > + "$storage_dir/import/raw.raw.qcow2", > > + "$storage_dir/import/raw.raw.qcow2.import.qcow2", > > + "$storage_dir/import/raw.raw.raw.your-boat.ova", > > + ], > > + expected =3D> [ > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/import.ova.ova", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/import.ova.ova.ova", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/import.ova.ova.ova.ova", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/ova.ova", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ovf', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/ova.ovf", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'vmdk', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/ova.vmdk", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'qcow2', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/raw.raw.qcow2", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'qcow2', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/raw.raw.qcow2.import.qcow2", > > + }, > > + { > > + content =3D> 'import', > > + ctime =3D> DEFAULT_CTIME, > > + format =3D> 'ova', > > + size =3D> DEFAULT_SIZE, > > + volid =3D> "local:import/raw.raw.raw.your-boat.ova", > > + }, > > + ], > > + }, > > ); > > > > # provide static vmlist for tests > > @@ -497,6 +663,10 @@ $mock_fsi->redefine( > > }, > > ); > > > > +my sub cmp_volinfo_by_volid { > > + return $a->{volid} cmp $b->{volid}; > > +} > > + > > my $plan =3D scalar @tests; > > plan tests =3D> $plan + 1; > > > > @@ -520,14 +690,14 @@ plan tests =3D> $plan + 1; > > > > { > > my $sid =3D 'local'; > > - my $types =3D ['rootdir', 'images', 'vztmpl', 'iso', 'backup', 'sn= ippets']; > > + my $types =3D [grep { $scfg->{content}->{$_} } keys $scfg->{conten= t}->%*]; > > my @suffixes =3D ('qcow2', 'raw', 'vmdk', 'vhdx'); > > > > # run through test cases > > foreach my $tt (@tests) { > > my $vmid =3D $tt->{vmid}; > > my $files =3D $tt->{files}; > > - my $expected =3D $tt->{expected}; > > + my $expected =3D [sort cmp_volinfo_by_volid $tt->{expected}->@= *]; > > my $description =3D $tt->{description}; > > my $parent =3D $tt->{parent}; > > > > @@ -550,8 +720,10 @@ plan tests =3D> $plan + 1; > > } > > } > > > > - my $got; > > - eval { $got =3D PVE::Storage::Plugin->list_volumes($sid, $scfg= , $vmid, $types) }; > > + my $got =3D eval { > > + my $volume_list =3D PVE::Storage::Plugin->list_volumes($si= d, $scfg, $vmid, $types); > > + return [sort cmp_volinfo_by_volid $volume_list->@*]; > > + }; > > $got =3D $@ if $@; > > > > is_deeply($got, $expected, $description) || diag(explain($got)= ); > > -- > > 2.47.3 > > > > > > > > > >