From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 88E9B1FF0AA for ; Tue, 17 Nov 2026 21:43:49 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 818B621680; Tue, 17 Nov 2026 21:41:07 +0100 (CET) From: "Max R. Carrara" To: pve-devel@lists.proxmox.com Subject: [PATCH pve-storage v3 23/51] tree-wide: replace usages of VZTMPL_EXT_RE_1 with parsing functions Date: Tue, 17 Nov 2026 21:38:55 +0100 Message-ID: <20261117203953.913035-24-m.carrara@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261117203953.913035-1-m.carrara@proxmox.com> References: <20261117203953.913035-1-m.carrara@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794948041754 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.257 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: SX3EYL36GYI7ORXH3XFSNCQ73D3KUWZ4 X-Message-ID-Hash: SX3EYL36GYI7ORXH3XFSNCQ73D3KUWZ4 X-MailFrom: m.carrara@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Wolfgang Bumiller X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add support for the 'vztmpl' volume type in `PVE::Storage::Common::Parse`. Also add corresponding test cases. Use the module's parsing functions to replace usages of the `PVE::Storage::VZTMPL_EXT_RE_1` regex across the repository. As done with the `ISO_EXT_RE_0` regex, add a FIXME comment to the `VZTMPL_EXT_RE_1` regex to mark it for removal on the next APIAGE reset. Its removal will be noted in the next APIVER + APIAGE bump. Signed-off-by: Max R. Carrara --- src/PVE/API2/Storage/Status.pm | 6 +- src/PVE/Storage.pm | 5 +- src/PVE/Storage/Common/Parse.pm | 54 +++++++++- src/PVE/Storage/Common/test/parser_tests.pl | 111 +++++++++++++++++++- src/PVE/Storage/Plugin.pm | 16 +-- 5 files changed, 178 insertions(+), 14 deletions(-) diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm index 66e8e141..80f9f3c4 100644 --- a/src/PVE/API2/Storage/Status.pm +++ b/src/PVE/API2/Storage/Status.pm @@ -83,15 +83,17 @@ my sub parse_transferred_file_path_extension : prototype($$) { my $ext = $parts->{ext}; return $ext; } if ($vtype eq 'vztmpl') { - if ($path !~ m![^/]+$PVE::Storage::VZTMPL_EXT_RE_1$!) { + my $parts = parse_rel_path_as_volname_parts($path, $vtype); + + if (!defined($parts)) { raise_param_exc({ filename => "wrong file extension" }); } - my $ext = $1; + my $ext = $parts->{ext}; return $ext; } if ($vtype eq 'import') { if ( diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index 474f1ee5..57355ffd 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -121,10 +121,11 @@ PVE::Storage::Plugin->init(); # CAUTION don't forget to update the digits accordingly after messing with the capture groups # FIXME: remove this regex on the next APIAGE reset. our $ISO_EXT_RE_0 = qr/\.(?:iso|img)/i; +# FIXME: remove this regex on the next APIAGE reset. our $VZTMPL_EXT_RE_1 = qr/\.(?|(tar)(?!\.)|tar\.(gz|xz|zst|bz2))/i; our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/; our $IMPORT_EXT_RE_1 = qr/\.(ova|ovf|qcow2|raw|vmdk)/; @@ -812,13 +813,11 @@ sub path_to_volume_id { if ($vtype eq 'iso') { return ($vtype, parse_abs_path_as_volid($sid, $scfg, $path, $vtype)); } if ($vtype eq 'vztmpl') { - return if $filename !~ m!^/([^/]+$VZTMPL_EXT_RE_1)$!; - my $name = $1; - return ($vtype, "$sid:vztmpl/$name"); + return ($vtype, parse_abs_path_as_volid($sid, $scfg, $path, $vtype)); } if ($vtype eq 'backup') { return if $filename !~ m!^/([^/]+$BACKUP_EXT_RE_2)$!; my $name = $1; diff --git a/src/PVE/Storage/Common/Parse.pm b/src/PVE/Storage/Common/Parse.pm index ab949d99..94dd75f1 100644 --- a/src/PVE/Storage/Common/Parse.pm +++ b/src/PVE/Storage/Common/Parse.pm @@ -31,10 +31,18 @@ Parsing functions are categorized by their main purpose / area of application and may be further subdivided depending on what kind of data type they are primarily related to. =cut +my @VZTMPL_COMPRESSION_EXTENSIONS = ('gz', 'xz', 'zst', 'bz2'); + +my sub join_to_re_alternations(@list) { + return join('|', map { quotemeta } @list); +} + +my $RE_VZTMPL_COMPRESSION_EXTENSIONS = join_to_re_alternations(@VZTMPL_COMPRESSION_EXTENSIONS); + my $RE_PARENT_DIR = quotemeta('..'); my $RE_CONTAINS_PARENT_DIR = qr! ( ^$RE_PARENT_DIR$ ) # .. --> Entire path | ( ^$RE_PARENT_DIR/ ) # ../ --> Beginning of path @@ -48,16 +56,31 @@ my $RE_ISO_FILE_PATH = qr! (? (? [^/]+ \. (? (?i: iso|img) ) ) ) !xn; +my $RE_VZTMPL_FILE_PATH = qr! + (? + (? + [^/]+ + \. + (? + (? (?i: tar) ) + ( \. (? (?i: $RE_VZTMPL_COMPRESSION_EXTENSIONS)) )? + ) + ) + ) +!xn; + my $RE_FILE_PATH_FOR_VTYPE = { iso => qr/^$RE_ISO_FILE_PATH$/, + vztmpl => qr/^$RE_VZTMPL_FILE_PATH$/, }; my $RE_VOLNAME_FOR_VTYPE = { iso => qr/^$RE_ISO_FILE_PATH$/, + vztmpl => qr/^$RE_VZTMPL_FILE_PATH$/, }; my sub contains_parent_dir($path) { return $path =~ $RE_CONTAINS_PARENT_DIR; } @@ -138,10 +161,35 @@ These are: # The volname, which is "${vtype}/${path}" volname => 'iso/custom-debian.iso', } +The following volume types have more specific parts in addition to the common +ones: + +=over + +=item C + +C, C (optional): + + { + file => 'archlinux-base_20190924-1_amd64.tar.gz', + ext => 'tar.gz', + + # Used to further differentiate 'ext' + 'ext-archive' => 'tar', + 'ext-compression' => 'gz', + + 'disk-path' => 'archlinux-base_20190924-1_amd64.tar.gz', + path => 'archlinux-base_20190924-1_amd64.tar.gz', + vtype => 'vztmpl', + volname => 'vztmpl/archlinux-base_20190924-1_amd64.tar.gz', + } + +=back + =cut =head3 parse_rel_path_as_volname_parts Parses the given relative file path C<$path> according to the given C<$vtype>, @@ -157,11 +205,11 @@ into: path => 'custom-debian.iso', vtype => 'iso', volname => 'iso/custom-debian.iso', } -See L for more details. +See L for more details and special cases. On failure or when the provided C<$vtype> is not supported or does not exist, returns C in scalar context, and an empty list in list context. B This function assumes that C<$path> is already relative to the @@ -287,10 +335,14 @@ These are: # Exclusive to volid parsing volid => 'local:iso/custom-debian.iso', } +The special cases described in L also apply for the parser +functions in this section. In fact, the volid parsing functions here build on +the volname ones. + =cut my $RE_VOLID = qr! ^ (? diff --git a/src/PVE/Storage/Common/test/parser_tests.pl b/src/PVE/Storage/Common/test/parser_tests.pl index ea770bb4..687ad6ae 100755 --- a/src/PVE/Storage/Common/test/parser_tests.pl +++ b/src/PVE/Storage/Common/test/parser_tests.pl @@ -84,10 +84,117 @@ my $volname_cases_iso_invalid = [ }, expected => undef, }, ]; +my $volname_cases_vztmpl_valid = [ + # plain paths + { + path => 'archlinux-base_20190924-1_amd64.tar', + expected => { + file => 'archlinux-base_20190924-1_amd64.tar', + ext => 'tar', + 'ext-archive' => 'tar', + 'disk-path' => 'archlinux-base_20190924-1_amd64.tar', + path => 'archlinux-base_20190924-1_amd64.tar', + vtype => 'vztmpl', + volname => 'vztmpl/archlinux-base_20190924-1_amd64.tar', + }, + }, + { + path => 'archlinux-base_20190924-1_amd64.tar.gz', + expected => { + file => 'archlinux-base_20190924-1_amd64.tar.gz', + ext => 'tar.gz', + 'ext-archive' => 'tar', + 'ext-compression' => 'gz', + 'disk-path' => 'archlinux-base_20190924-1_amd64.tar.gz', + path => 'archlinux-base_20190924-1_amd64.tar.gz', + vtype => 'vztmpl', + volname => 'vztmpl/archlinux-base_20190924-1_amd64.tar.gz', + }, + }, + { + path => 'alpine-3.10-default_20190626_amd64.tar.xz', + expected => { + file => 'alpine-3.10-default_20190626_amd64.tar.xz', + ext => 'tar.xz', + 'ext-archive' => 'tar', + 'ext-compression' => 'xz', + 'disk-path' => 'alpine-3.10-default_20190626_amd64.tar.xz', + path => 'alpine-3.10-default_20190626_amd64.tar.xz', + vtype => 'vztmpl', + volname => 'vztmpl/alpine-3.10-default_20190626_amd64.tar.xz', + }, + }, + { + path => 'debian-10.0-standard_10.0-1_amd64.tar.zst', + expected => { + file => 'debian-10.0-standard_10.0-1_amd64.tar.zst', + ext => 'tar.zst', + 'ext-archive' => 'tar', + 'ext-compression' => 'zst', + 'disk-path' => 'debian-10.0-standard_10.0-1_amd64.tar.zst', + path => 'debian-10.0-standard_10.0-1_amd64.tar.zst', + vtype => 'vztmpl', + volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.zst', + }, + }, + { + path => 'debian-11.0-standard_11.0-1_amd64.tar.bz2', + expected => { + file => 'debian-11.0-standard_11.0-1_amd64.tar.bz2', + ext => 'tar.bz2', + 'ext-archive' => 'tar', + 'ext-compression' => 'bz2', + 'disk-path' => 'debian-11.0-standard_11.0-1_amd64.tar.bz2', + path => 'debian-11.0-standard_11.0-1_amd64.tar.bz2', + vtype => 'vztmpl', + volname => 'vztmpl/debian-11.0-standard_11.0-1_amd64.tar.bz2', + }, + }, + + # case-insensitive file extensions + { + path => 'ARCHLINUX-BASE 20190924-1 AMD64.TAR.GZ', + expected => { + file => 'ARCHLINUX-BASE 20190924-1 AMD64.TAR.GZ', + ext => 'TAR.GZ', + 'ext-archive' => 'TAR', + 'ext-compression' => 'GZ', + 'disk-path' => 'ARCHLINUX-BASE 20190924-1 AMD64.TAR.GZ', + path => 'ARCHLINUX-BASE 20190924-1 AMD64.TAR.GZ', + vtype => 'vztmpl', + volname => 'vztmpl/ARCHLINUX-BASE 20190924-1 AMD64.TAR.GZ', + }, + }, + { + path => 'Alpine 3.10 default_20190626_amd64.Tar.xZ', + expected => { + file => 'Alpine 3.10 default_20190626_amd64.Tar.xZ', + ext => 'Tar.xZ', + 'ext-archive' => 'Tar', + 'ext-compression' => 'xZ', + 'disk-path' => 'Alpine 3.10 default_20190626_amd64.Tar.xZ', + path => 'Alpine 3.10 default_20190626_amd64.Tar.xZ', + vtype => 'vztmpl', + volname => 'vztmpl/Alpine 3.10 default_20190626_amd64.Tar.xZ', + }, + }, +]; + +my $volname_cases_vztmpl_invalid = [ + { + description => "Invalid file extension (vztmpl)", + args => { + path => 'archlinux-base_20190924-1_amd64.zip', + vtype => 'vztmpl', + }, + expected => undef, + }, +]; + my $volname_cases_newline_invalid = [ { description => "Trailing newline (iso)", args => { path => "debian.iso\n", @@ -112,15 +219,15 @@ my $volname_cases_newline_invalid = [ expected => undef, }, ]; my $cases_valid_all = [ - $volname_cases_iso_valid, + $volname_cases_iso_valid, $volname_cases_vztmpl_valid, ]; my $cases_invalid_all = [ - $volname_cases_iso_invalid, $volname_cases_newline_invalid, + $volname_cases_iso_invalid, $volname_cases_vztmpl_invalid, $volname_cases_newline_invalid, ]; { for my $cases_list ($cases_valid_all->@*) { for my $case ($cases_list->@*) { diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm index c5aed4ef..3786b6f5 100644 --- a/src/PVE/Storage/Plugin.pm +++ b/src/PVE/Storage/Plugin.pm @@ -903,14 +903,14 @@ sub parse_volname { my ($vtype, $volume_path) = $parts->@{qw(vtype path)}; if ($vtype eq 'iso') { return ($vtype, $volume_path, undef, undef, undef, undef, 'raw'); } - } - if ($volname =~ m!^vztmpl/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) { - return ('vztmpl', $1, undef, undef, undef, undef, 'raw'); + if ($vtype eq 'vztmpl') { + return ($vtype, $volume_path, undef, undef, undef, undef, 'raw'); + } } if ($volname =~ m!^backup/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!) { my $fn = $1; if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) { @@ -1906,15 +1906,19 @@ my sub get_subdir_files { format => 'iso', # always 'iso' even if we have a file ending in .img }; } if ($vtype eq 'vztmpl') { - return if $filename !~ m!/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!; + my $parts = parse_abs_path_as_volid_parts($storeid, $scfg, $path, $vtype); + return if !defined($parts); + + my ($ext_archive, $ext_compression) = $parts->@{qw(ext-archive ext-compression)}; + my $format = defined($ext_compression) ? "t$ext_compression" : $ext_archive; return { - volid => "$storeid:vztmpl/$1", - format => $2 eq 'tar' ? $2 : "t$2", + volid => $parts->{volid}, + format => $format, }; } if ($vtype eq 'backup') { return if $filename !~ m!/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!; -- 2.47.3