From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 460F21FF0B2 for ; Sun, 25 Oct 2026 15:27:48 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 7A77E20B89; Sun, 25 Oct 2026 15:27:47 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=romanondracek.cz; s=dkim2019; t=1792938455; bh=jDnfYC2Djw2PDa8jcOXRJD8JynD/bOdLwmbyYt49KYg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=HW+2yv3y9rKgyqCmwuyCf5v2N8T0b4nnqfUQdDJiJvxO0jK9yhKaI9Yo2QXUVGKa6 jCxQRxZYIGfW9D86zc3dxH32zjUtf4FMjEEqRYHaF78yPqBu+zcZPN99CxqxZ13YB1 VaM/pa5y70aYpjweQ2f00ra2OgYYS2okudmRjlrFvbXKyx1tyZtnDqCuzCW1rNIV3M jVKn0GCsbVKhGoRp2EF4gcMgHygqUL/iuxwvR+qtgrooyWbH/0RbXG8ET3iUL2VuCu MgZQsc1u9w9jOgB5qKGxKwzEcvr/C8wtGxcemgGLMjds9fPhtlbesb0laLuiHEkazd 6AoQIHLTeQ0bg== From: =?UTF-8?q?Roman=20Ondr=C3=A1=C4=8Dek?= To: pve-devel@lists.proxmox.com Subject: [PATCH storage v3 1/2] add xz to known compression formats Date: Sun, 25 Oct 2026 15:27:19 +0100 Message-ID: <20261025142720.1784119-2-mail@romanondracek.cz> X-Mailer: git-send-email 2.55.0 In-Reply-To: <20261025142720.1784119-1-mail@romanondracek.cz> References: <20261025142720.1784119-1-mail@romanondracek.cz> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Spam-Status: No, score=-1.2 required=5.0 tests=ALL_TRUSTED,DKIM_SIGNED, DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF autolearn=ham autolearn_force=no version=4.0.1 X-Spam-Checker-Version: SpamAssassin 4.0.1 (2024-03-25) on vps.romanondracek.cz X-SPAM-LEVEL: Spam detection results: 0 AWL 0.350 Adjusted score from AWL reputation of From: address DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy RCVD_IN_DNSWL_LOW -0.7 Sender listed at https://www.dnswl.org/, low trust SPF_HELO_PASS -0.001 SPF: HELO matches SPF record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: SFRUA3CAJPAR2FD6WHW5V2O6LM3IUBJP X-Message-ID-Hash: SFRUA3CAJPAR2FD6WHW5V2O6LM3IUBJP X-MailFrom: mail@romanondracek.cz X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: =?UTF-8?q?Roman=20Ondr=C3=A1=C4=8Dek?= X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Some distributions only provide their ISO images compressed with xz, for example FreeBSD [1]. Such images cannot be decompressed when downloading them to a storage, as xz is not a known compression format. Add xz to the known compression formats, together with a decompressor using xz-utils. As a side effect, backup archives compressed with xz (e.g. vma.xz or tar.xz) are now recognized too, which was previously explicitly tested as a non-match. [1] https://download.freebsd.org/releases/amd64/amd64/ISO-IMAGES/15.0/ Signed-off-by: Roman Ondráček --- Changes v2 -> v3: * rebase onto current master * adapt to the simplified decompressor map shared by all archive formats, terminating the decompressor options with `--` debian/control | 1 + src/PVE/Storage.pm | 1 + src/PVE/Storage/Plugin.pm | 2 +- src/test/archive_info_test.pm | 5 +++-- src/test/list_volumes_test.pm | 11 ++++++++++- src/test/parse_volname_test.pm | 8 ++++---- src/test/path_to_volume_id_test.pm | 12 +++++++----- 7 files changed, 27 insertions(+), 13 deletions(-) diff --git a/debian/control b/debian/control index 497ef0c..9649177 100644 --- a/debian/control +++ b/debian/control @@ -50,6 +50,7 @@ Depends: bzip2, smbclient, thin-provisioning-tools, udev, + xz-utils, zstd, ${misc:Depends}, ${perl:Depends}, diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index 78756d4..6700308 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -1767,6 +1767,7 @@ sub decompressor_info { lzo => ['lzop', '-d', '-c', '--'], zst => ['zstd', '-q', '-d', '-c', '--'], bz2 => ['bzcat', '-q', '--'], + xz => ['xz', '-q', '-d', '-c', '--'], }; die "ERROR: archive format not defined\n" diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm index 3d07f74..05ca230 100644 --- a/src/PVE/Storage/Plugin.pm +++ b/src/PVE/Storage/Plugin.pm @@ -21,7 +21,7 @@ use JSON; use base qw(PVE::SectionConfig); -use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2'); +use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2', 'xz'); use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS); use constant LOG_EXT => ".log"; diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm index b25cb39..2a662de 100644 --- a/src/test/archive_info_test.pm +++ b/src/test/archive_info_test.pm @@ -115,12 +115,13 @@ my $tests = [ }, ]; -# add new compression fromats to test +# add new compression formats to test my $decompressor = { gz => ['zcat', '--'], lzo => ['lzop', '-d', '-c', '--'], zst => ['zstd', '-q', '-d', '-c', '--'], bz2 => ['bzcat', '-q', '--'], + xz => ['xz', '-q', '-d', '-c', '--'], }; my $bkp_suffix = { @@ -159,7 +160,7 @@ for my $virt (sort keys %$bkp_suffix) { my $non_bkp_suffix = { 'openvz' => ['zip', 'tgz.lzo', 'zip.gz', ''], 'lxc' => ['zip', 'tgz.lzo', 'zip.gz', ''], - 'qemu' => ['vma.xz', 'vms.gz', 'vmx.zst', ''], + 'qemu' => ['vms.gz', 'vmx.zst', ''], 'none' => ['tar.gz'], }; diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm index 0876902..d79e991 100644 --- a/src/test/list_volumes_test.pm +++ b/src/test/list_volumes_test.pm @@ -90,6 +90,7 @@ my @tests = ( "$storage_dir/images/16110/vm-16110-disk-1.raw", "$storage_dir/images/16110/vm-16110-disk-2.vmdk", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz", + "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", @@ -136,6 +137,15 @@ my @tests = ( 'vmid' => '16110', 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz', }, + { + 'content' => 'backup', + 'ctime' => 1585602760, + 'format' => 'vma.xz', + 'size' => DEFAULT_SIZE, + 'subtype' => 'qemu', + 'vmid' => '16110', + 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz', + }, { 'content' => 'backup', 'ctime' => 1585602765, @@ -457,7 +467,6 @@ my @tests = ( "$storage_dir/private/subvol-19254-disk-0/19254", "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz", "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo", - "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz", "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz", ], expected => [], # returns empty list diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm index 7e937aa..d31c0e4 100644 --- a/src/test/parse_volname_test.pm +++ b/src/test/parse_volname_test.pm @@ -247,9 +247,9 @@ foreach my $s (@$disk_suffix) { # create more test cases for backup files matches my $bkp_suffix = { - qemu => ['vma', 'vma.gz', 'vma.lzo', 'vma.zst'], - lxc => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2'], - openvz => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst'], + qemu => ['vma', 'vma.gz', 'vma.lzo', 'vma.xz', 'vma.zst'], + lxc => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.xz', 'tar.zst', 'tar.bz2'], + openvz => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.xz', 'tar.zst'], }; foreach my $virt (keys %$bkp_suffix) { @@ -277,7 +277,7 @@ foreach my $virt (keys %$bkp_suffix) { # create more test cases for failed backup files matches my $non_bkp_suffix = { - qemu => ['vms.gz', 'vma.xz'], + qemu => ['vms.gz'], lxc => ['zip.gz', 'tgz.lzo'], }; foreach my $virt (keys %$non_bkp_suffix) { diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm index dfa51e6..ad3f39a 100644 --- a/src/test/path_to_volume_id_test.pm +++ b/src/test/path_to_volume_id_test.pm @@ -92,6 +92,13 @@ my @tests = ( 'backup', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo', ], }, + { + description => 'Backup, vma.xz', + volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz", + expected => [ + 'backup', 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz', + ], + }, { description => 'Backup, vma.zst', volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst", @@ -212,11 +219,6 @@ my @tests = ( volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo", expected => [''], }, - { - description => 'Backup, wrong ending, qemu, vma.xz', - volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz", - expected => [''], - }, { description => 'Backup, wrong format, qemu, vms.gz', volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz", -- 2.55.0