[PATCH storage v3 1/2] add xz to known compression formats

["Roman Ondráček" <mail@romanondracek.cz>]

Some distributions only provide their ISO images compressed with xz,
for example FreeBSD [1]. Such images cannot be decompressed when
downloading them to a storage, as xz is not a known compression format.

Add xz to the known compression formats, together with a decompressor
using xz-utils. As a side effect, backup archives compressed with xz
(e.g. vma.xz or tar.xz) are now recognized too, which was previously
explicitly tested as a non-match.

[1] https://download.freebsd.org/releases/amd64/amd64/ISO-IMAGES/15.0/

Signed-off-by: Roman Ondráček <mail@romanondracek.cz>
---

Changes v2 -> v3:
* rebase onto current master
* adapt to the simplified decompressor map shared by all archive
  formats, terminating the decompressor options with `--`

 debian/control                     |  1 +
 src/PVE/Storage.pm                 |  1 +
 src/PVE/Storage/Plugin.pm          |  2 +-
 src/test/archive_info_test.pm      |  5 +++--
 src/test/list_volumes_test.pm      | 11 ++++++++++-
 src/test/parse_volname_test.pm     |  8 ++++----
 src/test/path_to_volume_id_test.pm | 12 +++++++-----
 7 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/debian/control b/debian/control
index 497ef0c..9649177 100644
--- a/debian/control
+++ b/debian/control
@@ -50,6 +50,7 @@ Depends: bzip2,
          smbclient,
          thin-provisioning-tools,
          udev,
+         xz-utils,
          zstd,
          ${misc:Depends},
          ${perl:Depends},
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 78756d4..6700308 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1767,6 +1767,7 @@ sub decompressor_info {
         lzo => ['lzop', '-d', '-c', '--'],
         zst => ['zstd', '-q', '-d', '-c', '--'],
         bz2 => ['bzcat', '-q', '--'],
+        xz => ['xz', '-q', '-d', '-c', '--'],
     };
 
     die "ERROR: archive format not defined\n"
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 3d07f74..05ca230 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -21,7 +21,7 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2');
+use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2', 'xz');
 use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS);
 
 use constant LOG_EXT => ".log";
diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm
index b25cb39..2a662de 100644
--- a/src/test/archive_info_test.pm
+++ b/src/test/archive_info_test.pm
@@ -115,12 +115,13 @@ my $tests = [
     },
 ];
 
-# add new compression fromats to test
+# add new compression formats to test
 my $decompressor = {
     gz => ['zcat', '--'],
     lzo => ['lzop', '-d', '-c', '--'],
     zst => ['zstd', '-q', '-d', '-c', '--'],
     bz2 => ['bzcat', '-q', '--'],
+    xz => ['xz', '-q', '-d', '-c', '--'],
 };
 
 my $bkp_suffix = {
@@ -159,7 +160,7 @@ for my $virt (sort keys %$bkp_suffix) {
 my $non_bkp_suffix = {
     'openvz' => ['zip', 'tgz.lzo', 'zip.gz', ''],
     'lxc' => ['zip', 'tgz.lzo', 'zip.gz', ''],
-    'qemu' => ['vma.xz', 'vms.gz', 'vmx.zst', ''],
+    'qemu' => ['vms.gz', 'vmx.zst', ''],
     'none' => ['tar.gz'],
 };
 
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index 0876902..d79e991 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -90,6 +90,7 @@ my @tests = (
             "$storage_dir/images/16110/vm-16110-disk-1.raw",
             "$storage_dir/images/16110/vm-16110-disk-2.vmdk",
             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
+            "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
@@ -136,6 +137,15 @@ my @tests = (
                 'vmid' => '16110',
                 'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
             },
+            {
+                'content' => 'backup',
+                'ctime' => 1585602760,
+                'format' => 'vma.xz',
+                'size' => DEFAULT_SIZE,
+                'subtype' => 'qemu',
+                'vmid' => '16110',
+                'volid' => 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz',
+            },
             {
                 'content' => 'backup',
                 'ctime' => 1585602765,
@@ -457,7 +467,6 @@ my @tests = (
             "$storage_dir/private/subvol-19254-disk-0/19254",
             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
             "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
-            "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
             "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
         ],
         expected => [], # returns empty list
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index 7e937aa..d31c0e4 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -247,9 +247,9 @@ foreach my $s (@$disk_suffix) {
 
 # create more test cases for backup files matches
 my $bkp_suffix = {
-    qemu => ['vma', 'vma.gz', 'vma.lzo', 'vma.zst'],
-    lxc => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2'],
-    openvz => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst'],
+    qemu => ['vma', 'vma.gz', 'vma.lzo', 'vma.xz', 'vma.zst'],
+    lxc => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.xz', 'tar.zst', 'tar.bz2'],
+    openvz => ['tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.xz', 'tar.zst'],
 };
 
 foreach my $virt (keys %$bkp_suffix) {
@@ -277,7 +277,7 @@ foreach my $virt (keys %$bkp_suffix) {
 
 # create more test cases for failed backup files matches
 my $non_bkp_suffix = {
-    qemu => ['vms.gz', 'vma.xz'],
+    qemu => ['vms.gz'],
     lxc => ['zip.gz', 'tgz.lzo'],
 };
 foreach my $virt (keys %$non_bkp_suffix) {
diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm
index dfa51e6..ad3f39a 100644
--- a/src/test/path_to_volume_id_test.pm
+++ b/src/test/path_to_volume_id_test.pm
@@ -92,6 +92,13 @@ my @tests = (
             'backup', 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo',
         ],
     },
+    {
+        description => 'Backup, vma.xz',
+        volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
+        expected => [
+            'backup', 'local:backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz',
+        ],
+    },
     {
         description => 'Backup, vma.zst',
         volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
@@ -212,11 +219,6 @@ my @tests = (
         volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
         expected => [''],
     },
-    {
-        description => 'Backup, wrong ending, qemu, vma.xz',
-        volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
-        expected => [''],
-    },
     {
         description => 'Backup, wrong format, qemu, vms.gz',
         volname => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
-- 
2.55.0