all lists on lists.proxmox.com
 help / color / mirror / Atom feed
* [pve-devel] [PATCH storage v4 1/2] fix #5267: storage: add bzip2 support
@ 2024-09-13  8:13 Maximiliano Sandoval
  2024-09-13  8:13 ` [pve-devel] [PATCH storage v4 2/2] d/control: dependencies: add bzip2, gzip, lzop, zstd Maximiliano Sandoval
                   ` (2 more replies)
  0 siblings, 3 replies; 10+ messages in thread
From: Maximiliano Sandoval @ 2024-09-13  8:13 UTC (permalink / raw)
  To: pve-devel

A popular ISO compressed exclusively with bz2 is OPNsense [2].

Since this requires adding `bz2` to the list of known compression
formats we add decompression methods for vmz and tar.

[2] https://opnsense.org/download/

Suggested-by: Stoiko Ivanov <s.ivanov@proxmox.com>
Signed-off-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---

Differences from v3:
 - Remove gzip dependency from d/control
 - Fix typo in commit message
 - Add bz2 to VZTMPL_EXT_RE_1 regex which allow us to:
 - Fix tests

Differences from v2:
 - Fix typo s/lzpo/lzop
 - Add fix to commit message

Differences from v1:
 - Add more context to commit message
 - Add decompression utilities as explicit dependencies

 src/PVE/Storage.pm                 |  5 ++++-
 src/PVE/Storage/Plugin.pm          |  2 +-
 src/test/archive_info_test.pm      |  6 ++++--
 src/test/list_volumes_test.pm      | 20 ++++++++++++++++++--
 src/test/parse_volname_test.pm     | 14 +++++++-------
 src/test/path_to_volume_id_test.pm | 26 ++++++++++++++++----------
 6 files changed, 50 insertions(+), 23 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 57b2038e..2586753b 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -110,7 +110,7 @@ PVE::Storage::Plugin->init();
 
 our $ISO_EXT_RE_0 = qr/\.(?:iso|img)/i;
 
-our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i;
+our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst|bz2)/i;
 
 our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/;
 
@@ -1550,16 +1550,19 @@ sub decompressor_info {
 	    gz => ['tar', '-z'],
 	    lzo => ['tar', '--lzop'],
 	    zst => ['tar', '--zstd'],
+	    bz2 => ['tar', '--bzip2'],
 	},
 	vma => {
 	    gz => ['zcat'],
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
+	    bz2 => ['bzcat', '-q'],
 	},
 	iso => {
 	    gz => ['zcat'],
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
+	    bz2 => ['bzcat', '-q'],
 	},
     };
 
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 8cc693c7..445577bc 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -19,7 +19,7 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant KNOWN_COMPRESSION_FORMATS =>  ('gz', 'lzo', 'zst');
+use constant KNOWN_COMPRESSION_FORMATS =>  ('gz', 'lzo', 'zst', 'bz2');
 use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS);
 
 use constant LOG_EXT => ".log";
diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm
index cf03c3d8..53e37be1 100644
--- a/src/test/archive_info_test.pm
+++ b/src/test/archive_info_test.pm
@@ -121,11 +121,13 @@ my $decompressor = {
 	gz  => ['tar', '-z'],
 	lzo => ['tar', '--lzop'],
 	zst => ['tar', '--zstd'],
+	bz2 => ['tar', '--bzip2'],
     },
     vma => {
 	gz  => ['zcat'],
 	lzo => ['lzop', '-d', '-c'],
 	zst => ['zstd', '-q', '-d', '-c'],
+	bz2 => ['bzcat', '-q'],
     },
 };
 
@@ -163,8 +165,8 @@ for my $virt (sort keys %$bkp_suffix) {
 
 # add compression formats to test failed matches
 my $non_bkp_suffix = {
-    'openvz' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
-    'lxc'    => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
+    'openvz' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
+    'lxc'    => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
     'qemu'   => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ],
     'none'   => [ 'tar.gz', ],
 };
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index d155cb9f..e2648029 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -189,6 +189,7 @@ my @tests = (
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
 	    "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
+	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
 	],
 	expected => [
 	    {
@@ -237,6 +238,15 @@ my @tests = (
 		'vmid'    => '16112',
 		'volid'   => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz',
 	    },
+	    {
+		'content' => 'backup',
+		'ctime'   => 1585604370,
+		'format'  => 'tar.bz2',
+		'size'    => DEFAULT_SIZE,
+		'subtype' => 'openvz',
+		'vmid'    => '16112',
+		'volid'   => 'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2',
+	    },
 	],
     },
     {
@@ -315,6 +325,7 @@ my @tests = (
 	    "$storage_dir/template/iso/proxmox-ve_6.1-1.iso",
 	    "$storage_dir/template/cache/archlinux-base_20190924-1_amd64.tar.gz",
 	    "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz",
+	    "$storage_dir/template/cache/debian-11.0-standard_11.0-1_amd64.tar.bz2",
 	    "$storage_dir/template/cache/alpine-3.10-default_20190626_amd64.tar.xz",
 	    "$storage_dir/snippets/userconfig.yaml",
 	    "$storage_dir/snippets/hookscript.pl",
@@ -343,6 +354,13 @@ my @tests = (
 		'size'    => DEFAULT_SIZE,
 		'volid'   => 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
 	    },
+	    {
+		'content' => 'vztmpl',
+		'ctime'   => DEFAULT_CTIME,
+		'format'  => 'tbz2',
+		'size'    => DEFAULT_SIZE,
+		'volid'   => 'local:vztmpl/debian-11.0-standard_11.0-1_amd64.tar.bz2',
+	    },
 	    {
 		'content' => 'iso',
 		'ctime'   => DEFAULT_CTIME,
@@ -438,9 +456,7 @@ my @tests = (
 	    "$storage_dir/images/ssss/vm-1234-disk-0.qcow2",
 	    "$storage_dir/template/iso/yet-again-a-installation-disk.dvd",
 	    "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz",
-	    "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
 	    "$storage_dir/private/subvol-19254-disk-0/19254",
-	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
 	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
 	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index d6ac8854..6c5ba043 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -49,6 +49,11 @@ my $tests = [
 	volname     => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.xz',
 	expected    => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.xz'],
     },
+    {
+	description => 'Container template tar.bz2',
+	volname     => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2',
+	expected    => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.bz2'],
+    },
     #
     # container rootdir
     #
@@ -98,11 +103,6 @@ my $tests = [
 	volname     => 'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz',
 	expected    => "unable to parse directory volume name 'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz'\n",
     },
-    {
-	description => 'Failed match: Container template, tar.bz2',
-	volname     => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2',
-	expected    => "unable to parse directory volume name 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2'\n",
-    },
     {
 	description => 'Failed match: Container rootdir, subvol',
 	volname     => "rootdir/subvol-$vmid-disk-0",
@@ -177,7 +177,7 @@ foreach my $s (@$disk_suffix) {
 # create more test cases for backup files matches
 my $bkp_suffix = {
     qemu   => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst' ],
-    lxc    => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
+    lxc    => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2' ],
     openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
 };
 
@@ -204,7 +204,7 @@ foreach my $virt (keys %$bkp_suffix) {
 # create more test cases for failed backup files matches
 my $non_bkp_suffix = {
     qemu   => [ 'vms.gz', 'vma.xz' ],
-    lxc    => [ 'tar.bz2', 'zip.gz', 'tgz.lzo' ],
+    lxc    => [ 'zip.gz', 'tgz.lzo' ],
 };
 foreach my $virt (keys %$non_bkp_suffix) {
     my $suffix = $non_bkp_suffix->{$virt};
diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm
index 8149c881..3198752f 100644
--- a/src/test/path_to_volume_id_test.pm
+++ b/src/test/path_to_volume_id_test.pm
@@ -116,6 +116,14 @@ my @tests = (
 	    'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst'
 	],
     },
+    {
+	description => 'Backup, tar.bz2',
+	volname     => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
+	expected    => [
+	    'backup',
+	    'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2',
+	],
+    },
 
     {
 	description => 'ISO file',
@@ -133,6 +141,14 @@ my @tests = (
 	    'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
 	],
     },
+    {
+	description => 'CT template, wrong ending, tar bz2',
+	volname     => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
+	expected    => [
+	    'vztmpl',
+	    'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2',
+	],
+    },
 
     {
 	description => 'Rootdir',
@@ -191,21 +207,11 @@ my @tests = (
 	volname     => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz",
 	expected    => [''],
     },
-    {
-	description => 'CT template, wrong ending, tar bz2',
-	volname     => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
-	expected    => [''],
-    },
     {
 	description => 'Rootdir as subvol, wrong path',
 	volname     => "$storage_dir/private/subvol-19254-disk-0/",
 	expected    => [''],
     },
-    {
-	description => 'Backup, wrong ending, openvz, tar.bz2',
-	volname     => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
-	expected    => [''],
-    },
     {
 	description => 'Backup, wrong format, openvz, zip.gz',
 	volname     => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
-- 
2.39.2



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel


^ permalink raw reply	[flat|nested] 10+ messages in thread
end of thread, other threads:[~2024-11-11  9:41 UTC | newest]

Thread overview: 10+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-09-13  8:13 [pve-devel] [PATCH storage v4 1/2] fix #5267: storage: add bzip2 support Maximiliano Sandoval
2024-09-13  8:13 ` [pve-devel] [PATCH storage v4 2/2] d/control: dependencies: add bzip2, gzip, lzop, zstd Maximiliano Sandoval
2024-09-13  8:23   ` Fabian Grünbichler
2024-09-13  8:25     ` Maximiliano Sandoval
2024-10-01 11:03       ` Maximiliano Sandoval
2024-10-07 11:57         ` Maximiliano Sandoval
2024-10-14 12:19           ` Maximiliano Sandoval
2024-11-05 10:31             ` Maximiliano Sandoval
2024-11-07 10:35 ` [pve-devel] [PATCH storage v4 1/2] fix #5267: storage: add bzip2 support Aaron Lauterer
2024-11-11  9:41 ` [pve-devel] applied-series: " Fabian Grünbichler

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal