From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 87A921FF183 for ; Mon, 11 Nov 2024 10:41:58 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 78BF23E7F9; Mon, 11 Nov 2024 10:41:57 +0100 (CET) Date: Mon, 11 Nov 2024 10:41:19 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox VE development discussion References: <20240913081352.76932-1-m.sandoval@proxmox.com> In-Reply-To: <20240913081352.76932-1-m.sandoval@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1731317977.q2i514tlv6.astroid@yuna.none> X-SPAM-LEVEL: Spam detection results: 0 AWL -0.002 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment PROLO_LEO1 0.1 Meta Catches all Leo drug variations so far SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] applied-series: [PATCH storage v4 1/2] fix #5267: storage: add bzip2 support X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox VE development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pve-devel-bounces@lists.proxmox.com Sender: "pve-devel" with Aaron's T-b on the first patch, and the second patch's commit message adapted. On September 13, 2024 10:13 am, Maximiliano Sandoval wrote: > A popular ISO compressed exclusively with bz2 is OPNsense [2]. > > Since this requires adding `bz2` to the list of known compression > formats we add decompression methods for vmz and tar. > > [2] https://opnsense.org/download/ > > Suggested-by: Stoiko Ivanov > Signed-off-by: Maximiliano Sandoval > --- > > Differences from v3: > - Remove gzip dependency from d/control > - Fix typo in commit message > - Add bz2 to VZTMPL_EXT_RE_1 regex which allow us to: > - Fix tests > > Differences from v2: > - Fix typo s/lzpo/lzop > - Add fix to commit message > > Differences from v1: > - Add more context to commit message > - Add decompression utilities as explicit dependencies > > src/PVE/Storage.pm | 5 ++++- > src/PVE/Storage/Plugin.pm | 2 +- > src/test/archive_info_test.pm | 6 ++++-- > src/test/list_volumes_test.pm | 20 ++++++++++++++++++-- > src/test/parse_volname_test.pm | 14 +++++++------- > src/test/path_to_volume_id_test.pm | 26 ++++++++++++++++---------- > 6 files changed, 50 insertions(+), 23 deletions(-) > > diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm > index 57b2038e..2586753b 100755 > --- a/src/PVE/Storage.pm > +++ b/src/PVE/Storage.pm > @@ -110,7 +110,7 @@ PVE::Storage::Plugin->init(); > > our $ISO_EXT_RE_0 = qr/\.(?:iso|img)/i; > > -our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i; > +our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst|bz2)/i; > > our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/; > > @@ -1550,16 +1550,19 @@ sub decompressor_info { > gz => ['tar', '-z'], > lzo => ['tar', '--lzop'], > zst => ['tar', '--zstd'], > + bz2 => ['tar', '--bzip2'], > }, > vma => { > gz => ['zcat'], > lzo => ['lzop', '-d', '-c'], > zst => ['zstd', '-q', '-d', '-c'], > + bz2 => ['bzcat', '-q'], > }, > iso => { > gz => ['zcat'], > lzo => ['lzop', '-d', '-c'], > zst => ['zstd', '-q', '-d', '-c'], > + bz2 => ['bzcat', '-q'], > }, > }; > > diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm > index 8cc693c7..445577bc 100644 > --- a/src/PVE/Storage/Plugin.pm > +++ b/src/PVE/Storage/Plugin.pm > @@ -19,7 +19,7 @@ use JSON; > > use base qw(PVE::SectionConfig); > > -use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst'); > +use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2'); > use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS); > > use constant LOG_EXT => ".log"; > diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm > index cf03c3d8..53e37be1 100644 > --- a/src/test/archive_info_test.pm > +++ b/src/test/archive_info_test.pm > @@ -121,11 +121,13 @@ my $decompressor = { > gz => ['tar', '-z'], > lzo => ['tar', '--lzop'], > zst => ['tar', '--zstd'], > + bz2 => ['tar', '--bzip2'], > }, > vma => { > gz => ['zcat'], > lzo => ['lzop', '-d', '-c'], > zst => ['zstd', '-q', '-d', '-c'], > + bz2 => ['bzcat', '-q'], > }, > }; > > @@ -163,8 +165,8 @@ for my $virt (sort keys %$bkp_suffix) { > > # add compression formats to test failed matches > my $non_bkp_suffix = { > - 'openvz' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ], > - 'lxc' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ], > + 'openvz' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ], > + 'lxc' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ], > 'qemu' => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ], > 'none' => [ 'tar.gz', ], > }; > diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm > index d155cb9f..e2648029 100644 > --- a/src/test/list_volumes_test.pm > +++ b/src/test/list_volumes_test.pm > @@ -189,6 +189,7 @@ my @tests = ( > "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz", > "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst", > "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz", > + "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2", > ], > expected => [ > { > @@ -237,6 +238,15 @@ my @tests = ( > 'vmid' => '16112', > 'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz', > }, > + { > + 'content' => 'backup', > + 'ctime' => 1585604370, > + 'format' => 'tar.bz2', > + 'size' => DEFAULT_SIZE, > + 'subtype' => 'openvz', > + 'vmid' => '16112', > + 'volid' => 'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2', > + }, > ], > }, > { > @@ -315,6 +325,7 @@ my @tests = ( > "$storage_dir/template/iso/proxmox-ve_6.1-1.iso", > "$storage_dir/template/cache/archlinux-base_20190924-1_amd64.tar.gz", > "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz", > + "$storage_dir/template/cache/debian-11.0-standard_11.0-1_amd64.tar.bz2", > "$storage_dir/template/cache/alpine-3.10-default_20190626_amd64.tar.xz", > "$storage_dir/snippets/userconfig.yaml", > "$storage_dir/snippets/hookscript.pl", > @@ -343,6 +354,13 @@ my @tests = ( > 'size' => DEFAULT_SIZE, > 'volid' => 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz', > }, > + { > + 'content' => 'vztmpl', > + 'ctime' => DEFAULT_CTIME, > + 'format' => 'tbz2', > + 'size' => DEFAULT_SIZE, > + 'volid' => 'local:vztmpl/debian-11.0-standard_11.0-1_amd64.tar.bz2', > + }, > { > 'content' => 'iso', > 'ctime' => DEFAULT_CTIME, > @@ -438,9 +456,7 @@ my @tests = ( > "$storage_dir/images/ssss/vm-1234-disk-0.qcow2", > "$storage_dir/template/iso/yet-again-a-installation-disk.dvd", > "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz", > - "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2", > "$storage_dir/private/subvol-19254-disk-0/19254", > - "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2", > "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz", > "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo", > "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz", > diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm > index d6ac8854..6c5ba043 100644 > --- a/src/test/parse_volname_test.pm > +++ b/src/test/parse_volname_test.pm > @@ -49,6 +49,11 @@ my $tests = [ > volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.xz', > expected => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.xz'], > }, > + { > + description => 'Container template tar.bz2', > + volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2', > + expected => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.bz2'], > + }, > # > # container rootdir > # > @@ -98,11 +103,6 @@ my $tests = [ > volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz', > expected => "unable to parse directory volume name 'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz'\n", > }, > - { > - description => 'Failed match: Container template, tar.bz2', > - volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2', > - expected => "unable to parse directory volume name 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2'\n", > - }, > { > description => 'Failed match: Container rootdir, subvol', > volname => "rootdir/subvol-$vmid-disk-0", > @@ -177,7 +177,7 @@ foreach my $s (@$disk_suffix) { > # create more test cases for backup files matches > my $bkp_suffix = { > qemu => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst' ], > - lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ], > + lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2' ], > openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ], > }; > > @@ -204,7 +204,7 @@ foreach my $virt (keys %$bkp_suffix) { > # create more test cases for failed backup files matches > my $non_bkp_suffix = { > qemu => [ 'vms.gz', 'vma.xz' ], > - lxc => [ 'tar.bz2', 'zip.gz', 'tgz.lzo' ], > + lxc => [ 'zip.gz', 'tgz.lzo' ], > }; > foreach my $virt (keys %$non_bkp_suffix) { > my $suffix = $non_bkp_suffix->{$virt}; > diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm > index 8149c881..3198752f 100644 > --- a/src/test/path_to_volume_id_test.pm > +++ b/src/test/path_to_volume_id_test.pm > @@ -116,6 +116,14 @@ my @tests = ( > 'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst' > ], > }, > + { > + description => 'Backup, tar.bz2', > + volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2", > + expected => [ > + 'backup', > + 'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2', > + ], > + }, > > { > description => 'ISO file', > @@ -133,6 +141,14 @@ my @tests = ( > 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz', > ], > }, > + { > + description => 'CT template, wrong ending, tar bz2', > + volname => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2", > + expected => [ > + 'vztmpl', > + 'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2', > + ], > + }, > > { > description => 'Rootdir', > @@ -191,21 +207,11 @@ my @tests = ( > volname => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz", > expected => [''], > }, > - { > - description => 'CT template, wrong ending, tar bz2', > - volname => "$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2", > - expected => [''], > - }, > { > description => 'Rootdir as subvol, wrong path', > volname => "$storage_dir/private/subvol-19254-disk-0/", > expected => [''], > }, > - { > - description => 'Backup, wrong ending, openvz, tar.bz2', > - volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2", > - expected => [''], > - }, > { > description => 'Backup, wrong format, openvz, zip.gz', > volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz", > -- > 2.39.2 > > > > _______________________________________________ > pve-devel mailing list > pve-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel > > > _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel