From: Maximiliano Sandoval <m.sandoval@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH storage v2 1/2] storage: add bzip2 support
Date: Tue, 6 Aug 2024 14:58:08 +0200 [thread overview]
Message-ID: <20240806125809.185601-1-m.sandoval@proxmox.com> (raw)
Support for bz2 was requested at [1]. A popular ISO compressed
exclusively with bz2 is OPNsense [2].
Since this requires adding `bz2` to the list of known compression
formats we add decompression methods for vmz a tar.
[1] https://bugzilla.proxmox.com/show_bug.cgi?id=5267
[2] https://opnsense.org/download/
Suggested-by: Stoiko Ivanov <s.ivanov@proxmox.com>
Signed-off-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---
For the sake of not having a commit where tests fail, this is all in one commit.
A follow-up patch against pve-manager exposes bz2 decompression in the web UI
when downloading ISOs.
Differences from v1:
- Add more context to commit message
- Add decompression utilities as explicit dependencies
src/PVE/Storage.pm | 3 +++
src/PVE/Storage/Plugin.pm | 2 +-
src/test/archive_info_test.pm | 6 ++++--
src/test/list_volumes_test.pm | 11 ++++++++++-
src/test/parse_volname_test.pm | 4 ++--
src/test/path_to_volume_id_test.pm | 13 ++++++++-----
6 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 57b2038..0caac71 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1550,16 +1550,19 @@ sub decompressor_info {
gz => ['tar', '-z'],
lzo => ['tar', '--lzop'],
zst => ['tar', '--zstd'],
+ bz2 => ['tar', '--bzip2'],
},
vma => {
gz => ['zcat'],
lzo => ['lzop', '-d', '-c'],
zst => ['zstd', '-q', '-d', '-c'],
+ bz2 => ['bzcat', '-q'],
},
iso => {
gz => ['zcat'],
lzo => ['lzop', '-d', '-c'],
zst => ['zstd', '-q', '-d', '-c'],
+ bz2 => ['bzcat', '-q'],
},
};
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 6444390..e5ef9de 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -19,7 +19,7 @@ use JSON;
use base qw(PVE::SectionConfig);
-use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst');
+use constant KNOWN_COMPRESSION_FORMATS => ('gz', 'lzo', 'zst', 'bz2');
use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS);
use constant LOG_EXT => ".log";
diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm
index cf03c3d..53e37be 100644
--- a/src/test/archive_info_test.pm
+++ b/src/test/archive_info_test.pm
@@ -121,11 +121,13 @@ my $decompressor = {
gz => ['tar', '-z'],
lzo => ['tar', '--lzop'],
zst => ['tar', '--zstd'],
+ bz2 => ['tar', '--bzip2'],
},
vma => {
gz => ['zcat'],
lzo => ['lzop', '-d', '-c'],
zst => ['zstd', '-q', '-d', '-c'],
+ bz2 => ['bzcat', '-q'],
},
};
@@ -163,8 +165,8 @@ for my $virt (sort keys %$bkp_suffix) {
# add compression formats to test failed matches
my $non_bkp_suffix = {
- 'openvz' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
- 'lxc' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
+ 'openvz' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
+ 'lxc' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
'qemu' => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ],
'none' => [ 'tar.gz', ],
};
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index d155cb9..ba959ff 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -189,6 +189,7 @@ my @tests = (
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
+ "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
],
expected => [
{
@@ -237,6 +238,15 @@ my @tests = (
'vmid' => '16112',
'volid' => 'local:backup/vzdump-lxc-16112-2020_03_30-21_59_30.tgz',
},
+ {
+ 'content' => 'backup',
+ 'ctime' => 1585604370,
+ 'format' => 'tar.bz2',
+ 'size' => DEFAULT_SIZE,
+ 'subtype' => 'openvz',
+ 'vmid' => '16112',
+ 'volid' => 'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2',
+ },
],
},
{
@@ -440,7 +450,6 @@ my @tests = (
"$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.zip.gz",
"$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.bz2",
"$storage_dir/private/subvol-19254-disk-0/19254",
- "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
"$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
"$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index d6ac885..5f63941 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -177,7 +177,7 @@ foreach my $s (@$disk_suffix) {
# create more test cases for backup files matches
my $bkp_suffix = {
qemu => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst' ],
- lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
+ lxc => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2' ],
openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
};
@@ -204,7 +204,7 @@ foreach my $virt (keys %$bkp_suffix) {
# create more test cases for failed backup files matches
my $non_bkp_suffix = {
qemu => [ 'vms.gz', 'vma.xz' ],
- lxc => [ 'tar.bz2', 'zip.gz', 'tgz.lzo' ],
+ lxc => [ 'zip.gz', 'tgz.lzo' ],
};
foreach my $virt (keys %$non_bkp_suffix) {
my $suffix = $non_bkp_suffix->{$virt};
diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm
index 8149c88..0f39b03 100644
--- a/src/test/path_to_volume_id_test.pm
+++ b/src/test/path_to_volume_id_test.pm
@@ -116,6 +116,14 @@ my @tests = (
'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.zst'
],
},
+ {
+ description => 'Backup, tar.bz2',
+ volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
+ expected => [
+ 'backup',
+ 'local:backup/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2',
+ ],
+ },
{
description => 'ISO file',
@@ -201,11 +209,6 @@ my @tests = (
volname => "$storage_dir/private/subvol-19254-disk-0/",
expected => [''],
},
- {
- description => 'Backup, wrong ending, openvz, tar.bz2',
- volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tar.bz2",
- expected => [''],
- },
{
description => 'Backup, wrong format, openvz, zip.gz',
volname => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
--
2.39.2
_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
next reply other threads:[~2024-08-06 12:58 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-08-06 12:58 Maximiliano Sandoval [this message]
2024-08-06 12:58 ` [pve-devel] [PATCH storage v2 2/2] d/control: dependencies: add bzip2, gzip, lzpo, zstd Maximiliano Sandoval
2024-08-08 9:34 ` [pve-devel] [PATCH storage v2 1/2] storage: add bzip2 support Fabian Grünbichler
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240806125809.185601-1-m.sandoval@proxmox.com \
--to=m.sandoval@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.