[pve-devel] [PATCH guest-common/manager/storage/vma-to-pbs 00/12] align supported compression formats for templates and backups

[pve-devel] [PATCH guest-common/manager/storage/vma-to-pbs 00/12] align supported compression formats for templates and backups

[Filip Schauer]

Make the supported compression formats consistent across templates and
backups.

Currently we support:
* VM/container backups compressed with: gz, lzo, zst, bz2
* container templates compressed with: gz, xz, zst, bz2

This patch series makes this consistent, so that we support gz, xz, lzo,
zst, bz2 in every case.

In summary, this adds support for:
* .tar.lzo container templates
* .tar.xz container backups
* .vma.xz VM backups

Additionally, this patch series completes support for bzip2, which is
listed in the PVE::Storage::VZTMPL_EXT_RE_1 and
PVE::Storage::BACKUP_EXT_RE_2 regexes, but was forgotten to be fully
implemented in some places.

Note: .tar.lzo container templates are already supported in
PVE::LXC::Create::restore_tar_archive, since this code is shared with
container backups.

pve-storage:

Filip Schauer (3):
  add support for .vma.bz2
  add support for xz compressed VM and container backups
  align supported compression formats for templates and backups

 debian/control                     |  1 +
 src/PVE/Storage.pm                 | 16 +++++++++++++---
 src/PVE/Storage/Plugin.pm          |  2 +-
 src/test/archive_info_test.pm      |  4 +++-
 src/test/list_volumes_test.pm      | 21 ++++++++++++++++++++-
 src/test/parse_volname_test.pm     |  6 +++---
 src/test/path_to_volume_id_test.pm | 13 ++++++++-----
 7 files changed, 49 insertions(+), 14 deletions(-)


pve-guest-common:

Filip Schauer (2):
  vzdump: add support for bzip2 compression
  vzdump: add support for xz compression

 src/PVE/VZDump/Common.pm | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)


pve-manager:

Filip Schauer (5):
  accept .tar.bz2 ct templates in the storage upload dialog
  vzdump: add support for bzip2 compression
  vzdump: add support for xz compression
  allow download of xz compressed files
  accept .tar.lzo ct templates in the storage upload dialog

 PVE/VZDump.pm                                  | 9 +++++++++
 www/manager6/form/BackupCompressionSelector.js | 2 ++
 www/manager6/window/DownloadUrlToStorage.js    | 3 ++-
 www/manager6/window/UploadToStorage.js         | 2 +-
 4 files changed, 14 insertions(+), 2 deletions(-)


vma-to-pbs:

Filip Schauer (2):
  add support for bzip2 compressed VMA files
  add support for xz compressed VMA files

 src/main.rs    |  4 +++-
 src/vma2pbs.rs | 12 ++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)


Summary over all repositories:
  14 files changed, 86 insertions(+), 18 deletions(-)

-- 
Generated by git-murpp 0.6.0


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage 01/12] add support for .vma.bz2

[Filip Schauer]

This aligns with the PVE::Storage::BACKUP_EXT_RE_2 regex

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/PVE/Storage.pm             | 11 +++++++++--
 src/test/list_volumes_test.pm  | 10 ++++++++++
 src/test/parse_volname_test.pm |  2 +-
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index d0a696a..6031261 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1720,13 +1720,20 @@ sub extract_vzdump_config_vma {
     if ($comp) {
 	my $cmd = [ [@$decompressor, $archive], ["vma", "config", "-"] ];
 
-	# lzop/zcat exits with 1 when the pipe is closed early by vma, detect this and ignore the exit code later
+	# lzop/zcat/zstd/bzcat exits with 1 when the pipe is closed early by vma,
+	# detect this and ignore the exit code later
 	my $broken_pipe;
 	my $errstring;
 	my $err = sub {
 	    my $output = shift;
-	    if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error.*Broken pipe/) {
+	    if (
+		$output =~ m/lzop: Broken pipe: <stdout>/
+		|| $output =~ m/gzip: stdout: Broken pipe/
+		|| $output =~ m/zstd: error 70 : Write error.*Broken pipe/
+		|| $output =~ m/bzcat: Broken pipe/
+	    ) {
 		$broken_pipe = 1;
+		$errstring = "";
 	    } elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
 		$errstring = "Failed to extract config from VMA archive: $output\n";
 	    }
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index 7b6df6a..5adba03 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -94,6 +94,7 @@ my @tests = (
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
+	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_15_00.vma.bz2",
 	    "$storage_dir/snippets/userconfig.yaml",
 	    "$storage_dir/snippets/hookscript.pl",
 	],
@@ -164,6 +165,15 @@ my @tests = (
 		'vmid'    => '16110',
 		'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst',
 	    },
+	    {
+		'content' => 'backup',
+		'ctime'   => 1585602900,
+		'format'  => 'vma.bz2',
+		'size'    => DEFAULT_SIZE,
+		'subtype' => 'qemu',
+		'vmid'    => '16110',
+		'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_15_00.vma.bz2',
+	    },
 	    {
 		'content' => 'snippets',
 		'ctime'   => DEFAULT_CTIME,
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index 175500d..0ede982 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -214,7 +214,7 @@ foreach my $s (@$disk_suffix) {
 
 # create more test cases for backup files matches
 my $bkp_suffix = {
-    qemu   => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst' ],
+    qemu   => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst', 'vma.bz2' ],
     lxc    => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2' ],
     openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
 };
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage 02/12] add support for xz compressed VM and container backups

[Filip Schauer]

Add support for xz compressed VMA files and allow the use of xz
compressed container backups, which are already explicitly supported in
PVE::LXC::Create::restore_tar_archive.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 debian/control                     |  1 +
 src/PVE/Storage.pm                 |  3 +++
 src/PVE/Storage/Plugin.pm          |  2 +-
 src/test/archive_info_test.pm      |  4 +++-
 src/test/list_volumes_test.pm      | 11 ++++++++++-
 src/test/parse_volname_test.pm     |  6 +++---
 src/test/path_to_volume_id_test.pm | 13 ++++++++-----
 7 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/debian/control b/debian/control
index 4e1a046..c8e4f4e 100644
--- a/debian/control
+++ b/debian/control
@@ -51,6 +51,7 @@ Depends: bzip2,
          smbclient,
          thin-provisioning-tools,
          udev,
+         xz-utils,
          zstd,
          ${misc:Depends},
          ${perl:Depends},
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 6031261..0f0fad8 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1578,18 +1578,21 @@ sub decompressor_info {
     my $decompressor = {
 	tar => {
 	    gz => ['tar', '-z'],
+	    xz => ['tar', '--xz'],
 	    lzo => ['tar', '--lzop'],
 	    zst => ['tar', '--zstd'],
 	    bz2 => ['tar', '--bzip2'],
 	},
 	vma => {
 	    gz => ['zcat'],
+	    xz => ['xzcat', '-q'],
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
 	    bz2 => ['bzcat', '-q'],
 	},
 	iso => {
 	    gz => ['zcat'],
+	    xz => ['xzcat', '-q'],
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
 	    bz2 => ['bzcat', '-q'],
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 4e16420..9bc04ad 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -21,7 +21,7 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant KNOWN_COMPRESSION_FORMATS =>  ('gz', 'lzo', 'zst', 'bz2');
+use constant KNOWN_COMPRESSION_FORMATS =>  ('gz', 'xz', 'lzo', 'zst', 'bz2');
 use constant COMPRESSOR_RE => join('|', KNOWN_COMPRESSION_FORMATS);
 
 use constant LOG_EXT => ".log";
diff --git a/src/test/archive_info_test.pm b/src/test/archive_info_test.pm
index 53e37be..f8bf7b1 100644
--- a/src/test/archive_info_test.pm
+++ b/src/test/archive_info_test.pm
@@ -119,12 +119,14 @@ my $tests = [
 my $decompressor = {
     tar => {
 	gz  => ['tar', '-z'],
+	xz => ['tar', '--xz'],
 	lzo => ['tar', '--lzop'],
 	zst => ['tar', '--zstd'],
 	bz2 => ['tar', '--bzip2'],
     },
     vma => {
 	gz  => ['zcat'],
+	xz => ['xzcat', '-q'],
 	lzo => ['lzop', '-d', '-c'],
 	zst => ['zstd', '-q', '-d', '-c'],
 	bz2 => ['bzcat', '-q'],
@@ -167,7 +169,7 @@ for my $virt (sort keys %$bkp_suffix) {
 my $non_bkp_suffix = {
     'openvz' => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
     'lxc'    => [ 'zip', 'tgz.lzo', 'zip.gz', '', ],
-    'qemu'   => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ],
+    'qemu'   => [ 'vms.gz', 'vmx.zst', '', ],
     'none'   => [ 'tar.gz', ],
 };
 
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index 5adba03..588cba1 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -95,6 +95,7 @@ my @tests = (
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_15_00.vma.bz2",
+	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_16_05.vma.xz",
 	    "$storage_dir/snippets/userconfig.yaml",
 	    "$storage_dir/snippets/hookscript.pl",
 	],
@@ -174,6 +175,15 @@ my @tests = (
 		'vmid'    => '16110',
 		'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_15_00.vma.bz2',
 	    },
+	    {
+		'content' => 'backup',
+		'ctime'   => 1585602965,
+		'format'  => 'vma.xz',
+		'size'    => DEFAULT_SIZE,
+		'subtype' => 'qemu',
+		'vmid'    => '16110',
+		'volid'   => 'local:backup/vzdump-qemu-16110-2020_03_30-21_16_05.vma.xz',
+	    },
 	    {
 		'content' => 'snippets',
 		'ctime'   => DEFAULT_CTIME,
@@ -469,7 +479,6 @@ my @tests = (
 	    "$storage_dir/private/subvol-19254-disk-0/19254",
 	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.zip.gz",
 	    "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
-	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
 	    "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
 	],
 	expected => [], # returns empty list
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index 0ede982..8e32945 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -214,8 +214,8 @@ foreach my $s (@$disk_suffix) {
 
 # create more test cases for backup files matches
 my $bkp_suffix = {
-    qemu   => [ 'vma', 'vma.gz', 'vma.lzo', 'vma.zst', 'vma.bz2' ],
-    lxc    => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst', 'tar.bz2' ],
+    qemu   => [ 'vma', 'vma.gz', 'vma.xz', 'vma.lzo', 'vma.zst', 'vma.bz2' ],
+    lxc    => [ 'tar', 'tgz', 'tar.gz', 'tar.xz', 'tar.lzo', 'tar.zst', 'tar.bz2' ],
     openvz => [ 'tar', 'tgz', 'tar.gz', 'tar.lzo', 'tar.zst' ],
 };
 
@@ -245,7 +245,7 @@ foreach my $virt (keys %$bkp_suffix) {
 
 # create more test cases for failed backup files matches
 my $non_bkp_suffix = {
-    qemu   => [ 'vms.gz', 'vma.xz' ],
+    qemu   => [ 'vms.gz' ],
     lxc    => [ 'zip.gz', 'tgz.lzo' ],
 };
 foreach my $virt (keys %$non_bkp_suffix) {
diff --git a/src/test/path_to_volume_id_test.pm b/src/test/path_to_volume_id_test.pm
index 23c5a23..9c7c8c4 100644
--- a/src/test/path_to_volume_id_test.pm
+++ b/src/test/path_to_volume_id_test.pm
@@ -84,6 +84,14 @@ my @tests = (
 	    'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',
 	],
     },
+    {
+	description => 'Backup, vma.xz',
+	volname     => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
+	expected    => [
+	    'backup',
+	    'local:backup/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz',
+	],
+    },
     {
 	description => 'Backup, vma',
 	volname     => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
@@ -238,11 +246,6 @@ my @tests = (
 	volname     => "$storage_dir/dump/vzdump-openvz-16112-2020_03_30-21_39_30.tgz.lzo",
 	expected    => [''],
     },
-    {
-	description => 'Backup, wrong ending, qemu, vma.xz',
-	volname     => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vma.xz",
-	expected    => [''],
-    },
     {
 	description => 'Backup, wrong format, qemu, vms.gz',
 	volname     => "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_40.vms.gz",
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage 03/12] align supported compression formats for templates and backups

[Filip Schauer]

Make the supported compression formats consistent across templates and
backups.

Specifically this enables the use of .tar.lzo container templates. This
is already explicitly supported in PVE::LXC::Create::restore_tar_archive

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/PVE/Storage.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index 0f0fad8..1e0ecc8 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -110,7 +110,7 @@ PVE::Storage::Plugin->init();
 
 our $ISO_EXT_RE_0 = qr/\.(?:iso|img)/i;
 
-our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst|bz2)/i;
+our $VZTMPL_EXT_RE_1 = qr/\.tar\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})/i;
 
 our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/;
 
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH guest-common 04/12] vzdump: add support for bzip2 compression

[Filip Schauer]

This aligns with the PVE::Storage::BACKUP_EXT_RE_2 regex

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/PVE/VZDump/Common.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/PVE/VZDump/Common.pm b/src/PVE/VZDump/Common.pm
index 8fce316..9e13faf 100644
--- a/src/PVE/VZDump/Common.pm
+++ b/src/PVE/VZDump/Common.pm
@@ -158,7 +158,7 @@ my $confdesc = {
 	type => 'string',
 	description => "Compress dump file.",
 	optional => 1,
-	enum => ['0', '1', 'gzip', 'lzo', 'zstd'],
+	enum => ['0', '1', 'gzip', 'lzo', 'zstd', 'bzip2'],
 	default => '0',
     },
     pigz=> {
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH guest-common 05/12] vzdump: add support for xz compression

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/PVE/VZDump/Common.pm | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/PVE/VZDump/Common.pm b/src/PVE/VZDump/Common.pm
index 9e13faf..1e65047 100644
--- a/src/PVE/VZDump/Common.pm
+++ b/src/PVE/VZDump/Common.pm
@@ -158,7 +158,7 @@ my $confdesc = {
 	type => 'string',
 	description => "Compress dump file.",
 	optional => 1,
-	enum => ['0', '1', 'gzip', 'lzo', 'zstd', 'bzip2'],
+	enum => ['0', '1', 'gzip', 'xz', 'lzo', 'zstd', 'bzip2'],
 	default => '0',
     },
     pigz=> {
@@ -168,6 +168,13 @@ my $confdesc = {
 	optional => 1,
 	default => 0,
     },
+    xz => {
+	type => "integer",
+	description => "XZ threads. N=0 uses half of the available cores. N>1 uses N as the thread"
+	    ." count.",
+	optional => 1,
+	default => 0,
+    },
     zstd => {
 	type => "integer",
 	description => "Zstd threads. N=0 uses half of the available cores, if N is set to a value"
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager 06/12] accept .tar.bz2 ct templates in the storage upload dialog

[Filip Schauer]

This aligns with the PVE::Storage::VZTMPL_EXT_RE_1 regex.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 www/manager6/window/UploadToStorage.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/www/manager6/window/UploadToStorage.js b/www/manager6/window/UploadToStorage.js
index f6cad0ec..93dc9993 100644
--- a/www/manager6/window/UploadToStorage.js
+++ b/www/manager6/window/UploadToStorage.js
@@ -11,7 +11,7 @@ Ext.define('PVE.window.UploadToStorage', {
     acceptedExtensions: {
 	'import': ['.ova', '.qcow2', '.raw', '.vmdk'],
 	iso: ['.img', '.iso'],
-	vztmpl: ['.tar.gz', '.tar.xz', '.tar.zst'],
+	vztmpl: ['.tar.gz', '.tar.xz', '.tar.zst', '.tar.bz2'],
     },
 
     // accepted for file selection, will be renamed to real extension
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager 07/12] vzdump: add support for bzip2 compression

[Filip Schauer]

This aligns with the PVE::Storage::BACKUP_EXT_RE_2 regex.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 PVE/VZDump.pm                                  | 2 ++
 www/manager6/form/BackupCompressionSelector.js | 1 +
 2 files changed, 3 insertions(+)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 58fa0f64..06a887a3 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -879,6 +879,8 @@ sub compressor_info {
 	    $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
 	}
 	return ("zstd --threads=${zstd_threads}", 'zst');
+    } elsif ($opt_compress eq 'bzip2') {
+	return ('bzip2', 'bz2');
     } else {
 	die "internal error - unknown compression option '$opt_compress'";
     }
diff --git a/www/manager6/form/BackupCompressionSelector.js b/www/manager6/form/BackupCompressionSelector.js
index 014b8f7e..df2091f3 100644
--- a/www/manager6/form/BackupCompressionSelector.js
+++ b/www/manager6/form/BackupCompressionSelector.js
@@ -6,5 +6,6 @@ Ext.define('PVE.form.BackupCompressionSelector', {
                 ['lzo', 'LZO (' + gettext('fast') + ')'],
                 ['gzip', 'GZIP (' + gettext('good') + ')'],
                 ['zstd', 'ZSTD (' + gettext('fast and good') + ')'],
+                ['bzip2', 'BZIP2 (' + gettext('good') + ')'],
     ],
 });
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager 08/12] vzdump: add support for xz compression

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 PVE/VZDump.pm                                  | 7 +++++++
 www/manager6/form/BackupCompressionSelector.js | 1 +
 2 files changed, 8 insertions(+)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index 06a887a3..0b99f88d 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -872,6 +872,13 @@ sub compressor_info {
 	} else {
 	    return ('gzip --rsyncable', 'gz');
 	}
+    } elsif ($opt_compress eq 'xz') {
+	my $xz_threads = $opts->{xz} // 1;
+	if ($xz_threads == 0) {
+	    my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
+	    $xz_threads = int(($cpuinfo->{cpus} + 1)/2);
+	}
+	return ("xz --threads=$xz_threads", 'xz');
     } elsif ($opt_compress eq 'zstd') {
 	my $zstd_threads = $opts->{zstd} // 1;
 	if ($zstd_threads == 0) {
diff --git a/www/manager6/form/BackupCompressionSelector.js b/www/manager6/form/BackupCompressionSelector.js
index df2091f3..355b7880 100644
--- a/www/manager6/form/BackupCompressionSelector.js
+++ b/www/manager6/form/BackupCompressionSelector.js
@@ -5,6 +5,7 @@ Ext.define('PVE.form.BackupCompressionSelector', {
                 ['0', Proxmox.Utils.noneText],
                 ['lzo', 'LZO (' + gettext('fast') + ')'],
                 ['gzip', 'GZIP (' + gettext('good') + ')'],
+                ['xz', 'XZ (' + gettext('slow but very good') + ')'],
                 ['zstd', 'ZSTD (' + gettext('fast and good') + ')'],
                 ['bzip2', 'BZIP2 (' + gettext('good') + ')'],
     ],
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager 09/12] allow download of xz compressed files

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 www/manager6/window/DownloadUrlToStorage.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
index aabc9d3c..28e9b501 100644
--- a/www/manager6/window/DownloadUrlToStorage.js
+++ b/www/manager6/window/DownloadUrlToStorage.js
@@ -84,7 +84,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 		    let filename = data.filename || "";
 		    let compression = '__default__';
 		    if (view.content === 'iso') {
-			const matches = filename.match(/^(.+)\.(gz|lzo|zst|bz2)$/i);
+			const matches = filename.match(/^(.+)\.(gz|xz|lzo|zst|bz2)$/i);
 			if (matches) {
 			    filename = matches[1];
 			    compression = matches[2].toLowerCase();
@@ -231,6 +231,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 				['__default__', Proxmox.Utils.NoneText],
 				['lzo', 'LZO'],
 				['gz', 'GZIP'],
+				['xz', 'XZ'],
 				['zst', 'ZSTD'],
 				['bz2', 'BZIP2'],
 		    ],
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager 10/12] accept .tar.lzo ct templates in the storage upload dialog

[Filip Schauer]

.tar.lzo container templates are already explicitly supported in
PVE::LXC::Create::restore_tar_archive

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 www/manager6/window/UploadToStorage.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/www/manager6/window/UploadToStorage.js b/www/manager6/window/UploadToStorage.js
index 93dc9993..f21b40b5 100644
--- a/www/manager6/window/UploadToStorage.js
+++ b/www/manager6/window/UploadToStorage.js
@@ -11,7 +11,7 @@ Ext.define('PVE.window.UploadToStorage', {
     acceptedExtensions: {
 	'import': ['.ova', '.qcow2', '.raw', '.vmdk'],
 	iso: ['.img', '.iso'],
-	vztmpl: ['.tar.gz', '.tar.xz', '.tar.zst', '.tar.bz2'],
+	vztmpl: ['.tar.gz', '.tar.xz', '.tar.lzo', '.tar.zst', '.tar.bz2'],
     },
 
     // accepted for file selection, will be renamed to real extension
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH vma-to-pbs 11/12] add support for bzip2 compressed VMA files

[Filip Schauer]

This aligns with the PVE::Storage::BACKUP_EXT_RE_2 regex

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/main.rs    | 3 ++-
 src/vma2pbs.rs | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main.rs b/src/main.rs
index 59bc40b..5d7c3bf 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -254,7 +254,7 @@ fn parse_args() -> Result<BackupVmaToPbsArgs, Error> {
 
     let grouped_vmas = if let Some(dump_dir_path) = bulk {
         let re = Regex::new(
-            r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz)$",
+            r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz|.bz2)$",
         )?;
 
         let mut vmas = Vec::new();
@@ -291,6 +291,7 @@ fn parse_args() -> Result<BackupVmaToPbsArgs, Error> {
                     ".zst" => Some(Compression::Zstd),
                     ".lzo" => Some(Compression::Lzo),
                     ".gz" => Some(Compression::GZip),
+                    ".bz2" => Some(Compression::BZip2),
                     _ => bail!("Unexpected file extension: {ext}"),
                 };
 
diff --git a/src/vma2pbs.rs b/src/vma2pbs.rs
index 1be52b9..a27a07d 100644
--- a/src/vma2pbs.rs
+++ b/src/vma2pbs.rs
@@ -53,6 +53,7 @@ pub enum Compression {
     Zstd,
     Lzo,
     GZip,
+    BZip2,
 }
 
 pub struct VmaBackupArgs {
@@ -533,6 +534,11 @@ fn upload_vma_file(pbs_args: &PbsArgs, backup_args: &VmaBackupArgs) -> Result<()
                     cmd
                 }
                 Compression::GZip => Command::new("zcat"),
+                Compression::BZip2 => {
+                    let mut cmd = Command::new("bzcat");
+                    cmd.args(["-q"]);
+                    cmd
+                }
             };
             let mut process = cmd.arg(vma_file_path).stdout(Stdio::piped()).spawn()?;
             let stdout = process.stdout.take().expect("Failed to capture stdout");
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH vma-to-pbs 12/12] add support for xz compressed VMA files

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 src/main.rs    | 3 ++-
 src/vma2pbs.rs | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main.rs b/src/main.rs
index 5d7c3bf..03dd64f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -254,7 +254,7 @@ fn parse_args() -> Result<BackupVmaToPbsArgs, Error> {
 
     let grouped_vmas = if let Some(dump_dir_path) = bulk {
         let re = Regex::new(
-            r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz|.bz2)$",
+            r"vzdump-qemu-(\d+)-(\d{4}_\d{2}_\d{2}-\d{2}_\d{2}_\d{2}).vma(|.zst|.lzo|.gz|.xz|.bz2)$",
         )?;
 
         let mut vmas = Vec::new();
@@ -291,6 +291,7 @@ fn parse_args() -> Result<BackupVmaToPbsArgs, Error> {
                     ".zst" => Some(Compression::Zstd),
                     ".lzo" => Some(Compression::Lzo),
                     ".gz" => Some(Compression::GZip),
+                    ".xz" => Some(Compression::XZ),
                     ".bz2" => Some(Compression::BZip2),
                     _ => bail!("Unexpected file extension: {ext}"),
                 };
diff --git a/src/vma2pbs.rs b/src/vma2pbs.rs
index a27a07d..d8ccf84 100644
--- a/src/vma2pbs.rs
+++ b/src/vma2pbs.rs
@@ -53,6 +53,7 @@ pub enum Compression {
     Zstd,
     Lzo,
     GZip,
+    XZ,
     BZip2,
 }
 
@@ -534,6 +535,11 @@ fn upload_vma_file(pbs_args: &PbsArgs, backup_args: &VmaBackupArgs) -> Result<()
                     cmd
                 }
                 Compression::GZip => Command::new("zcat"),
+                Compression::XZ => {
+                    let mut cmd = Command::new("xzcat");
+                    cmd.args(["-q"]);
+                    cmd
+                }
                 Compression::BZip2 => {
                     let mut cmd = Command::new("bzcat");
                     cmd.args(["-q"]);
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel