[pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs

[pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs

[Philipp Hufnagl]

Many web pages offer the download of the disk images compressed. This
patch allows the download of archives (like .gz), automatically detects
the format and decompresses it

Changes since v2:
  * move compression code to the download function in common
  * minor code improvements


Changes since v1:
  * Improve code quality as suggested by feedback


Philipp Hufnagl (1):
  fix #4849: download-url: allow download and decompression of
    compressed ISOs

 src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
 src/PVE/Storage.pm             |  6 ++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

Philipp Hufnagl (1):
  fix #4849: download file from url: add opt parameter for a
    decompression command

 src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 21 deletions(-)

Philipp Hufnagl (1):
  fix #4849: download to storage: automatically dectect and configure
    compression

 PVE/API2/Nodes.pm                           | 21 ++++++++++++++++++++-
 www/manager6/Makefile                       |  1 +
 www/manager6/form/DecompressionSelector.js  | 14 ++++++++++++++
 www/manager6/window/DownloadUrlToStorage.js | 21 +++++++++++++++++++--
 4 files changed, 54 insertions(+), 3 deletions(-)
 create mode 100644 www/manager6/form/DecompressionSelector.js
-- 
2.39.2

[pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 38 insertions(+), 21 deletions(-)

diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 9ffac12..ab97129 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -92,23 +92,23 @@ our $EMAIL_USER_RE = qr/[\w\+\-\~]+(\.[\w\+\-\~]+)*/;
 our $EMAIL_RE = qr/$EMAIL_USER_RE@[a-zA-Z0-9\-]+(\.[a-zA-Z0-9\-]+)*/;
 
 use constant {CLONE_NEWNS   => 0x00020000,
-              CLONE_NEWUTS  => 0x04000000,
-              CLONE_NEWIPC  => 0x08000000,
-              CLONE_NEWUSER => 0x10000000,
-              CLONE_NEWPID  => 0x20000000,
-              CLONE_NEWNET  => 0x40000000};
+	      CLONE_NEWUTS  => 0x04000000,
+	      CLONE_NEWIPC  => 0x08000000,
+	      CLONE_NEWUSER => 0x10000000,
+	      CLONE_NEWPID  => 0x20000000,
+	      CLONE_NEWNET  => 0x40000000};
 
 use constant {O_PATH    => 0x00200000,
-              O_CLOEXEC => 0x00080000,
-              O_TMPFILE => 0x00400000 | O_DIRECTORY};
+	      O_CLOEXEC => 0x00080000,
+	      O_TMPFILE => 0x00400000 | O_DIRECTORY};
 
 use constant {AT_EMPTY_PATH => 0x1000,
-              AT_FDCWD => -100};
+	      AT_FDCWD => -100};
 
 # from <linux/fs.h>
 use constant {RENAME_NOREPLACE => (1 << 0),
-              RENAME_EXCHANGE  => (1 << 1),
-              RENAME_WHITEOUT  => (1 << 2)};
+	      RENAME_EXCHANGE  => (1 << 1),
+	      RENAME_WHITEOUT  => (1 << 2)};
 
 sub run_with_timeout {
     my ($timeout, $code, @param) = @_;
@@ -579,7 +579,7 @@ sub run_command {
 	    }
 	}
 
-        alarm(0);
+	alarm(0);
     };
 
     my $err = $@;
@@ -1354,7 +1354,7 @@ sub dump_journal {
     my $parser = sub {
 	my $line = shift;
 
-        return if $count++ < $start;
+	return if $count++ < $start;
 	return if $limit <= 0;
 	push @$lines, { n => int($count), t => $line};
 	$limit--;
@@ -1441,7 +1441,7 @@ sub unpack_sockaddr_in46 {
     my ($sin) = @_;
     my $family = Socket::sockaddr_family($sin);
     my ($port, $host) = ($family == AF_INET6 ? Socket::unpack_sockaddr_in6($sin)
-                                             : Socket::unpack_sockaddr_in($sin));
+					     : Socket::unpack_sockaddr_in($sin));
     return ($family, $port, $host);
 }
 
@@ -1485,8 +1485,8 @@ sub get_fqdn {
 sub parse_host_and_port {
     my ($address) = @_;
     if ($address =~ /^($IPV4RE|[[:alnum:]\-.]+)(?::(\d+))?$/ ||             # ipv4 or host with optional ':port'
-        $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
-        $address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
+	$address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
+	$address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
     {
 	return ($1, $2, 1); # end with 1 to support simple if(parse...) tests
     }
@@ -2013,10 +2013,13 @@ sub download_file_from_url {
 	}
     }
 
-    my $tmpdest = "$dest.tmp.$$";
+    my $tmp_download = "$dest.tmp_dwnl.$$";
+    my $tmp_decomp = "$dest.tmp.dcom.$$";
     eval {
 	local $SIG{INT} = sub {
-	    unlink $tmpdest or warn "could not cleanup temporary file: $!";
+	    unlink $tmp_download or warn "could not cleanup temporary file: $!";
+	    die "got interrupted by signal\n";
+	    unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
 	    die "got interrupted by signal\n";
 	};
 
@@ -2029,7 +2032,7 @@ sub download_file_from_url {
 		$ENV{https_proxy} = $opts->{https_proxy};
 	    }
 
-	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
+	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
 
 	    if (!($opts->{verify_certificates} // 1)) { # default to true
 		push @$cmd, '--no-check-certificate';
@@ -2041,7 +2044,7 @@ sub download_file_from_url {
 	if ($checksum_algorithm) {
 	    print "calculating checksum...";
 
-	    my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
+	    my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
 
 	    if (lc($checksum_got) eq lc($checksum_expected)) {
 		print "OK, checksum verified\n";
@@ -2051,10 +2054,24 @@ sub download_file_from_url {
 	    }
 	}
 
-	rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
+    if (my $cmd = $opts->{decompression_command}) {
+	push @$cmd, $tmp_download;
+
+
+    my $fh;
+    if (!open($fh, ">", "$tmp_decomp")) {
+	die "cant open temporary file $tmp_decomp for decompresson: $!\n";
+    }
+	print "decompressing $tmp_download to $tmp_decomp\n";
+	eval { run_command($cmd, output => '>&'.fileno($fh)); };
+	my $rerr = $@;
+	unlink $tmp_download;
+	die "$rerr\n" if $rerr;
+    }
+	rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
     };
     if (my $err = $@) {
-	unlink $tmpdest or warn "could not cleanup temporary file: $!";
+	unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
 	die $err;
     }
 
-- 
2.39.2

[pve-devel] [V3 PATCH manager 1/1] fix #4849: download to storage: automatically dectect and configure compression

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 PVE/API2/Nodes.pm                           | 21 ++++++++++++++++++++-
 www/manager6/Makefile                       |  1 +
 www/manager6/form/DecompressionSelector.js  | 14 ++++++++++++++
 www/manager6/window/DownloadUrlToStorage.js | 21 +++++++++++++++++++--
 4 files changed, 54 insertions(+), 3 deletions(-)
 create mode 100644 www/manager6/form/DecompressionSelector.js

diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 9269694d..3e9ec034 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -1564,6 +1564,12 @@ __PACKAGE__->register_method({
 		type => 'boolean',
 		optional => 1,
 		default => 1,
+	    },
+	    'detect-compression' => {
+		description => "If true an auto detect of used compression will be attempted",
+		type => 'boolean',
+		optional => 1,
+		default => 0,
 	    }
 	},
     },
@@ -1583,6 +1589,11 @@ __PACKAGE__->register_method({
 		type => 'string',
 		optional => 1,
 	    },
+	    compression => {
+		type => 'string',
+                enum => $PVE::Storage::Status::KNOWN_COMPRESSION_FORMATS,
+		optional => 1,
+	    },
 	},
     },
     code => sub {
@@ -1606,6 +1617,8 @@ __PACKAGE__->register_method({
 	    );
 	}
 
+        my $detect_compression = $param->{'detect-compression'};
+
 	my $req = HTTP::Request->new(HEAD => $url);
 	my $res = $ua->request($req);
 
@@ -1614,7 +1627,7 @@ __PACKAGE__->register_method({
 	my $size = $res->header("Content-Length");
 	my $disposition = $res->header("Content-Disposition");
 	my $type = $res->header("Content-Type");
-
+        my $compression;
 	my $filename;
 
 	if ($disposition && ($disposition =~ m/filename="([^"]*)"/ || $disposition =~ m/filename=([^;]*)/)) {
@@ -1628,10 +1641,16 @@ __PACKAGE__->register_method({
 	    $type = $1;
 	}
 
+        if ($detect_compression && $filename =~ m!^((.+)\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))$!) {
+            $filename = $2;
+            $compression = $3;
+        }
+
 	my $ret = {};
 	$ret->{filename} = $filename if $filename;
 	$ret->{size} = $size + 0 if $size;
 	$ret->{mimetype} = $type if $type;
+	$ret->{compression} = $compression if $compression;
 
 	return $ret;
     }});
diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index 7ec9d7a5..42a27548 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -34,6 +34,7 @@ JSSRC= 							\
 	form/ContentTypeSelector.js			\
 	form/ControllerSelector.js			\
 	form/DayOfWeekSelector.js			\
+	form/DecompressionSelector.js       \
 	form/DiskFormatSelector.js			\
 	form/DiskStorageSelector.js			\
 	form/EmailNotificationSelector.js		\
diff --git a/www/manager6/form/DecompressionSelector.js b/www/manager6/form/DecompressionSelector.js
new file mode 100644
index 00000000..87c0ae95
--- /dev/null
+++ b/www/manager6/form/DecompressionSelector.js
@@ -0,0 +1,14 @@
+
+Ext.define('PVE.form.DecompressionSelector', {
+    extend: 'Proxmox.form.KVComboBox',
+    alias: ['widget.pveDecompressionSelector'],
+    config: {
+	deleteEmpty: false,
+    },
+    comboItems: [
+                ['__default__', Proxmox.Utils.noneText],
+                ['lzo', 'LZO'],
+                ['gz', 'GZIP'],
+                ['zst', 'ZSTD'],
+    ],
+});
diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
index 48543d28..559a1c05 100644
--- a/www/manager6/window/DownloadUrlToStorage.js
+++ b/www/manager6/window/DownloadUrlToStorage.js
@@ -49,6 +49,9 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 	    vm.set('size', '-');
 	    vm.set('mimetype', '-');
 	},
+	decompressionPossible: function() {
+	    return this.view.content === 'iso';
+	},
 
 	urlCheck: function(field) {
 	    let me = this;
@@ -66,6 +69,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 		params: {
 		    url: queryParam.url,
 		    'verify-certificates': queryParam['verify-certificates'],
+		    'detect-compression': me.decompressionPossible() ? 1 : 0,
 		},
 		waitMsgTarget: view,
 		failure: res => {
@@ -84,6 +88,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 			filename: data.filename || "",
 			size: (data.size && Proxmox.Utils.format_size(data.size)) || gettext("Unknown"),
 			mimetype: data.mimetype || gettext("Unknown"),
+			compression: data.compression || '__default__',
 		    });
 		},
 	    });
@@ -215,7 +220,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
     ],
 
     initComponent: function() {
-        var me = this;
+	var me = this;
 
 	if (!me.nodename) {
 	    throw "no node name specified";
@@ -223,8 +228,20 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 	if (!me.storage) {
 	    throw "no storage ID specified";
 	}
+	if (me.content === 'iso') {
+	    me.items[0].advancedColumn2.push(
+
+		{
+		    xtype: 'pveDecompressionSelector',
+		    name: 'compression',
+		    fieldLabel: gettext('Decompression algorithm'),
+		    allowBlank: true,
+		    hasNoneOption: true,
+		    value: '__default__',
+		});
+	}
 
-        me.callParent();
+	me.callParent();
     },
 });
 
-- 
2.39.2

[pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
 src/PVE/Storage.pm             |  6 ++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
index 2aaeff6..1d73c96 100644
--- a/src/PVE/API2/Storage/Status.pm
+++ b/src/PVE/API2/Storage/Status.pm
@@ -23,6 +23,8 @@ use PVE::Storage;
 
 use base qw(PVE::RESTHandler);
 
+our $KNOWN_COMPRESSION_FORMATS = ['zst', 'gz', 'lzo'];
+
 __PACKAGE__->register_method ({
     subclass => "PVE::API2::Storage::PruneBackups",
     path => '{storage}/prunebackups',
@@ -578,6 +580,12 @@ __PACKAGE__->register_method({
 		requires => 'checksum-algorithm',
 		optional => 1,
 	    },
+	    compression => {
+		description => "Decompress the downloaded file using specified compression algorithm",
+		type => 'string',
+		enum => $KNOWN_COMPRESSION_FORMATS,
+		optional => 1,
+	    },
 	    'checksum-algorithm' => {
 		description => "The algorithm to calculate the checksum of the file.",
 		type => 'string',
@@ -604,7 +612,7 @@ __PACKAGE__->register_method({
 
 	my $cfg = PVE::Storage::config();
 
-	my ($node, $storage) = $param->@{'node', 'storage'};
+	my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
 	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 
 	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
@@ -642,13 +650,18 @@ __PACKAGE__->register_method({
 	    http_proxy => $dccfg->{http_proxy},
 	};
 
-	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
+	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm' };
 	if ($checksum) {
 	    $opts->{"${checksum_algorithm}sum"} = $checksum;
 	    $opts->{hash_required} = 1;
 	}
 
 	my $worker = sub {
+	    if ($compression) {
+		die "decompression not supported for $content\n" if $content ne 'iso';
+		my $info = PVE::Storage::decompressor_info('iso', $compression);
+		$opts->{decompression_command} = $info->{decompressor};
+	    }
 	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 	};
 
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index a4d85e1..cb70113 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1531,6 +1531,12 @@ sub decompressor_info {
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
 	},
+	iso => {
+	    # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
+	    gz => ['zcat'],
+	    lzo => ['lzop', '-d', '-c'],
+	    zst => ['zstd', '-q', '-d', '-c'],
+	},
     };
 
     die "ERROR: archive format not defined\n"
-- 
2.39.2

Re: [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command

[Fabian Grünbichler]

On July 31, 2023 10:39 am, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
>  1 file changed, 38 insertions(+), 21 deletions(-)
> 
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index 9ffac12..ab97129 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm

[snipped lots of unrelated changes here!]

> @@ -2013,10 +2013,13 @@ sub download_file_from_url {
>  	}
>      }
>  
> -    my $tmpdest = "$dest.tmp.$$";
> +    my $tmp_download = "$dest.tmp_dwnl.$$";
> +    my $tmp_decomp = "$dest.tmp.dcom.$$";

nit: the naming scheme should be unified ;)

>      eval {
>  	local $SIG{INT} = sub {
> -	    unlink $tmpdest or warn "could not cleanup temporary file: $!";
> +	    unlink $tmp_download or warn "could not cleanup temporary file: $!";
> +	    die "got interrupted by signal\n";

this die here

> +	    unlink $tmp_decomp or warn "could not cleanup temporary file: $!";

means this part never gets to run. also, this part should be guarded by
an if, since for the uncompressed case, $tmp_decomp never gets created
in the first place (see below).

>  	    die "got interrupted by signal\n";
>  	};
>  
> @@ -2029,7 +2032,7 @@ sub download_file_from_url {
>  		$ENV{https_proxy} = $opts->{https_proxy};
>  	    }
>  
> -	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
> +	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
>  
>  	    if (!($opts->{verify_certificates} // 1)) { # default to true
>  		push @$cmd, '--no-check-certificate';
> @@ -2041,7 +2044,7 @@ sub download_file_from_url {
>  	if ($checksum_algorithm) {
>  	    print "calculating checksum...";
>  
> -	    my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
> +	    my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
>  
>  	    if (lc($checksum_got) eq lc($checksum_expected)) {
>  		print "OK, checksum verified\n";
> @@ -2051,10 +2054,24 @@ sub download_file_from_url {
>  	    }
>  	}
>  
> -	rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
> +    if (my $cmd = $opts->{decompression_command}) {

nit: indentation (starting) here is wrong

> +	push @$cmd, $tmp_download;
> +
> +

nit: extra blank line?

> +    my $fh;
> +    if (!open($fh, ">", "$tmp_decomp")) {


> +	die "cant open temporary file $tmp_decomp for decompresson: $!\n";
> +    }
> +	print "decompressing $tmp_download to $tmp_decomp\n";
> +	eval { run_command($cmd, output => '>&'.fileno($fh)); };
> +	my $rerr = $@;

nit: $rerr should just be $err

> +	unlink $tmp_download;
> +	die "$rerr\n" if $rerr;
> +    }
> +	rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";

this needs to be properly handling the no-decompression case - did you
test that? because for me it's broken:

$ pveam download ext4 rockylinux-9-default_20221109_amd64.tar.xz

downloading http://download.proxmox.com/images/system/rockylinux-9-default_20221109_amd64.tar.xz to /mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz
--2023-07-31 15:29:34--  http://download.proxmox.com/images/system/rockylinux-9-default_20221109_amd64.tar.xz
Resolving download.proxmox.com (download.proxmox.com)... 212.224.123.70, 2a01:7e0:0:424::249
Connecting to download.proxmox.com (download.proxmox.com)|212.224.123.70|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 102704656 (98M) [application/octet-stream]
Saving to: '/mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888'
     0K ........ ........ ........ ........ 32% 54.8M 1s
 32768K ........ ........ ........ ........ 65% 45.3M 1s
 65536K ........ ........ ........ ........ 98% 45.1M 0s
 98304K .                                  100% 46.8M=2.0s
2023-07-31 15:29:36 (47.9 MB/s) - '/mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888' saved [102704656/102704656]
calculating checksum...OK, checksum verified
could not cleanup temporary file: No such file or directory at /usr/share/perl5/PVE/Tools.pm line 2073.
unable to rename temporary file: No such file or directory


>      };
>      if (my $err = $@) {
> -	unlink $tmpdest or warn "could not cleanup temporary file: $!";
> +	unlink $tmp_decomp or warn "could not cleanup temporary file: $!";

same here - this should cleanup $tmp_download if uncompressed, or
$tmp_decomp if decompression is enabled..

also, pre-existing (see output above), missing newline in error message.

after the above 'pveam' invocation:

$ ls -lh /mnt/pve/ext4/template/cache
total 98M
-rw-r--r-- 1 root root  98M Nov  9  2022 rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888

>  	die $err;
>      }
>  
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

Re: [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs

[Fabian Grünbichler]

On July 31, 2023 10:39 am, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
>  src/PVE/Storage.pm             |  6 ++++++
>  2 files changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
> index 2aaeff6..1d73c96 100644
> --- a/src/PVE/API2/Storage/Status.pm
> +++ b/src/PVE/API2/Storage/Status.pm
> @@ -23,6 +23,8 @@ use PVE::Storage;
>  
>  use base qw(PVE::RESTHandler);
> 
> +our $KNOWN_COMPRESSION_FORMATS = ['zst', 'gz', 'lzo'];

might want a comment here that adding new formats means handling them in
decompressor_info below, and vice-versa?

also, this is just another way to write PVE::Storage::Plugin::COMPRESSOR_RE

maybe they could be unified to avoid accidentally desyncing them?
technically, the RE could be used as 'pattern' in the API schema, but
the generated docs are less human-friendly for patterns compared to
enums..

> +
>  __PACKAGE__->register_method ({
>      subclass => "PVE::API2::Storage::PruneBackups",
>      path => '{storage}/prunebackups',
> @@ -578,6 +580,12 @@ __PACKAGE__->register_method({
>  		requires => 'checksum-algorithm',
>  		optional => 1,
>  	    },
> +	    compression => {
> +		description => "Decompress the downloaded file using specified compression algorithm",
> +		type => 'string',
> +		enum => $KNOWN_COMPRESSION_FORMATS,
> +		optional => 1,
> +	    },
>  	    'checksum-algorithm' => {
>  		description => "The algorithm to calculate the checksum of the file.",
>  		type => 'string',
> @@ -604,7 +612,7 @@ __PACKAGE__->register_method({
>  
>  	my $cfg = PVE::Storage::config();
>  
> -	my ($node, $storage) = $param->@{'node', 'storage'};
> +	my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
>  	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
>  
>  	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
> @@ -642,13 +650,18 @@ __PACKAGE__->register_method({
>  	    http_proxy => $dccfg->{http_proxy},
>  	};
>  
> -	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
> +	my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm' };
>  	if ($checksum) {
>  	    $opts->{"${checksum_algorithm}sum"} = $checksum;
>  	    $opts->{hash_required} = 1;
>  	}
>  
>  	my $worker = sub {
> +	    if ($compression) {
> +		die "decompression not supported for $content\n" if $content ne 'iso';
> +		my $info = PVE::Storage::decompressor_info('iso', $compression);
> +		$opts->{decompression_command} = $info->{decompressor};

note: $info->{decompressor} might be undef, should probably be caught
here, else decompression will be silently skipped

> +	    }
>  	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
>  	};
>  
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index a4d85e1..cb70113 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -1531,6 +1531,12 @@ sub decompressor_info {
>  	    lzo => ['lzop', '-d', '-c'],
>  	    zst => ['zstd', '-q', '-d', '-c'],
>  	},
> +	iso => {
> +	    # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
> +	    gz => ['zcat'],
> +	    lzo => ['lzop', '-d', '-c'],
> +	    zst => ['zstd', '-q', '-d', '-c'],
> +	},
>      };
>  
>      die "ERROR: archive format not defined\n"
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
>