[pve-devel] [PATCH storage/manager/common v4 0/6] fix #4849: allow download of compressed ISOs

[pve-devel] [PATCH storage/manager/common v4 0/6] fix #4849: allow download of compressed ISOs

[Philipp Hufnagl]

Changes since v3:
  * generate compression regex from compression list
  * fix logic errors

Changes since v2:
  * move compression code to the download function in common
  * minor code improvements


Changes since v1:
  * Improve code quality as suggested by feedback


Philipp Hufnagl (2):
  fix #4849: download-url: allow download and decompression of
    compressed ISOs
  fix whitespaces

 src/PVE/API2/Storage/Status.pm | 14 +++++++++++++-
 src/PVE/Storage.pm             |  6 ++++++
 src/PVE/Storage/Plugin.pm      | 25 +++++++++++++------------
 3 files changed, 32 insertions(+), 13 deletions(-)

Philipp Hufnagl (2):
  fix #4849: download file from url: add opt parameter for a
    decompression command
  fix whitespaces

 src/PVE/Tools.pm | 61 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 21 deletions(-)

Philipp Hufnagl (2):
  fix #4849: download to storage: automatically dectect and configure
    compression
  fix whitespaces

 PVE/API2/Nodes.pm                           | 37 ++++++++++++++++-----
 www/manager6/Makefile                       |  1 +
 www/manager6/form/DecompressionSelector.js  | 13 ++++++++
 www/manager6/window/DownloadUrlToStorage.js | 21 ++++++++++--
 4 files changed, 61 insertions(+), 11 deletions(-)
 create mode 100644 www/manager6/form/DecompressionSelector.js

-- 
2.39.2

[pve-devel] [PATCH storage v4 1/2] fix #4849: download-url: allow download and decompression of compressed ISOs

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/API2/Storage/Status.pm | 14 +++++++++++++-
 src/PVE/Storage.pm             |  6 ++++++
 src/PVE/Storage/Plugin.pm      |  3 ++-
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
index 2aaeff6..7875530 100644
--- a/src/PVE/API2/Storage/Status.pm
+++ b/src/PVE/API2/Storage/Status.pm
@@ -578,6 +578,12 @@ __PACKAGE__->register_method({
 		requires => 'checksum-algorithm',
 		optional => 1,
 	    },
+	    compression => {
+		description => "Decompress the downloaded file using specified compression algorithm",
+		type => 'string',
+		enum => $PVE::Storage::Plugin::KNOWN_COMPRESSION_FORMATS,
+		optional => 1,
+	    },
 	    'checksum-algorithm' => {
 		description => "The algorithm to calculate the checksum of the file.",
 		type => 'string',
@@ -604,7 +610,7 @@ __PACKAGE__->register_method({
 
 	my $cfg = PVE::Storage::config();
 
-	my ($node, $storage) = $param->@{'node', 'storage'};
+	my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
 	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
 
 	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
@@ -649,6 +655,12 @@ __PACKAGE__->register_method({
 	}
 
 	my $worker = sub {
+	    if ($compression) {
+		die "decompression not supported for $content\n" if $content ne 'iso';
+		my $info = PVE::Storage::decompressor_info('iso', $compression);
+		die "no decompression method found\n" if (! $info->{decompressor});
+		$opts->{decompression_command} = $info->{decompressor};
+	    }
 	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
 	};
 
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index a4d85e1..cb70113 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1531,6 +1531,12 @@ sub decompressor_info {
 	    lzo => ['lzop', '-d', '-c'],
 	    zst => ['zstd', '-q', '-d', '-c'],
 	},
+	iso => {
+	    # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
+	    gz => ['zcat'],
+	    lzo => ['lzop', '-d', '-c'],
+	    zst => ['zstd', '-q', '-d', '-c'],
+	},
     };
 
     die "ERROR: archive format not defined\n"
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 9d3b1ae..18cb5d5 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -19,7 +19,8 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant COMPRESSOR_RE => 'gz|lzo|zst';
+use constant KNOWN_COMPRESSION_FORMATS =>  ( 'gz', 'lzo', 'zst');
+use constant COMPRESSOR_RE => join( '|', KNOWN_COMPRESSION_FORMATS);
 
 use constant LOG_EXT => ".log";
 use constant NOTES_EXT => ".notes";
-- 
2.39.2

[pve-devel] [PATCH storage v4 2/2] fix whitespaces

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/Storage/Plugin.pm | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 18cb5d5..1795ae3 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -264,7 +264,7 @@ sub verify_server {
     my ($server, $noerr) = @_;
 
     if (!(PVE::JSONSchema::pve_verify_ip($server, 1) ||
-          PVE::JSONSchema::pve_verify_dns_name($server, 1)))
+	  PVE::JSONSchema::pve_verify_dns_name($server, 1)))
     {
 	return undef if $noerr;
 	die "value does not look like a valid server name or IP address\n";
@@ -472,7 +472,7 @@ sub encode_value {
     my ($class, $type, $key, $value) = @_;
 
     if ($key eq 'nodes') {
-        return join(',', keys(%$value));
+	return join(',', keys(%$value));
     } elsif ($key eq 'content') {
 	my $res = content_hash_to_string($value) || 'none';
 	return $res;
@@ -1153,9 +1153,9 @@ sub volume_has_feature {
 
     my $key = undef;
     if($snapname){
-        $key = 'snap';
+	$key = 'snap';
     }else{
-        $key =  $isBase ? 'base' : 'current';
+	$key =  $isBase ? 'base' : 'current';
     }
 
     return 1 if defined($features->{$feature}->{$key}->{$format});
@@ -1199,14 +1199,14 @@ sub list_images {
 	    next if !$found;
 	}
 
-        my $info = {
+	my $info = {
 	    volid => $volid, format => $format,
 	    size => $size, vmid => $owner, used => $used, parent => $parent
 	};
 
-        $info->{ctime} = $ctime if $ctime;
+	$info->{ctime} = $ctime if $ctime;
 
-        push @$res, $info;
+	push @$res, $info;
     }
 
     return $res;
@@ -1580,7 +1580,7 @@ sub volume_export {
 		run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
 	    } else {
 		run_command(['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
-		            output => '>&'.fileno($fh));
+			    output => '>&'.fileno($fh));
 	    }
 	    return;
 	} elsif ($format =~ /^(qcow2|vmdk)\+size$/) {
@@ -1593,7 +1593,7 @@ sub volume_export {
 	    goto unsupported if $file_format ne 'subvol';
 	    write_common_header($fh, $size);
 	    run_command(['tar', @COMMON_TAR_FLAGS, '-cf', '-', '-C', $file, '.'],
-	                output => '>&'.fileno($fh));
+			output => '>&'.fileno($fh));
 	    return;
 	}
     }
@@ -1662,10 +1662,10 @@ sub volume_import {
 	    or die "internal error: failed to get path to newly allocated volume $volname\n";
 	if ($data_format eq 'raw' || $data_format eq 'qcow2' || $data_format eq 'vmdk') {
 	    run_command(['dd', "of=$file", 'conv=sparse', 'bs=64k'],
-	                input => '<&'.fileno($fh));
+			input => '<&'.fileno($fh));
 	} elsif ($data_format eq 'tar') {
 	    run_command(['tar', @COMMON_TAR_FLAGS, '-C', $file, '-xf', '-'],
-	                input => '<&'.fileno($fh));
+			input => '<&'.fileno($fh));
 	} else {
 	    die "volume import format '$format' not available for $class";
 	}
-- 
2.39.2

[pve-devel] [PATCH common v4 1/2] fix #4849: download file from url: add opt parameter for a decompression command

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/Tools.pm | 31 +++++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 9ffac12..159ec82 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -2013,10 +2013,13 @@ sub download_file_from_url {
 	}
     }
 
-    my $tmpdest = "$dest.tmp.$$";
+    my $tmp_download = "$dest.tmp_dwnl.$$";
+    my $tmp_decomp = "$dest.tmp_dcom.$$";
     eval {
 	local $SIG{INT} = sub {
-	    unlink $tmpdest or warn "could not cleanup temporary file: $!";
+	    unlink $tmp_download or warn "could not cleanup temporary file: $!";
+	    unlink $tmp_decomp or warn "could not cleanup temporary file: $!"
+		if $opts->{decompression_command};
 	    die "got interrupted by signal\n";
 	};
 
@@ -2029,7 +2032,7 @@ sub download_file_from_url {
 		$ENV{https_proxy} = $opts->{https_proxy};
 	    }
 
-	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
+	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
 
 	    if (!($opts->{verify_certificates} // 1)) { # default to true
 		push @$cmd, '--no-check-certificate';
@@ -2041,7 +2044,7 @@ sub download_file_from_url {
 	if ($checksum_algorithm) {
 	    print "calculating checksum...";
 
-	    my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
+	    my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
 
 	    if (lc($checksum_got) eq lc($checksum_expected)) {
 		print "OK, checksum verified\n";
@@ -2051,10 +2054,26 @@ sub download_file_from_url {
 	    }
 	}
 
-	rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
+    if (my $cmd = $opts->{decompression_command}) {
+	push @$cmd, $tmp_download;
+    my $fh;
+    if (!open($fh, ">", "$tmp_decomp")) {
+	die "cant open temporary file $tmp_decomp for decompresson: $!\n";
+    }
+	print "decompressing $tmp_download to $tmp_decomp\n";
+	eval { run_command($cmd, output => '>&'.fileno($fh)); };
+	my $err = $@;
+	unlink $tmp_download;
+	die "$err\n" if $err;
+	rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
+    } else {
+	rename($tmp_download, $dest) or die "unable to rename temporary file: $!\n";
+    }
     };
     if (my $err = $@) {
-	unlink $tmpdest or warn "could not cleanup temporary file: $!";
+	unlink $tmp_download or warn "could not cleanup temporary file: $!";
+	unlink $tmp_decomp or warn "could not cleanup temporary file: $!"
+	    if $opts->{decompression_command};
 	die $err;
     }
 
-- 
2.39.2

[pve-devel] [PATCH common v4 2/2] fix whitespaces

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 src/PVE/Tools.pm | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 159ec82..28e70f0 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -92,23 +92,23 @@ our $EMAIL_USER_RE = qr/[\w\+\-\~]+(\.[\w\+\-\~]+)*/;
 our $EMAIL_RE = qr/$EMAIL_USER_RE@[a-zA-Z0-9\-]+(\.[a-zA-Z0-9\-]+)*/;
 
 use constant {CLONE_NEWNS   => 0x00020000,
-              CLONE_NEWUTS  => 0x04000000,
-              CLONE_NEWIPC  => 0x08000000,
-              CLONE_NEWUSER => 0x10000000,
-              CLONE_NEWPID  => 0x20000000,
-              CLONE_NEWNET  => 0x40000000};
+	    CLONE_NEWUTS  => 0x04000000,
+	    CLONE_NEWIPC  => 0x08000000,
+	    CLONE_NEWUSER => 0x10000000,
+	    CLONE_NEWPID  => 0x20000000,
+	    CLONE_NEWNET  => 0x40000000};
 
 use constant {O_PATH    => 0x00200000,
-              O_CLOEXEC => 0x00080000,
-              O_TMPFILE => 0x00400000 | O_DIRECTORY};
+	    O_CLOEXEC => 0x00080000,
+	    O_TMPFILE => 0x00400000 | O_DIRECTORY};
 
 use constant {AT_EMPTY_PATH => 0x1000,
-              AT_FDCWD => -100};
+	    AT_FDCWD => -100};
 
 # from <linux/fs.h>
 use constant {RENAME_NOREPLACE => (1 << 0),
-              RENAME_EXCHANGE  => (1 << 1),
-              RENAME_WHITEOUT  => (1 << 2)};
+	    RENAME_EXCHANGE  => (1 << 1),
+	    RENAME_WHITEOUT  => (1 << 2)};
 
 sub run_with_timeout {
     my ($timeout, $code, @param) = @_;
@@ -579,7 +579,7 @@ sub run_command {
 	    }
 	}
 
-        alarm(0);
+	alarm(0);
     };
 
     my $err = $@;
@@ -1354,7 +1354,7 @@ sub dump_journal {
     my $parser = sub {
 	my $line = shift;
 
-        return if $count++ < $start;
+	return if $count++ < $start;
 	return if $limit <= 0;
 	push @$lines, { n => int($count), t => $line};
 	$limit--;
@@ -1441,7 +1441,7 @@ sub unpack_sockaddr_in46 {
     my ($sin) = @_;
     my $family = Socket::sockaddr_family($sin);
     my ($port, $host) = ($family == AF_INET6 ? Socket::unpack_sockaddr_in6($sin)
-                                             : Socket::unpack_sockaddr_in($sin));
+					    : Socket::unpack_sockaddr_in($sin));
     return ($family, $port, $host);
 }
 
@@ -1485,8 +1485,8 @@ sub get_fqdn {
 sub parse_host_and_port {
     my ($address) = @_;
     if ($address =~ /^($IPV4RE|[[:alnum:]\-.]+)(?::(\d+))?$/ ||             # ipv4 or host with optional ':port'
-        $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
-        $address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
+	$address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
+	$address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
     {
 	return ($1, $2, 1); # end with 1 to support simple if(parse...) tests
     }
-- 
2.39.2

[pve-devel] [PATCH manager v4 1/2] fix #4849: download to storage: automatically dectect and configure compression

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 PVE/API2/Nodes.pm                           | 21 ++++++++++++++++++++-
 www/manager6/Makefile                       |  1 +
 www/manager6/form/DecompressionSelector.js  | 13 +++++++++++++
 www/manager6/window/DownloadUrlToStorage.js | 17 +++++++++++++++++
 4 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 www/manager6/form/DecompressionSelector.js

diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 9269694d..2bae4e6f 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -1564,6 +1564,12 @@ __PACKAGE__->register_method({
 		type => 'boolean',
 		optional => 1,
 		default => 1,
+	    },
+	    'detect-compression' => {
+		description => "If true an auto detect of used compression will be attempted",
+		type => 'boolean',
+		optional => 1,
+		default => 0,
 	    }
 	},
     },
@@ -1583,6 +1589,11 @@ __PACKAGE__->register_method({
 		type => 'string',
 		optional => 1,
 	    },
+	    compression => {
+		type => 'string',
+		enum => $PVE::Storage::Plugin::KNOWN_COMPRESSION_FORMATS,
+		optional => 1,
+	    },
 	},
     },
     code => sub {
@@ -1606,6 +1617,8 @@ __PACKAGE__->register_method({
 	    );
 	}
 
+	my $detect_compression = $param->{'detect-compression'};
+
 	my $req = HTTP::Request->new(HEAD => $url);
 	my $res = $ua->request($req);
 
@@ -1614,7 +1627,7 @@ __PACKAGE__->register_method({
 	my $size = $res->header("Content-Length");
 	my $disposition = $res->header("Content-Disposition");
 	my $type = $res->header("Content-Type");
-
+	my $compression;
 	my $filename;
 
 	if ($disposition && ($disposition =~ m/filename="([^"]*)"/ || $disposition =~ m/filename=([^;]*)/)) {
@@ -1628,10 +1641,16 @@ __PACKAGE__->register_method({
 	    $type = $1;
 	}
 
+	if ($detect_compression && $filename =~ m!^((.+)\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))$!) {
+	    $filename = $2;
+	    $compression = $3;
+	}
+
 	my $ret = {};
 	$ret->{filename} = $filename if $filename;
 	$ret->{size} = $size + 0 if $size;
 	$ret->{mimetype} = $type if $type;
+	$ret->{compression} = $compression if $compression;
 
 	return $ret;
     }});
diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index 7ec9d7a5..42a27548 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -34,6 +34,7 @@ JSSRC= 							\
 	form/ContentTypeSelector.js			\
 	form/ControllerSelector.js			\
 	form/DayOfWeekSelector.js			\
+	form/DecompressionSelector.js       \
 	form/DiskFormatSelector.js			\
 	form/DiskStorageSelector.js			\
 	form/EmailNotificationSelector.js		\
diff --git a/www/manager6/form/DecompressionSelector.js b/www/manager6/form/DecompressionSelector.js
new file mode 100644
index 00000000..b85e050c
--- /dev/null
+++ b/www/manager6/form/DecompressionSelector.js
@@ -0,0 +1,13 @@
+Ext.define('PVE.form.DecompressionSelector', {
+    extend: 'Proxmox.form.KVComboBox',
+    alias: ['widget.pveDecompressionSelector'],
+    config: {
+	deleteEmpty: false,
+    },
+    comboItems: [
+		['__default__', Proxmox.Utils.noneText],
+		['lzo', 'LZO'],
+		['gz', 'GZIP'],
+		['zst', 'ZSTD'],
+    ],
+});
diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
index 48543d28..7a472ce9 100644
--- a/www/manager6/window/DownloadUrlToStorage.js
+++ b/www/manager6/window/DownloadUrlToStorage.js
@@ -49,6 +49,9 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 	    vm.set('size', '-');
 	    vm.set('mimetype', '-');
 	},
+	decompressionPossible: function() {
+	    return this.view.content === 'iso';
+	},
 
 	urlCheck: function(field) {
 	    let me = this;
@@ -66,6 +69,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 		params: {
 		    url: queryParam.url,
 		    'verify-certificates': queryParam['verify-certificates'],
+		    'detect-compression': me.decompressionPossible() ? 1 : 0,
 		},
 		waitMsgTarget: view,
 		failure: res => {
@@ -84,6 +88,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 			filename: data.filename || "",
 			size: (data.size && Proxmox.Utils.format_size(data.size)) || gettext("Unknown"),
 			mimetype: data.mimetype || gettext("Unknown"),
+			compression: data.compression || '__default__',
 		    });
 		},
 	    });
@@ -223,6 +228,18 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 	if (!me.storage) {
 	    throw "no storage ID specified";
 	}
+	if (me.content === 'iso') {
+	    me.items[0].advancedColumn2.push(
+
+		{
+		    xtype: 'pveDecompressionSelector',
+		    name: 'compression',
+		    fieldLabel: gettext('Decompression algorithm'),
+		    allowBlank: true,
+		    hasNoneOption: true,
+		    value: '__default__',
+		});
+	}
 
         me.callParent();
     },
-- 
2.39.2

[pve-devel] [PATCH manager v4 2/2] fix whitespaces

[Philipp Hufnagl]

Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
 PVE/API2/Nodes.pm                           | 16 ++++++++--------
 www/manager6/window/DownloadUrlToStorage.js |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 2bae4e6f..6fa138fb 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -496,7 +496,7 @@ __PACKAGE__->register_method({
 	# just parse the json again, it should already be validated
 	my $commands = eval { decode_json($param->{commands}); };
 
-        foreach my $cmd (@$commands) {
+	foreach my $cmd (@$commands) {
 	    eval {
 		$cmd->{args} //= {};
 
@@ -654,11 +654,11 @@ __PACKAGE__->register_method({
 	    },
 	    ds => {
 		description => "The list of datasources you want to display.",
- 		type => 'string', format => 'pve-configid-list',
+		type => 'string', format => 'pve-configid-list',
 	    },
 	    cf => {
 		description => "The RRD consolidation function",
- 		type => 'string',
+		type => 'string',
 		enum => [ 'AVERAGE', 'MAX' ],
 		optional => 1,
 	    },
@@ -699,7 +699,7 @@ __PACKAGE__->register_method({
 	    },
 	    cf => {
 		description => "The RRD consolidation function",
- 		type => 'string',
+		type => 'string',
 		enum => [ 'AVERAGE', 'MAX' ],
 		optional => 1,
 	    },
@@ -1368,7 +1368,7 @@ __PACKAGE__->register_method({
     description => "Read server time and time zone settings.",
     proxyto => 'node',
     parameters => {
-    	additionalProperties => 0,
+	additionalProperties => 0,
 	properties => {
 	    node => get_standard_option('pve-node'),
 	},
@@ -1393,7 +1393,7 @@ __PACKAGE__->register_method({
 		minimum => 1297163644,
 		renderer => 'timestamp_gmt',
 	    },
-        },
+	},
     },
     code => sub {
 	my ($param) = @_;
@@ -2105,14 +2105,14 @@ __PACKAGE__->register_method ({
 	additionalProperties => 0,
 	properties => {
 	    node => get_standard_option('pve-node'),
-            target => get_standard_option('pve-node', { description => "Target node." }),
+	    target => get_standard_option('pve-node', { description => "Target node." }),
 	    maxworkers => {
 		description => "Maximal number of parallel migration job. If not set, uses"
 		    ."'max_workers' from datacenter.cfg. One of both must be set!",
 		optional => 1,
 		type => 'integer',
 		minimum => 1
-            },
+	    },
 	    vms => {
 		description => "Only consider Guests with these IDs.",
 		type => 'string',  format => 'pve-vmid-list',
diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
index 7a472ce9..559a1c05 100644
--- a/www/manager6/window/DownloadUrlToStorage.js
+++ b/www/manager6/window/DownloadUrlToStorage.js
@@ -220,7 +220,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
     ],
 
     initComponent: function() {
-        var me = this;
+	var me = this;
 
 	if (!me.nodename) {
 	    throw "no node name specified";
@@ -241,7 +241,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
 		});
 	}
 
-        me.callParent();
+	me.callParent();
     },
 });
 
-- 
2.39.2

[pve-devel] applied: [PATCH common v4 1/2] fix #4849: download file from url: add opt parameter for a decompression command

[Fabian Grünbichler]

with a few follow-ups.

On August 1, 2023 4:46 pm, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/Tools.pm | 31 +++++++++++++++++++++++++------
>  1 file changed, 25 insertions(+), 6 deletions(-)
> 
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index 9ffac12..159ec82 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm
> @@ -2013,10 +2013,13 @@ sub download_file_from_url {
>  	}
>      }
>  
> -    my $tmpdest = "$dest.tmp.$$";
> +    my $tmp_download = "$dest.tmp_dwnl.$$";
> +    my $tmp_decomp = "$dest.tmp_dcom.$$";
>      eval {
>  	local $SIG{INT} = sub {
> -	    unlink $tmpdest or warn "could not cleanup temporary file: $!";
> +	    unlink $tmp_download or warn "could not cleanup temporary file: $!";
> +	    unlink $tmp_decomp or warn "could not cleanup temporary file: $!"
> +		if $opts->{decompression_command};
>  	    die "got interrupted by signal\n";
>  	};
>  
> @@ -2029,7 +2032,7 @@ sub download_file_from_url {
>  		$ENV{https_proxy} = $opts->{https_proxy};
>  	    }
>  
> -	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
> +	    my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
>  
>  	    if (!($opts->{verify_certificates} // 1)) { # default to true
>  		push @$cmd, '--no-check-certificate';
> @@ -2041,7 +2044,7 @@ sub download_file_from_url {
>  	if ($checksum_algorithm) {
>  	    print "calculating checksum...";
>  
> -	    my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
> +	    my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
>  
>  	    if (lc($checksum_got) eq lc($checksum_expected)) {
>  		print "OK, checksum verified\n";
> @@ -2051,10 +2054,26 @@ sub download_file_from_url {
>  	    }
>  	}
>  
> -	rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
> +    if (my $cmd = $opts->{decompression_command}) {
> +	push @$cmd, $tmp_download;
> +    my $fh;
> +    if (!open($fh, ">", "$tmp_decomp")) {
> +	die "cant open temporary file $tmp_decomp for decompresson: $!\n";
> +    }
> +	print "decompressing $tmp_download to $tmp_decomp\n";
> +	eval { run_command($cmd, output => '>&'.fileno($fh)); };
> +	my $err = $@;
> +	unlink $tmp_download;
> +	die "$err\n" if $err;
> +	rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
> +    } else {
> +	rename($tmp_download, $dest) or die "unable to rename temporary file: $!\n";
> +    }
>      };
>      if (my $err = $@) {
> -	unlink $tmpdest or warn "could not cleanup temporary file: $!";
> +	unlink $tmp_download or warn "could not cleanup temporary file: $!";
> +	unlink $tmp_decomp or warn "could not cleanup temporary file: $!"
> +	    if $opts->{decompression_command};
>  	die $err;
>      }
>  
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

[pve-devel] applied: [PATCH common v4 2/2] fix whitespaces

[Fabian Grünbichler]

the indentation part, skipping the alignment hunks as discussed
off-list.

On August 1, 2023 4:46 pm, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/Tools.pm | 30 +++++++++++++++---------------
>  1 file changed, 15 insertions(+), 15 deletions(-)
> 
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index 159ec82..28e70f0 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm
> @@ -92,23 +92,23 @@ our $EMAIL_USER_RE = qr/[\w\+\-\~]+(\.[\w\+\-\~]+)*/;
>  our $EMAIL_RE = qr/$EMAIL_USER_RE@[a-zA-Z0-9\-]+(\.[a-zA-Z0-9\-]+)*/;
>  
>  use constant {CLONE_NEWNS   => 0x00020000,
> -              CLONE_NEWUTS  => 0x04000000,
> -              CLONE_NEWIPC  => 0x08000000,
> -              CLONE_NEWUSER => 0x10000000,
> -              CLONE_NEWPID  => 0x20000000,
> -              CLONE_NEWNET  => 0x40000000};
> +	    CLONE_NEWUTS  => 0x04000000,
> +	    CLONE_NEWIPC  => 0x08000000,
> +	    CLONE_NEWUSER => 0x10000000,
> +	    CLONE_NEWPID  => 0x20000000,
> +	    CLONE_NEWNET  => 0x40000000};
>  
>  use constant {O_PATH    => 0x00200000,
> -              O_CLOEXEC => 0x00080000,
> -              O_TMPFILE => 0x00400000 | O_DIRECTORY};
> +	    O_CLOEXEC => 0x00080000,
> +	    O_TMPFILE => 0x00400000 | O_DIRECTORY};
>  
>  use constant {AT_EMPTY_PATH => 0x1000,
> -              AT_FDCWD => -100};
> +	    AT_FDCWD => -100};
>  
>  # from <linux/fs.h>
>  use constant {RENAME_NOREPLACE => (1 << 0),
> -              RENAME_EXCHANGE  => (1 << 1),
> -              RENAME_WHITEOUT  => (1 << 2)};
> +	    RENAME_EXCHANGE  => (1 << 1),
> +	    RENAME_WHITEOUT  => (1 << 2)};
>  
>  sub run_with_timeout {
>      my ($timeout, $code, @param) = @_;
> @@ -579,7 +579,7 @@ sub run_command {
>  	    }
>  	}
>  
> -        alarm(0);
> +	alarm(0);
>      };
>  
>      my $err = $@;
> @@ -1354,7 +1354,7 @@ sub dump_journal {
>      my $parser = sub {
>  	my $line = shift;
>  
> -        return if $count++ < $start;
> +	return if $count++ < $start;
>  	return if $limit <= 0;
>  	push @$lines, { n => int($count), t => $line};
>  	$limit--;
> @@ -1441,7 +1441,7 @@ sub unpack_sockaddr_in46 {
>      my ($sin) = @_;
>      my $family = Socket::sockaddr_family($sin);
>      my ($port, $host) = ($family == AF_INET6 ? Socket::unpack_sockaddr_in6($sin)
> -                                             : Socket::unpack_sockaddr_in($sin));
> +					    : Socket::unpack_sockaddr_in($sin));
>      return ($family, $port, $host);
>  }
>  
> @@ -1485,8 +1485,8 @@ sub get_fqdn {
>  sub parse_host_and_port {
>      my ($address) = @_;
>      if ($address =~ /^($IPV4RE|[[:alnum:]\-.]+)(?::(\d+))?$/ ||             # ipv4 or host with optional ':port'
> -        $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
> -        $address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
> +	$address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
> +	$address =~ /^($IPV6RE)(?:\.(\d+))?$/)                              # ipv6 with optional port separated by dot
>      {
>  	return ($1, $2, 1); # end with 1 to support simple if(parse...) tests
>      }
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

[pve-devel] applied: [PATCH storage v4 2/2] fix whitespaces

[Fabian Grünbichler]

again, limited to the non-alignmnet changes

On August 1, 2023 4:46 pm, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/Storage/Plugin.pm | 22 +++++++++++-----------
>  1 file changed, 11 insertions(+), 11 deletions(-)
> 
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 18cb5d5..1795ae3 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -264,7 +264,7 @@ sub verify_server {
>      my ($server, $noerr) = @_;
>  
>      if (!(PVE::JSONSchema::pve_verify_ip($server, 1) ||
> -          PVE::JSONSchema::pve_verify_dns_name($server, 1)))
> +	  PVE::JSONSchema::pve_verify_dns_name($server, 1)))
>      {
>  	return undef if $noerr;
>  	die "value does not look like a valid server name or IP address\n";
> @@ -472,7 +472,7 @@ sub encode_value {
>      my ($class, $type, $key, $value) = @_;
>  
>      if ($key eq 'nodes') {
> -        return join(',', keys(%$value));
> +	return join(',', keys(%$value));
>      } elsif ($key eq 'content') {
>  	my $res = content_hash_to_string($value) || 'none';
>  	return $res;
> @@ -1153,9 +1153,9 @@ sub volume_has_feature {
>  
>      my $key = undef;
>      if($snapname){
> -        $key = 'snap';
> +	$key = 'snap';
>      }else{
> -        $key =  $isBase ? 'base' : 'current';
> +	$key =  $isBase ? 'base' : 'current';
>      }
>  
>      return 1 if defined($features->{$feature}->{$key}->{$format});
> @@ -1199,14 +1199,14 @@ sub list_images {
>  	    next if !$found;
>  	}
>  
> -        my $info = {
> +	my $info = {
>  	    volid => $volid, format => $format,
>  	    size => $size, vmid => $owner, used => $used, parent => $parent
>  	};
>  
> -        $info->{ctime} = $ctime if $ctime;
> +	$info->{ctime} = $ctime if $ctime;
>  
> -        push @$res, $info;
> +	push @$res, $info;
>      }
>  
>      return $res;
> @@ -1580,7 +1580,7 @@ sub volume_export {
>  		run_command(['dd', "if=$file", "bs=4k"], output => '>&'.fileno($fh));
>  	    } else {
>  		run_command(['qemu-img', 'convert', '-f', $file_format, '-O', 'raw', $file, '/dev/stdout'],
> -		            output => '>&'.fileno($fh));
> +			    output => '>&'.fileno($fh));
>  	    }
>  	    return;
>  	} elsif ($format =~ /^(qcow2|vmdk)\+size$/) {
> @@ -1593,7 +1593,7 @@ sub volume_export {
>  	    goto unsupported if $file_format ne 'subvol';
>  	    write_common_header($fh, $size);
>  	    run_command(['tar', @COMMON_TAR_FLAGS, '-cf', '-', '-C', $file, '.'],
> -	                output => '>&'.fileno($fh));
> +			output => '>&'.fileno($fh));
>  	    return;
>  	}
>      }
> @@ -1662,10 +1662,10 @@ sub volume_import {
>  	    or die "internal error: failed to get path to newly allocated volume $volname\n";
>  	if ($data_format eq 'raw' || $data_format eq 'qcow2' || $data_format eq 'vmdk') {
>  	    run_command(['dd', "of=$file", 'conv=sparse', 'bs=64k'],
> -	                input => '<&'.fileno($fh));
> +			input => '<&'.fileno($fh));
>  	} elsif ($data_format eq 'tar') {
>  	    run_command(['tar', @COMMON_TAR_FLAGS, '-C', $file, '-xf', '-'],
> -	                input => '<&'.fileno($fh));
> +			input => '<&'.fileno($fh));
>  	} else {
>  	    die "volume import format '$format' not available for $class";
>  	}
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

[pve-devel] applied: [PATCH manager v4 2/2] fix whitespaces

[Fabian Grünbichler]

On August 1, 2023 4:46 pm, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  PVE/API2/Nodes.pm                           | 16 ++++++++--------
>  www/manager6/window/DownloadUrlToStorage.js |  4 ++--
>  2 files changed, 10 insertions(+), 10 deletions(-)
> 
> diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
> index 2bae4e6f..6fa138fb 100644
> --- a/PVE/API2/Nodes.pm
> +++ b/PVE/API2/Nodes.pm
> @@ -496,7 +496,7 @@ __PACKAGE__->register_method({
>  	# just parse the json again, it should already be validated
>  	my $commands = eval { decode_json($param->{commands}); };
>  
> -        foreach my $cmd (@$commands) {
> +	foreach my $cmd (@$commands) {
>  	    eval {
>  		$cmd->{args} //= {};
>  
> @@ -654,11 +654,11 @@ __PACKAGE__->register_method({
>  	    },
>  	    ds => {
>  		description => "The list of datasources you want to display.",
> - 		type => 'string', format => 'pve-configid-list',
> +		type => 'string', format => 'pve-configid-list',
>  	    },
>  	    cf => {
>  		description => "The RRD consolidation function",
> - 		type => 'string',
> +		type => 'string',
>  		enum => [ 'AVERAGE', 'MAX' ],
>  		optional => 1,
>  	    },
> @@ -699,7 +699,7 @@ __PACKAGE__->register_method({
>  	    },
>  	    cf => {
>  		description => "The RRD consolidation function",
> - 		type => 'string',
> +		type => 'string',
>  		enum => [ 'AVERAGE', 'MAX' ],
>  		optional => 1,
>  	    },
> @@ -1368,7 +1368,7 @@ __PACKAGE__->register_method({
>      description => "Read server time and time zone settings.",
>      proxyto => 'node',
>      parameters => {
> -    	additionalProperties => 0,
> +	additionalProperties => 0,
>  	properties => {
>  	    node => get_standard_option('pve-node'),
>  	},
> @@ -1393,7 +1393,7 @@ __PACKAGE__->register_method({
>  		minimum => 1297163644,
>  		renderer => 'timestamp_gmt',
>  	    },
> -        },
> +	},
>      },
>      code => sub {
>  	my ($param) = @_;
> @@ -2105,14 +2105,14 @@ __PACKAGE__->register_method ({
>  	additionalProperties => 0,
>  	properties => {
>  	    node => get_standard_option('pve-node'),
> -            target => get_standard_option('pve-node', { description => "Target node." }),
> +	    target => get_standard_option('pve-node', { description => "Target node." }),
>  	    maxworkers => {
>  		description => "Maximal number of parallel migration job. If not set, uses"
>  		    ."'max_workers' from datacenter.cfg. One of both must be set!",
>  		optional => 1,
>  		type => 'integer',
>  		minimum => 1
> -            },
> +	    },
>  	    vms => {
>  		description => "Only consider Guests with these IDs.",
>  		type => 'string',  format => 'pve-vmid-list',
> diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
> index 7a472ce9..559a1c05 100644
> --- a/www/manager6/window/DownloadUrlToStorage.js
> +++ b/www/manager6/window/DownloadUrlToStorage.js
> @@ -220,7 +220,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>      ],
>  
>      initComponent: function() {
> -        var me = this;
> +	var me = this;
>  
>  	if (!me.nodename) {
>  	    throw "no node name specified";
> @@ -241,7 +241,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>  		});
>  	}
>  
> -        me.callParent();
> +	me.callParent();
>      },
>  });
>  
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

Re: [pve-devel] [PATCH storage v4 1/2] fix #4849: download-url: allow download and decompression of compressed ISOs

[Fabian Grünbichler]

On August 1, 2023 4:45 pm, Philipp Hufnagl wrote:

a bit of description here would be nice..

that being said,

Reviewed-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
Tested-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>

can be applied with a corresponding bumped dep on libpve-common-perl
after that one has been released!

> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  src/PVE/API2/Storage/Status.pm | 14 +++++++++++++-
>  src/PVE/Storage.pm             |  6 ++++++
>  src/PVE/Storage/Plugin.pm      |  3 ++-
>  3 files changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
> index 2aaeff6..7875530 100644
> --- a/src/PVE/API2/Storage/Status.pm
> +++ b/src/PVE/API2/Storage/Status.pm
> @@ -578,6 +578,12 @@ __PACKAGE__->register_method({
>  		requires => 'checksum-algorithm',
>  		optional => 1,
>  	    },
> +	    compression => {
> +		description => "Decompress the downloaded file using specified compression algorithm",
> +		type => 'string',
> +		enum => $PVE::Storage::Plugin::KNOWN_COMPRESSION_FORMATS,
> +		optional => 1,
> +	    },
>  	    'checksum-algorithm' => {
>  		description => "The algorithm to calculate the checksum of the file.",
>  		type => 'string',
> @@ -604,7 +610,7 @@ __PACKAGE__->register_method({
>  
>  	my $cfg = PVE::Storage::config();
>  
> -	my ($node, $storage) = $param->@{'node', 'storage'};
> +	my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
>  	my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
>  
>  	die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
> @@ -649,6 +655,12 @@ __PACKAGE__->register_method({
>  	}
>  
>  	my $worker = sub {
> +	    if ($compression) {
> +		die "decompression not supported for $content\n" if $content ne 'iso';
> +		my $info = PVE::Storage::decompressor_info('iso', $compression);
> +		die "no decompression method found\n" if (! $info->{decompressor});
> +		$opts->{decompression_command} = $info->{decompressor};
> +	    }
>  	    PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
>  	};
>  
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index a4d85e1..cb70113 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -1531,6 +1531,12 @@ sub decompressor_info {
>  	    lzo => ['lzop', '-d', '-c'],
>  	    zst => ['zstd', '-q', '-d', '-c'],
>  	},
> +	iso => {
> +	    # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
> +	    gz => ['zcat'],
> +	    lzo => ['lzop', '-d', '-c'],
> +	    zst => ['zstd', '-q', '-d', '-c'],
> +	},
>      };
>  
>      die "ERROR: archive format not defined\n"
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 9d3b1ae..18cb5d5 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -19,7 +19,8 @@ use JSON;
>  
>  use base qw(PVE::SectionConfig);
>  
> -use constant COMPRESSOR_RE => 'gz|lzo|zst';
> +use constant KNOWN_COMPRESSION_FORMATS =>  ( 'gz', 'lzo', 'zst');
> +use constant COMPRESSOR_RE => join( '|', KNOWN_COMPRESSION_FORMATS);
>  
>  use constant LOG_EXT => ".log";
>  use constant NOTES_EXT => ".notes";
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
> 

Re: [pve-devel] [PATCH manager v4 1/2] fix #4849: download to storage: automatically dectect and configure compression

[Fabian Grünbichler]

On August 1, 2023 4:46 pm, Philipp Hufnagl wrote:

same here - please add a commit message!

with one nit below that could be folded in together with the commit
message, after pve-common and pve-storage have been bumped, since this
one requires their changes and a corresponding bump in d/control:

Reviewed-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
Tested-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>

> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
>  PVE/API2/Nodes.pm                           | 21 ++++++++++++++++++++-
>  www/manager6/Makefile                       |  1 +
>  www/manager6/form/DecompressionSelector.js  | 13 +++++++++++++
>  www/manager6/window/DownloadUrlToStorage.js | 17 +++++++++++++++++
>  4 files changed, 51 insertions(+), 1 deletion(-)
>  create mode 100644 www/manager6/form/DecompressionSelector.js
> 
> diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
> index 9269694d..2bae4e6f 100644
> --- a/PVE/API2/Nodes.pm
> +++ b/PVE/API2/Nodes.pm
> @@ -1564,6 +1564,12 @@ __PACKAGE__->register_method({
>  		type => 'boolean',
>  		optional => 1,
>  		default => 1,
> +	    },
> +	    'detect-compression' => {
> +		description => "If true an auto detect of used compression will be attempted",
> +		type => 'boolean',
> +		optional => 1,
> +		default => 0,
>  	    }
>  	},
>      },
> @@ -1583,6 +1589,11 @@ __PACKAGE__->register_method({
>  		type => 'string',
>  		optional => 1,
>  	    },
> +	    compression => {
> +		type => 'string',
> +		enum => $PVE::Storage::Plugin::KNOWN_COMPRESSION_FORMATS,
> +		optional => 1,
> +	    },
>  	},
>      },
>      code => sub {
> @@ -1606,6 +1617,8 @@ __PACKAGE__->register_method({
>  	    );
>  	}
>  
> +	my $detect_compression = $param->{'detect-compression'};
> +
>  	my $req = HTTP::Request->new(HEAD => $url);
>  	my $res = $ua->request($req);
>  
> @@ -1614,7 +1627,7 @@ __PACKAGE__->register_method({
>  	my $size = $res->header("Content-Length");
>  	my $disposition = $res->header("Content-Disposition");
>  	my $type = $res->header("Content-Type");
> -
> +	my $compression;
>  	my $filename;
>  
>  	if ($disposition && ($disposition =~ m/filename="([^"]*)"/ || $disposition =~ m/filename=([^;]*)/)) {
> @@ -1628,10 +1641,16 @@ __PACKAGE__->register_method({
>  	    $type = $1;
>  	}
>  
> +	if ($detect_compression && $filename =~ m!^((.+)\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))$!) {
> +	    $filename = $2;
> +	    $compression = $3;
> +	}
> +
>  	my $ret = {};
>  	$ret->{filename} = $filename if $filename;
>  	$ret->{size} = $size + 0 if $size;
>  	$ret->{mimetype} = $type if $type;
> +	$ret->{compression} = $compression if $compression;
>  
>  	return $ret;
>      }});
> diff --git a/www/manager6/Makefile b/www/manager6/Makefile
> index 7ec9d7a5..42a27548 100644
> --- a/www/manager6/Makefile
> +++ b/www/manager6/Makefile
> @@ -34,6 +34,7 @@ JSSRC= 							\
>  	form/ContentTypeSelector.js			\
>  	form/ControllerSelector.js			\
>  	form/DayOfWeekSelector.js			\
> +	form/DecompressionSelector.js       \
>  	form/DiskFormatSelector.js			\
>  	form/DiskStorageSelector.js			\
>  	form/EmailNotificationSelector.js		\
> diff --git a/www/manager6/form/DecompressionSelector.js b/www/manager6/form/DecompressionSelector.js
> new file mode 100644
> index 00000000..b85e050c
> --- /dev/null
> +++ b/www/manager6/form/DecompressionSelector.js
> @@ -0,0 +1,13 @@
> +Ext.define('PVE.form.DecompressionSelector', {
> +    extend: 'Proxmox.form.KVComboBox',
> +    alias: ['widget.pveDecompressionSelector'],
> +    config: {
> +	deleteEmpty: false,
> +    },
> +    comboItems: [
> +		['__default__', Proxmox.Utils.noneText],

nit: this should be NoneText , not noneText, to get the right
capitalization.

> +		['lzo', 'LZO'],
> +		['gz', 'GZIP'],
> +		['zst', 'ZSTD'],
> +    ],
> +});
> diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
> index 48543d28..7a472ce9 100644
> --- a/www/manager6/window/DownloadUrlToStorage.js
> +++ b/www/manager6/window/DownloadUrlToStorage.js
> @@ -49,6 +49,9 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>  	    vm.set('size', '-');
>  	    vm.set('mimetype', '-');
>  	},
> +	decompressionPossible: function() {
> +	    return this.view.content === 'iso';
> +	},
>  
>  	urlCheck: function(field) {
>  	    let me = this;
> @@ -66,6 +69,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>  		params: {
>  		    url: queryParam.url,
>  		    'verify-certificates': queryParam['verify-certificates'],
> +		    'detect-compression': me.decompressionPossible() ? 1 : 0,
>  		},
>  		waitMsgTarget: view,
>  		failure: res => {
> @@ -84,6 +88,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>  			filename: data.filename || "",
>  			size: (data.size && Proxmox.Utils.format_size(data.size)) || gettext("Unknown"),
>  			mimetype: data.mimetype || gettext("Unknown"),
> +			compression: data.compression || '__default__',
>  		    });
>  		},
>  	    });
> @@ -223,6 +228,18 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
>  	if (!me.storage) {
>  	    throw "no storage ID specified";
>  	}
> +	if (me.content === 'iso') {
> +	    me.items[0].advancedColumn2.push(
> +
> +		{
> +		    xtype: 'pveDecompressionSelector',
> +		    name: 'compression',
> +		    fieldLabel: gettext('Decompression algorithm'),
> +		    allowBlank: true,
> +		    hasNoneOption: true,
> +		    value: '__default__',
> +		});
> +	}
>  
>          me.callParent();
>      },
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
> 
> 
>