* [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command
2023-07-31 8:39 [pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs Philipp Hufnagl
@ 2023-07-31 8:39 ` Philipp Hufnagl
2023-07-31 13:42 ` Fabian Grünbichler
2023-07-31 8:39 ` [pve-devel] [V3 PATCH manager 1/1] fix #4849: download to storage: automatically dectect and configure compression Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs Philipp Hufnagl
2 siblings, 1 reply; 6+ messages in thread
From: Philipp Hufnagl @ 2023-07-31 8:39 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
1 file changed, 38 insertions(+), 21 deletions(-)
diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 9ffac12..ab97129 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -92,23 +92,23 @@ our $EMAIL_USER_RE = qr/[\w\+\-\~]+(\.[\w\+\-\~]+)*/;
our $EMAIL_RE = qr/$EMAIL_USER_RE@[a-zA-Z0-9\-]+(\.[a-zA-Z0-9\-]+)*/;
use constant {CLONE_NEWNS => 0x00020000,
- CLONE_NEWUTS => 0x04000000,
- CLONE_NEWIPC => 0x08000000,
- CLONE_NEWUSER => 0x10000000,
- CLONE_NEWPID => 0x20000000,
- CLONE_NEWNET => 0x40000000};
+ CLONE_NEWUTS => 0x04000000,
+ CLONE_NEWIPC => 0x08000000,
+ CLONE_NEWUSER => 0x10000000,
+ CLONE_NEWPID => 0x20000000,
+ CLONE_NEWNET => 0x40000000};
use constant {O_PATH => 0x00200000,
- O_CLOEXEC => 0x00080000,
- O_TMPFILE => 0x00400000 | O_DIRECTORY};
+ O_CLOEXEC => 0x00080000,
+ O_TMPFILE => 0x00400000 | O_DIRECTORY};
use constant {AT_EMPTY_PATH => 0x1000,
- AT_FDCWD => -100};
+ AT_FDCWD => -100};
# from <linux/fs.h>
use constant {RENAME_NOREPLACE => (1 << 0),
- RENAME_EXCHANGE => (1 << 1),
- RENAME_WHITEOUT => (1 << 2)};
+ RENAME_EXCHANGE => (1 << 1),
+ RENAME_WHITEOUT => (1 << 2)};
sub run_with_timeout {
my ($timeout, $code, @param) = @_;
@@ -579,7 +579,7 @@ sub run_command {
}
}
- alarm(0);
+ alarm(0);
};
my $err = $@;
@@ -1354,7 +1354,7 @@ sub dump_journal {
my $parser = sub {
my $line = shift;
- return if $count++ < $start;
+ return if $count++ < $start;
return if $limit <= 0;
push @$lines, { n => int($count), t => $line};
$limit--;
@@ -1441,7 +1441,7 @@ sub unpack_sockaddr_in46 {
my ($sin) = @_;
my $family = Socket::sockaddr_family($sin);
my ($port, $host) = ($family == AF_INET6 ? Socket::unpack_sockaddr_in6($sin)
- : Socket::unpack_sockaddr_in($sin));
+ : Socket::unpack_sockaddr_in($sin));
return ($family, $port, $host);
}
@@ -1485,8 +1485,8 @@ sub get_fqdn {
sub parse_host_and_port {
my ($address) = @_;
if ($address =~ /^($IPV4RE|[[:alnum:]\-.]+)(?::(\d+))?$/ || # ipv4 or host with optional ':port'
- $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
- $address =~ /^($IPV6RE)(?:\.(\d+))?$/) # ipv6 with optional port separated by dot
+ $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
+ $address =~ /^($IPV6RE)(?:\.(\d+))?$/) # ipv6 with optional port separated by dot
{
return ($1, $2, 1); # end with 1 to support simple if(parse...) tests
}
@@ -2013,10 +2013,13 @@ sub download_file_from_url {
}
}
- my $tmpdest = "$dest.tmp.$$";
+ my $tmp_download = "$dest.tmp_dwnl.$$";
+ my $tmp_decomp = "$dest.tmp.dcom.$$";
eval {
local $SIG{INT} = sub {
- unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ unlink $tmp_download or warn "could not cleanup temporary file: $!";
+ die "got interrupted by signal\n";
+ unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
die "got interrupted by signal\n";
};
@@ -2029,7 +2032,7 @@ sub download_file_from_url {
$ENV{https_proxy} = $opts->{https_proxy};
}
- my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
+ my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
if (!($opts->{verify_certificates} // 1)) { # default to true
push @$cmd, '--no-check-certificate';
@@ -2041,7 +2044,7 @@ sub download_file_from_url {
if ($checksum_algorithm) {
print "calculating checksum...";
- my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
+ my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
if (lc($checksum_got) eq lc($checksum_expected)) {
print "OK, checksum verified\n";
@@ -2051,10 +2054,24 @@ sub download_file_from_url {
}
}
- rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
+ if (my $cmd = $opts->{decompression_command}) {
+ push @$cmd, $tmp_download;
+
+
+ my $fh;
+ if (!open($fh, ">", "$tmp_decomp")) {
+ die "cant open temporary file $tmp_decomp for decompresson: $!\n";
+ }
+ print "decompressing $tmp_download to $tmp_decomp\n";
+ eval { run_command($cmd, output => '>&'.fileno($fh)); };
+ my $rerr = $@;
+ unlink $tmp_download;
+ die "$rerr\n" if $rerr;
+ }
+ rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
};
if (my $err = $@) {
- unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
die $err;
}
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command
2023-07-31 8:39 ` [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command Philipp Hufnagl
@ 2023-07-31 13:42 ` Fabian Grünbichler
0 siblings, 0 replies; 6+ messages in thread
From: Fabian Grünbichler @ 2023-07-31 13:42 UTC (permalink / raw)
To: Proxmox VE development discussion
On July 31, 2023 10:39 am, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
> src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
> 1 file changed, 38 insertions(+), 21 deletions(-)
>
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index 9ffac12..ab97129 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm
[snipped lots of unrelated changes here!]
> @@ -2013,10 +2013,13 @@ sub download_file_from_url {
> }
> }
>
> - my $tmpdest = "$dest.tmp.$$";
> + my $tmp_download = "$dest.tmp_dwnl.$$";
> + my $tmp_decomp = "$dest.tmp.dcom.$$";
nit: the naming scheme should be unified ;)
> eval {
> local $SIG{INT} = sub {
> - unlink $tmpdest or warn "could not cleanup temporary file: $!";
> + unlink $tmp_download or warn "could not cleanup temporary file: $!";
> + die "got interrupted by signal\n";
this die here
> + unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
means this part never gets to run. also, this part should be guarded by
an if, since for the uncompressed case, $tmp_decomp never gets created
in the first place (see below).
> die "got interrupted by signal\n";
> };
>
> @@ -2029,7 +2032,7 @@ sub download_file_from_url {
> $ENV{https_proxy} = $opts->{https_proxy};
> }
>
> - my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
> + my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
>
> if (!($opts->{verify_certificates} // 1)) { # default to true
> push @$cmd, '--no-check-certificate';
> @@ -2041,7 +2044,7 @@ sub download_file_from_url {
> if ($checksum_algorithm) {
> print "calculating checksum...";
>
> - my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
> + my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
>
> if (lc($checksum_got) eq lc($checksum_expected)) {
> print "OK, checksum verified\n";
> @@ -2051,10 +2054,24 @@ sub download_file_from_url {
> }
> }
>
> - rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
> + if (my $cmd = $opts->{decompression_command}) {
nit: indentation (starting) here is wrong
> + push @$cmd, $tmp_download;
> +
> +
nit: extra blank line?
> + my $fh;
> + if (!open($fh, ">", "$tmp_decomp")) {
> + die "cant open temporary file $tmp_decomp for decompresson: $!\n";
> + }
> + print "decompressing $tmp_download to $tmp_decomp\n";
> + eval { run_command($cmd, output => '>&'.fileno($fh)); };
> + my $rerr = $@;
nit: $rerr should just be $err
> + unlink $tmp_download;
> + die "$rerr\n" if $rerr;
> + }
> + rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
this needs to be properly handling the no-decompression case - did you
test that? because for me it's broken:
$ pveam download ext4 rockylinux-9-default_20221109_amd64.tar.xz
downloading http://download.proxmox.com/images/system/rockylinux-9-default_20221109_amd64.tar.xz to /mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz
--2023-07-31 15:29:34-- http://download.proxmox.com/images/system/rockylinux-9-default_20221109_amd64.tar.xz
Resolving download.proxmox.com (download.proxmox.com)... 212.224.123.70, 2a01:7e0:0:424::249
Connecting to download.proxmox.com (download.proxmox.com)|212.224.123.70|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 102704656 (98M) [application/octet-stream]
Saving to: '/mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888'
0K ........ ........ ........ ........ 32% 54.8M 1s
32768K ........ ........ ........ ........ 65% 45.3M 1s
65536K ........ ........ ........ ........ 98% 45.1M 0s
98304K . 100% 46.8M=2.0s
2023-07-31 15:29:36 (47.9 MB/s) - '/mnt/pve/ext4/template/cache/rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888' saved [102704656/102704656]
calculating checksum...OK, checksum verified
could not cleanup temporary file: No such file or directory at /usr/share/perl5/PVE/Tools.pm line 2073.
unable to rename temporary file: No such file or directory
> };
> if (my $err = $@) {
> - unlink $tmpdest or warn "could not cleanup temporary file: $!";
> + unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
same here - this should cleanup $tmp_download if uncompressed, or
$tmp_decomp if decompression is enabled..
also, pre-existing (see output above), missing newline in error message.
after the above 'pveam' invocation:
$ ls -lh /mnt/pve/ext4/template/cache
total 98M
-rw-r--r-- 1 root root 98M Nov 9 2022 rockylinux-9-default_20221109_amd64.tar.xz.tmp_dwnl.665888
> die $err;
> }
>
> --
> 2.39.2
>
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [V3 PATCH manager 1/1] fix #4849: download to storage: automatically dectect and configure compression
2023-07-31 8:39 [pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command Philipp Hufnagl
@ 2023-07-31 8:39 ` Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs Philipp Hufnagl
2 siblings, 0 replies; 6+ messages in thread
From: Philipp Hufnagl @ 2023-07-31 8:39 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
PVE/API2/Nodes.pm | 21 ++++++++++++++++++++-
www/manager6/Makefile | 1 +
www/manager6/form/DecompressionSelector.js | 14 ++++++++++++++
www/manager6/window/DownloadUrlToStorage.js | 21 +++++++++++++++++++--
4 files changed, 54 insertions(+), 3 deletions(-)
create mode 100644 www/manager6/form/DecompressionSelector.js
diff --git a/PVE/API2/Nodes.pm b/PVE/API2/Nodes.pm
index 9269694d..3e9ec034 100644
--- a/PVE/API2/Nodes.pm
+++ b/PVE/API2/Nodes.pm
@@ -1564,6 +1564,12 @@ __PACKAGE__->register_method({
type => 'boolean',
optional => 1,
default => 1,
+ },
+ 'detect-compression' => {
+ description => "If true an auto detect of used compression will be attempted",
+ type => 'boolean',
+ optional => 1,
+ default => 0,
}
},
},
@@ -1583,6 +1589,11 @@ __PACKAGE__->register_method({
type => 'string',
optional => 1,
},
+ compression => {
+ type => 'string',
+ enum => $PVE::Storage::Status::KNOWN_COMPRESSION_FORMATS,
+ optional => 1,
+ },
},
},
code => sub {
@@ -1606,6 +1617,8 @@ __PACKAGE__->register_method({
);
}
+ my $detect_compression = $param->{'detect-compression'};
+
my $req = HTTP::Request->new(HEAD => $url);
my $res = $ua->request($req);
@@ -1614,7 +1627,7 @@ __PACKAGE__->register_method({
my $size = $res->header("Content-Length");
my $disposition = $res->header("Content-Disposition");
my $type = $res->header("Content-Type");
-
+ my $compression;
my $filename;
if ($disposition && ($disposition =~ m/filename="([^"]*)"/ || $disposition =~ m/filename=([^;]*)/)) {
@@ -1628,10 +1641,16 @@ __PACKAGE__->register_method({
$type = $1;
}
+ if ($detect_compression && $filename =~ m!^((.+)\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))$!) {
+ $filename = $2;
+ $compression = $3;
+ }
+
my $ret = {};
$ret->{filename} = $filename if $filename;
$ret->{size} = $size + 0 if $size;
$ret->{mimetype} = $type if $type;
+ $ret->{compression} = $compression if $compression;
return $ret;
}});
diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index 7ec9d7a5..42a27548 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -34,6 +34,7 @@ JSSRC= \
form/ContentTypeSelector.js \
form/ControllerSelector.js \
form/DayOfWeekSelector.js \
+ form/DecompressionSelector.js \
form/DiskFormatSelector.js \
form/DiskStorageSelector.js \
form/EmailNotificationSelector.js \
diff --git a/www/manager6/form/DecompressionSelector.js b/www/manager6/form/DecompressionSelector.js
new file mode 100644
index 00000000..87c0ae95
--- /dev/null
+++ b/www/manager6/form/DecompressionSelector.js
@@ -0,0 +1,14 @@
+
+Ext.define('PVE.form.DecompressionSelector', {
+ extend: 'Proxmox.form.KVComboBox',
+ alias: ['widget.pveDecompressionSelector'],
+ config: {
+ deleteEmpty: false,
+ },
+ comboItems: [
+ ['__default__', Proxmox.Utils.noneText],
+ ['lzo', 'LZO'],
+ ['gz', 'GZIP'],
+ ['zst', 'ZSTD'],
+ ],
+});
diff --git a/www/manager6/window/DownloadUrlToStorage.js b/www/manager6/window/DownloadUrlToStorage.js
index 48543d28..559a1c05 100644
--- a/www/manager6/window/DownloadUrlToStorage.js
+++ b/www/manager6/window/DownloadUrlToStorage.js
@@ -49,6 +49,9 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
vm.set('size', '-');
vm.set('mimetype', '-');
},
+ decompressionPossible: function() {
+ return this.view.content === 'iso';
+ },
urlCheck: function(field) {
let me = this;
@@ -66,6 +69,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
params: {
url: queryParam.url,
'verify-certificates': queryParam['verify-certificates'],
+ 'detect-compression': me.decompressionPossible() ? 1 : 0,
},
waitMsgTarget: view,
failure: res => {
@@ -84,6 +88,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
filename: data.filename || "",
size: (data.size && Proxmox.Utils.format_size(data.size)) || gettext("Unknown"),
mimetype: data.mimetype || gettext("Unknown"),
+ compression: data.compression || '__default__',
});
},
});
@@ -215,7 +220,7 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
],
initComponent: function() {
- var me = this;
+ var me = this;
if (!me.nodename) {
throw "no node name specified";
@@ -223,8 +228,20 @@ Ext.define('PVE.window.DownloadUrlToStorage', {
if (!me.storage) {
throw "no storage ID specified";
}
+ if (me.content === 'iso') {
+ me.items[0].advancedColumn2.push(
+
+ {
+ xtype: 'pveDecompressionSelector',
+ name: 'compression',
+ fieldLabel: gettext('Decompression algorithm'),
+ allowBlank: true,
+ hasNoneOption: true,
+ value: '__default__',
+ });
+ }
- me.callParent();
+ me.callParent();
},
});
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs
2023-07-31 8:39 [pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH manager 1/1] fix #4849: download to storage: automatically dectect and configure compression Philipp Hufnagl
@ 2023-07-31 8:39 ` Philipp Hufnagl
2023-07-31 13:42 ` Fabian Grünbichler
2 siblings, 1 reply; 6+ messages in thread
From: Philipp Hufnagl @ 2023-07-31 8:39 UTC (permalink / raw)
To: pve-devel
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
src/PVE/Storage.pm | 6 ++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
index 2aaeff6..1d73c96 100644
--- a/src/PVE/API2/Storage/Status.pm
+++ b/src/PVE/API2/Storage/Status.pm
@@ -23,6 +23,8 @@ use PVE::Storage;
use base qw(PVE::RESTHandler);
+our $KNOWN_COMPRESSION_FORMATS = ['zst', 'gz', 'lzo'];
+
__PACKAGE__->register_method ({
subclass => "PVE::API2::Storage::PruneBackups",
path => '{storage}/prunebackups',
@@ -578,6 +580,12 @@ __PACKAGE__->register_method({
requires => 'checksum-algorithm',
optional => 1,
},
+ compression => {
+ description => "Decompress the downloaded file using specified compression algorithm",
+ type => 'string',
+ enum => $KNOWN_COMPRESSION_FORMATS,
+ optional => 1,
+ },
'checksum-algorithm' => {
description => "The algorithm to calculate the checksum of the file.",
type => 'string',
@@ -604,7 +612,7 @@ __PACKAGE__->register_method({
my $cfg = PVE::Storage::config();
- my ($node, $storage) = $param->@{'node', 'storage'};
+ my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
@@ -642,13 +650,18 @@ __PACKAGE__->register_method({
http_proxy => $dccfg->{http_proxy},
};
- my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
+ my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm' };
if ($checksum) {
$opts->{"${checksum_algorithm}sum"} = $checksum;
$opts->{hash_required} = 1;
}
my $worker = sub {
+ if ($compression) {
+ die "decompression not supported for $content\n" if $content ne 'iso';
+ my $info = PVE::Storage::decompressor_info('iso', $compression);
+ $opts->{decompression_command} = $info->{decompressor};
+ }
PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
};
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index a4d85e1..cb70113 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -1531,6 +1531,12 @@ sub decompressor_info {
lzo => ['lzop', '-d', '-c'],
zst => ['zstd', '-q', '-d', '-c'],
},
+ iso => {
+ # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
+ gz => ['zcat'],
+ lzo => ['lzop', '-d', '-c'],
+ zst => ['zstd', '-q', '-d', '-c'],
+ },
};
die "ERROR: archive format not defined\n"
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* Re: [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs
2023-07-31 8:39 ` [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs Philipp Hufnagl
@ 2023-07-31 13:42 ` Fabian Grünbichler
0 siblings, 0 replies; 6+ messages in thread
From: Fabian Grünbichler @ 2023-07-31 13:42 UTC (permalink / raw)
To: Proxmox VE development discussion
On July 31, 2023 10:39 am, Philipp Hufnagl wrote:
> Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
> ---
> src/PVE/API2/Storage/Status.pm | 17 +++++++++++++++--
> src/PVE/Storage.pm | 6 ++++++
> 2 files changed, 21 insertions(+), 2 deletions(-)
>
> diff --git a/src/PVE/API2/Storage/Status.pm b/src/PVE/API2/Storage/Status.pm
> index 2aaeff6..1d73c96 100644
> --- a/src/PVE/API2/Storage/Status.pm
> +++ b/src/PVE/API2/Storage/Status.pm
> @@ -23,6 +23,8 @@ use PVE::Storage;
>
> use base qw(PVE::RESTHandler);
>
> +our $KNOWN_COMPRESSION_FORMATS = ['zst', 'gz', 'lzo'];
might want a comment here that adding new formats means handling them in
decompressor_info below, and vice-versa?
also, this is just another way to write PVE::Storage::Plugin::COMPRESSOR_RE
maybe they could be unified to avoid accidentally desyncing them?
technically, the RE could be used as 'pattern' in the API schema, but
the generated docs are less human-friendly for patterns compared to
enums..
> +
> __PACKAGE__->register_method ({
> subclass => "PVE::API2::Storage::PruneBackups",
> path => '{storage}/prunebackups',
> @@ -578,6 +580,12 @@ __PACKAGE__->register_method({
> requires => 'checksum-algorithm',
> optional => 1,
> },
> + compression => {
> + description => "Decompress the downloaded file using specified compression algorithm",
> + type => 'string',
> + enum => $KNOWN_COMPRESSION_FORMATS,
> + optional => 1,
> + },
> 'checksum-algorithm' => {
> description => "The algorithm to calculate the checksum of the file.",
> type => 'string',
> @@ -604,7 +612,7 @@ __PACKAGE__->register_method({
>
> my $cfg = PVE::Storage::config();
>
> - my ($node, $storage) = $param->@{'node', 'storage'};
> + my ($node, $storage, $compression) = $param->@{'node', 'storage','compression'};
> my $scfg = PVE::Storage::storage_check_enabled($cfg, $storage, $node);
>
> die "can't upload to storage type '$scfg->{type}', not a file based storage!\n"
> @@ -642,13 +650,18 @@ __PACKAGE__->register_method({
> http_proxy => $dccfg->{http_proxy},
> };
>
> - my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm'};
> + my ($checksum, $checksum_algorithm) = $param->@{'checksum', 'checksum-algorithm' };
> if ($checksum) {
> $opts->{"${checksum_algorithm}sum"} = $checksum;
> $opts->{hash_required} = 1;
> }
>
> my $worker = sub {
> + if ($compression) {
> + die "decompression not supported for $content\n" if $content ne 'iso';
> + my $info = PVE::Storage::decompressor_info('iso', $compression);
> + $opts->{decompression_command} = $info->{decompressor};
note: $info->{decompressor} might be undef, should probably be caught
here, else decompression will be silently skipped
> + }
> PVE::Tools::download_file_from_url("$path/$filename", $url, $opts);
> };
>
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index a4d85e1..cb70113 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -1531,6 +1531,12 @@ sub decompressor_info {
> lzo => ['lzop', '-d', '-c'],
> zst => ['zstd', '-q', '-d', '-c'],
> },
> + iso => {
> + # zstd seem to be able to handle .gzip fine. Therefore we dont need additional other tool
> + gz => ['zcat'],
> + lzo => ['lzop', '-d', '-c'],
> + zst => ['zstd', '-q', '-d', '-c'],
> + },
> };
>
> die "ERROR: archive format not defined\n"
> --
> 2.39.2
>
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread