From: Philipp Hufnagl <p.hufnagl@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command
Date: Mon, 31 Jul 2023 10:39:22 +0200 [thread overview]
Message-ID: <20230731083924.18234-2-p.hufnagl@proxmox.com> (raw)
In-Reply-To: <20230731083924.18234-1-p.hufnagl@proxmox.com>
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
src/PVE/Tools.pm | 59 +++++++++++++++++++++++++++++++-----------------
1 file changed, 38 insertions(+), 21 deletions(-)
diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 9ffac12..ab97129 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -92,23 +92,23 @@ our $EMAIL_USER_RE = qr/[\w\+\-\~]+(\.[\w\+\-\~]+)*/;
our $EMAIL_RE = qr/$EMAIL_USER_RE@[a-zA-Z0-9\-]+(\.[a-zA-Z0-9\-]+)*/;
use constant {CLONE_NEWNS => 0x00020000,
- CLONE_NEWUTS => 0x04000000,
- CLONE_NEWIPC => 0x08000000,
- CLONE_NEWUSER => 0x10000000,
- CLONE_NEWPID => 0x20000000,
- CLONE_NEWNET => 0x40000000};
+ CLONE_NEWUTS => 0x04000000,
+ CLONE_NEWIPC => 0x08000000,
+ CLONE_NEWUSER => 0x10000000,
+ CLONE_NEWPID => 0x20000000,
+ CLONE_NEWNET => 0x40000000};
use constant {O_PATH => 0x00200000,
- O_CLOEXEC => 0x00080000,
- O_TMPFILE => 0x00400000 | O_DIRECTORY};
+ O_CLOEXEC => 0x00080000,
+ O_TMPFILE => 0x00400000 | O_DIRECTORY};
use constant {AT_EMPTY_PATH => 0x1000,
- AT_FDCWD => -100};
+ AT_FDCWD => -100};
# from <linux/fs.h>
use constant {RENAME_NOREPLACE => (1 << 0),
- RENAME_EXCHANGE => (1 << 1),
- RENAME_WHITEOUT => (1 << 2)};
+ RENAME_EXCHANGE => (1 << 1),
+ RENAME_WHITEOUT => (1 << 2)};
sub run_with_timeout {
my ($timeout, $code, @param) = @_;
@@ -579,7 +579,7 @@ sub run_command {
}
}
- alarm(0);
+ alarm(0);
};
my $err = $@;
@@ -1354,7 +1354,7 @@ sub dump_journal {
my $parser = sub {
my $line = shift;
- return if $count++ < $start;
+ return if $count++ < $start;
return if $limit <= 0;
push @$lines, { n => int($count), t => $line};
$limit--;
@@ -1441,7 +1441,7 @@ sub unpack_sockaddr_in46 {
my ($sin) = @_;
my $family = Socket::sockaddr_family($sin);
my ($port, $host) = ($family == AF_INET6 ? Socket::unpack_sockaddr_in6($sin)
- : Socket::unpack_sockaddr_in($sin));
+ : Socket::unpack_sockaddr_in($sin));
return ($family, $port, $host);
}
@@ -1485,8 +1485,8 @@ sub get_fqdn {
sub parse_host_and_port {
my ($address) = @_;
if ($address =~ /^($IPV4RE|[[:alnum:]\-.]+)(?::(\d+))?$/ || # ipv4 or host with optional ':port'
- $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
- $address =~ /^($IPV6RE)(?:\.(\d+))?$/) # ipv6 with optional port separated by dot
+ $address =~ /^\[($IPV6RE|$IPV4RE|[[:alnum:]\-.]+)\](?::(\d+))?$/ || # anything in brackets with optional ':port'
+ $address =~ /^($IPV6RE)(?:\.(\d+))?$/) # ipv6 with optional port separated by dot
{
return ($1, $2, 1); # end with 1 to support simple if(parse...) tests
}
@@ -2013,10 +2013,13 @@ sub download_file_from_url {
}
}
- my $tmpdest = "$dest.tmp.$$";
+ my $tmp_download = "$dest.tmp_dwnl.$$";
+ my $tmp_decomp = "$dest.tmp.dcom.$$";
eval {
local $SIG{INT} = sub {
- unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ unlink $tmp_download or warn "could not cleanup temporary file: $!";
+ die "got interrupted by signal\n";
+ unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
die "got interrupted by signal\n";
};
@@ -2029,7 +2032,7 @@ sub download_file_from_url {
$ENV{https_proxy} = $opts->{https_proxy};
}
- my $cmd = ['wget', '--progress=dot:giga', '-O', $tmpdest, $url];
+ my $cmd = ['wget', '--progress=dot:giga', '-O', $tmp_download, $url];
if (!($opts->{verify_certificates} // 1)) { # default to true
push @$cmd, '--no-check-certificate';
@@ -2041,7 +2044,7 @@ sub download_file_from_url {
if ($checksum_algorithm) {
print "calculating checksum...";
- my $checksum_got = get_file_hash($checksum_algorithm, $tmpdest);
+ my $checksum_got = get_file_hash($checksum_algorithm, $tmp_download);
if (lc($checksum_got) eq lc($checksum_expected)) {
print "OK, checksum verified\n";
@@ -2051,10 +2054,24 @@ sub download_file_from_url {
}
}
- rename($tmpdest, $dest) or die "unable to rename temporary file: $!\n";
+ if (my $cmd = $opts->{decompression_command}) {
+ push @$cmd, $tmp_download;
+
+
+ my $fh;
+ if (!open($fh, ">", "$tmp_decomp")) {
+ die "cant open temporary file $tmp_decomp for decompresson: $!\n";
+ }
+ print "decompressing $tmp_download to $tmp_decomp\n";
+ eval { run_command($cmd, output => '>&'.fileno($fh)); };
+ my $rerr = $@;
+ unlink $tmp_download;
+ die "$rerr\n" if $rerr;
+ }
+ rename($tmp_decomp, $dest) or die "unable to rename temporary file: $!\n";
};
if (my $err = $@) {
- unlink $tmpdest or warn "could not cleanup temporary file: $!";
+ unlink $tmp_decomp or warn "could not cleanup temporary file: $!";
die $err;
}
--
2.39.2
next prev parent reply other threads:[~2023-07-31 8:39 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-07-31 8:39 [pve-devel] [V3 PATCH storage/manager/common 0/3]fix #4849: allow download of compressed ISOs Philipp Hufnagl
2023-07-31 8:39 ` Philipp Hufnagl [this message]
2023-07-31 13:42 ` [pve-devel] [V3 PATCH common 1/1] fix #4849: download file from url: add opt parameter for a decompression command Fabian Grünbichler
2023-07-31 8:39 ` [pve-devel] [V3 PATCH manager 1/1] fix #4849: download to storage: automatically dectect and configure compression Philipp Hufnagl
2023-07-31 8:39 ` [pve-devel] [V3 PATCH storage 1/1] fix #4849: download-url: allow download and decompression of compressed ISOs Philipp Hufnagl
2023-07-31 13:42 ` Fabian Grünbichler
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230731083924.18234-2-p.hufnagl@proxmox.com \
--to=p.hufnagl@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox