From: Lorenz Stechauner <l.stechauner@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH v6 common 2/7] tools: add download_file_from_url
Date: Mon, 17 May 2021 10:03:13 +0200 [thread overview]
Message-ID: <20210517080318.51405-3-l.stechauner@proxmox.com> (raw)
In-Reply-To: <20210517080318.51405-1-l.stechauner@proxmox.com>
code is based on
manager:PVE/API2/Nodes.pm:aplinfo
Signed-off-by: Lorenz Stechauner <l.stechauner@proxmox.com>
---
src/PVE/Tools.pm | 124 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 124 insertions(+)
diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index 16ae3d2..7b82e00 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -1829,4 +1829,128 @@ sub safe_compare {
return $cmp->($left, $right);
}
+
+# opts
+# -> hash_required
+# if 1, at least one checksum has to be specified otherwise an error will be thrown
+# -> http_proxy
+# -> https_proxy
+# -> verify_certificates
+# -> sha(1|224|256|384|512)sum
+# -> md5sum
+sub download_file_from_url {
+ my ($dest, $url, $opts) = @_;
+
+ my $tmpdest = "$dest.tmp.$$";
+
+ my $algorithm;
+ my $expected;
+
+ for ('sha512', 'sha384', 'sha256', 'sha224', 'sha1', 'md5') {
+ if (defined($opts->{"${_}sum"})) {
+ $algorithm = $_;
+ $expected = $opts->{"${_}sum"};
+ last;
+ }
+ }
+
+ die "checksum required but not specified\n" if ($opts->{hash_required} && !$algorithm);
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "downloading $url to $dest\n";
+
+ eval {
+ if (-f $dest && $algorithm) {
+ print "calculating checksum of existing file...\n";
+ my $correct = check_file_hash($algorithm, $expected, $dest);
+
+ if ($correct) {
+ print "file already exists, no need to download\n";
+ return;
+ } else {
+ print "mismatch, downloading\n";
+ }
+ }
+
+ my @cmd = ('/usr/bin/wget', '--progress=dot:mega', '-O', $tmpdest, $url);
+
+ local %ENV;
+ if ($opts->{http_proxy}) {
+ $ENV{http_proxy} = $opts->{http_proxy};
+ }
+ if ($opts->{https_proxy}) {
+ $ENV{https_proxy} = $opts->{https_proxy};
+ }
+
+ my $verify = $opts->{verify_certificates} // 1;
+ if (!$verify) {
+ push @cmd, '--no-check-certificate';
+ }
+
+ if (run_command([[@cmd]]) != 0) {
+ die "download failed: $!\n";
+ }
+
+ if ($algorithm) {
+ print "calculating checksum...\n";
+
+ my $correct = check_file_hash($algorithm, $expected, $tmpdest);
+
+ if ($correct) {
+ print "checksum verified\n";
+ } else {
+ die "checksum mismatch\n";
+ }
+ } else {
+ print "no checksum for verification specified\n";
+ }
+
+ if (!rename($tmpdest, $dest)) {
+ die "unable to save file: $!\n";
+ }
+ };
+ my $err = $@;
+
+ unlink $tmpdest;
+
+ if ($err) {
+ print "\n";
+ die $err;
+ }
+
+ print "download finished\n";
+ };
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+ my $user = $rpcenv->get_user();
+
+ (my $filename = $dest) =~ s!.*/([^/]*)$!$1!;
+
+ return $rpcenv->fork_worker('download', $filename, $user, $worker);
+}
+
+sub check_file_hash {
+ my ($algorithm, $expected, $filename) = @_;
+
+ my $algorithm_map = {
+ 'md5' => sub { Digest::MD5->new },
+ 'sha1' => sub { Digest::SHA->new(1) },
+ 'sha224' => sub { Digest::SHA->new(224) },
+ 'sha256' => sub { Digest::SHA->new(256) },
+ 'sha384' => sub { Digest::SHA->new(384) },
+ 'sha512' => sub { Digest::SHA->new(512) },
+ };
+
+ my $digester = $algorithm_map->{$algorithm}->() or die "unknown algorithm '$algorithm'\n";
+
+ open(my $fh, '<', $filename) or die "unable to open '$filename': $!\n";
+ binmode($fh);
+
+ my $digest = $digester->addfile($fh)->hexdigest;
+
+ return lc($digest) eq lc($expected);
+}
+
1;
--
2.20.1
next prev parent reply other threads:[~2021-05-17 8:03 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-17 8:03 [pve-devel] [PATCH-SERIES v6 manager/common/storage] fix #1710: add download from url button Lorenz Stechauner
2021-05-17 8:03 ` [pve-devel] [PATCH v6 manager 1/7] api: nodes: add query_url_metadata method Lorenz Stechauner
2021-06-11 7:27 ` Dominic Jäger
2021-05-17 8:03 ` Lorenz Stechauner [this message]
2021-05-17 8:03 ` [pve-devel] [PATCH v6 manager 3/7] api: nodes: refactor aplinfo to use common download function Lorenz Stechauner
2021-05-17 8:03 ` [pve-devel] [PATCH v6 storage 4/7] status: add download_url method Lorenz Stechauner
2021-05-17 8:03 ` [pve-devel] [PATCH v6 manager 5/7] ui: add HashAlgorithmSelector Lorenz Stechauner
2021-05-17 8:03 ` [pve-devel] [PATCH v6 manager 6/7] ui: Utils: change download task format Lorenz Stechauner
2021-05-17 8:03 ` [pve-devel] [PATCH v6 manager 7/7] fix #1710: ui: storage: add download from url button Lorenz Stechauner
2021-06-11 7:27 ` Dominic Jäger
2021-06-11 8:30 ` Lorenz Stechauner
2021-06-09 6:50 ` [pve-devel] [PATCH-SERIES v6 manager/common/storage] fix #1710: " Lorenz Stechauner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210517080318.51405-3-l.stechauner@proxmox.com \
--to=l.stechauner@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal