From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D848670EDB for ; Mon, 17 May 2021 10:03:41 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BAE1A28ABA for ; Mon, 17 May 2021 10:03:41 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id E503128A31 for ; Mon, 17 May 2021 10:03:36 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id B498A465AC for ; Mon, 17 May 2021 10:03:36 +0200 (CEST) From: Lorenz Stechauner To: pve-devel@lists.proxmox.com Date: Mon, 17 May 2021 10:03:13 +0200 Message-Id: <20210517080318.51405-3-l.stechauner@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210517080318.51405-1-l.stechauner@proxmox.com> References: <20210517080318.51405-1-l.stechauner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.455 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v6 common 2/7] tools: add download_file_from_url X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 17 May 2021 08:03:41 -0000 code is based on manager:PVE/API2/Nodes.pm:aplinfo Signed-off-by: Lorenz Stechauner --- src/PVE/Tools.pm | 124 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm index 16ae3d2..7b82e00 100644 --- a/src/PVE/Tools.pm +++ b/src/PVE/Tools.pm @@ -1829,4 +1829,128 @@ sub safe_compare { return $cmp->($left, $right); } + +# opts +# -> hash_required +# if 1, at least one checksum has to be specified otherwise an error will be thrown +# -> http_proxy +# -> https_proxy +# -> verify_certificates +# -> sha(1|224|256|384|512)sum +# -> md5sum +sub download_file_from_url { + my ($dest, $url, $opts) = @_; + + my $tmpdest = "$dest.tmp.$$"; + + my $algorithm; + my $expected; + + for ('sha512', 'sha384', 'sha256', 'sha224', 'sha1', 'md5') { + if (defined($opts->{"${_}sum"})) { + $algorithm = $_; + $expected = $opts->{"${_}sum"}; + last; + } + } + + die "checksum required but not specified\n" if ($opts->{hash_required} && !$algorithm); + + my $worker = sub { + my $upid = shift; + + print "downloading $url to $dest\n"; + + eval { + if (-f $dest && $algorithm) { + print "calculating checksum of existing file...\n"; + my $correct = check_file_hash($algorithm, $expected, $dest); + + if ($correct) { + print "file already exists, no need to download\n"; + return; + } else { + print "mismatch, downloading\n"; + } + } + + my @cmd = ('/usr/bin/wget', '--progress=dot:mega', '-O', $tmpdest, $url); + + local %ENV; + if ($opts->{http_proxy}) { + $ENV{http_proxy} = $opts->{http_proxy}; + } + if ($opts->{https_proxy}) { + $ENV{https_proxy} = $opts->{https_proxy}; + } + + my $verify = $opts->{verify_certificates} // 1; + if (!$verify) { + push @cmd, '--no-check-certificate'; + } + + if (run_command([[@cmd]]) != 0) { + die "download failed: $!\n"; + } + + if ($algorithm) { + print "calculating checksum...\n"; + + my $correct = check_file_hash($algorithm, $expected, $tmpdest); + + if ($correct) { + print "checksum verified\n"; + } else { + die "checksum mismatch\n"; + } + } else { + print "no checksum for verification specified\n"; + } + + if (!rename($tmpdest, $dest)) { + die "unable to save file: $!\n"; + } + }; + my $err = $@; + + unlink $tmpdest; + + if ($err) { + print "\n"; + die $err; + } + + print "download finished\n"; + }; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $user = $rpcenv->get_user(); + + (my $filename = $dest) =~ s!.*/([^/]*)$!$1!; + + return $rpcenv->fork_worker('download', $filename, $user, $worker); +} + +sub check_file_hash { + my ($algorithm, $expected, $filename) = @_; + + my $algorithm_map = { + 'md5' => sub { Digest::MD5->new }, + 'sha1' => sub { Digest::SHA->new(1) }, + 'sha224' => sub { Digest::SHA->new(224) }, + 'sha256' => sub { Digest::SHA->new(256) }, + 'sha384' => sub { Digest::SHA->new(384) }, + 'sha512' => sub { Digest::SHA->new(512) }, + }; + + my $digester = $algorithm_map->{$algorithm}->() or die "unknown algorithm '$algorithm'\n"; + + open(my $fh, '<', $filename) or die "unable to open '$filename': $!\n"; + binmode($fh); + + my $digest = $digester->addfile($fh)->hexdigest; + + return lc($digest) eq lc($expected); +} + 1; -- 2.20.1