From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5A4BE78F62 for ; Mon, 3 May 2021 12:20:52 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 57C8A1C10C for ; Mon, 3 May 2021 12:20:52 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 8D9331C103 for ; Mon, 3 May 2021 12:20:51 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 681D242A16 for ; Mon, 3 May 2021 12:20:51 +0200 (CEST) From: Lorenz Stechauner To: pve-devel@lists.proxmox.com Date: Mon, 3 May 2021 12:20:45 +0200 Message-Id: <20210503102045.134007-1-l.stechauner@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210503102016.133870-1-l.stechauner@proxmox.com> References: <20210503102016.133870-1-l.stechauner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 1 AWL 1.257 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH v2 storage 1/2] fix #1710: add retrieve method for storage X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 May 2021 10:20:52 -0000 Users are now able to download/retrieve any .iso/... file onto their storages and verify file integrity with checksums. Signed-off-by: Lorenz Stechauner --- PVE/API2/Storage/Status.pm | 156 +++++++++++++++++++++++++++++++++++-- PVE/Storage.pm | 10 +++ 2 files changed, 161 insertions(+), 5 deletions(-) diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm index 897b4a7..a64101f 100644 --- a/PVE/API2/Storage/Status.pm +++ b/PVE/API2/Storage/Status.pm @@ -5,6 +5,8 @@ use warnings; use File::Path; use File::Basename; +use HTTP::Request; +use LWP::UserAgent; use PVE::Tools; use PVE::INotify; use PVE::Cluster; @@ -412,11 +414,7 @@ __PACKAGE__->register_method ({ my $size = -s $tmpfilename; die "temporary file '$tmpfilename' does not exist\n" if !defined($size); - my $filename = $param->{filename}; - - chomp $filename; - $filename =~ s/^.*[\/\\]//; - $filename =~ s/[^-a-zA-Z0-9_.]/_/g; + my $filename = PVE::Storage::normalize_content_filename($param->{filename}); my $path; @@ -497,4 +495,152 @@ __PACKAGE__->register_method ({ return $upid; }}); +__PACKAGE__->register_method({ + name => 'retrieve', + path => '{storage}/retrieve', + method => 'POST', + description => "Download templates and ISO images by using an URL.", + proxyto => 'node', + permissions => { + check => [ 'and', + ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]], + ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]], + ], + }, + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + storage => get_standard_option('pve-storage-id'), + url => { + description => "The URL to retrieve the file from.", + type => 'string', + }, + content => { + description => "Content type.", + type => 'string', format => 'pve-storage-content', + }, + filename => { + description => "The name of the file to create. Alternatively the file name given by the server will be used.", + type => 'string', + }, + checksum => { + description => "The expected checksum of the file.", + type => 'string', + requires => 'checksumalg', + optional => 1, + }, + checksumalg => { + description => "The algorithm to claculate the checksum of the file.", + type => 'string', enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'], + requires => 'checksum', + optional => 1, + }, + insecure => { + description => "Allow TLS certificates to be invalid.", + type => 'boolean', + optional => 1, + } + }, + }, + returns => { type => "string" }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + + my $user = $rpcenv->get_user(); + + my $cfg = PVE::Storage::config(); + + my $node = $param->{node}; + my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node); + + die "can't upload to storage type '$scfg->{type}'" + if !defined($scfg->{path}); + + my $content = $param->{content}; + + my $url = $param->{url}; + + die "invalid https or http url" + if $url !~ qr!^https?://!; + + my $checksum = $param->{checksum}; + my $hash_alg = $param->{checksumalg}; + + my $filename = PVE::Storage::normalize_content_filename($param->{filename}); + + my $path; + + # MIME type is checked in front end only + # this check is omitted here intentionally and replaced by file extension check + if ($content eq 'iso') { + if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) { + raise_param_exc({ filename => "missing '.iso' or '.img' extension" }); + } + $path = PVE::Storage::get_iso_dir($cfg, $param->{storage}); + } elsif ($content eq 'vztmpl') { + if ($filename !~ m![^/]+\.tar\.[gx]z$!) { + raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" }); + } + $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage}); + } else { + raise_param_exc({ content => "upload content type '$content' not allowed" }); + } + + die "storage '$param->{storage}' does not support '$content' content" + if !$scfg->{content}->{$content}; + + my $dest = "$path/$filename"; + + PVE::Storage::activate_storage($cfg, $param->{storage}); + File::Path::make_path($path); + + # -L follows redirects + # -f silent fail on error + my @curlcmd = ('curl', '-L', '-o', $dest, '-f'); + push @curlcmd, '--insecure' + if $param->{insecure}; + + my $cmd = [@curlcmd, $url]; + + my $cmd_check = [ [ 'echo', $checksum, $dest ], [ "${hash_alg}sum", '-c' ] ]; + my $cmd_check_flat = [ 'echo', $checksum, $dest, '|', "${hash_alg}sum", '-c' ]; # only used for logging + + my $worker = sub { + my $upid = shift; + + print "starting file download from: $url\n"; + print "target node: $node\n"; + print "target file: $dest\n"; + print "command: " . join(' ', @$cmd) . "\n"; + + eval { PVE::Tools::run_command($cmd, errmsg => 'download failed'); }; + if (my $err = $@) { + unlink $dest; + die $err; + } + print "finished file download successfully\n"; + + if ($checksum) { + print "validating checksum...\n"; + print "expected $hash_alg checksum is: $checksum\n"; + print cmd_check print "checksum validation command: " . join(' ', @$cmd_check_flat) . "\n"; + + eval { PVE::Tools::run_command($cmd_check, errmsg => 'checksum mismatch'); }; + if (my $err = $@) { + unlink $dest; + die $err; + } + print "validated checksum successfully\n"; + } + }; + + my $upid = $rpcenv->fork_worker('imgdownload', undef, $user, $worker); + + return $upid; + }}); + 1; diff --git a/PVE/Storage.pm b/PVE/Storage.pm index 122c3e9..d57fd43 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -1931,4 +1931,14 @@ sub assert_sid_unused { return undef; } +sub normalize_content_filename { + my ($filename) = @_; + + chomp $filename; + $filename =~ s/^.*[\/\\]//; + $filename =~ s/[^-a-zA-Z0-9_.]/_/g; + + return $filename; +} + 1; -- 2.20.1