From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 40E437796A for ; Wed, 28 Apr 2021 16:14:03 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 318DCFAC3 for ; Wed, 28 Apr 2021 16:14:03 +0200 (CEST) Received: from dana.proxmox.com (unknown [94.136.29.99]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0F4E6FAA5 for ; Wed, 28 Apr 2021 16:13:58 +0200 (CEST) Received: by dana.proxmox.com (Postfix, from userid 10037) id D38FC1C088B; Wed, 28 Apr 2021 16:13:58 +0200 (CEST) From: Lorenz Stechauner To: pve-devel@lists.proxmox.com Date: Wed, 28 Apr 2021 16:13:45 +0200 Message-Id: <20210428141346.240896-2-l.stechauner@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210428141346.240896-1-l.stechauner@proxmox.com> References: <20210428141346.240896-1-l.stechauner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 2 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pve-devel] [PATCH storage 1/1] fix #1710: add retrieve method for storage X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 28 Apr 2021 14:14:03 -0000 Users are now able to download/retrieve any .iso/... file onto their storages and verify file integrity with checksums. Signed-off-by: Lorenz Stechauner --- PVE/API2/Storage/Status.pm | 244 +++++++++++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm index 897b4a7..3919be7 100644 --- a/PVE/API2/Storage/Status.pm +++ b/PVE/API2/Storage/Status.pm @@ -5,6 +5,8 @@ use warnings; use File::Path; use File::Basename; +use HTTP::Request; +use LWP::UserAgent; use PVE::Tools; use PVE::INotify; use PVE::Cluster; @@ -497,4 +499,246 @@ __PACKAGE__->register_method ({ return $upid; }}); +__PACKAGE__->register_method({ + name => 'retrieve', + path => '{storage}/retrieve', + method => 'POST', + description => "Download templates and ISO images by using an URL.", + permissions => { + check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']], + }, + protected => 1, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + storage => get_standard_option('pve-storage-id'), + url => { + description => "The URL to retrieve the file from.", + type => 'string', + }, + content => { + description => "Content type.", + type => 'string', format => 'pve-storage-content', + }, + filename => { + description => "The name of the file to create. Alternatively the file name given by the server will be used.", + type => 'string', + optional => 1, + }, + checksum => { + description => "The expected checksum of the file.", + type => 'string', + optional => 1, + }, + checksumalg => { + description => "The algorithm to claculate the checksum of the file.", + type => 'string', + optional => 1, + }, + metaonly => { + description => "Only return the file name and size.", + type => 'boolean', + optional => 1, + }, + insecure => { + description => "Allow TLS certificates to be invalid.", + type => 'boolean', + optional => 1, + } + }, + }, + returns => { + type => "object", + properties => { + filename => { type => 'string' }, + upid => { type => 'string' }, + size => { + type => 'integer', + renderer => 'bytes', + }, + }, + }, + code => sub { + my ($param) = @_; + + my @hash_algs = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']; + + my $rpcenv = PVE::RPCEnvironment::get(); + + my $user = $rpcenv->get_user(); + + my $cfg = PVE::Storage::config(); + + my $node = $param->{node}; + my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node); + + die "can't upload to storage type '$scfg->{type}'" + if !defined($scfg->{path}); + + my $content = $param->{content}; + + my $url = $param->{url}; + + die "invalid https or http url" + if $url !~ qr!^https?://!; + + my $checksum = $param->{checksum}; + my $hash_alg = $param->{checksumalg}; + + $hash_alg = undef + if $hash_alg eq 'none'; + + die "either 'checksum' and 'checksumalg' or none of these have to be specified" + if ($checksum && !$hash_alg) || (!$checksum && $hash_alg); + + die "unsupported checksumalg: '$hash_alg'" + if $hash_alg && ($hash_alg !~ /^(md5|sha1|sha(224|256|384|512))$/); + + my $req = HTTP::Request->new(HEAD => $url); + my $ua = LWP::UserAgent->new(); + my $res = $ua->request($req); + + die "invalid server response: '" . $res->status_line() . "'" + if ($res->code() != 200); + + my $size = $res->header("Content-Length"); + my $dispo = $res->header("Content-Disposition"); + + my $filename_remote; + + if ($dispo && $dispo =~ m/filename=(.+)/) { + $filename_remote = $1; + } elsif ($url =~ m!^[^?]+/([^?/]*)(?:\?.*)?$!) { + $filename_remote = $1; + } + + chomp $filename_remote; + $filename_remote =~ s/^.*[\/\\]//; + $filename_remote =~ s/[^-a-zA-Z0-9_.]/_/g; + + if ($param->{metaonly}) { + return { + filename => $filename_remote, + upid => 0, + size => $size + 0, + }; + } + + my $filename = $param->{filename} || $filename_remote; + chomp $filename; + $filename =~ s/^.*[\/\\]//; + $filename =~ s/[^-a-zA-Z0-9_.]/_/g; + + my $path; + + if ($content eq 'iso') { + if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) { + raise_param_exc({ filename => "missing '.iso' or '.img' extension" }); + } + $path = PVE::Storage::get_iso_dir($cfg, $param->{storage}); + } elsif ($content eq 'vztmpl') { + if ($filename !~ m![^/]+\.tar\.[gx]z$!) { + raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" }); + } + $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage}); + } else { + raise_param_exc({ content => "upload content type '$content' not allowed" }); + } + + die "storage '$param->{storage}' does not support '$content' content" + if !$scfg->{content}->{$content}; + + my $dest = "$path/$filename"; + my $dirname = dirname($dest); + + my $cmd; + my $cmd_check; + my $cmd_check_flat; + my $cmd_del; + + my @curlcmd = ('curl', '-L', '-o', $dest, '-f'); + push @curlcmd, '--insecure' + if $param->{insecure}; + + my @check1 = ('echo', $checksum, $dest); + my @check2 = ("${hash_alg}sum", '-c'); + + my @unlinkcmd = ('rm', '-f', $dest); + + if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) { + my $remip = PVE::Cluster::remote_node_ip($node); + + my @ssh_options = ('-o', 'BatchMode=yes'); + + my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--'); + + eval { + # activate remote storage + PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]); + }; + die "can't activate storage '$param->{storage}' on node '$node': $@" + if $@; + + PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', '--', $dirname], + errmsg => "mkdir failed"); + + $cmd = [@remcmd, @curlcmd, PVE::Tools::shell_quote($url)]; + + $cmd_check = [@remcmd, @check1, '|', @check2]; + $cmd_check_flat = [@remcmd, @check1, '|', @check2]; + + $cmd_del = [@remcmd, @unlinkcmd]; + } else { + PVE::Storage::activate_storage($cfg, $param->{storage}); + File::Path::make_path($dirname); + + $cmd = [@curlcmd, $url]; + + $cmd_check = [[@check1], [@check2]]; + $cmd_check_flat = [@check1, '|', @check2]; + + $cmd_del = [@unlinkcmd]; + } + + my $worker = sub { + my $upid = shift; + + print "starting file download from: $url\n"; + print "target node: $node\n"; + print "target file: $dest\n"; + print "file size is: $size\n"; + print "command: " . join(' ', @$cmd) . "\n"; + + eval { PVE::Tools::run_command($cmd, errmsg => 'download failed'); }; + if (my $err = $@) { + PVE::Tools::run_command($cmd_del); + die $err; + } + print "finished file download successfully\n"; + + if ($checksum) { + print "validating checksum...\n"; + print "expected $hash_alg checksum is: $checksum\n"; + print "checksum validation command: " . join(' ', @$cmd_check_flat) . "\n"; + + eval { PVE::Tools::run_command($cmd_check, errmsg => 'checksum mismatch'); }; + if (my $err = $@) { + PVE::Tools::run_command($cmd_del); + die $err; + } + print "validated checksum successfully\n"; + } + }; + + my $upid = $rpcenv->fork_worker('imgdownload', undef, $user, $worker); + + return { + filename => $filename, + upid => $upid, + size => $size + 0, + }; + }}); + + 1; -- 2.20.1