From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <lstechauner@dana.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 40E437796A
for <pve-devel@lists.proxmox.com>; Wed, 28 Apr 2021 16:14:03 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 318DCFAC3
for <pve-devel@lists.proxmox.com>; Wed, 28 Apr 2021 16:14:03 +0200 (CEST)
Received: from dana.proxmox.com (unknown [94.136.29.99])
by firstgate.proxmox.com (Proxmox) with ESMTP id 0F4E6FAA5
for <pve-devel@lists.proxmox.com>; Wed, 28 Apr 2021 16:13:58 +0200 (CEST)
Received: by dana.proxmox.com (Postfix, from userid 10037)
id D38FC1C088B; Wed, 28 Apr 2021 16:13:58 +0200 (CEST)
From: Lorenz Stechauner <l.stechauner@proxmox.com>
To: pve-devel@lists.proxmox.com
Date: Wed, 28 Apr 2021 16:13:45 +0200
Message-Id: <20210428141346.240896-2-l.stechauner@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20210428141346.240896-1-l.stechauner@proxmox.com>
References: <20210428141346.240896-1-l.stechauner@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 2
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery
methods
NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records
RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_NONE 0.001 SPF: sender does not publish an SPF Record
Subject: [pve-devel] [PATCH storage 1/1] fix #1710: add retrieve method for
storage
X-BeenThere: pve-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox VE development discussion <pve-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pve-devel/>
List-Post: <mailto:pve-devel@lists.proxmox.com>
List-Help: <mailto:pve-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel>,
<mailto:pve-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Wed, 28 Apr 2021 14:14:03 -0000
Users are now able to download/retrieve any .iso/... file onto their
storages and verify file integrity with checksums.
Signed-off-by: Lorenz Stechauner <l.stechauner@proxmox.com>
---
PVE/API2/Storage/Status.pm | 244 +++++++++++++++++++++++++++++++++++++
1 file changed, 244 insertions(+)
diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm
index 897b4a7..3919be7 100644
--- a/PVE/API2/Storage/Status.pm
+++ b/PVE/API2/Storage/Status.pm
@@ -5,6 +5,8 @@ use warnings;
use File::Path;
use File::Basename;
+use HTTP::Request;
+use LWP::UserAgent;
use PVE::Tools;
use PVE::INotify;
use PVE::Cluster;
@@ -497,4 +499,246 @@ __PACKAGE__->register_method ({
return $upid;
}});
+__PACKAGE__->register_method({
+ name => 'retrieve',
+ path => '{storage}/retrieve',
+ method => 'POST',
+ description => "Download templates and ISO images by using an URL.",
+ permissions => {
+ check => ['perm', '/storage/{storage}', ['Datastore.AllocateTemplate']],
+ },
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ storage => get_standard_option('pve-storage-id'),
+ url => {
+ description => "The URL to retrieve the file from.",
+ type => 'string',
+ },
+ content => {
+ description => "Content type.",
+ type => 'string', format => 'pve-storage-content',
+ },
+ filename => {
+ description => "The name of the file to create. Alternatively the file name given by the server will be used.",
+ type => 'string',
+ optional => 1,
+ },
+ checksum => {
+ description => "The expected checksum of the file.",
+ type => 'string',
+ optional => 1,
+ },
+ checksumalg => {
+ description => "The algorithm to claculate the checksum of the file.",
+ type => 'string',
+ optional => 1,
+ },
+ metaonly => {
+ description => "Only return the file name and size.",
+ type => 'boolean',
+ optional => 1,
+ },
+ insecure => {
+ description => "Allow TLS certificates to be invalid.",
+ type => 'boolean',
+ optional => 1,
+ }
+ },
+ },
+ returns => {
+ type => "object",
+ properties => {
+ filename => { type => 'string' },
+ upid => { type => 'string' },
+ size => {
+ type => 'integer',
+ renderer => 'bytes',
+ },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my @hash_algs = ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'];
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+
+ my $user = $rpcenv->get_user();
+
+ my $cfg = PVE::Storage::config();
+
+ my $node = $param->{node};
+ my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
+
+ die "can't upload to storage type '$scfg->{type}'"
+ if !defined($scfg->{path});
+
+ my $content = $param->{content};
+
+ my $url = $param->{url};
+
+ die "invalid https or http url"
+ if $url !~ qr!^https?://!;
+
+ my $checksum = $param->{checksum};
+ my $hash_alg = $param->{checksumalg};
+
+ $hash_alg = undef
+ if $hash_alg eq 'none';
+
+ die "either 'checksum' and 'checksumalg' or none of these have to be specified"
+ if ($checksum && !$hash_alg) || (!$checksum && $hash_alg);
+
+ die "unsupported checksumalg: '$hash_alg'"
+ if $hash_alg && ($hash_alg !~ /^(md5|sha1|sha(224|256|384|512))$/);
+
+ my $req = HTTP::Request->new(HEAD => $url);
+ my $ua = LWP::UserAgent->new();
+ my $res = $ua->request($req);
+
+ die "invalid server response: '" . $res->status_line() . "'"
+ if ($res->code() != 200);
+
+ my $size = $res->header("Content-Length");
+ my $dispo = $res->header("Content-Disposition");
+
+ my $filename_remote;
+
+ if ($dispo && $dispo =~ m/filename=(.+)/) {
+ $filename_remote = $1;
+ } elsif ($url =~ m!^[^?]+/([^?/]*)(?:\?.*)?$!) {
+ $filename_remote = $1;
+ }
+
+ chomp $filename_remote;
+ $filename_remote =~ s/^.*[\/\\]//;
+ $filename_remote =~ s/[^-a-zA-Z0-9_.]/_/g;
+
+ if ($param->{metaonly}) {
+ return {
+ filename => $filename_remote,
+ upid => 0,
+ size => $size + 0,
+ };
+ }
+
+ my $filename = $param->{filename} || $filename_remote;
+ chomp $filename;
+ $filename =~ s/^.*[\/\\]//;
+ $filename =~ s/[^-a-zA-Z0-9_.]/_/g;
+
+ my $path;
+
+ if ($content eq 'iso') {
+ if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
+ raise_param_exc({ filename => "missing '.iso' or '.img' extension" });
+ }
+ $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
+ } elsif ($content eq 'vztmpl') {
+ if ($filename !~ m![^/]+\.tar\.[gx]z$!) {
+ raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" });
+ }
+ $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
+ } else {
+ raise_param_exc({ content => "upload content type '$content' not allowed" });
+ }
+
+ die "storage '$param->{storage}' does not support '$content' content"
+ if !$scfg->{content}->{$content};
+
+ my $dest = "$path/$filename";
+ my $dirname = dirname($dest);
+
+ my $cmd;
+ my $cmd_check;
+ my $cmd_check_flat;
+ my $cmd_del;
+
+ my @curlcmd = ('curl', '-L', '-o', $dest, '-f');
+ push @curlcmd, '--insecure'
+ if $param->{insecure};
+
+ my @check1 = ('echo', $checksum, $dest);
+ my @check2 = ("${hash_alg}sum", '-c');
+
+ my @unlinkcmd = ('rm', '-f', $dest);
+
+ if ($node ne 'localhost' && $node ne PVE::INotify::nodename()) {
+ my $remip = PVE::Cluster::remote_node_ip($node);
+
+ my @ssh_options = ('-o', 'BatchMode=yes');
+
+ my @remcmd = ('/usr/bin/ssh', @ssh_options, $remip, '--');
+
+ eval {
+ # activate remote storage
+ PVE::Tools::run_command([@remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage}]);
+ };
+ die "can't activate storage '$param->{storage}' on node '$node': $@"
+ if $@;
+
+ PVE::Tools::run_command([@remcmd, '/bin/mkdir', '-p', '--', $dirname],
+ errmsg => "mkdir failed");
+
+ $cmd = [@remcmd, @curlcmd, PVE::Tools::shell_quote($url)];
+
+ $cmd_check = [@remcmd, @check1, '|', @check2];
+ $cmd_check_flat = [@remcmd, @check1, '|', @check2];
+
+ $cmd_del = [@remcmd, @unlinkcmd];
+ } else {
+ PVE::Storage::activate_storage($cfg, $param->{storage});
+ File::Path::make_path($dirname);
+
+ $cmd = [@curlcmd, $url];
+
+ $cmd_check = [[@check1], [@check2]];
+ $cmd_check_flat = [@check1, '|', @check2];
+
+ $cmd_del = [@unlinkcmd];
+ }
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting file download from: $url\n";
+ print "target node: $node\n";
+ print "target file: $dest\n";
+ print "file size is: $size\n";
+ print "command: " . join(' ', @$cmd) . "\n";
+
+ eval { PVE::Tools::run_command($cmd, errmsg => 'download failed'); };
+ if (my $err = $@) {
+ PVE::Tools::run_command($cmd_del);
+ die $err;
+ }
+ print "finished file download successfully\n";
+
+ if ($checksum) {
+ print "validating checksum...\n";
+ print "expected $hash_alg checksum is: $checksum\n";
+ print "checksum validation command: " . join(' ', @$cmd_check_flat) . "\n";
+
+ eval { PVE::Tools::run_command($cmd_check, errmsg => 'checksum mismatch'); };
+ if (my $err = $@) {
+ PVE::Tools::run_command($cmd_del);
+ die $err;
+ }
+ print "validated checksum successfully\n";
+ }
+ };
+
+ my $upid = $rpcenv->fork_worker('imgdownload', undef, $user, $worker);
+
+ return {
+ filename => $filename,
+ upid => $upid,
+ size => $size + 0,
+ };
+ }});
+
+
1;
--
2.20.1