From: Lorenz Stechauner <l.stechauner@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH v2 storage 1/2] fix #1710: add retrieve method for storage
Date: Mon, 3 May 2021 12:20:45 +0200 [thread overview]
Message-ID: <20210503102045.134007-1-l.stechauner@proxmox.com> (raw)
In-Reply-To: <20210503102016.133870-1-l.stechauner@proxmox.com>
Users are now able to download/retrieve any .iso/... file onto their
storages and verify file integrity with checksums.
Signed-off-by: Lorenz Stechauner <l.stechauner@proxmox.com>
---
PVE/API2/Storage/Status.pm | 156 +++++++++++++++++++++++++++++++++++--
PVE/Storage.pm | 10 +++
2 files changed, 161 insertions(+), 5 deletions(-)
diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm
index 897b4a7..a64101f 100644
--- a/PVE/API2/Storage/Status.pm
+++ b/PVE/API2/Storage/Status.pm
@@ -5,6 +5,8 @@ use warnings;
use File::Path;
use File::Basename;
+use HTTP::Request;
+use LWP::UserAgent;
use PVE::Tools;
use PVE::INotify;
use PVE::Cluster;
@@ -412,11 +414,7 @@ __PACKAGE__->register_method ({
my $size = -s $tmpfilename;
die "temporary file '$tmpfilename' does not exist\n" if !defined($size);
- my $filename = $param->{filename};
-
- chomp $filename;
- $filename =~ s/^.*[\/\\]//;
- $filename =~ s/[^-a-zA-Z0-9_.]/_/g;
+ my $filename = PVE::Storage::normalize_content_filename($param->{filename});
my $path;
@@ -497,4 +495,152 @@ __PACKAGE__->register_method ({
return $upid;
}});
+__PACKAGE__->register_method({
+ name => 'retrieve',
+ path => '{storage}/retrieve',
+ method => 'POST',
+ description => "Download templates and ISO images by using an URL.",
+ proxyto => 'node',
+ permissions => {
+ check => [ 'and',
+ ['perm', '/storage/{storage}', [ 'Datastore.AllocateTemplate' ]],
+ ['perm', '/', [ 'Sys.Audit', 'Sys.Modify' ]],
+ ],
+ },
+ protected => 1,
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ storage => get_standard_option('pve-storage-id'),
+ url => {
+ description => "The URL to retrieve the file from.",
+ type => 'string',
+ },
+ content => {
+ description => "Content type.",
+ type => 'string', format => 'pve-storage-content',
+ },
+ filename => {
+ description => "The name of the file to create. Alternatively the file name given by the server will be used.",
+ type => 'string',
+ },
+ checksum => {
+ description => "The expected checksum of the file.",
+ type => 'string',
+ requires => 'checksumalg',
+ optional => 1,
+ },
+ checksumalg => {
+ description => "The algorithm to claculate the checksum of the file.",
+ type => 'string', enum => ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512'],
+ requires => 'checksum',
+ optional => 1,
+ },
+ insecure => {
+ description => "Allow TLS certificates to be invalid.",
+ type => 'boolean',
+ optional => 1,
+ }
+ },
+ },
+ returns => { type => "string" },
+ code => sub {
+ my ($param) = @_;
+
+ my $rpcenv = PVE::RPCEnvironment::get();
+
+ my $user = $rpcenv->get_user();
+
+ my $cfg = PVE::Storage::config();
+
+ my $node = $param->{node};
+ my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, $node);
+
+ die "can't upload to storage type '$scfg->{type}'"
+ if !defined($scfg->{path});
+
+ my $content = $param->{content};
+
+ my $url = $param->{url};
+
+ die "invalid https or http url"
+ if $url !~ qr!^https?://!;
+
+ my $checksum = $param->{checksum};
+ my $hash_alg = $param->{checksumalg};
+
+ my $filename = PVE::Storage::normalize_content_filename($param->{filename});
+
+ my $path;
+
+ # MIME type is checked in front end only
+ # this check is omitted here intentionally and replaced by file extension check
+ if ($content eq 'iso') {
+ if ($filename !~ m![^/]+$PVE::Storage::iso_extension_re$!) {
+ raise_param_exc({ filename => "missing '.iso' or '.img' extension" });
+ }
+ $path = PVE::Storage::get_iso_dir($cfg, $param->{storage});
+ } elsif ($content eq 'vztmpl') {
+ if ($filename !~ m![^/]+\.tar\.[gx]z$!) {
+ raise_param_exc({ filename => "missing '.tar.gz' or '.tar.xz' extension" });
+ }
+ $path = PVE::Storage::get_vztmpl_dir($cfg, $param->{storage});
+ } else {
+ raise_param_exc({ content => "upload content type '$content' not allowed" });
+ }
+
+ die "storage '$param->{storage}' does not support '$content' content"
+ if !$scfg->{content}->{$content};
+
+ my $dest = "$path/$filename";
+
+ PVE::Storage::activate_storage($cfg, $param->{storage});
+ File::Path::make_path($path);
+
+ # -L follows redirects
+ # -f silent fail on error
+ my @curlcmd = ('curl', '-L', '-o', $dest, '-f');
+ push @curlcmd, '--insecure'
+ if $param->{insecure};
+
+ my $cmd = [@curlcmd, $url];
+
+ my $cmd_check = [ [ 'echo', $checksum, $dest ], [ "${hash_alg}sum", '-c' ] ];
+ my $cmd_check_flat = [ 'echo', $checksum, $dest, '|', "${hash_alg}sum", '-c' ]; # only used for logging
+
+ my $worker = sub {
+ my $upid = shift;
+
+ print "starting file download from: $url\n";
+ print "target node: $node\n";
+ print "target file: $dest\n";
+ print "command: " . join(' ', @$cmd) . "\n";
+
+ eval { PVE::Tools::run_command($cmd, errmsg => 'download failed'); };
+ if (my $err = $@) {
+ unlink $dest;
+ die $err;
+ }
+ print "finished file download successfully\n";
+
+ if ($checksum) {
+ print "validating checksum...\n";
+ print "expected $hash_alg checksum is: $checksum\n";
+ print cmd_check print "checksum validation command: " . join(' ', @$cmd_check_flat) . "\n";
+
+ eval { PVE::Tools::run_command($cmd_check, errmsg => 'checksum mismatch'); };
+ if (my $err = $@) {
+ unlink $dest;
+ die $err;
+ }
+ print "validated checksum successfully\n";
+ }
+ };
+
+ my $upid = $rpcenv->fork_worker('imgdownload', undef, $user, $worker);
+
+ return $upid;
+ }});
+
1;
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 122c3e9..d57fd43 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1931,4 +1931,14 @@ sub assert_sid_unused {
return undef;
}
+sub normalize_content_filename {
+ my ($filename) = @_;
+
+ chomp $filename;
+ $filename =~ s/^.*[\/\\]//;
+ $filename =~ s/[^-a-zA-Z0-9_.]/_/g;
+
+ return $filename;
+}
+
1;
--
2.20.1
next prev parent reply other threads:[~2021-05-03 10:20 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-03 10:20 [pve-devel] [PATCH-SERIES v2 manager/storage 0/2] fix #1710: add retrieve method for Lorenz Stechauner
2021-05-03 10:20 ` Lorenz Stechauner [this message]
2021-05-03 10:21 ` [pve-devel] [PATCH v2 manager 2/2] fix #1710: add retrieve from url button for storage Lorenz Stechauner
2021-05-04 8:55 ` [pve-devel] [PATCH-SERIES v3 manager/storage 0/2] " Lorenz Stechauner
2021-05-04 8:56 ` [pve-devel] [PATCH v3 storage 1/2] fix #1710: add retrieve method " Lorenz Stechauner
2021-05-04 9:31 ` Thomas Lamprecht
2021-05-04 8:57 ` [pve-devel] [PATCH v3 manager 2/2] fix #1710: add retrieve from url button " Lorenz Stechauner
2021-05-04 9:47 ` Thomas Lamprecht
2021-05-06 9:10 ` [pve-devel] [PATCH-SERIES v4 manager/common/storage 0/7] fix #1710: add download from url button Lorenz Stechauner
2021-05-06 9:10 ` [pve-devel] [PATCH v4 manager 1/7] api: nodes: add query_url_metadata method Lorenz Stechauner
2021-05-06 9:11 ` [pve-devel] [PATCH v4 common 2/7] tools: add download_file_from_url Lorenz Stechauner
2021-05-06 10:04 ` Oguz Bektas
2021-05-06 12:15 ` Thomas Lamprecht
2021-05-06 12:17 ` Thomas Lamprecht
2021-05-06 9:11 ` [pve-devel] [PATCH v4 manager 3/7] api: nodes: refactor aplinfo to use common download function Lorenz Stechauner
2021-05-06 9:11 ` [pve-devel] [PATCH v4 storage 4/7] status: add download_url method Lorenz Stechauner
2021-05-06 9:23 ` [pve-devel] [PATCH v5 storage] " Lorenz Stechauner
2021-05-06 9:11 ` [pve-devel] [PATCH v4 manager 5/7] ui: add HashAlgorithmSelector Lorenz Stechauner
2021-05-06 9:11 ` [pve-devel] [PATCH v4 manager 6/7] ui: Utils: change download task format Lorenz Stechauner
2021-05-06 9:11 ` [pve-devel] [PATCH v4 manager 7/7] fix #1710: ui: storage: add download from url button Lorenz Stechauner
2021-05-06 13:15 ` [pve-devel] [PATCH v4 manager 1/7] api: nodes: add query_url_metadata method Dominik Csapak
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210503102045.134007-1-l.stechauner@proxmox.com \
--to=l.stechauner@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox