From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D186BA183F for ; Thu, 15 Jun 2023 14:04:04 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B30F4266A0 for ; Thu, 15 Jun 2023 14:03:34 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 15 Jun 2023 14:03:33 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7387F4584E for ; Thu, 15 Jun 2023 14:03:33 +0200 (CEST) From: Noel Ullreich To: pve-devel@lists.proxmox.com Date: Thu, 15 Jun 2023 14:03:26 +0200 Message-Id: <20230615120329.28764-2-n.ullreich@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230615120329.28764-1-n.ullreich@proxmox.com> References: <20230615120329.28764-1-n.ullreich@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.056 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [plugin.pm, pluggin.pm, storage.pm] Subject: [pve-devel] [PATCH pve-storage v3 1/3] recursively go through subdirs to find files X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Jun 2023 12:04:04 -0000 This patch allows `get_subdir_files` to recursively call itself, so that subdirectories of set depth can be searched. We allow searching for isos, vztmpl and snippets but not backups. As a security measure, when parsing a given path, parent directories (`/../`) are forbidden. The feature is opt-in, i.e. the searchdepth is 0 by default. It can be changed via the API, the web interface and `pvesm` (see the other patches). Signed-off-by: Noel Ullreich --- changes from v2: * fixed the path of the volid for snippets in Pluggin.pm (thanks @Markus) src/PVE/Storage.pm | 7 +++++ src/PVE/Storage/Plugin.pm | 56 ++++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm index b99ed35..250a892 100755 --- a/src/PVE/Storage.pm +++ b/src/PVE/Storage.pm @@ -113,6 +113,13 @@ our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i; our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/; +# '..' is forbidden at the beginning, between two '/' and at the end +my $dots = quotemeta('..'); +my $beginning = qr!^$dots/!; +my $between = qr!/$dots/!; +my $end = qr!/$dots$!; +our $forbidden_double_dots_re = qr!(?:$beginning|$between|$end)!; + # FIXME remove with PVE 8.0, add versioned breaks for pve-manager our $vztmpl_extension_re = $VZTMPL_EXT_RE_1; diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm index ab6b675..87a08c3 100644 --- a/src/PVE/Storage/Plugin.pm +++ b/src/PVE/Storage/Plugin.pm @@ -621,6 +621,8 @@ sub parse_name_dir { sub parse_volname { my ($class, $volname) = @_; + die "volname must not contain parent directories '/../'\n" if $volname =~ $PVE::Storage::forbidden_double_dots_re; + if ($volname =~ m!^(\d+)/(\S+)/(\d+)/(\S+)$!) { my ($basedvmid, $basename) = ($1, $2); parse_name_dir($basename); @@ -631,9 +633,9 @@ sub parse_volname { my ($vmid, $name) = ($1, $2); my (undef, $format, $isBase) = parse_name_dir($name); return ('images', $name, $vmid, undef, undef, $isBase, $format); - } elsif ($volname =~ m!^iso/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!) { + } elsif ($volname =~ m!^iso/((?:[0-9A-z\_\-\.]+\/)*[^\/]+$PVE::Storage::ISO_EXT_RE_0)$!) { return ('iso', $1); - } elsif ($volname =~ m!^vztmpl/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) { + } elsif ($volname =~ m!^vztmpl/((?:[0-9A-z\_\-\.]+\/)*[^\/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) { return ('vztmpl', $1); } elsif ($volname =~ m!^rootdir/(\d+)$!) { return ('rootdir', $1, $1); @@ -643,7 +645,7 @@ sub parse_volname { return ('backup', $fn, $2); } return ('backup', $fn); - } elsif ($volname =~ m!^snippets/([^/]+)$!) { + } elsif ($volname =~ m!^snippets/((?:[0-9A-z\_\-\.]+\/)*[^\/]+)$!) { return ('snippets', $1); } @@ -1212,28 +1214,33 @@ sub list_images { } # list templates ($tt = ) -my $get_subdir_files = sub { - my ($sid, $path, $tt, $vmid) = @_; +sub get_subdir_files { + my ($sid, $path, $tt, $scfg, $vmid, $remaining_depth) = @_; + my $storage_path = $scfg->{path}; + my $content_dir = $scfg->{"content-dirs"}->{$tt} // $vtype_subdirs->{$tt}; my $res = []; foreach my $fn (<$path/*>) { - my $st = File::stat::stat($fn); + my $st = File::stat::lstat($fn); + + next if (!$st); - next if (!$st || S_ISDIR($st->mode)); + if (S_ISDIR($st->mode)) { + if ($remaining_depth) { + push @$res, get_subdir_files($sid, $fn, $tt, $scfg, $vmid, $remaining_depth-1); + } + next; + } my $info; if ($tt eq 'iso') { - next if $fn !~ m!/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!i; - + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)((?:[0-9A-z\_\-\.]+\/)*[^\/]+$PVE::Storage::ISO_EXT_RE_0)/; $info = { volid => "$sid:iso/$1", format => 'iso' }; - } elsif ($tt eq 'vztmpl') { - next if $fn !~ m!/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!; - + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)((?:[0-9A-z\_\-\.]+\/)*[^\/]+$PVE::Storage::VZTMPL_EXT_RE_1)/; $info = { volid => "$sid:vztmpl/$1", format => "t$2" }; - } elsif ($tt eq 'backup') { next if $fn !~ m!/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!; my $original = $fn; @@ -1262,9 +1269,9 @@ my $get_subdir_files = sub { $info->{protected} = 1 if -e PVE::Storage::protection_file_path($original); } elsif ($tt eq 'snippets') { - + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)((?:[0-9A-z\_\-\.]+\/)*.+)/; $info = { - volid => "$sid:snippets/". basename($fn), + volid => "$sid:snippets/$1", #basename($fn), format => 'snippet', }; } @@ -1274,14 +1281,18 @@ my $get_subdir_files = sub { push @$res, $info; } - return $res; }; +sub flatten { + map { ref eq 'ARRAY' ? flatten(@{$_}) : $_ } @_; +} + # If attributes are set on a volume, they should be included in the result. # See get_volume_attribute for a list of possible attributes. sub list_volumes { my ($class, $storeid, $scfg, $vmid, $content_types) = @_; + my $max_depth = $scfg->{'subdir-depth'} // 0; my $res = []; my $vmlist = PVE::Cluster::get_vmlist(); @@ -1294,17 +1305,19 @@ sub list_volumes { my $path = $class->get_subdir($scfg, $type); if ($type eq 'iso' && !defined($vmid)) { - $data = $get_subdir_files->($storeid, $path, 'iso'); + $data = get_subdir_files($storeid, $path, 'iso', $scfg, undef, $max_depth); } elsif ($type eq 'vztmpl'&& !defined($vmid)) { - $data = $get_subdir_files->($storeid, $path, 'vztmpl'); + $data = get_subdir_files($storeid, $path , 'vztmpl', $scfg, undef, $max_depth); } elsif ($type eq 'backup') { - $data = $get_subdir_files->($storeid, $path, 'backup', $vmid); + $data = get_subdir_files($storeid, $path, 'backup', $scfg, $vmid, $max_depth); } elsif ($type eq 'snippets') { - $data = $get_subdir_files->($storeid, $path, 'snippets'); + $data = get_subdir_files($storeid, $path, 'snippets', $scfg, undef, $max_depth); } } - next if !$data; + $data = [flatten($data)]; + + next if !@$data[0]; foreach my $item (@$data) { if ($type eq 'images' || $type eq 'rootdir') { @@ -1322,7 +1335,6 @@ sub list_volumes { } else { $item->{content} = $type; } - push @$res, $item; } } -- 2.30.2