* [pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files
2023-07-21 12:23 [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
@ 2023-07-21 12:23 ` Noel Ullreich
2023-07-26 9:34 ` Fabian Grünbichler
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 2/3] add `scan-depth` option to filesystems Noel Ullreich
` (3 subsequent siblings)
4 siblings, 1 reply; 7+ messages in thread
From: Noel Ullreich @ 2023-07-21 12:23 UTC (permalink / raw)
To: pve-devel
This patch allows `get_subdir_files` to recursively call itself, so that
subdirectories of set depth can be searched. We allow searching for
isos, vztmpl and snippets but not backups.
As a security measure, when parsing a given path, parent
directories (`/../`) are forbidden.
The feature is opt-in, i.e. the searchdepth is 0 by default. It can be
changed via the API, the web interface and `pvesm` (see the other
patches).
Signed-off-by: Noel Ullreich <n.ullreich@proxmox.com>
---
src/PVE/Storage.pm | 11 ++++++++
src/PVE/Storage/Plugin.pm | 54 ++++++++++++++++++++++++---------------
2 files changed, 44 insertions(+), 21 deletions(-)
diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
index b99ed35..02abacf 100755
--- a/src/PVE/Storage.pm
+++ b/src/PVE/Storage.pm
@@ -113,6 +113,17 @@ our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i;
our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/;
+our $INTERMEDIATE_SUBDIR_EXT_RE_3 = qr/(?:[0-9A-z\_\-\.]+\/)*[^\/]+/i;
+
+our $SUBDIR_ANY_FILEEXTENSION_EXT_RE_4 = qr/(?:[0-9A-z\_\-\.]+\/)*.+/i;
+
+# '..' is forbidden at the beginning, between two '/' and at the end
+my $dots = quotemeta('..');
+my $beginning = qr!^$dots/!;
+my $between = qr!/$dots/!;
+my $end = qr!/$dots$!;
+our $forbidden_double_dots_re = qr!(?:$beginning|$between|$end)!;
+
# FIXME remove with PVE 8.0, add versioned breaks for pve-manager
our $vztmpl_extension_re = $VZTMPL_EXT_RE_1;
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 9d3b1ae..8831fbb 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -621,6 +621,8 @@ sub parse_name_dir {
sub parse_volname {
my ($class, $volname) = @_;
+ die "volname must not contain parent directories '/../'\n" if $volname =~ $PVE::Storage::forbidden_double_dots_re;
+
if ($volname =~ m!^(\d+)/(\S+)/(\d+)/(\S+)$!) {
my ($basedvmid, $basename) = ($1, $2);
parse_name_dir($basename);
@@ -631,9 +633,9 @@ sub parse_volname {
my ($vmid, $name) = ($1, $2);
my (undef, $format, $isBase) = parse_name_dir($name);
return ('images', $name, $vmid, undef, undef, $isBase, $format);
- } elsif ($volname =~ m!^iso/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!) {
+ } elsif ($volname =~ m!^iso/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)$!) {
return ('iso', $1);
- } elsif ($volname =~ m!^vztmpl/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
+ } elsif ($volname =~ m!^vztmpl/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
return ('vztmpl', $1);
} elsif ($volname =~ m!^rootdir/(\d+)$!) {
return ('rootdir', $1, $1);
@@ -643,7 +645,7 @@ sub parse_volname {
return ('backup', $fn, $2);
}
return ('backup', $fn);
- } elsif ($volname =~ m!^snippets/([^/]+)$!) {
+ } elsif ($volname =~ m!^snippets/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3)$!) {
return ('snippets', $1);
}
@@ -1212,28 +1214,33 @@ sub list_images {
}
# list templates ($tt = <iso|vztmpl|backup|snippets>)
-my $get_subdir_files = sub {
- my ($sid, $path, $tt, $vmid) = @_;
+sub get_subdir_files {
+ my ($sid, $path, $tt, $scfg, $vmid, $remaining_depth) = @_;
+ my $storage_path = $scfg->{path};
+ my $content_dir = $scfg->{"content-dirs"}->{$tt} // $vtype_subdirs->{$tt};
my $res = [];
foreach my $fn (<$path/*>) {
my $st = File::stat::stat($fn);
- next if (!$st || S_ISDIR($st->mode));
+ next if (!$st);
+
+ if (S_ISDIR($st->mode)) {
+ if ($remaining_depth) {
+ push @$res, get_subdir_files($sid, $fn, $tt, $scfg, $vmid, $remaining_depth-1);
+ }
+ next;
+ }
my $info;
if ($tt eq 'iso') {
- next if $fn !~ m!/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!i;
-
+ next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)/;
$info = { volid => "$sid:iso/$1", format => 'iso' };
-
} elsif ($tt eq 'vztmpl') {
- next if $fn !~ m!/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!;
-
+ next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)/;
$info = { volid => "$sid:vztmpl/$1", format => "t$2" };
-
} elsif ($tt eq 'backup') {
next if $fn !~ m!/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!;
my $original = $fn;
@@ -1262,9 +1269,9 @@ my $get_subdir_files = sub {
$info->{protected} = 1 if -e PVE::Storage::protection_file_path($original);
} elsif ($tt eq 'snippets') {
-
+ next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::SUBDIR_ANY_FILEEXTENSION_EXT_RE_4)/;
$info = {
- volid => "$sid:snippets/". basename($fn),
+ volid => "$sid:snippets/$1", #basename($fn),
format => 'snippet',
};
}
@@ -1274,14 +1281,18 @@ my $get_subdir_files = sub {
push @$res, $info;
}
-
return $res;
};
+sub flatten {
+ map { ref eq 'ARRAY' ? flatten(@{$_}) : $_ } @_;
+}
+
# If attributes are set on a volume, they should be included in the result.
# See get_volume_attribute for a list of possible attributes.
sub list_volumes {
my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
+ my $max_depth = $scfg->{'scan-depth'} // 0;
my $res = [];
my $vmlist = PVE::Cluster::get_vmlist();
@@ -1294,17 +1305,19 @@ sub list_volumes {
my $path = $class->get_subdir($scfg, $type);
if ($type eq 'iso' && !defined($vmid)) {
- $data = $get_subdir_files->($storeid, $path, 'iso');
+ $data = get_subdir_files($storeid, $path, 'iso', $scfg, undef, $max_depth);
} elsif ($type eq 'vztmpl'&& !defined($vmid)) {
- $data = $get_subdir_files->($storeid, $path, 'vztmpl');
+ $data = get_subdir_files($storeid, $path , 'vztmpl', $scfg, undef, $max_depth);
} elsif ($type eq 'backup') {
- $data = $get_subdir_files->($storeid, $path, 'backup', $vmid);
+ $data = get_subdir_files($storeid, $path, 'backup', $scfg, $vmid, $max_depth);
} elsif ($type eq 'snippets') {
- $data = $get_subdir_files->($storeid, $path, 'snippets');
+ $data = get_subdir_files($storeid, $path, 'snippets', $scfg, undef, $max_depth);
}
}
- next if !$data;
+ $data = [flatten($data)];
+
+ next if !@$data[0];
foreach my $item (@$data) {
if ($type eq 'images' || $type eq 'rootdir') {
@@ -1322,7 +1335,6 @@ sub list_volumes {
} else {
$item->{content} = $type;
}
-
push @$res, $item;
}
}
--
2.39.2
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files Noel Ullreich
@ 2023-07-26 9:34 ` Fabian Grünbichler
0 siblings, 0 replies; 7+ messages in thread
From: Fabian Grünbichler @ 2023-07-26 9:34 UTC (permalink / raw)
To: Proxmox VE development discussion
On July 21, 2023 2:23 pm, Noel Ullreich wrote:
> This patch allows `get_subdir_files` to recursively call itself, so that
> subdirectories of set depth can be searched. We allow searching for
> isos, vztmpl and snippets but not backups.
>
> As a security measure, when parsing a given path, parent
> directories (`/../`) are forbidden.
>
> The feature is opt-in, i.e. the searchdepth is 0 by default. It can be
> changed via the API, the web interface and `pvesm` (see the other
> patches).
>
> Signed-off-by: Noel Ullreich <n.ullreich@proxmox.com>
> ---
> src/PVE/Storage.pm | 11 ++++++++
> src/PVE/Storage/Plugin.pm | 54 ++++++++++++++++++++++++---------------
> 2 files changed, 44 insertions(+), 21 deletions(-)
>
> diff --git a/src/PVE/Storage.pm b/src/PVE/Storage.pm
> index b99ed35..02abacf 100755
> --- a/src/PVE/Storage.pm
> +++ b/src/PVE/Storage.pm
> @@ -113,6 +113,17 @@ our $VZTMPL_EXT_RE_1 = qr/\.tar\.(gz|xz|zst)/i;
>
> our $BACKUP_EXT_RE_2 = qr/\.(tgz|(?:tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)/;
>
> +our $INTERMEDIATE_SUBDIR_EXT_RE_3 = qr/(?:[0-9A-z\_\-\.]+\/)*[^\/]+/i;
> +
> +our $SUBDIR_ANY_FILEEXTENSION_EXT_RE_4 = qr/(?:[0-9A-z\_\-\.]+\/)*.+/i;
> +
1. common RE pitfall: use a-z instead of A-z, or a-zA-Z and drop the
`/i` - else this matches the characters [\]^` as well, since they sit
between 'Z' and 'a'. not all of them will work in practice, especially
the '\' one would require special handling - but we don't want them
anyhow ;)
2. these two should end with _0 , right? (they don't have any capturing
groups)
3. the first regular expression matches any string that doesn't contain
a '/', or any string containing slashes, where everything up to the last
slash must be from the restricted char set - the name is a bit weird
IMHO ;) maybe it would make more sense to keep the [^\/]+ part out of
it, and leave that at the call site? then it would actually just match
the intermediate sub-dirs, like the name implies (although I would still
drop the `_EXT`, it refers to a file extension, and that is the only
thing it *does not* match at the moment ;))..
4. the second RE basically matches any string of length >=1 ?
if we do the above change, a single RE should be enough (and the one
usage of $SUBDIR_ANY_FILEEXTENSION_EXT_RE_4 could also just be
$INTERMEDIATE_SUBDIR_RE_0[^\/]+ , like the other ones, but without any
specific extension/suffix..). it would also mean that parse_volname and
get_subdir_files accept the same things, instead of the latter allowing
more characters in subdir components (although in practice, its return
value will likely be filtered through parse_volume_id -> parse_volname
anyhow - not sure of *all* the code paths though..)
other than these, the patches look good to me and seem to work as
expected, the above is mostly (except for number 1) code
hygiene/style/naming :) symlinks are also handled transparently as
expected (e.g., no resolved path leaks into path/volume id/.., and the
size is correctly retrieved from the target, dangling symlinks are just
ignored).
> +# '..' is forbidden at the beginning, between two '/' and at the end
> +my $dots = quotemeta('..');
> +my $beginning = qr!^$dots/!;
> +my $between = qr!/$dots/!;
> +my $end = qr!/$dots$!;
> +our $forbidden_double_dots_re = qr!(?:$beginning|$between|$end)!;
> +
> # FIXME remove with PVE 8.0, add versioned breaks for pve-manager
> our $vztmpl_extension_re = $VZTMPL_EXT_RE_1;
>
> diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
> index 9d3b1ae..8831fbb 100644
> --- a/src/PVE/Storage/Plugin.pm
> +++ b/src/PVE/Storage/Plugin.pm
> @@ -621,6 +621,8 @@ sub parse_name_dir {
> sub parse_volname {
> my ($class, $volname) = @_;
>
> + die "volname must not contain parent directories '/../'\n" if $volname =~ $PVE::Storage::forbidden_double_dots_re;
> +
> if ($volname =~ m!^(\d+)/(\S+)/(\d+)/(\S+)$!) {
> my ($basedvmid, $basename) = ($1, $2);
> parse_name_dir($basename);
> @@ -631,9 +633,9 @@ sub parse_volname {
> my ($vmid, $name) = ($1, $2);
> my (undef, $format, $isBase) = parse_name_dir($name);
> return ('images', $name, $vmid, undef, undef, $isBase, $format);
> - } elsif ($volname =~ m!^iso/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!) {
> + } elsif ($volname =~ m!^iso/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)$!) {
> return ('iso', $1);
> - } elsif ($volname =~ m!^vztmpl/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
> + } elsif ($volname =~ m!^vztmpl/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)$!) {
> return ('vztmpl', $1);
> } elsif ($volname =~ m!^rootdir/(\d+)$!) {
> return ('rootdir', $1, $1);
> @@ -643,7 +645,7 @@ sub parse_volname {
> return ('backup', $fn, $2);
> }
> return ('backup', $fn);
> - } elsif ($volname =~ m!^snippets/([^/]+)$!) {
> + } elsif ($volname =~ m!^snippets/($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3)$!) {
> return ('snippets', $1);
> }
>
> @@ -1212,28 +1214,33 @@ sub list_images {
> }
>
> # list templates ($tt = <iso|vztmpl|backup|snippets>)
> -my $get_subdir_files = sub {
> - my ($sid, $path, $tt, $vmid) = @_;
> +sub get_subdir_files {
> + my ($sid, $path, $tt, $scfg, $vmid, $remaining_depth) = @_;
> + my $storage_path = $scfg->{path};
> + my $content_dir = $scfg->{"content-dirs"}->{$tt} // $vtype_subdirs->{$tt};
>
> my $res = [];
>
> foreach my $fn (<$path/*>) {
> my $st = File::stat::stat($fn);
>
> - next if (!$st || S_ISDIR($st->mode));
> + next if (!$st);
> +
> + if (S_ISDIR($st->mode)) {
> + if ($remaining_depth) {
> + push @$res, get_subdir_files($sid, $fn, $tt, $scfg, $vmid, $remaining_depth-1);
> + }
> + next;
> + }
>
> my $info;
>
> if ($tt eq 'iso') {
> - next if $fn !~ m!/([^/]+$PVE::Storage::ISO_EXT_RE_0)$!i;
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::ISO_EXT_RE_0)/;
> $info = { volid => "$sid:iso/$1", format => 'iso' };
> -
> } elsif ($tt eq 'vztmpl') {
> - next if $fn !~ m!/([^/]+$PVE::Storage::VZTMPL_EXT_RE_1)$!;
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::INTERMEDIATE_SUBDIR_EXT_RE_3$PVE::Storage::VZTMPL_EXT_RE_1)/;
> $info = { volid => "$sid:vztmpl/$1", format => "t$2" };
> -
> } elsif ($tt eq 'backup') {
> next if $fn !~ m!/([^/]+$PVE::Storage::BACKUP_EXT_RE_2)$!;
> my $original = $fn;
> @@ -1262,9 +1269,9 @@ my $get_subdir_files = sub {
>
> $info->{protected} = 1 if -e PVE::Storage::protection_file_path($original);
> } elsif ($tt eq 'snippets') {
> -
> + next if $fn !~ m/(?:^$storage_path\/$content_dir\/)($PVE::Storage::SUBDIR_ANY_FILEEXTENSION_EXT_RE_4)/;
> $info = {
> - volid => "$sid:snippets/". basename($fn),
> + volid => "$sid:snippets/$1", #basename($fn),
leftover comment? ;)
> format => 'snippet',
> };
> }
> @@ -1274,14 +1281,18 @@ my $get_subdir_files = sub {
>
> push @$res, $info;
> }
> -
> return $res;
> };
>
> +sub flatten {
> + map { ref eq 'ARRAY' ? flatten(@{$_}) : $_ } @_;
> +}
> +
> # If attributes are set on a volume, they should be included in the result.
> # See get_volume_attribute for a list of possible attributes.
> sub list_volumes {
> my ($class, $storeid, $scfg, $vmid, $content_types) = @_;
> + my $max_depth = $scfg->{'scan-depth'} // 0;
>
> my $res = [];
> my $vmlist = PVE::Cluster::get_vmlist();
> @@ -1294,17 +1305,19 @@ sub list_volumes {
> my $path = $class->get_subdir($scfg, $type);
>
> if ($type eq 'iso' && !defined($vmid)) {
> - $data = $get_subdir_files->($storeid, $path, 'iso');
> + $data = get_subdir_files($storeid, $path, 'iso', $scfg, undef, $max_depth);
> } elsif ($type eq 'vztmpl'&& !defined($vmid)) {
> - $data = $get_subdir_files->($storeid, $path, 'vztmpl');
> + $data = get_subdir_files($storeid, $path , 'vztmpl', $scfg, undef, $max_depth);
> } elsif ($type eq 'backup') {
> - $data = $get_subdir_files->($storeid, $path, 'backup', $vmid);
> + $data = get_subdir_files($storeid, $path, 'backup', $scfg, $vmid, $max_depth);
> } elsif ($type eq 'snippets') {
> - $data = $get_subdir_files->($storeid, $path, 'snippets');
> + $data = get_subdir_files($storeid, $path, 'snippets', $scfg, undef, $max_depth);
> }
> }
>
> - next if !$data;
> + $data = [flatten($data)];
> +
> + next if !@$data[0];
>
> foreach my $item (@$data) {
> if ($type eq 'images' || $type eq 'rootdir') {
> @@ -1322,7 +1335,6 @@ sub list_volumes {
> } else {
> $item->{content} = $type;
> }
> -
> push @$res, $item;
> }
> }
> --
> 2.39.2
>
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>
>
^ permalink raw reply [flat|nested] 7+ messages in thread
* [pve-devel] [PATCH pve-storage v4 2/3] add `scan-depth` option to filesystems
2023-07-21 12:23 [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files Noel Ullreich
@ 2023-07-21 12:23 ` Noel Ullreich
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 3/3] update test for recursive subdir search Noel Ullreich
` (2 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Noel Ullreich @ 2023-07-21 12:23 UTC (permalink / raw)
To: pve-devel
Add the `subdir-depth` to the filesystems that can hold
isos/vztmpl/snippets.
Signed-off-by: Noel Ullreich <n.ullreich@proxmox.com>
---
src/PVE/Storage/CIFSPlugin.pm | 1 +
src/PVE/Storage/CephFSPlugin.pm | 1 +
src/PVE/Storage/DirPlugin.pm | 1 +
src/PVE/Storage/GlusterfsPlugin.pm | 1 +
src/PVE/Storage/NFSPlugin.pm | 1 +
src/PVE/Storage/Plugin.pm | 7 +++++++
6 files changed, 12 insertions(+)
diff --git a/src/PVE/Storage/CIFSPlugin.pm b/src/PVE/Storage/CIFSPlugin.pm
index 4cab2e1..34a6d3a 100644
--- a/src/PVE/Storage/CIFSPlugin.pm
+++ b/src/PVE/Storage/CIFSPlugin.pm
@@ -155,6 +155,7 @@ sub options {
'create-subdirs' => { optional => 1 },
bwlimit => { optional => 1 },
preallocation => { optional => 1 },
+ 'scan-depth' => { optional => 1},
options => { optional => 1 },
};
}
diff --git a/src/PVE/Storage/CephFSPlugin.pm b/src/PVE/Storage/CephFSPlugin.pm
index 8aad518..8f22973 100644
--- a/src/PVE/Storage/CephFSPlugin.pm
+++ b/src/PVE/Storage/CephFSPlugin.pm
@@ -156,6 +156,7 @@ sub options {
'prune-backups' => { optional => 1 },
'max-protected-backups' => { optional => 1 },
'fs-name' => { optional => 1 },
+ 'scan-depth' => { optional => 1},
};
}
diff --git a/src/PVE/Storage/DirPlugin.pm b/src/PVE/Storage/DirPlugin.pm
index 2efa8d5..33ea4f7 100644
--- a/src/PVE/Storage/DirPlugin.pm
+++ b/src/PVE/Storage/DirPlugin.pm
@@ -80,6 +80,7 @@ sub options {
is_mountpoint => { optional => 1 },
bwlimit => { optional => 1 },
preallocation => { optional => 1 },
+ 'scan-depth' => { optional => 1},
};
}
diff --git a/src/PVE/Storage/GlusterfsPlugin.pm b/src/PVE/Storage/GlusterfsPlugin.pm
index 2b7f9e1..a249a04 100644
--- a/src/PVE/Storage/GlusterfsPlugin.pm
+++ b/src/PVE/Storage/GlusterfsPlugin.pm
@@ -141,6 +141,7 @@ sub options {
'create-subdirs' => { optional => 1 },
bwlimit => { optional => 1 },
preallocation => { optional => 1 },
+ 'scan-depth' => { optional => 1},
};
}
diff --git a/src/PVE/Storage/NFSPlugin.pm b/src/PVE/Storage/NFSPlugin.pm
index f2e4c0d..0d26432 100644
--- a/src/PVE/Storage/NFSPlugin.pm
+++ b/src/PVE/Storage/NFSPlugin.pm
@@ -91,6 +91,7 @@ sub options {
'create-subdirs' => { optional => 1 },
bwlimit => { optional => 1 },
preallocation => { optional => 1 },
+ 'scan-depth' => { optional => 1},
};
}
diff --git a/src/PVE/Storage/Plugin.pm b/src/PVE/Storage/Plugin.pm
index 8831fbb..46a9bd0 100644
--- a/src/PVE/Storage/Plugin.pm
+++ b/src/PVE/Storage/Plugin.pm
@@ -192,6 +192,13 @@ my $defaultData = {
type => "string", format => "pve-dir-override-list",
optional => 1,
},
+ 'scan-depth' => {
+ description => "Maximal depth of subdirectories to look though when
+ searching for isos/container templates/snippets in a directory",
+ type => 'integer',
+ default => 0,
+ optional => 1,
+ },
options => {
description => "NFS/CIFS mount options (see 'man nfs' or 'man mount.cifs')",
type => 'string',
--
2.39.2
^ permalink raw reply [flat|nested] 7+ messages in thread* [pve-devel] [PATCH pve-storage v4 3/3] update test for recursive subdir search
2023-07-21 12:23 [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 1/3] recursively go through subdirs to find files Noel Ullreich
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 2/3] add `scan-depth` option to filesystems Noel Ullreich
@ 2023-07-21 12:23 ` Noel Ullreich
2023-07-21 12:23 ` [pve-devel] [PATCH pve-manager v4 1/1] ui: add field to set subdir-depth in web interface Noel Ullreich
2023-07-25 8:00 ` [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
4 siblings, 0 replies; 7+ messages in thread
From: Noel Ullreich @ 2023-07-21 12:23 UTC (permalink / raw)
To: pve-devel
Tests now also test if traversing subdirectories works. That means
checking if:
- parent directories in paths are caught
- checking that nested files are found
- checking that files below the maximum depth set are not found
Signed-off-by: Noel Ullreich <n.ullreich@proxmox.com>
---
src/test/filesystem_path_test.pm | 18 +++++++++
src/test/list_volumes_test.pm | 68 ++++++++++++++++++++++++++++++++
src/test/parse_volname_test.pm | 40 +++++++++++++++++++
3 files changed, 126 insertions(+)
diff --git a/src/test/filesystem_path_test.pm b/src/test/filesystem_path_test.pm
index c1b6d90..35a1d4e 100644
--- a/src/test/filesystem_path_test.pm
+++ b/src/test/filesystem_path_test.pm
@@ -56,6 +56,24 @@ my $tests = [
'iso'
],
},
+ {
+ volname => 'iso/a/b/c/my-awesome-proxmox.iso',
+ snapname => undef,
+ expected => [
+ "$path/template/iso/a/b/c/my-awesome-proxmox.iso",
+ undef,
+ 'iso'
+ ],
+ },
+ {
+ volname => 'vztmpl/a/b/c/my-awesome-template.tar.gz',
+ snapname => undef,
+ expected => [
+ "$path/template/cache/a/b/c/my-awesome-template.tar.gz",
+ undef,
+ 'vztmpl'
+ ],
+ },
{
volname => "backup/vzdump-qemu-1234-2020_03_30-21_12_40.vma",
snapname => undef,
diff --git a/src/test/list_volumes_test.pm b/src/test/list_volumes_test.pm
index d155cb9..bfc8582 100644
--- a/src/test/list_volumes_test.pm
+++ b/src/test/list_volumes_test.pm
@@ -74,6 +74,7 @@ my $scfg = {
'snippets' => 1,
'backup' => 1,
},
+ 'scan-depth' => 10,
};
# The test cases are comprised of an arry of hashes with the following keys:
@@ -448,6 +449,73 @@ my @tests = (
],
expected => [], # returns empty list
},
+ {
+ descripton => 'VMID: none, subdirectory',
+ # find files in subdirectories, test if maximum depth is working
+ vmid => undef,
+ files => [
+ "$storage_dir/template/cache/dir1/dir2/uwuntu-3.10-default_20190626_amd64.tar.xz",
+ "$storage_dir/template/iso/a/b/c/installation-disk.iso",
+ "$storage_dir/template/iso/a/b/c/d.d/yet-again-an-installation-disk.iso",
+ "$storage_dir/template/iso/a/b/template/iso/c/d/another-installation-disk.iso",
+ "$storage_dir/template/iso/1/2/3/4/5/6/7/8/9/10//wunkus.iso", # test depth
+ "$storage_dir/template/iso/1/2/3/4/5/6/7/8/9/10/11/wunkus.iso", # this should not match
+ "$storage_dir/template/iso/a/../somedir/little.iso",
+ "$storage_dir/template/iso/a/../../somedir/little.iso", # this should not match
+ "$storage_dir/snippets/a/b/c/d/userconfig.yaml",
+ ],
+ expected => [
+ {
+ 'content' => 'vztmpl',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'txz',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:vztmpl/dir1/dir2/uwuntu-3.10-default_20190626_amd64.tar.xz',
+ },
+ {
+ 'content' => 'iso',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'iso',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:iso/1/2/3/4/5/6/7/8/9/10/wunkus.iso',
+ },
+ {
+ 'content' => 'iso',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'iso',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:iso/a/b/c/d.d/yet-again-an-installation-disk.iso',
+ },
+ {
+ 'content' => 'iso',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'iso',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:iso/a/b/c/installation-disk.iso',
+ },
+ {
+ 'content' => 'iso',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'iso',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:iso/a/b/template/iso/c/d/another-installation-disk.iso',
+ },
+ {
+ 'content' => 'iso',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'iso',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:iso/somedir/little.iso',
+ },
+ {
+ 'content' => 'snippets',
+ 'ctime' => DEFAULT_CTIME,
+ 'format' => 'snippet',
+ 'size' => DEFAULT_SIZE,
+ 'volid' => 'local:snippets/a/b/c/d/userconfig.yaml',
+ },
+ ],
+ }
);
diff --git a/src/test/parse_volname_test.pm b/src/test/parse_volname_test.pm
index d6ac885..737ae9e 100644
--- a/src/test/parse_volname_test.pm
+++ b/src/test/parse_volname_test.pm
@@ -31,11 +31,26 @@ my $tests = [
volname => 'iso/some-installation-disk.iso',
expected => ['iso', 'some-installation-disk.iso'],
},
+ {
+ description => 'ISO image, iso, nested',
+ volname => 'iso/a/b/c/some-installation-disk.iso',
+ expected => ['iso', 'a/b/c/some-installation-disk.iso'],
+ },
{
description => 'ISO image, img',
volname => 'iso/some-other-installation-disk.img',
expected => ['iso', 'some-other-installation-disk.img'],
},
+ {
+ description => 'ISO image, img, nested',
+ volname => 'iso/a/b/c/some-other-installation-disk.img',
+ expected => ['iso', 'a/b/c/some-other-installation-disk.img'],
+ },
+ {
+ description => 'ISO image, img, nested with dots',
+ volname => 'iso/a/b.b/c/some-other-installation-disk.img',
+ expected => ['iso', 'a/b.b/c/some-other-installation-disk.img'],
+ },
#
# container templates
#
@@ -44,11 +59,21 @@ my $tests = [
volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
expected => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.gz'],
},
+ {
+ description => 'Container template tar.gz nested',
+ volname => 'vztmpl/a/b/c/debian-10.0-standard_10.0-1_amd64.tar.gz',
+ expected => ['vztmpl', 'a/b/c/debian-10.0-standard_10.0-1_amd64.tar.gz'],
+ },
{
description => 'Container template tar.xz',
volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.xz',
expected => ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.xz'],
},
+ {
+ description => 'Container template tar.xz nested',
+ volname => 'vztmpl/a/b/c/debian-10.0-standard_10.0-1_amd64.tar.xz',
+ expected => ['vztmpl', 'a/b/c/debian-10.0-standard_10.0-1_amd64.tar.xz'],
+ },
#
# container rootdir
#
@@ -123,6 +148,21 @@ my $tests = [
volname => "$vmid/base-$vmid-disk-0.qcow2/ssss/vm-$vmid-disk-0.qcow2",
expected => "unable to parse volume filename 'base-$vmid-disk-0.qcow2/ssss/vm-$vmid-disk-0.qcow2'\n",
},
+ {
+ description => 'Failed match: ISO image, forbidden double dots between two /',
+ volname => 'iso/a/../c/some-installation-disk.iso',
+ expected => "volname must not contain parent directories '/../'\n",
+ },
+ {
+ description => 'Failed match: ISO image, forbidden double dots in the beginning',
+ volname => '../iso/some-installation-disk.iso',
+ expected => "volname must not contain parent directories '/../'\n",
+ },
+ {
+ description => 'Failed match: ISO image, forbidden double dots in the end',
+ volname => 'iso/some-installation-disk.iso/..',
+ expected => "volname must not contain parent directories '/../'\n",
+ },
];
# create more test cases for VM disk images matches
--
2.39.2
^ permalink raw reply [flat|nested] 7+ messages in thread* [pve-devel] [PATCH pve-manager v4 1/1] ui: add field to set subdir-depth in web interface
2023-07-21 12:23 [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
` (2 preceding siblings ...)
2023-07-21 12:23 ` [pve-devel] [PATCH pve-storage v4 3/3] update test for recursive subdir search Noel Ullreich
@ 2023-07-21 12:23 ` Noel Ullreich
2023-07-25 8:00 ` [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
4 siblings, 0 replies; 7+ messages in thread
From: Noel Ullreich @ 2023-07-21 12:23 UTC (permalink / raw)
To: pve-devel
When adding or editing a storage device in Datacenter->Storage in the
web interface, the subdirectory depth can be set in the advanced menu.
Signed-off-by: Noel Ullreich <n.ullreich@proxmox.com>
---
www/manager6/storage/Base.js | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/www/manager6/storage/Base.js b/www/manager6/storage/Base.js
index c8c735f3..e79ec6d6 100644
--- a/www/manager6/storage/Base.js
+++ b/www/manager6/storage/Base.js
@@ -48,6 +48,7 @@ Ext.define('PVE.panel.StorageBase', {
name: 'enable',
checked: true,
uncheckedValue: 0,
+ minValue: 0,
fieldLabel: gettext('Enable'),
},
);
@@ -63,13 +64,23 @@ Ext.define('PVE.panel.StorageBase', {
deleteEmpty: !me.isCreate,
value: '__default__',
};
+ const recursionDepth = {
+ xtype: 'proxmoxintegerfield',
+ name: 'scan-depth',
+ fieldLabel: gettext('Directory Scan Depth'),
+ allowBlank: false,
+ value: 0,
+ minValue: 0,
+ };
me.advancedColumn1 = me.advancedColumn1 || [];
me.advancedColumn2 = me.advancedColumn2 || [];
if (me.advancedColumn2.length < me.advancedColumn1.length) {
me.advancedColumn2.unshift(preallocSelector);
+ me.advancedColumn2.unshift(recursionDepth);
} else {
me.advancedColumn1.unshift(preallocSelector);
+ me.advancedColumn1.unshift(recursionDepth);
}
}
--
2.39.2
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs
2023-07-21 12:23 [pve-devel] [PATCH pve-storage v4; pve-manager 0/4] fix #623: show isos/vztmpl/snippets in subdirs Noel Ullreich
` (3 preceding siblings ...)
2023-07-21 12:23 ` [pve-devel] [PATCH pve-manager v4 1/1] ui: add field to set subdir-depth in web interface Noel Ullreich
@ 2023-07-25 8:00 ` Noel Ullreich
4 siblings, 0 replies; 7+ messages in thread
From: Noel Ullreich @ 2023-07-25 8:00 UTC (permalink / raw)
To: pve-devel
Forgot to add to the changelog that symlinked files now also show the
size of the file not of the symlink
On 21-07-2023 14:23, Noel Ullreich wrote:
> This patch fixes #623, allowing isos/vztmpl/snippets in subdirectories.
> This feature is opt-in and can be set from the API, web interface or
> with `pvesm`.
>
> changes from v3:
> * moved regex that is often used into variables
> * enabled going through symlinked directories as well.
> NB: there is no logic in this patch that looks for cycles, (so a symlink
> to the directory it is in will recurse). This is not a security concern
> since it can only recurse as many times as the depth has been set to, it
> might however lead to isos/templates/etc appearing multiple times with
> cycles in their paths.
> Imo, while it is ugly, it is an edgecase of a niche feature that
> doesn't break anything, so it's ok.
> * changed name of parameter from subdir-depth to scan-depth
>
> Noel Ullreich (3):
> recursively go through subdirs to find files
> add `scan-depth` option to filesystems
> update test for recursive subdir search
>
> src/PVE/Storage.pm | 11 +++++
> src/PVE/Storage/CIFSPlugin.pm | 1 +
> src/PVE/Storage/CephFSPlugin.pm | 1 +
> src/PVE/Storage/DirPlugin.pm | 1 +
> src/PVE/Storage/GlusterfsPlugin.pm | 1 +
> src/PVE/Storage/NFSPlugin.pm | 1 +
> src/PVE/Storage/Plugin.pm | 61 ++++++++++++++++++---------
> src/test/filesystem_path_test.pm | 18 ++++++++
> src/test/list_volumes_test.pm | 68 ++++++++++++++++++++++++++++++
> src/test/parse_volname_test.pm | 40 ++++++++++++++++++
> 10 files changed, 182 insertions(+), 21 deletions(-)
>
> Noel Ullreich (1):
> ui: add field to set subdir-depth in web interface
>
> www/manager6/storage/Base.js | 11 +++++++++++
> 1 file changed, 11 insertions(+)
>
>
^ permalink raw reply [flat|nested] 7+ messages in thread