Re: [pve-devel] [PATCH v5 container] Add device passthrough

[Wolfgang Bumiller <w.bumiller@proxmox.com>]

On Thu, Nov 16, 2023 at 12:50:44PM +0100, Filip Schauer wrote:
> Add a dev[n] argument to the container config to pass devices through to
> a container. A device can be passed by its path. Additionally the access
> mode, uid and gid can be specified through their respective properties.
> 
> Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
> ---
> Changes since v4:
> * Rename device lists to "mounts" and "devices" respectively
>   and move them into the tmpfs mounted to the passthrough directory
> * Add detailed $! error messages
> * Enforce stricter config formatting on passthrough devices
> * Combine regex in verify_lxc_dev_string and describe what it does in
>   a comment
> * Remove unnecessary int() in map_ct_id_to_host since Perl automatically
>   parses a string as a number when compared to a number
> * Cosmetic changes (foreach --> for, unless --> if)
> 
>  src/PVE/LXC.pm            | 53 +++++++++++++++++++++++-
>  src/PVE/LXC/Config.pm     | 84 +++++++++++++++++++++++++++++++++++++++
>  src/PVE/LXC/Tools.pm      | 23 ++++++++---
>  src/lxc-pve-autodev-hook  | 20 ++++++++--
>  src/lxc-pve-prestart-hook | 62 +++++++++++++++++++++++++++--
>  5 files changed, 229 insertions(+), 13 deletions(-)
> 
> diff --git a/src/PVE/LXC.pm b/src/PVE/LXC.pm
> index 8f53b53..98eb909 100644
> --- a/src/PVE/LXC.pm
> +++ b/src/PVE/LXC.pm
> @@ -5,7 +5,7 @@ use warnings;
>  
>  use Cwd qw();
>  use Errno qw(ELOOP ENOTDIR EROFS ECONNREFUSED EEXIST);
> -use Fcntl qw(O_RDONLY O_WRONLY O_NOFOLLOW O_DIRECTORY);
> +use Fcntl qw(O_RDONLY O_WRONLY O_NOFOLLOW O_DIRECTORY :mode);
>  use File::Path;
>  use File::Spec;
>  use IO::Poll qw(POLLIN POLLHUP);
> @@ -639,6 +639,27 @@ sub update_lxc_config {
>  	$raw .= "lxc.mount.auto = sys:mixed\n";
>      }
>  
> +    PVE::LXC::Config->foreach_passthrough_device($conf, sub {
> +	my ($key, $device) = @_;
> +
> +	die "Path is not defined for passthrough device $key"
> +	    unless (defined($device->{path}));
> +
> +	my $absolute_path = $device->{path};
> +	my ($mode, $rdev) = (stat($absolute_path))[2, 6];
> +
> +	die "Device $absolute_path does not exist\n" 
> +	    if (!defined($mode) || !defined($rdev));

^ The above is only true for ENOENT, either check it explicitly or use
something like "Error accessing device $absolute_path: $!\n".

> +
> +	die "$absolute_path is not a device\n"
> +	    if (!S_ISBLK($mode) && !S_ISCHR($mode));
> +
> +	my $major = PVE::Tools::dev_t_major($rdev);
> +	my $minor = PVE::Tools::dev_t_minor($rdev);
> +	my $device_type_char = S_ISBLK($mode) ? 'b' : 'c';
> +	$raw .= "lxc.cgroup2.devices.allow = $device_type_char $major:$minor rw\n";
> +    });
> +
>      # WARNING: DO NOT REMOVE this without making sure that loop device nodes
>      # cannot be exposed to the container with r/w access (cgroup perms).
>      # When this is enabled mounts will still remain in the monitor's namespace
> diff --git a/src/PVE/LXC/Config.pm b/src/PVE/LXC/Config.pm
> index 56e1f10..4feae20 100644
> --- a/src/PVE/LXC/Config.pm
> +++ b/src/PVE/LXC/Config.pm
> @@ -1255,6 +1313,20 @@ sub parse_volume {
>      return;
>  }
>  
> +sub parse_device {
> +    my ($class, $device_string, $noerr) = @_;
> +
> +    my $res = eval { PVE::JSONSchema::parse_property_string($dev_desc, $device_string) };
> +    if ($@) {
> +	return undef if $noerr;
> +	die $@;
> +    }
> +
> +    die "Path has to be defined" if (!$noerr && !defined($res->{path}));

^ an error with $noerr should still return `undef` - here we'd fall back
to `return $res` on error with $noerr currently, follow the same pattern
as above

> +
> +    return $res;
> +}
> +
>  sub print_volume {
>      my ($class, $key, $volume) = @_;
>  
> diff --git a/src/lxc-pve-autodev-hook b/src/lxc-pve-autodev-hook
> index 3c45949..e860fef 100755
> --- a/src/lxc-pve-autodev-hook
> +++ b/src/lxc-pve-autodev-hook
> @@ -3,18 +3,32 @@
>  use strict;
>  use warnings;
>  
> -use File::Path;
> +use Fcntl qw(S_IFREG);
>  use File::Basename;
> +use File::Path;
>  
>  use PVE::LXC::Tools;
> -use PVE::Tools;
> +use PVE::Tools qw(MS_BIND);
>  
>  PVE::LXC::Tools::lxc_hook('autodev', 'lxc', sub {
>      my ($vmid, $vars, undef, undef) = @_;
>  
>      my $root = $vars->{ROOTFS_MOUNT};
>  
> -    PVE::LXC::Tools::for_current_devices($vmid, sub {
> +    PVE::LXC::Tools::for_current_passthrough_devices($vmid, sub {
> +	my ($type, $major, $minor, $dev) = @_;
> +
> +	my $rel_devpath = "/dev/$dev";
> +	my $rel_dir = dirname($rel_devpath);
> +	File::Path::mkpath("$root/$rel_dir");
> +	PVE::Tools::mknod("$root/dev/$dev", S_IFREG, 0)
> +	    or die("Could not mknod $root/dev/$dev: $!\n");
> +
> +	PVE::Tools::mount("/var/lib/lxc/$vmid/passthrough/dev/$dev", "$root/dev/$dev", 0, MS_BIND, 0)
> +	    or die("Bind mount of device $dev into container failed: $!\n");
> +    });
> +
> +    PVE::LXC::Tools::for_current_passthrough_mounts($vmid, sub {
>  	my ($type, $major, $minor, $dev) = @_;
>  
>  	my $rel_devpath = "/dev/$dev";
> diff --git a/src/lxc-pve-prestart-hook b/src/lxc-pve-prestart-hook
> index 936d0bf..f0cc08d 100755
> --- a/src/lxc-pve-prestart-hook
> +++ b/src/lxc-pve-prestart-hook
> @@ -6,6 +6,7 @@ use strict;
>  use warnings;
>  
>  use Fcntl qw(O_DIRECTORY :mode);
> +use File::Basename;
>  use File::Path;
>  use POSIX;
>  
> @@ -58,11 +59,9 @@ PVE::LXC::Tools::lxc_hook('pre-start', 'lxc', sub {
>      # Delete any leftover reboot-trigger file
>      unlink("/var/lib/lxc/$vmid/reboot");
>  
> -    my $devlist_file = "/var/lib/lxc/$vmid/devices";

^ It's now technically possible for a startup during a package upgrade
to have issues if we move this since in *theory* the prestart hook could
run with the old package code and the autodev hook with the new package.

Since the pre-start hook unlinks the file, we *could* potentially
fallback to that path in the autodev hook to catch this case...
But then we should keep `unlink`ing on the old path for a while to make
sure we don't fall back to an old leftover file.

> -    unlink $devlist_file;
>      my $devices = [];
>  
> -    my (undef, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
> +    my ($id_map, $rootuid, $rootgid) = PVE::LXC::parse_id_maps($conf);
>  
>      # Unmount first when the user mounted the container with "pct mount".
>      eval {