From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 3CFBF8B6E5 for ; Thu, 25 Aug 2022 11:25:38 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A1D65233A3 for ; Thu, 25 Aug 2022 11:25:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 25 Aug 2022 11:24:56 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 3983643FC9 for ; Thu, 25 Aug 2022 11:24:45 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com Date: Thu, 25 Aug 2022 11:24:26 +0200 Message-Id: <20220825092440.1810328-18-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220825092440.1810328-1-d.csapak@proxmox.com> References: <20220825092440.1810328-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.094 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [pci.pm, qemuserver.pm] Subject: [pve-devel] [PATCH qemu-server v2 12/13] fix #3574: enable multi pci device mapping from config X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 25 Aug 2022 09:25:38 -0000 The hardware config now supports multiple devices as a semicolon seperated list. With this, instead of only having one device in a pci mapping, we now have a list of which we can choose from on vm start. This way one can dynamically start vms with a pool of (identical) pci devices without having to manually assign the proper ids. For that we have to change the internal representation of a parsed device, such that we have the seperately configured paths in the mapping in different lists (because multifunction devices still are interpreted as single devices) For mdev devices we now can also have multiple devices, where we simply try to create the appropriate type on each until we either have one created, or bail out. Since we now have to reserve the pci ids in print_hostpci_devices, we have to add a 'reserve' parameter to config_to_command (and chain it through to reserve_pci_usage) so that a 'qm showcmd' does not actually reserve any pci id (this would break when using that on running vms). Additionally this also prevents the migration tests from failing (they use vm_commandline which in turn uses config_to_command) Signed-off-by: Dominik Csapak --- PVE/QemuServer.pm | 43 ++++++++++++++++++++++++++++++------- PVE/QemuServer/PCI.pm | 49 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index 4551e2b..ce24b19 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -3507,8 +3507,9 @@ my sub should_disable_smm { sub config_to_command { my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu, - $pbs_backing) = @_; + $pbs_backing, $reserve) = @_; + $reserve //= 1; my $cmd = []; my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []); my $devices = []; @@ -3724,7 +3725,7 @@ sub config_to_command { # host pci device passthrough my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices( - $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder); + $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder, $reserve); # usb devices my $usb_dev_features = {}; @@ -5623,13 +5624,30 @@ sub vm_start_nolock { my $uuid; for my $id (sort keys %$pci_devices) { my $d = $pci_devices->{$id}->{device}; - for my $dev ($d->{pciid}->@*) { - my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev}); - # nvidia grid needs the uuid of the mdev as qemu parameter - if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') { - $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id); + # used pci devices for non-mdev + if (!$d->{mdev}) { + for my $dev ($pci_devices->{$id}->{used}->@*) { + PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id); } + next; + } + + # try each configured pci device for mdevs + my $devs = [map { $_->{id} } map { @$_ } $d->{ids}->@*]; # flatten ids + + my $info; + for my $dev (@$devs) { + $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev, $id, $d->{mdev}) }; + warn $@ if $@; + last if $info; # if successful, we're done + } + + die "could not create mediated device\n" if !defined($info); + + # nvidia grid needs the uuid of the mdev as qemu parameter + if (!defined($uuid) && $info->{vendor} eq '10de') { + $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id); } } push @$cmd, '-uuid', $uuid if defined($uuid); @@ -5862,7 +5880,16 @@ sub vm_commandline { my $defaults = load_defaults(); - my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu); + my $cmd = config_to_command( + $storecfg, + $vmid, + $conf, + $defaults, + $forcemachine, + $forcecpu, + undef, + 0, + ); return PVE::Tools::cmd2string($cmd); } diff --git a/PVE/QemuServer/PCI.pm b/PVE/QemuServer/PCI.pm index 8c171f3..df8d16a 100644 --- a/PVE/QemuServer/PCI.pm +++ b/PVE/QemuServer/PCI.pm @@ -386,6 +386,7 @@ sub parse_hostpci { my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value); + my $idlist = []; if ($res->{host} !~ m/:/) { # we have no ordinary pci id, must be a mapping my $device = PVE::HardwareMap::find_device_on_current_node('pci', $res->{host}); @@ -396,15 +397,27 @@ sub parse_hostpci { if (my $err = $@) { die "PCI device mapping invalid (hardware probably changed): $err\n"; } - $res->{host} = $device->{path}; + $idlist = [split(/;/, $device->{path})]; + # if we have a list of mapped devices, we want to choose the first available one + $res->{choose} = 1 if scalar(@$idlist > 1); + } else { + $idlist = [split(/;/, $res->{host})]; } - my @idlist = split(/;/, $res->{host}); delete $res->{host}; - foreach my $id (@idlist) { + my $ignore_mdev = !$res->{choose} && scalar(@$idlist) > 1; + + $res->{ids} = []; + foreach my $id (@$idlist) { my $devs = PVE::SysFSTools::lspci($id); die "no PCI device found for '$id'\n" if !scalar(@$devs); - push @{$res->{pciid}}, @$devs; + $ignore_mdev = 1 if scalar(@$devs) > 1; + push @{$res->{ids}}, $devs; + } + # ignore mdev for multiple devices, except when from mapping + if ($res->{mdev} && $ignore_mdev) { + warn "ignoring mediated device with multifunction device\n"; + delete $res->{mdev}; } return $res; } @@ -433,11 +446,13 @@ my $print_pci_device = sub { }; sub print_hostpci_devices { - my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder) = @_; + my ($vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder, $reserve) = @_; + $reserve //= 1; my $kvm_off = 0; my $gpu_passthrough = 0; my $legacy_igd = 0; + my $used_pci_ids = {}; my $parsed_devices = {}; my $pciaddr; @@ -469,7 +484,24 @@ sub print_hostpci_devices { $pciaddr = print_pci_addr($pci_name, $bridges, $arch, $machine_type); } - my $pcidevices = $d->{pciid}; + # choose devices + my $pcidevices = []; + if (!$d->{mdev}) { + for my $devs ($d->{ids}->@*) { + my $ids = [map { $_->{id} } @$devs]; + + if ($d->{choose}) { + next if grep { defined($used_pci_ids->{$_}) } @$ids; # already used + eval { reserve_pci_usage($ids, $vmid, 10, undef, $reserve) }; + next if $@; + } + + map { $used_pci_ids->{$_} = 1 } @$ids; + push @$pcidevices, @$devs; + last if $d->{choose}; + } + die "could not find a free device\n" if scalar(@$pcidevices) < 1; + } $parsed_devices->{$i}->{used} = $pcidevices; my $multifunction = @$pcidevices > 1; @@ -599,8 +631,9 @@ sub remove_pci_reservation { } sub reserve_pci_usage { - my ($requested_ids, $vmid, $timeout, $pid) = @_; + my ($requested_ids, $vmid, $timeout, $pid, $reserve) = @_; + $reserve //= 1; $requested_ids = [ $requested_ids ] if !ref($requested_ids); return if !scalar(@$requested_ids); # do nothing for empty list @@ -633,7 +666,7 @@ sub reserve_pci_usage { $reservation_list->{$id}->{time} = $ctime + $timeout + 5; } } - $write_pci_reservation_unlocked->($reservation_list); + $write_pci_reservation_unlocked->($reservation_list) if $reserve; }); die $@ if $@; } -- 2.30.2