[pve-devel] [PATCH qemu-server] pci: add 'driver' option

[Dominik Csapak <d.csapak@proxmox.com>]

by default, pci devices will be bound to 'vfio-pci' driver and reset.
For most devices this is necessary, but there are a few exceptions,
e.g.:

* some mellanox nics have support for the driver 'mlx5_vfio_pci'
* intel flex gpus have support for 'i915_vfio_pci'
* (maybe some more i don't know about)

both of these drivers play the role of the vfio-pci drivers themselves,
so no rebinding or resetting necessary. Those drivers usually have more
functionality than the default vfio driver, like support for
live-migration.

To be able to configure that on our side, introduce the 'driver'
option for 'hostpciX', which will not rebind/reset the device if it set
to 'keep'. The default 'vfio' option upholds the current behavior.

Adds some basic config parsing test for the 'keep' and 'vfio' option.
This does not test the actual driver preparation behavior, just that
the option is parseable from the config

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes from RFC:
* use thomas suggestion to use an enum for the option for future
  extensibility
* add some basic config parsing test for cfg2cmd, copied from the q35
  linux test

 src/PVE/QemuServer/PCI.pm                     | 11 ++++-
 .../q35-linux-hostpci-driver-keep.conf        | 20 +++++++++
 .../q35-linux-hostpci-driver-keep.conf.cmd    | 44 +++++++++++++++++++
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf
 create mode 100644 src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf.cmd

diff --git a/src/PVE/QemuServer/PCI.pm b/src/PVE/QemuServer/PCI.pm
index e7a9a610..c9cf8de0 100644
--- a/src/PVE/QemuServer/PCI.pm
+++ b/src/PVE/QemuServer/PCI.pm
@@ -124,6 +124,14 @@ EODESCR
         optional => 1,
         description => "Override PCI subsystem device ID visible to guest",
     },
+    'driver' => {
+        type => 'string',
+        optional => 1,
+        default => 'vfio',
+        enum => [qw(vfio keep)],
+        description => "If set to 'keep' the device will neither be reset nor bound to the "
+            . "'vfio-pci' driver. Useful for devices that already have the correct driver loaded.",
+    },
 };
 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
 
@@ -736,7 +744,8 @@ sub prepare_pci_device {
         if !PVE::SysFSTools::check_iommu_support();
     die "no pci device info for device '$pciid'\n" if !$info;
 
-    if ($device->{nvidia}) {
+    my $driver = $device->{driver} // 'vfio';
+    if ($device->{nvidia} || $driver eq "keep") {
         # nothing to do
     } elsif (my $mdev = $device->{mdev}) {
         my $uuid = generate_mdev_uuid($vmid, $index);
diff --git a/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf b/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf
new file mode 100644
index 00000000..9b1136d5
--- /dev/null
+++ b/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf
@@ -0,0 +1,20 @@
+# TEST: Config with q35, NUMA, hostpci passthrough, EFI & Linux & driver option
+bios: ovmf
+bootdisk: scsi0
+cores: 1
+efidisk0: local:100/vm-100-disk-1.qcow2,size=128K
+hostpci0: 00:ff.1,driver=keep
+hostpci1: d0:13.0,pcie=1,driver=vfio
+hostpci2: 00:f4.0
+hostpci3: d0:15.1,pcie=1
+hostpci4: d0:17.0,pcie=1,rombar=0
+hostpci7: d0:15.2,pcie=1
+machine: q35
+memory: 512
+net0: virtio=2E:01:68:F9:9C:87,bridge=vmbr0
+numa: 1
+ostype: l26
+scsihw: virtio-scsi-pci
+smbios1: uuid=3dd750ce-d910-44d0-9493-525c0be4e687
+sockets: 2
+vmgenid: 54d1c06c-8f5b-440f-b5b2-6eab1380e13d
diff --git a/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf.cmd b/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf.cmd
new file mode 100644
index 00000000..7226d478
--- /dev/null
+++ b/src/test/cfg2cmd/q35-linux-hostpci-driver-keep.conf.cmd
@@ -0,0 +1,44 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'vm8006,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smbios 'type=1,uuid=3dd750ce-d910-44d0-9493-525c0be4e687' \
+  -object '{"id":"throttle-drive-efidisk0","limits":{},"qom-type":"throttle-group"}' \
+  -blockdev '{"driver":"raw","file":{"driver":"file","filename":"/usr/share/pve-edk2-firmware//OVMF_CODE.fd"},"node-name":"pflash0","read-only":true}' \
+  -blockdev '{"detect-zeroes":"on","discard":"ignore","driver":"throttle","file":{"cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"qcow2","file":{"aio":"io_uring","cache":{"direct":false,"no-flush":false},"detect-zeroes":"on","discard":"ignore","driver":"file","filename":"/var/lib/vz/images/100/vm-100-disk-1.qcow2","node-name":"e70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"f70e3017c5a79fdee5a04aa92ac1e9c","read-only":false},"node-name":"drive-efidisk0","read-only":false,"throttle-group":"throttle-drive-efidisk0"}' \
+  -global 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off' \
+  -smp '2,sockets=2,cores=1,maxcpus=2' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 512 \
+  -object 'memory-backend-ram,id=ram-node0,size=256M' \
+  -numa 'node,nodeid=0,cpus=0,memdev=ram-node0' \
+  -object 'memory-backend-ram,id=ram-node1,size=256M' \
+  -numa 'node,nodeid=1,cpus=1,memdev=ram-node1' \
+  -global 'ICH9-LPC.disable_s3=1' \
+  -global 'ICH9-LPC.disable_s4=1' \
+  -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+  -device 'vmgenid,guid=54d1c06c-8f5b-440f-b5b2-6eab1380e13d' \
+  -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+  -device 'vfio-pci,host=0000:00:ff.1,id=hostpci0,bus=pci.0,addr=0x10' \
+  -device 'vfio-pci,host=0000:d0:13.0,id=hostpci1,bus=ich9-pcie-port-2,addr=0x0' \
+  -device 'vfio-pci,host=0000:00:f4.0,id=hostpci2,bus=pci.0,addr=0x1b' \
+  -device 'vfio-pci,host=0000:d0:15.1,id=hostpci3,bus=ich9-pcie-port-4,addr=0x0' \
+  -device 'pcie-root-port,id=ich9-pcie-port-5,addr=10.0,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=5,chassis=5' \
+  -device 'vfio-pci,host=0000:d0:17.0,id=hostpci4,bus=ich9-pcie-port-5,addr=0x0,rombar=0' \
+  -device 'pcie-root-port,id=ich9-pcie-port-8,addr=10.3,x-speed=16,x-width=32,multifunction=on,bus=pcie.0,port=8,chassis=8' \
+  -device 'vfio-pci,host=0000:d0:15.2,id=hostpci7,bus=ich9-pcie-port-8,addr=0x0' \
+  -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -netdev 'type=tap,id=net0,ifname=tap8006i0,script=/usr/libexec/qemu-server/pve-bridge,downscript=/usr/libexec/qemu-server/pve-bridgedown,vhost=on' \
+  -device 'virtio-net-pci,mac=2E:01:68:F9:9C:87,netdev=net0,bus=pci.0,addr=0x12,id=net0,rx_queue_size=1024,tx_queue_size=256,bootindex=300,host_mtu=1500' \
+  -machine 'pflash0=pflash0,pflash1=drive-efidisk0,hpet=off,type=q35+pve0'
-- 
2.47.3



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel