[pve-devel] [PATCH qemu-server v7 2/4] feature #3784: Parameter for guest vIOMMU + test-cases

[Markus Frank <m.frank@proxmox.com>]

vIOMMU enables the option to passthrough pci devices to L2 VMs
in L1 VMs via Nested Virtualisation.

Currently there are two vIOMMU implementation in QEMU to choose:
intel & virtio

virtio-iommu is more recent but less used in production than intel-iommu.

Intel IOMMU:
-machine ...,kernel-irqchip=split
-device intel-iommu

Virtio IOMMU:
-device virtio-iommu-pci

Signed-off-by: Markus Frank <m.frank@proxmox.com>
---
 PVE/API2/Qemu.pm                        |  2 ++
 PVE/QemuServer.pm                       | 12 ++++++++++++
 PVE/QemuServer/Machine.pm               | 20 +++++++++++++++++++-
 test/cfg2cmd/q35-viommu-intel.conf      |  1 +
 test/cfg2cmd/q35-viommu-intel.conf.cmd  | 23 +++++++++++++++++++++++
 test/cfg2cmd/q35-viommu-virtio.conf     |  1 +
 test/cfg2cmd/q35-viommu-virtio.conf.cmd | 23 +++++++++++++++++++++++
 7 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 test/cfg2cmd/q35-viommu-intel.conf
 create mode 100644 test/cfg2cmd/q35-viommu-intel.conf.cmd
 create mode 100644 test/cfg2cmd/q35-viommu-virtio.conf
 create mode 100644 test/cfg2cmd/q35-viommu-virtio.conf.cmd

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index 93f0f9b..7177d55 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -1052,6 +1052,7 @@ __PACKAGE__->register_method({
 			    $conf->{machine} = PVE::QemuServer::Machine::print_machine($machine_conf);
 			}
 		    }
+		    PVE::QemuServer::Machine::check_machine_config($conf, $machine_conf);
 
 		    PVE::QemuConfig->write_config($vmid, $conf);
 
@@ -1884,6 +1885,7 @@ my $update_vm_api  = sub {
 		    $conf->{pending}->{$opt} = $param->{$opt};
 		} elsif ($opt eq 'machine') {
 		    my $machine_conf = PVE::QemuServer::Machine::parse_machine($param->{$opt});
+		    PVE::QemuServer::Machine::check_machine_config($conf, $machine_conf);
 		    $conf->{pending}->{$opt} = $param->{$opt};
 		} else {
 		    $conf->{pending}->{$opt} = $param->{$opt};
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 7791373..fa2a773 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -4170,6 +4170,18 @@ sub config_to_command {
     }
     push @$machineFlags, "type=${machine_type_min}";
 
+    PVE::QemuServer::Machine::check_machine_config($conf, $machine_conf);
+
+    if ($machine_conf->{viommu}) {
+	if ($machine_conf->{viommu} eq 'intel') {
+	    unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
+	    push @$machineFlags, 'kernel-irqchip=split';
+	}
+	if ($machine_conf->{viommu} eq 'virtio') {
+	    push @$devices, '-device', 'virtio-iommu-pci';
+	}
+    }
+
     push @$cmd, @$devices;
     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
diff --git a/PVE/QemuServer/Machine.pm b/PVE/QemuServer/Machine.pm
index cb729ca..66e0041 100644
--- a/PVE/QemuServer/Machine.pm
+++ b/PVE/QemuServer/Machine.pm
@@ -23,12 +23,19 @@ my $machine_fmt = {
 	format_description => 'machine type',
 	optional => 1,
     },
+    viommu => {
+	type => 'string',
+	description => "Enable/disable guest vIOMMU"
+	    ." (needs kvm to be enabled and q35 to be set as machine type).",
+	enum => ['intel', 'virtio'],
+	optional => 1,
+    },
 };
 
 PVE::JSONSchema::register_format('pve-qemu-machine-fmt', $machine_fmt);
 
 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
-    description => "Specify the QEMU machine type.",
+    description => "Specify the QEMU machine type & enable/disable vIOMMU.",
     type => 'string',
     optional => 1,
     format => PVE::JSONSchema::get_format('pve-qemu-machine-fmt'),
@@ -48,6 +55,17 @@ sub print_machine {
     return print_property_string($machine_conf, $machine_fmt);
 }
 
+sub check_machine_config {
+    my ($conf, $machine_conf) = @_;
+    my $q35 = $machine_conf->{type} && ($machine_conf->{type} =~ m/q35/) ? 1 : 0;
+    my $kvm = $conf->{kvm};
+    my $arch = PVE::QemuServer::get_vm_arch($conf);
+    $kvm //= 1 if PVE::QemuServer::is_native($arch);
+    if ($machine_conf->{viommu} && $machine_conf->{viommu} eq "intel" && !$q35) {
+	die "to use Intel vIOMMU please set the machine type to q35\n";
+    }
+}
+
 sub machine_type_is_q35 {
     my ($conf) = @_;
 
diff --git a/test/cfg2cmd/q35-viommu-intel.conf b/test/cfg2cmd/q35-viommu-intel.conf
new file mode 100644
index 0000000..e500ab0
--- /dev/null
+++ b/test/cfg2cmd/q35-viommu-intel.conf
@@ -0,0 +1 @@
+machine: q35,viommu=intel
diff --git a/test/cfg2cmd/q35-viommu-intel.conf.cmd b/test/cfg2cmd/q35-viommu-intel.conf.cmd
new file mode 100644
index 0000000..24e873d
--- /dev/null
+++ b/test/cfg2cmd/q35-viommu-intel.conf.cmd
@@ -0,0 +1,23 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'vm8006,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smp '1,sockets=1,cores=1,maxcpus=1' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 512 \
+  -device 'intel-iommu,intremap=on,caching-mode=on' \
+  -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+  -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+  -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -machine 'type=q35+pve0,kernel-irqchip=split'
diff --git a/test/cfg2cmd/q35-viommu-virtio.conf b/test/cfg2cmd/q35-viommu-virtio.conf
new file mode 100644
index 0000000..d31b339
--- /dev/null
+++ b/test/cfg2cmd/q35-viommu-virtio.conf
@@ -0,0 +1 @@
+machine: type=q35,viommu=virtio
diff --git a/test/cfg2cmd/q35-viommu-virtio.conf.cmd b/test/cfg2cmd/q35-viommu-virtio.conf.cmd
new file mode 100644
index 0000000..294c353
--- /dev/null
+++ b/test/cfg2cmd/q35-viommu-virtio.conf.cmd
@@ -0,0 +1,23 @@
+/usr/bin/kvm \
+  -id 8006 \
+  -name 'vm8006,debug-threads=on' \
+  -no-shutdown \
+  -chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' \
+  -mon 'chardev=qmp,mode=control' \
+  -chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5' \
+  -mon 'chardev=qmp-event,mode=control' \
+  -pidfile /var/run/qemu-server/8006.pid \
+  -daemonize \
+  -smp '1,sockets=1,cores=1,maxcpus=1' \
+  -nodefaults \
+  -boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' \
+  -vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' \
+  -cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep \
+  -m 512 \
+  -readconfig /usr/share/qemu-server/pve-q35-4.0.cfg \
+  -device 'usb-tablet,id=tablet,bus=ehci.0,port=1' \
+  -device 'VGA,id=vga,bus=pcie.0,addr=0x1' \
+  -device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' \
+  -iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' \
+  -device virtio-iommu-pci \
+  -machine 'type=q35+pve0'
-- 
2.39.2