[RFC qemu-server 05/13] add experimental kyber-gl display

[Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>]

kyber-gl display is mapping on vhost-user-gpu-pci.

virgl in a helper process rather than inside QEMU: it exports a dmabuf, so QEMU
emits ScanoutDMABUF2 and the encoder imports it instead of reading pixels back.

Currently experimental because of detected bugs:

 - tried vhost-user-gpu instead vhost-user-gpu-pci, but deadlock intermittently
   on QEMU 11.0.2, blocking the main thread under the BQL,
   so the guest freezes rather than just the stream.
 - Modes whose stride is not a multiple of 256 - 800x600 and 1360x768 - fail
   to import on radeonsi and are never streamed. The rest import once the
   retry is made with an explicit DRM_FORMAT_MOD_LINEAR, QEMU labelling its
   scanouts DRM_FORMAT_MOD_INVALID; those two need a copy instead.
 - The picture is flipped vertically through the kernel and plymouth phase,
   while GRUB and the desktop are both correct, so the origin changes twice
   within a session and y0_top does not track it.

Signed-off-by: Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>
---
 src/PVE/API2/Qemu.pm               |   9 ++-
 src/PVE/QemuServer.pm              |  29 ++++++--
 src/PVE/QemuServer/Kyber.pm        |   4 +-
 src/PVE/QemuServer/Makefile        |   1 +
 src/PVE/QemuServer/VhostUserGPU.pm | 103 +++++++++++++++++++++++++++++
 src/test/cfg2cmd/kyber-gl.conf     |   3 +
 src/test/cfg2cmd/kyber-gl.conf.cmd |  29 ++++++++
 7 files changed, 169 insertions(+), 9 deletions(-)
 create mode 100644 src/PVE/QemuServer/VhostUserGPU.pm
 create mode 100644 src/test/cfg2cmd/kyber-gl.conf
 create mode 100644 src/test/cfg2cmd/kyber-gl.conf.cmd

diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm
index bae345b..79aab86 100644
--- a/src/PVE/API2/Qemu.pm
+++ b/src/PVE/API2/Qemu.pm
@@ -3371,9 +3371,10 @@ __PACKAGE__->register_method({
         my $conf = PVE::QemuConfig->load_config($vmid, $node);
 
         my $vga = PVE::QemuServer::parse_vga($conf->{vga} // '');
+        my $vga_type = $vga->{type} // '';
         die "VM $vmid is not configured for the Kyber console"
-            . " - set its display to 'kyber' and restart it\n"
-            if ($vga->{type} // '') ne 'kyber';
+            . " - set its display to 'kyber' or 'kyber-gl' and restart it\n"
+            if $vga_type !~ /^kyber(?:-gl)?$/;
 
         die "VM $vmid is not running\n" if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
 
@@ -3397,7 +3398,9 @@ __PACKAGE__->register_method({
             # Only 'vnc' adds the vdagent chardev the guest needs to share a clipboard.
             my $clipboard = ($vga->{clipboard} // '') eq 'vnc';
 
-            PVE::QemuServer::Kyber::write_env($vmid, $secret, $port, $clipboard);
+            my $dmabuf = $vga_type eq 'kyber-gl';
+
+            PVE::QemuServer::Kyber::write_env($vmid, $secret, $port, $clipboard, $dmabuf);
             PVE::QemuServer::Kyber::restart_controller($vmid);
         }
 
diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 33a5e40..652f3ea 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -97,6 +97,7 @@ use PVE::QemuServer::RNG qw(parse_rng print_rng_device_commandline print_rng_obj
 use PVE::QemuServer::RunState;
 use PVE::QemuServer::StateFile;
 use PVE::QemuServer::USB;
+use PVE::QemuServer::VhostUserGPU;
 use PVE::QemuServer::Virtiofs qw(max_virtiofs start_all_virtiofsd);
 use PVE::QemuServer::VolumeChain;
 use PVE::QemuServer::DBusDisplay;
@@ -171,7 +172,7 @@ my $vga_fmt = {
         optional => 1,
         default_key => 1,
         enum => [
-            qw(cirrus kyber qxl qxl2 qxl3 qxl4 none rdp serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)
+            qw(cirrus kyber kyber-gl qxl qxl2 qxl3 qxl4 none rdp serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)
         ],
     },
     memory => {
@@ -1491,6 +1492,7 @@ my $vga_map = {
     # A display transport, not a card, so it picks one: virtio-vga rather than a GL
     # variant, both of which currently break QEMU.
     'kyber' => 'virtio-vga',
+    'kyber-gl' => 'vhost-user-vga',
     # Same reasoning for the RDP console: it reads the same D-Bus display.
     'rdp' => 'virtio-vga',
 };
@@ -1501,6 +1503,7 @@ my $vga_map_aarch64 = {
     'virtio' => 'virtio-gpu',
     'virtio-gl' => 'virtio-gpu-gl',
     'kyber' => 'virtio-gpu',
+    'kyber-gl' => 'vhost-user-gpu-pci',
     'rdp' => 'virtio-gpu',
 };
 
@@ -1548,6 +1551,12 @@ sub print_vga_device {
         $memory = ",ram_size=67108864,vram_size=33554432";
     }
 
+    my $chardev = "";
+    if ($vga->{type} eq 'kyber-gl') {
+        $memory = "";
+        $chardev = ",chardev=" . PVE::QemuServer::VhostUserGPU::chardev_id();
+    }
+
     my $edidoff = "";
     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
         $edidoff = ",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
@@ -1576,7 +1585,7 @@ sub print_vga_device {
             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
     }
 
-    return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
+    return "$type,id=${vgaid}${memory}${chardev}${max_outputs}${pciaddr}${edidoff}";
 }
 
 sub vm_is_volid_owner {
@@ -3429,9 +3438,15 @@ sub config_to_command {
 
         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
 
-        if ($vga->{type} =~ /^(?:kyber|rdp)$/) {
+        if ($vga->{type} =~ /^(?:kyber|kyber-gl|rdp)$/) {
             my $dbus = PVE::QemuServer::Helpers::dbus_socket($vmid);
             my $display = "dbus,addr=unix:path=$dbus";
+            if ($vga->{type} eq 'kyber-gl') {
+                $display .= ",gl=on";
+                my $socket = PVE::QemuServer::VhostUserGPU::socket_file($vmid);
+                my $id = PVE::QemuServer::VhostUserGPU::chardev_id();
+                push @$cmd, '-chardev', "socket,id=$id,path=$socket";
+            }
 
             # The display exports org.qemu.Display1.Audio only when told which audiodev to
             # read, and nothing else can consume a dbus audiodev.
@@ -3474,6 +3489,7 @@ sub config_to_command {
     }
 
     my $virtiofs_enabled = PVE::QemuServer::Virtiofs::virtiofs_enabled($conf);
+    my $shared_memory = $virtiofs_enabled || ($vga->{type} // '') eq 'kyber-gl';
 
     PVE::QemuServer::Memory::config(
         $conf,
@@ -3481,7 +3497,7 @@ sub config_to_command {
         $sockets,
         $cores,
         $hotplug_features->{memory},
-        $virtiofs_enabled,
+        $shared_memory,
         $cmd,
         $machineFlags,
     );
@@ -5849,7 +5865,9 @@ sub vm_start_nolock {
             # QEMU connects to the D-Bus address, so the bus has to be listening first.
             my $dbus_vga = parse_vga($conf->{vga} // '');
             PVE::QemuServer::DBusDisplay::start($vmid)
-                if ($dbus_vga->{type} // '') =~ /^(?:kyber|rdp)$/;
+                if ($dbus_vga->{type} // '') =~ /^(?:kyber|kyber-gl|rdp)$/;
+            PVE::QemuServer::VhostUserGPU::start($vmid)
+                if ($dbus_vga->{type} // '') eq 'kyber-gl';
 
             my $tpmpid;
             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
@@ -6242,6 +6260,7 @@ sub vm_stop_cleanup {
         PVE::QemuServer::Kyber::stop_controller($vmid);
         PVE::QemuServer::RDP::stop_server($vmid);
         PVE::QemuServer::DBusDisplay::stop($vmid);
+        PVE::QemuServer::VhostUserGPU::stop($vmid);
     };
     warn $@ if $@;
 
diff --git a/src/PVE/QemuServer/Kyber.pm b/src/PVE/QemuServer/Kyber.pm
index 3115f82..953413b 100644
--- a/src/PVE/QemuServer/Kyber.pm
+++ b/src/PVE/QemuServer/Kyber.pm
@@ -84,14 +84,16 @@ sub running_secret {
 }
 
 sub write_env {
-    my ($vmid, $secret, $dataplane_port, $clipboard) = @_;
+    my ($vmid, $secret, $dataplane_port, $clipboard, $dmabuf) = @_;
 
     my $clipboard_env = $clipboard ? 1 : 0;
+    my $dmabuf_env = $dmabuf ? 1 : 0;
 
     my $env = <<"EOF";
 KYBER_JWT_KEY=$secret
 KYBER_DATAPLANE_PORT=$dataplane_port
 KQS_CLIPBOARD=$clipboard_env
+KQS_DMABUF=$dmabuf_env
 EOF
 
     my $path = env_file($vmid);
diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile
index 38e5aa6..1bd7a2f 100644
--- a/src/PVE/QemuServer/Makefile
+++ b/src/PVE/QemuServer/Makefile
@@ -32,6 +32,7 @@ SOURCES=Agent.pm	\
 	RunState.pm	\
 	StateFile.pm	\
 	USB.pm		\
+	VhostUserGPU.pm	\
 	Virtiofs.pm	\
 	VolumeChain.pm
 
diff --git a/src/PVE/QemuServer/VhostUserGPU.pm b/src/PVE/QemuServer/VhostUserGPU.pm
new file mode 100644
index 0000000..63d571b
--- /dev/null
+++ b/src/PVE/QemuServer/VhostUserGPU.pm
@@ -0,0 +1,103 @@
+package PVE::QemuServer::VhostUserGPU;
+
+# virgl in a helper process rather than inside QEMU, for the 'kyber-gl' display.
+# The helper renders and exports a dmabuf, so QEMU emits ScanoutDMABUF2 and the
+# console encoder imports it instead of reading back pixels.
+
+use strict;
+use warnings;
+
+use POSIX;
+use Time::HiRes qw(usleep);
+
+use PVE::ProcFSTools;
+use PVE::Tools qw(file_set_contents file_read_firstline);
+use PVE::QemuServer::Helpers;
+
+my $BINARY = '/usr/lib/kvm/vhost-user-gpu';
+
+sub socket_file {
+    my ($vmid) = @_;
+    return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.vhost-user-gpu.sock";
+}
+
+sub pidfile {
+    my ($vmid) = @_;
+    return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.vhost-user-gpu.pid";
+}
+
+sub chardev_id {
+    return 'vhost-user-gpu';
+}
+
+sub start {
+    my ($vmid) = @_;
+
+    die "vhost-user-gpu is not installed ($BINARY)\n" if !-x $BINARY;
+
+    stop($vmid);
+
+    my $socket = socket_file($vmid);
+    my $pidfile = pidfile($vmid);
+    my $log = "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid-vhost-user-gpu.log";
+
+    my $pid = fork();
+    die "could not fork to start vhost-user-gpu for VM $vmid\n" if !defined($pid);
+
+    if ($pid == 0) {
+        POSIX::setsid();
+        $0 = "task pve-vm$vmid-vhost-user-gpu";
+
+        my $pid2 = fork();
+        if (!defined($pid2)) {
+            POSIX::_exit(1);
+        } elsif ($pid2 == 0) {
+            open(STDIN, '<', '/dev/null');
+            open(STDOUT, '>>', $log);
+            open(STDERR, '>&', \*STDOUT);
+            exec($BINARY, '--virgl', '--socket-path', $socket);
+            POSIX::_exit(1);
+        }
+
+        eval { file_set_contents($pidfile, "$pid2\n") };
+        POSIX::_exit(0);
+    }
+
+    waitpid($pid, 0);
+
+    for (my $waited = 0; $waited < 5; $waited += 0.05) {
+        last if -S $socket;
+        usleep(50_000);
+    }
+    die "vhost-user-gpu for VM $vmid did not create $socket - see $log\n" if !-S $socket;
+
+    return;
+}
+
+sub stop {
+    my ($vmid) = @_;
+
+    my $pidfile = pidfile($vmid);
+    if (my $pid = eval { file_read_firstline($pidfile) }) {
+        if ($pid =~ m/^(\d+)$/) {
+            $pid = $1;
+            kill('TERM', $pid);
+
+            for (my $waited = 0; $waited < 5; $waited += 0.05) {
+                last if !PVE::ProcFSTools::check_process_running($pid);
+                usleep(50_000);
+            }
+            if (PVE::ProcFSTools::check_process_running($pid)) {
+                warn "vhost-user-gpu for VM $vmid did not exit, killing it\n";
+                kill('KILL', $pid);
+            }
+        }
+    }
+
+    unlink $pidfile;
+    unlink socket_file($vmid);
+
+    return;
+}
+
+1;
diff --git a/src/test/cfg2cmd/kyber-gl.conf b/src/test/cfg2cmd/kyber-gl.conf
new file mode 100644
index 0000000..e127bea
--- /dev/null
+++ b/src/test/cfg2cmd/kyber-gl.conf
@@ -0,0 +1,3 @@
+# TEST: Kyber console on the vhost-user GPU
+memory: 2048
+vga: kyber-gl
diff --git a/src/test/cfg2cmd/kyber-gl.conf.cmd b/src/test/cfg2cmd/kyber-gl.conf.cmd
new file mode 100644
index 0000000..d95b7df
--- /dev/null
+++ b/src/test/cfg2cmd/kyber-gl.conf.cmd
@@ -0,0 +1,29 @@
+/usr/bin/kvm
+-id 8006
+-name vm8006
+-no-shutdown
+-chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off'
+-mon 'chardev=qmp,mode=control'
+-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000'
+-mon 'chardev=qmp-event,mode=control'
+-pidfile /var/run/qemu-server/8006.pid
+-daemonize
+-smp '1,sockets=1,cores=1,maxcpus=1'
+-nodefaults
+-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg'
+-chardev 'socket,id=vhost-user-gpu,path=/var/run/qemu-server/8006.vhost-user-gpu.sock'
+-display 'dbus,addr=unix:path=/var/run/qemu-server/8006.dbusdisplay,gl=on'
+-vnc 'unix:/var/run/qemu-server/8006.vnc,password=on'
+-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep
+-m 2048
+-object 'memory-backend-memfd,id=virtiofs-mem,size=2048M,share=on'
+-global 'PIIX4_PM.disable_s3=1'
+-global 'PIIX4_PM.disable_s4=1'
+-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e'
+-device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f'
+-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2'
+-device 'usb-tablet,id=tablet,bus=uhci.0,port=1'
+-device 'vhost-user-vga,id=vga,chardev=vhost-user-gpu,bus=pci.0,addr=0x2'
+-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on'
+-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff'
+-machine 'memory-backend=virtiofs-mem,type=pc+pve0'
\ No newline at end of file
-- 
2.55.0