SPAM: [RFC qemu-server 03/13] add kyber display

[Alexandre Derumier <alexandre.derumier@groupe-cyllene.com>]

From: Alexandre Derumier <aderumier@groupe-cyllene.com>

kyber maps to virtio-vga, with a controller started per VM on demand by
the kyberproxy API call.

Signed-off-by: Alexandre Derumier <aderumier@groupe-cyllene.com>
---
 src/PVE/API2/Qemu.pm            |  94 +++++++++++++++++++++++
 src/PVE/QemuServer.pm           |  50 ++++++++++++-
 src/PVE/QemuServer/Kyber.pm     | 128 ++++++++++++++++++++++++++++++++
 src/PVE/QemuServer/Makefile     |   2 +
 src/test/cfg2cmd/kyber.conf     |   3 +
 src/test/cfg2cmd/kyber.conf.cmd |  27 +++++++
 src/usr/Makefile                |   1 +
 src/usr/pve-qemu-kyber@.service |  25 +++++++
 8 files changed, 326 insertions(+), 4 deletions(-)
 create mode 100644 src/PVE/QemuServer/Kyber.pm
 create mode 100644 src/test/cfg2cmd/kyber.conf
 create mode 100644 src/test/cfg2cmd/kyber.conf.cmd
 create mode 100644 src/usr/pve-qemu-kyber@.service

diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm
index 71247ee..55befc1 100644
--- a/src/PVE/API2/Qemu.pm
+++ b/src/PVE/API2/Qemu.pm
@@ -12,6 +12,7 @@ use IPC::Open3;
 use JSON;
 use URI::Escape;
 use Socket qw(SOCK_STREAM);
+use Time::HiRes qw(usleep);
 
 use PVE::APIClient::LWP;
 use PVE::CGroup;
@@ -35,6 +36,7 @@ use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CPUConfig;
 use PVE::QemuServer::Drive qw(checked_volume_format checked_parse_volname);
 use PVE::QemuServer::Helpers;
+use PVE::QemuServer::Kyber;
 use PVE::QemuServer::ImportDisk;
 use PVE::QemuServer::Monitor qw(mon_cmd vm_qmp_peer);
 use PVE::QemuServer::Machine;
@@ -3329,6 +3331,92 @@ __PACKAGE__->register_method({
     },
 });
 
+__PACKAGE__->register_method({
+    name => 'kyberproxy',
+    path => '{vmid}/kyberproxy',
+    method => 'POST',
+    protected => 1,
+    proxyto => 'node',
+    permissions => {
+        check => ['perm', '/vms/{vmid}', ['VM.Console']],
+    },
+    description => "Start a Kyber console controller for the VM and return how to reach it.",
+    parameters => {
+        additionalProperties => 0,
+        properties => {
+            node => get_standard_option('pve-node'),
+            vmid => get_standard_option('pve-vmid'),
+        },
+    },
+    returns => {
+        additionalProperties => 0,
+        properties => {
+            user => { type => 'string' },
+            ticket => {
+                type => 'string',
+                description => "Short-lived token authenticating this user to the controller.",
+            },
+        },
+    },
+    code => sub {
+        my ($param) = @_;
+
+        my $rpcenv = PVE::RPCEnvironment::get();
+        my $authuser = $rpcenv->get_user();
+
+        my $vmid = $param->{vmid};
+        my $node = $param->{node};
+
+        my $conf = PVE::QemuConfig->load_config($vmid, $node);
+
+        my $vga = PVE::QemuServer::parse_vga($conf->{vga} // '');
+        die "VM $vmid is not configured for the Kyber console"
+            . " - set its display to 'kyber' and restart it\n"
+            if ($vga->{type} // '') ne 'kyber';
+
+        die "VM $vmid is not running\n" if !PVE::QemuServer::Helpers::vm_running_locally($vmid);
+
+        my $socket = PVE::QemuServer::Helpers::dbus_socket($vmid);
+        die "VM $vmid has no D-Bus display socket at $socket"
+            . " - it was started before its display was set to 'kyber',"
+            . " so it needs a restart\n"
+            if !-S $socket;
+
+        # Join a controller that is already streaming: it shares one capture between
+        # clients, and restarting to install a new secret would cut the first off.
+        my ($secret, $port) = PVE::QemuServer::Kyber::running_secret($vmid);
+
+        if (!$secret) {
+            my $family = PVE::Tools::get_host_address_family($node);
+            $port = PVE::QemuServer::Kyber::next_port($family);
+
+            # Fresh per controller: with none running there is nothing to cut off.
+            $secret = PVE::QemuServer::Kyber::generate_secret();
+
+            # Only 'vnc' adds the vdagent chardev the guest needs to share a clipboard.
+            my $clipboard = ($vga->{clipboard} // '') eq 'vnc';
+
+            PVE::QemuServer::Kyber::write_env($vmid, $secret, $port, $clipboard);
+            PVE::QemuServer::Kyber::restart_controller($vmid);
+        }
+
+        my $ticket = PVE::QemuServer::Kyber::assemble_ticket($secret, $authuser);
+
+        # Listening within ~30ms, so waiting here saves the client a retry loop.
+        my $kybersocket = PVE::QemuServer::Kyber::socket_file($vmid);
+        for (my $waited = 0; $waited < 5; $waited += 0.05) {
+            last if -S $kybersocket;
+            usleep(50_000);
+        }
+        die "Kyber console controller for VM $vmid did not start\n" if !-S $kybersocket;
+
+        return {
+            user => $authuser,
+            ticket => $ticket,
+        };
+    },
+});
+
 __PACKAGE__->register_method({
     name => 'spiceproxy',
     path => '{vmid}/spiceproxy',
@@ -3452,6 +3540,11 @@ __PACKAGE__->register_method({
                 type => 'boolean',
                 optional => 1,
             },
+            kyber => {
+                description => "QEMU VGA configuration supports the Kyber console.",
+                type => 'boolean',
+                optional => 1,
+            },
             agent => {
                 description => "QEMU Guest Agent is enabled in config.",
                 type => 'boolean',
@@ -3482,6 +3575,7 @@ __PACKAGE__->register_method({
             my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/;
             $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga});
             $status->{spice} = 1 if $spice;
+            $status->{kyber} = 1 if ($vga->{type} // '') eq 'kyber';
             $status->{clipboard} = $vga->{clipboard};
         }
         $status->{agent} = 1 if PVE::QemuServer::Agent::get_qga_key($conf, 'enabled');
diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 2f43faa..8c7f023 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -80,6 +80,7 @@ use PVE::QemuServer::Drive qw(
     storage_allows_io_uring_default
 );
 use PVE::QemuServer::DriveDevice qw(print_drivedevice_full scsihw_infos);
+use PVE::QemuServer::Kyber;
 use PVE::QemuServer::Machine;
 use PVE::QemuServer::Memory qw(get_current_memory);
 use PVE::QemuServer::MetaInfo;
@@ -98,6 +99,7 @@ use PVE::QemuServer::StateFile;
 use PVE::QemuServer::USB;
 use PVE::QemuServer::Virtiofs qw(max_virtiofs start_all_virtiofsd);
 use PVE::QemuServer::VolumeChain;
+use PVE::QemuServer::DBusDisplay;
 use PVE::QemuServer::DBusVMState;
 
 my $have_ha_config;
@@ -168,7 +170,7 @@ my $vga_fmt = {
         optional => 1,
         default_key => 1,
         enum => [
-            qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)
+            qw(cirrus kyber qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)
         ],
     },
     memory => {
@@ -213,10 +215,12 @@ my $audio_fmt = {
     },
     driver => {
         type => 'string',
-        enum => ['spice', 'none'],
+        enum => ['spice', 'dbus', 'none'],
         default => 'spice',
         optional => 1,
-        description => "Driver backend for the audio device.",
+        description => "Driver backend for the audio device."
+            . " 'dbus' exposes it on the VM's D-Bus display, which is what the"
+            . " Kyber and RDP consoles read.",
     },
 };
 
@@ -1481,6 +1485,9 @@ my $vga_map = {
     'vmware' => 'vmware-svga',
     'virtio' => 'virtio-vga',
     'virtio-gl' => 'virtio-vga-gl',
+    # A display transport, not a card, so it picks one: virtio-vga rather than a GL
+    # variant, both of which currently break QEMU.
+    'kyber' => 'virtio-vga',
 };
 
 # QEMU builds only the non-VGA variants of the virtio GPU for aarch64
@@ -1488,6 +1495,7 @@ my $vga_map_aarch64 = {
     $vga_map->%*,
     'virtio' => 'virtio-gpu',
     'virtio-gl' => 'virtio-gpu-gl',
+    'kyber' => 'virtio-gpu',
 };
 
 my sub map_vga_model {
@@ -2839,7 +2847,14 @@ sub audio_devs {
         die "unknown audio device '$audio->{dev}', implement me!";
     }
 
-    push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
+    my $backend = "$audio->{backend},id=$audio->{backend_id}";
+
+    # Pinned for the D-Bus backend: what reads it is an Opus encoder and libopus takes
+    # 48kHz only. QEMU already resamples, so this costs nothing new.
+    $backend .= ',out.frequency=48000,out.channels=2,out.format=s16'
+        if $audio->{backend} eq 'dbus';
+
+    push @$devs, '-audiodev', $backend;
 
     return $devs;
 }
@@ -3408,6 +3423,19 @@ sub config_to_command {
 
         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
 
+        if ($vga->{type} eq 'kyber') {
+            my $dbus = PVE::QemuServer::Helpers::dbus_socket($vmid);
+            my $display = "dbus,addr=unix:path=$dbus";
+
+            # The display exports org.qemu.Display1.Audio only when told which audiodev to
+            # read, and nothing else can consume a dbus audiodev.
+            my $audio = conf_has_audio($conf);
+            $display .= ",audiodev=$audio->{backend_id}"
+                if $audio && $audio->{backend} eq 'dbus';
+
+            push @$cmd, '-display', $display;
+        }
+
         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
         push @$cmd, '-vnc', "unix:$socket,password=on";
     } else {
@@ -5812,6 +5840,11 @@ sub vm_start_nolock {
 
             my $virtiofs_sockets = start_all_virtiofsd($conf, $vmid);
 
+            # QEMU connects to the D-Bus address, so the bus has to be listening first.
+            my $dbus_vga = parse_vga($conf->{vga} // '');
+            PVE::QemuServer::DBusDisplay::start($vmid)
+                if ($dbus_vga->{type} // '') eq 'kyber';
+
             my $tpmpid;
             if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
                 # start the TPM emulator so QEMU can connect on start
@@ -6196,6 +6229,15 @@ sub vm_stop_cleanup {
     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes, $noerr, $skip_hookscript) =
         @_;
 
+    # Before the cleanup flag is consulted, deliberately: the bus is forked into the
+    # VM's systemd scope, and a survivor keeps that cgroup from emptying, so the next
+    # start fails with "timeout waiting on systemd".
+    eval {
+        PVE::QemuServer::Kyber::stop_controller($vmid);
+        PVE::QemuServer::DBusDisplay::stop($vmid);
+    };
+    warn $@ if $@;
+
     my $can_use_cleanup_flag = PVE::QemuServer::RunState::can_use_cleanup_flag();
     if ($can_use_cleanup_flag) {
         return if !PVE::QemuServer::RunState::cleanup_flag_exists($vmid);
diff --git a/src/PVE/QemuServer/Kyber.pm b/src/PVE/QemuServer/Kyber.pm
new file mode 100644
index 0000000..3115f82
--- /dev/null
+++ b/src/PVE/QemuServer/Kyber.pm
@@ -0,0 +1,128 @@
+package PVE::QemuServer::Kyber;
+
+# Per-VM Kyber console controller: one kycontroller per VM, spawned on demand and
+# reaped when it exits.
+
+use strict;
+use warnings;
+
+use Digest::SHA qw(hmac_sha256);
+use JSON;
+use Crypt::OpenSSL::Random;
+use MIME::Base64 qw(encode_base64url);
+
+use PVE::Tools qw(file_set_contents);
+use PVE::QemuServer::DBusDisplay;
+use PVE::QemuServer::Helpers;
+
+sub socket_file {
+    my ($vmid) = @_;
+    return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.kyber.sock";
+}
+
+# Carries the signing key, and only that: /proc/<pid>/cmdline is world-readable,
+# so it cannot go on the command line. systemd passes it as KYBER_JWT_KEY.
+sub env_file {
+    my ($vmid) = @_;
+    return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.kyber.env";
+}
+
+sub next_port {
+    my ($family) = @_;
+    return PVE::Tools::next_unused_port(63000, 63099, $family, '127.0.0.1');
+}
+
+sub generate_secret {
+    my $bytes = Crypt::OpenSSL::Random::random_bytes(32)
+        or die "unable to generate a random secret\n";
+    return unpack('H*', $bytes);
+}
+
+# An HS256 token the controller will accept. The secret is regenerated whenever a
+# controller starts and never leaves the node, so it is useless against other VMs.
+sub assemble_ticket {
+    my ($secret, $username, $ttl) = @_;
+
+    # As long as a VNC ticket, and for the same reason: the client re-presents it to
+    # renew its session. It is worth little alone - the controller is on loopback.
+    $ttl //= 3600;
+    my $now = time();
+
+    my $header = encode_base64url(encode_json({ alg => 'HS256', typ => 'JWT' }));
+    # The controller requires aud=kyber (auth/jwt.rs) and rejects a token without it
+    # as malformed, which reads like a signing failure.
+    my $claims = encode_base64url(
+        encode_json({
+            aud => 'kyber',
+            sub => $username,
+            iat => $now,
+            exp => $now + $ttl,
+        }),
+    );
+
+    my $signature = encode_base64url(hmac_sha256("$header.$claims", $secret));
+
+    return "$header.$claims.$signature";
+}
+
+# The secret a running controller is verifying against, or undef when there is
+# none. Kept in a root-only env file so a second console can join instead of
+# restarting the controller and cutting the first viewer off.
+sub running_secret {
+    my ($vmid) = @_;
+
+    return undef if !-S socket_file($vmid);
+
+    my $env = eval { PVE::Tools::file_get_contents(env_file($vmid)) };
+    return undef if !defined($env);
+
+    my ($secret) = $env =~ m/^KYBER_JWT_KEY=(\S+)$/m;
+    my ($port) = $env =~ m/^KYBER_DATAPLANE_PORT=(\d+)$/m;
+    return undef if !$secret || !$port;
+
+    return ($secret, $port);
+}
+
+sub write_env {
+    my ($vmid, $secret, $dataplane_port, $clipboard) = @_;
+
+    my $clipboard_env = $clipboard ? 1 : 0;
+
+    my $env = <<"EOF";
+KYBER_JWT_KEY=$secret
+KYBER_DATAPLANE_PORT=$dataplane_port
+KQS_CLIPBOARD=$clipboard_env
+EOF
+
+    my $path = env_file($vmid);
+    file_set_contents($path, $env, 0600);
+
+    return $path;
+}
+
+sub restart_controller {
+    my ($vmid) = @_;
+
+    PVE::Tools::run_command(
+        ['systemctl', 'restart', "pve-qemu-kyber\@$vmid"],
+        errmsg => "failed to start Kyber console controller for VM $vmid",
+    );
+
+    return;
+}
+
+sub stop_controller {
+    my ($vmid) = @_;
+
+    eval {
+        PVE::Tools::run_command(['systemctl', 'stop', "pve-qemu-kyber\@$vmid"]);
+    };
+    warn $@ if $@;
+
+    unlink env_file($vmid);
+    unlink socket_file($vmid);
+
+    return;
+}
+
+1;
diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile
index 060fac2..061d61f 100644
--- a/src/PVE/QemuServer/Makefile
+++ b/src/PVE/QemuServer/Makefile
@@ -10,11 +10,13 @@ SOURCES=Agent.pm	\
 	Cloudinit.pm	\
 	CPUConfig.pm	\
 	CPUFlags.pm	\
+	DBusDisplay.pm	\
 	DBusVMState.pm	\
 	Drive.pm	\
 	DriveDevice.pm	\
 	Helpers.pm	\
 	ImportDisk.pm	\
+	Kyber.pm	\
 	Machine.pm	\
 	Memory.pm	\
 	MetaInfo.pm	\
diff --git a/src/test/cfg2cmd/kyber.conf b/src/test/cfg2cmd/kyber.conf
new file mode 100644
index 0000000..31000dd
--- /dev/null
+++ b/src/test/cfg2cmd/kyber.conf
@@ -0,0 +1,3 @@
+# TEST: Kyber console display
+memory: 2048
+vga: kyber
diff --git a/src/test/cfg2cmd/kyber.conf.cmd b/src/test/cfg2cmd/kyber.conf.cmd
new file mode 100644
index 0000000..dfb2e99
--- /dev/null
+++ b/src/test/cfg2cmd/kyber.conf.cmd
@@ -0,0 +1,27 @@
+/usr/bin/kvm
+-id 8006
+-name vm8006
+-no-shutdown
+-chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off'
+-mon 'chardev=qmp,mode=control'
+-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000'
+-mon 'chardev=qmp-event,mode=control'
+-pidfile /var/run/qemu-server/8006.pid
+-daemonize
+-smp '1,sockets=1,cores=1,maxcpus=1'
+-nodefaults
+-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg'
+-display 'dbus,addr=unix:path=/var/run/qemu-server/8006.dbusdisplay'
+-vnc 'unix:/var/run/qemu-server/8006.vnc,password=on'
+-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep
+-m 2048
+-global 'PIIX4_PM.disable_s3=1'
+-global 'PIIX4_PM.disable_s4=1'
+-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e'
+-device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f'
+-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2'
+-device 'usb-tablet,id=tablet,bus=uhci.0,port=1'
+-device 'virtio-vga,id=vga,bus=pci.0,addr=0x2'
+-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on'
+-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff'
+-machine 'type=pc+pve0'
\ No newline at end of file
diff --git a/src/usr/Makefile b/src/usr/Makefile
index 1365544..58dbb1d 100644
--- a/src/usr/Makefile
+++ b/src/usr/Makefile
@@ -22,6 +22,7 @@ install: pve-usb.cfg pve-q35.cfg pve-q35-4.0.cfg bootsplash.jpg modules-load.con
 	install -D -m 0755 dbus-vmstate $(LIBEXECDIR)/dbus-vmstate
 	install -d $(LIBSYSTEMDDIR)
 	install -D -m 0644 pve-dbus-vmstate@.service $(LIBSYSTEMDDIR)/system/pve-dbus-vmstate@.service
+	install -D -m 0644 pve-qemu-kyber@.service $(LIBSYSTEMDDIR)/system/pve-qemu-kyber@.service
 	install -d $(DBUSDIR)
 	install -D -m 0644 org.qemu.VMState1.conf $(DBUSDIR)/system.d/org.qemu.VMState1.conf
 
diff --git a/src/usr/pve-qemu-kyber@.service b/src/usr/pve-qemu-kyber@.service
new file mode 100644
index 0000000..46e394e
--- /dev/null
+++ b/src/usr/pve-qemu-kyber@.service
@@ -0,0 +1,25 @@
+[Unit]
+Description=PVE Kyber Console Controller (VM %i)
+# Tie it to the VM's scope: it goes away with the VM.
+PartOf=%i.scope
+After=%i.scope
+
+[Service]
+Slice=qemu.slice
+Type=simple
+# The adapters find their VM's QEMU here rather than on a session bus.
+Environment=KQS_DBUS_ADDR=unix:path=/var/run/qemu-server/%i.dbusdisplay
+# Carries KYBER_JWT_KEY: /proc/<pid>/cmdline is readable by every user.
+EnvironmentFile=/var/run/qemu-server/%i.kyber.env
+# No configuration file: one line is all this needs. The control plane is a unix
+# socket; the data plane needs a UDP port, and --dataplane-addr keeps it off
+# every other interface - upstream binds the wildcard.
+ExecStart=/usr/bin/kycontroller \
+    --listen-socket /var/run/qemu-server/%i.kyber.sock \
+    --dataplane-addr 127.0.0.1 \
+    --tls-cert /etc/pve/local/pve-ssl.pem \
+    --tls-key /etc/pve/local/pve-ssl.key \
+    --no-basic-auth \
+    --no-oidc-auth \
+    --no-tray
+Restart=no
-- 
2.55.0