From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 86DC91FF0B7 for ; Tue, 25 Aug 2026 13:09:43 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 7D9362167C; Tue, 25 Aug 2026 13:09:15 +0200 (CEST) From: Alexandre Derumier To: pve-devel@lists.proxmox.com subject: SPAM: [RFC qemu-server 03/13] add kyber display Date: Tue, 25 Aug 2026 13:08:35 +0200 Message-ID: <20260825110849.2967694-4-alexandre.derumier@groupe-cyllene.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260825110849.2967694-1-alexandre.derumier@groupe-cyllene.com> References: <20260825110849.2967694-1-alexandre.derumier@groupe-cyllene.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 6 AWL 0.100 Adjusted score from AWL reputation of From: address DMARC_QUAR 0.1 DMARC quarantine policy HEADER_FROM_DIFFERENT_DOMAINS 0.25 From and EnvelopeFrom 2nd level mail domains are different KAM_DMARC_QUARANTINE 4 DKIM has Failed or SPF has failed on the message and the domain has a DMARC quarantine policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods KAM_SHORT 0.001 Use of a URL Shortener for very short URL RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Message-ID-Hash: 4YPMIHGSA44J4N5HWRLP4L66UXQ6M74T X-Message-ID-Hash: 4YPMIHGSA44J4N5HWRLP4L66UXQ6M74T X-MailFrom: root@formationkvm1.odiso.net X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Alexandre Derumier X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: From: Alexandre Derumier kyber maps to virtio-vga, with a controller started per VM on demand by the kyberproxy API call. Signed-off-by: Alexandre Derumier --- src/PVE/API2/Qemu.pm | 94 +++++++++++++++++++++++ src/PVE/QemuServer.pm | 50 ++++++++++++- src/PVE/QemuServer/Kyber.pm | 128 ++++++++++++++++++++++++++++++++ src/PVE/QemuServer/Makefile | 2 + src/test/cfg2cmd/kyber.conf | 3 + src/test/cfg2cmd/kyber.conf.cmd | 27 +++++++ src/usr/Makefile | 1 + src/usr/pve-qemu-kyber@.service | 25 +++++++ 8 files changed, 326 insertions(+), 4 deletions(-) create mode 100644 src/PVE/QemuServer/Kyber.pm create mode 100644 src/test/cfg2cmd/kyber.conf create mode 100644 src/test/cfg2cmd/kyber.conf.cmd create mode 100644 src/usr/pve-qemu-kyber@.service diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm index 71247ee..55befc1 100644 --- a/src/PVE/API2/Qemu.pm +++ b/src/PVE/API2/Qemu.pm @@ -12,6 +12,7 @@ use IPC::Open3; use JSON; use URI::Escape; use Socket qw(SOCK_STREAM); +use Time::HiRes qw(usleep); use PVE::APIClient::LWP; use PVE::CGroup; @@ -35,6 +36,7 @@ use PVE::QemuServer::Cloudinit; use PVE::QemuServer::CPUConfig; use PVE::QemuServer::Drive qw(checked_volume_format checked_parse_volname); use PVE::QemuServer::Helpers; +use PVE::QemuServer::Kyber; use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd vm_qmp_peer); use PVE::QemuServer::Machine; @@ -3329,6 +3331,92 @@ __PACKAGE__->register_method({ }, }); +__PACKAGE__->register_method({ + name => 'kyberproxy', + path => '{vmid}/kyberproxy', + method => 'POST', + protected => 1, + proxyto => 'node', + permissions => { + check => ['perm', '/vms/{vmid}', ['VM.Console']], + }, + description => "Start a Kyber console controller for the VM and return how to reach it.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid'), + }, + }, + returns => { + additionalProperties => 0, + properties => { + user => { type => 'string' }, + ticket => { + type => 'string', + description => "Short-lived token authenticating this user to the controller.", + }, + }, + }, + code => sub { + my ($param) = @_; + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $vmid = $param->{vmid}; + my $node = $param->{node}; + + my $conf = PVE::QemuConfig->load_config($vmid, $node); + + my $vga = PVE::QemuServer::parse_vga($conf->{vga} // ''); + die "VM $vmid is not configured for the Kyber console" + . " - set its display to 'kyber' and restart it\n" + if ($vga->{type} // '') ne 'kyber'; + + die "VM $vmid is not running\n" if !PVE::QemuServer::Helpers::vm_running_locally($vmid); + + my $socket = PVE::QemuServer::Helpers::dbus_socket($vmid); + die "VM $vmid has no D-Bus display socket at $socket" + . " - it was started before its display was set to 'kyber'," + . " so it needs a restart\n" + if !-S $socket; + + # Join a controller that is already streaming: it shares one capture between + # clients, and restarting to install a new secret would cut the first off. + my ($secret, $port) = PVE::QemuServer::Kyber::running_secret($vmid); + + if (!$secret) { + my $family = PVE::Tools::get_host_address_family($node); + $port = PVE::QemuServer::Kyber::next_port($family); + + # Fresh per controller: with none running there is nothing to cut off. + $secret = PVE::QemuServer::Kyber::generate_secret(); + + # Only 'vnc' adds the vdagent chardev the guest needs to share a clipboard. + my $clipboard = ($vga->{clipboard} // '') eq 'vnc'; + + PVE::QemuServer::Kyber::write_env($vmid, $secret, $port, $clipboard); + PVE::QemuServer::Kyber::restart_controller($vmid); + } + + my $ticket = PVE::QemuServer::Kyber::assemble_ticket($secret, $authuser); + + # Listening within ~30ms, so waiting here saves the client a retry loop. + my $kybersocket = PVE::QemuServer::Kyber::socket_file($vmid); + for (my $waited = 0; $waited < 5; $waited += 0.05) { + last if -S $kybersocket; + usleep(50_000); + } + die "Kyber console controller for VM $vmid did not start\n" if !-S $kybersocket; + + return { + user => $authuser, + ticket => $ticket, + }; + }, +}); + __PACKAGE__->register_method({ name => 'spiceproxy', path => '{vmid}/spiceproxy', @@ -3452,6 +3540,11 @@ __PACKAGE__->register_method({ type => 'boolean', optional => 1, }, + kyber => { + description => "QEMU VGA configuration supports the Kyber console.", + type => 'boolean', + optional => 1, + }, agent => { description => "QEMU Guest Agent is enabled in config.", type => 'boolean', @@ -3482,6 +3575,7 @@ __PACKAGE__->register_method({ my $spice = defined($vga->{type}) && $vga->{type} =~ /^virtio/; $spice ||= PVE::QemuServer::vga_conf_has_spice($conf->{vga}); $status->{spice} = 1 if $spice; + $status->{kyber} = 1 if ($vga->{type} // '') eq 'kyber'; $status->{clipboard} = $vga->{clipboard}; } $status->{agent} = 1 if PVE::QemuServer::Agent::get_qga_key($conf, 'enabled'); diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm index 2f43faa..8c7f023 100644 --- a/src/PVE/QemuServer.pm +++ b/src/PVE/QemuServer.pm @@ -80,6 +80,7 @@ use PVE::QemuServer::Drive qw( storage_allows_io_uring_default ); use PVE::QemuServer::DriveDevice qw(print_drivedevice_full scsihw_infos); +use PVE::QemuServer::Kyber; use PVE::QemuServer::Machine; use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::MetaInfo; @@ -98,6 +99,7 @@ use PVE::QemuServer::StateFile; use PVE::QemuServer::USB; use PVE::QemuServer::Virtiofs qw(max_virtiofs start_all_virtiofsd); use PVE::QemuServer::VolumeChain; +use PVE::QemuServer::DBusDisplay; use PVE::QemuServer::DBusVMState; my $have_ha_config; @@ -168,7 +170,7 @@ my $vga_fmt = { optional => 1, default_key => 1, enum => [ - qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware) + qw(cirrus kyber qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware) ], }, memory => { @@ -213,10 +215,12 @@ my $audio_fmt = { }, driver => { type => 'string', - enum => ['spice', 'none'], + enum => ['spice', 'dbus', 'none'], default => 'spice', optional => 1, - description => "Driver backend for the audio device.", + description => "Driver backend for the audio device." + . " 'dbus' exposes it on the VM's D-Bus display, which is what the" + . " Kyber and RDP consoles read.", }, }; @@ -1481,6 +1485,9 @@ my $vga_map = { 'vmware' => 'vmware-svga', 'virtio' => 'virtio-vga', 'virtio-gl' => 'virtio-vga-gl', + # A display transport, not a card, so it picks one: virtio-vga rather than a GL + # variant, both of which currently break QEMU. + 'kyber' => 'virtio-vga', }; # QEMU builds only the non-VGA variants of the virtio GPU for aarch64 @@ -1488,6 +1495,7 @@ my $vga_map_aarch64 = { $vga_map->%*, 'virtio' => 'virtio-gpu', 'virtio-gl' => 'virtio-gpu-gl', + 'kyber' => 'virtio-gpu', }; my sub map_vga_model { @@ -2839,7 +2847,14 @@ sub audio_devs { die "unknown audio device '$audio->{dev}', implement me!"; } - push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}"; + my $backend = "$audio->{backend},id=$audio->{backend_id}"; + + # Pinned for the D-Bus backend: what reads it is an Opus encoder and libopus takes + # 48kHz only. QEMU already resamples, so this costs nothing new. + $backend .= ',out.frequency=48000,out.channels=2,out.format=s16' + if $audio->{backend} eq 'dbus'; + + push @$devs, '-audiodev', $backend; return $devs; } @@ -3408,6 +3423,19 @@ sub config_to_command { push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL + if ($vga->{type} eq 'kyber') { + my $dbus = PVE::QemuServer::Helpers::dbus_socket($vmid); + my $display = "dbus,addr=unix:path=$dbus"; + + # The display exports org.qemu.Display1.Audio only when told which audiodev to + # read, and nothing else can consume a dbus audiodev. + my $audio = conf_has_audio($conf); + $display .= ",audiodev=$audio->{backend_id}" + if $audio && $audio->{backend} eq 'dbus'; + + push @$cmd, '-display', $display; + } + my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid); push @$cmd, '-vnc', "unix:$socket,password=on"; } else { @@ -5812,6 +5840,11 @@ sub vm_start_nolock { my $virtiofs_sockets = start_all_virtiofsd($conf, $vmid); + # QEMU connects to the D-Bus address, so the bus has to be listening first. + my $dbus_vga = parse_vga($conf->{vga} // ''); + PVE::QemuServer::DBusDisplay::start($vmid) + if ($dbus_vga->{type} // '') eq 'kyber'; + my $tpmpid; if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) { # start the TPM emulator so QEMU can connect on start @@ -6196,6 +6229,15 @@ sub vm_stop_cleanup { my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes, $noerr, $skip_hookscript) = @_; + # Before the cleanup flag is consulted, deliberately: the bus is forked into the + # VM's systemd scope, and a survivor keeps that cgroup from emptying, so the next + # start fails with "timeout waiting on systemd". + eval { + PVE::QemuServer::Kyber::stop_controller($vmid); + PVE::QemuServer::DBusDisplay::stop($vmid); + }; + warn $@ if $@; + my $can_use_cleanup_flag = PVE::QemuServer::RunState::can_use_cleanup_flag(); if ($can_use_cleanup_flag) { return if !PVE::QemuServer::RunState::cleanup_flag_exists($vmid); diff --git a/src/PVE/QemuServer/Kyber.pm b/src/PVE/QemuServer/Kyber.pm new file mode 100644 index 0000000..3115f82 --- /dev/null +++ b/src/PVE/QemuServer/Kyber.pm @@ -0,0 +1,128 @@ +package PVE::QemuServer::Kyber; + +# Per-VM Kyber console controller: one kycontroller per VM, spawned on demand and +# reaped when it exits. + +use strict; +use warnings; + +use Digest::SHA qw(hmac_sha256); +use JSON; +use Crypt::OpenSSL::Random; +use MIME::Base64 qw(encode_base64url); + +use PVE::Tools qw(file_set_contents); +use PVE::QemuServer::DBusDisplay; +use PVE::QemuServer::Helpers; + +sub socket_file { + my ($vmid) = @_; + return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.kyber.sock"; +} + +# Carries the signing key, and only that: /proc//cmdline is world-readable, +# so it cannot go on the command line. systemd passes it as KYBER_JWT_KEY. +sub env_file { + my ($vmid) = @_; + return "$PVE::QemuServer::Helpers::var_run_tmpdir/$vmid.kyber.env"; +} + +sub next_port { + my ($family) = @_; + return PVE::Tools::next_unused_port(63000, 63099, $family, '127.0.0.1'); +} + +sub generate_secret { + my $bytes = Crypt::OpenSSL::Random::random_bytes(32) + or die "unable to generate a random secret\n"; + return unpack('H*', $bytes); +} + +# An HS256 token the controller will accept. The secret is regenerated whenever a +# controller starts and never leaves the node, so it is useless against other VMs. +sub assemble_ticket { + my ($secret, $username, $ttl) = @_; + + # As long as a VNC ticket, and for the same reason: the client re-presents it to + # renew its session. It is worth little alone - the controller is on loopback. + $ttl //= 3600; + my $now = time(); + + my $header = encode_base64url(encode_json({ alg => 'HS256', typ => 'JWT' })); + # The controller requires aud=kyber (auth/jwt.rs) and rejects a token without it + # as malformed, which reads like a signing failure. + my $claims = encode_base64url( + encode_json({ + aud => 'kyber', + sub => $username, + iat => $now, + exp => $now + $ttl, + }), + ); + + my $signature = encode_base64url(hmac_sha256("$header.$claims", $secret)); + + return "$header.$claims.$signature"; +} + +# The secret a running controller is verifying against, or undef when there is +# none. Kept in a root-only env file so a second console can join instead of +# restarting the controller and cutting the first viewer off. +sub running_secret { + my ($vmid) = @_; + + return undef if !-S socket_file($vmid); + + my $env = eval { PVE::Tools::file_get_contents(env_file($vmid)) }; + return undef if !defined($env); + + my ($secret) = $env =~ m/^KYBER_JWT_KEY=(\S+)$/m; + my ($port) = $env =~ m/^KYBER_DATAPLANE_PORT=(\d+)$/m; + return undef if !$secret || !$port; + + return ($secret, $port); +} + +sub write_env { + my ($vmid, $secret, $dataplane_port, $clipboard) = @_; + + my $clipboard_env = $clipboard ? 1 : 0; + + my $env = <<"EOF"; +KYBER_JWT_KEY=$secret +KYBER_DATAPLANE_PORT=$dataplane_port +KQS_CLIPBOARD=$clipboard_env +EOF + + my $path = env_file($vmid); + file_set_contents($path, $env, 0600); + + return $path; +} + +sub restart_controller { + my ($vmid) = @_; + + PVE::Tools::run_command( + ['systemctl', 'restart', "pve-qemu-kyber\@$vmid"], + errmsg => "failed to start Kyber console controller for VM $vmid", + ); + + return; +} + +sub stop_controller { + my ($vmid) = @_; + + eval { + PVE::Tools::run_command(['systemctl', 'stop', "pve-qemu-kyber\@$vmid"]); + }; + warn $@ if $@; + + unlink env_file($vmid); + unlink socket_file($vmid); + + return; +} + +1; diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile index 060fac2..061d61f 100644 --- a/src/PVE/QemuServer/Makefile +++ b/src/PVE/QemuServer/Makefile @@ -10,11 +10,13 @@ SOURCES=Agent.pm \ Cloudinit.pm \ CPUConfig.pm \ CPUFlags.pm \ + DBusDisplay.pm \ DBusVMState.pm \ Drive.pm \ DriveDevice.pm \ Helpers.pm \ ImportDisk.pm \ + Kyber.pm \ Machine.pm \ Memory.pm \ MetaInfo.pm \ diff --git a/src/test/cfg2cmd/kyber.conf b/src/test/cfg2cmd/kyber.conf new file mode 100644 index 0000000..31000dd --- /dev/null +++ b/src/test/cfg2cmd/kyber.conf @@ -0,0 +1,3 @@ +# TEST: Kyber console display +memory: 2048 +vga: kyber diff --git a/src/test/cfg2cmd/kyber.conf.cmd b/src/test/cfg2cmd/kyber.conf.cmd new file mode 100644 index 0000000..dfb2e99 --- /dev/null +++ b/src/test/cfg2cmd/kyber.conf.cmd @@ -0,0 +1,27 @@ +/usr/bin/kvm +-id 8006 +-name vm8006 +-no-shutdown +-chardev 'socket,id=qmp,path=/var/run/qemu-server/8006.qmp,server=on,wait=off' +-mon 'chardev=qmp,mode=control' +-chardev 'socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect-ms=5000' +-mon 'chardev=qmp-event,mode=control' +-pidfile /var/run/qemu-server/8006.pid +-daemonize +-smp '1,sockets=1,cores=1,maxcpus=1' +-nodefaults +-boot 'menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg' +-display 'dbus,addr=unix:path=/var/run/qemu-server/8006.dbusdisplay' +-vnc 'unix:/var/run/qemu-server/8006.vnc,password=on' +-cpu kvm64,enforce,+kvm_pv_eoi,+kvm_pv_unhalt,+lahf_lm,+sep +-m 2048 +-global 'PIIX4_PM.disable_s3=1' +-global 'PIIX4_PM.disable_s4=1' +-device 'pci-bridge,id=pci.1,chassis_nr=1,bus=pci.0,addr=0x1e' +-device 'pci-bridge,id=pci.2,chassis_nr=2,bus=pci.0,addr=0x1f' +-device 'piix3-usb-uhci,id=uhci,bus=pci.0,addr=0x1.0x2' +-device 'usb-tablet,id=tablet,bus=uhci.0,port=1' +-device 'virtio-vga,id=vga,bus=pci.0,addr=0x2' +-device 'virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3,free-page-reporting=on' +-iscsi 'initiator-name=iqn.1993-08.org.debian:01:aabbccddeeff' +-machine 'type=pc+pve0' \ No newline at end of file diff --git a/src/usr/Makefile b/src/usr/Makefile index 1365544..58dbb1d 100644 --- a/src/usr/Makefile +++ b/src/usr/Makefile @@ -22,6 +22,7 @@ install: pve-usb.cfg pve-q35.cfg pve-q35-4.0.cfg bootsplash.jpg modules-load.con install -D -m 0755 dbus-vmstate $(LIBEXECDIR)/dbus-vmstate install -d $(LIBSYSTEMDDIR) install -D -m 0644 pve-dbus-vmstate@.service $(LIBSYSTEMDDIR)/system/pve-dbus-vmstate@.service + install -D -m 0644 pve-qemu-kyber@.service $(LIBSYSTEMDDIR)/system/pve-qemu-kyber@.service install -d $(DBUSDIR) install -D -m 0644 org.qemu.VMState1.conf $(DBUSDIR)/system.d/org.qemu.VMState1.conf diff --git a/src/usr/pve-qemu-kyber@.service b/src/usr/pve-qemu-kyber@.service new file mode 100644 index 0000000..46e394e --- /dev/null +++ b/src/usr/pve-qemu-kyber@.service @@ -0,0 +1,25 @@ +[Unit] +Description=PVE Kyber Console Controller (VM %i) +# Tie it to the VM's scope: it goes away with the VM. +PartOf=%i.scope +After=%i.scope + +[Service] +Slice=qemu.slice +Type=simple +# The adapters find their VM's QEMU here rather than on a session bus. +Environment=KQS_DBUS_ADDR=unix:path=/var/run/qemu-server/%i.dbusdisplay +# Carries KYBER_JWT_KEY: /proc//cmdline is readable by every user. +EnvironmentFile=/var/run/qemu-server/%i.kyber.env +# No configuration file: one line is all this needs. The control plane is a unix +# socket; the data plane needs a UDP port, and --dataplane-addr keeps it off +# every other interface - upstream binds the wildcard. +ExecStart=/usr/bin/kycontroller \ + --listen-socket /var/run/qemu-server/%i.kyber.sock \ + --dataplane-addr 127.0.0.1 \ + --tls-cert /etc/pve/local/pve-ssl.pem \ + --tls-key /etc/pve/local/pve-ssl.key \ + --no-basic-auth \ + --no-oidc-auth \ + --no-tray +Restart=no -- 2.55.0