From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 582BE1FF142 for ; Fri, 05 Jun 2026 21:40:59 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 098EA32E78; Fri, 5 Jun 2026 21:40:55 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=budzowski.pl; s=default; t=1780672528; h=from:from:reply-to:subject:subject:date:date:message-id:message-id: to:to:cc:cc:mime-version:mime-version: content-transfer-encoding:content-transfer-encoding; bh=weXYh0Fw+D3eKxmUS8fT1+DZKRV63EhiztOP4i78y/Y=; b=q2tL5TN1/1efEIaEPB89BQRRMs3meUo0Gfw2MKKDlOtygCy4hQ6+iL9xedbrbb4nf3XrS+ C1Q3Ka2hnUpzRraPqn1tKhloG1DwL6Ia3M0GHYLqR/v1jjsj/1tio/Mtt4DdgKJUjNOJlp +ZxjdJjkwi8IEwz/jtQ3rycuOFHMDuhrFdZ1D6ahkCb8X2MAkdsV3qdx2B3L92QLsOeKIU JeYZren/UH+rmzjAOGDXr9v5QXf0lRNjmaHDVtLk66fy73Rqlf+cFsNXGufvW+qoweQilw WQYOhrqLbE7irNIs2NuZMmGuNtlgLkqPSk2nMfY35rv7aeB7zTKHfEXkNUZdRw== Authentication-Results: mail.alfaline.pl; auth=pass smtp.auth=dominik@budzowski.pl smtp.mailfrom=dominik@budzowski.pl From: Dominik Budzowski To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server] qemu-server: add iothread-vq-mapping support for virtio/scsi drives Date: Fri, 5 Jun 2026 17:15:27 +0200 Message-ID: <20260605151527.6538-1-dominik@budzowski.pl> X-Mailer: git-send-email 2.47.3 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 BAYES_00 -1.9 Bayes spam probability is 0 to 1% DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record X-MailFrom: dominik@budzowski.pl X-Mailman-Rule-Hits: nonmember-moderation X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation Message-ID-Hash: E3D3F2TE7UUIX6LIT7JWEZYYGQKF7W4X X-Message-ID-Hash: E3D3F2TE7UUIX6LIT7JWEZYYGQKF7W4X X-Mailman-Approved-At: Fri, 05 Jun 2026 21:40:56 +0200 CC: Dominik Budzowski X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a new drive option 'iothread_vq_mapping' (integer, 2-16) that enables QEMU's iothread-vq-mapping feature. Unlike the existing per-drive iothread option, vq-mapping assigns a pool of N iothreads shared across all VQs of a virtio-blk-pci or virtio-scsi-single device, using QEMU's JSON device syntax to pass the mapping array. Changes: - Drive.pm: define %iothread_vq_mapping_fmt and add it to scsi, virtio and alldrive formats - DriveDevice.pm: add _generate_iothread_vq_mapping() helper; emit JSON-syntax device string for virtio-blk-pci and virtio-scsi-single when iothread_vq_mapping is set - QemuServer.pm: create/reuse shared iothread objects keyed by vmid+index at VM start and hotplug; treat iothread_vq_mapping change as requiring VM restart (same as iothread) Resolves: https://bugzilla.proxmox.com/show_bug.cgi?id=6350 Signed-off-by: Dominik Budzowski --- src/PVE/QemuServer.pm | 121 ++++++++++++++++++++++++------ src/PVE/QemuServer/Drive.pm | 13 +++- src/PVE/QemuServer/DriveDevice.pm | 74 +++++++++++++++++- 3 files changed, 182 insertions(+), 26 deletions(-) diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm index 55e9f52..cb85971 100644 --- a/src/PVE/QemuServer.pm +++ b/src/PVE/QemuServer.pm @@ -3542,7 +3542,14 @@ sub config_to_command { $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds}; if ($drive->{interface} eq 'virtio') { - push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread}; + if ($drive->{iothread_vq_mapping}) { + for my $i (0 .. $drive->{iothread_vq_mapping} - 1) { + my $id = "iothread,id=iothread-$vmid-$i"; + push @$cmd, ('-object', $id) unless grep { $_ eq $id } @$cmd; + } + } elsif ($drive->{iothread}) { + push @$cmd, '-object', "iothread,id=iothread-$ds"; + } } if ($drive->{interface} eq 'scsi') { @@ -3559,14 +3566,19 @@ sub config_to_command { $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw; my $iothread = ''; - if ( - $conf->{scsihw} - && $conf->{scsihw} eq "virtio-scsi-single" - && $drive->{iothread} - ) { - $iothread .= ",iothread=iothread-$controller_prefix$controller"; - push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller"; - } elsif ($drive->{iothread}) { + my $use_vq_mapping_scsi = 0; + if ($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single") { + if ($drive->{iothread_vq_mapping}) { + $use_vq_mapping_scsi = 1; + for my $i (0 .. $drive->{iothread_vq_mapping} - 1) { + my $id = "iothread,id=iothread-$vmid-$i"; + push @$cmd, ('-object', $id) unless grep { $_ eq $id } @$cmd; + } + } elsif ($drive->{iothread}) { + $iothread = ",iothread=iothread-$controller_prefix$controller"; + push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller"; + } + } elsif ($drive->{iothread} || $drive->{iothread_vq_mapping}) { log_warn( "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n" ); @@ -3581,10 +3593,35 @@ sub config_to_command { $queues = ",num_queues=$drive->{queues}"; } - push @$devices, '-device', - "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" - if !$scsicontroller->{$controller}; - $scsicontroller->{$controller} = 1; + if (!$scsicontroller->{$controller}) { + if ($use_vq_mapping_scsi) { + my @vq_map = map { { iothread => "iothread-${vmid}-$_" } } + 0 .. $drive->{iothread_vq_mapping} - 1; + my ($bus, $addr) = ('', ''); + if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) { + ($bus, $addr) = ($1, $2); + } + my $json = JSON->new->canonical(1); + my @fields = ( + [ driver => $scsihw_type ], + [ 'iothread-vq-mapping' => \@vq_map ], + [ id => "$controller_prefix$controller" ], + [ bus => $bus ], + [ addr => $addr ], + ); + push @fields, [ num_queues => int($drive->{queues}) ] if $drive->{queues}; + my @parts; + for my $fld (@fields) { + my ($k, $v) = @$fld; + push @parts, $json->encode($k) . ':' . $json->encode($v); + } + push @$devices, '-device', '{' . join(',', @parts) . '}'; + } else { + push @$devices, '-device', + "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"; + } + $scsicontroller->{$controller} = 1; + } } if ($drive->{interface} eq 'sata') { @@ -3883,13 +3920,25 @@ sub vm_deviceplug { PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1), ); } elsif ($deviceid =~ m/^(virtio)(\d+)$/) { - qemu_iothread_add($vmid, $deviceid, $device); + if ($device->{iothread_vq_mapping}) { + my $iothreads = vm_iothreads_list($vmid); + for my $i (0 .. $device->{iothread_vq_mapping} - 1) { + qemu_objectadd($vmid, "iothread-$vmid-$i", "iothread") + if !$iothreads->{"iothread-$vmid-$i"}; + } + } else { + qemu_iothread_add($vmid, $deviceid, $device); + } qemu_driveadd($storecfg, $vmid, $device); my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type); - qemu_deviceadd($vmid, $devicefull); + if ($device->{iothread_vq_mapping}) { + mon_cmd($vmid, "device_add", %{ decode_json($devicefull) }); + } else { + qemu_deviceadd($vmid, $devicefull); + } eval { qemu_deviceaddverify($vmid, $deviceid); }; if (my $err = $@) { eval { qemu_drivedel($vmid, $deviceid); }; @@ -3901,18 +3950,41 @@ sub vm_deviceplug { my $pciaddr = print_pci_addr($deviceid, undef, $arch); my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw; - my $devicefull = "$scsihw_type,id=$deviceid$pciaddr"; + if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread_vq_mapping}) { + my $iothreads = vm_iothreads_list($vmid); + for my $i (0 .. $device->{iothread_vq_mapping} - 1) { + qemu_objectadd($vmid, "iothread-$vmid-$i", "iothread") + if !$iothreads->{"iothread-$vmid-$i"}; + } + my @vq_map = map { { iothread => "iothread-${vmid}-$_" } } + 0 .. $device->{iothread_vq_mapping} - 1; + my ($bus, $addr) = ('', ''); + if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) { + ($bus, $addr) = ($1, $2); + } + mon_cmd( + $vmid, "device_add", + driver => $scsihw_type, + id => $deviceid, + bus => $bus, + addr => $addr, + 'iothread-vq-mapping' => \@vq_map, + ($device->{queues} ? (num_queues => int($device->{queues})) : ()), + ); + } else { + my $devicefull = "$scsihw_type,id=$deviceid$pciaddr"; + + if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) { + qemu_iothread_add($vmid, $deviceid, $device); + $devicefull .= ",iothread=iothread-$deviceid"; + } - if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) { - qemu_iothread_add($vmid, $deviceid, $device); - $devicefull .= ",iothread=iothread-$deviceid"; - } + if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) { + $devicefull .= ",num_queues=$device->{queues}"; + } - if ($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) { - $devicefull .= ",num_queues=$device->{queues}"; + qemu_deviceadd($vmid, $devicefull); } - - qemu_deviceadd($vmid, $devicefull); qemu_deviceaddverify($vmid, $deviceid); } elsif ($deviceid =~ m/^(scsi)(\d+)$/) { qemu_findorcreatescsihw($storecfg, $conf, $vmid, $device, $arch, $machine_type); @@ -5211,6 +5283,7 @@ sub vmconfig_update_disk { safe_string_ne($drive->{aio}, $old_drive->{aio}) || safe_string_ne($drive->{discard}, $old_drive->{discard}) || safe_string_ne($drive->{iothread}, $old_drive->{iothread}) + || safe_string_ne($drive->{iothread_vq_mapping}, $old_drive->{iothread_vq_mapping}) || safe_string_ne($drive->{queues}, $old_drive->{queues}) || safe_string_ne($drive->{product}, $old_drive->{product}) || safe_string_ne($drive->{cache}, $old_drive->{cache}) diff --git a/src/PVE/QemuServer/Drive.pm b/src/PVE/QemuServer/Drive.pm index b80b7db..647a475 100644 --- a/src/PVE/QemuServer/Drive.pm +++ b/src/PVE/QemuServer/Drive.pm @@ -266,6 +266,15 @@ my %iothread_fmt = ( }, ); +my %iothread_vq_mapping_fmt = ( + iothread_vq_mapping => { + type => 'integer', + description => "Whether to use iothread-vq-mapping for this drive", + minimum => 2, + maximum => 16, + optional => 1, +}); + my %product_fmt = ( product => { type => 'string', @@ -463,6 +472,7 @@ PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc); my $scsi_fmt = { %drivedesc_base, %iothread_fmt, + %iothread_vq_mapping_fmt, %product_fmt, %queues_fmt, %readonly_fmt, @@ -493,7 +503,7 @@ my $satadesc = { PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc); my $virtio_fmt = { - %drivedesc_base, %iothread_fmt, %readonly_fmt, + %drivedesc_base, %iothread_fmt, %iothread_vq_mapping_fmt, %readonly_fmt, }; my $virtiodesc = { optional => 1, @@ -606,6 +616,7 @@ use constant TPMSTATE_DISK_SIZE => 4 * 1024 * 1024; my $alldrive_fmt = { %drivedesc_base, %iothread_fmt, + %iothread_vq_mapping_fmt, %model_fmt, %product_fmt, %queues_fmt, diff --git a/src/PVE/QemuServer/DriveDevice.pm b/src/PVE/QemuServer/DriveDevice.pm index 37b611f..dbf7b92 100644 --- a/src/PVE/QemuServer/DriveDevice.pm +++ b/src/PVE/QemuServer/DriveDevice.pm @@ -3,6 +3,7 @@ package PVE::QemuServer::DriveDevice; use strict; use warnings; +use JSON; use URI::Escape; use PVE::QemuServer::Drive qw (drive_is_cdrom); @@ -49,6 +50,21 @@ sub scsihw_infos { return ($maxdev, $controller, $controller_prefix); } +sub _generate_iothread_vq_mapping { + my ($vmid, $drive) = @_; + my ($use_iothread_vq_mapping, $use_iothread, @vq_map); + + if ($drive->{iothread_vq_mapping}) { + $use_iothread_vq_mapping = 1; + @vq_map = map { { iothread => "iothread-${vmid}-$_" } } + 0 .. $drive->{iothread_vq_mapping} - 1; + } elsif ($drive->{iothread}) { + $use_iothread = 1; + } + + return ($use_iothread_vq_mapping, $use_iothread, \@vq_map); +} + sub print_drivedevice_full { my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_; @@ -62,13 +78,46 @@ sub print_drivedevice_full { my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive); if ($drive->{interface} eq 'virtio') { my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch); + + my ($use_iothread_vq_mapping, $use_iothread, $vq_map_ref) = + _generate_iothread_vq_mapping($vmid, $drive); + + if ($use_iothread_vq_mapping) { + my ($bus, $addr) = ('', ''); + if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) { + ($bus, $addr) = ($1, $2); + } + + my $json = JSON->new->canonical(1); + my @fields = ( + [ driver => 'virtio-blk-pci' ], + [ 'iothread-vq-mapping' => $vq_map_ref ], + [ 'queue-size' => 1024 ], + [ 'config-wce' => JSON::false ], + [ drive => "drive-$drive_id" ], + [ id => $drive_id ], + [ bus => $bus ], + [ addr => $addr ], + ($drive->{bootindex} ? [ bootindex => $drive->{bootindex} ] : ()), + ); + + my @parts; + for my $fld (@fields) { + my ($k, $v) = @$fld; + push @parts, $json->encode($k) . ':' . $json->encode($v); + } + return '{' . join(',', @parts) . '}'; + } + $device = 'virtio-blk-pci'; # for the switch to -blockdev, there is no blockdev for 'none' if (!min_version($machine_version, 10, 0) || $drive->{file} ne 'none') { $device .= ",drive=drive-$drive_id"; } $device .= ",id=${drive_id}${pciaddr}"; - $device .= ",iothread=iothread-$drive_id" if $drive->{iothread}; + if ($use_iothread) { + $device .= ",iothread=iothread-$drive_id"; + } } elsif ($drive->{interface} eq 'scsi') { my ($maxdev, $controller, $controller_prefix) = @@ -192,6 +241,29 @@ sub print_drivedevice_full { } } + if ($drive->{interface} eq 'scsi' + && $drive->{iothread_vq_mapping} + && $conf->{scsihw} + && $conf->{scsihw} eq 'virtio-scsi-single') + { + my @parts = split(/,/, $device); + my $driver = shift @parts; + my $json = JSON->new->canonical(1); + my %int_props = map { $_ => 1 } qw(channel scsi-id lun rotation_rate bootindex); + my @fields = ([ driver => $driver ]); + for my $part (@parts) { + next unless $part =~ /^([^=]+)=(.*)$/; + my ($k, $v) = ($1, $2); + push @fields, [$k, $int_props{$k} ? int($v) : $v]; + } + my @json_parts; + for my $fld (@fields) { + my ($k, $v) = @$fld; + push @json_parts, $json->encode($k) . ':' . $json->encode($v); + } + return '{' . join(',', @json_parts) . '}'; + } + return $device; } -- 2.47.3