Re: [pve-devel] [PATCH] Add patches for iothread_vq

["DERUMIER, Alexandre via pve-devel" <pve-devel@lists.proxmox.com>]

[-- Attachment #1: Type: message/rfc822, Size: 23659 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>
Subject: Re: [pve-devel] [PATCH] Add patches for iothread_vq
Date: Fri, 27 Jun 2025 07:01:46 +0000
Message-ID: <43035b37e68c44ca6c30799dfa454d1bb8b23fec.camel@groupe-cyllene.com>

Hi Dominik,

I'm going to send patch to use json format by default for -device 
options, It should help here


I have a patch for multiple iothreads too on my side (for both virtio
&& virtio-scsi), maybe we could compare implementation.

on my side, I'm using use same implementation than libvirt,

iothreads: x  , are defined globally, and shared accross all disk by
default. (redhat said that it's better to not have too much threads, or
not busy thread, so shared iothreads are better by default).

optionally, user is able to define specific iothreads list by disk


some notes: 

iothreads need to be < host cores.  (I thinked it was vm cores but
finally the iothreads are unrelated to vm vcpu)

Redhat doc said that it's not playing fine with cache=writeback &&
aio=threads  (not sure about aio=io_uring). So this need to be tested,
but maybe it need to fallback to 1thread is this case.

Also, your implementation is missing hotplug



-------- Message initial --------
De: Dominik Budzowski <dbudzowski@alfaline.pl>
Répondre à: Proxmox VE development discussion <pve-
devel@lists.proxmox.com>
À: pve-devel@lists.proxmox.com
Objet: [pve-devel] [PATCH] Add patches for iothread_vq
Date: 26/06/2025 00:15:35

---
 drive-iothread-vq-pve8.4.patch      |  33 ++++++++
 qemuserver-iothread-vq-pve8.4.patch | 123 ++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+)
 create mode 100644 drive-iothread-vq-pve8.4.patch
 create mode 100644 qemuserver-iothread-vq-pve8.4.patch

diff --git a/drive-iothread-vq-pve8.4.patch b/drive-iothread-vq-
pve8.4.patch
new file mode 100644
index 0000000..0fd39be
--- /dev/null
+++ b/drive-iothread-vq-pve8.4.patch
@@ -0,0 +1,33 @@
+--- Drive.pm	2025-06-09 18:31:45.482659331 +0200
++++ Drive.pm	2025-06-09 19:00:29.962125989 +0200
+@@ -275,6 +275,14 @@
+ 	optional => 1,
+ });
+ 
++my %iothread_vq_mapping_fmt = ( iothread_vq_mapping => {
++    type => 'integer',
++    description => "Whether to use iothread-vq-mapping for this
drive",
++    minimum => 2,
++    maximum => 16,
++    optional => 1,
++});
++
+ my %product_fmt = (
+     product => {
+ 	type => 'string',
+@@ -442,6 +450,7 @@
+ my $virtio_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %readonly_fmt,
+ };
+ my $virtiodesc = {
+@@ -537,6 +546,7 @@
+ my $alldrive_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %model_fmt,
+     %product_fmt,
+     %queues_fmt,
diff --git a/qemuserver-iothread-vq-pve8.4.patch b/qemuserver-iothread-
vq-pve8.4.patch
new file mode 100644
index 0000000..9719091
--- /dev/null
+++ b/qemuserver-iothread-vq-pve8.4.patch
@@ -0,0 +1,123 @@
+--- QemuServer.pm	2025-06-25 22:36:12.414594136 +0200
++++ QemuServer.pm	2025-06-25 23:22:38.522270102 +0200
+@@ -1301,17 +1301,79 @@
+     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
+ }
+ 
++# Helper to generate iothread/VQ mapping for block devices
++sub generate_iothread_vq_mapping {
++    my ($vmid, $drive) = @_;
++    my ($use_iothread_vq_mapping, $use_iothread, @vq_map);
++
++    if ($drive->{iothread_vq_mapping}) {
++        $use_iothread_vq_mapping = 1;
++        @vq_map = map { { iothread => "iothread-${vmid}-$_" } }
++                  0 .. $drive->{iothread_vq_mapping} - 1;
++    } elsif ($drive->{iothread}) {
++        $use_iothread = 1;
++    }
++
++    return ($use_iothread_vq_mapping, $use_iothread, \@vq_map);
++}
++
++# Main sub: JSON encoder for ordered key-value pairs and full drive
device construction
+ sub print_drivedevice_full {
+     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch,
$machine_type) = @_;
+ 
+-    my $device = '';
+-    my $maxdev = 0;
+-
++    # Compute drive ID and PCI address for virtio
+     my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
++    my $pciaddr = '';
++    if ($drive->{interface} eq 'virtio') {
++        $pciaddr = print_pci_addr($drive_id, $bridges, $arch,
$machine_type);
++    }
++
++    # Generate iothread/VQ mapping flags and mapping array
++    my ($use_iothread_vq_mapping, $use_iothread, $vq_map_ref) =
++        generate_iothread_vq_mapping($vmid, $drive);
++
++    # Prepare base JSON encoder
++    my $json = JSON->new->canonical(1);
++    my $device;
++
++    # Virtio interface handling
+     if ($drive->{interface} eq 'virtio') {
+-	my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch,
$machine_type);
+-	$device = "virtio-blk-pci,drive=drive-
$drive_id,id=${drive_id}${pciaddr}";
+-	$device .= ",iothread=iothread-$drive_id" if $drive-
>{iothread};
++        if ($use_iothread_vq_mapping) {
++	    my ($bus, $addr) = ();
++	    if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) {
++    	    ($bus, $addr) = ($1, $2);
++	    }
++            my @fields = (
++                [ driver              => 'virtio-blk-pci'   ],
++                [ 'iothread-vq-mapping' => $vq_map_ref      ],
++                [ 'queue-size'        => 1024               ],
++                [ 'config-wce'        => JSON::false        ],
++                [ drive               => "drive-$drive_id"  ],
++                [ id                  => $drive_id          ],
++                [ bus                 => $bus               ],
++                [ addr                => $addr              ],
++                ($drive->{bootindex} ? [ bootindex => $drive-
>{bootindex} ] : ()),
++            );
++            my @parts;
++            for my $fld (@fields) {
++                my ($k, $v) = @$fld;
++                push @parts, $json->encode($k) . ':' . $json-
>encode($v);
++            }
++            $device = '{' . join(',', @parts) . '}';
++        }
++        elsif ($use_iothread) {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-
%s,id=%s%s,iothread=iothread-%s',
++                $drive_id, $drive_id, $pciaddr, $drive_id
++            );
++        }
++        else {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-%s,id=%s%s',
++                $drive_id, $drive_id, $pciaddr
++            );
++        }
++
+     } elsif ($drive->{interface} eq 'scsi') {
+ 
+ 	my ($maxdev, $controller, $controller_prefix) =
scsihw_infos($conf, $drive);
+@@ -1386,7 +1448,7 @@
+ 	die "unsupported interface type";
+     }
+ 
+-    $device .= ",bootindex=$drive->{bootindex}" if $drive-
>{bootindex};
++    $device .= ",bootindex=$drive->{bootindex}" if $drive-
>{bootindex} && !$use_iothread_vq_mapping;
+ 
+     if (my $serial = $drive->{serial}) {
+ 	$serial = URI::Escape::uri_unescape($serial);
+@@ -3911,8 +3973,23 @@
+ 
+ 	$drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
+ 
+-	if ($drive->{interface} eq 'virtio'){
+-           push @$cmd, '-object', "iothread,id=iothread-$ds" if
$drive->{iothread};
++#	if ($drive->{interface} eq 'virtio'){
++#           push @$cmd, '-object', "iothread,id=iothread-$ds" if
$drive->{iothread};
++#	}
++
++	if ($drive->{interface} eq 'virtio') {
++
++	    if ($drive->{iothread_vq_mapping}) {
++
++		for my $i (0 .. $drive->{iothread_vq_mapping} - 1) {
++		    my $id = "iothread,id=iothread-$vmid-$i";
++		    push @$cmd, ('-object', $id) unless grep { $_ eq
$id } @$cmd;
++		}
++	    } elsif ($drive->{iothread}) {
++
++		my $id = "iothread,id=iothread-$ds";
++		push @$cmd, ('-object', $id) unless grep { $_ eq $id }
@$cmd;
++	    }
+ 	}
+ 
+ 	if ($drive->{interface} eq 'scsi') {

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel