[pve-devel] [PATCH qemu-server] Add iothread_vq_mapping support for virtio-blk (PVE 8.4)

[pve-devel] [PATCH qemu-server] Add iothread_vq_mapping support for virtio-blk (PVE 8.4)

[Dominik Budzowski]

Cover letter for series of patches adding iothread_vq_mapping support to
virtio-blk devices in Proxmox VE 8.4.

This feature was introduced in QEMU 9 and allows mapping multiple vhost
queues onto separate IO threads, dramatically improving RAW block I/O
throughput.

See discussion and background here:
https://blogs.oracle.com/linux/post/virtioblk-using-iothread-vq-mapping

Patches:

  1. drive-iothread-vq-pve8.4.patch
     – extend PVE/QemuServer/Drive.pm to expose `iothread_vq_mapping`
       parameter in `Drive` objects.

  2. qemuserver-iothread-vq-pve8.4.patch
     – update PVE/QemuServer.pm to consume `iothread_vq_mapping`,
       generate `-object iothread,id=…` entries and JSON `-device`
       parameters with separate bus/addr fields.

Installation:

  cp /usr/share/perl5/PVE/QemuServer/Drive.pm \
     /usr/share/perl5/PVE/QemuServer/Drive.pm.backup
  cp /usr/share/perl5/PVE/QemuServer.pm \
     /usr/share/perl5/PVE/QemuServer.pm.backup
  patch /usr/share/perl5/PVE/QemuServer/Drive.pm \
     < drive-iothread-vq-pve8.4.patch
  patch /usr/share/perl5/PVE/QemuServer.pm \
     < qemuserver-iothread-vq-pve8.4.patch
  systemctl restart pvedaemon pveproxy

Usage:

  Add `iothread_vq_mapping=<num>` (2–16) to your disk line in
  `/etc/pve/qemu-server/<VMID>.conf`, for example:

    virtio0: local-lvm:vm-100-disk-0,aio=native,iothread_vq_mapping=8,size=50G

Test Results:

  fio 4k randread RAW:
   - legacy `iothread=1`: ~200k IOPS  
   - new `iothread_vq_mapping=8`: ~800k IOPS  

Please review this series to bring enhanced virtio-blk performance to PVE 8.4.

— 
Dominik Budzowski <dbudzowski@alfaline.pl>

Signed-off-by: Dominik Budzowski <dbudzowski@alfaline.pl>

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH] Add patches for iothread_vq

[Dominik Budzowski]

---
 drive-iothread-vq-pve8.4.patch      |  33 ++++++++
 qemuserver-iothread-vq-pve8.4.patch | 123 ++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+)
 create mode 100644 drive-iothread-vq-pve8.4.patch
 create mode 100644 qemuserver-iothread-vq-pve8.4.patch

diff --git a/drive-iothread-vq-pve8.4.patch b/drive-iothread-vq-pve8.4.patch
new file mode 100644
index 0000000..0fd39be
--- /dev/null
+++ b/drive-iothread-vq-pve8.4.patch
@@ -0,0 +1,33 @@
+--- Drive.pm	2025-06-09 18:31:45.482659331 +0200
++++ Drive.pm	2025-06-09 19:00:29.962125989 +0200
+@@ -275,6 +275,14 @@
+ 	optional => 1,
+ });
+ 
++my %iothread_vq_mapping_fmt = ( iothread_vq_mapping => {
++    type => 'integer',
++    description => "Whether to use iothread-vq-mapping for this drive",
++    minimum => 2,
++    maximum => 16,
++    optional => 1,
++});
++
+ my %product_fmt = (
+     product => {
+ 	type => 'string',
+@@ -442,6 +450,7 @@
+ my $virtio_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %readonly_fmt,
+ };
+ my $virtiodesc = {
+@@ -537,6 +546,7 @@
+ my $alldrive_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %model_fmt,
+     %product_fmt,
+     %queues_fmt,
diff --git a/qemuserver-iothread-vq-pve8.4.patch b/qemuserver-iothread-vq-pve8.4.patch
new file mode 100644
index 0000000..9719091
--- /dev/null
+++ b/qemuserver-iothread-vq-pve8.4.patch
@@ -0,0 +1,123 @@
+--- QemuServer.pm	2025-06-25 22:36:12.414594136 +0200
++++ QemuServer.pm	2025-06-25 23:22:38.522270102 +0200
+@@ -1301,17 +1301,79 @@
+     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
+ }
+ 
++# Helper to generate iothread/VQ mapping for block devices
++sub generate_iothread_vq_mapping {
++    my ($vmid, $drive) = @_;
++    my ($use_iothread_vq_mapping, $use_iothread, @vq_map);
++
++    if ($drive->{iothread_vq_mapping}) {
++        $use_iothread_vq_mapping = 1;
++        @vq_map = map { { iothread => "iothread-${vmid}-$_" } }
++                  0 .. $drive->{iothread_vq_mapping} - 1;
++    } elsif ($drive->{iothread}) {
++        $use_iothread = 1;
++    }
++
++    return ($use_iothread_vq_mapping, $use_iothread, \@vq_map);
++}
++
++# Main sub: JSON encoder for ordered key-value pairs and full drive device construction
+ sub print_drivedevice_full {
+     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
+ 
+-    my $device = '';
+-    my $maxdev = 0;
+-
++    # Compute drive ID and PCI address for virtio
+     my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
++    my $pciaddr = '';
++    if ($drive->{interface} eq 'virtio') {
++        $pciaddr = print_pci_addr($drive_id, $bridges, $arch, $machine_type);
++    }
++
++    # Generate iothread/VQ mapping flags and mapping array
++    my ($use_iothread_vq_mapping, $use_iothread, $vq_map_ref) =
++        generate_iothread_vq_mapping($vmid, $drive);
++
++    # Prepare base JSON encoder
++    my $json = JSON->new->canonical(1);
++    my $device;
++
++    # Virtio interface handling
+     if ($drive->{interface} eq 'virtio') {
+-	my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
+-	$device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
+-	$device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
++        if ($use_iothread_vq_mapping) {
++	    my ($bus, $addr) = ();
++	    if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) {
++    	    ($bus, $addr) = ($1, $2);
++	    }
++            my @fields = (
++                [ driver              => 'virtio-blk-pci'   ],
++                [ 'iothread-vq-mapping' => $vq_map_ref      ],
++                [ 'queue-size'        => 1024               ],
++                [ 'config-wce'        => JSON::false        ],
++                [ drive               => "drive-$drive_id"  ],
++                [ id                  => $drive_id          ],
++                [ bus                 => $bus               ],
++                [ addr                => $addr              ],
++                ($drive->{bootindex} ? [ bootindex => $drive->{bootindex} ] : ()),
++            );
++            my @parts;
++            for my $fld (@fields) {
++                my ($k, $v) = @$fld;
++                push @parts, $json->encode($k) . ':' . $json->encode($v);
++            }
++            $device = '{' . join(',', @parts) . '}';
++        }
++        elsif ($use_iothread) {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-%s,id=%s%s,iothread=iothread-%s',
++                $drive_id, $drive_id, $pciaddr, $drive_id
++            );
++        }
++        else {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-%s,id=%s%s',
++                $drive_id, $drive_id, $pciaddr
++            );
++        }
++
+     } elsif ($drive->{interface} eq 'scsi') {
+ 
+ 	my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
+@@ -1386,7 +1448,7 @@
+ 	die "unsupported interface type";
+     }
+ 
+-    $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
++    $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex} && !$use_iothread_vq_mapping;
+ 
+     if (my $serial = $drive->{serial}) {
+ 	$serial = URI::Escape::uri_unescape($serial);
+@@ -3911,8 +3973,23 @@
+ 
+ 	$drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
+ 
+-	if ($drive->{interface} eq 'virtio'){
+-           push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
++#	if ($drive->{interface} eq 'virtio'){
++#           push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
++#	}
++
++	if ($drive->{interface} eq 'virtio') {
++
++	    if ($drive->{iothread_vq_mapping}) {
++
++		for my $i (0 .. $drive->{iothread_vq_mapping} - 1) {
++		    my $id = "iothread,id=iothread-$vmid-$i";
++		    push @$cmd, ('-object', $id) unless grep { $_ eq $id } @$cmd;
++		}
++	    } elsif ($drive->{iothread}) {
++
++		my $id = "iothread,id=iothread-$ds";
++		push @$cmd, ('-object', $id) unless grep { $_ eq $id } @$cmd;
++	    }
+ 	}
+ 
+ 	if ($drive->{interface} eq 'scsi') {
-- 
2.39.5


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH] Add patches for iothread_vq

[DERUMIER, Alexandre via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 23659 bytes --]

From: "DERUMIER, Alexandre" <alexandre.derumier@groupe-cyllene.com>
To: "pve-devel@lists.proxmox.com" <pve-devel@lists.proxmox.com>
Subject: Re: [pve-devel] [PATCH] Add patches for iothread_vq
Date: Fri, 27 Jun 2025 07:01:46 +0000
Message-ID: <43035b37e68c44ca6c30799dfa454d1bb8b23fec.camel@groupe-cyllene.com>

Hi Dominik,

I'm going to send patch to use json format by default for -device 
options, It should help here


I have a patch for multiple iothreads too on my side (for both virtio
&& virtio-scsi), maybe we could compare implementation.

on my side, I'm using use same implementation than libvirt,

iothreads: x  , are defined globally, and shared accross all disk by
default. (redhat said that it's better to not have too much threads, or
not busy thread, so shared iothreads are better by default).

optionally, user is able to define specific iothreads list by disk


some notes: 

iothreads need to be < host cores.  (I thinked it was vm cores but
finally the iothreads are unrelated to vm vcpu)

Redhat doc said that it's not playing fine with cache=writeback &&
aio=threads  (not sure about aio=io_uring). So this need to be tested,
but maybe it need to fallback to 1thread is this case.

Also, your implementation is missing hotplug



-------- Message initial --------
De: Dominik Budzowski <dbudzowski@alfaline.pl>
Répondre à: Proxmox VE development discussion <pve-
devel@lists.proxmox.com>
À: pve-devel@lists.proxmox.com
Objet: [pve-devel] [PATCH] Add patches for iothread_vq
Date: 26/06/2025 00:15:35

---
 drive-iothread-vq-pve8.4.patch      |  33 ++++++++
 qemuserver-iothread-vq-pve8.4.patch | 123 ++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+)
 create mode 100644 drive-iothread-vq-pve8.4.patch
 create mode 100644 qemuserver-iothread-vq-pve8.4.patch

diff --git a/drive-iothread-vq-pve8.4.patch b/drive-iothread-vq-
pve8.4.patch
new file mode 100644
index 0000000..0fd39be
--- /dev/null
+++ b/drive-iothread-vq-pve8.4.patch
@@ -0,0 +1,33 @@
+--- Drive.pm	2025-06-09 18:31:45.482659331 +0200
++++ Drive.pm	2025-06-09 19:00:29.962125989 +0200
+@@ -275,6 +275,14 @@
+ 	optional => 1,
+ });
+ 
++my %iothread_vq_mapping_fmt = ( iothread_vq_mapping => {
++    type => 'integer',
++    description => "Whether to use iothread-vq-mapping for this
drive",
++    minimum => 2,
++    maximum => 16,
++    optional => 1,
++});
++
+ my %product_fmt = (
+     product => {
+ 	type => 'string',
+@@ -442,6 +450,7 @@
+ my $virtio_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %readonly_fmt,
+ };
+ my $virtiodesc = {
+@@ -537,6 +546,7 @@
+ my $alldrive_fmt = {
+     %drivedesc_base,
+     %iothread_fmt,
++    %iothread_vq_mapping_fmt,
+     %model_fmt,
+     %product_fmt,
+     %queues_fmt,
diff --git a/qemuserver-iothread-vq-pve8.4.patch b/qemuserver-iothread-
vq-pve8.4.patch
new file mode 100644
index 0000000..9719091
--- /dev/null
+++ b/qemuserver-iothread-vq-pve8.4.patch
@@ -0,0 +1,123 @@
+--- QemuServer.pm	2025-06-25 22:36:12.414594136 +0200
++++ QemuServer.pm	2025-06-25 23:22:38.522270102 +0200
+@@ -1301,17 +1301,79 @@
+     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
+ }
+ 
++# Helper to generate iothread/VQ mapping for block devices
++sub generate_iothread_vq_mapping {
++    my ($vmid, $drive) = @_;
++    my ($use_iothread_vq_mapping, $use_iothread, @vq_map);
++
++    if ($drive->{iothread_vq_mapping}) {
++        $use_iothread_vq_mapping = 1;
++        @vq_map = map { { iothread => "iothread-${vmid}-$_" } }
++                  0 .. $drive->{iothread_vq_mapping} - 1;
++    } elsif ($drive->{iothread}) {
++        $use_iothread = 1;
++    }
++
++    return ($use_iothread_vq_mapping, $use_iothread, \@vq_map);
++}
++
++# Main sub: JSON encoder for ordered key-value pairs and full drive
device construction
+ sub print_drivedevice_full {
+     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch,
$machine_type) = @_;
+ 
+-    my $device = '';
+-    my $maxdev = 0;
+-
++    # Compute drive ID and PCI address for virtio
+     my $drive_id = PVE::QemuServer::Drive::get_drive_id($drive);
++    my $pciaddr = '';
++    if ($drive->{interface} eq 'virtio') {
++        $pciaddr = print_pci_addr($drive_id, $bridges, $arch,
$machine_type);
++    }
++
++    # Generate iothread/VQ mapping flags and mapping array
++    my ($use_iothread_vq_mapping, $use_iothread, $vq_map_ref) =
++        generate_iothread_vq_mapping($vmid, $drive);
++
++    # Prepare base JSON encoder
++    my $json = JSON->new->canonical(1);
++    my $device;
++
++    # Virtio interface handling
+     if ($drive->{interface} eq 'virtio') {
+-	my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch,
$machine_type);
+-	$device = "virtio-blk-pci,drive=drive-
$drive_id,id=${drive_id}${pciaddr}";
+-	$device .= ",iothread=iothread-$drive_id" if $drive-
>{iothread};
++        if ($use_iothread_vq_mapping) {
++	    my ($bus, $addr) = ();
++	    if ($pciaddr =~ /^,bus=([^,]+),addr=(.+)$/) {
++    	    ($bus, $addr) = ($1, $2);
++	    }
++            my @fields = (
++                [ driver              => 'virtio-blk-pci'   ],
++                [ 'iothread-vq-mapping' => $vq_map_ref      ],
++                [ 'queue-size'        => 1024               ],
++                [ 'config-wce'        => JSON::false        ],
++                [ drive               => "drive-$drive_id"  ],
++                [ id                  => $drive_id          ],
++                [ bus                 => $bus               ],
++                [ addr                => $addr              ],
++                ($drive->{bootindex} ? [ bootindex => $drive-
>{bootindex} ] : ()),
++            );
++            my @parts;
++            for my $fld (@fields) {
++                my ($k, $v) = @$fld;
++                push @parts, $json->encode($k) . ':' . $json-
>encode($v);
++            }
++            $device = '{' . join(',', @parts) . '}';
++        }
++        elsif ($use_iothread) {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-
%s,id=%s%s,iothread=iothread-%s',
++                $drive_id, $drive_id, $pciaddr, $drive_id
++            );
++        }
++        else {
++            $device = sprintf(
++                'virtio-blk-pci,drive=drive-%s,id=%s%s',
++                $drive_id, $drive_id, $pciaddr
++            );
++        }
++
+     } elsif ($drive->{interface} eq 'scsi') {
+ 
+ 	my ($maxdev, $controller, $controller_prefix) =
scsihw_infos($conf, $drive);
+@@ -1386,7 +1448,7 @@
+ 	die "unsupported interface type";
+     }
+ 
+-    $device .= ",bootindex=$drive->{bootindex}" if $drive-
>{bootindex};
++    $device .= ",bootindex=$drive->{bootindex}" if $drive-
>{bootindex} && !$use_iothread_vq_mapping;
+ 
+     if (my $serial = $drive->{serial}) {
+ 	$serial = URI::Escape::uri_unescape($serial);
+@@ -3911,8 +3973,23 @@
+ 
+ 	$drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
+ 
+-	if ($drive->{interface} eq 'virtio'){
+-           push @$cmd, '-object', "iothread,id=iothread-$ds" if
$drive->{iothread};
++#	if ($drive->{interface} eq 'virtio'){
++#           push @$cmd, '-object', "iothread,id=iothread-$ds" if
$drive->{iothread};
++#	}
++
++	if ($drive->{interface} eq 'virtio') {
++
++	    if ($drive->{iothread_vq_mapping}) {
++
++		for my $i (0 .. $drive->{iothread_vq_mapping} - 1) {
++		    my $id = "iothread,id=iothread-$vmid-$i";
++		    push @$cmd, ('-object', $id) unless grep { $_ eq
$id } @$cmd;
++		}
++	    } elsif ($drive->{iothread}) {
++
++		my $id = "iothread,id=iothread-$ds";
++		push @$cmd, ('-object', $id) unless grep { $_ eq $id }
@$cmd;
++	    }
+ 	}
+ 
+ 	if ($drive->{interface} eq 'scsi') {

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel