[PATCH qemu-server v4 4/4] cpu: add cpu binding

[Elias Huhsovitz <e.huhsovitz@proxmox.com>]

Add a new qm.conf option `cpu-binding-mode`: none | one-to-one | numa

When cpu-binding-mode is unset or "none", behavior is unchanged.
Otherwise, vCPU threads are pinned to host CPUs after QEMU has spawned
them.

Two functional modes are supported: (1) one-to-one - each vCPU gets its
own cpu, defined by the affinity-pool. (2) numa - per-vNUMA-node
set-to-set binding, defined by numa [N] config.

Two entry points are provided in PVE::QemuServer::CPUBind. (1)
gen_vcpu_bindings() - called before VM QEMU spawns. Generates a binding
map, specifying vCPU:host cpu(s). (2) bind_vcpus() - called after QEMU
started. Binds the vCPUs according to the generated binding map.
information as JSON and hands it to Rust together with the mode and cfg
string.

Originally-by: Dominik Csapak <d.csapak@proxmox.com>
Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
 src/PVE/QemuServer.pm         |  32 ++++++++++-
 src/PVE/QemuServer/CPUBind.pm | 103 ++++++++++++++++++++++++++++++++++
 src/PVE/QemuServer/Makefile   |   1 +
 src/PVE/QemuServer/Memory.pm  |  52 ++++++++++++++++-
 4 files changed, 186 insertions(+), 2 deletions(-)
 create mode 100644 src/PVE/QemuServer/CPUBind.pm

diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 8149184e..a8269e36 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -59,6 +59,7 @@ use PVE::QemuServer::Helpers
     qw(config_aware_timeout get_iscsi_initiator_name get_host_arch min_version kvm_user_version windows_version);
 use PVE::QemuServer::Cloudinit;
 use PVE::QemuServer::CGroup;
+use PVE::QemuServer::CPUBind;
 use PVE::QemuServer::CPUConfig qw(
     print_cpu_device
     get_cpu_options
@@ -741,6 +742,12 @@ EODESCR
         optional => 1,
         default => 1,
     },
+    'cpu-binding-mode' => {
+        type => 'string',
+        format => $PVE::QemuServer::Bind::bind_mode_fmt,
+        description => "Set CPU binding mode for the guest",
+        optional => 1,
+    },
 };
 
 my $cicustom_fmt = {
@@ -3189,7 +3196,7 @@ sub config_to_command {
     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
 
     my $cmd = [];
-    if ($conf->{affinity}) {
+    if ($conf->{affinity} && !$conf->{'cpu-binding-mode'}) {
         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
     }
 
@@ -5514,6 +5521,8 @@ sub vm_start_nolock {
 
     my ($cmd, $spice_port, $start_timeout);
     my $pci_reserve_list = [];
+    my $vcpu_binding_map;
+    my $freeze_for_binding = 0;
     eval {
         # With -blockdev, it is necessary to activate the volumes before generating the command line
         # Plugins can safely deactivate already-active volumes here if needed
@@ -5539,10 +5548,21 @@ sub vm_start_nolock {
             },
         );
 
+        if ($conf->{'cpu-binding-mode'}) {
+            $vcpu_binding_map = PVE::QemuServer::CPUBind::conf_to_vcpu_bindings($conf);
+        }
+
+        $freeze_for_binding =
+            $vcpu_binding_map
+            && !$params->{paused}
+            && !$statefile
+            && !$conf->{freeze};
+
         my $memory = get_current_memory($conf->{memory});
         $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
 
         push $cmd->@*, $state_cmdline->@*;
+        push @$cmd, '-S' if $freeze_for_binding;
 
         ($pci_reserve_list, my $uuid) =
             PVE::QemuServer::PCI::prepare_pci_devices($conf, $vmid, $pci_devices, $start_timeout);
@@ -5759,6 +5779,16 @@ sub vm_start_nolock {
     my $qemu_pkg_string = PVE::QemuServer::QMPHelpers::get_qemu_package_string($vmid);
     syslog("info", "VM $vmid started with PID $pid using $qemu_pkg_string.");
 
+    if ($vcpu_binding_map) {
+        eval { PVE::QemuServer::CPUBind::bind_vcpus($vmid, $vcpu_binding_map) };
+        log_warn("unable to bind vCPU threads - $@") if $@;
+    }
+
+    if ($freeze_for_binding) {
+        eval { mon_cmd($vmid, "cont") };
+        log_warn("failed to resume VM after CPU binding: $@") if $@;
+    }
+
     PVE::QemuServer::RunState::create_cleanup_flag($vmid);
 
     if (defined(my $migrate = $res->{migrate})) {
diff --git a/src/PVE/QemuServer/CPUBind.pm b/src/PVE/QemuServer/CPUBind.pm
new file mode 100644
index 00000000..0566b08b
--- /dev/null
+++ b/src/PVE/QemuServer/CPUBind.pm
@@ -0,0 +1,103 @@
+package PVE::QemuServer::CPUBind;
+
+use v5.36;
+
+use JSON;
+
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::Monitor;
+use PVE::RS::CPU::Bind;
+
+our $bind_mode_fmt = {
+    mode => {
+        type => 'string',
+        enum => [qw(none one-to-one numa)],
+        description => "Set the CPU binding mode for the guest.",
+        verbose_description => <<'EODESCR',
+Determines how guest vCPUs are bound/pinned to host CPUs.
+
+* none (default): no binding. Regular scheduling.
+
+* one-to-one: pin each vCPU to its own physical core, chosen from the
+  pool configured by affinity. The configured affinity string acts as
+  an upper bound on the host cpus, that can be used for binding.
+
+  If the guest has N vCPUS and the affinity-pool has M host cpus, each vCPU
+  is fixed to a single one of the M host cpus. (M >= N, must be true).
+
+  Each vCPU is fixed to a single host cpu, it does not float within
+  the pool.
+
+  Example 1: 4-vCPU guest with affinity=4-9:
+  The resulting mapping would be: vCPU0:CPU4, vCPU1:CPU5, vCPU2:CPU6,
+  vCPU3:CPU7.
+
+  Example 2: 4-vCPU guest with affinity=0,2:
+  The resulting mapping would be: vCPU0:CPU0, vCPU1:CPU2
+
+  An empty affinity string means all host cpus will be considered.
+  On a host with 16 cores, that pins an N-vCPU VM to host cpus
+  0, 1, ..., N-1.
+
+* numa: bind the set of vCPUs of each vNUMA node to the set of
+  CPUs of a host NUMA node. The cpu bindings are determined by the
+  numa[N] configuration of the guest. If no numa[N] config is
+  explicitly provided, then for each socket a numa[N] node is created.
+
+  Each guest vNUMA node's vCPUs form a group that may run on any CPU
+  of the assigned host NUMA node. The host scheduler is free to migrate
+  the vCPUs within the host node. This is the intended trade-off:
+  memory allocated by a vCPU stays local to the host node it runs on
+  (preserving NUMA memory locality), while the scheduler still has room
+  to balance load across the node's cores and hyperthreads.
+
+  Examples:
+  numa0: cpus=0-1,hostnodes=0,memory=2048,policy=bind
+  Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0
+
+  numa1: cpus=6-7,hostnodes=3,memory=2048,policy=bind
+  Allows vCPUs 6-7 to execute on all cpus of host NUMA node 3
+
+  numa:1, no numa[N], sockets: 2, cores: 2, total vCPUs: 4
+  Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0
+  Allows vCPUs 2-3 to execute on all cpus of host NUMA node 1
+
+EODESCR
+        default => 'none',
+        default_key => 1,
+    },
+};
+
+my sub gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg) {
+    my $cfg_json = ''; # currently only relevant for numa mode.
+    if ($mode eq 'numa') {
+        $cfg_json = JSON::encode_json($cfg);
+    }
+
+    # PVE::RS::CPU::Bind::gen_vcpu_bindings returns the map as JSON. This is currently
+    # not decoded, because at the moment we are simply passing the JSON bind map to
+    # bind_vcpus. Should this change in the future, decode the JSON string before returning.
+    return PVE::RS::CPU::Bind::gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg_json);
+}
+
+sub conf_to_vcpu_bindings($conf) {
+    my $mode = $conf->{'cpu-binding-mode'};
+    return if $mode eq 'none';
+
+    my $cfg = '';
+    $cfg = PVE::QemuServer::Memory::numa_conf_to_map($conf) if $mode eq 'numa';
+
+    my $vcpu_binding_map = gen_vcpu_bindings(
+        $mode, $conf->{sockets}, $conf->{cores}, $conf->{affinity} // '', $cfg,
+    );
+
+    return $vcpu_binding_map;
+}
+
+sub bind_vcpus($vmid, $bind_map_json) {
+    my $cpuinfo = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-cpus-fast');
+    my $qmp_cpuinfo_json = JSON::encode_json($cpuinfo);
+    PVE::RS::CPU::Bind::bind_vcpus($bind_map_json, $qmp_cpuinfo_json);
+}
+
+1;
diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile
index 61ed1a4e..c4cd9077 100644
--- a/src/PVE/QemuServer/Makefile
+++ b/src/PVE/QemuServer/Makefile
@@ -8,6 +8,7 @@ SOURCES=Agent.pm	\
 	Cfg2Cmd.pm	\
 	CGroup.pm	\
 	Cloudinit.pm	\
+	CPUBind.pm	\
 	CPUConfig.pm	\
 	CPUFlags.pm	\
 	DBusVMState.pm	\
diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm
index 8cbbac84..282d5e23 100644
--- a/src/PVE/QemuServer/Memory.pm
+++ b/src/PVE/QemuServer/Memory.pm
@@ -87,6 +87,57 @@ our $memory_fmt = {
     },
 };
 
+sub numa_conf_to_map {
+    my ($conf) = @_;
+
+    my $numa_cfg = {};
+    my $has_custom_topology = 0;
+
+    for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) {
+        my $entry = $conf->{"numa$i"} or next;
+        $has_custom_topology = 1;
+
+        my $numa = PVE::QemuServer::Memory::parse_numa($entry);
+
+        $numa_cfg->{"numa$i"} = {
+            vcpus => $numa->{cpus} // [],
+            hostnodes => $numa->{hostnodes} // [],
+            memory => 0 + ($numa->{memory} // 0),
+            policy => $numa->{policy},
+        };
+    }
+
+    return $numa_cfg if $has_custom_topology;
+
+    die "NUMA must be enabled in the VM configuration\n"
+        if !$conf->{numa};
+
+    my $sockets = $conf->{sockets} || 1;
+    my $cores = $conf->{cores} || 1;
+    my $memory = get_current_memory($conf->{memory});
+    my $numa_memory = int($memory / $sockets);
+
+    for (my $i = 0; $i < $sockets; $i++) {
+        my $start_cpu = $cores * $i;
+        my $end_cpu = $start_cpu + $cores - 1;
+
+        # Format matches parse_number_sets output: [[start, end]] or [[start, undef]]
+        my $cpus = $cores > 1 ? [[$start_cpu, $end_cpu]] : [[$start_cpu, undef]];
+
+        # Fallback maps guest node $i directly to host node $i
+        my $hostnodes = [[$i, undef]];
+
+        $numa_cfg->{"numa$i"} = {
+            vcpus => $cpus,
+            hostnodes => $hostnodes,
+            memory => 0 + $numa_memory,
+            policy => undef,
+        };
+    }
+
+    return $numa_cfg;
+}
+
 sub print_memory {
     my $memory = shift;
 
@@ -825,4 +876,3 @@ sub hugepages_update_locked {
     return $res;
 }
 1;
-
-- 
2.47.3