From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 39F3A1FF0B3 for ; Wed, 11 Nov 2026 16:21:27 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 35323216AB; Wed, 11 Nov 2026 16:21:08 +0100 (CET) From: Elias Huhsovitz To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server v4 4/4] cpu: add cpu binding Date: Wed, 11 Nov 2026 16:20:27 +0100 Message-ID: <20261111152027.260117-5-e.huhsovitz@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261111152027.260117-1-e.huhsovitz@proxmox.com> References: <20261111152027.260117-1-e.huhsovitz@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794410460064 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.428 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 4OPH7XEIXRZJ2H2QCVHCTSZEYME2NKES X-Message-ID-Hash: 4OPH7XEIXRZJ2H2QCVHCTSZEYME2NKES X-MailFrom: e.huhsovitz@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Elias Huhsovitz X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a new qm.conf option `cpu-binding-mode`: none | one-to-one | numa When cpu-binding-mode is unset or "none", behavior is unchanged. Otherwise, vCPU threads are pinned to host CPUs after QEMU has spawned them. Two functional modes are supported: (1) one-to-one - each vCPU gets its own cpu, defined by the affinity-pool. (2) numa - per-vNUMA-node set-to-set binding, defined by numa [N] config. Two entry points are provided in PVE::QemuServer::CPUBind. (1) gen_vcpu_bindings() - called before VM QEMU spawns. Generates a binding map, specifying vCPU:host cpu(s). (2) bind_vcpus() - called after QEMU started. Binds the vCPUs according to the generated binding map. information as JSON and hands it to Rust together with the mode and cfg string. Originally-by: Dominik Csapak Signed-off-by: Elias Huhsovitz --- src/PVE/QemuServer.pm | 32 ++++++++++- src/PVE/QemuServer/CPUBind.pm | 103 ++++++++++++++++++++++++++++++++++ src/PVE/QemuServer/Makefile | 1 + src/PVE/QemuServer/Memory.pm | 52 ++++++++++++++++- 4 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 src/PVE/QemuServer/CPUBind.pm diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm index 8149184e..a8269e36 100644 --- a/src/PVE/QemuServer.pm +++ b/src/PVE/QemuServer.pm @@ -59,6 +59,7 @@ use PVE::QemuServer::Helpers qw(config_aware_timeout get_iscsi_initiator_name get_host_arch min_version kvm_user_version windows_version); use PVE::QemuServer::Cloudinit; use PVE::QemuServer::CGroup; +use PVE::QemuServer::CPUBind; use PVE::QemuServer::CPUConfig qw( print_cpu_device get_cpu_options @@ -741,6 +742,12 @@ EODESCR optional => 1, default => 1, }, + 'cpu-binding-mode' => { + type => 'string', + format => $PVE::QemuServer::Bind::bind_mode_fmt, + description => "Set CPU binding mode for the guest", + optional => 1, + }, }; my $cicustom_fmt = { @@ -3189,7 +3196,7 @@ sub config_to_command { ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); my $cmd = []; - if ($conf->{affinity}) { + if ($conf->{affinity} && !$conf->{'cpu-binding-mode'}) { push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity}; } @@ -5514,6 +5521,8 @@ sub vm_start_nolock { my ($cmd, $spice_port, $start_timeout); my $pci_reserve_list = []; + my $vcpu_binding_map; + my $freeze_for_binding = 0; eval { # With -blockdev, it is necessary to activate the volumes before generating the command line # Plugins can safely deactivate already-active volumes here if needed @@ -5539,10 +5548,21 @@ sub vm_start_nolock { }, ); + if ($conf->{'cpu-binding-mode'}) { + $vcpu_binding_map = PVE::QemuServer::CPUBind::conf_to_vcpu_bindings($conf); + } + + $freeze_for_binding = + $vcpu_binding_map + && !$params->{paused} + && !$statefile + && !$conf->{freeze}; + my $memory = get_current_memory($conf->{memory}); $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume); push $cmd->@*, $state_cmdline->@*; + push @$cmd, '-S' if $freeze_for_binding; ($pci_reserve_list, my $uuid) = PVE::QemuServer::PCI::prepare_pci_devices($conf, $vmid, $pci_devices, $start_timeout); @@ -5759,6 +5779,16 @@ sub vm_start_nolock { my $qemu_pkg_string = PVE::QemuServer::QMPHelpers::get_qemu_package_string($vmid); syslog("info", "VM $vmid started with PID $pid using $qemu_pkg_string."); + if ($vcpu_binding_map) { + eval { PVE::QemuServer::CPUBind::bind_vcpus($vmid, $vcpu_binding_map) }; + log_warn("unable to bind vCPU threads - $@") if $@; + } + + if ($freeze_for_binding) { + eval { mon_cmd($vmid, "cont") }; + log_warn("failed to resume VM after CPU binding: $@") if $@; + } + PVE::QemuServer::RunState::create_cleanup_flag($vmid); if (defined(my $migrate = $res->{migrate})) { diff --git a/src/PVE/QemuServer/CPUBind.pm b/src/PVE/QemuServer/CPUBind.pm new file mode 100644 index 00000000..0566b08b --- /dev/null +++ b/src/PVE/QemuServer/CPUBind.pm @@ -0,0 +1,103 @@ +package PVE::QemuServer::CPUBind; + +use v5.36; + +use JSON; + +use PVE::QemuServer::Memory; +use PVE::QemuServer::Monitor; +use PVE::RS::CPU::Bind; + +our $bind_mode_fmt = { + mode => { + type => 'string', + enum => [qw(none one-to-one numa)], + description => "Set the CPU binding mode for the guest.", + verbose_description => <<'EODESCR', +Determines how guest vCPUs are bound/pinned to host CPUs. + +* none (default): no binding. Regular scheduling. + +* one-to-one: pin each vCPU to its own physical core, chosen from the + pool configured by affinity. The configured affinity string acts as + an upper bound on the host cpus, that can be used for binding. + + If the guest has N vCPUS and the affinity-pool has M host cpus, each vCPU + is fixed to a single one of the M host cpus. (M >= N, must be true). + + Each vCPU is fixed to a single host cpu, it does not float within + the pool. + + Example 1: 4-vCPU guest with affinity=4-9: + The resulting mapping would be: vCPU0:CPU4, vCPU1:CPU5, vCPU2:CPU6, + vCPU3:CPU7. + + Example 2: 4-vCPU guest with affinity=0,2: + The resulting mapping would be: vCPU0:CPU0, vCPU1:CPU2 + + An empty affinity string means all host cpus will be considered. + On a host with 16 cores, that pins an N-vCPU VM to host cpus + 0, 1, ..., N-1. + +* numa: bind the set of vCPUs of each vNUMA node to the set of + CPUs of a host NUMA node. The cpu bindings are determined by the + numa[N] configuration of the guest. If no numa[N] config is + explicitly provided, then for each socket a numa[N] node is created. + + Each guest vNUMA node's vCPUs form a group that may run on any CPU + of the assigned host NUMA node. The host scheduler is free to migrate + the vCPUs within the host node. This is the intended trade-off: + memory allocated by a vCPU stays local to the host node it runs on + (preserving NUMA memory locality), while the scheduler still has room + to balance load across the node's cores and hyperthreads. + + Examples: + numa0: cpus=0-1,hostnodes=0,memory=2048,policy=bind + Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0 + + numa1: cpus=6-7,hostnodes=3,memory=2048,policy=bind + Allows vCPUs 6-7 to execute on all cpus of host NUMA node 3 + + numa:1, no numa[N], sockets: 2, cores: 2, total vCPUs: 4 + Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0 + Allows vCPUs 2-3 to execute on all cpus of host NUMA node 1 + +EODESCR + default => 'none', + default_key => 1, + }, +}; + +my sub gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg) { + my $cfg_json = ''; # currently only relevant for numa mode. + if ($mode eq 'numa') { + $cfg_json = JSON::encode_json($cfg); + } + + # PVE::RS::CPU::Bind::gen_vcpu_bindings returns the map as JSON. This is currently + # not decoded, because at the moment we are simply passing the JSON bind map to + # bind_vcpus. Should this change in the future, decode the JSON string before returning. + return PVE::RS::CPU::Bind::gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg_json); +} + +sub conf_to_vcpu_bindings($conf) { + my $mode = $conf->{'cpu-binding-mode'}; + return if $mode eq 'none'; + + my $cfg = ''; + $cfg = PVE::QemuServer::Memory::numa_conf_to_map($conf) if $mode eq 'numa'; + + my $vcpu_binding_map = gen_vcpu_bindings( + $mode, $conf->{sockets}, $conf->{cores}, $conf->{affinity} // '', $cfg, + ); + + return $vcpu_binding_map; +} + +sub bind_vcpus($vmid, $bind_map_json) { + my $cpuinfo = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-cpus-fast'); + my $qmp_cpuinfo_json = JSON::encode_json($cpuinfo); + PVE::RS::CPU::Bind::bind_vcpus($bind_map_json, $qmp_cpuinfo_json); +} + +1; diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile index 61ed1a4e..c4cd9077 100644 --- a/src/PVE/QemuServer/Makefile +++ b/src/PVE/QemuServer/Makefile @@ -8,6 +8,7 @@ SOURCES=Agent.pm \ Cfg2Cmd.pm \ CGroup.pm \ Cloudinit.pm \ + CPUBind.pm \ CPUConfig.pm \ CPUFlags.pm \ DBusVMState.pm \ diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm index 8cbbac84..282d5e23 100644 --- a/src/PVE/QemuServer/Memory.pm +++ b/src/PVE/QemuServer/Memory.pm @@ -87,6 +87,57 @@ our $memory_fmt = { }, }; +sub numa_conf_to_map { + my ($conf) = @_; + + my $numa_cfg = {}; + my $has_custom_topology = 0; + + for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) { + my $entry = $conf->{"numa$i"} or next; + $has_custom_topology = 1; + + my $numa = PVE::QemuServer::Memory::parse_numa($entry); + + $numa_cfg->{"numa$i"} = { + vcpus => $numa->{cpus} // [], + hostnodes => $numa->{hostnodes} // [], + memory => 0 + ($numa->{memory} // 0), + policy => $numa->{policy}, + }; + } + + return $numa_cfg if $has_custom_topology; + + die "NUMA must be enabled in the VM configuration\n" + if !$conf->{numa}; + + my $sockets = $conf->{sockets} || 1; + my $cores = $conf->{cores} || 1; + my $memory = get_current_memory($conf->{memory}); + my $numa_memory = int($memory / $sockets); + + for (my $i = 0; $i < $sockets; $i++) { + my $start_cpu = $cores * $i; + my $end_cpu = $start_cpu + $cores - 1; + + # Format matches parse_number_sets output: [[start, end]] or [[start, undef]] + my $cpus = $cores > 1 ? [[$start_cpu, $end_cpu]] : [[$start_cpu, undef]]; + + # Fallback maps guest node $i directly to host node $i + my $hostnodes = [[$i, undef]]; + + $numa_cfg->{"numa$i"} = { + vcpus => $cpus, + hostnodes => $hostnodes, + memory => 0 + $numa_memory, + policy => undef, + }; + } + + return $numa_cfg; +} + sub print_memory { my $memory = shift; @@ -825,4 +876,3 @@ sub hugepages_update_locked { return $res; } 1; - -- 2.47.3