From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 3829C1FF09C for ; Mon, 21 Sep 2026 11:54:59 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 9C16221612; Mon, 21 Sep 2026 11:54:33 +0200 (CEST) From: Elias Huhsovitz To: pve-devel@lists.proxmox.com Subject: [RFC qemu-server v2 4/4] pinning: integrate cpu pinning into vm lifecycle Date: Mon, 21 Sep 2026 11:54:03 +0200 Message-ID: <20260921095404.61552-5-e.huhsovitz@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260921095404.61552-1-e.huhsovitz@proxmox.com> References: <20260921095404.61552-1-e.huhsovitz@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1789984465783 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.625 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: SDBZNFGLTMHFHNYQ3HVCC5FNLSXSQYAI X-Message-ID-Hash: SDBZNFGLTMHFHNYQ3HVCC5FNLSXSQYAI X-MailFrom: e.huhsovitz@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Elias Huhsovitz X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add new Pinning.pm to handle the core logic of pinning via taskset. Integrate the new pinning system into QemuServer.pm so that CPU placement actually takes effect when VMs start, stop, or change. config_to_command now computes the effective topology and startup CPU set before building the QEMU command line. The taskset wrapper uses this computed set instead of the raw affinity value. This confines QEMU qemu to the correct CPUs immediately. vm_start_nolock now receives a pinning map from config_to_command and applies per-vCPU thread affinity via QMP once the guest is running. qemu_cpu_hotplug recalculates the pinning map after adding or removing vCPUs and reapplies thread affinity. vm_stop_cleanup clears the VM's entry from the NUMA reservation tracker so that freed resources are immediately available for other VMs instead of waiting for the next start cycle to prune stale entries. Originally-by: Dominik Csapak Signed-off-by: Elias Huhsovitz --- src/PVE/QemuServer.pm | 97 +++++-- src/PVE/QemuServer/Makefile | 2 + src/PVE/QemuServer/Pinning.pm | 396 ++++++++++++++++++++++++++++ src/PVE/QemuServer/Pinning/Makefile | 9 + 4 files changed, 481 insertions(+), 23 deletions(-) create mode 100644 src/PVE/QemuServer/Pinning.pm create mode 100644 src/PVE/QemuServer/Pinning/Makefile diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm index 759f7db2..ba007915 100644 --- a/src/PVE/QemuServer.pm +++ b/src/PVE/QemuServer.pm @@ -85,6 +85,7 @@ use PVE::QemuServer::Memory qw(get_current_memory); use PVE::QemuServer::MetaInfo; use PVE::QemuServer::Monitor qw(mon_cmd qmp_cmd vm_qmp_peer); use PVE::QemuServer::Network; +use PVE::QemuServer::Pinning; use PVE::QemuServer::OVMF; use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr parse_hostpci get_pci_bridges); use PVE::QemuServer::PCI::Mdev; @@ -739,6 +740,12 @@ EODESCR optional => 1, default => 1, }, + pinning => { + type => 'string', + format => $PVE::QemuServer::Pinning::pinning_fmt, + description => "Set pinning options for the guest", + optional => 1, + }, }; my $cicustom_fmt = { @@ -3202,8 +3209,27 @@ sub config_to_command { ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type); my $cmd = []; - if ($conf->{affinity}) { - push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity}; + + my ($sockets, $cores, $maxcpus, $vcpus, $startup_cpuset, $pinning_map); + + my $topo; + if ($dry_run) { + ($sockets, $cores, $maxcpus, $vcpus) = + PVE::QemuServer::Pinning::Config::get_config_topology($conf); + $startup_cpuset = $conf->{affinity}; + $pinning_map = {}; + } else { + $topo = PVE::QemuServer::Pinning::get_effective_topology($conf, $vmid); + ($sockets, $cores, $maxcpus, $vcpus) = @$topo{qw(sockets cores maxcpus vcpus)}; + $startup_cpuset = $topo->{startup_cpuset}; + $pinning_map = $topo->{pinning_map}; + } + + my $allowed_vcpus = $cpuinfo->{cpus}; + die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus); + + if (defined($startup_cpuset) && $startup_cpuset ne '') { + push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $startup_cpuset; } push @$cmd, $kvm_binary; @@ -3375,20 +3401,6 @@ sub config_to_command { # to support backing up template VMs even if the TPM disk is write-protected. add_tpm_device($vmid, $devices, $conf) if !$is_template; - my $sockets = 1; - $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused - $sockets = $conf->{sockets} if $conf->{sockets}; - - my $cores = $conf->{cores} || 1; - - my $maxcpus = $sockets * $cores; - - my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus; - - my $allowed_vcpus = $cpuinfo->{cpus}; - - die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus); - if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) { push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus"; for (my $i = 2; $i <= $vcpus; $i++) { @@ -3448,6 +3460,18 @@ sub config_to_command { my $virtiofs_enabled = PVE::QemuServer::Virtiofs::virtiofs_enabled($conf); + my $pinning_info = {}; + if (!$dry_run) { + my $vmap = $topo->{vcpu_to_numa_map} // {}; + my $nmap = $topo->{guest_node_to_host_node} // {}; + if (%$pinning_map || %$vmap) { + $pinning_info = { + vcpu_to_numa_map => $vmap, + guest_node_to_host_node => $nmap, + }; + } + } + PVE::QemuServer::Memory::config( $conf, $vmid, @@ -3457,6 +3481,7 @@ sub config_to_command { $virtiofs_enabled, $cmd, $machineFlags, + $pinning_info ); push @$cmd, '-S' if $conf->{freeze}; @@ -3809,7 +3834,7 @@ sub config_to_command { push @$cmd, @$aa; } - return wantarray ? ($cmd, $spice_port, $pci_devices, $conf) : $cmd; + return wantarray ? ($cmd, $spice_port, $pci_devices, $conf, $pinning_map) : $cmd; } sub spice_port { @@ -4261,11 +4286,8 @@ sub qemu_cpu_hotplug { my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid); - my $sockets = 1; - $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused - $sockets = $conf->{sockets} if $conf->{sockets}; - my $cores = $conf->{cores} || 1; - my $maxcpus = $sockets * $cores; + my $topo = PVE::QemuServer::Pinning::get_effective_topology($conf, $vmid); + my $maxcpus = $topo->{maxcpus}; $vcpus = $maxcpus if !$vcpus; @@ -4330,6 +4352,18 @@ sub qemu_cpu_hotplug { mon_cmd($vmid, "cpu-add", id => int($i)); } } + + # Recalculate the pinning map AFTER the hotplug operations have changed the vCPU count. + my $new_vcpus = $conf->{vcpus} || $maxcpus; + eval { + my ($new_pinning_map, undef, undef, undef) = PVE::QemuServer::Pinning::get_pinning_map($conf, $vmid, $new_vcpus); + PVE::QemuServer::Pinning::pin_vcpu_threads_to_host_cpus($vmid, $new_pinning_map); + 1; + } or do { + my $err = $@ || 'unknown error'; + log_warn("could not re-pin vCPU threads after CPU hotplug - $err"); + }; + } sub qemu_volume_snapshot { @@ -5619,6 +5653,7 @@ sub vm_start_nolock { my ($cmd, $spice_port, $start_timeout); my $pci_reserve_list = []; + my $pinning_map = {}; eval { # With -blockdev, it is necessary to activate the volumes before generating the command line # Plugins can safely deactivate already-active volumes here if needed @@ -5629,7 +5664,7 @@ sub vm_start_nolock { # Note that for certain cases like templates, the configuration is minimized, so need to ensure # the rest of the function here uses the same configuration that was used to build the command - ($cmd, $spice_port, my $pci_devices, $conf) = config_to_command( + ($cmd, $spice_port, my $pci_devices, $conf, $pinning_map) = config_to_command( $storecfg, $vmid, $conf, @@ -5836,6 +5871,14 @@ sub vm_start_nolock { my $qemu_pkg_string = PVE::QemuServer::QMPHelpers::get_qemu_package_string($vmid); syslog("info", "VM $vmid started with PID $pid using $qemu_pkg_string."); + eval { + PVE::QemuServer::Pinning::pin_vcpu_threads_to_host_cpus($vmid, $pinning_map); + 1; + } or do { + my $err = $@ || 'unknown error'; + log_warn("could not pin vCPU threads - $err"); + }; + PVE::QemuServer::RunState::create_cleanup_flag($vmid); if (defined(my $migrate = $res->{migrate})) { @@ -6186,6 +6229,14 @@ sub vm_stop_cleanup { warn $err; } + eval { + PVE::QemuServer::Pinning::Allocator::clear_reservation($vmid); + 1; + } or do { + my $err = $@ || 'Unknown failure'; + warn "Failed to clear NUMA reservation for VM $vmid: $err"; + }; + # under the new mechanism the hookscript is fired here, so all callers of # vm_stop_cleanup observe it; with the old one 'qm cleanup' still drives it. # callers that should not trigger the post-stop hook (e.g. the live-migration diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile index 060fac23..f5981868 100644 --- a/src/PVE/QemuServer/Makefile +++ b/src/PVE/QemuServer/Makefile @@ -22,6 +22,7 @@ SOURCES=Agent.pm \ Network.pm \ OVMF.pm \ PCI.pm \ + Pinning.pm \ QemuImage.pm \ QMPHelpers.pm \ QSD.pm \ @@ -36,4 +37,5 @@ SOURCES=Agent.pm \ install: $(SOURCES) for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuServer/$$i; done $(MAKE) -C Cfg2Cmd install + $(MAKE) -C Pinning install $(MAKE) -C PCI install diff --git a/src/PVE/QemuServer/Pinning.pm b/src/PVE/QemuServer/Pinning.pm new file mode 100644 index 00000000..36cb743d --- /dev/null +++ b/src/PVE/QemuServer/Pinning.pm @@ -0,0 +1,396 @@ +package PVE::QemuServer::Pinning; + +use v5.36; + +use PVE::CpuSet; +use PVE::QemuServer::Monitor; +use PVE::QemuServer::Pinning::Config; +use PVE::QemuServer::Pinning::Topology; +use PVE::QemuServer::Pinning::Allocator; +use PVE::Tools qw(run_command); + + +# Returns: sorted cpu ids +# Warn if the assigned host CPUs span multiple CPU models. +# in: $pinning_map = {0=>"0", 1=>"1", 2=>"4"} +# out: [0, 1, 4] +my sub collect_assigned_cpu_ids($pinning_map) { + my %unique_cpus; + for my $vcpu_id (keys $pinning_map->%*) { + my (undef, $members) = PVE::CpuSet::parse_cpuset($pinning_map->{$vcpu_id}); + $unique_cpus{$_} = 1 for keys %$members; + } + my @cpu_ids = sort { $a <=> $b } keys %unique_cpus; + + if (PVE::QemuServer::Pinning::Topology::are_cpus_heterogenous(\@cpu_ids)) { + warn "WARNING: The assigned CPU pinning spans multiple CPU models." + . " This may cause KVM initialization failures or performance degradation.\n"; + } + + return \@cpu_ids; +} + +# On heterogeneous systems (ARM big.LITTLE, Intel P+E), KVM vCPU init can +# fail if the QEMU process schedules across CPU types at startup. +# Restrict a base cpuset to the largest same-type subset. +# Returns undef on homogeneous systems or empty input. +my sub restrict_to_largest_cpu_type($cpuset_str) { + my ($groups, $keys) = PVE::QemuServer::Pinning::Topology::group_cpus_by_type(); + return undef if !@$keys || @$keys <= 1; + return undef if !defined($cpuset_str) || $cpuset_str eq ''; + + my (undef, $members) = PVE::CpuSet::parse_cpuset($cpuset_str); + return undef if !keys %$members; + + my ($best_key, $best_count) = (undef, -1); + for my $key (@$keys) { + my @matches = grep { $members->{$_} } @{$groups->{$key}{cpus}}; + if (scalar @matches > $best_count) { + $best_count = scalar @matches; + $best_key = $key; + } + } + return undef if !defined $best_key || $best_count == 0; + + my @sorted = sort { $a <=> $b } + grep { $members->{$_} } @{$groups->{$best_key}{cpus}}; + return @sorted ? join(',', @sorted) : undef; +} + +# Used when there's no existing cpuset to restrict (pinning: none, no affinity). +my sub pick_highest_capacity_cpu_type() { + my ($groups, $keys) = PVE::QemuServer::Pinning::Topology::group_cpus_by_type(); + return undef if !@$keys || @$keys <= 1; + + my ($best_key, $best_capacity) = (undef, -1); + for my $key (@$keys) { + my $cap = $groups->{$key}{capacity}; + if ($cap > $best_capacity) { + $best_capacity = $cap; + $best_key = $key; + } + } + return undef if !defined $best_key; + + my @sorted = sort { $a <=> $b } @{$groups->{$best_key}{cpus}}; + return @sorted ? join(',', @sorted) : undef; +} + +my sub get_filtered_topology($conf) { + my $topology = PVE::QemuServer::Pinning::Topology::get_host_topology(); + return PVE::QemuServer::Pinning::Topology::filter_by_affinity( + $topology, $conf->{affinity}); +} + +my sub allocate_for_mode($mode, $topology, $vcpu_to_numa_map, $vcpu_count, $has_numa) { + return {} if $mode eq 'balanced'; + return PVE::QemuServer::Pinning::Allocator::allocate_numa( + $topology, $vcpu_to_numa_map) + if $mode eq 'numa'; + if ($mode eq 'one-to-one') { + return PVE::QemuServer::Pinning::Allocator::allocate_one_to_one_flat( + $topology, $vcpu_count) + if !$has_numa; + return PVE::QemuServer::Pinning::Allocator::allocate_one_to_one( + $topology, $vcpu_to_numa_map, $vcpu_count); + } + return undef; +} + +my sub validate_topology_for_mode($mode, $topology, $assigned_cpu_ids, $conf) { + return { valid => 1 } if $mode ne 'one-to-one'; + + my $guest_sockets = $conf->{sockets} // 1; + my $guest_cores = $conf->{cores} // 1; + + my $result = PVE::QemuServer::Pinning::Topology::validate_pinned_topology( + $topology, $assigned_cpu_ids, $guest_sockets, $guest_cores); + + warn "WARNING: $result->{warning}\n" if !$result->{valid}; + return $result; +} + +=head2 get_pinning_map + +Parses the VM's C property, reads host topology, +assigns host NUMA nodes to guest nodes if needed, computes +the per-vCPU CPU pinning map for the configured mode. + +Returns four values: + +=over + +=item C<$pinning_map> - C<< { $vcpu_id => $cpus } >>. For C +and C, C<$cpus> is a comma-joined CPU set. For C, C<{}>: +memory binding is done by the caller using C<$vcpu_to_numa_map>. + +=item C<$topo_result> - validation result from C. + +=item C<$vcpu_to_numa_map> - C<< { $vcpu_id => $host_node_id } >>. + +=item C<$topology> - affinity-filtered host topology. Returned so callers +do not have to re-read sysfs. + +=back + +Returns C<({}, {valid=>1}, {}, undef)> immediately when pinning mode is +C. Dies if C or C is used without C. + +=cut +sub get_pinning_map($conf, $vmid, $vcpu_count) { + my $pinning = PVE::QemuServer::Pinning::Config::parse_pinning($conf->{pinning}); + my $mode = $pinning->{mode}; + + return ({}, { valid => 1 }, {}, undef) + if $mode eq PVE::QemuServer::Pinning::Config::DEFAULT_VCPU_PINNING; + + my $topology = get_filtered_topology($conf); + my $has_numa = $conf->{numa}; + + # balanced and numa modes require numa: 1 for memory binding. + if (($mode eq 'balanced' || $mode eq 'numa') && !$has_numa) { + die "pinning mode '$mode' requires numa: 1 to be enabled\n"; + } + + my $vcpu_to_numa_map; + + if ($mode eq 'balanced') { + $vcpu_to_numa_map = PVE::QemuServer::Pinning::Allocator::balance_numa_nodes( + $conf, $topology, $vmid); + } elsif ($mode eq 'one-to-one' && !$has_numa) { + $vcpu_to_numa_map = {}; + } else { + $vcpu_to_numa_map = PVE::QemuServer::Pinning::Allocator::assign_numa_nodes( + $conf, $topology, $vmid); + } + + my $pinning_map = allocate_for_mode( + $mode, $topology, $vcpu_to_numa_map, $vcpu_count, $has_numa); + return ({}, { valid => 1 }, {}, $topology) if !defined $pinning_map; + + my $assigned_cpu_ids = collect_assigned_cpu_ids($pinning_map); + my $topo_result = validate_topology_for_mode( + $mode, $topology, $assigned_cpu_ids, $conf); + + return ($pinning_map, $topo_result, $vcpu_to_numa_map, $topology); +} + +# QEMU requires -numa node,nodeid=N to start from 0 and be sequential. +# If host nodes are [1, 3], guest IDs become {0 => 1, 1 => 3}. +my sub build_sequential_guest_node_to_host_node($vcpu_to_numa_map) { + return {} if !defined($vcpu_to_numa_map) || !%$vcpu_to_numa_map; + + my %seen; + $seen{$_} = 1 for values %$vcpu_to_numa_map; + my @host_nodes = sort { $a <=> $b } keys %seen; + + my $map = {}; + my $guest_id = 0; + for my $host_node (@host_nodes) { + $map->{$guest_id} = $host_node; + $guest_id++; + } + return $map; +} + +# Rewrite vcpu→host_node values to vcpu→guest_node values. +# +# Why this is needed: the allocator works in host node id space, because +# that is what the kernel and the reservations file use. QEMU, however, +# requires -numa node,nodeid=N to start at 0 and be contiguous. On a host +# where the allocator picked nodes {2, 3}, QEMU must still see nodeid=0 +# and nodeid=1. This function translates between the two id spaces. +# +# On the common case where the allocator picked {0, 1, 2, ...}, +# the output equals the input. +# +# in: {0=>1, 1=>1, 2=>3}, {0=>1, 1=>3} +# out: {0=>0, 1=>0, 2=>1} +my sub remap_to_guest_ids($vcpu_to_numa_map, $guest_node_to_host_node) { + return {} if !%$vcpu_to_numa_map || !%$guest_node_to_host_node; + + my %host_to_guest; + for my $guest_id (keys %$guest_node_to_host_node) { + my $host_id = $guest_node_to_host_node->{$guest_id}; + $host_to_guest{$host_id} = $guest_id; + } + + my %remapped; + for my $vcpu (keys %$vcpu_to_numa_map) { + my $host_node = $vcpu_to_numa_map->{$vcpu}; + $remapped{$vcpu} = $host_to_guest{$host_node} // $host_node; + } + return \%remapped; +} + +# Union of all host CPUs referenced by the pinning map. +# in: {0=>"0", 1=>"1", 2=>"4,5"} +# out: "0,1,4,5" +my sub cpuset_from_pinning_map($pinning_map) { + return undef if !%$pinning_map; + + my %unique_cpus; + for my $vcpu_id (keys %$pinning_map) { + my (undef, $members) = PVE::CpuSet::parse_cpuset($pinning_map->{$vcpu_id}); + $unique_cpus{$_} = 1 for keys %$members; + } + + return undef if !%unique_cpus; + return join(',', sort { $a <=> $b } keys %unique_cpus); +} + +# Union of CPUs on the host nodes selected in balanced mode. +# in: {0=>1, 1=>3}, $topology +# out: "4,5,6,7,20,21,22,23,12,13,14,15,28,29,30,31" +my sub cpuset_for_balanced_mode($guest_node_to_host_node, $topology) { + my %host_nodes = map { $_ => 1 } values %$guest_node_to_host_node; + my @cpus; + for my $host_node (sort { $a <=> $b } keys %host_nodes) { + next if !defined($topology->{numa_nodes}->{$host_node}); + push @cpus, sort { $a <=> $b } + keys $topology->{numa_nodes}->{$host_node}->{cpus}->%*; + } + return @cpus ? join(',', @cpus) : undef; +} + +=head2 compute_startup_cpuset + + my $cpuset = compute_startup_cpuset( + $pinning_map, $guest_node_to_host_node, $affinity, $topology); + +Determines the cpuset that the QEMU main thread is confined to at startup. +The scheduler only ever runs QEMU on these CPUs, so this must cover every +CPU that any vCPU might be pinned to. + +Priority: + +=over + +=item 1. Union of CPUs in C<$pinning_map> (numa, one-to-one). + +=item 2. Union of CPUs on selected host nodes (balanced). + +=item 3. VM's C setting. + +=item 4. All online CPUs. + +=back + +The chosen set is then restricted to a single CPU type if the host is +heterogeneous (ARM big.LITTLE, Intel P+E). KVM vCPU init can fail if the +QEMU process is allowed to migrate across CPU types during startup, so on +such hosts we narrow the cpuset to the largest same-type subset — or, if +no cpuset has been established yet, to the highest-capacity type. + +Returns a comma-joined cpuset string, or the original C<$affinity> if no +restriction is possible. + +=cut +my sub compute_startup_cpuset($pinning_map, $guest_node_to_host_node, $affinity, $topology) { + my $base_cpuset = $affinity; + + if (%$pinning_map) { + my $pinning_cpuset = cpuset_from_pinning_map($pinning_map); + $base_cpuset = $pinning_cpuset if defined($pinning_cpuset); + } elsif (%$guest_node_to_host_node) { + my $balanced = cpuset_for_balanced_mode($guest_node_to_host_node, $topology); + $base_cpuset = $balanced if defined($balanced); + } + + if (defined($base_cpuset) && $base_cpuset ne '') { + my $restricted = restrict_to_largest_cpu_type($base_cpuset); + $base_cpuset = $restricted if defined($restricted) && $restricted ne ''; + } else { + my $best_type = pick_highest_capacity_cpu_type(); + $base_cpuset = $best_type if defined($best_type); + } + + return $base_cpuset; +} + +my sub taskset_thread($cpus, $thread_id) { + run_command( + ['/usr/bin/taskset', '-c', '-p', $cpus, $thread_id], + errmsg => "failed to pin thread $thread_id to cpus $cpus", + quiet => 1, + ); +} + +sub pin_vcpu_threads_to_host_cpus($vmid, $pinning_map) { + return if !%$pinning_map; + + my $cpuinfo = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-cpus-fast'); + + for my $vcpu ($cpuinfo->@*) { + my $vcpu_index = $vcpu->{'cpu-index'}; + my $cpus = $pinning_map->{$vcpu_index}; + + die "no cpus selected for pinning vcpu $vcpu_index\n" + if !defined($cpus); + + my $tid = $vcpu->{'thread-id'}; + print "pinning vcpu $vcpu_index (thread $tid) to cpu(s) $cpus\n"; + taskset_thread($cpus, $tid); + } +} + +=head2 get_effective_topology + +Computes the effective CPU topology and pinning for a VM start. + +Returns a hashref with: + +=over + +=item C, C, C, C - the guest CPU topology +to pass to QEMU. + +=item C - cpuset string for the QEMU main thread. + +=item C - C<< { $vcpu_id => $cpus } >> for C and +C; empty for C and C. + +=item C - C<< { $vcpu_id => $guest_node_id } >> with +sequential guest node ids. + +=item C - C<< { $guest_node_id => $host_node_id } >>. + +=back + +Dies via C on invalid pinning configurations. + +=cut +sub get_effective_topology($conf, $vmid) { + my ($sockets, $cores, $maxcpus, $vcpus) = + PVE::QemuServer::Pinning::Config::get_config_topology($conf); + + my ($pinning_map, $topo_result, $vcpu_to_numa_map, $topology) = + get_pinning_map($conf, $vmid, $vcpus); + + my $guest_node_to_host_node = build_sequential_guest_node_to_host_node($vcpu_to_numa_map); + $vcpu_to_numa_map = remap_to_guest_ids($vcpu_to_numa_map, $guest_node_to_host_node); + + my $startup_cpuset = compute_startup_cpuset( + $pinning_map, $guest_node_to_host_node, $conf->{affinity}, $topology); + + # Fall back to flat topology if one-to-one pinning didn't match hardware. + if (%$pinning_map && !$topo_result->{valid}) { + $sockets = $topo_result->{sockets}; + $cores = $topo_result->{cores}; + $maxcpus = $sockets * $cores; + $vcpus = $maxcpus; + } + + return { + sockets => $sockets, + cores => $cores, + maxcpus => $maxcpus, + vcpus => $vcpus, + startup_cpuset => $startup_cpuset, + pinning_map => $pinning_map, + vcpu_to_numa_map => $vcpu_to_numa_map, + guest_node_to_host_node => $guest_node_to_host_node, + }; +} + +1; diff --git a/src/PVE/QemuServer/Pinning/Makefile b/src/PVE/QemuServer/Pinning/Makefile new file mode 100644 index 00000000..aa2c9603 --- /dev/null +++ b/src/PVE/QemuServer/Pinning/Makefile @@ -0,0 +1,9 @@ +DESTDIR= +PREFIX=/usr +PERLDIR=$(PREFIX)/share/perl5 + +SOURCES=Allocator.pm Config.pm Topology.pm + +.PHONY: install +install: $(SOURCES) + for i in $(SOURCES); do install -D -m 0644 $$i $(DESTDIR)$(PERLDIR)/PVE/QemuServer/Pinning/$$i; done -- 2.47.3