From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id EDDA91FF09C for ; Mon, 21 Sep 2026 11:54:32 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 9937521569; Mon, 21 Sep 2026 11:54:31 +0200 (CEST) From: Elias Huhsovitz To: pve-devel@lists.proxmox.com Subject: [RFC qemu-server v2 3/4] memory: integrate pinning-aware NUMA memory binding Date: Mon, 21 Sep 2026 11:54:02 +0200 Message-ID: <20260921095404.61552-4-e.huhsovitz@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260921095404.61552-1-e.huhsovitz@proxmox.com> References: <20260921095404.61552-1-e.huhsovitz@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1789984465635 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.637 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: L4ECI6OFAPEM7V7LWDQQABKEZDDEWSZN X-Message-ID-Hash: L4ECI6OFAPEM7V7LWDQQABKEZDDEWSZN X-MailFrom: e.huhsovitz@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Elias Huhsovitz X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Extend config to accept pinning decisions and emit the corresponding QEMU NUMA arguments. Currently memory binding is not automatically configured and must be set via policy=bind. In the context of NUMA aware CPU Pinning, omitting memory binding rarely makes sense. When NUMA + pinning is active, the memory module now automatically binds guest memory to the host NUMA nodes selected by the allocator using policy=bind, even if the user did not explicitly configure 'hostnodes' in the numaX settings. Emit detailed '-numa cpu' arguments to pass the exact socket, core, and thread topology to the guest OS. Signed-off-by: Elias Huhsovitz --- src/PVE/QemuServer/Memory.pm | 240 +++++++++++++++++++++++++---------- 1 file changed, 171 insertions(+), 69 deletions(-) diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm index 27738cd9..310b6b04 100644 --- a/src/PVE/QemuServer/Memory.pm +++ b/src/PVE/QemuServer/Memory.pm @@ -364,8 +364,165 @@ sub qemu_memdevices_list { return $dimms; } +my sub pass_detailed_numa_topology { + my ($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info) = @_; + + my $threads_per_core = 1; + if ($conf->{vcpus} && $conf->{vcpus} > $sockets * $cores) { + $threads_per_core = int($conf->{vcpus} / ($sockets * $cores)); + } + + my $pinning_active = $pinning_info + && $pinning_info->{vcpu_to_numa_map} + && %{$pinning_info->{vcpu_to_numa_map}}; + + my $vcpu_id = 0; + for (my $s = 0; $s < $sockets; $s++) { + for (my $c = 0; $c < $cores; $c++) { + for (my $t = 0; $t < $threads_per_core; $t++) { + my $nodeid; + + if ($numa_totalmemory) { + # Custom numaX topology: find which numaX contains this vcpu_id. + for (my $i = 0; $i < $MAX_NUMA; $i++) { + next if !$conf->{"numa$i"}; + my $numa = parse_numa($conf->{"numa$i"}); + next if !$numa || !$numa->{cpus}; + + for my $range (@{$numa->{cpus}}) { + my ($start, $end) = @$range; + $end //= $start; + if ($vcpu_id >= $start && $vcpu_id <= $end) { + $nodeid = $i; + last; + } + } + last if defined $nodeid; + } + } elsif ($pinning_active) { + # No numaX, but pinning decided the layout: use it. + # vcpu_to_numa_map values are host node IDs; with our 1:1 + # guest_node_to_host_node scheme the guest nodeid equals + # the host node id. + my $host_node = $pinning_info->{vcpu_to_numa_map}->{$vcpu_id}; + $nodeid = $host_node if defined $host_node; + } else { + # Default topology: 1 node per socket. + $nodeid = $s; + } + + if (defined $nodeid) { + push @$cmd, '-numa', "cpu,node-id=$nodeid,socket-id=$s,core-id=$c,thread-id=$t"; + } + $vcpu_id++; + } + } + } +} + +# Emit one -object memory-backend + one -numa node for guest node $i +# Returns: node's memory size, or undef if numa$i is not set +my sub emit_explicit_numa_node { + my ($cmd, $conf, $node, $virtiofs_enabled, $pinning_active, $pinning_info) = @_; + my $numa = parse_numa($conf->{"numa$node"}); + return if !$numa; + + die "missing NUMA node$node memory value\n" if !$numa->{memory}; + my $numa_memory = $numa->{memory}; + + my $memdev = $virtiofs_enabled ? "virtiofs-mem$node" : "ram-node$node"; + my $mem_object = print_mem_object($conf, $memdev, $numa_memory); + + my $cpulists = $numa->{cpus}; + die "missing NUMA node$node cpus\n" if !defined($cpulists); + my $cpus = join(';', + map { + my ($start, $end) = @$_; + defined($end) ? "$start-$end" : $start + } @$cpulists, + ); + + my $hostnodelists = $numa->{hostnodes}; + if (defined($hostnodelists)) { + my $hostnodes = print_numa_hostnodes($hostnodelists); + my $policy = $numa->{policy}; + die "you need to define a policy for hostnode $hostnodes\n" if !$policy; + $mem_object .= ",host-nodes=$hostnodes,policy=$policy"; + } elsif ($pinning_active + && defined($pinning_info->{guest_node_to_host_node}->{$node})) { + # No explicit hostnodes, but pinning is set: bind memory to the + # host node the pinning decision placed guest node $i on. + my $host_node = $pinning_info->{guest_node_to_host_node}->{$node}; + $mem_object .= ",host-nodes=$host_node,policy=bind"; + } else { + die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages}; + } + + push @$cmd, '-object', $mem_object; + push @$cmd, '-numa', "node,nodeid=$node,cpus=$cpus,memdev=$memdev"; + + return $numa_memory; +} + +# Emit one -object memory-backend + one -numa node per guest node chosen +# by the pinning allocator. Memory is split evenly (remainder to the first +# node) and bound with policy=bind. +my sub emit_pinning_aware_numa { + my ($cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory) = @_; + my @guest_nodes = sort { $a <=> $b } + keys %{$pinning_info->{guest_node_to_host_node}}; + my $num_nodes = scalar @guest_nodes; + die "pinning selected no host NUMA nodes\n" if $num_nodes == 0; + + my $numa_memory = int($static_memory / $num_nodes); + my $remainder = $static_memory - ($numa_memory * $num_nodes); + + my $first = 1; + for my $guest_node (@guest_nodes) { + my $host_node = $pinning_info->{guest_node_to_host_node}->{$guest_node}; + my $mem_for_node = $numa_memory + ($first ? $remainder : 0); + $first = 0; + + my $memdev = $virtiofs_enabled + ? "virtiofs-mem$guest_node" + : "ram-node$guest_node"; + my $mem_object = print_mem_object($conf, $memdev, $mem_for_node); + $mem_object .= ",host-nodes=$host_node,policy=bind"; + + push @$cmd, '-object', $mem_object; + push @$cmd, '-numa', "node,nodeid=$guest_node,memdev=$memdev"; + } +} + +# Emit one -object memory-backend & one -numa node per guest socket. +# Used when numa: 1 but no numaX entries and no pinning info. No host +# binding. Memory is split evenly; cpus= covers each socket's vCPU range. +my sub emit_per_socket_numa { + my ($cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory) = @_; + my $numa_memory = ($static_memory / $sockets); + + for (my $i = 0; $i < $sockets; $i++) { + die "host NUMA node$i doesn't exist\n" + if !host_numanode_exists($i) && $conf->{hugepages}; + + my $cpus = ($cores * $i); + $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1; + + my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i"; + my $mem_object = print_mem_object($conf, $memdev, $numa_memory); + push @$cmd, '-object', $mem_object; + push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev"; + } +} + sub config { - my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags) = @_; + my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags, $pinning_info) = @_; + + $pinning_info //= {}; + my $pinning_active = $pinning_info->{vcpu_to_numa_map} + && %{$pinning_info->{vcpu_to_numa_map}} + && $pinning_info->{guest_node_to_host_node} + && %{$pinning_info->{guest_node_to_host_node}}; my $memory = get_current_memory($conf->{memory}); my $static_memory = 0; @@ -381,94 +538,46 @@ sub config { } my $sockets = $conf->{sockets} || 1; - $static_memory = $STATICMEM; $static_memory = $static_memory * $sockets if ($conf->{hugepages} && $conf->{hugepages} == 1024); die "minimum memory must be ${static_memory}MB\n" if ($memory < $static_memory); push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M"; - } else { - $static_memory = $memory; push @$cmd, '-m', $static_memory; } die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa}; - die "Memory hotplug does not work in combination with virtio-fs.\n" if $hotplug && $virtiofs_enabled; if ($conf->{numa}) { - my $numa_totalmemory = undef; + for (my $i = 0; $i < $MAX_NUMA; $i++) { next if !$conf->{"numa$i"}; - my $numa = parse_numa($conf->{"numa$i"}); - next if !$numa; - # memory - die "missing NUMA node$i memory value\n" if !$numa->{memory}; - my $numa_memory = $numa->{memory}; - $numa_totalmemory += $numa_memory; - - my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i"; - my $mem_object = print_mem_object($conf, $memdev, $numa_memory); - - # cpus - my $cpulists = $numa->{cpus}; - die "missing NUMA node$i cpus\n" if !defined($cpulists); - my $cpus = join( - ',cpus=', - map { - my ($start, $end) = @$_; - defined($end) ? "$start-$end" : $start - } @$cpulists, - ); - - # hostnodes - my $hostnodelists = $numa->{hostnodes}; - if (defined($hostnodelists)) { - - my $hostnodes = print_numa_hostnodes($hostnodelists); - - # policy - note that the value 'default' is not exposed, because it can't be used in - # combination with host-nodes: - # > kvm: host-nodes must be empty for policy default, or you should explicitly - # > specify a policy other than default - my $policy = $numa->{policy}; - die "you need to define a policy for hostnode $hostnodes\n" if !$policy; - $mem_object .= ",host-nodes=$hostnodes,policy=$policy"; - } else { - die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages}; - } - - push @$cmd, '-object', $mem_object; - push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev"; + $numa_totalmemory += emit_explicit_numa_node( + $cmd, $conf, $i, $virtiofs_enabled, $pinning_active, $pinning_info); } die "total memory for NUMA nodes must be equal to vm static memory\n" if $numa_totalmemory && $numa_totalmemory != $static_memory; - #if no custom tology, we split memory and cores across numa nodes if (!$numa_totalmemory) { - my $numa_memory = ($static_memory / $sockets); - - for (my $i = 0; $i < $sockets; $i++) { - die "host NUMA node$i doesn't exist\n" - if !host_numanode_exists($i) && $conf->{hugepages}; - - my $cpus = ($cores * $i); - $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1; - - my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i"; - my $mem_object = print_mem_object($conf, $memdev, $numa_memory); - push @$cmd, '-object', $mem_object; - push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev"; + if ($pinning_active) { + emit_pinning_aware_numa( + $cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory); + } else { + emit_per_socket_numa( + $cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory); } } + + pass_detailed_numa_topology($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info); + } elsif ($virtiofs_enabled) { - # kvm: '-machine memory-backend' and '-numa memdev' properties are mutually exclusive push @$cmd, '-object', 'memory-backend-memfd,id=virtiofs-mem' . ",size=$conf->{memory}M,share=on"; push @$machine_flags, 'memory-backend=virtiofs-mem'; @@ -476,18 +585,12 @@ sub config { if ($hotplug) { foreach_dimm( - $conf, - $vmid, - $memory, - $static_memory, + $conf, $vmid, $memory, $static_memory, sub { my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_; - my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size); - push @$cmd, "-object", $mem_object; push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode"; - die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n" if $current_size > $memory; }, @@ -816,4 +919,3 @@ sub hugepages_update_locked { return $res; } 1; - -- 2.47.3