From: Elias Huhsovitz <e.huhsovitz@proxmox.com>
To: pve-devel@lists.proxmox.com
Cc: Elias Huhsovitz <e.huhsovitz@proxmox.com>
Subject: [RFC qemu-server v2 3/4] memory: integrate pinning-aware NUMA memory binding
Date: Mon, 21 Sep 2026 11:54:02 +0200 [thread overview]
Message-ID: <20260921095404.61552-4-e.huhsovitz@proxmox.com> (raw)
In-Reply-To: <20260921095404.61552-1-e.huhsovitz@proxmox.com>
Extend config to accept pinning decisions and emit the corresponding
QEMU NUMA arguments.
Currently memory binding is not automatically configured and must be set
via policy=bind. In the context of NUMA aware CPU Pinning, omitting
memory binding rarely makes sense.
When NUMA + pinning is active, the memory module now automatically binds
guest memory to the host NUMA nodes selected by the allocator using
policy=bind, even if the user did not explicitly configure 'hostnodes'
in the numaX settings.
Emit detailed '-numa cpu' arguments to pass the exact socket, core, and
thread topology to the guest OS.
Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
src/PVE/QemuServer/Memory.pm | 240 +++++++++++++++++++++++++----------
1 file changed, 171 insertions(+), 69 deletions(-)
diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm
index 27738cd9..310b6b04 100644
--- a/src/PVE/QemuServer/Memory.pm
+++ b/src/PVE/QemuServer/Memory.pm
@@ -364,8 +364,165 @@ sub qemu_memdevices_list {
return $dimms;
}
+my sub pass_detailed_numa_topology {
+ my ($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info) = @_;
+
+ my $threads_per_core = 1;
+ if ($conf->{vcpus} && $conf->{vcpus} > $sockets * $cores) {
+ $threads_per_core = int($conf->{vcpus} / ($sockets * $cores));
+ }
+
+ my $pinning_active = $pinning_info
+ && $pinning_info->{vcpu_to_numa_map}
+ && %{$pinning_info->{vcpu_to_numa_map}};
+
+ my $vcpu_id = 0;
+ for (my $s = 0; $s < $sockets; $s++) {
+ for (my $c = 0; $c < $cores; $c++) {
+ for (my $t = 0; $t < $threads_per_core; $t++) {
+ my $nodeid;
+
+ if ($numa_totalmemory) {
+ # Custom numaX topology: find which numaX contains this vcpu_id.
+ for (my $i = 0; $i < $MAX_NUMA; $i++) {
+ next if !$conf->{"numa$i"};
+ my $numa = parse_numa($conf->{"numa$i"});
+ next if !$numa || !$numa->{cpus};
+
+ for my $range (@{$numa->{cpus}}) {
+ my ($start, $end) = @$range;
+ $end //= $start;
+ if ($vcpu_id >= $start && $vcpu_id <= $end) {
+ $nodeid = $i;
+ last;
+ }
+ }
+ last if defined $nodeid;
+ }
+ } elsif ($pinning_active) {
+ # No numaX, but pinning decided the layout: use it.
+ # vcpu_to_numa_map values are host node IDs; with our 1:1
+ # guest_node_to_host_node scheme the guest nodeid equals
+ # the host node id.
+ my $host_node = $pinning_info->{vcpu_to_numa_map}->{$vcpu_id};
+ $nodeid = $host_node if defined $host_node;
+ } else {
+ # Default topology: 1 node per socket.
+ $nodeid = $s;
+ }
+
+ if (defined $nodeid) {
+ push @$cmd, '-numa', "cpu,node-id=$nodeid,socket-id=$s,core-id=$c,thread-id=$t";
+ }
+ $vcpu_id++;
+ }
+ }
+ }
+}
+
+# Emit one -object memory-backend + one -numa node for guest node $i
+# Returns: node's memory size, or undef if numa$i is not set
+my sub emit_explicit_numa_node {
+ my ($cmd, $conf, $node, $virtiofs_enabled, $pinning_active, $pinning_info) = @_;
+ my $numa = parse_numa($conf->{"numa$node"});
+ return if !$numa;
+
+ die "missing NUMA node$node memory value\n" if !$numa->{memory};
+ my $numa_memory = $numa->{memory};
+
+ my $memdev = $virtiofs_enabled ? "virtiofs-mem$node" : "ram-node$node";
+ my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
+
+ my $cpulists = $numa->{cpus};
+ die "missing NUMA node$node cpus\n" if !defined($cpulists);
+ my $cpus = join(';',
+ map {
+ my ($start, $end) = @$_;
+ defined($end) ? "$start-$end" : $start
+ } @$cpulists,
+ );
+
+ my $hostnodelists = $numa->{hostnodes};
+ if (defined($hostnodelists)) {
+ my $hostnodes = print_numa_hostnodes($hostnodelists);
+ my $policy = $numa->{policy};
+ die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
+ $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
+ } elsif ($pinning_active
+ && defined($pinning_info->{guest_node_to_host_node}->{$node})) {
+ # No explicit hostnodes, but pinning is set: bind memory to the
+ # host node the pinning decision placed guest node $i on.
+ my $host_node = $pinning_info->{guest_node_to_host_node}->{$node};
+ $mem_object .= ",host-nodes=$host_node,policy=bind";
+ } else {
+ die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
+ }
+
+ push @$cmd, '-object', $mem_object;
+ push @$cmd, '-numa', "node,nodeid=$node,cpus=$cpus,memdev=$memdev";
+
+ return $numa_memory;
+}
+
+# Emit one -object memory-backend + one -numa node per guest node chosen
+# by the pinning allocator. Memory is split evenly (remainder to the first
+# node) and bound with policy=bind.
+my sub emit_pinning_aware_numa {
+ my ($cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory) = @_;
+ my @guest_nodes = sort { $a <=> $b }
+ keys %{$pinning_info->{guest_node_to_host_node}};
+ my $num_nodes = scalar @guest_nodes;
+ die "pinning selected no host NUMA nodes\n" if $num_nodes == 0;
+
+ my $numa_memory = int($static_memory / $num_nodes);
+ my $remainder = $static_memory - ($numa_memory * $num_nodes);
+
+ my $first = 1;
+ for my $guest_node (@guest_nodes) {
+ my $host_node = $pinning_info->{guest_node_to_host_node}->{$guest_node};
+ my $mem_for_node = $numa_memory + ($first ? $remainder : 0);
+ $first = 0;
+
+ my $memdev = $virtiofs_enabled
+ ? "virtiofs-mem$guest_node"
+ : "ram-node$guest_node";
+ my $mem_object = print_mem_object($conf, $memdev, $mem_for_node);
+ $mem_object .= ",host-nodes=$host_node,policy=bind";
+
+ push @$cmd, '-object', $mem_object;
+ push @$cmd, '-numa', "node,nodeid=$guest_node,memdev=$memdev";
+ }
+}
+
+# Emit one -object memory-backend & one -numa node per guest socket.
+# Used when numa: 1 but no numaX entries and no pinning info. No host
+# binding. Memory is split evenly; cpus= covers each socket's vCPU range.
+my sub emit_per_socket_numa {
+ my ($cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory) = @_;
+ my $numa_memory = ($static_memory / $sockets);
+
+ for (my $i = 0; $i < $sockets; $i++) {
+ die "host NUMA node$i doesn't exist\n"
+ if !host_numanode_exists($i) && $conf->{hugepages};
+
+ my $cpus = ($cores * $i);
+ $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
+
+ my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
+ my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
+ push @$cmd, '-object', $mem_object;
+ push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+ }
+}
+
sub config {
- my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags) = @_;
+ my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags, $pinning_info) = @_;
+
+ $pinning_info //= {};
+ my $pinning_active = $pinning_info->{vcpu_to_numa_map}
+ && %{$pinning_info->{vcpu_to_numa_map}}
+ && $pinning_info->{guest_node_to_host_node}
+ && %{$pinning_info->{guest_node_to_host_node}};
my $memory = get_current_memory($conf->{memory});
my $static_memory = 0;
@@ -381,94 +538,46 @@ sub config {
}
my $sockets = $conf->{sockets} || 1;
-
$static_memory = $STATICMEM;
$static_memory = $static_memory * $sockets
if ($conf->{hugepages} && $conf->{hugepages} == 1024);
die "minimum memory must be ${static_memory}MB\n" if ($memory < $static_memory);
push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
-
} else {
-
$static_memory = $memory;
push @$cmd, '-m', $static_memory;
}
die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa};
-
die "Memory hotplug does not work in combination with virtio-fs.\n"
if $hotplug && $virtiofs_enabled;
if ($conf->{numa}) {
-
my $numa_totalmemory = undef;
+
for (my $i = 0; $i < $MAX_NUMA; $i++) {
next if !$conf->{"numa$i"};
- my $numa = parse_numa($conf->{"numa$i"});
- next if !$numa;
- # memory
- die "missing NUMA node$i memory value\n" if !$numa->{memory};
- my $numa_memory = $numa->{memory};
- $numa_totalmemory += $numa_memory;
-
- my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
- my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
-
- # cpus
- my $cpulists = $numa->{cpus};
- die "missing NUMA node$i cpus\n" if !defined($cpulists);
- my $cpus = join(
- ',cpus=',
- map {
- my ($start, $end) = @$_;
- defined($end) ? "$start-$end" : $start
- } @$cpulists,
- );
-
- # hostnodes
- my $hostnodelists = $numa->{hostnodes};
- if (defined($hostnodelists)) {
-
- my $hostnodes = print_numa_hostnodes($hostnodelists);
-
- # policy - note that the value 'default' is not exposed, because it can't be used in
- # combination with host-nodes:
- # > kvm: host-nodes must be empty for policy default, or you should explicitly
- # > specify a policy other than default
- my $policy = $numa->{policy};
- die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
- $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
- } else {
- die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
- }
-
- push @$cmd, '-object', $mem_object;
- push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+ $numa_totalmemory += emit_explicit_numa_node(
+ $cmd, $conf, $i, $virtiofs_enabled, $pinning_active, $pinning_info);
}
die "total memory for NUMA nodes must be equal to vm static memory\n"
if $numa_totalmemory && $numa_totalmemory != $static_memory;
- #if no custom tology, we split memory and cores across numa nodes
if (!$numa_totalmemory) {
- my $numa_memory = ($static_memory / $sockets);
-
- for (my $i = 0; $i < $sockets; $i++) {
- die "host NUMA node$i doesn't exist\n"
- if !host_numanode_exists($i) && $conf->{hugepages};
-
- my $cpus = ($cores * $i);
- $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
-
- my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
- my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
- push @$cmd, '-object', $mem_object;
- push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+ if ($pinning_active) {
+ emit_pinning_aware_numa(
+ $cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory);
+ } else {
+ emit_per_socket_numa(
+ $cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory);
}
}
+
+ pass_detailed_numa_topology($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info);
+
} elsif ($virtiofs_enabled) {
- # kvm: '-machine memory-backend' and '-numa memdev' properties are mutually exclusive
push @$cmd, '-object',
'memory-backend-memfd,id=virtiofs-mem' . ",size=$conf->{memory}M,share=on";
push @$machine_flags, 'memory-backend=virtiofs-mem';
@@ -476,18 +585,12 @@ sub config {
if ($hotplug) {
foreach_dimm(
- $conf,
- $vmid,
- $memory,
- $static_memory,
+ $conf, $vmid, $memory, $static_memory,
sub {
my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
-
my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
-
push @$cmd, "-object", $mem_object;
push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
-
die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n"
if $current_size > $memory;
},
@@ -816,4 +919,3 @@ sub hugepages_update_locked {
return $res;
}
1;
-
--
2.47.3
next prev parent reply other threads:[~2026-09-21 9:54 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-21 9:53 [RFC qemu-server v2 0/4] fix #7282: allow (NUMA aware) vCPU pinning Elias Huhsovitz
2026-09-21 9:54 ` [RFC qemu-server v2 1/4] pinning: add topology discovery and config parsing Elias Huhsovitz
2026-09-21 9:54 ` [RFC qemu-server v2 2/4] pinning: add NUMA allocator and reservation tracking Elias Huhsovitz
2026-09-21 9:54 ` Elias Huhsovitz [this message]
2026-09-21 9:54 ` [RFC qemu-server v2 4/4] pinning: integrate cpu pinning into vm lifecycle Elias Huhsovitz
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260921095404.61552-4-e.huhsovitz@proxmox.com \
--to=e.huhsovitz@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox