[RFC qemu-server v2 3/4] memory: integrate pinning-aware NUMA memory binding

[Elias Huhsovitz <e.huhsovitz@proxmox.com>]

Extend config to accept pinning decisions and emit the corresponding
QEMU NUMA arguments.

Currently memory binding is not automatically configured and must be set
via policy=bind. In the context of NUMA aware CPU Pinning, omitting
memory binding rarely makes sense.

When NUMA + pinning is active, the memory module now automatically binds
guest memory to the host NUMA nodes selected by the allocator using
policy=bind, even if the user did not explicitly configure 'hostnodes'
in the numaX settings.

Emit detailed '-numa cpu' arguments to pass the exact socket, core, and
thread topology to the guest OS.

Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
 src/PVE/QemuServer/Memory.pm | 240 +++++++++++++++++++++++++----------
 1 file changed, 171 insertions(+), 69 deletions(-)

diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm
index 27738cd9..310b6b04 100644
--- a/src/PVE/QemuServer/Memory.pm
+++ b/src/PVE/QemuServer/Memory.pm
@@ -364,8 +364,165 @@ sub qemu_memdevices_list {
     return $dimms;
 }
 
+my sub pass_detailed_numa_topology {
+    my ($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info) = @_;
+
+    my $threads_per_core = 1;
+    if ($conf->{vcpus} && $conf->{vcpus} > $sockets * $cores) {
+        $threads_per_core = int($conf->{vcpus} / ($sockets * $cores));
+    }
+
+    my $pinning_active = $pinning_info
+        && $pinning_info->{vcpu_to_numa_map}
+        && %{$pinning_info->{vcpu_to_numa_map}};
+
+    my $vcpu_id = 0;
+    for (my $s = 0; $s < $sockets; $s++) {
+        for (my $c = 0; $c < $cores; $c++) {
+            for (my $t = 0; $t < $threads_per_core; $t++) {
+                my $nodeid;
+
+                if ($numa_totalmemory) {
+                    # Custom numaX topology: find which numaX contains this vcpu_id.
+                    for (my $i = 0; $i < $MAX_NUMA; $i++) {
+                        next if !$conf->{"numa$i"};
+                        my $numa = parse_numa($conf->{"numa$i"});
+                        next if !$numa || !$numa->{cpus};
+
+                        for my $range (@{$numa->{cpus}}) {
+                            my ($start, $end) = @$range;
+                            $end //= $start;
+                            if ($vcpu_id >= $start && $vcpu_id <= $end) {
+                                $nodeid = $i;
+                                last;
+                            }
+                        }
+                        last if defined $nodeid;
+                    }
+                } elsif ($pinning_active) {
+                    # No numaX, but pinning decided the layout: use it.
+                    # vcpu_to_numa_map values are host node IDs; with our 1:1
+                    # guest_node_to_host_node scheme the guest nodeid equals
+                    # the host node id.
+                    my $host_node = $pinning_info->{vcpu_to_numa_map}->{$vcpu_id};
+                    $nodeid = $host_node if defined $host_node;
+                } else {
+                    # Default topology: 1 node per socket.
+                    $nodeid = $s;
+                }
+
+                if (defined $nodeid) {
+                    push @$cmd, '-numa', "cpu,node-id=$nodeid,socket-id=$s,core-id=$c,thread-id=$t";
+                }
+                $vcpu_id++;
+            }
+        }
+    }
+}
+
+# Emit one -object memory-backend + one -numa node for guest node $i
+# Returns: node's memory size, or undef if numa$i is not set
+my sub emit_explicit_numa_node {
+    my ($cmd, $conf, $node, $virtiofs_enabled, $pinning_active, $pinning_info) = @_;
+    my $numa = parse_numa($conf->{"numa$node"});
+    return if !$numa;
+
+    die "missing NUMA node$node memory value\n" if !$numa->{memory};
+    my $numa_memory = $numa->{memory};
+
+    my $memdev = $virtiofs_enabled ? "virtiofs-mem$node" : "ram-node$node";
+    my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
+
+    my $cpulists = $numa->{cpus};
+    die "missing NUMA node$node cpus\n" if !defined($cpulists);
+    my $cpus = join(';',
+        map {
+            my ($start, $end) = @$_;
+            defined($end) ? "$start-$end" : $start
+        } @$cpulists,
+    );
+
+    my $hostnodelists = $numa->{hostnodes};
+    if (defined($hostnodelists)) {
+        my $hostnodes = print_numa_hostnodes($hostnodelists);
+        my $policy = $numa->{policy};
+        die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
+        $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
+    } elsif ($pinning_active
+            && defined($pinning_info->{guest_node_to_host_node}->{$node})) {
+        # No explicit hostnodes, but pinning is set: bind memory to the
+        # host node the pinning decision placed guest node $i on.
+        my $host_node = $pinning_info->{guest_node_to_host_node}->{$node};
+        $mem_object .= ",host-nodes=$host_node,policy=bind";
+    } else {
+        die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
+    }
+
+    push @$cmd, '-object', $mem_object;
+    push @$cmd, '-numa', "node,nodeid=$node,cpus=$cpus,memdev=$memdev";
+
+    return $numa_memory;
+}
+
+# Emit one -object memory-backend + one -numa node per guest node chosen
+# by the pinning allocator. Memory is split evenly (remainder to the first
+# node) and bound with policy=bind.
+my sub emit_pinning_aware_numa {
+    my ($cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory) = @_;
+    my @guest_nodes = sort { $a <=> $b }
+        keys %{$pinning_info->{guest_node_to_host_node}};
+    my $num_nodes = scalar @guest_nodes;
+    die "pinning selected no host NUMA nodes\n" if $num_nodes == 0;
+
+    my $numa_memory = int($static_memory / $num_nodes);
+    my $remainder   = $static_memory - ($numa_memory * $num_nodes);
+
+    my $first = 1;
+    for my $guest_node (@guest_nodes) {
+        my $host_node = $pinning_info->{guest_node_to_host_node}->{$guest_node};
+        my $mem_for_node = $numa_memory + ($first ? $remainder : 0);
+        $first = 0;
+
+        my $memdev = $virtiofs_enabled
+            ? "virtiofs-mem$guest_node"
+            : "ram-node$guest_node";
+        my $mem_object = print_mem_object($conf, $memdev, $mem_for_node);
+        $mem_object .= ",host-nodes=$host_node,policy=bind";
+
+        push @$cmd, '-object', $mem_object;
+        push @$cmd, '-numa', "node,nodeid=$guest_node,memdev=$memdev";
+    }
+}
+
+# Emit one -object memory-backend & one -numa node per guest socket.
+# Used when numa: 1 but no numaX entries and no pinning info. No host
+# binding. Memory is split evenly; cpus= covers each socket's vCPU range.
+my sub emit_per_socket_numa {
+    my ($cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory) = @_;
+    my $numa_memory = ($static_memory / $sockets);
+
+    for (my $i = 0; $i < $sockets; $i++) {
+        die "host NUMA node$i doesn't exist\n"
+            if !host_numanode_exists($i) && $conf->{hugepages};
+
+        my $cpus = ($cores * $i);
+        $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
+
+        my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
+        my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
+        push @$cmd, '-object', $mem_object;
+        push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+    }
+}
+
 sub config {
-    my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags) = @_;
+    my ($conf, $vmid, $sockets, $cores, $hotplug, $virtiofs_enabled, $cmd, $machine_flags, $pinning_info) = @_;
+
+    $pinning_info //= {};
+    my $pinning_active = $pinning_info->{vcpu_to_numa_map}
+        && %{$pinning_info->{vcpu_to_numa_map}}
+        && $pinning_info->{guest_node_to_host_node}
+        && %{$pinning_info->{guest_node_to_host_node}};
 
     my $memory = get_current_memory($conf->{memory});
     my $static_memory = 0;
@@ -381,94 +538,46 @@ sub config {
         }
 
         my $sockets = $conf->{sockets} || 1;
-
         $static_memory = $STATICMEM;
         $static_memory = $static_memory * $sockets
             if ($conf->{hugepages} && $conf->{hugepages} == 1024);
 
         die "minimum memory must be ${static_memory}MB\n" if ($memory < $static_memory);
         push @$cmd, '-m', "size=${static_memory},slots=255,maxmem=${MAX_MEM}M";
-
     } else {
-
         $static_memory = $memory;
         push @$cmd, '-m', $static_memory;
     }
 
     die "numa needs to be enabled to use hugepages" if $conf->{hugepages} && !$conf->{numa};
-
     die "Memory hotplug does not work in combination with virtio-fs.\n"
         if $hotplug && $virtiofs_enabled;
 
     if ($conf->{numa}) {
-
         my $numa_totalmemory = undef;
+
         for (my $i = 0; $i < $MAX_NUMA; $i++) {
             next if !$conf->{"numa$i"};
-            my $numa = parse_numa($conf->{"numa$i"});
-            next if !$numa;
-            # memory
-            die "missing NUMA node$i memory value\n" if !$numa->{memory};
-            my $numa_memory = $numa->{memory};
-            $numa_totalmemory += $numa_memory;
-
-            my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
-            my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
-
-            # cpus
-            my $cpulists = $numa->{cpus};
-            die "missing NUMA node$i cpus\n" if !defined($cpulists);
-            my $cpus = join(
-                ',cpus=',
-                map {
-                    my ($start, $end) = @$_;
-                    defined($end) ? "$start-$end" : $start
-                } @$cpulists,
-            );
-
-            # hostnodes
-            my $hostnodelists = $numa->{hostnodes};
-            if (defined($hostnodelists)) {
-
-                my $hostnodes = print_numa_hostnodes($hostnodelists);
-
-                # policy - note that the value 'default' is not exposed, because it can't be used in
-                # combination with host-nodes:
-                # > kvm: host-nodes must be empty for policy default, or you should explicitly
-                # > specify a policy other than default
-                my $policy = $numa->{policy};
-                die "you need to define a policy for hostnode $hostnodes\n" if !$policy;
-                $mem_object .= ",host-nodes=$hostnodes,policy=$policy";
-            } else {
-                die "numa hostnodes need to be defined to use hugepages" if $conf->{hugepages};
-            }
-
-            push @$cmd, '-object', $mem_object;
-            push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+            $numa_totalmemory += emit_explicit_numa_node(
+                $cmd, $conf, $i, $virtiofs_enabled, $pinning_active, $pinning_info);
         }
 
         die "total memory for NUMA nodes must be equal to vm static memory\n"
             if $numa_totalmemory && $numa_totalmemory != $static_memory;
 
-        #if no custom tology, we split memory and cores across numa nodes
         if (!$numa_totalmemory) {
-            my $numa_memory = ($static_memory / $sockets);
-
-            for (my $i = 0; $i < $sockets; $i++) {
-                die "host NUMA node$i doesn't exist\n"
-                    if !host_numanode_exists($i) && $conf->{hugepages};
-
-                my $cpus = ($cores * $i);
-                $cpus .= "-" . ($cpus + $cores - 1) if $cores > 1;
-
-                my $memdev = $virtiofs_enabled ? "virtiofs-mem$i" : "ram-node$i";
-                my $mem_object = print_mem_object($conf, $memdev, $numa_memory);
-                push @$cmd, '-object', $mem_object;
-                push @$cmd, '-numa', "node,nodeid=$i,cpus=$cpus,memdev=$memdev";
+            if ($pinning_active) {
+                emit_pinning_aware_numa(
+                    $cmd, $conf, $pinning_info, $virtiofs_enabled, $static_memory);
+            } else {
+                emit_per_socket_numa(
+                    $cmd, $conf, $sockets, $cores, $virtiofs_enabled, $static_memory);
             }
         }
+
+        pass_detailed_numa_topology($cmd, $conf, $sockets, $cores, $numa_totalmemory, $pinning_info);
+
     } elsif ($virtiofs_enabled) {
-        # kvm: '-machine memory-backend' and '-numa memdev' properties are mutually exclusive
         push @$cmd, '-object',
             'memory-backend-memfd,id=virtiofs-mem' . ",size=$conf->{memory}M,share=on";
         push @$machine_flags, 'memory-backend=virtiofs-mem';
@@ -476,18 +585,12 @@ sub config {
 
     if ($hotplug) {
         foreach_dimm(
-            $conf,
-            $vmid,
-            $memory,
-            $static_memory,
+            $conf, $vmid, $memory, $static_memory,
             sub {
                 my ($conf, $vmid, $name, $dimm_size, $numanode, $current_size, $memory) = @_;
-
                 my $mem_object = print_mem_object($conf, "mem-$name", $dimm_size);
-
                 push @$cmd, "-object", $mem_object;
                 push @$cmd, "-device", "pc-dimm,id=$name,memdev=mem-$name,node=$numanode";
-
                 die "memory size ($memory) must be aligned to $dimm_size for hotplugging\n"
                     if $current_size > $memory;
             },
@@ -816,4 +919,3 @@ sub hugepages_update_locked {
     return $res;
 }
 1;
-
-- 
2.47.3