* [PATCH qemu-server-rs v4 1/4] cpu: add pve-qemu-server-cpu crate for vCPU binding
2026-11-11 15:20 [RFC proxmox-perl-rs/qemu-server/qemu-server-rs v4 0/4] vCPU binding for qemu guests Elias Huhsovitz
@ 2026-11-11 15:20 ` Elias Huhsovitz
2026-11-11 15:20 ` [PATCH proxmox-perl-rs v4 2/4] pve: cpu: add binding for pve-qemu-server-cpu crate Elias Huhsovitz
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Elias Huhsovitz @ 2026-11-11 15:20 UTC (permalink / raw)
To: pve-devel; +Cc: Elias Huhsovitz
Add a new library crate that implements vCPU to host CPU binding of qemu
guests.
2 main functions are exposed: gen_vcpu_bindings: (1) Reads the CPU
Toplogy from sysfs, validates the configured cpu binding config and
Generates a vCPU:hostCPU binding map. This function should be called
before the guest starts. (2) bind_vcpus: Applies the binding map
generated by gen_vcpu_bindings to the specified vm. The cpu binding is
applied using sched_setaffinity(2).
2 modes are implemented: (1) one-to-one: pin each vCPU to its own host
cpu. (2) numa: bind the set of vCPUs of each vNUMA node to the set of
CPUs of a host NUMA node. The cpu bindings are determined by the numa[N]
configuration of the guest.
Both settings respect the configured affinity.
Thread binding uses sched_setaffinity(2), which accepts a kernel
TID directly. The TID used is reported by qemu via query-cpus-fast.
Originally-by: Dominik Csapak <d.csapak@proxmox.com>
Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
Cargo.toml | 2 +-
build.sh | 2 +-
pve-qemu-server-cpu/Cargo.toml | 16 ++
pve-qemu-server-cpu/debian/changelog | 5 +
pve-qemu-server-cpu/debian/control | 41 ++++
pve-qemu-server-cpu/debian/copyright | 18 ++
pve-qemu-server-cpu/debian/debcargo.toml | 9 +
pve-qemu-server-cpu/src/bind.rs | 136 +++++++++++++
pve-qemu-server-cpu/src/config.rs | 146 ++++++++++++++
pve-qemu-server-cpu/src/cpuset.rs | 125 ++++++++++++
pve-qemu-server-cpu/src/error.rs | 32 ++++
pve-qemu-server-cpu/src/lib.rs | 92 +++++++++
pve-qemu-server-cpu/src/qmp.rs | 91 +++++++++
pve-qemu-server-cpu/src/syscall.rs | 24 +++
pve-qemu-server-cpu/src/topology/mod.rs | 222 ++++++++++++++++++++++
pve-qemu-server-cpu/src/topology/sysfs.rs | 6 +
16 files changed, 965 insertions(+), 2 deletions(-)
create mode 100644 pve-qemu-server-cpu/Cargo.toml
create mode 100644 pve-qemu-server-cpu/debian/changelog
create mode 100644 pve-qemu-server-cpu/debian/control
create mode 100644 pve-qemu-server-cpu/debian/copyright
create mode 100644 pve-qemu-server-cpu/debian/debcargo.toml
create mode 100644 pve-qemu-server-cpu/src/bind.rs
create mode 100644 pve-qemu-server-cpu/src/config.rs
create mode 100644 pve-qemu-server-cpu/src/cpuset.rs
create mode 100644 pve-qemu-server-cpu/src/error.rs
create mode 100644 pve-qemu-server-cpu/src/lib.rs
create mode 100644 pve-qemu-server-cpu/src/qmp.rs
create mode 100644 pve-qemu-server-cpu/src/syscall.rs
create mode 100644 pve-qemu-server-cpu/src/topology/mod.rs
create mode 100644 pve-qemu-server-cpu/src/topology/sysfs.rs
diff --git a/Cargo.toml b/Cargo.toml
index 5eefc79..80f87ff 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,6 @@
[workspace]
members = [
+ "pve-qemu-server-cpu",
"pve-qemu-server-pci"
]
exclude = [
@@ -25,4 +26,3 @@ strum = { version = "0.26", features = ["derive"] }
# proxmox dependencies
pve-api-types = "8.1"
-
diff --git a/build.sh b/build.sh
index 3c2293b..c3ee7b6 100755
--- a/build.sh
+++ b/build.sh
@@ -28,7 +28,7 @@ debcargo package \
--no-overlay-write-back \
--directory "$PWD/"${BUILDDIR}"/${CRATE}" \
"${CRATE}" \
- "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//')"
+ "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//' -e 's/~/-/')"
cd "${BUILDDIR}/${CRATE}"
rm -f debian/source/format.debcargo.hint
diff --git a/pve-qemu-server-cpu/Cargo.toml b/pve-qemu-server-cpu/Cargo.toml
new file mode 100644
index 0000000..b236796
--- /dev/null
+++ b/pve-qemu-server-cpu/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "pve-qemu-server-cpu"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+exclude.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+thiserror = "2"
+libc = "0.2"
diff --git a/pve-qemu-server-cpu/debian/changelog b/pve-qemu-server-cpu/debian/changelog
new file mode 100644
index 0000000..3731cfd
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/changelog
@@ -0,0 +1,5 @@
+rust-pve-qemu-server-cpu (0.1.0) trixie; urgency=medium
+
+ * initial release
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 01 Sep 2026 14:02:25 +0200
diff --git a/pve-qemu-server-cpu/debian/control b/pve-qemu-server-cpu/debian/control
new file mode 100644
index 0000000..1298d61
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/control
@@ -0,0 +1,41 @@
+Source: rust-pve-qemu-server-cpu
+Section: rust
+Priority: optional
+Build-Depends: debhelper-compat (= 13),
+ dh-sequence-cargo,
+ libhwloc-dev (>= 2.12)
+Build-Depends-Arch: cargo:native <!nocheck>,
+ rustc:native (>= 1.94) <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-libc-0.2+default-dev <!nocheck>,
+ librust-serde-1+default-dev <!nocheck>,
+ librust-serde-1+derive-dev <!nocheck>,
+ librust-serde-json-1+default-dev <!nocheck>,
+ librust-thiserror-2+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.7.2
+Vcs-Git: git://git.proxmox.com/git/pve-qemu-server-rs.git
+Vcs-Browser: https://git.proxmox.com/?p=pve-qemu-server-rs.git
+Homepage: https://proxmox.com
+X-Cargo-Crate: pve-qemu-server-cpu
+
+Package: librust-pve-qemu-server-cpu-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-libc-0.2+default-dev,
+ librust-serde-1+default-dev,
+ librust-serde-1+derive-dev,
+ librust-serde-json-1+default-dev,
+ librust-thiserror-2+default-dev
+Provides:
+ librust-pve-qemu-server-cpu+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1.0-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1.0+default-dev (= ${binary:Version})
+Description: Rust crate "pve-qemu-server-cpu" - Rust source code
+ Source code for Debianized Rust crate "pve-qemu-server-cpu"
diff --git a/pve-qemu-server-cpu/debian/copyright b/pve-qemu-server-cpu/debian/copyright
new file mode 100644
index 0000000..01138fa
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/copyright
@@ -0,0 +1,18 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files:
+ *
+Copyright: 2026 Proxmox Server Solutions GmbH <support@proxmox.com>
+License: AGPL-3.0-or-later
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU Affero General Public License as published by the Free
+ Software Foundation, either version 3 of the License, or (at your option) any
+ later version.
+ .
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ details.
+ .
+ You should have received a copy of the GNU Affero General Public License along
+ with this program. If not, see <https://www.gnu.org/licenses/>.
diff --git a/pve-qemu-server-cpu/debian/debcargo.toml b/pve-qemu-server-cpu/debian/debcargo.toml
new file mode 100644
index 0000000..23feea4
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/debcargo.toml
@@ -0,0 +1,9 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+allow_prerelease_deps = true
+
+[source]
+vcs_git = "git://git.proxmox.com/git/pve-qemu-server-rs.git"
+vcs_browser = "https://git.proxmox.com/?p=pve-qemu-server-rs.git"
+build_depends = ["libhwloc-dev (>= 2.12)"]
diff --git a/pve-qemu-server-cpu/src/bind.rs b/pve-qemu-server-cpu/src/bind.rs
new file mode 100644
index 0000000..d7f52f8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/bind.rs
@@ -0,0 +1,136 @@
+use std::collections::BTreeMap;
+
+use crate::cpuset::CpuSet;
+use crate::config::NumaConfig;
+use crate::qmp::QmpCpu;
+use crate::syscall::sched_setaffinity;
+use crate::topology::Topology;
+use crate::error::CpuError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Mode {
+ None,
+ OneToOne,
+ Numa,
+}
+
+impl std::str::FromStr for Mode {
+ type Err = CpuError;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "none" => Ok(Mode::None),
+ "one-to-one" => Ok(Mode::OneToOne),
+ "numa" => Ok(Mode::Numa),
+ other => Err(CpuError::BadSyntax(format!(
+ "unknown cpu-bind-mode '{other}'"
+ ))),
+ }
+ }
+}
+
+pub fn gen_one_to_one_bindings(
+ sockets: u32,
+ cores: u32,
+ affinity: &Option<CpuSet>,
+ only_real_cores: bool,
+) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let topo = Topology::new()?;
+ let mut allowed = topo.online_cpus().clone();
+
+ if let Some(aff) = affinity {
+ allowed.intersect(aff);
+ }
+
+ let nr_vcpus = sockets * cores;
+
+ if only_real_cores {
+ allowed = topo.singlify_per_core(&allowed);
+ }
+
+ if allowed.size() < nr_vcpus as usize {
+ return Err(CpuError::BadSyntax(format!(
+ "one-to-one mode requires {nr_vcpus} host cpus but only {} are available",
+ allowed.size()
+ )));
+ }
+
+ let available: Vec<u32> = allowed.to_vec();
+
+ let mut bindings: BTreeMap<u32, CpuSet> = BTreeMap::new();
+
+ // qmp::parse_cpuinfo should yield vCPU IDs that are 0..N-1.
+ // If not: we proably messed up somewhere else
+ for (vcpu_id, cpu) in available.iter().take(nr_vcpus as usize).enumerate() {
+ let mut single_cpu = CpuSet::new();
+ single_cpu.set(*cpu);
+ bindings.insert(vcpu_id as u32, single_cpu);
+ }
+
+ Ok(bindings)
+}
+
+pub fn gen_numa_bindings(
+ numa_cfg: &Vec<NumaConfig>,
+ affinity: &Option<CpuSet>,
+) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let topo = Topology::new()?;
+ let mut bindings = BTreeMap::new();
+
+ for (vnode_id, cfg) in numa_cfg.iter().enumerate() {
+ if cfg.hostnodes.len() != 1 {
+ return Err(CpuError::BadSyntax(format!(
+ "vNUMA node {vnode_id} must be bound to exactly one host NUMA node"
+ )));
+ }
+
+ let host_node_id = *cfg.hostnodes.first().unwrap();
+ let host_cpus = topo.numa_cpus(host_node_id).ok_or_else(|| {
+ CpuError::BadSyntax(format!(
+ "host NUMA node {host_node_id} does not exist on this host"
+ ))
+ })?;
+
+ let mut allowed = host_cpus.clone();
+
+ if let Some(aff) = affinity {
+ allowed.intersect(aff);
+ }
+
+ if allowed.is_empty() {
+ return Err(CpuError::BadSyntax(format!(
+ "no allowed CPUs available for host NUMA node {host_node_id} \
+ after applying affinity mask."
+ )));
+ }
+
+ for &vcpu_id in &cfg.vcpus {
+ bindings.insert(vcpu_id, allowed.clone());
+ }
+ }
+
+ Ok(bindings)
+}
+
+/// Applies the generated CPU bindings to the running vCPU threads.
+pub fn apply_bindings(
+ bindings: &BTreeMap<u32, CpuSet>,
+ qmp_cpuinfo: &[QmpCpu],
+) -> Result<(), CpuError> {
+ for (&vcpu_id, cpuset) in bindings {
+ let vcpu_index = vcpu_id as usize;
+ if vcpu_index >= qmp_cpuinfo.len() {
+ return Err(CpuError::VcpuOutOfRange {
+ vcpu: vcpu_id,
+ max: qmp_cpuinfo.len() as u32,
+ });
+ }
+
+ let tid = qmp_cpuinfo[vcpu_index].thread_id;
+ sched_setaffinity(tid as libc::pid_t, cpuset).map_err(|e| CpuError::Bind {
+ vcpu: vcpu_id,
+ message: e.to_string(),
+ })?;
+ }
+ Ok(())
+}
diff --git a/pve-qemu-server-cpu/src/config.rs b/pve-qemu-server-cpu/src/config.rs
new file mode 100644
index 0000000..a89c62c
--- /dev/null
+++ b/pve-qemu-server-cpu/src/config.rs
@@ -0,0 +1,146 @@
+//! Parsing of the Proxmox VE CPU and memory configuration.
+
+use serde::de::{self, Deserializer, SeqAccess, Visitor};
+use serde::Deserialize;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::fmt;
+
+use crate::cpuset::CpuSet;
+use crate::error::CpuError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum NumaPolicy {
+ Bind,
+ Interleave,
+ Preferred,
+}
+
+/// Configuration for a single numa[N] node.
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
+pub struct NumaConfig {
+ /// Guest vCPU IDs assigned to this node.
+ #[serde(default, deserialize_with = "deserialize_range_set")]
+ pub vcpus: BTreeSet<u32>,
+
+ /// Host NUMA node IDs.
+ #[serde(default, deserialize_with = "deserialize_range_set")]
+ pub hostnodes: BTreeSet<u32>,
+
+ /// Memory provided by this node, in MiB.
+ pub memory: u32,
+
+ /// NUMA allocation policy.
+ pub policy: Option<NumaPolicy>,
+}
+
+/// Parses the NUMA configuration from a JSON string.
+///
+/// Returns a `Vec` where the index corresponds to the NUMA node ID.
+/// Assumes the Perl layer has already validated that node IDs are
+/// contiguous from `0` to `N-1`.
+/// Parses the NUMA configuration from a JSON string.
+///
+/// Returns a `Vec` where the index corresponds to the NUMA node ID.
+/// Assumes the Perl layer has already validated that node IDs are
+/// contiguous from `0` to `N-1`.
+pub fn numa_config_from_json(cfg_json: &str) -> Result<Vec<NumaConfig>, CpuError> {
+ let raw_map: HashMap<String, NumaConfig> = serde_json::from_str(cfg_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid NUMA JSON: {e}")))?;
+
+ let len = raw_map.len();
+
+ let mut configs: Vec<Option<NumaConfig>> = vec![None; len];
+
+ for (key, config) in raw_map {
+ let id_str = key.strip_prefix("numa").unwrap_or(&key);
+ let id = id_str.parse::<usize>().map_err(|e| {
+ CpuError::BadSyntax(format!("invalid NUMA node ID '{key}': {e}"))
+ })?;
+
+ configs[id] = Some(config);
+ }
+
+ // Unwrap the options. If Perl did its job, every slot will be filled.
+ // We return an error instead of panicking just in case the invariant is broken.
+ configs
+ .into_iter()
+ .enumerate()
+ .map(|(i, opt)| {
+ opt.ok_or_else(|| CpuError::BadSyntax(format!("missing NUMA node {i}")))
+ })
+ .collect()
+}
+
+/// Expands a JSON array of ranges into a `BTreeSet<u32>`.
+///
+/// Perl outputs ranges as `[[0, 3], [8, null]]`. This visitor expands them
+/// into a flat set of integers: `{0, 1, 2, 3, 8}`.
+fn deserialize_range_set<'de, D>(deserializer: D) -> Result<BTreeSet<u32>, D::Error>
+where
+ D: Deserializer<'de>,
+{
+ struct RangeSetVisitor;
+
+ impl<'de> Visitor<'de> for RangeSetVisitor {
+ type Value = BTreeSet<u32>;
+
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("a JSON array of ranges (e.g., [[0, 3], [8, null]])")
+ }
+
+ fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+ where
+ A: SeqAccess<'de>,
+ {
+ let mut set = BTreeSet::new();
+
+ while let Some((start, end_opt)) = seq.next_element::<(u32, Option<u32>)>()? {
+ let end = end_opt.unwrap_or(start);
+ if end < start {
+ return Err(de::Error::custom(format!(
+ "invalid range: end ({end}) < start ({start})"
+ )));
+ }
+ set.extend(start..=end);
+ }
+
+ Ok(set)
+ }
+
+ fn visit_unit<E>(self) -> Result<Self::Value, E> {
+ Ok(BTreeSet::new())
+ }
+
+ fn visit_none<E>(self) -> Result<Self::Value, E> {
+ Ok(BTreeSet::new())
+ }
+ }
+
+ deserializer.deserialize_any(RangeSetVisitor)
+}
+
+pub fn bindings_from_json(bind_map_json: &str) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let raw_map: BTreeMap<u32, Vec<u32>> = serde_json::from_str(bind_map_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid binding map JSON: {e}")))?;
+
+ let mut bindings = BTreeMap::new();
+ for (vcpu, cpus) in raw_map {
+ let mut set = CpuSet::new();
+ for cpu in cpus {
+ set.set(cpu);
+ }
+ bindings.insert(vcpu, set);
+ }
+ Ok(bindings)
+}
+
+pub fn bindings_to_json(bindings: &BTreeMap<u32, CpuSet>) -> Result<String, CpuError> {
+ let map: BTreeMap<u32, Vec<u32>> = bindings
+ .iter()
+ .map(|(&vcpu, cpuset)| (vcpu, cpuset.iter().collect()))
+ .collect();
+
+ serde_json::to_string(&map)
+ .map_err(|e| CpuError::BadSyntax(format!("failed to serialize bindings: {e}")))
+}
diff --git a/pve-qemu-server-cpu/src/cpuset.rs b/pve-qemu-server-cpu/src/cpuset.rs
new file mode 100644
index 0000000..d2ffcf8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/cpuset.rs
@@ -0,0 +1,125 @@
+//! Arbitrary-precision CPU bitmask for efficient set operations.
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CpuSet {
+ bits: Vec<u64>,
+}
+
+struct WordBits {
+ base: u32,
+ remaining: u64,
+}
+
+impl Iterator for WordBits {
+ type Item = u32;
+
+ fn next(&mut self) -> Option<u32> {
+ if self.remaining == 0 {
+ return None;
+ }
+ let bit = self.remaining.trailing_zeros();
+ // Clear the lowest set bit so the next call finds the following one.
+ self.remaining &= self.remaining - 1;
+ Some(self.base + bit)
+ }
+}
+
+impl CpuSet {
+ pub fn new() -> Self {
+ Self { bits: Vec::new() }
+ }
+
+ pub fn set(&mut self, cpu: u32) {
+ let idx = (cpu / 64) as usize;
+ let bit = cpu % 64;
+ if idx >= self.bits.len() {
+ self.bits.resize(idx + 1, 0);
+ }
+ self.bits[idx] |= 1u64 << bit;
+ }
+
+ pub fn intersect(&mut self, other: &CpuSet) {
+ for (i, word) in self.bits.iter_mut().enumerate() {
+ if i < other.bits.len() {
+ *word &= other.bits[i];
+ } else {
+ *word = 0;
+ }
+ }
+ }
+
+ pub fn union(&mut self, other: &CpuSet) {
+ if other.bits.len() > self.bits.len() {
+ self.bits.resize(other.bits.len(), 0);
+ }
+ for (i, word) in other.bits.iter().enumerate() {
+ self.bits[i] |= word;
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.bits.iter().all(|&w| w == 0)
+ }
+
+ pub fn size(&self) -> usize {
+ self.bits.iter().map(|w| w.count_ones() as usize).sum()
+ }
+
+ pub fn first_set(&self) -> Option<u32> {
+ self.iter().next()
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = u32> + '_ {
+ self.bits
+ .iter()
+ .enumerate()
+ .flat_map(|(idx, &word)| WordBits {
+ base: (idx as u32) * 64,
+ remaining: word,
+ })
+ }
+
+ /// Returns the set members as an ascending vector.
+ pub fn to_vec(&self) -> Vec<u32> {
+ self.iter().collect()
+ }
+
+ /// Converts the bitmask to a `libc::cpu_set_t` for `sched_setaffinity`.
+ pub fn to_cpu_set_t(&self) -> libc::cpu_set_t {
+ unsafe {
+ let mut set: libc::cpu_set_t = std::mem::zeroed();
+ for cpu in self.iter() {
+ libc::CPU_SET(cpu as usize, &mut set);
+ }
+ set
+ }
+ }
+
+ /// Parses sysfs CPU lists, for example `online`, `cpulist`, `core_cpus_list`.
+ ///
+ /// Format: comma-separated CPU IDs and inclusive ranges, such as `0-15,18-31`.
+ pub fn from_list(list: &str) -> Self {
+ let mut set = CpuSet::new();
+ let list = list.trim();
+ if list.is_empty() {
+ return set;
+ }
+
+ for part in list.split(',') {
+ let part = part.trim();
+ if part.is_empty() {
+ continue;
+ }
+ if let Some((start_raw, end_raw)) = part.split_once('-') {
+ let start: u32 = start_raw.trim().parse().unwrap_or(0);
+ let end: u32 = end_raw.trim().parse().unwrap_or(start);
+ for i in start..=end {
+ set.set(i);
+ }
+ } else if let Ok(cpu) = part.parse::<u32>() {
+ set.set(cpu);
+ }
+ }
+ set
+ }
+}
diff --git a/pve-qemu-server-cpu/src/error.rs b/pve-qemu-server-cpu/src/error.rs
new file mode 100644
index 0000000..2efbc12
--- /dev/null
+++ b/pve-qemu-server-cpu/src/error.rs
@@ -0,0 +1,32 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+#[error("cpu binding: {0}")]
+pub enum CpuError {
+ #[error("invalid syntax in: '{0}'")]
+ BadSyntax(String),
+
+ // Validation errors (pre-start)
+ #[error("host CPU {0} does not exist in this topology")]
+ UnknownHostCpu(u32),
+
+ #[error("vCPU {vcpu} is out of range (VM has vCPUs 0..{max})")]
+ VcpuOutOfRange { vcpu: u32, max: u32 },
+
+ // Runtime errors (post-start)
+ #[error("no running thread found for vCPU {0}")]
+ ThreadNotFound(u32),
+
+ #[error("invalid kernel TID {tid} for vCPU {vcpu}")]
+ BadThreadId { vcpu: u32, tid: u32 },
+
+ #[error("failed to init host topology: '{0}'")]
+ TopologyInit(String),
+
+ #[error("binding vCPU {vcpu} failed: {message}")]
+ Bind {
+ vcpu: u32,
+ /// Human-readable error from sched_setaffinity(2)
+ message: String,
+ },
+}
diff --git a/pve-qemu-server-cpu/src/lib.rs b/pve-qemu-server-cpu/src/lib.rs
new file mode 100644
index 0000000..cf19aa8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/lib.rs
@@ -0,0 +1,92 @@
+mod bind;
+mod cpuset;
+mod error;
+mod config;
+mod qmp;
+mod syscall;
+mod topology;
+
+/// Generate vcpu:cpu(s) binding map for a guest
+///
+/// # Arguments
+///
+/// * `mode` - Pinning mode.
+/// * `sockets` - number of sockets configured in <vmid.conf>
+/// * `cores` - number of cores configured in <vmid.conf>
+/// * `affinity` - Upper bound of host cpus that can be used for binding.
+/// * `cfg_json` - Additional config options. (e.g. numa config)
+/// * `cpuinfo_json` - JSON output from QMP `query-cpus-fast`.
+///
+/// # Modes
+///
+/// | Mode | Meaning |
+/// |---|---|
+/// | `none` | Do not pin vCPUs. |
+/// | `one-to-one` | Assign each vCPU to its own cpu. |
+/// | `numa` | Map guest NUMA nodes to host NUMA nodes |
+///
+/// # Examples
+///
+/// ```ignore
+/// // One-to-one mode:
+/// // Distribute vCPUs over host CPUs 0-7.
+/// gen_vcpu_bindings("one-to-one", 1, 8, "0-7", "", &cpuinfo_json)?;
+///
+/// // One-to-one mode over all allowed host CPUs.
+/// gen_vcpu_bindings("one-to-one", 1, 4 "", "", &cpuinfo_json)?;
+///
+/// // NUMA mode:
+/// // NUMA mode over all allowed host NUMA nodes.
+/// gen_vcpu_bindings("numa", 2, 4, "", &cfg_json, &cpuinfo_json)?;
+///
+/// // NUMA mode respecting the set affinity
+/// gen_vcpu_bindings("numa", 4, 2, "0-7", &cfg_json, &cpuinfo_json)?;
+/// ```
+///
+/// # Errors
+///
+/// Returns [`CpuError`] if the mode or binding syntax is invalid, if the
+/// requested host CPUs or NUMA nodes do not exist, or if binding fails.
+pub fn gen_vcpu_bindings(
+ mode: &str,
+ sockets: u32,
+ cores: u32,
+ affinity: &str,
+ cfg_json: &str,
+) -> Result<String, error::CpuError> {
+ let cpu_affinity = if affinity.is_empty() {
+ None
+ } else {
+ Some(cpuset::CpuSet::from_list(affinity))
+ };
+
+ let bindings: std::collections::BTreeMap<u32, cpuset::CpuSet> = match mode.parse::<bind::Mode>()? {
+ bind::Mode::None => return Ok("{}".to_string()),
+ bind::Mode::OneToOne => {
+ // only_real_cores is hardcoded to false for now, until we device if we actually want this setting
+ bind::gen_one_to_one_bindings(sockets, cores, &cpu_affinity, false)?
+ }
+ bind::Mode::Numa => {
+ let numa_cfg = config::numa_config_from_json(cfg_json)?;
+ bind::gen_numa_bindings(&numa_cfg, &cpu_affinity)?
+ }
+ };
+
+ // This shouldn't happen, but we should still avoid passing empty config to perl
+ if bindings.is_empty() {
+ return Err(error::CpuError::BadSyntax(
+ "cpu-binding-mode '{mode}' is enabled but no vCPU bindings were generated".to_string(),
+ ));
+ }
+
+ config::bindings_to_json(&bindings)
+}
+
+/// Apply vcpu:cpu(s) the binding map
+/// The function parses the QMP `query-cpus-fast` response and binds each
+/// vCPU thread according to the selected mode.
+pub fn bind_vcpus(bind_map_json: &str, qmp_cpuinfo_json: &str) -> Result<(), error::CpuError> {
+ let bindings = config::bindings_from_json(bind_map_json)?;
+ let qmp_cpuinfo = qmp::parse_cpuinfo(qmp_cpuinfo_json)?;
+ bind::apply_bindings(&bindings, &qmp_cpuinfo)
+}
diff --git a/pve-qemu-server-cpu/src/qmp.rs b/pve-qemu-server-cpu/src/qmp.rs
new file mode 100644
index 0000000..78fb8bd
--- /dev/null
+++ b/pve-qemu-server-cpu/src/qmp.rs
@@ -0,0 +1,91 @@
+//! Parsing of the QEMU `query-cpus-fast` QMP response.
+//!
+//! The response is a JSON array; each element describes one guest vCPU
+//! and carries both its guest-visible topology (socket/core/NUMA node)
+//! and its host kernel TID. Every binding mode consumes this shape, so
+//! the parsing lives here rather than inside any individual mode.
+//!
+//! The full QEMU schema is modeled, including fields not currently used
+//! by any binding mode. Serde ignores fields it doesn't know about, but
+//! keeping them explicit documents the wire format and makes future
+//! modes cheaper to implement.
+
+use serde::Deserialize;
+
+use crate::error::CpuError;
+
+/// One entry of the `query-cpus-fast` array.
+#[derive(Debug, Clone, Deserialize)]
+pub struct QmpCpu {
+ /// Guest-visible vCPU index (0 .. sockets*cores).
+ #[serde(rename = "cpu-index")]
+ pub cpu_index: u32,
+
+ /// Host kernel TID of the QEMU vCPU thread.
+ #[serde(rename = "thread-id")]
+ pub thread_id: u32,
+
+ /// QOM path of the CPU object, e.g. `/machine/unattached/device[0]`.
+ #[expect(dead_code)]
+ #[serde(rename = "qom-path")]
+ pub qom_path: String,
+
+ /// QOM type of the CPU object, e.g. `host-x86_64-cpu`.
+ #[expect(dead_code)]
+ #[serde(rename = "qom-type")]
+ pub qom_type: String,
+
+ /// Target architecture, e.g. `x86_64`.
+ #[expect(dead_code)]
+ pub target: String,
+
+ /// Guest-visible topology properties of the vCPU.
+ #[expect(dead_code)]
+ pub props: QmpCpuProps,
+}
+
+/// Guest-visible topology properties of a vCPU.
+#[derive(Debug, Clone, Deserialize)]
+pub struct QmpCpuProps {
+ /// Guest socket the vCPU belongs to.
+ #[expect(dead_code)]
+ #[serde(rename = "socket-id")]
+ pub socket_id: u32,
+
+ /// Guest core within that socket.
+ #[expect(dead_code)]
+ #[serde(rename = "core-id")]
+ pub core_id: u32,
+
+ /// Guest NUMA node the vCPU belongs to.
+ ///
+ /// QEMU reports this even when `numa` is disabled in the VM config,
+ /// in which case every vCPU is on node 0.
+ #[expect(dead_code)]
+ #[serde(rename = "node-id")]
+ pub node_id: u32,
+
+ /// Guest thread within the core.
+ ///
+ /// Always 0 today: QEMU's CPU object model carries this field for
+ /// SMT support in the guest, but PVE does not expose SMT to guests,
+ /// so it is always zero in practice. Modeled for completeness.
+ /// TODO: PVE should expose SMT to the guests
+ #[expect(dead_code)]
+ #[serde(rename = "thread-id")]
+ pub thread_id: u32,
+}
+
+// returned vec is sorted by vCPU id (0,1,2,3...)
+// Guarantees vec[id] == vec[id].cpu_index
+pub fn parse_cpuinfo(cpuinfo_json: &str) -> Result<Vec<QmpCpu>, CpuError> {
+ let mut cpus: Vec<QmpCpu> = serde_json::from_str(cpuinfo_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid query-cpus-fast JSON: {e}")))?;
+
+ // QEMU assigns cpu-index sequentially from 0. Sort by it so
+ // `vec[i]` indexes vCPU i,
+ // value should be 0..N-1 so that assumption holds.
+ cpus.sort_unstable_by_key(|c| c.cpu_index);
+
+ Ok(cpus)
+}
diff --git a/pve-qemu-server-cpu/src/syscall.rs b/pve-qemu-server-cpu/src/syscall.rs
new file mode 100644
index 0000000..59b74a9
--- /dev/null
+++ b/pve-qemu-server-cpu/src/syscall.rs
@@ -0,0 +1,24 @@
+//! Safe wrappers around Linux syscalls.
+
+#[cfg(target_os = "linux")]
+use std::io;
+
+use crate::cpuset::CpuSet;
+
+pub fn sched_setaffinity(tid: libc::pid_t, cpus: &CpuSet) -> io::Result<()> {
+ let set = cpus.to_cpu_set_t();
+
+ unsafe {
+ let rc = libc::sched_setaffinity(
+ tid,
+ std::mem::size_of::<libc::cpu_set_t>(),
+ &set,
+ );
+
+ if rc != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ }
+
+ Ok(())
+}
diff --git a/pve-qemu-server-cpu/src/topology/mod.rs b/pve-qemu-server-cpu/src/topology/mod.rs
new file mode 100644
index 0000000..3d7ffaa
--- /dev/null
+++ b/pve-qemu-server-cpu/src/topology/mod.rs
@@ -0,0 +1,222 @@
+//! Hierarchical hardware topology discovered from sysfs.
+//!
+//! The topology is a tree rooted at the machine. Every node owns its
+//! children and there are no parent pointers, so traversal is top-down.
+//!
+//! On modern systems (like AMD EPYC), the hierarchy typically looks like:
+//!
+//! ```text
+//! Machine -> Package -> Die -> NumaNode -> Core -> Pu
+//! ```
+
+mod sysfs;
+
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+
+use crate::cpuset::CpuSet;
+use crate::error::CpuError;
+
+const CPU_DIR: &str = "/sys/devices/system/cpu";
+const NODE_DIR: &str = "/sys/devices/system/node";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ObjectKind {
+ Machine,
+ Package,
+ Die,
+ NumaNode,
+ // Cache, // Future extension
+ Core,
+ Pu,
+}
+
+/// A node in the hardware topology tree.
+///
+/// Each node owns its children. The `cpuset` covers every CPU at or below
+/// this node. `os_index` is the operating-system index (hwloc's `P#`).
+#[derive(Debug, Clone)]
+pub struct TopoObject {
+ pub kind: ObjectKind,
+ pub os_index: u32,
+ pub cpuset: CpuSet,
+ pub children: Vec<TopoObject>,
+}
+
+impl TopoObject {
+ fn new(kind: ObjectKind, os_index: u32) -> Self {
+ Self {
+ kind,
+ os_index,
+ cpuset: CpuSet::new(),
+ children: Vec::new(),
+ }
+ }
+
+ /// Finds an existing child by kind and index, or creates and appends a new one.
+ fn get_or_add_child(&mut self, kind: ObjectKind, os_index: u32) -> &mut TopoObject {
+ let pos = self
+ .children
+ .iter()
+ .position(|c| c.kind == kind && c.os_index == os_index)
+ .unwrap_or_else(|| {
+ self.children.push(TopoObject::new(kind, os_index));
+ self.children.len() - 1
+ });
+
+ &mut self.children[pos]
+ }
+
+ /// Finds the first node matching `kind` and `os_index` in this subtree.
+ pub fn find(&self, kind: ObjectKind, os_index: u32) -> Option<&TopoObject> {
+ if self.kind == kind && self.os_index == os_index {
+ return Some(self);
+ }
+ for child in &self.children {
+ if let Some(found) = child.find(kind, os_index) {
+ return Some(found);
+ }
+ }
+ None
+ }
+
+ /// Calls `f` for every node matching `kind` in this subtree.
+ pub fn for_each(&self, kind: ObjectKind, f: &mut dyn FnMut(&TopoObject)) {
+ if self.kind == kind {
+ f(self);
+ }
+ for child in &self.children {
+ child.for_each(kind, &mut *f);
+ }
+ }
+
+ /// Computes this node's cpuset bottom-up from its children.
+ fn compute_cpuset(&mut self) {
+ self.cpuset = CpuSet::new();
+ if self.kind == ObjectKind::Pu {
+ self.cpuset.set(self.os_index);
+ } else {
+ for child in &mut self.children {
+ child.compute_cpuset();
+ self.cpuset.union(&child.cpuset);
+ }
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct Topology {
+ online_cpus: CpuSet,
+ root: TopoObject,
+}
+
+impl Topology {
+ pub fn new() -> Result<Self, CpuError> {
+ let online_cpus = sysfs::read_file(&Path::new(CPU_DIR).join("online"))
+ .map(|s| CpuSet::from_list(&s))
+ .ok_or_else(|| {
+ CpuError::TopologyInit("cannot read the online CPU mask".to_string())
+ })?;
+
+ let cpu_to_numa = Self::read_cpu_to_numa();
+
+ let mut root = TopoObject::new(ObjectKind::Machine, 0);
+ Self::build_tree(&mut root, &online_cpus, &cpu_to_numa)?;
+
+ root.compute_cpuset();
+
+ Ok(Topology { online_cpus, root })
+ }
+
+ fn build_tree(
+ root: &mut TopoObject,
+ online_cpus: &CpuSet,
+ cpu_to_numa: &HashMap<u32, u32>,
+ ) -> Result<(), CpuError> {
+ let cpu_dir = Path::new(CPU_DIR);
+
+ for cpu_id in online_cpus.iter() {
+ let topo_dir = cpu_dir.join(format!("cpu{cpu_id}")).join("topology");
+
+ let pkg_id = sysfs::read_file(&topo_dir.join("physical_package_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+
+ let die_id = sysfs::read_file(&topo_dir.join("die_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+
+ let core_id = sysfs::read_file(&topo_dir.join("core_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(cpu_id);
+
+ let pkg = root.get_or_add_child(ObjectKind::Package, pkg_id);
+ let die = pkg.get_or_add_child(ObjectKind::Die, die_id);
+
+ // NumaNode is optional. If present, it sits between Die and Core.
+ let parent_of_core = if let Some(numa_id) = cpu_to_numa.get(&cpu_id).copied() {
+ die.get_or_add_child(ObjectKind::NumaNode, numa_id)
+ } else {
+ &mut *die
+ };
+
+ let core = parent_of_core.get_or_add_child(ObjectKind::Core, core_id);
+ core.children.push(TopoObject::new(ObjectKind::Pu, cpu_id));
+ }
+
+ Ok(())
+ }
+
+ pub fn online_cpus(&self) -> &CpuSet {
+ &self.online_cpus
+ }
+
+ pub fn numa_cpus(&self, os_index: u32) -> Option<&CpuSet> {
+ self.root
+ .find(ObjectKind::NumaNode, os_index)
+ .map(|node| &node.cpuset)
+ }
+
+ /// Reduces `cpus` to at most one thread per physical core.
+ /// (Avoids SMT siblings)
+ pub fn singlify_per_core(&self, cpus: &CpuSet) -> CpuSet {
+ let mut result = CpuSet::new();
+ self.root.for_each(ObjectKind::Core, &mut |core| {
+ let mut core_cpus = core.cpuset.clone();
+ core_cpus.intersect(cpus);
+ if let Some(first) = core_cpus.first_set() {
+ result.set(first);
+ }
+ });
+ result
+ }
+
+ fn read_cpu_to_numa() -> HashMap<u32, u32> {
+ let mut map = HashMap::new();
+ let node_dir = Path::new(NODE_DIR);
+
+ let entries = match fs::read_dir(node_dir) {
+ Ok(entries) => entries,
+ Err(_) => return map,
+ };
+
+ for entry in entries.flatten() {
+ let name = entry.file_name();
+ let name_str = name.to_string_lossy();
+ let node_id = match name_str.strip_prefix("node").and_then(|s| s.parse::<u32>().ok()) {
+ Some(id) => id,
+ None => continue,
+ };
+
+ let cpuset = sysfs::read_file(&entry.path().join("cpulist"))
+ .map(|s| CpuSet::from_list(&s))
+ .unwrap_or_else(CpuSet::new);
+
+ for cpu in cpuset.iter() {
+ map.insert(cpu, node_id);
+ }
+ }
+ map
+ }
+}
diff --git a/pve-qemu-server-cpu/src/topology/sysfs.rs b/pve-qemu-server-cpu/src/topology/sysfs.rs
new file mode 100644
index 0000000..a1ff67a
--- /dev/null
+++ b/pve-qemu-server-cpu/src/topology/sysfs.rs
@@ -0,0 +1,6 @@
+use std::fs;
+use std::path::Path;
+
+pub fn read_file(path: &Path) -> Option<String> {
+ fs::read_to_string(path).ok().map(|s| s.trim().to_string())
+}
--
2.47.3
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH qemu-server v4 4/4] cpu: add cpu binding
2026-11-11 15:20 [RFC proxmox-perl-rs/qemu-server/qemu-server-rs v4 0/4] vCPU binding for qemu guests Elias Huhsovitz
` (2 preceding siblings ...)
2026-11-11 15:20 ` [PATCH qemu-server v4 3/4] helpers: return integer values for parse_number_sets Elias Huhsovitz
@ 2026-11-11 15:20 ` Elias Huhsovitz
3 siblings, 0 replies; 5+ messages in thread
From: Elias Huhsovitz @ 2026-11-11 15:20 UTC (permalink / raw)
To: pve-devel; +Cc: Elias Huhsovitz
Add a new qm.conf option `cpu-binding-mode`: none | one-to-one | numa
When cpu-binding-mode is unset or "none", behavior is unchanged.
Otherwise, vCPU threads are pinned to host CPUs after QEMU has spawned
them.
Two functional modes are supported: (1) one-to-one - each vCPU gets its
own cpu, defined by the affinity-pool. (2) numa - per-vNUMA-node
set-to-set binding, defined by numa [N] config.
Two entry points are provided in PVE::QemuServer::CPUBind. (1)
gen_vcpu_bindings() - called before VM QEMU spawns. Generates a binding
map, specifying vCPU:host cpu(s). (2) bind_vcpus() - called after QEMU
started. Binds the vCPUs according to the generated binding map.
information as JSON and hands it to Rust together with the mode and cfg
string.
Originally-by: Dominik Csapak <d.csapak@proxmox.com>
Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
src/PVE/QemuServer.pm | 32 ++++++++++-
src/PVE/QemuServer/CPUBind.pm | 103 ++++++++++++++++++++++++++++++++++
src/PVE/QemuServer/Makefile | 1 +
src/PVE/QemuServer/Memory.pm | 52 ++++++++++++++++-
4 files changed, 186 insertions(+), 2 deletions(-)
create mode 100644 src/PVE/QemuServer/CPUBind.pm
diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 8149184e..a8269e36 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -59,6 +59,7 @@ use PVE::QemuServer::Helpers
qw(config_aware_timeout get_iscsi_initiator_name get_host_arch min_version kvm_user_version windows_version);
use PVE::QemuServer::Cloudinit;
use PVE::QemuServer::CGroup;
+use PVE::QemuServer::CPUBind;
use PVE::QemuServer::CPUConfig qw(
print_cpu_device
get_cpu_options
@@ -741,6 +742,12 @@ EODESCR
optional => 1,
default => 1,
},
+ 'cpu-binding-mode' => {
+ type => 'string',
+ format => $PVE::QemuServer::Bind::bind_mode_fmt,
+ description => "Set CPU binding mode for the guest",
+ optional => 1,
+ },
};
my $cicustom_fmt = {
@@ -3189,7 +3196,7 @@ sub config_to_command {
($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
my $cmd = [];
- if ($conf->{affinity}) {
+ if ($conf->{affinity} && !$conf->{'cpu-binding-mode'}) {
push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
}
@@ -5514,6 +5521,8 @@ sub vm_start_nolock {
my ($cmd, $spice_port, $start_timeout);
my $pci_reserve_list = [];
+ my $vcpu_binding_map;
+ my $freeze_for_binding = 0;
eval {
# With -blockdev, it is necessary to activate the volumes before generating the command line
# Plugins can safely deactivate already-active volumes here if needed
@@ -5539,10 +5548,21 @@ sub vm_start_nolock {
},
);
+ if ($conf->{'cpu-binding-mode'}) {
+ $vcpu_binding_map = PVE::QemuServer::CPUBind::conf_to_vcpu_bindings($conf);
+ }
+
+ $freeze_for_binding =
+ $vcpu_binding_map
+ && !$params->{paused}
+ && !$statefile
+ && !$conf->{freeze};
+
my $memory = get_current_memory($conf->{memory});
$start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
push $cmd->@*, $state_cmdline->@*;
+ push @$cmd, '-S' if $freeze_for_binding;
($pci_reserve_list, my $uuid) =
PVE::QemuServer::PCI::prepare_pci_devices($conf, $vmid, $pci_devices, $start_timeout);
@@ -5759,6 +5779,16 @@ sub vm_start_nolock {
my $qemu_pkg_string = PVE::QemuServer::QMPHelpers::get_qemu_package_string($vmid);
syslog("info", "VM $vmid started with PID $pid using $qemu_pkg_string.");
+ if ($vcpu_binding_map) {
+ eval { PVE::QemuServer::CPUBind::bind_vcpus($vmid, $vcpu_binding_map) };
+ log_warn("unable to bind vCPU threads - $@") if $@;
+ }
+
+ if ($freeze_for_binding) {
+ eval { mon_cmd($vmid, "cont") };
+ log_warn("failed to resume VM after CPU binding: $@") if $@;
+ }
+
PVE::QemuServer::RunState::create_cleanup_flag($vmid);
if (defined(my $migrate = $res->{migrate})) {
diff --git a/src/PVE/QemuServer/CPUBind.pm b/src/PVE/QemuServer/CPUBind.pm
new file mode 100644
index 00000000..0566b08b
--- /dev/null
+++ b/src/PVE/QemuServer/CPUBind.pm
@@ -0,0 +1,103 @@
+package PVE::QemuServer::CPUBind;
+
+use v5.36;
+
+use JSON;
+
+use PVE::QemuServer::Memory;
+use PVE::QemuServer::Monitor;
+use PVE::RS::CPU::Bind;
+
+our $bind_mode_fmt = {
+ mode => {
+ type => 'string',
+ enum => [qw(none one-to-one numa)],
+ description => "Set the CPU binding mode for the guest.",
+ verbose_description => <<'EODESCR',
+Determines how guest vCPUs are bound/pinned to host CPUs.
+
+* none (default): no binding. Regular scheduling.
+
+* one-to-one: pin each vCPU to its own physical core, chosen from the
+ pool configured by affinity. The configured affinity string acts as
+ an upper bound on the host cpus, that can be used for binding.
+
+ If the guest has N vCPUS and the affinity-pool has M host cpus, each vCPU
+ is fixed to a single one of the M host cpus. (M >= N, must be true).
+
+ Each vCPU is fixed to a single host cpu, it does not float within
+ the pool.
+
+ Example 1: 4-vCPU guest with affinity=4-9:
+ The resulting mapping would be: vCPU0:CPU4, vCPU1:CPU5, vCPU2:CPU6,
+ vCPU3:CPU7.
+
+ Example 2: 4-vCPU guest with affinity=0,2:
+ The resulting mapping would be: vCPU0:CPU0, vCPU1:CPU2
+
+ An empty affinity string means all host cpus will be considered.
+ On a host with 16 cores, that pins an N-vCPU VM to host cpus
+ 0, 1, ..., N-1.
+
+* numa: bind the set of vCPUs of each vNUMA node to the set of
+ CPUs of a host NUMA node. The cpu bindings are determined by the
+ numa[N] configuration of the guest. If no numa[N] config is
+ explicitly provided, then for each socket a numa[N] node is created.
+
+ Each guest vNUMA node's vCPUs form a group that may run on any CPU
+ of the assigned host NUMA node. The host scheduler is free to migrate
+ the vCPUs within the host node. This is the intended trade-off:
+ memory allocated by a vCPU stays local to the host node it runs on
+ (preserving NUMA memory locality), while the scheduler still has room
+ to balance load across the node's cores and hyperthreads.
+
+ Examples:
+ numa0: cpus=0-1,hostnodes=0,memory=2048,policy=bind
+ Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0
+
+ numa1: cpus=6-7,hostnodes=3,memory=2048,policy=bind
+ Allows vCPUs 6-7 to execute on all cpus of host NUMA node 3
+
+ numa:1, no numa[N], sockets: 2, cores: 2, total vCPUs: 4
+ Allows vCPUs 0-1 to execute on all cpus of host NUMA node 0
+ Allows vCPUs 2-3 to execute on all cpus of host NUMA node 1
+
+EODESCR
+ default => 'none',
+ default_key => 1,
+ },
+};
+
+my sub gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg) {
+ my $cfg_json = ''; # currently only relevant for numa mode.
+ if ($mode eq 'numa') {
+ $cfg_json = JSON::encode_json($cfg);
+ }
+
+ # PVE::RS::CPU::Bind::gen_vcpu_bindings returns the map as JSON. This is currently
+ # not decoded, because at the moment we are simply passing the JSON bind map to
+ # bind_vcpus. Should this change in the future, decode the JSON string before returning.
+ return PVE::RS::CPU::Bind::gen_vcpu_bindings($mode, $sockets, $cores, $affinity, $cfg_json);
+}
+
+sub conf_to_vcpu_bindings($conf) {
+ my $mode = $conf->{'cpu-binding-mode'};
+ return if $mode eq 'none';
+
+ my $cfg = '';
+ $cfg = PVE::QemuServer::Memory::numa_conf_to_map($conf) if $mode eq 'numa';
+
+ my $vcpu_binding_map = gen_vcpu_bindings(
+ $mode, $conf->{sockets}, $conf->{cores}, $conf->{affinity} // '', $cfg,
+ );
+
+ return $vcpu_binding_map;
+}
+
+sub bind_vcpus($vmid, $bind_map_json) {
+ my $cpuinfo = PVE::QemuServer::Monitor::mon_cmd($vmid, 'query-cpus-fast');
+ my $qmp_cpuinfo_json = JSON::encode_json($cpuinfo);
+ PVE::RS::CPU::Bind::bind_vcpus($bind_map_json, $qmp_cpuinfo_json);
+}
+
+1;
diff --git a/src/PVE/QemuServer/Makefile b/src/PVE/QemuServer/Makefile
index 61ed1a4e..c4cd9077 100644
--- a/src/PVE/QemuServer/Makefile
+++ b/src/PVE/QemuServer/Makefile
@@ -8,6 +8,7 @@ SOURCES=Agent.pm \
Cfg2Cmd.pm \
CGroup.pm \
Cloudinit.pm \
+ CPUBind.pm \
CPUConfig.pm \
CPUFlags.pm \
DBusVMState.pm \
diff --git a/src/PVE/QemuServer/Memory.pm b/src/PVE/QemuServer/Memory.pm
index 8cbbac84..282d5e23 100644
--- a/src/PVE/QemuServer/Memory.pm
+++ b/src/PVE/QemuServer/Memory.pm
@@ -87,6 +87,57 @@ our $memory_fmt = {
},
};
+sub numa_conf_to_map {
+ my ($conf) = @_;
+
+ my $numa_cfg = {};
+ my $has_custom_topology = 0;
+
+ for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) {
+ my $entry = $conf->{"numa$i"} or next;
+ $has_custom_topology = 1;
+
+ my $numa = PVE::QemuServer::Memory::parse_numa($entry);
+
+ $numa_cfg->{"numa$i"} = {
+ vcpus => $numa->{cpus} // [],
+ hostnodes => $numa->{hostnodes} // [],
+ memory => 0 + ($numa->{memory} // 0),
+ policy => $numa->{policy},
+ };
+ }
+
+ return $numa_cfg if $has_custom_topology;
+
+ die "NUMA must be enabled in the VM configuration\n"
+ if !$conf->{numa};
+
+ my $sockets = $conf->{sockets} || 1;
+ my $cores = $conf->{cores} || 1;
+ my $memory = get_current_memory($conf->{memory});
+ my $numa_memory = int($memory / $sockets);
+
+ for (my $i = 0; $i < $sockets; $i++) {
+ my $start_cpu = $cores * $i;
+ my $end_cpu = $start_cpu + $cores - 1;
+
+ # Format matches parse_number_sets output: [[start, end]] or [[start, undef]]
+ my $cpus = $cores > 1 ? [[$start_cpu, $end_cpu]] : [[$start_cpu, undef]];
+
+ # Fallback maps guest node $i directly to host node $i
+ my $hostnodes = [[$i, undef]];
+
+ $numa_cfg->{"numa$i"} = {
+ vcpus => $cpus,
+ hostnodes => $hostnodes,
+ memory => 0 + $numa_memory,
+ policy => undef,
+ };
+ }
+
+ return $numa_cfg;
+}
+
sub print_memory {
my $memory = shift;
@@ -825,4 +876,3 @@ sub hugepages_update_locked {
return $res;
}
1;
-
--
2.47.3
^ permalink raw reply related [flat|nested] 5+ messages in thread