From: Elias Huhsovitz <e.huhsovitz@proxmox.com>
To: pve-devel@lists.proxmox.com
Cc: Elias Huhsovitz <e.huhsovitz@proxmox.com>
Subject: [PATCH qemu-server-rs v4 1/4] cpu: add pve-qemu-server-cpu crate for vCPU binding
Date: Wed, 11 Nov 2026 16:20:24 +0100 [thread overview]
Message-ID: <20261111152027.260117-2-e.huhsovitz@proxmox.com> (raw)
In-Reply-To: <20261111152027.260117-1-e.huhsovitz@proxmox.com>
Add a new library crate that implements vCPU to host CPU binding of qemu
guests.
2 main functions are exposed: gen_vcpu_bindings: (1) Reads the CPU
Toplogy from sysfs, validates the configured cpu binding config and
Generates a vCPU:hostCPU binding map. This function should be called
before the guest starts. (2) bind_vcpus: Applies the binding map
generated by gen_vcpu_bindings to the specified vm. The cpu binding is
applied using sched_setaffinity(2).
2 modes are implemented: (1) one-to-one: pin each vCPU to its own host
cpu. (2) numa: bind the set of vCPUs of each vNUMA node to the set of
CPUs of a host NUMA node. The cpu bindings are determined by the numa[N]
configuration of the guest.
Both settings respect the configured affinity.
Thread binding uses sched_setaffinity(2), which accepts a kernel
TID directly. The TID used is reported by qemu via query-cpus-fast.
Originally-by: Dominik Csapak <d.csapak@proxmox.com>
Signed-off-by: Elias Huhsovitz <e.huhsovitz@proxmox.com>
---
Cargo.toml | 2 +-
build.sh | 2 +-
pve-qemu-server-cpu/Cargo.toml | 16 ++
pve-qemu-server-cpu/debian/changelog | 5 +
pve-qemu-server-cpu/debian/control | 41 ++++
pve-qemu-server-cpu/debian/copyright | 18 ++
pve-qemu-server-cpu/debian/debcargo.toml | 9 +
pve-qemu-server-cpu/src/bind.rs | 136 +++++++++++++
pve-qemu-server-cpu/src/config.rs | 146 ++++++++++++++
pve-qemu-server-cpu/src/cpuset.rs | 125 ++++++++++++
pve-qemu-server-cpu/src/error.rs | 32 ++++
pve-qemu-server-cpu/src/lib.rs | 92 +++++++++
pve-qemu-server-cpu/src/qmp.rs | 91 +++++++++
pve-qemu-server-cpu/src/syscall.rs | 24 +++
pve-qemu-server-cpu/src/topology/mod.rs | 222 ++++++++++++++++++++++
pve-qemu-server-cpu/src/topology/sysfs.rs | 6 +
16 files changed, 965 insertions(+), 2 deletions(-)
create mode 100644 pve-qemu-server-cpu/Cargo.toml
create mode 100644 pve-qemu-server-cpu/debian/changelog
create mode 100644 pve-qemu-server-cpu/debian/control
create mode 100644 pve-qemu-server-cpu/debian/copyright
create mode 100644 pve-qemu-server-cpu/debian/debcargo.toml
create mode 100644 pve-qemu-server-cpu/src/bind.rs
create mode 100644 pve-qemu-server-cpu/src/config.rs
create mode 100644 pve-qemu-server-cpu/src/cpuset.rs
create mode 100644 pve-qemu-server-cpu/src/error.rs
create mode 100644 pve-qemu-server-cpu/src/lib.rs
create mode 100644 pve-qemu-server-cpu/src/qmp.rs
create mode 100644 pve-qemu-server-cpu/src/syscall.rs
create mode 100644 pve-qemu-server-cpu/src/topology/mod.rs
create mode 100644 pve-qemu-server-cpu/src/topology/sysfs.rs
diff --git a/Cargo.toml b/Cargo.toml
index 5eefc79..80f87ff 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,6 @@
[workspace]
members = [
+ "pve-qemu-server-cpu",
"pve-qemu-server-pci"
]
exclude = [
@@ -25,4 +26,3 @@ strum = { version = "0.26", features = ["derive"] }
# proxmox dependencies
pve-api-types = "8.1"
-
diff --git a/build.sh b/build.sh
index 3c2293b..c3ee7b6 100755
--- a/build.sh
+++ b/build.sh
@@ -28,7 +28,7 @@ debcargo package \
--no-overlay-write-back \
--directory "$PWD/"${BUILDDIR}"/${CRATE}" \
"${CRATE}" \
- "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//')"
+ "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//' -e 's/~/-/')"
cd "${BUILDDIR}/${CRATE}"
rm -f debian/source/format.debcargo.hint
diff --git a/pve-qemu-server-cpu/Cargo.toml b/pve-qemu-server-cpu/Cargo.toml
new file mode 100644
index 0000000..b236796
--- /dev/null
+++ b/pve-qemu-server-cpu/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "pve-qemu-server-cpu"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+exclude.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+thiserror = "2"
+libc = "0.2"
diff --git a/pve-qemu-server-cpu/debian/changelog b/pve-qemu-server-cpu/debian/changelog
new file mode 100644
index 0000000..3731cfd
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/changelog
@@ -0,0 +1,5 @@
+rust-pve-qemu-server-cpu (0.1.0) trixie; urgency=medium
+
+ * initial release
+
+ -- Proxmox Support Team <support@proxmox.com> Tue, 01 Sep 2026 14:02:25 +0200
diff --git a/pve-qemu-server-cpu/debian/control b/pve-qemu-server-cpu/debian/control
new file mode 100644
index 0000000..1298d61
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/control
@@ -0,0 +1,41 @@
+Source: rust-pve-qemu-server-cpu
+Section: rust
+Priority: optional
+Build-Depends: debhelper-compat (= 13),
+ dh-sequence-cargo,
+ libhwloc-dev (>= 2.12)
+Build-Depends-Arch: cargo:native <!nocheck>,
+ rustc:native (>= 1.94) <!nocheck>,
+ libstd-rust-dev <!nocheck>,
+ librust-libc-0.2+default-dev <!nocheck>,
+ librust-serde-1+default-dev <!nocheck>,
+ librust-serde-1+derive-dev <!nocheck>,
+ librust-serde-json-1+default-dev <!nocheck>,
+ librust-thiserror-2+default-dev <!nocheck>
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Standards-Version: 4.7.2
+Vcs-Git: git://git.proxmox.com/git/pve-qemu-server-rs.git
+Vcs-Browser: https://git.proxmox.com/?p=pve-qemu-server-rs.git
+Homepage: https://proxmox.com
+X-Cargo-Crate: pve-qemu-server-cpu
+
+Package: librust-pve-qemu-server-cpu-dev
+Architecture: any
+Multi-Arch: same
+Depends:
+ ${misc:Depends},
+ librust-libc-0.2+default-dev,
+ librust-serde-1+default-dev,
+ librust-serde-1+derive-dev,
+ librust-serde-json-1+default-dev,
+ librust-thiserror-2+default-dev
+Provides:
+ librust-pve-qemu-server-cpu+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1+default-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1.0-dev (= ${binary:Version}),
+ librust-pve-qemu-server-cpu-0.1.0+default-dev (= ${binary:Version})
+Description: Rust crate "pve-qemu-server-cpu" - Rust source code
+ Source code for Debianized Rust crate "pve-qemu-server-cpu"
diff --git a/pve-qemu-server-cpu/debian/copyright b/pve-qemu-server-cpu/debian/copyright
new file mode 100644
index 0000000..01138fa
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/copyright
@@ -0,0 +1,18 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+
+Files:
+ *
+Copyright: 2026 Proxmox Server Solutions GmbH <support@proxmox.com>
+License: AGPL-3.0-or-later
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU Affero General Public License as published by the Free
+ Software Foundation, either version 3 of the License, or (at your option) any
+ later version.
+ .
+ This program is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ details.
+ .
+ You should have received a copy of the GNU Affero General Public License along
+ with this program. If not, see <https://www.gnu.org/licenses/>.
diff --git a/pve-qemu-server-cpu/debian/debcargo.toml b/pve-qemu-server-cpu/debian/debcargo.toml
new file mode 100644
index 0000000..23feea4
--- /dev/null
+++ b/pve-qemu-server-cpu/debian/debcargo.toml
@@ -0,0 +1,9 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+allow_prerelease_deps = true
+
+[source]
+vcs_git = "git://git.proxmox.com/git/pve-qemu-server-rs.git"
+vcs_browser = "https://git.proxmox.com/?p=pve-qemu-server-rs.git"
+build_depends = ["libhwloc-dev (>= 2.12)"]
diff --git a/pve-qemu-server-cpu/src/bind.rs b/pve-qemu-server-cpu/src/bind.rs
new file mode 100644
index 0000000..d7f52f8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/bind.rs
@@ -0,0 +1,136 @@
+use std::collections::BTreeMap;
+
+use crate::cpuset::CpuSet;
+use crate::config::NumaConfig;
+use crate::qmp::QmpCpu;
+use crate::syscall::sched_setaffinity;
+use crate::topology::Topology;
+use crate::error::CpuError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Mode {
+ None,
+ OneToOne,
+ Numa,
+}
+
+impl std::str::FromStr for Mode {
+ type Err = CpuError;
+
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
+ match s {
+ "none" => Ok(Mode::None),
+ "one-to-one" => Ok(Mode::OneToOne),
+ "numa" => Ok(Mode::Numa),
+ other => Err(CpuError::BadSyntax(format!(
+ "unknown cpu-bind-mode '{other}'"
+ ))),
+ }
+ }
+}
+
+pub fn gen_one_to_one_bindings(
+ sockets: u32,
+ cores: u32,
+ affinity: &Option<CpuSet>,
+ only_real_cores: bool,
+) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let topo = Topology::new()?;
+ let mut allowed = topo.online_cpus().clone();
+
+ if let Some(aff) = affinity {
+ allowed.intersect(aff);
+ }
+
+ let nr_vcpus = sockets * cores;
+
+ if only_real_cores {
+ allowed = topo.singlify_per_core(&allowed);
+ }
+
+ if allowed.size() < nr_vcpus as usize {
+ return Err(CpuError::BadSyntax(format!(
+ "one-to-one mode requires {nr_vcpus} host cpus but only {} are available",
+ allowed.size()
+ )));
+ }
+
+ let available: Vec<u32> = allowed.to_vec();
+
+ let mut bindings: BTreeMap<u32, CpuSet> = BTreeMap::new();
+
+ // qmp::parse_cpuinfo should yield vCPU IDs that are 0..N-1.
+ // If not: we proably messed up somewhere else
+ for (vcpu_id, cpu) in available.iter().take(nr_vcpus as usize).enumerate() {
+ let mut single_cpu = CpuSet::new();
+ single_cpu.set(*cpu);
+ bindings.insert(vcpu_id as u32, single_cpu);
+ }
+
+ Ok(bindings)
+}
+
+pub fn gen_numa_bindings(
+ numa_cfg: &Vec<NumaConfig>,
+ affinity: &Option<CpuSet>,
+) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let topo = Topology::new()?;
+ let mut bindings = BTreeMap::new();
+
+ for (vnode_id, cfg) in numa_cfg.iter().enumerate() {
+ if cfg.hostnodes.len() != 1 {
+ return Err(CpuError::BadSyntax(format!(
+ "vNUMA node {vnode_id} must be bound to exactly one host NUMA node"
+ )));
+ }
+
+ let host_node_id = *cfg.hostnodes.first().unwrap();
+ let host_cpus = topo.numa_cpus(host_node_id).ok_or_else(|| {
+ CpuError::BadSyntax(format!(
+ "host NUMA node {host_node_id} does not exist on this host"
+ ))
+ })?;
+
+ let mut allowed = host_cpus.clone();
+
+ if let Some(aff) = affinity {
+ allowed.intersect(aff);
+ }
+
+ if allowed.is_empty() {
+ return Err(CpuError::BadSyntax(format!(
+ "no allowed CPUs available for host NUMA node {host_node_id} \
+ after applying affinity mask."
+ )));
+ }
+
+ for &vcpu_id in &cfg.vcpus {
+ bindings.insert(vcpu_id, allowed.clone());
+ }
+ }
+
+ Ok(bindings)
+}
+
+/// Applies the generated CPU bindings to the running vCPU threads.
+pub fn apply_bindings(
+ bindings: &BTreeMap<u32, CpuSet>,
+ qmp_cpuinfo: &[QmpCpu],
+) -> Result<(), CpuError> {
+ for (&vcpu_id, cpuset) in bindings {
+ let vcpu_index = vcpu_id as usize;
+ if vcpu_index >= qmp_cpuinfo.len() {
+ return Err(CpuError::VcpuOutOfRange {
+ vcpu: vcpu_id,
+ max: qmp_cpuinfo.len() as u32,
+ });
+ }
+
+ let tid = qmp_cpuinfo[vcpu_index].thread_id;
+ sched_setaffinity(tid as libc::pid_t, cpuset).map_err(|e| CpuError::Bind {
+ vcpu: vcpu_id,
+ message: e.to_string(),
+ })?;
+ }
+ Ok(())
+}
diff --git a/pve-qemu-server-cpu/src/config.rs b/pve-qemu-server-cpu/src/config.rs
new file mode 100644
index 0000000..a89c62c
--- /dev/null
+++ b/pve-qemu-server-cpu/src/config.rs
@@ -0,0 +1,146 @@
+//! Parsing of the Proxmox VE CPU and memory configuration.
+
+use serde::de::{self, Deserializer, SeqAccess, Visitor};
+use serde::Deserialize;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::fmt;
+
+use crate::cpuset::CpuSet;
+use crate::error::CpuError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum NumaPolicy {
+ Bind,
+ Interleave,
+ Preferred,
+}
+
+/// Configuration for a single numa[N] node.
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
+pub struct NumaConfig {
+ /// Guest vCPU IDs assigned to this node.
+ #[serde(default, deserialize_with = "deserialize_range_set")]
+ pub vcpus: BTreeSet<u32>,
+
+ /// Host NUMA node IDs.
+ #[serde(default, deserialize_with = "deserialize_range_set")]
+ pub hostnodes: BTreeSet<u32>,
+
+ /// Memory provided by this node, in MiB.
+ pub memory: u32,
+
+ /// NUMA allocation policy.
+ pub policy: Option<NumaPolicy>,
+}
+
+/// Parses the NUMA configuration from a JSON string.
+///
+/// Returns a `Vec` where the index corresponds to the NUMA node ID.
+/// Assumes the Perl layer has already validated that node IDs are
+/// contiguous from `0` to `N-1`.
+/// Parses the NUMA configuration from a JSON string.
+///
+/// Returns a `Vec` where the index corresponds to the NUMA node ID.
+/// Assumes the Perl layer has already validated that node IDs are
+/// contiguous from `0` to `N-1`.
+pub fn numa_config_from_json(cfg_json: &str) -> Result<Vec<NumaConfig>, CpuError> {
+ let raw_map: HashMap<String, NumaConfig> = serde_json::from_str(cfg_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid NUMA JSON: {e}")))?;
+
+ let len = raw_map.len();
+
+ let mut configs: Vec<Option<NumaConfig>> = vec![None; len];
+
+ for (key, config) in raw_map {
+ let id_str = key.strip_prefix("numa").unwrap_or(&key);
+ let id = id_str.parse::<usize>().map_err(|e| {
+ CpuError::BadSyntax(format!("invalid NUMA node ID '{key}': {e}"))
+ })?;
+
+ configs[id] = Some(config);
+ }
+
+ // Unwrap the options. If Perl did its job, every slot will be filled.
+ // We return an error instead of panicking just in case the invariant is broken.
+ configs
+ .into_iter()
+ .enumerate()
+ .map(|(i, opt)| {
+ opt.ok_or_else(|| CpuError::BadSyntax(format!("missing NUMA node {i}")))
+ })
+ .collect()
+}
+
+/// Expands a JSON array of ranges into a `BTreeSet<u32>`.
+///
+/// Perl outputs ranges as `[[0, 3], [8, null]]`. This visitor expands them
+/// into a flat set of integers: `{0, 1, 2, 3, 8}`.
+fn deserialize_range_set<'de, D>(deserializer: D) -> Result<BTreeSet<u32>, D::Error>
+where
+ D: Deserializer<'de>,
+{
+ struct RangeSetVisitor;
+
+ impl<'de> Visitor<'de> for RangeSetVisitor {
+ type Value = BTreeSet<u32>;
+
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("a JSON array of ranges (e.g., [[0, 3], [8, null]])")
+ }
+
+ fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+ where
+ A: SeqAccess<'de>,
+ {
+ let mut set = BTreeSet::new();
+
+ while let Some((start, end_opt)) = seq.next_element::<(u32, Option<u32>)>()? {
+ let end = end_opt.unwrap_or(start);
+ if end < start {
+ return Err(de::Error::custom(format!(
+ "invalid range: end ({end}) < start ({start})"
+ )));
+ }
+ set.extend(start..=end);
+ }
+
+ Ok(set)
+ }
+
+ fn visit_unit<E>(self) -> Result<Self::Value, E> {
+ Ok(BTreeSet::new())
+ }
+
+ fn visit_none<E>(self) -> Result<Self::Value, E> {
+ Ok(BTreeSet::new())
+ }
+ }
+
+ deserializer.deserialize_any(RangeSetVisitor)
+}
+
+pub fn bindings_from_json(bind_map_json: &str) -> Result<BTreeMap<u32, CpuSet>, CpuError> {
+ let raw_map: BTreeMap<u32, Vec<u32>> = serde_json::from_str(bind_map_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid binding map JSON: {e}")))?;
+
+ let mut bindings = BTreeMap::new();
+ for (vcpu, cpus) in raw_map {
+ let mut set = CpuSet::new();
+ for cpu in cpus {
+ set.set(cpu);
+ }
+ bindings.insert(vcpu, set);
+ }
+ Ok(bindings)
+}
+
+pub fn bindings_to_json(bindings: &BTreeMap<u32, CpuSet>) -> Result<String, CpuError> {
+ let map: BTreeMap<u32, Vec<u32>> = bindings
+ .iter()
+ .map(|(&vcpu, cpuset)| (vcpu, cpuset.iter().collect()))
+ .collect();
+
+ serde_json::to_string(&map)
+ .map_err(|e| CpuError::BadSyntax(format!("failed to serialize bindings: {e}")))
+}
diff --git a/pve-qemu-server-cpu/src/cpuset.rs b/pve-qemu-server-cpu/src/cpuset.rs
new file mode 100644
index 0000000..d2ffcf8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/cpuset.rs
@@ -0,0 +1,125 @@
+//! Arbitrary-precision CPU bitmask for efficient set operations.
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CpuSet {
+ bits: Vec<u64>,
+}
+
+struct WordBits {
+ base: u32,
+ remaining: u64,
+}
+
+impl Iterator for WordBits {
+ type Item = u32;
+
+ fn next(&mut self) -> Option<u32> {
+ if self.remaining == 0 {
+ return None;
+ }
+ let bit = self.remaining.trailing_zeros();
+ // Clear the lowest set bit so the next call finds the following one.
+ self.remaining &= self.remaining - 1;
+ Some(self.base + bit)
+ }
+}
+
+impl CpuSet {
+ pub fn new() -> Self {
+ Self { bits: Vec::new() }
+ }
+
+ pub fn set(&mut self, cpu: u32) {
+ let idx = (cpu / 64) as usize;
+ let bit = cpu % 64;
+ if idx >= self.bits.len() {
+ self.bits.resize(idx + 1, 0);
+ }
+ self.bits[idx] |= 1u64 << bit;
+ }
+
+ pub fn intersect(&mut self, other: &CpuSet) {
+ for (i, word) in self.bits.iter_mut().enumerate() {
+ if i < other.bits.len() {
+ *word &= other.bits[i];
+ } else {
+ *word = 0;
+ }
+ }
+ }
+
+ pub fn union(&mut self, other: &CpuSet) {
+ if other.bits.len() > self.bits.len() {
+ self.bits.resize(other.bits.len(), 0);
+ }
+ for (i, word) in other.bits.iter().enumerate() {
+ self.bits[i] |= word;
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.bits.iter().all(|&w| w == 0)
+ }
+
+ pub fn size(&self) -> usize {
+ self.bits.iter().map(|w| w.count_ones() as usize).sum()
+ }
+
+ pub fn first_set(&self) -> Option<u32> {
+ self.iter().next()
+ }
+
+ pub fn iter(&self) -> impl Iterator<Item = u32> + '_ {
+ self.bits
+ .iter()
+ .enumerate()
+ .flat_map(|(idx, &word)| WordBits {
+ base: (idx as u32) * 64,
+ remaining: word,
+ })
+ }
+
+ /// Returns the set members as an ascending vector.
+ pub fn to_vec(&self) -> Vec<u32> {
+ self.iter().collect()
+ }
+
+ /// Converts the bitmask to a `libc::cpu_set_t` for `sched_setaffinity`.
+ pub fn to_cpu_set_t(&self) -> libc::cpu_set_t {
+ unsafe {
+ let mut set: libc::cpu_set_t = std::mem::zeroed();
+ for cpu in self.iter() {
+ libc::CPU_SET(cpu as usize, &mut set);
+ }
+ set
+ }
+ }
+
+ /// Parses sysfs CPU lists, for example `online`, `cpulist`, `core_cpus_list`.
+ ///
+ /// Format: comma-separated CPU IDs and inclusive ranges, such as `0-15,18-31`.
+ pub fn from_list(list: &str) -> Self {
+ let mut set = CpuSet::new();
+ let list = list.trim();
+ if list.is_empty() {
+ return set;
+ }
+
+ for part in list.split(',') {
+ let part = part.trim();
+ if part.is_empty() {
+ continue;
+ }
+ if let Some((start_raw, end_raw)) = part.split_once('-') {
+ let start: u32 = start_raw.trim().parse().unwrap_or(0);
+ let end: u32 = end_raw.trim().parse().unwrap_or(start);
+ for i in start..=end {
+ set.set(i);
+ }
+ } else if let Ok(cpu) = part.parse::<u32>() {
+ set.set(cpu);
+ }
+ }
+ set
+ }
+}
diff --git a/pve-qemu-server-cpu/src/error.rs b/pve-qemu-server-cpu/src/error.rs
new file mode 100644
index 0000000..2efbc12
--- /dev/null
+++ b/pve-qemu-server-cpu/src/error.rs
@@ -0,0 +1,32 @@
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+#[error("cpu binding: {0}")]
+pub enum CpuError {
+ #[error("invalid syntax in: '{0}'")]
+ BadSyntax(String),
+
+ // Validation errors (pre-start)
+ #[error("host CPU {0} does not exist in this topology")]
+ UnknownHostCpu(u32),
+
+ #[error("vCPU {vcpu} is out of range (VM has vCPUs 0..{max})")]
+ VcpuOutOfRange { vcpu: u32, max: u32 },
+
+ // Runtime errors (post-start)
+ #[error("no running thread found for vCPU {0}")]
+ ThreadNotFound(u32),
+
+ #[error("invalid kernel TID {tid} for vCPU {vcpu}")]
+ BadThreadId { vcpu: u32, tid: u32 },
+
+ #[error("failed to init host topology: '{0}'")]
+ TopologyInit(String),
+
+ #[error("binding vCPU {vcpu} failed: {message}")]
+ Bind {
+ vcpu: u32,
+ /// Human-readable error from sched_setaffinity(2)
+ message: String,
+ },
+}
diff --git a/pve-qemu-server-cpu/src/lib.rs b/pve-qemu-server-cpu/src/lib.rs
new file mode 100644
index 0000000..cf19aa8
--- /dev/null
+++ b/pve-qemu-server-cpu/src/lib.rs
@@ -0,0 +1,92 @@
+mod bind;
+mod cpuset;
+mod error;
+mod config;
+mod qmp;
+mod syscall;
+mod topology;
+
+/// Generate vcpu:cpu(s) binding map for a guest
+///
+/// # Arguments
+///
+/// * `mode` - Pinning mode.
+/// * `sockets` - number of sockets configured in <vmid.conf>
+/// * `cores` - number of cores configured in <vmid.conf>
+/// * `affinity` - Upper bound of host cpus that can be used for binding.
+/// * `cfg_json` - Additional config options. (e.g. numa config)
+/// * `cpuinfo_json` - JSON output from QMP `query-cpus-fast`.
+///
+/// # Modes
+///
+/// | Mode | Meaning |
+/// |---|---|
+/// | `none` | Do not pin vCPUs. |
+/// | `one-to-one` | Assign each vCPU to its own cpu. |
+/// | `numa` | Map guest NUMA nodes to host NUMA nodes |
+///
+/// # Examples
+///
+/// ```ignore
+/// // One-to-one mode:
+/// // Distribute vCPUs over host CPUs 0-7.
+/// gen_vcpu_bindings("one-to-one", 1, 8, "0-7", "", &cpuinfo_json)?;
+///
+/// // One-to-one mode over all allowed host CPUs.
+/// gen_vcpu_bindings("one-to-one", 1, 4 "", "", &cpuinfo_json)?;
+///
+/// // NUMA mode:
+/// // NUMA mode over all allowed host NUMA nodes.
+/// gen_vcpu_bindings("numa", 2, 4, "", &cfg_json, &cpuinfo_json)?;
+///
+/// // NUMA mode respecting the set affinity
+/// gen_vcpu_bindings("numa", 4, 2, "0-7", &cfg_json, &cpuinfo_json)?;
+/// ```
+///
+/// # Errors
+///
+/// Returns [`CpuError`] if the mode or binding syntax is invalid, if the
+/// requested host CPUs or NUMA nodes do not exist, or if binding fails.
+pub fn gen_vcpu_bindings(
+ mode: &str,
+ sockets: u32,
+ cores: u32,
+ affinity: &str,
+ cfg_json: &str,
+) -> Result<String, error::CpuError> {
+ let cpu_affinity = if affinity.is_empty() {
+ None
+ } else {
+ Some(cpuset::CpuSet::from_list(affinity))
+ };
+
+ let bindings: std::collections::BTreeMap<u32, cpuset::CpuSet> = match mode.parse::<bind::Mode>()? {
+ bind::Mode::None => return Ok("{}".to_string()),
+ bind::Mode::OneToOne => {
+ // only_real_cores is hardcoded to false for now, until we device if we actually want this setting
+ bind::gen_one_to_one_bindings(sockets, cores, &cpu_affinity, false)?
+ }
+ bind::Mode::Numa => {
+ let numa_cfg = config::numa_config_from_json(cfg_json)?;
+ bind::gen_numa_bindings(&numa_cfg, &cpu_affinity)?
+ }
+ };
+
+ // This shouldn't happen, but we should still avoid passing empty config to perl
+ if bindings.is_empty() {
+ return Err(error::CpuError::BadSyntax(
+ "cpu-binding-mode '{mode}' is enabled but no vCPU bindings were generated".to_string(),
+ ));
+ }
+
+ config::bindings_to_json(&bindings)
+}
+
+/// Apply vcpu:cpu(s) the binding map
+/// The function parses the QMP `query-cpus-fast` response and binds each
+/// vCPU thread according to the selected mode.
+pub fn bind_vcpus(bind_map_json: &str, qmp_cpuinfo_json: &str) -> Result<(), error::CpuError> {
+ let bindings = config::bindings_from_json(bind_map_json)?;
+ let qmp_cpuinfo = qmp::parse_cpuinfo(qmp_cpuinfo_json)?;
+ bind::apply_bindings(&bindings, &qmp_cpuinfo)
+}
diff --git a/pve-qemu-server-cpu/src/qmp.rs b/pve-qemu-server-cpu/src/qmp.rs
new file mode 100644
index 0000000..78fb8bd
--- /dev/null
+++ b/pve-qemu-server-cpu/src/qmp.rs
@@ -0,0 +1,91 @@
+//! Parsing of the QEMU `query-cpus-fast` QMP response.
+//!
+//! The response is a JSON array; each element describes one guest vCPU
+//! and carries both its guest-visible topology (socket/core/NUMA node)
+//! and its host kernel TID. Every binding mode consumes this shape, so
+//! the parsing lives here rather than inside any individual mode.
+//!
+//! The full QEMU schema is modeled, including fields not currently used
+//! by any binding mode. Serde ignores fields it doesn't know about, but
+//! keeping them explicit documents the wire format and makes future
+//! modes cheaper to implement.
+
+use serde::Deserialize;
+
+use crate::error::CpuError;
+
+/// One entry of the `query-cpus-fast` array.
+#[derive(Debug, Clone, Deserialize)]
+pub struct QmpCpu {
+ /// Guest-visible vCPU index (0 .. sockets*cores).
+ #[serde(rename = "cpu-index")]
+ pub cpu_index: u32,
+
+ /// Host kernel TID of the QEMU vCPU thread.
+ #[serde(rename = "thread-id")]
+ pub thread_id: u32,
+
+ /// QOM path of the CPU object, e.g. `/machine/unattached/device[0]`.
+ #[expect(dead_code)]
+ #[serde(rename = "qom-path")]
+ pub qom_path: String,
+
+ /// QOM type of the CPU object, e.g. `host-x86_64-cpu`.
+ #[expect(dead_code)]
+ #[serde(rename = "qom-type")]
+ pub qom_type: String,
+
+ /// Target architecture, e.g. `x86_64`.
+ #[expect(dead_code)]
+ pub target: String,
+
+ /// Guest-visible topology properties of the vCPU.
+ #[expect(dead_code)]
+ pub props: QmpCpuProps,
+}
+
+/// Guest-visible topology properties of a vCPU.
+#[derive(Debug, Clone, Deserialize)]
+pub struct QmpCpuProps {
+ /// Guest socket the vCPU belongs to.
+ #[expect(dead_code)]
+ #[serde(rename = "socket-id")]
+ pub socket_id: u32,
+
+ /// Guest core within that socket.
+ #[expect(dead_code)]
+ #[serde(rename = "core-id")]
+ pub core_id: u32,
+
+ /// Guest NUMA node the vCPU belongs to.
+ ///
+ /// QEMU reports this even when `numa` is disabled in the VM config,
+ /// in which case every vCPU is on node 0.
+ #[expect(dead_code)]
+ #[serde(rename = "node-id")]
+ pub node_id: u32,
+
+ /// Guest thread within the core.
+ ///
+ /// Always 0 today: QEMU's CPU object model carries this field for
+ /// SMT support in the guest, but PVE does not expose SMT to guests,
+ /// so it is always zero in practice. Modeled for completeness.
+ /// TODO: PVE should expose SMT to the guests
+ #[expect(dead_code)]
+ #[serde(rename = "thread-id")]
+ pub thread_id: u32,
+}
+
+// returned vec is sorted by vCPU id (0,1,2,3...)
+// Guarantees vec[id] == vec[id].cpu_index
+pub fn parse_cpuinfo(cpuinfo_json: &str) -> Result<Vec<QmpCpu>, CpuError> {
+ let mut cpus: Vec<QmpCpu> = serde_json::from_str(cpuinfo_json)
+ .map_err(|e| CpuError::BadSyntax(format!("invalid query-cpus-fast JSON: {e}")))?;
+
+ // QEMU assigns cpu-index sequentially from 0. Sort by it so
+ // `vec[i]` indexes vCPU i,
+ // value should be 0..N-1 so that assumption holds.
+ cpus.sort_unstable_by_key(|c| c.cpu_index);
+
+ Ok(cpus)
+}
diff --git a/pve-qemu-server-cpu/src/syscall.rs b/pve-qemu-server-cpu/src/syscall.rs
new file mode 100644
index 0000000..59b74a9
--- /dev/null
+++ b/pve-qemu-server-cpu/src/syscall.rs
@@ -0,0 +1,24 @@
+//! Safe wrappers around Linux syscalls.
+
+#[cfg(target_os = "linux")]
+use std::io;
+
+use crate::cpuset::CpuSet;
+
+pub fn sched_setaffinity(tid: libc::pid_t, cpus: &CpuSet) -> io::Result<()> {
+ let set = cpus.to_cpu_set_t();
+
+ unsafe {
+ let rc = libc::sched_setaffinity(
+ tid,
+ std::mem::size_of::<libc::cpu_set_t>(),
+ &set,
+ );
+
+ if rc != 0 {
+ return Err(io::Error::last_os_error());
+ }
+ }
+
+ Ok(())
+}
diff --git a/pve-qemu-server-cpu/src/topology/mod.rs b/pve-qemu-server-cpu/src/topology/mod.rs
new file mode 100644
index 0000000..3d7ffaa
--- /dev/null
+++ b/pve-qemu-server-cpu/src/topology/mod.rs
@@ -0,0 +1,222 @@
+//! Hierarchical hardware topology discovered from sysfs.
+//!
+//! The topology is a tree rooted at the machine. Every node owns its
+//! children and there are no parent pointers, so traversal is top-down.
+//!
+//! On modern systems (like AMD EPYC), the hierarchy typically looks like:
+//!
+//! ```text
+//! Machine -> Package -> Die -> NumaNode -> Core -> Pu
+//! ```
+
+mod sysfs;
+
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+
+use crate::cpuset::CpuSet;
+use crate::error::CpuError;
+
+const CPU_DIR: &str = "/sys/devices/system/cpu";
+const NODE_DIR: &str = "/sys/devices/system/node";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ObjectKind {
+ Machine,
+ Package,
+ Die,
+ NumaNode,
+ // Cache, // Future extension
+ Core,
+ Pu,
+}
+
+/// A node in the hardware topology tree.
+///
+/// Each node owns its children. The `cpuset` covers every CPU at or below
+/// this node. `os_index` is the operating-system index (hwloc's `P#`).
+#[derive(Debug, Clone)]
+pub struct TopoObject {
+ pub kind: ObjectKind,
+ pub os_index: u32,
+ pub cpuset: CpuSet,
+ pub children: Vec<TopoObject>,
+}
+
+impl TopoObject {
+ fn new(kind: ObjectKind, os_index: u32) -> Self {
+ Self {
+ kind,
+ os_index,
+ cpuset: CpuSet::new(),
+ children: Vec::new(),
+ }
+ }
+
+ /// Finds an existing child by kind and index, or creates and appends a new one.
+ fn get_or_add_child(&mut self, kind: ObjectKind, os_index: u32) -> &mut TopoObject {
+ let pos = self
+ .children
+ .iter()
+ .position(|c| c.kind == kind && c.os_index == os_index)
+ .unwrap_or_else(|| {
+ self.children.push(TopoObject::new(kind, os_index));
+ self.children.len() - 1
+ });
+
+ &mut self.children[pos]
+ }
+
+ /// Finds the first node matching `kind` and `os_index` in this subtree.
+ pub fn find(&self, kind: ObjectKind, os_index: u32) -> Option<&TopoObject> {
+ if self.kind == kind && self.os_index == os_index {
+ return Some(self);
+ }
+ for child in &self.children {
+ if let Some(found) = child.find(kind, os_index) {
+ return Some(found);
+ }
+ }
+ None
+ }
+
+ /// Calls `f` for every node matching `kind` in this subtree.
+ pub fn for_each(&self, kind: ObjectKind, f: &mut dyn FnMut(&TopoObject)) {
+ if self.kind == kind {
+ f(self);
+ }
+ for child in &self.children {
+ child.for_each(kind, &mut *f);
+ }
+ }
+
+ /// Computes this node's cpuset bottom-up from its children.
+ fn compute_cpuset(&mut self) {
+ self.cpuset = CpuSet::new();
+ if self.kind == ObjectKind::Pu {
+ self.cpuset.set(self.os_index);
+ } else {
+ for child in &mut self.children {
+ child.compute_cpuset();
+ self.cpuset.union(&child.cpuset);
+ }
+ }
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct Topology {
+ online_cpus: CpuSet,
+ root: TopoObject,
+}
+
+impl Topology {
+ pub fn new() -> Result<Self, CpuError> {
+ let online_cpus = sysfs::read_file(&Path::new(CPU_DIR).join("online"))
+ .map(|s| CpuSet::from_list(&s))
+ .ok_or_else(|| {
+ CpuError::TopologyInit("cannot read the online CPU mask".to_string())
+ })?;
+
+ let cpu_to_numa = Self::read_cpu_to_numa();
+
+ let mut root = TopoObject::new(ObjectKind::Machine, 0);
+ Self::build_tree(&mut root, &online_cpus, &cpu_to_numa)?;
+
+ root.compute_cpuset();
+
+ Ok(Topology { online_cpus, root })
+ }
+
+ fn build_tree(
+ root: &mut TopoObject,
+ online_cpus: &CpuSet,
+ cpu_to_numa: &HashMap<u32, u32>,
+ ) -> Result<(), CpuError> {
+ let cpu_dir = Path::new(CPU_DIR);
+
+ for cpu_id in online_cpus.iter() {
+ let topo_dir = cpu_dir.join(format!("cpu{cpu_id}")).join("topology");
+
+ let pkg_id = sysfs::read_file(&topo_dir.join("physical_package_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+
+ let die_id = sysfs::read_file(&topo_dir.join("die_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(0);
+
+ let core_id = sysfs::read_file(&topo_dir.join("core_id"))
+ .and_then(|s| s.parse().ok())
+ .unwrap_or(cpu_id);
+
+ let pkg = root.get_or_add_child(ObjectKind::Package, pkg_id);
+ let die = pkg.get_or_add_child(ObjectKind::Die, die_id);
+
+ // NumaNode is optional. If present, it sits between Die and Core.
+ let parent_of_core = if let Some(numa_id) = cpu_to_numa.get(&cpu_id).copied() {
+ die.get_or_add_child(ObjectKind::NumaNode, numa_id)
+ } else {
+ &mut *die
+ };
+
+ let core = parent_of_core.get_or_add_child(ObjectKind::Core, core_id);
+ core.children.push(TopoObject::new(ObjectKind::Pu, cpu_id));
+ }
+
+ Ok(())
+ }
+
+ pub fn online_cpus(&self) -> &CpuSet {
+ &self.online_cpus
+ }
+
+ pub fn numa_cpus(&self, os_index: u32) -> Option<&CpuSet> {
+ self.root
+ .find(ObjectKind::NumaNode, os_index)
+ .map(|node| &node.cpuset)
+ }
+
+ /// Reduces `cpus` to at most one thread per physical core.
+ /// (Avoids SMT siblings)
+ pub fn singlify_per_core(&self, cpus: &CpuSet) -> CpuSet {
+ let mut result = CpuSet::new();
+ self.root.for_each(ObjectKind::Core, &mut |core| {
+ let mut core_cpus = core.cpuset.clone();
+ core_cpus.intersect(cpus);
+ if let Some(first) = core_cpus.first_set() {
+ result.set(first);
+ }
+ });
+ result
+ }
+
+ fn read_cpu_to_numa() -> HashMap<u32, u32> {
+ let mut map = HashMap::new();
+ let node_dir = Path::new(NODE_DIR);
+
+ let entries = match fs::read_dir(node_dir) {
+ Ok(entries) => entries,
+ Err(_) => return map,
+ };
+
+ for entry in entries.flatten() {
+ let name = entry.file_name();
+ let name_str = name.to_string_lossy();
+ let node_id = match name_str.strip_prefix("node").and_then(|s| s.parse::<u32>().ok()) {
+ Some(id) => id,
+ None => continue,
+ };
+
+ let cpuset = sysfs::read_file(&entry.path().join("cpulist"))
+ .map(|s| CpuSet::from_list(&s))
+ .unwrap_or_else(CpuSet::new);
+
+ for cpu in cpuset.iter() {
+ map.insert(cpu, node_id);
+ }
+ }
+ map
+ }
+}
diff --git a/pve-qemu-server-cpu/src/topology/sysfs.rs b/pve-qemu-server-cpu/src/topology/sysfs.rs
new file mode 100644
index 0000000..a1ff67a
--- /dev/null
+++ b/pve-qemu-server-cpu/src/topology/sysfs.rs
@@ -0,0 +1,6 @@
+use std::fs;
+use std::path::Path;
+
+pub fn read_file(path: &Path) -> Option<String> {
+ fs::read_to_string(path).ok().map(|s| s.trim().to_string())
+}
--
2.47.3
next prev parent reply other threads:[~2026-11-11 15:21 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-11 15:20 [RFC proxmox-perl-rs/qemu-server/qemu-server-rs v4 0/4] vCPU binding for qemu guests Elias Huhsovitz
2026-11-11 15:20 ` Elias Huhsovitz [this message]
2026-11-11 15:20 ` [PATCH proxmox-perl-rs v4 2/4] pve: cpu: add binding for pve-qemu-server-cpu crate Elias Huhsovitz
2026-11-11 15:20 ` [PATCH qemu-server v4 3/4] helpers: return integer values for parse_number_sets Elias Huhsovitz
2026-11-11 15:20 ` [PATCH qemu-server v4 4/4] cpu: add cpu binding Elias Huhsovitz
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261111152027.260117-2-e.huhsovitz@proxmox.com \
--to=e.huhsovitz@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox