From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 989C11FF0B3 for ; Wed, 11 Nov 2026 16:21:54 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 0FF292170E; Wed, 11 Nov 2026 16:21:16 +0100 (CET) From: Elias Huhsovitz To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server-rs v4 1/4] cpu: add pve-qemu-server-cpu crate for vCPU binding Date: Wed, 11 Nov 2026 16:20:24 +0100 Message-ID: <20261111152027.260117-2-e.huhsovitz@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261111152027.260117-1-e.huhsovitz@proxmox.com> References: <20261111152027.260117-1-e.huhsovitz@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794410459752 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.424 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_SHORT 0.001 Use of a URL Shortener for very short URL RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: XGHI2X3GXLAICUPE4AHMCVQBCGFAFY7H X-Message-ID-Hash: XGHI2X3GXLAICUPE4AHMCVQBCGFAFY7H X-MailFrom: e.huhsovitz@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Elias Huhsovitz X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a new library crate that implements vCPU to host CPU binding of qemu guests. 2 main functions are exposed: gen_vcpu_bindings: (1) Reads the CPU Toplogy from sysfs, validates the configured cpu binding config and Generates a vCPU:hostCPU binding map. This function should be called before the guest starts. (2) bind_vcpus: Applies the binding map generated by gen_vcpu_bindings to the specified vm. The cpu binding is applied using sched_setaffinity(2). 2 modes are implemented: (1) one-to-one: pin each vCPU to its own host cpu. (2) numa: bind the set of vCPUs of each vNUMA node to the set of CPUs of a host NUMA node. The cpu bindings are determined by the numa[N] configuration of the guest. Both settings respect the configured affinity. Thread binding uses sched_setaffinity(2), which accepts a kernel TID directly. The TID used is reported by qemu via query-cpus-fast. Originally-by: Dominik Csapak Signed-off-by: Elias Huhsovitz --- Cargo.toml | 2 +- build.sh | 2 +- pve-qemu-server-cpu/Cargo.toml | 16 ++ pve-qemu-server-cpu/debian/changelog | 5 + pve-qemu-server-cpu/debian/control | 41 ++++ pve-qemu-server-cpu/debian/copyright | 18 ++ pve-qemu-server-cpu/debian/debcargo.toml | 9 + pve-qemu-server-cpu/src/bind.rs | 136 +++++++++++++ pve-qemu-server-cpu/src/config.rs | 146 ++++++++++++++ pve-qemu-server-cpu/src/cpuset.rs | 125 ++++++++++++ pve-qemu-server-cpu/src/error.rs | 32 ++++ pve-qemu-server-cpu/src/lib.rs | 92 +++++++++ pve-qemu-server-cpu/src/qmp.rs | 91 +++++++++ pve-qemu-server-cpu/src/syscall.rs | 24 +++ pve-qemu-server-cpu/src/topology/mod.rs | 222 ++++++++++++++++++++++ pve-qemu-server-cpu/src/topology/sysfs.rs | 6 + 16 files changed, 965 insertions(+), 2 deletions(-) create mode 100644 pve-qemu-server-cpu/Cargo.toml create mode 100644 pve-qemu-server-cpu/debian/changelog create mode 100644 pve-qemu-server-cpu/debian/control create mode 100644 pve-qemu-server-cpu/debian/copyright create mode 100644 pve-qemu-server-cpu/debian/debcargo.toml create mode 100644 pve-qemu-server-cpu/src/bind.rs create mode 100644 pve-qemu-server-cpu/src/config.rs create mode 100644 pve-qemu-server-cpu/src/cpuset.rs create mode 100644 pve-qemu-server-cpu/src/error.rs create mode 100644 pve-qemu-server-cpu/src/lib.rs create mode 100644 pve-qemu-server-cpu/src/qmp.rs create mode 100644 pve-qemu-server-cpu/src/syscall.rs create mode 100644 pve-qemu-server-cpu/src/topology/mod.rs create mode 100644 pve-qemu-server-cpu/src/topology/sysfs.rs diff --git a/Cargo.toml b/Cargo.toml index 5eefc79..80f87ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "pve-qemu-server-cpu", "pve-qemu-server-pci" ] exclude = [ @@ -25,4 +26,3 @@ strum = { version = "0.26", features = ["derive"] } # proxmox dependencies pve-api-types = "8.1" - diff --git a/build.sh b/build.sh index 3c2293b..c3ee7b6 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ debcargo package \ --no-overlay-write-back \ --directory "$PWD/"${BUILDDIR}"/${CRATE}" \ "${CRATE}" \ - "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//')" + "$(dpkg-parsechangelog -l "${CRATE}/debian/changelog" -SVersion | sed -e 's/-.*//' -e 's/~/-/')" cd "${BUILDDIR}/${CRATE}" rm -f debian/source/format.debcargo.hint diff --git a/pve-qemu-server-cpu/Cargo.toml b/pve-qemu-server-cpu/Cargo.toml new file mode 100644 index 0000000..b236796 --- /dev/null +++ b/pve-qemu-server-cpu/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "pve-qemu-server-cpu" +version = "0.1.0" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +exclude.workspace = true +rust-version.workspace = true + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +libc = "0.2" diff --git a/pve-qemu-server-cpu/debian/changelog b/pve-qemu-server-cpu/debian/changelog new file mode 100644 index 0000000..3731cfd --- /dev/null +++ b/pve-qemu-server-cpu/debian/changelog @@ -0,0 +1,5 @@ +rust-pve-qemu-server-cpu (0.1.0) trixie; urgency=medium + + * initial release + + -- Proxmox Support Team Tue, 01 Sep 2026 14:02:25 +0200 diff --git a/pve-qemu-server-cpu/debian/control b/pve-qemu-server-cpu/debian/control new file mode 100644 index 0000000..1298d61 --- /dev/null +++ b/pve-qemu-server-cpu/debian/control @@ -0,0 +1,41 @@ +Source: rust-pve-qemu-server-cpu +Section: rust +Priority: optional +Build-Depends: debhelper-compat (= 13), + dh-sequence-cargo, + libhwloc-dev (>= 2.12) +Build-Depends-Arch: cargo:native , + rustc:native (>= 1.94) , + libstd-rust-dev , + librust-libc-0.2+default-dev , + librust-serde-1+default-dev , + librust-serde-1+derive-dev , + librust-serde-json-1+default-dev , + librust-thiserror-2+default-dev +Maintainer: Proxmox Support Team +Standards-Version: 4.7.2 +Vcs-Git: git://git.proxmox.com/git/pve-qemu-server-rs.git +Vcs-Browser: https://git.proxmox.com/?p=pve-qemu-server-rs.git +Homepage: https://proxmox.com +X-Cargo-Crate: pve-qemu-server-cpu + +Package: librust-pve-qemu-server-cpu-dev +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + librust-libc-0.2+default-dev, + librust-serde-1+default-dev, + librust-serde-1+derive-dev, + librust-serde-json-1+default-dev, + librust-thiserror-2+default-dev +Provides: + librust-pve-qemu-server-cpu+default-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0+default-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0.1-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0.1+default-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0.1.0-dev (= ${binary:Version}), + librust-pve-qemu-server-cpu-0.1.0+default-dev (= ${binary:Version}) +Description: Rust crate "pve-qemu-server-cpu" - Rust source code + Source code for Debianized Rust crate "pve-qemu-server-cpu" diff --git a/pve-qemu-server-cpu/debian/copyright b/pve-qemu-server-cpu/debian/copyright new file mode 100644 index 0000000..01138fa --- /dev/null +++ b/pve-qemu-server-cpu/debian/copyright @@ -0,0 +1,18 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + +Files: + * +Copyright: 2026 Proxmox Server Solutions GmbH +License: AGPL-3.0-or-later + This program is free software: you can redistribute it and/or modify it under + the terms of the GNU Affero General Public License as published by the Free + Software Foundation, either version 3 of the License, or (at your option) any + later version. + . + This program is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + details. + . + You should have received a copy of the GNU Affero General Public License along + with this program. If not, see . diff --git a/pve-qemu-server-cpu/debian/debcargo.toml b/pve-qemu-server-cpu/debian/debcargo.toml new file mode 100644 index 0000000..23feea4 --- /dev/null +++ b/pve-qemu-server-cpu/debian/debcargo.toml @@ -0,0 +1,9 @@ +overlay = "." +crate_src_path = ".." +maintainer = "Proxmox Support Team " +allow_prerelease_deps = true + +[source] +vcs_git = "git://git.proxmox.com/git/pve-qemu-server-rs.git" +vcs_browser = "https://git.proxmox.com/?p=pve-qemu-server-rs.git" +build_depends = ["libhwloc-dev (>= 2.12)"] diff --git a/pve-qemu-server-cpu/src/bind.rs b/pve-qemu-server-cpu/src/bind.rs new file mode 100644 index 0000000..d7f52f8 --- /dev/null +++ b/pve-qemu-server-cpu/src/bind.rs @@ -0,0 +1,136 @@ +use std::collections::BTreeMap; + +use crate::cpuset::CpuSet; +use crate::config::NumaConfig; +use crate::qmp::QmpCpu; +use crate::syscall::sched_setaffinity; +use crate::topology::Topology; +use crate::error::CpuError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + None, + OneToOne, + Numa, +} + +impl std::str::FromStr for Mode { + type Err = CpuError; + + fn from_str(s: &str) -> Result { + match s { + "none" => Ok(Mode::None), + "one-to-one" => Ok(Mode::OneToOne), + "numa" => Ok(Mode::Numa), + other => Err(CpuError::BadSyntax(format!( + "unknown cpu-bind-mode '{other}'" + ))), + } + } +} + +pub fn gen_one_to_one_bindings( + sockets: u32, + cores: u32, + affinity: &Option, + only_real_cores: bool, +) -> Result, CpuError> { + let topo = Topology::new()?; + let mut allowed = topo.online_cpus().clone(); + + if let Some(aff) = affinity { + allowed.intersect(aff); + } + + let nr_vcpus = sockets * cores; + + if only_real_cores { + allowed = topo.singlify_per_core(&allowed); + } + + if allowed.size() < nr_vcpus as usize { + return Err(CpuError::BadSyntax(format!( + "one-to-one mode requires {nr_vcpus} host cpus but only {} are available", + allowed.size() + ))); + } + + let available: Vec = allowed.to_vec(); + + let mut bindings: BTreeMap = BTreeMap::new(); + + // qmp::parse_cpuinfo should yield vCPU IDs that are 0..N-1. + // If not: we proably messed up somewhere else + for (vcpu_id, cpu) in available.iter().take(nr_vcpus as usize).enumerate() { + let mut single_cpu = CpuSet::new(); + single_cpu.set(*cpu); + bindings.insert(vcpu_id as u32, single_cpu); + } + + Ok(bindings) +} + +pub fn gen_numa_bindings( + numa_cfg: &Vec, + affinity: &Option, +) -> Result, CpuError> { + let topo = Topology::new()?; + let mut bindings = BTreeMap::new(); + + for (vnode_id, cfg) in numa_cfg.iter().enumerate() { + if cfg.hostnodes.len() != 1 { + return Err(CpuError::BadSyntax(format!( + "vNUMA node {vnode_id} must be bound to exactly one host NUMA node" + ))); + } + + let host_node_id = *cfg.hostnodes.first().unwrap(); + let host_cpus = topo.numa_cpus(host_node_id).ok_or_else(|| { + CpuError::BadSyntax(format!( + "host NUMA node {host_node_id} does not exist on this host" + )) + })?; + + let mut allowed = host_cpus.clone(); + + if let Some(aff) = affinity { + allowed.intersect(aff); + } + + if allowed.is_empty() { + return Err(CpuError::BadSyntax(format!( + "no allowed CPUs available for host NUMA node {host_node_id} \ + after applying affinity mask." + ))); + } + + for &vcpu_id in &cfg.vcpus { + bindings.insert(vcpu_id, allowed.clone()); + } + } + + Ok(bindings) +} + +/// Applies the generated CPU bindings to the running vCPU threads. +pub fn apply_bindings( + bindings: &BTreeMap, + qmp_cpuinfo: &[QmpCpu], +) -> Result<(), CpuError> { + for (&vcpu_id, cpuset) in bindings { + let vcpu_index = vcpu_id as usize; + if vcpu_index >= qmp_cpuinfo.len() { + return Err(CpuError::VcpuOutOfRange { + vcpu: vcpu_id, + max: qmp_cpuinfo.len() as u32, + }); + } + + let tid = qmp_cpuinfo[vcpu_index].thread_id; + sched_setaffinity(tid as libc::pid_t, cpuset).map_err(|e| CpuError::Bind { + vcpu: vcpu_id, + message: e.to_string(), + })?; + } + Ok(()) +} diff --git a/pve-qemu-server-cpu/src/config.rs b/pve-qemu-server-cpu/src/config.rs new file mode 100644 index 0000000..a89c62c --- /dev/null +++ b/pve-qemu-server-cpu/src/config.rs @@ -0,0 +1,146 @@ +//! Parsing of the Proxmox VE CPU and memory configuration. + +use serde::de::{self, Deserializer, SeqAccess, Visitor}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fmt; + +use crate::cpuset::CpuSet; +use crate::error::CpuError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum NumaPolicy { + Bind, + Interleave, + Preferred, +} + +/// Configuration for a single numa[N] node. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct NumaConfig { + /// Guest vCPU IDs assigned to this node. + #[serde(default, deserialize_with = "deserialize_range_set")] + pub vcpus: BTreeSet, + + /// Host NUMA node IDs. + #[serde(default, deserialize_with = "deserialize_range_set")] + pub hostnodes: BTreeSet, + + /// Memory provided by this node, in MiB. + pub memory: u32, + + /// NUMA allocation policy. + pub policy: Option, +} + +/// Parses the NUMA configuration from a JSON string. +/// +/// Returns a `Vec` where the index corresponds to the NUMA node ID. +/// Assumes the Perl layer has already validated that node IDs are +/// contiguous from `0` to `N-1`. +/// Parses the NUMA configuration from a JSON string. +/// +/// Returns a `Vec` where the index corresponds to the NUMA node ID. +/// Assumes the Perl layer has already validated that node IDs are +/// contiguous from `0` to `N-1`. +pub fn numa_config_from_json(cfg_json: &str) -> Result, CpuError> { + let raw_map: HashMap = serde_json::from_str(cfg_json) + .map_err(|e| CpuError::BadSyntax(format!("invalid NUMA JSON: {e}")))?; + + let len = raw_map.len(); + + let mut configs: Vec> = vec![None; len]; + + for (key, config) in raw_map { + let id_str = key.strip_prefix("numa").unwrap_or(&key); + let id = id_str.parse::().map_err(|e| { + CpuError::BadSyntax(format!("invalid NUMA node ID '{key}': {e}")) + })?; + + configs[id] = Some(config); + } + + // Unwrap the options. If Perl did its job, every slot will be filled. + // We return an error instead of panicking just in case the invariant is broken. + configs + .into_iter() + .enumerate() + .map(|(i, opt)| { + opt.ok_or_else(|| CpuError::BadSyntax(format!("missing NUMA node {i}"))) + }) + .collect() +} + +/// Expands a JSON array of ranges into a `BTreeSet`. +/// +/// Perl outputs ranges as `[[0, 3], [8, null]]`. This visitor expands them +/// into a flat set of integers: `{0, 1, 2, 3, 8}`. +fn deserialize_range_set<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct RangeSetVisitor; + + impl<'de> Visitor<'de> for RangeSetVisitor { + type Value = BTreeSet; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON array of ranges (e.g., [[0, 3], [8, null]])") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut set = BTreeSet::new(); + + while let Some((start, end_opt)) = seq.next_element::<(u32, Option)>()? { + let end = end_opt.unwrap_or(start); + if end < start { + return Err(de::Error::custom(format!( + "invalid range: end ({end}) < start ({start})" + ))); + } + set.extend(start..=end); + } + + Ok(set) + } + + fn visit_unit(self) -> Result { + Ok(BTreeSet::new()) + } + + fn visit_none(self) -> Result { + Ok(BTreeSet::new()) + } + } + + deserializer.deserialize_any(RangeSetVisitor) +} + +pub fn bindings_from_json(bind_map_json: &str) -> Result, CpuError> { + let raw_map: BTreeMap> = serde_json::from_str(bind_map_json) + .map_err(|e| CpuError::BadSyntax(format!("invalid binding map JSON: {e}")))?; + + let mut bindings = BTreeMap::new(); + for (vcpu, cpus) in raw_map { + let mut set = CpuSet::new(); + for cpu in cpus { + set.set(cpu); + } + bindings.insert(vcpu, set); + } + Ok(bindings) +} + +pub fn bindings_to_json(bindings: &BTreeMap) -> Result { + let map: BTreeMap> = bindings + .iter() + .map(|(&vcpu, cpuset)| (vcpu, cpuset.iter().collect())) + .collect(); + + serde_json::to_string(&map) + .map_err(|e| CpuError::BadSyntax(format!("failed to serialize bindings: {e}"))) +} diff --git a/pve-qemu-server-cpu/src/cpuset.rs b/pve-qemu-server-cpu/src/cpuset.rs new file mode 100644 index 0000000..d2ffcf8 --- /dev/null +++ b/pve-qemu-server-cpu/src/cpuset.rs @@ -0,0 +1,125 @@ +//! Arbitrary-precision CPU bitmask for efficient set operations. + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CpuSet { + bits: Vec, +} + +struct WordBits { + base: u32, + remaining: u64, +} + +impl Iterator for WordBits { + type Item = u32; + + fn next(&mut self) -> Option { + if self.remaining == 0 { + return None; + } + let bit = self.remaining.trailing_zeros(); + // Clear the lowest set bit so the next call finds the following one. + self.remaining &= self.remaining - 1; + Some(self.base + bit) + } +} + +impl CpuSet { + pub fn new() -> Self { + Self { bits: Vec::new() } + } + + pub fn set(&mut self, cpu: u32) { + let idx = (cpu / 64) as usize; + let bit = cpu % 64; + if idx >= self.bits.len() { + self.bits.resize(idx + 1, 0); + } + self.bits[idx] |= 1u64 << bit; + } + + pub fn intersect(&mut self, other: &CpuSet) { + for (i, word) in self.bits.iter_mut().enumerate() { + if i < other.bits.len() { + *word &= other.bits[i]; + } else { + *word = 0; + } + } + } + + pub fn union(&mut self, other: &CpuSet) { + if other.bits.len() > self.bits.len() { + self.bits.resize(other.bits.len(), 0); + } + for (i, word) in other.bits.iter().enumerate() { + self.bits[i] |= word; + } + } + + pub fn is_empty(&self) -> bool { + self.bits.iter().all(|&w| w == 0) + } + + pub fn size(&self) -> usize { + self.bits.iter().map(|w| w.count_ones() as usize).sum() + } + + pub fn first_set(&self) -> Option { + self.iter().next() + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.bits + .iter() + .enumerate() + .flat_map(|(idx, &word)| WordBits { + base: (idx as u32) * 64, + remaining: word, + }) + } + + /// Returns the set members as an ascending vector. + pub fn to_vec(&self) -> Vec { + self.iter().collect() + } + + /// Converts the bitmask to a `libc::cpu_set_t` for `sched_setaffinity`. + pub fn to_cpu_set_t(&self) -> libc::cpu_set_t { + unsafe { + let mut set: libc::cpu_set_t = std::mem::zeroed(); + for cpu in self.iter() { + libc::CPU_SET(cpu as usize, &mut set); + } + set + } + } + + /// Parses sysfs CPU lists, for example `online`, `cpulist`, `core_cpus_list`. + /// + /// Format: comma-separated CPU IDs and inclusive ranges, such as `0-15,18-31`. + pub fn from_list(list: &str) -> Self { + let mut set = CpuSet::new(); + let list = list.trim(); + if list.is_empty() { + return set; + } + + for part in list.split(',') { + let part = part.trim(); + if part.is_empty() { + continue; + } + if let Some((start_raw, end_raw)) = part.split_once('-') { + let start: u32 = start_raw.trim().parse().unwrap_or(0); + let end: u32 = end_raw.trim().parse().unwrap_or(start); + for i in start..=end { + set.set(i); + } + } else if let Ok(cpu) = part.parse::() { + set.set(cpu); + } + } + set + } +} diff --git a/pve-qemu-server-cpu/src/error.rs b/pve-qemu-server-cpu/src/error.rs new file mode 100644 index 0000000..2efbc12 --- /dev/null +++ b/pve-qemu-server-cpu/src/error.rs @@ -0,0 +1,32 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +#[error("cpu binding: {0}")] +pub enum CpuError { + #[error("invalid syntax in: '{0}'")] + BadSyntax(String), + + // Validation errors (pre-start) + #[error("host CPU {0} does not exist in this topology")] + UnknownHostCpu(u32), + + #[error("vCPU {vcpu} is out of range (VM has vCPUs 0..{max})")] + VcpuOutOfRange { vcpu: u32, max: u32 }, + + // Runtime errors (post-start) + #[error("no running thread found for vCPU {0}")] + ThreadNotFound(u32), + + #[error("invalid kernel TID {tid} for vCPU {vcpu}")] + BadThreadId { vcpu: u32, tid: u32 }, + + #[error("failed to init host topology: '{0}'")] + TopologyInit(String), + + #[error("binding vCPU {vcpu} failed: {message}")] + Bind { + vcpu: u32, + /// Human-readable error from sched_setaffinity(2) + message: String, + }, +} diff --git a/pve-qemu-server-cpu/src/lib.rs b/pve-qemu-server-cpu/src/lib.rs new file mode 100644 index 0000000..cf19aa8 --- /dev/null +++ b/pve-qemu-server-cpu/src/lib.rs @@ -0,0 +1,92 @@ +mod bind; +mod cpuset; +mod error; +mod config; +mod qmp; +mod syscall; +mod topology; + +/// Generate vcpu:cpu(s) binding map for a guest +/// +/// # Arguments +/// +/// * `mode` - Pinning mode. +/// * `sockets` - number of sockets configured in +/// * `cores` - number of cores configured in +/// * `affinity` - Upper bound of host cpus that can be used for binding. +/// * `cfg_json` - Additional config options. (e.g. numa config) +/// * `cpuinfo_json` - JSON output from QMP `query-cpus-fast`. +/// +/// # Modes +/// +/// | Mode | Meaning | +/// |---|---| +/// | `none` | Do not pin vCPUs. | +/// | `one-to-one` | Assign each vCPU to its own cpu. | +/// | `numa` | Map guest NUMA nodes to host NUMA nodes | +/// +/// # Examples +/// +/// ```ignore +/// // One-to-one mode: +/// // Distribute vCPUs over host CPUs 0-7. +/// gen_vcpu_bindings("one-to-one", 1, 8, "0-7", "", &cpuinfo_json)?; +/// +/// // One-to-one mode over all allowed host CPUs. +/// gen_vcpu_bindings("one-to-one", 1, 4 "", "", &cpuinfo_json)?; +/// +/// // NUMA mode: +/// // NUMA mode over all allowed host NUMA nodes. +/// gen_vcpu_bindings("numa", 2, 4, "", &cfg_json, &cpuinfo_json)?; +/// +/// // NUMA mode respecting the set affinity +/// gen_vcpu_bindings("numa", 4, 2, "0-7", &cfg_json, &cpuinfo_json)?; +/// ``` +/// +/// # Errors +/// +/// Returns [`CpuError`] if the mode or binding syntax is invalid, if the +/// requested host CPUs or NUMA nodes do not exist, or if binding fails. +pub fn gen_vcpu_bindings( + mode: &str, + sockets: u32, + cores: u32, + affinity: &str, + cfg_json: &str, +) -> Result { + let cpu_affinity = if affinity.is_empty() { + None + } else { + Some(cpuset::CpuSet::from_list(affinity)) + }; + + let bindings: std::collections::BTreeMap = match mode.parse::()? { + bind::Mode::None => return Ok("{}".to_string()), + bind::Mode::OneToOne => { + // only_real_cores is hardcoded to false for now, until we device if we actually want this setting + bind::gen_one_to_one_bindings(sockets, cores, &cpu_affinity, false)? + } + bind::Mode::Numa => { + let numa_cfg = config::numa_config_from_json(cfg_json)?; + bind::gen_numa_bindings(&numa_cfg, &cpu_affinity)? + } + }; + + // This shouldn't happen, but we should still avoid passing empty config to perl + if bindings.is_empty() { + return Err(error::CpuError::BadSyntax( + "cpu-binding-mode '{mode}' is enabled but no vCPU bindings were generated".to_string(), + )); + } + + config::bindings_to_json(&bindings) +} + +/// Apply vcpu:cpu(s) the binding map +/// The function parses the QMP `query-cpus-fast` response and binds each +/// vCPU thread according to the selected mode. +pub fn bind_vcpus(bind_map_json: &str, qmp_cpuinfo_json: &str) -> Result<(), error::CpuError> { + let bindings = config::bindings_from_json(bind_map_json)?; + let qmp_cpuinfo = qmp::parse_cpuinfo(qmp_cpuinfo_json)?; + bind::apply_bindings(&bindings, &qmp_cpuinfo) +} diff --git a/pve-qemu-server-cpu/src/qmp.rs b/pve-qemu-server-cpu/src/qmp.rs new file mode 100644 index 0000000..78fb8bd --- /dev/null +++ b/pve-qemu-server-cpu/src/qmp.rs @@ -0,0 +1,91 @@ +//! Parsing of the QEMU `query-cpus-fast` QMP response. +//! +//! The response is a JSON array; each element describes one guest vCPU +//! and carries both its guest-visible topology (socket/core/NUMA node) +//! and its host kernel TID. Every binding mode consumes this shape, so +//! the parsing lives here rather than inside any individual mode. +//! +//! The full QEMU schema is modeled, including fields not currently used +//! by any binding mode. Serde ignores fields it doesn't know about, but +//! keeping them explicit documents the wire format and makes future +//! modes cheaper to implement. + +use serde::Deserialize; + +use crate::error::CpuError; + +/// One entry of the `query-cpus-fast` array. +#[derive(Debug, Clone, Deserialize)] +pub struct QmpCpu { + /// Guest-visible vCPU index (0 .. sockets*cores). + #[serde(rename = "cpu-index")] + pub cpu_index: u32, + + /// Host kernel TID of the QEMU vCPU thread. + #[serde(rename = "thread-id")] + pub thread_id: u32, + + /// QOM path of the CPU object, e.g. `/machine/unattached/device[0]`. + #[expect(dead_code)] + #[serde(rename = "qom-path")] + pub qom_path: String, + + /// QOM type of the CPU object, e.g. `host-x86_64-cpu`. + #[expect(dead_code)] + #[serde(rename = "qom-type")] + pub qom_type: String, + + /// Target architecture, e.g. `x86_64`. + #[expect(dead_code)] + pub target: String, + + /// Guest-visible topology properties of the vCPU. + #[expect(dead_code)] + pub props: QmpCpuProps, +} + +/// Guest-visible topology properties of a vCPU. +#[derive(Debug, Clone, Deserialize)] +pub struct QmpCpuProps { + /// Guest socket the vCPU belongs to. + #[expect(dead_code)] + #[serde(rename = "socket-id")] + pub socket_id: u32, + + /// Guest core within that socket. + #[expect(dead_code)] + #[serde(rename = "core-id")] + pub core_id: u32, + + /// Guest NUMA node the vCPU belongs to. + /// + /// QEMU reports this even when `numa` is disabled in the VM config, + /// in which case every vCPU is on node 0. + #[expect(dead_code)] + #[serde(rename = "node-id")] + pub node_id: u32, + + /// Guest thread within the core. + /// + /// Always 0 today: QEMU's CPU object model carries this field for + /// SMT support in the guest, but PVE does not expose SMT to guests, + /// so it is always zero in practice. Modeled for completeness. + /// TODO: PVE should expose SMT to the guests + #[expect(dead_code)] + #[serde(rename = "thread-id")] + pub thread_id: u32, +} + +// returned vec is sorted by vCPU id (0,1,2,3...) +// Guarantees vec[id] == vec[id].cpu_index +pub fn parse_cpuinfo(cpuinfo_json: &str) -> Result, CpuError> { + let mut cpus: Vec = serde_json::from_str(cpuinfo_json) + .map_err(|e| CpuError::BadSyntax(format!("invalid query-cpus-fast JSON: {e}")))?; + + // QEMU assigns cpu-index sequentially from 0. Sort by it so + // `vec[i]` indexes vCPU i, + // value should be 0..N-1 so that assumption holds. + cpus.sort_unstable_by_key(|c| c.cpu_index); + + Ok(cpus) +} diff --git a/pve-qemu-server-cpu/src/syscall.rs b/pve-qemu-server-cpu/src/syscall.rs new file mode 100644 index 0000000..59b74a9 --- /dev/null +++ b/pve-qemu-server-cpu/src/syscall.rs @@ -0,0 +1,24 @@ +//! Safe wrappers around Linux syscalls. + +#[cfg(target_os = "linux")] +use std::io; + +use crate::cpuset::CpuSet; + +pub fn sched_setaffinity(tid: libc::pid_t, cpus: &CpuSet) -> io::Result<()> { + let set = cpus.to_cpu_set_t(); + + unsafe { + let rc = libc::sched_setaffinity( + tid, + std::mem::size_of::(), + &set, + ); + + if rc != 0 { + return Err(io::Error::last_os_error()); + } + } + + Ok(()) +} diff --git a/pve-qemu-server-cpu/src/topology/mod.rs b/pve-qemu-server-cpu/src/topology/mod.rs new file mode 100644 index 0000000..3d7ffaa --- /dev/null +++ b/pve-qemu-server-cpu/src/topology/mod.rs @@ -0,0 +1,222 @@ +//! Hierarchical hardware topology discovered from sysfs. +//! +//! The topology is a tree rooted at the machine. Every node owns its +//! children and there are no parent pointers, so traversal is top-down. +//! +//! On modern systems (like AMD EPYC), the hierarchy typically looks like: +//! +//! ```text +//! Machine -> Package -> Die -> NumaNode -> Core -> Pu +//! ``` + +mod sysfs; + +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use crate::cpuset::CpuSet; +use crate::error::CpuError; + +const CPU_DIR: &str = "/sys/devices/system/cpu"; +const NODE_DIR: &str = "/sys/devices/system/node"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectKind { + Machine, + Package, + Die, + NumaNode, + // Cache, // Future extension + Core, + Pu, +} + +/// A node in the hardware topology tree. +/// +/// Each node owns its children. The `cpuset` covers every CPU at or below +/// this node. `os_index` is the operating-system index (hwloc's `P#`). +#[derive(Debug, Clone)] +pub struct TopoObject { + pub kind: ObjectKind, + pub os_index: u32, + pub cpuset: CpuSet, + pub children: Vec, +} + +impl TopoObject { + fn new(kind: ObjectKind, os_index: u32) -> Self { + Self { + kind, + os_index, + cpuset: CpuSet::new(), + children: Vec::new(), + } + } + + /// Finds an existing child by kind and index, or creates and appends a new one. + fn get_or_add_child(&mut self, kind: ObjectKind, os_index: u32) -> &mut TopoObject { + let pos = self + .children + .iter() + .position(|c| c.kind == kind && c.os_index == os_index) + .unwrap_or_else(|| { + self.children.push(TopoObject::new(kind, os_index)); + self.children.len() - 1 + }); + + &mut self.children[pos] + } + + /// Finds the first node matching `kind` and `os_index` in this subtree. + pub fn find(&self, kind: ObjectKind, os_index: u32) -> Option<&TopoObject> { + if self.kind == kind && self.os_index == os_index { + return Some(self); + } + for child in &self.children { + if let Some(found) = child.find(kind, os_index) { + return Some(found); + } + } + None + } + + /// Calls `f` for every node matching `kind` in this subtree. + pub fn for_each(&self, kind: ObjectKind, f: &mut dyn FnMut(&TopoObject)) { + if self.kind == kind { + f(self); + } + for child in &self.children { + child.for_each(kind, &mut *f); + } + } + + /// Computes this node's cpuset bottom-up from its children. + fn compute_cpuset(&mut self) { + self.cpuset = CpuSet::new(); + if self.kind == ObjectKind::Pu { + self.cpuset.set(self.os_index); + } else { + for child in &mut self.children { + child.compute_cpuset(); + self.cpuset.union(&child.cpuset); + } + } + } +} + +#[derive(Debug, Clone)] +pub struct Topology { + online_cpus: CpuSet, + root: TopoObject, +} + +impl Topology { + pub fn new() -> Result { + let online_cpus = sysfs::read_file(&Path::new(CPU_DIR).join("online")) + .map(|s| CpuSet::from_list(&s)) + .ok_or_else(|| { + CpuError::TopologyInit("cannot read the online CPU mask".to_string()) + })?; + + let cpu_to_numa = Self::read_cpu_to_numa(); + + let mut root = TopoObject::new(ObjectKind::Machine, 0); + Self::build_tree(&mut root, &online_cpus, &cpu_to_numa)?; + + root.compute_cpuset(); + + Ok(Topology { online_cpus, root }) + } + + fn build_tree( + root: &mut TopoObject, + online_cpus: &CpuSet, + cpu_to_numa: &HashMap, + ) -> Result<(), CpuError> { + let cpu_dir = Path::new(CPU_DIR); + + for cpu_id in online_cpus.iter() { + let topo_dir = cpu_dir.join(format!("cpu{cpu_id}")).join("topology"); + + let pkg_id = sysfs::read_file(&topo_dir.join("physical_package_id")) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let die_id = sysfs::read_file(&topo_dir.join("die_id")) + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let core_id = sysfs::read_file(&topo_dir.join("core_id")) + .and_then(|s| s.parse().ok()) + .unwrap_or(cpu_id); + + let pkg = root.get_or_add_child(ObjectKind::Package, pkg_id); + let die = pkg.get_or_add_child(ObjectKind::Die, die_id); + + // NumaNode is optional. If present, it sits between Die and Core. + let parent_of_core = if let Some(numa_id) = cpu_to_numa.get(&cpu_id).copied() { + die.get_or_add_child(ObjectKind::NumaNode, numa_id) + } else { + &mut *die + }; + + let core = parent_of_core.get_or_add_child(ObjectKind::Core, core_id); + core.children.push(TopoObject::new(ObjectKind::Pu, cpu_id)); + } + + Ok(()) + } + + pub fn online_cpus(&self) -> &CpuSet { + &self.online_cpus + } + + pub fn numa_cpus(&self, os_index: u32) -> Option<&CpuSet> { + self.root + .find(ObjectKind::NumaNode, os_index) + .map(|node| &node.cpuset) + } + + /// Reduces `cpus` to at most one thread per physical core. + /// (Avoids SMT siblings) + pub fn singlify_per_core(&self, cpus: &CpuSet) -> CpuSet { + let mut result = CpuSet::new(); + self.root.for_each(ObjectKind::Core, &mut |core| { + let mut core_cpus = core.cpuset.clone(); + core_cpus.intersect(cpus); + if let Some(first) = core_cpus.first_set() { + result.set(first); + } + }); + result + } + + fn read_cpu_to_numa() -> HashMap { + let mut map = HashMap::new(); + let node_dir = Path::new(NODE_DIR); + + let entries = match fs::read_dir(node_dir) { + Ok(entries) => entries, + Err(_) => return map, + }; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + let node_id = match name_str.strip_prefix("node").and_then(|s| s.parse::().ok()) { + Some(id) => id, + None => continue, + }; + + let cpuset = sysfs::read_file(&entry.path().join("cpulist")) + .map(|s| CpuSet::from_list(&s)) + .unwrap_or_else(CpuSet::new); + + for cpu in cpuset.iter() { + map.insert(cpu, node_id); + } + } + map + } +} diff --git a/pve-qemu-server-cpu/src/topology/sysfs.rs b/pve-qemu-server-cpu/src/topology/sysfs.rs new file mode 100644 index 0000000..a1ff67a --- /dev/null +++ b/pve-qemu-server-cpu/src/topology/sysfs.rs @@ -0,0 +1,6 @@ +use std::fs; +use std::path::Path; + +pub fn read_file(path: &Path) -> Option { + fs::read_to_string(path).ok().map(|s| s.trim().to_string()) +} -- 2.47.3