From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 5D9E11FF0C0 for ; Wed, 18 Nov 2026 16:30:24 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id D611D218E1; Wed, 18 Nov 2026 16:29:25 +0100 (CET) From: Dominik Csapak To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server v5 11/13] pve-qemu-pci: topology: place hostpcie devices like on the host Date: Wed, 18 Nov 2026 16:27:53 +0100 Message-ID: <20261118152803.3843758-12-d.csapak@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118152803.3843758-1-d.csapak@proxmox.com> References: <20261118152803.3843758-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.288 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: JAYVLI36YL43ADHLQWJXH3QZN7P3FGHK X-Message-ID-Hash: JAYVLI36YL43ADHLQWJXH3QZN7P3FGHK X-MailFrom: d.csapak@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Guest software sometimes uses the PCI topology to choose CPUs and DMA paths. Since we put every hostpcie device on a root port of the root bus, the guest can't see which devices are close to which NUMA node or to each other. So add a hostpci topology setting for machines with a PCIe root bus (q35 and aarch64 virt) and the grouped layout: * 'flat': the current placement, a root port on the root bus per device * 'numa': puts the root ports of each guest NUMA node's devices behind a pxb-pcie for that node * 'host': additionally replicates the host PCIe switches that more than one passed through device is behind. We skip the other switches, since they don't add any information but cost bus numbers. The guest NUMA node and the host switches of each device are passed as placements separately from the hardware config, since the caller only knows them after choosing the host devices. All buses share the 256 bus numbers of a single PCI domain. We pack the bus number ranges of the expander bridges by demand and align them to 16, so a changed device normally only shifts its own node, and even 16 devices always fit. Each port gets its own slot, numbered after the lowest hostpcie index behind it, so its address does not depend on its siblings or on a function 0 being present. In the bindings, add 'hostpcie_bus_devices', which takes the placements and returns the expander bridges, switches and root ports, and let 'addr' take them as an optional parameter, so existing callers don't have to change. Since the topology is a required field, qemu-server passes 'flat' until the option to choose it is added. Signed-off-by: Dominik Csapak --- pve-qemu-rs/pve-qemu-pci/src/constants.rs | 1 + pve-qemu-rs/pve-qemu-pci/src/error.rs | 21 + .../pve-qemu-pci/src/hardware_config.rs | 5 +- .../pve-qemu-pci/src/layout/grouped.rs | 52 +- pve-qemu-rs/pve-qemu-pci/src/lib.rs | 49 +- pve-qemu-rs/pve-qemu-pci/src/topology.rs | 481 ++++++++++++++++++ .../pve-qemu-perl/src/pci/hardware_config.rs | 27 +- src/PVE/QemuServer/PCI.pm | 1 + 8 files changed, 617 insertions(+), 20 deletions(-) create mode 100644 pve-qemu-rs/pve-qemu-pci/src/topology.rs diff --git a/pve-qemu-rs/pve-qemu-pci/src/constants.rs b/pve-qemu-rs/pve-qemu-pci/src/constants.rs index 2f5f97aa..ed36f170 100644 --- a/pve-qemu-rs/pve-qemu-pci/src/constants.rs +++ b/pve-qemu-rs/pve-qemu-pci/src/constants.rs @@ -7,6 +7,7 @@ pub(crate) const MAX_SCSI_DEVICES: u8 = pve_api_types::QemuConfigScsiArray::MAX pub(crate) const MAX_VIRTIO_BLK_DEVICES: u8 = pve_api_types::QemuConfigVirtioArray::MAX as u8; pub(crate) const MAX_VIRTIOFS_DEVICES: u8 = pve_api_types::QemuConfigVirtiofsArray::MAX as u8; pub(crate) const MAX_HOSTPCI_DEVICES: u8 = pve_api_types::QemuConfigHostpciArray::MAX as u8; +pub(crate) const MAX_NUMA_NODES: u8 = pve_api_types::QemuConfigNumaArray::MAX as u8; /// Number of slots on a PCI bus. Slot 0x00 is always taken by the bridge /// itself, so only [`USABLE_SLOTS_PER_BUS`] of them can hold a device. diff --git a/pve-qemu-rs/pve-qemu-pci/src/error.rs b/pve-qemu-rs/pve-qemu-pci/src/error.rs index d67b3df6..0e024e84 100644 --- a/pve-qemu-rs/pve-qemu-pci/src/error.rs +++ b/pve-qemu-rs/pve-qemu-pci/src/error.rs @@ -12,6 +12,14 @@ pub enum LayoutError { NoAddressFound(Device), /// The selected layout is not known. InvalidLayout, + /// The hostpci topology requires a PCIe machine and the grouped PCI layout. + UnsupportedTopology, + /// The NUMA node or host switches given for `hostpcie` are invalid. + InvalidPlacement(u8), + /// The hostpci topology needs the host placement of `hostpcie`, but it was not given. + MissingPlacement(u8), + /// Not enough PCI bus numbers for the hostpci topology. + BusNumbersExhausted, } impl Display for LayoutError { @@ -20,6 +28,19 @@ impl Display for LayoutError { LayoutError::InvalidConfigId => write!(f, "invalid configuration id"), LayoutError::NoAddressFound(device) => write!(f, "address '{device}' not found"), LayoutError::InvalidLayout => write!(f, "invalid layout"), + LayoutError::UnsupportedTopology => write!( + f, + "hostpci topology requires a PCIe machine with the grouped PCI layout" + ), + LayoutError::InvalidPlacement(index) => { + write!(f, "invalid host placement for 'hostpcie{index}'") + } + LayoutError::MissingPlacement(index) => { + write!(f, "missing host placement for 'hostpcie{index}'") + } + LayoutError::BusNumbersExhausted => { + write!(f, "not enough PCI bus numbers for the hostpci topology") + } } } } diff --git a/pve-qemu-rs/pve-qemu-pci/src/hardware_config.rs b/pve-qemu-rs/pve-qemu-pci/src/hardware_config.rs index 1c1c1efb..b8fb6b74 100644 --- a/pve-qemu-rs/pve-qemu-pci/src/hardware_config.rs +++ b/pve-qemu-rs/pve-qemu-pci/src/hardware_config.rs @@ -6,7 +6,7 @@ use serde::Deserialize; use pve_api_types::{ClusterResourceHostArch, QemuConfigOstype, QemuConfigScsihw}; -use crate::{Device, Machine, Mainboard, PciLayout}; +use crate::{Device, HostpciTopology, Machine, Mainboard, PciLayout}; // internal type to make sure we only have valid ones #[derive(Default, Clone, Copy, Debug, Deserialize)] @@ -40,6 +40,7 @@ pub struct HardwareConfigParams { pub machine: Machine, pub hostpcie_devices: Vec, pub layout: PciLayout, + pub hostpci_topology: HostpciTopology, } /// A passed through PCIe device of the guest. @@ -70,6 +71,7 @@ pub struct HardwareConfig { /// The configured `hostpcie` devices, keyed by their index, which decide /// what root ports are needed. pub hostpcie_devices: BTreeMap, + pub hostpci_topology: HostpciTopology, } impl HardwareConfig { @@ -90,6 +92,7 @@ impl HardwareConfig { required_pve_version: Arc::new(AtomicU16::new(0)), layout: params.layout, hostpcie_devices, + hostpci_topology: params.hostpci_topology, } } diff --git a/pve-qemu-rs/pve-qemu-pci/src/layout/grouped.rs b/pve-qemu-rs/pve-qemu-pci/src/layout/grouped.rs index 02fcbdc5..6fa4f6cd 100644 --- a/pve-qemu-rs/pve-qemu-pci/src/layout/grouped.rs +++ b/pve-qemu-rs/pve-qemu-pci/src/layout/grouped.rs @@ -3,8 +3,10 @@ use crate::constants::{ MAX_VIRTIOFS_DEVICES, SLOTS_PER_BUS, }; use crate::layout::bridge_slots; +use crate::topology::Topology; use crate::{ - Bus, Device, Function, HardwareConfig, Layout, LayoutError, PciAddress, PveBridge, Slot, + Bus, Device, Function, HardwareConfig, HostpciTopology, HostpciePlacement, Layout, LayoutError, + PciAddress, PveBridge, Slot, }; /// The guest's built-in devices, which all live on a bridge of their own so @@ -129,14 +131,15 @@ pub static PCIE_LAYOUT: Layout = Layout { Function::RootPort(Bus::RootPort(14), Device::Hostpcie(14)), Function::RootPort(Bus::RootPort(15), Device::Hostpcie(15)), ]), - Slot::Unused, - Slot::Unused, - Slot::Unused, - Slot::Unused, - Slot::Unused, - Slot::Unused, - Slot::Unused, - Slot::Unused, + // expander bridges for the hostpci topology, one per guest NUMA node + single_device!(Bridge(Bus::Numa(0))), + single_device!(Bridge(Bus::Numa(1))), + single_device!(Bridge(Bus::Numa(2))), + single_device!(Bridge(Bus::Numa(3))), + single_device!(Bridge(Bus::Numa(4))), + single_device!(Bridge(Bus::Numa(5))), + single_device!(Bridge(Bus::Numa(6))), + single_device!(Bridge(Bus::Numa(7))), Slot::Unused, Slot::Unused, Slot::Unused, @@ -152,13 +155,25 @@ fn layout(pcie: bool) -> &'static Layout<'static> { if pcie { &PCIE_LAYOUT } else { &PCI_LAYOUT } } -pub(crate) fn address(hw: &HardwareConfig, id: &str) -> Result { - let layout = layout(hw.has_pcie()); - layout.find_device(&id.parse()?) +pub(crate) fn address( + hw: &HardwareConfig, + id: &str, + placements: Option<&[HostpciePlacement]>, +) -> Result { + let device = id.parse()?; + match device { + Device::Hostpcie(index) if hw.hostpci_topology != HostpciTopology::Flat => { + let placements = placements.ok_or(LayoutError::MissingPlacement(index))?; + let topology = Topology::new(hw, placements)?; + let addr = topology.addresses.get(&index).copied(); + addr.ok_or(LayoutError::NoAddressFound(device)) + } + _ => layout(hw.has_pcie()).find_device(&device), + } } pub(crate) fn pci_bridge_for_device(hw: &HardwareConfig, id: &str) -> Option { - let addr = address(hw, id).ok()?; + let addr = address(hw, id, None).ok()?; Some(addr.bus) } @@ -172,6 +187,8 @@ pub(crate) fn pci_bus_devices(hw: &HardwareConfig) -> Result, Layout { let addr = bridge.addr; let (model, id) = match bridge.device { + // only created by the hostpci topology + Device::Bridge(Bus::Numa(_)) => continue, Device::Bridge(bus) => (bus.bridge_model(addr.bus), bus), _ => continue, }; @@ -198,9 +215,12 @@ pub(crate) fn pci_bus_devices(hw: &HardwareConfig) -> Result, Layout )); } - for port in layout.root_ports(&hw.hostpcie()) { - bridges.push("-device".to_string()); - bridges.push(port.to_qemu_device()); + // other topologies need the host placements, see crate::hostpcie_bus_devices + if hw.hostpci_topology == HostpciTopology::Flat { + for port in layout.root_ports(&hw.hostpcie()) { + bridges.push("-device".to_string()); + bridges.push(port.to_qemu_device()); + } } Ok(bridges) diff --git a/pve-qemu-rs/pve-qemu-pci/src/lib.rs b/pve-qemu-rs/pve-qemu-pci/src/lib.rs index f998f2b3..0b183f78 100644 --- a/pve-qemu-rs/pve-qemu-pci/src/lib.rs +++ b/pve-qemu-rs/pve-qemu-pci/src/lib.rs @@ -21,6 +21,9 @@ pub use error::LayoutError; pub mod layout; +mod topology; +pub use topology::{HostpciTopology, HostpciePlacement}; + use serde::Deserialize; /// The set of layouts a guest's addresses are taken from. @@ -70,6 +73,9 @@ pub fn pci_bridge_for_device(hw: &HardwareConfig, id: &str) -> Option { /// /// They are ordered so that every bus is created before the devices on it, /// and have to be passed before any other device that sits on these buses. +/// +/// With a non-flat [`HostpciTopology`], the `hostpcie` root ports are not +/// included, see [`hostpcie_bus_devices`]. pub fn pci_bus_devices(hw: &HardwareConfig) -> Result, LayoutError> { match hw.layout { PciLayout::Legacy => layout::legacy::pci_bus_devices(hw), @@ -77,10 +83,51 @@ pub fn pci_bus_devices(hw: &HardwareConfig) -> Result, LayoutError> } } +/// Returns the QEMU arguments for the expander bridges, ports and switches of +/// a non-flat [`HostpciTopology`], as alternating `-device` flags and their +/// values. +/// +/// They depend on the chosen host devices, so they are not part of +/// [`pci_bus_devices`], but have to come after them and before the `hostpcie` +/// devices. With the flat topology, this is empty. +pub fn hostpcie_bus_devices( + hw: &HardwareConfig, + placements: &[HostpciePlacement], +) -> Result, LayoutError> { + if hw.hostpci_topology == HostpciTopology::Flat { + return Ok(Vec::new()); + } + let topology = topology::Topology::new(hw, placements)?; + let devices = topology.devices.into_iter(); + Ok(devices + .flat_map(|device| ["-device".to_string(), device]) + .collect()) +} + /// Returns the address of the device with the configuration id `id`. +/// +/// With a non-flat [`HostpciTopology`], `hostpcie` devices need their host +/// placements, see [`address_with_placements`]. pub fn address(hw: &HardwareConfig, id: &str) -> Result { match hw.layout { PciLayout::Legacy => layout::legacy::address(hw, id), - PciLayout::Grouped => layout::grouped::address(hw, id), + PciLayout::Grouped => layout::grouped::address(hw, id, None), + } +} + +/// Like [`address`], but with the host placements of the `hostpcie` devices, +/// which a non-flat [`HostpciTopology`] needs. +pub fn address_with_placements( + hw: &HardwareConfig, + id: &str, + placements: &[HostpciePlacement], +) -> Result { + match hw.layout { + // legacy variant can't use placements so deny all non flat topologies + PciLayout::Legacy if hw.hostpci_topology != HostpciTopology::Flat => { + Err(LayoutError::UnsupportedTopology) + } + PciLayout::Legacy => layout::legacy::address(hw, id), + PciLayout::Grouped => layout::grouped::address(hw, id, Some(placements)), } } diff --git a/pve-qemu-rs/pve-qemu-pci/src/topology.rs b/pve-qemu-rs/pve-qemu-pci/src/topology.rs new file mode 100644 index 00000000..d15d5e1e --- /dev/null +++ b/pve-qemu-rs/pve-qemu-pci/src/topology.rs @@ -0,0 +1,481 @@ +//! Places passed through PCIe devices in the guest similar to how they are connected on the host. +//! +//! All buses share the 256 bus numbers of one PCI domain. Each expander bridge gets its own range +//! starting at its `bus_nr`, aligned to [`BUS_RANGE_ALIGN`], so changing a device normally only +//! shifts the bus numbers of its own node. The guest sees them, e.g. in network interface names. + +use std::collections::BTreeMap; + +use serde::Deserialize; + +use crate::constants::{MAX_HOSTPCI_DEVICES, MAX_NUMA_NODES}; +use crate::layout::grouped::PCIE_LAYOUT; +use crate::{Bus, Device, HardwareConfig, LayoutError, PciAddress, PciLayout}; + +/// First bus number for the expander bridges, the ones below are for the root bus. +const FIRST_NUMA_BUS: usize = 0x20; + +const BUS_RANGE_ALIGN: usize = 16; + +// make sure the worst case always fits +const _: () = { + let devices = MAX_HOSTPCI_DEVICES as usize; + let nodes = MAX_NUMA_NODES as usize; + // a switch is only necessary with at least two devices behind it, so we can have at max + // devices - 1 switches. + let switches = devices - 1; + + // one bus for the port of each device, two per switch (its port and its upstream port) and + // one per expander bridge + let buses = devices + 2 * switches + nodes; + // the range of each expander bridge is rounded up to the alignment + let lost_to_alignment = nodes * (BUS_RANGE_ALIGN - 1); + + assert!(FIRST_NUMA_BUS + buses + lost_to_alignment <= 256); +}; + +/// Chassis numbers for the ports on expander bridges and switches, above the ones of the root +/// ports on the root bus. Each bus gets its own, so slots only have to be unique per bus. +const NUMA_CHASSIS_BASE: u8 = 0x80; +const SWITCH_CHASSIS_BASE: u8 = 0x90; + +/// How passed through PCIe devices are placed in the guest. +/// +/// Drivers for accelerators and RDMA NICs use the PCI topology to choose CPUs and peer-to-peer +/// paths, so we replicate the host topology from the [`HostpciePlacement`] of each device. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum HostpciTopology { + /// Each device gets its own root port on the root bus. + #[default] + Flat, + /// Groups the root ports behind a `pxb-pcie` expander bridge per guest NUMA node. + Numa, + /// Like `Numa`, but also replicates host switches with more than one device behind them. + Host, +} + +/// Where a passed through PCIe device is connected on the host. Only needed for topologies other +/// than [`HostpciTopology::Flat`]. +#[derive(Default, Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct HostpciePlacement { + /// The hostpci index of the device (e.g. '3' from 'hostpci3'). + pub index: u8, + /// The guest NUMA node backed by the host NUMA node of the device. + #[serde(default)] + pub numa_node: u8, + /// The host bridges above the device, starting at the root port. Each one needs a unique id, + /// e.g. its PCI address. Bridges shared by more than one device are replicated as a switch, + /// so the devices have the same common upstream bridge as on the host. Only used with + /// [`HostpciTopology::Host`]. + #[serde(default)] + pub switches: Vec, +} + +/// A device or replicated switch in the guest. +enum Node { + Device(u8), + Switch(BTreeMap), +} + +impl Node { + /// Returns the lowest `hostpcie` index below this node. We use it as slot number, so the + /// address of a port does not depend on its siblings. + fn key(&self) -> u8 { + match self { + Node::Device(index) => *index, + Node::Switch(children) => *children.keys().next().unwrap(), + } + } +} + +/// The devices and switches below a NUMA node or a switch on the host. +#[derive(Default)] +struct HostTree<'a> { + devices: Vec, + switches: BTreeMap<&'a str, HostTree<'a>>, +} + +impl<'a> HostTree<'a> { + /// Adds `hostpcie` below the given host switches. + fn insert(&mut self, index: u8, switches: &'a [String]) { + match switches.split_first() { + None => self.devices.push(index), + Some((first, rest)) => self.switches.entry(first).or_default().insert(index, rest), + } + } + + /// Returns the guest nodes directly below this one, keyed by [`Node::key`]. + fn children(self) -> BTreeMap { + let devices = self.devices.into_iter().map(Node::Device); + let switches = self.switches.into_values().map(HostTree::into_node); + devices + .chain(switches) + .map(|node| (node.key(), node)) + .collect() + } + + /// Converts the switch to a guest node. Switches with only one child are replaced by that + /// child, since they only use up bus numbers without giving the guest any information. + fn into_node(self) -> Node { + let mut children = self.children(); + if children.len() == 1 { + children.pop_first().unwrap().1 + } else { + Node::Switch(children) + } + } +} + +/// Returns the host tree for each guest NUMA node. Placements of devices that are not configured +/// are ignored. +fn numa_nodes<'a>( + hw: &HardwareConfig, + placements: &'a [HostpciePlacement], +) -> Result>, LayoutError> { + let placements: BTreeMap = + placements.iter().map(|p| (p.index, p)).collect(); + let mut nodes: BTreeMap = BTreeMap::new(); + let mut parents = BTreeMap::new(); + + for &index in hw.hostpcie_devices.keys() { + let device = placements + .get(&index) + .ok_or(LayoutError::MissingPlacement(index))?; + if index >= MAX_HOSTPCI_DEVICES || device.numa_node >= MAX_NUMA_NODES { + return Err(LayoutError::InvalidPlacement(index)); + } + let switches: &[String] = match hw.hostpci_topology { + HostpciTopology::Host => &device.switches, + _ => &[], + }; + // switches must be nested identically for all devices (e.g. switch X can't be behind Y for + // one device and behind Z for another) + for (depth, switch) in switches.iter().enumerate() { + let parent = (device.numa_node, &switches[..depth]); + if *parents.entry(switch).or_insert(parent) != parent { + return Err(LayoutError::InvalidPlacement(index)); + } + } + nodes + .entry(device.numa_node) + .or_default() + .insert(index, switches); + } + + Ok(nodes) +} + +/// Returns the `-device` value for the device creating `bus` at `addr`. +fn device(bus: Bus, addr: PciAddress, params: &str) -> String { + // all buses of the topology are created by a device + let model = bus.bridge_model(addr.bus).unwrap(); + format!("{model},id={bus}{}{params}", addr.to_qemu_addr()) +} + +/// The QEMU devices for a [`HostpciTopology`] and the resulting `hostpcie` addresses. +pub(crate) struct Topology { + /// `-device` values for all expander bridges, ports and switches, each bus before the devices + /// on it. + pub(crate) devices: Vec, + /// Addresses of the `hostpcie` devices, keyed by index. + pub(crate) addresses: BTreeMap, + switches: u8, +} + +impl Topology { + /// Computes the topology of `hw`, which must not be [`HostpciTopology::Flat`]. + pub(crate) fn new( + hw: &HardwareConfig, + placements: &[HostpciePlacement], + ) -> Result { + if !hw.has_pcie() || hw.layout != PciLayout::Grouped { + return Err(LayoutError::UnsupportedTopology); + } + + let mut topology = Self { + devices: Vec::new(), + addresses: BTreeMap::new(), + switches: 0, + }; + let mut bus_nr = FIRST_NUMA_BUS; + for (numa_node, tree) in numa_nodes(hw, placements)? { + let start = topology.devices.len(); + let bus = Bus::Numa(numa_node); + let addr = PCIE_LAYOUT.find_device(&Device::Bridge(bus))?; + let params = format!(",bus_nr={bus_nr},numa_node={numa_node}"); + topology.devices.push(device(bus, addr, ¶ms)); + for (key, node) in tree.children() { + // keep slot 0 free, like on all other bridges + let addr = PciAddress::new(bus, key + 1, 0); + topology.add_port(addr, NUMA_CHASSIS_BASE + numa_node, &node); + } + + // each device we added creates exactly one bus + let buses = topology.devices.len() - start; + if bus_nr + buses > 256 { + return Err(LayoutError::BusNumbersExhausted); + } + bus_nr += buses.next_multiple_of(BUS_RANGE_ALIGN); + } + + Ok(topology) + } + + /// Adds a port for `node` at `addr`, plus everything behind it. On expander bridges these are + /// root ports, on switches downstream ports. + fn add_port(&mut self, addr: PciAddress, chassis: u8, node: &Node) { + let root_port = matches!(addr.bus, Bus::Numa(_)); + let id = match node { + Node::Device(index) if root_port => Bus::RootPort(*index), + Node::Device(index) => Bus::SwitchPort(*index), + Node::Switch(_) => Bus::SwitchUplink(self.switches), + }; + let slot = node.key(); + let mut params = format!(",port={slot},chassis={chassis},slot={slot}"); + if root_port { + params.push_str(",x-speed=16,x-width=32"); + } + self.devices.push(device(id, addr, ¶ms)); + + match node { + Node::Device(index) => { + self.addresses.insert(*index, PciAddress::new(id, 0, 0)); + } + Node::Switch(children) => { + let nr = self.switches; + self.switches += 1; + + let switch = Bus::Switch(nr); + self.devices + .push(device(switch, PciAddress::new(id, 0, 0), "")); + for (key, child) in children { + let addr = PciAddress::new(switch, *key, 0); + self.add_port(addr, SWITCH_CHASSIS_BASE + nr, child); + } + } + } + } +} + +#[cfg(test)] +mod test { + use super::{HostpciTopology, HostpciePlacement, Topology}; + use crate::Mainboard; + use crate::constants::{MAX_HOSTPCI_DEVICES, MAX_NUMA_NODES}; + use crate::machine::MachineVersion; + use crate::{ + Bus, Device, HardwareConfig, HardwareConfigParams, HostpcieDevice, LayoutError, PciAddress, + PciLayout, + }; + + fn hw_config(topology: HostpciTopology, placements: &[HostpciePlacement]) -> HardwareConfig { + let devices = placements.iter().map(|p| HostpcieDevice { index: p.index }); + HardwareConfig::new(HardwareConfigParams { + arch: crate::hardware_config::Arch::X8664, + machine: crate::Machine { + mainboard: Mainboard::Q35, + version: MachineVersion::new(10, 0, Some(1)), + }, + layout: PciLayout::Grouped, + hostpci_topology: topology, + hostpcie_devices: devices.collect(), + ..Default::default() + }) + } + + fn place(index: u8, numa_node: u8, switches: &[&str]) -> HostpciePlacement { + let switches = switches.iter().map(|s| s.to_string()).collect(); + HostpciePlacement { + index, + numa_node, + switches, + } + } + + fn devices(topology: HostpciTopology, placements: &[HostpciePlacement]) -> Vec { + let hw = hw_config(topology, placements); + Topology::new(&hw, placements).unwrap().devices + } + + #[test] + fn flat() { + let placements = [place(0, 1, &[])]; + let hw = hw_config(HostpciTopology::Flat, &placements); + assert_eq!(crate::hostpcie_bus_devices(&hw, &placements), Ok(vec![])); + + // the root ports are part of the bus devices and need no placements + let bus_devices = crate::pci_bus_devices(&hw).unwrap().join(" "); + assert!(bus_devices.contains("id=pve.root-port0,")); + assert_eq!( + crate::address(&hw, "hostpcie0"), + crate::address_with_placements(&hw, "hostpcie0", &placements), + ); + } + + #[test] + fn machines() { + // aarch64 virt shares the PCIe layout with q35 + let mut virt = hw_config(HostpciTopology::Numa, &[]); + virt.machine.mainboard = Mainboard::Virt; + assert!(Topology::new(&virt, &[]).is_ok()); + + let mut i440fx = hw_config(HostpciTopology::Numa, &[]); + i440fx.machine.mainboard = Mainboard::I440fx; + assert_eq!( + Topology::new(&i440fx, &[]).err(), + Some(LayoutError::UnsupportedTopology) + ); + + let mut legacy = hw_config(HostpciTopology::Numa, &[]); + legacy.layout = PciLayout::Legacy; + assert_eq!( + crate::hostpcie_bus_devices(&legacy, &[]).err(), + Some(LayoutError::UnsupportedTopology) + ); + } + + #[test] + fn numa() { + // switches are ignored in this mode + let placements = [place(0, 1, &["a"]), place(1, 0, &[]), place(2, 1, &["a"])]; + assert_eq!( + devices(HostpciTopology::Numa, &placements), + [ + "pxb-pcie,id=pve.numa0,bus=pcie.0,addr=0x10,bus_nr=32,numa_node=0", + "pcie-root-port,id=pve.root-port1,bus=pve.numa0,addr=0x2,port=1,chassis=128,slot=1,x-speed=16,x-width=32", + // node 0 needs 2 bus numbers, which get aligned to 16 + "pxb-pcie,id=pve.numa1,bus=pcie.0,addr=0x11,bus_nr=48,numa_node=1", + "pcie-root-port,id=pve.root-port0,bus=pve.numa1,addr=0x1,port=0,chassis=129,slot=0,x-speed=16,x-width=32", + "pcie-root-port,id=pve.root-port2,bus=pve.numa1,addr=0x3,port=2,chassis=129,slot=2,x-speed=16,x-width=32", + ] + ); + + let hw = hw_config(HostpciTopology::Numa, &placements); + assert_eq!( + crate::address_with_placements(&hw, "hostpcie2", &placements), + Ok(PciAddress::new(Bus::RootPort(2), 0, 0)) + ); + assert_eq!( + crate::address_with_placements(&hw, "hostpcie3", &placements), + Err(LayoutError::NoAddressFound(Device::Hostpcie(3))) + ); + // the address of hostpcie devices depends on the placements + assert_eq!( + crate::address(&hw, "hostpcie2"), + Err(LayoutError::MissingPlacement(2)) + ); + assert_eq!( + crate::address_with_placements(&hw, "hostpcie2", &placements[..2]), + Err(LayoutError::MissingPlacement(2)) + ); + + // the topology replaces the root ports on the root bus + let bus_devices = crate::pci_bus_devices(&hw).unwrap().join(" "); + assert!(!bus_devices.contains("pve.numa0")); + assert!(!bus_devices.contains("bus=pcie.0,port=")); + let hostpcie_devices = crate::hostpcie_bus_devices(&hw, &placements).unwrap(); + assert_eq!(hostpcie_devices.len(), 10); + assert_eq!(hostpcie_devices[0], "-device"); + assert!(hostpcie_devices[1].starts_with("pxb-pcie,id=pve.numa0,")); + + // other devices are not affected and need no placements + let flat = hw_config(HostpciTopology::Flat, &[]); + assert_eq!(crate::address(&hw, "net0"), crate::address(&flat, "net0")); + } + + #[test] + fn host() { + let placements = [ + // 'b', 'd' and 'e' only have a single device below them, so they're not used + place(4, 0, &["d", "e"]), + place(3, 0, &["b"]), + place(2, 0, &["a", "c"]), + place(1, 0, &["a", "c"]), + place(0, 0, &["a"]), + ]; + assert_eq!( + devices(HostpciTopology::Host, &placements), + [ + "pxb-pcie,id=pve.numa0,bus=pcie.0,addr=0x10,bus_nr=32,numa_node=0", + "pcie-root-port,id=pve.switch-uplink0,bus=pve.numa0,addr=0x1,port=0,chassis=128,slot=0,x-speed=16,x-width=32", + "x3130-upstream,id=pve.switch0,bus=pve.switch-uplink0,addr=0x0", + "xio3130-downstream,id=pve.switch-port0,bus=pve.switch0,addr=0x0,port=0,chassis=144,slot=0", + "xio3130-downstream,id=pve.switch-uplink1,bus=pve.switch0,addr=0x1,port=1,chassis=144,slot=1", + "x3130-upstream,id=pve.switch1,bus=pve.switch-uplink1,addr=0x0", + "xio3130-downstream,id=pve.switch-port1,bus=pve.switch1,addr=0x1,port=1,chassis=145,slot=1", + "xio3130-downstream,id=pve.switch-port2,bus=pve.switch1,addr=0x2,port=2,chassis=145,slot=2", + "pcie-root-port,id=pve.root-port3,bus=pve.numa0,addr=0x4,port=3,chassis=128,slot=3,x-speed=16,x-width=32", + "pcie-root-port,id=pve.root-port4,bus=pve.numa0,addr=0x5,port=4,chassis=128,slot=4,x-speed=16,x-width=32", + ] + ); + + let hw = hw_config(HostpciTopology::Host, &placements); + let address = |id| crate::address_with_placements(&hw, id, &placements); + assert_eq!( + address("hostpcie0"), + Ok(PciAddress::new(Bus::SwitchPort(0), 0, 0)) + ); + assert_eq!( + address("hostpcie2"), + Ok(PciAddress::new(Bus::SwitchPort(2), 0, 0)) + ); + assert_eq!( + address("hostpcie4"), + Ok(PciAddress::new(Bus::RootPort(4), 0, 0)) + ); + } + + #[test] + fn invalid_placements() { + let check = |placements: &[HostpciePlacement], index: u8| { + let hw = hw_config(HostpciTopology::Host, placements); + assert_eq!( + Topology::new(&hw, placements).err(), + Some(LayoutError::InvalidPlacement(index)) + ); + }; + + check(&[place(MAX_HOSTPCI_DEVICES, 0, &[])], MAX_HOSTPCI_DEVICES); + check(&[place(1, MAX_NUMA_NODES, &[])], 1); + // the same switch on different nodes, below different switches or twice in one path + check(&[place(0, 0, &["a"]), place(1, 1, &["a"])], 1); + check(&[place(0, 0, &["a", "b"]), place(1, 0, &["b"])], 1); + check(&[place(0, 0, &["a", "a"])], 0); + } + + #[test] + fn bus_numbers_always_fit() { + // all devices on one node, below a binary tree of switches + let tree: Vec<_> = (0..MAX_HOSTPCI_DEVICES) + .map(|index| HostpciePlacement { + index, + numa_node: 0, + switches: (0..=4) + .map(|depth| format!("{depth}-{}", index >> (4 - depth))) + .collect(), + }) + .collect(); + let switches = devices(HostpciTopology::Host, &tree) + .iter() + .filter(|d| d.starts_with("x3130-upstream")) + .count(); + assert_eq!(switches, usize::from(MAX_HOSTPCI_DEVICES) - 1); + + // pairs of devices behind a switch on every node + let pairs: Vec<_> = (0..MAX_HOSTPCI_DEVICES) + .map(|index| HostpciePlacement { + index, + numa_node: index / 2, + switches: vec![format!("{}", index / 2)], + }) + .collect(); + let pxb = "pxb-pcie,id=pve.numa7,bus=pcie.0,addr=0x17,bus_nr=144,numa_node=7"; + assert!( + devices(HostpciTopology::Host, &pairs) + .iter() + .any(|d| d == pxb) + ); + } +} diff --git a/pve-qemu-rs/pve-qemu-perl/src/pci/hardware_config.rs b/pve-qemu-rs/pve-qemu-perl/src/pci/hardware_config.rs index a65aeac8..c5ca6ef0 100644 --- a/pve-qemu-rs/pve-qemu-perl/src/pci/hardware_config.rs +++ b/pve-qemu-rs/pve-qemu-perl/src/pci/hardware_config.rs @@ -14,7 +14,9 @@ pub mod pve_qemu_rs_pci_hardware_config { use perlmod::Value; - use pve_qemu_pci::{HardwareConfigParams, LayoutError, address}; + use pve_qemu_pci::{ + HardwareConfigParams, HostpciePlacement, LayoutError, address, address_with_placements, + }; perlmod::declare_magic!(Box: &HardwareConfig as "PVE::QemuServer::RS::PCI::HardwareConfig"); @@ -43,11 +45,19 @@ pub mod pve_qemu_rs_pci_hardware_config { #[export] /// Method: Returns the PCI or PCIe address of the chosen device on the given machine if the /// device exists on the layout. + /// + /// With a hostpci topology other than 'flat', `hostpcie` devices need the host placements of + /// all `hostpcie` devices, see `hostpcie_bus_devices`. pub fn addr( #[try_from_ref] this: &HardwareConfig, id: &str, + placements: Option>, ) -> Result, LayoutError> { - match address(this, id) { + let addr = match placements { + Some(placements) => address_with_placements(this, id, &placements), + None => address(this, id), + }; + match addr { Ok(addr) => Ok(Some(addr.to_qemu_addr())), Err(LayoutError::NoAddressFound(_)) => Ok(None), Err(other) => Err(other), @@ -63,6 +73,19 @@ pub mod pve_qemu_rs_pci_hardware_config { pve_qemu_pci::pci_bus_devices(this) } + #[export] + /// Method: Get the `-device` arguments of the expander bridges, ports and switches of a hostpci + /// topology other than 'flat', for the given host placements of the `hostpcie` devices. + /// + /// They have to come after the `pci_bus_devices` and before the `hostpcie` devices. For the + /// 'flat' topology this is empty, since its root ports are part of `pci_bus_devices`. + pub fn hostpcie_bus_devices( + #[try_from_ref] this: &HardwareConfig, + placements: Vec, + ) -> Result, LayoutError> { + pve_qemu_pci::hostpcie_bus_devices(this, &placements) + } + #[export] /// Method: Whether the q35 chipset devices still have to be loaded from `pve-q35-4.0.cfg`. pub fn uses_q35_config_file(#[try_from_ref] this: &HardwareConfig) -> bool { diff --git a/src/PVE/QemuServer/PCI.pm b/src/PVE/QemuServer/PCI.pm index 4a4d2531..ece4ed6d 100644 --- a/src/PVE/QemuServer/PCI.pm +++ b/src/PVE/QemuServer/PCI.pm @@ -731,6 +731,7 @@ sub hardware_config { 'ostype' => $conf->{ostype} // 'other', 'layout' => $machine_conf->{'pci-layout'} // 'legacy', 'hostpcie-devices' => $hostpcie, + 'hostpci-topology' => 'flat', }); } -- 2.47.3