From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 4633C1FF0B4 for ; Tue, 10 Nov 2026 14:51:15 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id E15F5216DF; Tue, 10 Nov 2026 14:50:39 +0100 (CET) From: Hannes Laimer To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox v9 3/5] ebpf: subsystem: add the shared tc program handling Date: Tue, 10 Nov 2026 14:50:03 +0100 Message-ID: <20261110135005.2847847-4-h.laimer@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261110135005.2847847-1-h.laimer@proxmox.com> References: <20261110135005.2847847-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794318620978 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.294 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: CUTLXXWSKMUE6JBKNDIYQ3YH6ADJZSRR X-Message-ID-Hash: CUTLXXWSKMUE6JBKNDIYQ3YH6ADJZSRR X-MailFrom: h.laimer@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A subsystem names its tc programs and maps. The crate loads them, attaches the programs to interfaces and pins everything in bpffs. The pins keep the programs running after the process exits, and a later process finds them through the pins. The programs come from an object file under /usr/lib. The object carries a schema version in an ELF section, and a process only loads it if that is the version it was built with. Maps and programs are pinned under that version too, so a process never opens maps of another layout. A subsystem either changes single map entries or replaces everything. A replace loads the object with new maps and fills them in a staging directory. Only then does each link switch to the new program, which already uses the new maps. If the fill fails, the old maps stay as they are, and no program sees a mix of old and new entries. The staging directory replaces the live one after the last link has switched, so if a live directory exists, every link uses it. Changes run under one lock per subsystem. A replace takes it exclusively, and a change to one entry or one interface takes it shared. A caller reads its input under the lock, so a replace and the changes around it reach the kernel in the order they read their input. Signed-off-by: Hannes Laimer --- Cargo.toml | 1 + proxmox-ebpf/Cargo.toml | 5 + proxmox-ebpf/src/lib.rs | 4 +- proxmox-ebpf/src/subsystem.rs | 1058 +++++++++++++++++++++++++++++++++ proxmox-ebpf/src/tc.rs | 17 +- 5 files changed, 1077 insertions(+), 8 deletions(-) create mode 100644 proxmox-ebpf/src/subsystem.rs diff --git a/Cargo.toml b/Cargo.toml index 681236207e0b..f0d88f607e16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ md5 = "0.7.0" native-tls = "0.2" nix = "0.29" nom = "7" +object = { version = "0.36", default-features = false } # used by proxmox-disks, can be replaced by OnceLock from std once it supports get_or_try_init once_cell = "1.3.1" openssl = "0.10" diff --git a/proxmox-ebpf/Cargo.toml b/proxmox-ebpf/Cargo.toml index de974f7a34a0..4ac7a7586a84 100644 --- a/proxmox-ebpf/Cargo.toml +++ b/proxmox-ebpf/Cargo.toml @@ -14,3 +14,8 @@ anyhow.workspace = true aya.workspace = true log.workspace = true nix = { workspace = true, features = ["fs"] } +object = { workspace = true, features = ["elf", "read_core", "std"] } +proxmox-sys = { workspace = true, features = ["timer"] } + +[dev-dependencies] +object = { workspace = true, features = ["write"] } diff --git a/proxmox-ebpf/src/lib.rs b/proxmox-ebpf/src/lib.rs index b9babe319a41..7be1c7475d09 100644 --- a/proxmox-ebpf/src/lib.rs +++ b/proxmox-ebpf/src/lib.rs @@ -1,5 +1,7 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] //! The shared code of eBPF subsystems, one subsystem per concern. Nothing here runs on its own, -//! a subsystem describes its tc programs and calls into this crate. +//! a subsystem describes its tc programs with a [`TcPrograms`](subsystem::TcPrograms) and calls +//! into this crate. +pub mod subsystem; pub mod tc; diff --git a/proxmox-ebpf/src/subsystem.rs b/proxmox-ebpf/src/subsystem.rs new file mode 100644 index 000000000000..ed17537e078b --- /dev/null +++ b/proxmox-ebpf/src/subsystem.rs @@ -0,0 +1,1058 @@ +//! The shared subsystem code. A subsystem is described by one [`TcPrograms`], and every subsystem +//! has its own directory in the bpffs. Its maps are pinned by name inside it, so two subsystems +//! never share a map, even when the names are the same. For [`TcPrograms`] they can specify one +//! program per [`tc::Direction`], both programs have access to the same maps. +//! +//! Programs, maps and links are all pinned in a per-subsystem directory under +//! ```text +//! /sys/fs/bpf/proxmox-ebpf// +//! ``` +//! Links are under `links`, and their filenames follow the format described in [`tc::LinkPin`]. +//! Maps and programs live in a `//` directory, programs access maps to read and +//! write data, so the shape a map has always has to match what the programs assume it to be. +//! Program object files have the schema version they were compiled with written into an ELF +//! section, and the subsystem's Rust code passes its own version to [`TcPrograms::new`]. Programs +//! only get pinned in the directory of their own version, the same way this library only ever +//! writes/reads maps under the schema directory of that version. This means map layout changes +//! *have* to bump the version passed to [`TcPrograms::new`], and the schema version of the BPF +//! programs that use them. They have to match. A change to the programs given to +//! [`TcPrograms::new`] needs a bump as well. A process still running with the old ones would +//! otherwise detach the links of a direction it has no program for on its next full apply. +//! ```text +//! links/ +//! 4:ingress:tap8999i0 :: +//! ... +//! 1/ +//! live/ +//! records , always complete state +//! prog/ +//! ingress +//! staging/ only during a full apply, or after one stopped partway +//! records +//! prog/ +//! ingress +//! old/ the previous live/, only while it is being removed +//! ``` +//! Writes to maps are either single entry updates that can be done atomically, or writes that +//! replace the whole map. For updates that involve more than a single write we write all the new +//! data into a new map under `staging/`, next to `live/`. This makes sure we don't have a map that +//! contains only half of an update, the single entries are still fine, but the map as a whole may +//! be in a state that neither the update nor the state before intended. Once all writes succeeded, +//! we rename `live/` to `old/` and remove it there. The rename is a single step, so `live/` is +//! either complete or gone, even if the process dies right then. Then we swap all the links over to +//! the new state we just wrote, this has to be done one-by-one, but is atomic. So a link either has +//! the old program with the complete old state, or the new program with the complete new state. +//! Only once all links moved over to the new state we rename staging to `live/`. New program does +//! not necessarily mean it is a different one, to use a new map though, we have to load the +//! programs again, and only then can they use the new map. +//! +//! A program update without a schema version bump goes through the same steps. Every full apply +//! loads the shipped program again, whether it was changed or not. So the first install, a new +//! program or a (not single entry) update to the maps all take this path. With the same schema +//! version the caller can read the old `live/` maps before the apply, to build the new content from +//! them. +//! +//! We have these pins, so everything outlives the process that put it there. And other processes +//! can continue working on state another process set up. The program and map pins are only another +//! reference next to the ones the links hold, a link holds its program and a program its maps. So +//! just removing those pins does nothing to the interfaces. The kernel only drops a program or map +//! once its last reference disappears. A link pin is different, removing it detaches the link. This +//! means deleting a schema-version dir makes the state it was for immutable for this library. Only +//! removing a link, or swapping a new program and maps onto it is possible. Interfaces with old +//! programs continue to run on the old version until the next full apply, which puts every link +//! onto the new programs or drops it. +//! +//! The first thing every full apply does is check that the schema version passed to +//! [`TcPrograms::new`] matches what the installed program object file has in its ELF section. In +//! case of a mismatch we're either stale, or the package shipping the program objects has not been +//! updated yet. Whatever the case, we don't load programs that assume a different schema version +//! than ours, and leave everything as it is. Then we remove all schema-version directories that do +//! not match our version. This makes sure processes still running another schema version have their +//! single entry updates and attaches fail with [`NotLoaded`], and their detaches do nothing. +//! +//! The programs and maps come from an object file that is shipped as a package of its own, so a +//! program can be updated without building this library. +//! ```text +//! /usr/lib/proxmox-ebpf/ +//! dhcp.bpf.o .bpf.o +//! ``` +//! The object has to follow a few rules for us to load it. Every program is a function in the +//! `classifier` section, under the name given to [`TcPrograms::new`]. Every map the subsystem lists +//! or opens is pinned by name, so it ends up in the directory we load into. The schema version is a +//! `__u32` in the `.proxmox.schema` section, in the byte order of the host. +//! ```c +//! const __u32 proxmox_schema SEC(".proxmox.schema") = 1; +//! +//! struct { +//! __uint(type, BPF_MAP_TYPE_HASH); +//! __type(key, __u32); +//! __type(value, struct record); +//! __uint(max_entries, 1024); +//! __uint(pinning, LIBBPF_PIN_BY_NAME); +//! } records SEC(".maps"); +//! +//! SEC("classifier") +//! int tc_ingress(struct __sk_buff *skb) +//! ``` +//! A full apply is [`ApplyLock::replace_state`]. The caller hands it the interfaces that should +//! have the programs and a `fill` function that writes the fresh maps, and it runs the steps above. +//! Links of interfaces the caller does not name, or whose interfaces disappeared, are detached +//! after the new maps are filled and before the programs of the existing links are swapped. +//! Interfaces the caller names that have no link yet are attached after `staging/` was renamed to +//! `live/`. +//! +//! Everything else works on single entries or on links and never reads the object. +//! [`ApplyLock::hash_map`] opens a map in `live/` to change entries one at a time, +//! [`ApplyLock::attach_iface`] attaches one interface, and [`ApplyLock::detach_named`] and +//! [`ApplyLock::detach_gone`] detach links again. Without `live/` the calls that need the programs +//! or maps return [`NotLoaded`], and only a full apply creates it again. +//! [`ApplyLock::clear`] removes everything the subsystem pinned, but like a full apply only when +//! the shipped object carries our schema version. +//! +//! Every change runs under the subsystem's apply lock, a file under `/run`. +//! ```text +//! /run/proxmox-ebpf/ +//! dhcp/lock /lock, the apply lock +//! ``` +//! A full apply and [`ApplyLock::clear`] take it exclusively, everything else takes it shared. The +//! caller reads the input of its change under the lock and writes it before it lets go, so changes +//! reach the kernel in the order they read their input. Two shared holders working on the same +//! interface can still write in either order. Waiting for the lock fails after 10 seconds, or +//! earlier when a signal handler without `SA_RESTART` runs. +use std::{ + cmp::Ordering, + collections::HashSet, + fs::File, + io::ErrorKind, + marker::PhantomData, + ops::{Deref, DerefMut}, + os::unix::fs::DirBuilderExt, + path::{Path, PathBuf}, + time::Duration, +}; + +use anyhow::{Context, Error, bail}; +use aya::{EbpfLoader, maps::MapType, programs::SchedClassifier}; +use nix::sys::stat::Mode; +use object::{Object, ObjectSection}; +use proxmox_sys::fs::{CreateOptions, open_file_locked}; + +use crate::tc::{self, Direction}; + +const PIN_ROOT: &str = "/sys/fs/bpf/proxmox-ebpf"; +const RUN_ROOT: &str = "/run/proxmox-ebpf"; +const OBJECT_DIR: &str = "/usr/lib/proxmox-ebpf"; +const SCHEMA_SECTION: &str = ".proxmox.schema"; + +// flock works on a read-only descriptor, so anyone who can open a lock file could hold it and block +// every change +fn private_dir(dir: &Path) -> std::io::Result<()> { + std::fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(dir) +} + +// a wait times out instead of hanging, and a signal handler without SA_RESTART ends it too +fn open_locked(path: &Path, exclusive: bool, what: &str, subsystem: &str) -> Result { + let options = CreateOptions::new().perm(Mode::from_bits_truncate(0o600)); + open_file_locked(path, Duration::from_secs(10), exclusive, options) + .with_context(|| format!("{subsystem}: take {what}")) +} + +/// A call that needs the programs or maps found none pinned. Only [`ApplyLock::replace_state`] pins +/// them. +#[derive(Debug)] +pub struct NotLoaded; + +impl std::fmt::Display for NotLoaded { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("programs not loaded") + } +} + +impl std::error::Error for NotLoaded {} + +/// A subsystem's description of its tc programs. The loaded programs live pinned in the kernel and +/// are reached through those pins. +pub struct TcPrograms { + name: &'static str, + programs: &'static [(Direction, &'static str)], + maps: &'static [&'static str], + schema_version: u32, +} + +impl TcPrograms { + /// Creates a new `TcPrograms`, describing its programs with their direction, their maps, the + /// name and current schema version. + /// + /// # Example + /// ``` + /// # use proxmox_ebpf::{subsystem::TcPrograms, tc::Direction}; + /// // loads /usr/lib/proxmox-ebpf/dhcp.bpf.o, whose `.proxmox.schema` section has to hold 1 + /// static PROGRAMS: TcPrograms = TcPrograms::new( + /// "dhcp", + /// &[(Direction::Ingress, "tc_ingress")], + /// &["records"], + /// 1, + /// ); + /// ``` + pub const fn new( + name: &'static str, + programs: &'static [(Direction, &'static str)], + maps: &'static [&'static str], + schema_version: u32, + ) -> Self { + Self { + name, + programs, + maps, + schema_version, + } + } + + fn object_path(&self) -> PathBuf { + PathBuf::from(OBJECT_DIR).join(format!("{}.bpf.o", self.name)) + } + + fn pin_root(&self) -> PathBuf { + PathBuf::from(PIN_ROOT).join(self.name) + } + + fn schema_root(&self) -> PathBuf { + self.pin_root().join(self.schema_version.to_string()) + } + + fn live_dir(&self) -> PathBuf { + self.schema_root().join("live") + } + + fn staging_dir(&self) -> PathBuf { + self.schema_root().join("staging") + } + + fn old_dir(&self) -> PathBuf { + self.schema_root().join("old") + } + + fn links_dir(&self) -> PathBuf { + self.pin_root().join(LINKS_DIR) + } + + fn link_pin_path(&self, pin: &tc::LinkPin) -> PathBuf { + self.links_dir().join(pin.filename()) + } + + fn prog_dir_in(&self, pin_dir: &Path) -> PathBuf { + pin_dir.join("prog") + } + + fn program_name(&self, dir: Direction) -> Option<&'static str> { + self.programs + .iter() + .find(|&&(d, _)| d == dir) + .map(|&(_, name)| name) + } + + fn run_dir(&self) -> PathBuf { + PathBuf::from(RUN_ROOT).join(self.name) + } + + fn hash_map( + &self, + name: &str, + ) -> Result, Error> { + self.hash_map_in(&self.live_dir(), name) + } + + fn hash_map_in( + &self, + pin_dir: &Path, + name: &str, + ) -> Result, Error> { + let path = pin_dir.join(name); + if !path.exists() { + // a pin dir is created with every map the object pins, so an existing one lacks only + // a map the object never had under that name + if pin_dir.exists() { + bail!( + "{}: no map {name} pinned in {}", + self.name, + pin_dir.display() + ); + } + return Err(NotLoaded.into()); + } + let map = aya::maps::MapData::from_pin(path) + .with_context(|| format!("{}: open pinned map {name}", self.name))?; + // aya only compares the key and value sizes from here on. A lookup in a per-CPU map would + // have the kernel write one value per CPU into a buffer for a single one + let map_type = map + .info() + .and_then(|info| info.map_type()) + .with_context(|| format!("{}: read the type of map {name}", self.name))?; + if !matches!(map_type, MapType::Hash | MapType::LruHash) { + bail!("{}: map {name} is not a hash map ({map_type:?})", self.name); + } + Ok(aya::maps::HashMap::try_from(aya::maps::Map::HashMap(map))?) + } + + fn program(&self, dir: Direction) -> Result { + let path = self.prog_dir_in(&self.live_dir()).join(dir.as_str()); + if !path.exists() { + return Err(NotLoaded.into()); + } + SchedClassifier::from_pin(path) + .with_context(|| format!("{}: open pinned program {}", self.name, dir.as_str())) + } + + // attaching interfaces without a pause could starve a writer waiting for an exclusive lock, + // such a writer times out. Writes to the pinned state are methods on the returned guard, so + // they can't go around the lock. + fn lock(&self, exclusive: bool) -> Result, Error> { + let dir = self.run_dir(); + private_dir(&dir).with_context(|| { + format!("{}: create the run directory {}", self.name, dir.display()) + })?; + let lock = open_locked(&dir.join("lock"), exclusive, "apply lock", self.name)?; + Ok(ApplyLock { + programs: self, + _lock: lock, + _mode: PhantomData, + }) + } + + /// Check if the link of the interface name is pinned. Fails if the links in links/ could not be + /// listed. + pub fn is_link_pinned(&self, name: &str) -> Result { + let links = self.live_links()?; + Ok(links.iter().any(|pin| pin.name == name)) + } + + /// Take the apply lock shared, for every change except a full apply or [`ApplyLock::clear`]. + /// + /// Use for single entry updates that don't depend on each other. + pub fn lock_shared(&self) -> Result, Error> { + self.lock(false) + } + + /// Take the apply lock exclusively, for a full apply or [`ApplyLock::clear`]. + /// + /// Timeout is 10s, like for the shared variant. + pub fn lock_exclusive(&self) -> Result, Error> { + self.lock(true) + } + + // check that we have all the programs and maps pinned under the correct schema-version live/ + // dir + fn is_current(&self) -> bool { + let dir = self.prog_dir_in(&self.live_dir()); + self.programs + .iter() + .all(|&(d, _)| dir.join(d.as_str()).exists()) + && self.maps_pinned() + } + + fn maps_pinned(&self) -> bool { + self.maps + .iter() + .all(|map| self.live_dir().join(map).exists()) + } + + // finds schema-version dirs for a version different than ours. Includes dirs that are not named + // after a schema version + fn other_schemas(&self) -> Result, Error> { + let entries = match std::fs::read_dir(self.pin_root()) { + Ok(entries) => entries, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e.into()), + }; + let mut others = Vec::new(); + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_dir() || entry.file_name() == LINKS_DIR { + continue; + } + let schema = entry + .file_name() + .to_str() + .and_then(|s| s.parse::().ok()); + if schema != Some(self.schema_version) { + others.push(entry.path()); + } + } + Ok(others) + } + + // take `live/` away in one step and remove it under its new name. Removed in place, a process + // that dies partway would leave a `live/` that lacks some of its pins + fn retire_live(&self) -> Result<(), Error> { + let old = self.old_dir(); + match std::fs::rename(self.live_dir(), &old) { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(e).with_context(|| format!("{}: rename live/ to old/", self.name)); + } + } + // `live/` is out of the way by now, a leftover `old/` only keeps the old maps and programs + // pinned until the next full apply removes it + if let Err(e) = std::fs::remove_dir_all(&old) { + log::warn!("{}: remove old/: {e:#}", self.name); + } + Ok(()) + } + + fn tear_down(&self) -> Result<(), Error> { + // `live/` goes first, the same way a full apply takes it away + remove_dir_if_present(&self.old_dir()) + .with_context(|| format!("{}: remove the leftover old/", self.name))?; + self.retire_live()?; + remove_dir_if_present(&self.pin_root()) + .with_context(|| format!("{}: remove the pinned state", self.name)) + } + + // load the object with all of its programs, create its maps and pin them under `pin_dir` + fn load_and_pin_maps(&self, pin_dir: &Path, object: &[u8]) -> Result { + let mut bpf = EbpfLoader::new() + .map_pin_path(pin_dir) + .load(object) + .with_context(|| format!("{}: load the object", self.name))?; + for map in self.maps { + if !pin_dir.join(map).exists() { + bail!("{}: the object does not pin a map {map}", self.name); + } + } + for &(_, name) in self.programs { + let prog: &mut SchedClassifier = bpf + .program_mut(name) + .with_context(|| format!("{}: program {name} not found in object", self.name))? + .try_into()?; + prog.load() + .with_context(|| format!("{}: load program {name}", self.name))?; + } + Ok(bpf) + } + + // swap all links under links/ onto the programs in `bpf`, the ones just pinned under `staging/` + fn swap_links(&self, bpf: &mut aya::Ebpf) -> Result<(), Error> { + let live = self.live_links()?; + for &(dir, _) in self.programs { + self.swap_links_onto(dir, self.program_in(bpf, dir)?, &live)?; + } + Ok(()) + } + + // swap the links of one direction onto `prog`. One that cannot be swapped is detached + fn swap_links_onto( + &self, + dir: Direction, + prog: &mut SchedClassifier, + links: &[tc::LinkPin], + ) -> Result<(), Error> { + for pin in links.iter().filter(|pin| pin.dir == dir) { + let path = self.link_pin_path(pin); + // a link pin that disappeared is ignored, we just care about existing ones being on the + // program we expect them to be on. Links that disappeared, but should exist are added + // again by `attach_missing`. Other errors are reported, since they indicate something + // being wrong with the system/process that we can't work around. + let link = match tc::open_pinned_link(&path) { + Ok(link) => link, + Err(tc::Error::PinGone) => continue, + Err(e) => { + return Err(e) + .with_context(|| format!("{}: open {}", self.name, pin.filename())); + } + }; + if let Err(e) = tc::swap_link(prog, link) { + log::warn!( + "{}: {} swap failed ({e:#}), detaching", + self.name, + pin.filename() + ); + self.detach_link(pin)?; + } + } + Ok(()) + } + + fn pin_programs(&self, bpf: &mut aya::Ebpf, pin_dir: &Path) -> Result<(), Error> { + let dir = self.prog_dir_in(pin_dir); + std::fs::create_dir_all(&dir) + .with_context(|| format!("{}: create the program directory", self.name))?; + for &(direction, _) in self.programs { + self.program_in(bpf, direction)? + .pin(dir.join(direction.as_str())) + .with_context(|| format!("{}: pin program {}", self.name, direction.as_str()))?; + } + Ok(()) + } + + fn read_object_checked(&self) -> Result, Error> { + let path = self.object_path(); + let object = std::fs::read(&path) + .with_context(|| format!("{}: read the object {}", self.name, path.display()))?; + let schema = object_schema(&object) + .with_context(|| format!("{}: read the schema of {}", self.name, path.display()))?; + check_schema(schema, self.schema_version) + .with_context(|| format!("{}: check the schema of {}", self.name, path.display()))?; + Ok(object) + } + + // Creates fresh maps under `staging/` and fills them using `fill`. `desired` is the set of + // indexes of the interfaces we want the program attached to. + // + // After the complete data is written under `staging/`, `live/` is deleted and existing links + // are swapped to this new state. When done `staging/` is renamed to `live/`. + fn replace_state( + &self, + desired: &HashSet, + fill: impl FnOnce(&Staged<'_>) -> Result<(), Error>, + ) -> Result<(), Error> { + let object = self.read_object_checked()?; + let others = self + .other_schemas() + .with_context(|| format!("{}: list the schema directories", self.name))?; + for dir in others { + if let Err(e) = std::fs::remove_dir_all(&dir) { + log::warn!( + "{}: remove the pinned state {}: {e:#}", + self.name, + dir.display() + ); + } + } + std::fs::create_dir_all(self.links_dir()) + .with_context(|| format!("{}: create the links directory", self.name))?; + let staging = self.staging_dir(); + let live = self.live_dir(); + remove_dir_if_present(&staging) + .with_context(|| format!("{}: remove the previous staging directory", self.name))?; + // left behind when the removal after the rename of `live/` did not finish + remove_dir_if_present(&self.old_dir()) + .with_context(|| format!("{}: remove the leftover old/", self.name))?; + let mut bpf = match self.setup_staging(&staging, &object, fill) { + Ok(bpf) => bpf, + Err(e) => { + // the pins would keep the fresh maps alive + if let Err(err) = std::fs::remove_dir_all(&staging) { + log::warn!("{}: remove staging/: {err:#}", self.name); + } + return Err(e); + } + }; + self.remove_foreign_pins() + .with_context(|| format!("{}: remove the pins it cannot use", self.name))?; + self.detach_unwanted(desired)?; + self.retire_live()?; + self.swap_links(&mut bpf)?; + std::fs::rename(&staging, &live) + .with_context(|| format!("{}: rename staging/ to live/", self.name))?; + self.attach_missing(&mut bpf, desired) + } + + // drop the links not in `desired` and the pins of interfaces that are gone + fn detach_unwanted(&self, desired: &HashSet) -> Result<(), Error> { + for pin in self.live_links()? { + if link_alive(&pin)? && desired.contains(&pin.ifindex) { + continue; + } + log::info!("{}: detach {}", self.name, pin.filename()); + if let Err(e) = self.detach_link(&pin) { + log::warn!("{e:#}"); + } + } + Ok(()) + } + + fn attach_missing(&self, bpf: &mut aya::Ebpf, desired: &HashSet) -> Result<(), Error> { + let mut attached = HashSet::new(); + for pin in self.live_links()? { + if link_alive(&pin)? { + attached.insert((pin.ifindex, pin.dir)); + } + } + let mut failed = 0usize; + for &(dir, _) in self.programs { + let prog = self.program_in(bpf, dir)?; + for &ifindex in desired { + if attached.contains(&(ifindex, dir)) { + continue; + } + let Some(name) = tc::iface_name(ifindex)? else { + log::info!("{}: {ifindex} vanished before the attach", self.name); + continue; + }; + let pin = tc::LinkPin { ifindex, name, dir }; + match tc::attach_and_pin(prog, &pin, &self.link_pin_path(&pin)) { + Ok(()) => {} + Err(tc::Error::IfaceGone) => { + log::info!("{}: {ifindex} vanished before the attach", self.name); + } + Err(e) => { + log::error!("{}: attach {}: {e:#}", self.name, pin.filename()); + failed += 1; + } + } + } + } + if failed > 0 { + bail!("{}: {failed} link(s) failed to attach", self.name); + } + Ok(()) + } + + fn setup_staging( + &self, + staging: &Path, + object: &[u8], + fill: impl FnOnce(&Staged<'_>) -> Result<(), Error>, + ) -> Result { + std::fs::create_dir_all(staging) + .with_context(|| format!("{}: create the staging directory", self.name))?; + let mut bpf = self.load_and_pin_maps(staging, object)?; + fill(&Staged { programs: self })?; + self.pin_programs(&mut bpf, staging)?; + Ok(bpf) + } + + fn program_in<'b>( + &self, + bpf: &'b mut aya::Ebpf, + dir: Direction, + ) -> Result<&'b mut SchedClassifier, Error> { + let name = self + .program_name(dir) + .expect("only directions with a program are loaded"); + let prog = bpf + .program_mut(name) + .expect("every program was resolved by the load loop") + .try_into()?; + Ok(prog) + } + + fn live_links(&self) -> Result, Error> { + tc::read_pinned_links(&self.links_dir()) + .with_context(|| format!("{}: list the links", self.name)) + } + + fn remove_foreign_pins(&self) -> Result<(), Error> { + for entry in std::fs::read_dir(self.links_dir()).context("list the links")? { + let entry = entry.context("list the links")?; + let name = entry.file_name(); + let ours = name + .to_str() + .and_then(tc::LinkPin::parse) + .is_some_and(|pin| self.program_name(pin.dir).is_some()); + if ours { + continue; + } + let name = name.to_string_lossy(); + log::warn!( + "{}: removing pin {name} this subsystem cannot use", + self.name + ); + let path = entry.path(); + if let Err(e) = tc::detach_pinned_link(&path) { + log::warn!("{}: detach {name}: {e:#}", self.name); + // a pin we cannot use keeps a link running that nothing updates, so it goes even + // when it does not open + let _ = std::fs::remove_file(&path); + } + } + Ok(()) + } + + fn detach_named( + &self, + name: &str, + keep: impl Fn(&tc::LinkPin) -> bool, + ) -> Result, Error> { + let mut ifindexes = Vec::new(); + for pin in self.live_links()? { + if pin.name != name || keep(&pin) { + continue; + } + self.detach_link(&pin)?; + if !ifindexes.contains(&pin.ifindex) { + ifindexes.push(pin.ifindex); + } + } + Ok(ifindexes) + } + + fn detach_gone(&self) -> Result, Error> { + let mut gone = Vec::new(); + for pin in self.live_links()? { + // an index a new netdev took is not gone, the state under it may be the new one's + if tc::iface_name(pin.ifindex)?.is_some() { + continue; + } + self.detach_link(&pin)?; + if !gone.contains(&pin.ifindex) { + gone.push(pin.ifindex); + } + } + Ok(gone) + } + + // detach a link through its pin. Pins that cannot be opened stay, a failed open says nothing + // about the link behind it + fn detach_link(&self, pin: &tc::LinkPin) -> Result<(), Error> { + let path = self.link_pin_path(pin); + match tc::detach_pinned_link(&path) { + Ok(()) => Ok(()), + // a lost race, another attach or detach took the pin first + Err(tc::Error::PinGone) => { + log::debug!("{}: detach {}: pin gone already", self.name, pin.filename()); + Ok(()) + } + Err(e) => Err(e).with_context(|| format!("{}: detach {}", self.name, pin.filename())), + } + } + + fn attach_iface(&self, ifindex: u32) -> Result<(), Error> { + // opened for every direction ahead of the pin checks, so a missing program fails here, and + // never counts as a dead link and leaves no direction attached alone + let mut progs = Vec::new(); + for &(dir, _) in self.programs { + progs.push((dir, self.program(dir)?)); + } + let Some(name) = tc::iface_name(ifindex)? else { + log::info!("{}: {ifindex} vanished before the attach", self.name); + return Ok(()); + }; + let live = self.live_links()?; + for (dir, prog) in &mut progs { + let dir = *dir; + let pin = tc::LinkPin { + ifindex, + name: name.clone(), + dir, + }; + let path = self.link_pin_path(&pin); + for stale in live.iter().filter(|p| p.ifindex == ifindex && p.dir == dir) { + if stale.name == name { + let link = match tc::open_pinned_link(&path) { + Ok(link) => link, + // gone since listing, attached again by `attach_link` further down, if + // another attach was faster, we just find the one it installed and leave + // it + Err(tc::Error::PinGone) => { + log::debug!( + "{}: {} pin gone since the listing", + self.name, + path.display() + ); + continue; + } + Err(e) => { + return Err(e).with_context(|| { + format!("{}: open {}", self.name, pin.filename()) + }); + } + }; + if let Err(e) = tc::swap_link(prog, link) { + log::warn!( + "{}: {} swap failed ({e:#}), rebuilding", + self.name, + pin.filename() + ); + self.detach_link(stale)?; + continue; + } + } else { + // a pin with this index but a different name is left from before a rename, + // or from a deleted interface whose index got reused. After a rename its link + // still runs here and the kernel would refuse the new one, so it goes + if let Err(e) = self.detach_link(stale) { + log::warn!("{e:#}"); + } + } + } + if path.exists() { + continue; + } + match tc::attach_and_pin(prog, &pin, &path) { + Ok(()) => {} + Err(tc::Error::IfaceGone) => { + log::info!("{}: {ifindex} vanished before the attach", self.name); + } + Err(tc::Error::AlreadyAttached) => { + log::debug!("{}: {} attached concurrently", self.name, pin.filename()); + } + Err(e) => { + return Err(e) + .with_context(|| format!("{}: attach {}", self.name, pin.filename())); + } + } + } + Ok(()) + } +} + +const LINKS_DIR: &str = "links"; + +fn object_schema(bytes: &[u8]) -> Result { + let file = object::File::parse(bytes).context("not an object")?; + let section = file + .section_by_name(SCHEMA_SECTION) + .with_context(|| format!("no section {SCHEMA_SECTION}"))?; + let data = section + .data() + .with_context(|| format!("read section {SCHEMA_SECTION}"))?; + let schema: [u8; 4] = data + .try_into() + .with_context(|| format!("section {SCHEMA_SECTION} has {} bytes, not 4", data.len()))?; + Ok(u32::from_ne_bytes(schema)) +} + +// the pin path comes from the process's schema, so an object of another schema would pin maps of +// its layout under the wrong number +fn check_schema(object: u32, process: u32) -> Result<(), Error> { + match object.cmp(&process) { + Ordering::Equal => Ok(()), + Ordering::Greater => bail!( + "the object has schema {object}, this process has {process}, it is stale and needs a \ + restart" + ), + Ordering::Less => bail!( + "the object has schema {object}, older than this process's {process}, the package \ + providing it is behind" + ), + } +} + +fn link_alive(pin: &tc::LinkPin) -> Result { + Ok(tc::iface_name(pin.ifindex)?.is_some_and(|name| name == pin.name)) +} + +fn remove_dir_if_present(dir: &Path) -> std::io::Result<()> { + match std::fs::remove_dir_all(dir) { + Err(e) if e.kind() != ErrorKind::NotFound => Err(e), + _ => Ok(()), + } +} + +/// Marker for a guard holding the apply lock shared. +pub struct Shared; + +/// Marker for a guard holding the apply lock exclusively. +pub struct Exclusive; + +/// The apply lock of one subsystem, released when the guard is dropped. `M` is [`Shared`] or +/// [`Exclusive`]. +pub struct ApplyLock<'a, M> { + programs: &'a TcPrograms, + _lock: File, + _mode: PhantomData, +} + +/// What both lock modes can do. +/// +/// Shared holders run next to each other. A single map write is atomic, but a write that depends +/// on an earlier read would overwrite what another holder wrote in between. Such a write needs the +/// exclusive lock, or the `BPF_NOEXIST` or `BPF_EXIST` flag on the update, which makes the kernel +/// refuse it if the entry appeared or went away since. +impl ApplyLock<'_, M> { + /// Whether every program and map is pinned in `live/`. A shared holder checks this before + /// it writes, since only a full apply pins them. Under the lock the program and map pins do not + /// change, only links come and go, so the answer holds for as long as the guard lives. + pub fn is_current(&self) -> bool { + self.programs.is_current() + } + + /// Open one of the maps in `live/` by name. Returns [`NotLoaded`] if there is no `live/`. + pub fn hash_map( + &self, + name: &str, + ) -> Result, Error> { + Ok(MapHandle { + map: self.programs.hash_map(name)?, + _opened_through: PhantomData, + }) + } + + /// Attach the programs to one interface, the other interfaces stay as they are. + /// + /// If the interface has a link already, we swap the program in place, so no packet passes + /// without one. A link that cannot be swapped is detached and attached again. Two attaches of + /// the same interface can run at the same time under the shared lock, the second one finds the + /// link of the first and is done. If the interface is gone, there is nothing to attach and this + /// returns `Ok`. Without the programs in `live/` this returns [`NotLoaded`]. + pub fn attach_iface(&self, ifindex: u32) -> Result<(), Error> { + self.programs.attach_iface(ifindex) + } + + /// Detach the programs from the interface of that name, even when the interface is gone + /// already. A pin for which `keep` returns true stays, such as one of a newer interface under + /// the same name. Returns the indexes the detached links had. Without a complete `live/` this + /// does nothing, since the pins may belong to a process of another schema. + pub fn detach_named( + &self, + name: &str, + keep: impl Fn(&tc::LinkPin) -> bool, + ) -> Result, Error> { + if !self.is_current() { + return Ok(Vec::new()); + } + self.programs.detach_named(name, keep) + } + + /// Remove the pins of interfaces that are gone and return their indexes. + /// + /// The kernel detaches a link when its interface goes away, but the pin stays and keeps the + /// dead link around. Without a complete `live/` this does nothing, since the pins may belong to + /// a process of another schema. + pub fn detach_gone(&self) -> Result, Error> { + if !self.is_current() { + return Ok(Vec::new()); + } + self.programs.detach_gone() + } + + /// The links of this subsystem as their pins name them. A shared holder sees a snapshot, + /// another attach may add to it meanwhile. The name is not unique. A leftover pin of an + /// interface that went away and the pin of its successor under the name share it, only the + /// index tells them apart. + pub fn attached(&self) -> Result, Error> { + self.programs.live_links() + } +} + +/// What only the exclusive lock can do, since nothing else may run next to it. +impl ApplyLock<'_, Exclusive> { + /// Replace everything in `live/` with the shipped object, loaded with fresh maps, and make the + /// attached interfaces match `desired`. This is the full apply the + /// [module docs](crate::subsystem) describe, so a program never reads a mix of old and new + /// entries. + /// + /// `desired` holds the indexes of the interfaces that should have the programs attached + /// afterwards. `fill` has to write everything the fresh maps should hold, through the + /// [`Staged`] it gets, since nothing carries over from `live/`. To keep something from + /// `live/`, read it through [`hash_map`](Self::hash_map) before the call. + /// + /// If the object cannot be read, carries another schema version or the kernel refuses to load + /// it, `live/` and the links stay as they were. The same goes for a failing `fill`. If the run + /// stops while the links move, there is no `live/` until the next full apply, and the calls + /// that need it return [`NotLoaded`]. + /// + /// An interface that fails to attach fails the call. The other interfaces are still tried. One + /// that is gone by then is skipped. A failed detach is logged and leaves that link in place. + /// + /// Takes `&mut self`, so every map handle has to be dropped before the call. One kept open + /// would write the old map, which no program reads anymore. + pub fn replace_state( + &mut self, + desired: &HashSet, + fill: impl FnOnce(&Staged<'_>) -> Result<(), Error>, + ) -> Result<(), Error> { + self.programs.replace_state(desired, fill) + } + + /// Remove everything this subsystem pinned, links included, so no program keeps running with + /// nothing left to update it. With nothing pinned this does nothing. + /// + /// Like a full apply, this only runs when the shipped object can be read and carries our + /// schema version, since the state may otherwise belong to a process of another + /// schema version. The lock file under `/run` stays. If we removed it, a process waiting on it + /// would get its lock on the removed file while the next one locks a new file, so both would + /// run at once. + pub fn clear(&mut self) -> Result<(), Error> { + if !self.programs.pin_root().exists() { + return Ok(()); + } + self.programs.read_object_checked()?; + self.programs.tear_down() + } +} + +/// The fresh maps of a [`replace_state`](ApplyLock::replace_state), seen only by the fill that +/// writes them. +pub struct Staged<'a> { + programs: &'a TcPrograms, +} + +impl Staged<'_> { + /// Open one of the staged maps by name, empty until the fill writes it. + pub fn hash_map( + &self, + name: &str, + ) -> Result, Error> { + Ok(MapHandle { + map: self + .programs + .hash_map_in(&self.programs.staging_dir(), name)?, + _opened_through: PhantomData, + }) + } +} + +/// A pinned hash map. It cannot outlive the lock or the fill it was opened under, so every write +/// to it runs under the lock and goes into the maps the programs use. +pub struct MapHandle<'a, K, V> { + map: aya::maps::HashMap, + _opened_through: PhantomData<&'a ()>, +} + +impl Deref for MapHandle<'_, K, V> { + type Target = aya::maps::HashMap; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl DerefMut for MapHandle<'_, K, V> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn object_with_section(name: &str, data: &[u8]) -> Vec { + let mut object = object::write::Object::new( + object::BinaryFormat::Elf, + object::Architecture::Bpf, + object::Endianness::default(), + ); + let section = object.add_section( + Vec::new(), + name.as_bytes().to_vec(), + object::SectionKind::Data, + ); + object.append_section_data(section, data, 1); + object.write().unwrap() + } + + #[test] + fn an_object_carries_its_schema_in_a_section_of_four_bytes() { + let object = object_with_section(SCHEMA_SECTION, &7u32.to_ne_bytes()); + assert_eq!(object_schema(&object).unwrap(), 7); + } + + #[test] + fn an_object_without_a_section_of_four_bytes_has_no_schema() { + assert!(object_schema(&object_with_section(".data", &7u32.to_ne_bytes())).is_err()); + assert!(object_schema(&object_with_section(SCHEMA_SECTION, &[7, 0])).is_err()); + assert!(object_schema(&object_with_section(SCHEMA_SECTION, &7u64.to_ne_bytes())).is_err()); + assert!(object_schema(b"no object").is_err()); + } + + #[test] + fn only_an_object_of_the_same_schema_passes() { + assert!(check_schema(2, 2).is_ok()); + assert_eq!( + check_schema(3, 2).unwrap_err().to_string(), + "the object has schema 3, this process has 2, it is stale and needs a restart" + ); + assert_eq!( + check_schema(1, 2).unwrap_err().to_string(), + "the object has schema 1, older than this process's 2, the package providing it is \ + behind" + ); + } +} diff --git a/proxmox-ebpf/src/tc.rs b/proxmox-ebpf/src/tc.rs index d99f705bfa88..58eb468acba6 100644 --- a/proxmox-ebpf/src/tc.rs +++ b/proxmox-ebpf/src/tc.rs @@ -20,7 +20,7 @@ use aya::{ /// Why a link operation failed, so a caller can tell a lost race or a vanished interface from a /// real failure. #[derive(Debug)] -pub enum Error { +pub(crate) enum Error { /// A link of the program is in place already. AlreadyAttached, /// The interface is gone, either no device under that ifindex exists, or the device at that @@ -187,7 +187,7 @@ impl LinkPin { /// Attach `prog` on the interface and direction of `pin` as a tcx link and pin it at `pin_path`. /// Programs of several subsystems run in attach order, each new link goes last. -pub fn attach_and_pin( +pub(crate) fn attach_and_pin( prog: &mut SchedClassifier, pin: &LinkPin, pin_path: &Path, @@ -215,14 +215,17 @@ pub fn attach_and_pin( Ok(()) } -pub fn open_pinned_link(pin_path: &Path) -> Result { +pub(crate) fn open_pinned_link(pin_path: &Path) -> Result { let fd_link: FdLink = PinnedLink::from_pin(pin_path)?.into(); Ok(fd_link.try_into()?) } /// Update the program a link points to to `prog`. This is done with LINK_UPDATE, which is atomic, /// and swaps the old program for `prog` without a single packet passing without a program. -pub fn swap_link(prog: &mut SchedClassifier, link: SchedClassifierLink) -> Result<(), Error> { +pub(crate) fn swap_link( + prog: &mut SchedClassifier, + link: SchedClassifierLink, +) -> Result<(), Error> { let new_id = prog.attach_to_link(link)?; // take the handle out of aya's internal tracking, we have the pin let _ = prog.take_link(new_id)?; @@ -241,7 +244,7 @@ const BPF_LINK_DETACH: nix::libc::c_long = 34; /// Removing the pin alone leaves the detach to the kernel. Kernels before 6.18 without the /// backport of 4f375ade6aa9 only get to that after an RCU grace period, and an attach of the same /// program in between is refused with `EEXIST`, since the old link still holds its place. -pub fn detach_pinned_link(pin_path: &Path) -> Result<(), Error> { +pub(crate) fn detach_pinned_link(pin_path: &Path) -> Result<(), Error> { let link = open_pinned_link(pin_path)?; // the `link_detach` member of the kernel's `union bpf_attr` is the link's fd and nothing else let link_fd = link.fd()?.as_raw_fd() as u32; @@ -263,7 +266,7 @@ pub fn detach_pinned_link(pin_path: &Path) -> Result<(), Error> { } /// The name of an interface by index, none when it is gone. Any other failure returns `Err`. -pub fn iface_name(ifindex: u32) -> Result, anyhow::Error> { +pub(crate) fn iface_name(ifindex: u32) -> Result, anyhow::Error> { let mut buf = [0u8; nix::libc::IF_NAMESIZE]; // nix's if_indextoname takes only -1 as its error, so a NULL from libc comes back as `Ok` with // an empty name @@ -289,7 +292,7 @@ pub fn iface_name(ifindex: u32) -> Result, anyhow::Error> { /// /// The found link files are parsed into [`LinkPin`] as described there. Filenames we cannot parse /// are ignored, we only care about the ones we place and manage. -pub fn read_pinned_links(links_dir: &Path) -> Result, anyhow::Error> { +pub(crate) fn read_pinned_links(links_dir: &Path) -> Result, anyhow::Error> { let mut out = Vec::new(); let dir = match std::fs::read_dir(links_dir) { Ok(d) => d, -- 2.47.3