[PATCH proxmox-ebpf v2 07/27] bpf: add bridge subsystem

[Hannes Laimer <h.laimer@proxmox.com>]

Attach programs to bridge-facing carrier interfaces that write the
identity into the IPv6 SRH tag on the way out and read it back into the
packet mark on the way in, so the identity survives an SRv6 underlay
between nodes. VXLAN-GBP needs none of this, the kernel carries the
mark natively. The programs detect SRv6 from the packet and no-op on
everything else, so they attach to any carrier without knowing the zone
type.

Which interfaces are carriers comes from the bridge entries in the
running config, filtered by node. Enforcement itself stays on the guest
NICs, this subsystem only transports the identity between them.

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/agent.rs              | 48 ++++++++++++++++++++-----
 src/bridge/bpf/srv6.bpf.c | 76 +++++++++++++++++++++++++++++++++++++++
 src/bridge/mod.rs         | 76 +++++++++++++++++++++++++++++++++++++++
 src/main.rs               |  1 +
 src/state.rs              | 33 ++++++++++++-----
 5 files changed, 217 insertions(+), 17 deletions(-)
 create mode 100644 src/bridge/bpf/srv6.bpf.c
 create mode 100644 src/bridge/mod.rs

diff --git a/src/agent.rs b/src/agent.rs
index 63f5f14..5b299b1 100644
--- a/src/agent.rs
+++ b/src/agent.rs
@@ -1,39 +1,60 @@
 //! Agent orchestrator. Builds the per-host [`DesiredState`] from the SDN running-config and applies
-//! it via the [`policy`](crate::policy) subsystem. The tap_plug path
-//! ([`apply_guest_iface_policy`](Agent::apply_guest_iface_policy)) programs a single interface.
+//! it. A full pass ([`apply`](Agent::apply)) covers both subsystems ([`policy`](crate::policy) and
+//! [`bridge`](crate::bridge)). The tap_plug path ([`apply_guest_iface_policy`](Agent::apply_guest_iface_policy))
+//! is policy only and programs a single interface.
 //!
 //! The binary runs one-shot per event (boot, an SDN apply, a tap_plug), not as a resident daemon.
 //! Every invocation first makes sure the programs are loaded (installing them on the first run or a
 //! version change), then applies. Maps and links are pinned in bpffs, so they persist across
 //! invocations and re-running never interrupts traffic.
 
-use crate::{policy::PolicySubsystem, running_config, state::DesiredState};
+use anyhow::Context;
+
+use crate::{
+    bridge::BridgeSubsystem, policy::PolicySubsystem, running_config, state::DesiredState,
+};
 
 pub struct Agent {
     policy: PolicySubsystem,
+    bridge: BridgeSubsystem,
 }
 
 impl Agent {
     pub fn new() -> Self {
         Self {
             policy: PolicySubsystem::new(),
+            bridge: BridgeSubsystem::new(),
         }
     }
 
-    /// Full pass over the host.
+    /// Full pass over the host, both subsystems.
     pub fn apply(&mut self) -> anyhow::Result<()> {
         let Some(state) = build_state()? else {
             return Ok(());
         };
         log::debug!(
-            "applying: {} rule cells, {} assignments",
+            "applying: {} rule cells, {} assignments, {} bridges",
             state.rules.len(),
             state.assignments.len(),
+            state.bridges.len(),
         );
-        self.policy.apply(&state)
+        let policy = self.policy.apply(&state);
+        let bridge = self.bridge.apply(&state);
+        let mut failed = false;
+        for (name, result) in [("policy", policy), ("bridge", bridge)] {
+            if let Err(e) = result {
+                log::error!("{name} subsystem: {e:#}");
+                failed = true;
+            }
+        }
+        if failed {
+            anyhow::bail!("one or more subsystems failed to apply");
+        }
+        Ok(())
     }
 
-    /// Fast path for a guest NIC that just appeared (a tap_plug).
+    /// Policy-only fast path for a guest NIC that just appeared (a tap_plug). Bridge is untouched,
+    /// a guest NIC is never a bridge-facing interface.
     ///
     /// Forwards only a genuine enforcement failure for an assigned NIC, so the agent exits non-zero
     /// and the caller does not bring the NIC up unenforced. A running config we cannot read or
@@ -52,13 +73,16 @@ impl Agent {
 
     /// Detach everything and drop all pinned/run state, for package removal.
     pub fn clear(&self) -> anyhow::Result<()> {
-        self.policy.clear()
+        self.policy.clear()?;
+        self.bridge.clear()?;
+        Ok(())
     }
 }
 
 fn build_state() -> anyhow::Result<Option<DesiredState>> {
     let microseg = running_config::load_microseg()?;
-    match DesiredState::build(&microseg) {
+    let hostname = read_hostname()?;
+    match DesiredState::build(&microseg, &hostname) {
         Ok(s) => Ok(Some(s)),
         Err(e) => {
             log::error!("desired state: {e:#}");
@@ -67,3 +91,9 @@ fn build_state() -> anyhow::Result<Option<DesiredState>> {
     }
 }
 
+fn read_hostname() -> anyhow::Result<String> {
+    let hostname = nix::unistd::gethostname().context("gethostname")?;
+    hostname
+        .into_string()
+        .map_err(|os| anyhow::anyhow!("non-utf8 hostname: {os:?}"))
+}
diff --git a/src/bridge/bpf/srv6.bpf.c b/src/bridge/bpf/srv6.bpf.c
new file mode 100644
index 0000000..8610418
--- /dev/null
+++ b/src/bridge/bpf/srv6.bpf.c
@@ -0,0 +1,76 @@
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include "mark.h"
+#include "bpf_debug.h"
+
+char LICENSE[] SEC("license") = "GPL";
+
+#define TC_ACT_OK 0
+
+#define ETH_HLEN  14
+#define IPV6_HLEN 40
+
+#define ETH_P_IPV6      0x86dd
+#define IPPROTO_ROUTING 43
+#define SRH_TYPE        4
+
+// SRH layout (IPv6 Routing Header type 4), starting from byte 0 of the header:
+//   0: next_hdr
+//   1: hdr_ext_len
+//   2: routing_type   (== SRH_TYPE)
+//   3: segments_left
+//   4: last_entry
+//   5: flags
+//   6..7: tag         <-- our carrier
+#define SRH_TAG_OFF (ETH_HLEN + IPV6_HLEN + 6)
+
+static __always_inline int is_srv6(struct __sk_buff *skb) {
+    void *data = (void *)(long)skb->data;
+    void *data_end = (void *)(long)skb->data_end;
+
+    if (data + ETH_HLEN + IPV6_HLEN + 8 > data_end) return 0;
+
+    struct ethhdr *eth = data;
+    if (eth->h_proto != bpf_htons(ETH_P_IPV6)) return 0;
+
+    struct ipv6hdr *ip6 = (void *)(eth + 1);
+    if (ip6->nexthdr != IPPROTO_ROUTING) return 0;
+
+    __u8 *rh = (__u8 *)(ip6 + 1);
+    if (rh[2] != SRH_TYPE) return 0;
+
+    return 1;
+}
+
+SEC("classifier")
+int tc_bridge_egress(struct __sk_buff *skb) {
+    if (!is_srv6(skb)) {
+        DBG("bridge_out(%u): skip, no SRv6", skb->ifindex);
+        return TC_ACT_OK;
+    }
+    __u16 group = microseg_mark_get(skb);
+    DBG("bridge_out(%u): putting %u onto SRH tag", skb->ifindex, group);
+    __u16 tag = bpf_htons(group);
+    bpf_skb_store_bytes(skb, SRH_TAG_OFF, &tag, sizeof(tag), 0);
+    return TC_ACT_OK;
+}
+
+SEC("classifier")
+int tc_bridge_ingress(struct __sk_buff *skb) {
+    if (!is_srv6(skb)) {
+        DBG("bridge_in(%u): skip, no SRv6", skb->ifindex);
+        return TC_ACT_OK;
+    }
+
+    __u16 tag = 0;
+    if (bpf_skb_load_bytes(skb, SRH_TAG_OFF, &tag, sizeof(tag)) < 0) {
+        DBG("bridge_in(%u): skip, no tag", skb->ifindex);
+        return TC_ACT_OK;
+    }
+
+    __u16 group = bpf_ntohs(tag);
+    microseg_mark_set(skb, group);
+    DBG("bridge_in(%u): pulled %u out of SRH", skb->ifindex, group);
+    return TC_ACT_OK;
+}
diff --git a/src/bridge/mod.rs b/src/bridge/mod.rs
new file mode 100644
index 0000000..f34f6c7
--- /dev/null
+++ b/src/bridge/mod.rs
@@ -0,0 +1,76 @@
+//! Bridge-side carrier translation. Attaches BPF programs to bridge-facing interfaces that write
+//! `skb->mark` into the IPv6 SRH tag (the SRv6 on-wire carrier) on egress and read it back into
+//! `skb->mark` on ingress. Driven by the bridge entries in the [`DesiredState`], which the
+//! agent has already filtered to interfaces that apply on this host.
+//!
+//! The program detects SRv6 from the packet and is a no-op on every other frame, so userspace
+//! attaches it to all bridge carriers without knowing the zone type. A VXLAN-GBP underlay needs no
+//! BPF here: the kernel carries the identity natively (GBP policy-id <-> `skb->mark`) and this
+//! program just no-ops on those frames.
+//!
+//! Pairs with the [`policy`](crate::policy) subsystem. Policy decides which group `skb->mark`
+//! carries on the local tap, bridge moves that value onto and off the wire so it survives across
+//! hosts.
+
+use std::collections::HashSet;
+
+use aya::include_bytes_aligned;
+
+use crate::state::{DesiredState, ResolvedBridge};
+use crate::subsystem::TcPrograms;
+use crate::tc::Direction;
+
+const NAME: &str = "bridge";
+
+const BRIDGE_OBJ: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/srv6.bpf.o"));
+const BRIDGE_FINGERPRINT: u64 = TcPrograms::obj_fingerprint(BRIDGE_OBJ);
+
+fn program_name(dir: Direction) -> &'static str {
+    match dir {
+        Direction::Ingress => "tc_bridge_ingress",
+        Direction::Egress => "tc_bridge_egress",
+    }
+}
+
+pub struct BridgeSubsystem {
+    programs: TcPrograms,
+}
+
+impl BridgeSubsystem {
+    pub fn new() -> Self {
+        Self {
+            programs: TcPrograms::new(NAME, BRIDGE_OBJ, BRIDGE_FINGERPRINT, program_name, None),
+        }
+    }
+
+    pub fn apply(&mut self, state: &DesiredState) -> anyhow::Result<()> {
+        log::trace!("bridge apply starting");
+        let _lock = self.programs.lock_exclusive()?;
+        self.programs.ensure_loaded()?;
+        self.programs
+            .reconcile(&resolve_local_bridges(&state.bridges))?;
+        log::trace!("bridge apply done");
+        Ok(())
+    }
+
+    /// Detach all bridge programs and drop pinned/run state, for package removal.
+    pub fn clear(&self) -> anyhow::Result<()> {
+        self.programs.clear()
+    }
+}
+
+fn resolve_local_bridges(bridges: &[ResolvedBridge]) -> HashSet<u32> {
+    let mut out = HashSet::new();
+    for b in bridges {
+        match nix::net::if_::if_nametoindex(b.interface.as_str()) {
+            Ok(ifidx) => {
+                log::debug!("resolved bridge iface {} -> ifindex {}", b.interface, ifidx);
+                out.insert(ifidx);
+            }
+            Err(_) => {
+                log::debug!("bridge: skipping {}, no such interface", b.interface);
+            }
+        }
+    }
+    out
+}
diff --git a/src/main.rs b/src/main.rs
index 6b3c16c..0334fcb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,4 +1,5 @@
 mod agent;
+mod bridge;
 mod policy;
 mod running_config;
 mod state;
diff --git a/src/state.rs b/src/state.rs
index fc67818..ccc9e0f 100644
--- a/src/state.rs
+++ b/src/state.rs
@@ -1,9 +1,9 @@
-//! Resolved desired state derived from the SDN running config.
+//! Resolved desired state derived from the SDN running config plus the local hostname.
 //!
 //! The running config carries the compiled identity registry (id to representative set) and the
 //! admin's groups, rules and assignments. This module turns that into what the data plane needs:
-//! the wire id for each managed interface and the folded `(src_id, dst_id) -> allow` rule cells.
-//! The hard, must-be-cluster-consistent work (allocation)
+//! the wire id for each managed interface, the folded `(src_id, dst_id) -> allow` rule cells, and
+//! the bridge carriers active on this host. The hard, must-be-cluster-consistent work (allocation)
 //! already happened in the render. Here the agent only resolves and folds, reading nothing but the
 //! replicated config.
 
@@ -18,6 +18,8 @@ pub struct DesiredState {
     pub rules: Vec<ResolvedRule>,
     /// The wire id each managed guest NIC should stamp.
     pub assignments: Vec<ResolvedAssignment>,
+    /// Bridge carriers active on this host.
+    pub bridges: Vec<ResolvedBridge>,
 }
 
 #[derive(Debug)]
@@ -35,8 +37,13 @@ pub struct ResolvedAssignment {
     pub id: u16,
 }
 
+#[derive(Debug)]
+pub struct ResolvedBridge {
+    pub interface: String,
+}
+
 impl DesiredState {
-    pub fn build(cfg: &MicrosegRunningConfig) -> anyhow::Result<Self> {
+    pub fn build(cfg: &MicrosegRunningConfig, this_node: &str) -> anyhow::Result<Self> {
         let entries = cfg.ids();
         let registry = cfg.identities();
         let policies = microseg::build_policies(entries).context("build policies")?;
@@ -62,8 +69,8 @@ impl DesiredState {
         // class. The source side additionally includes the untagged id (id 0, the no-group
         // identity) so an `untagged -> X` rule admits unstamped traffic: gateway / DHCP / ARP
         // replies, and cross-node frames that arrive without a GBP tag. An untagged *destination*
-        // would be a no-op (no program enforces there), so it is not iterated, we only emit allow
-        // cells.
+        // would be a no-op (no program enforces there), so it is not iterated, and the fold
+        // emits allow cells only.
         let classes = registry.classes();
         // its representative is the built-in untagged mark, so a predicate naming the 'untagged'
         // group fires on the untagged id.
@@ -89,9 +96,19 @@ impl DesiredState {
             }
         }
 
+        let mut bridges = Vec::new();
+        for (interface, bridge) in cfg.bridges() {
+            if bridge.applies_to(this_node) {
+                bridges.push(ResolvedBridge {
+                    interface: interface.to_string(),
+                });
+            }
+        }
+
         Ok(Self {
             rules: allow_cells,
             assignments,
+            bridges,
         })
     }
 }
@@ -102,7 +119,7 @@ mod tests {
 
     #[test]
     fn build_resolves_ids_and_folds_rules() {
-        // web=2 -> class 1, db=3 -> class 2, one rule web -> db allow.
+        // web=2 -> class 1, db=3 -> class 2, with one rule web -> db allow.
         let json = r#"{
             "ids": {
                 "web": {"type":"group","id":"web","mark":2},
@@ -114,7 +131,7 @@ mod tests {
             "realized": [{"vmid":100,"iface":0,"groups":["web"]}]
         }"#;
         let cfg: MicrosegRunningConfig = serde_json::from_str(json).expect("parse running config");
-        let state = DesiredState::build(&cfg).expect("build state");
+        let state = DesiredState::build(&cfg, "node1").expect("build state");
 
         // the web interface resolves to its class id 1
         let assignment = state
-- 
2.47.3