[PATCH pve-network v4 21/45] sdn: zones: add vxlan-gbp option to vxlan and evpn zones

[Hannes Laimer <h.laimer@proxmox.com>]

The option enables Group Based Policy (GBP) on the zone's VXLAN
interfaces. The low 16 bits of the packet mark then travel as the
group ID in the VXLAN header, and the receiving node puts them back
into the mark. A VXLAN interface without GBP drops frames with a
group ID, so every VTEP of the zone needs GBP.

Zones that share a VXLAN port must agree on the option, so an SDN
apply refuses a config where they do not. The kernel shares a port's
socket only between interfaces with the same GBP setting, and the
interfaces of the zone that comes second would stay down without any
error.

The VXLAN interfaces of an EVPN zone with the option also get VXLAN
learning off at creation. Since their bridge learning is off,
ifupdown2 otherwise changes them after creation to turn VXLAN
learning off as well. A change cannot carry the GBP flag, so the
kernel then treats the GBP bits as reserved again and drops incoming
GBP frames as malformed.

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/PVE/API2/Network/SDN/Zones.pm             |  8 ++
 src/PVE/Network/SDN.pm                        |  1 +
 src/PVE/Network/SDN/Zones.pm                  | 27 +++++++
 src/PVE/Network/SDN/Zones/EvpnPlugin.pm       |  8 ++
 src/PVE/Network/SDN/Zones/VxlanPlugin.pm      | 12 +++
 .../zones/evpn/gbp/expected_controller_config | 77 +++++++++++++++++++
 .../zones/evpn/gbp/expected_sdn_interfaces    | 45 +++++++++++
 src/test/zones/evpn/gbp/interfaces            |  7 ++
 src/test/zones/evpn/gbp/sdn_config            | 26 +++++++
 .../zones/vxlan/gbp/expected_sdn_interfaces   | 17 ++++
 src/test/zones/vxlan/gbp/interfaces           |  7 ++
 src/test/zones/vxlan/gbp/sdn_config           | 11 +++
 12 files changed, 246 insertions(+)
 create mode 100644 src/test/zones/evpn/gbp/expected_controller_config
 create mode 100644 src/test/zones/evpn/gbp/expected_sdn_interfaces
 create mode 100644 src/test/zones/evpn/gbp/interfaces
 create mode 100644 src/test/zones/evpn/gbp/sdn_config
 create mode 100644 src/test/zones/vxlan/gbp/expected_sdn_interfaces
 create mode 100644 src/test/zones/vxlan/gbp/interfaces
 create mode 100644 src/test/zones/vxlan/gbp/sdn_config

diff --git a/src/PVE/API2/Network/SDN/Zones.pm b/src/PVE/API2/Network/SDN/Zones.pm
index 74b10f032bf4..3d395cb8e836 100644
--- a/src/PVE/API2/Network/SDN/Zones.pm
+++ b/src/PVE/API2/Network/SDN/Zones.pm
@@ -159,6 +159,14 @@ my $ZONE_PROPERTIES = {
         optional => 1,
         default => 4789,
     },
+    'vxlan-gbp' => {
+        type => 'boolean',
+        description => "Enable VXLAN Group Based Policy (GBP) on the zone's VXLAN interfaces."
+            . " Carries the 16-bit group ID of the packet mark across hosts. Every VTEP"
+            . " in the zone must have it enabled. Changing it on a zone in use recreates its"
+            . " VXLAN interfaces on the next apply. VXLAN and EVPN zone only.",
+        optional => 1,
+    },
     'bridge' => {
         type => 'string',
         description => 'the bridge for which VLANs should be managed. VLAN & QinQ zone only.',
diff --git a/src/PVE/Network/SDN.pm b/src/PVE/Network/SDN.pm
index ebfe7a68b3a5..4c3bdf447830 100644
--- a/src/PVE/Network/SDN.pm
+++ b/src/PVE/Network/SDN.pm
@@ -210,6 +210,7 @@ sub compile_running_cfg {
 
     my $vnets_cfg = PVE::Network::SDN::Vnets::config();
     my $zones_cfg = PVE::Network::SDN::Zones::config();
+    PVE::Network::SDN::Zones::check_vxlan_gbp_agrees($zones_cfg);
     my $controllers_cfg = PVE::Network::SDN::Controllers::config();
     my $subnets_cfg = PVE::Network::SDN::Subnets::config();
     my $fabrics_cfg = PVE::Network::SDN::Fabrics::config();
diff --git a/src/PVE/Network/SDN/Zones.pm b/src/PVE/Network/SDN/Zones.pm
index 2652d66e0b3e..c7345907c67c 100644
--- a/src/PVE/Network/SDN/Zones.pm
+++ b/src/PVE/Network/SDN/Zones.pm
@@ -59,6 +59,33 @@ sub config {
     return cfs_read_file("sdn/zones.cfg");
 }
 
+my sub zones_share_a_node {
+    my ($nodes, $other_nodes) = @_;
+    return 1 if !defined($nodes) || !defined($other_nodes);
+    return scalar(grep { $other_nodes->{$_} } keys $nodes->%*);
+}
+
+# the kernel shares a VXLAN port's socket only between interfaces with the same GBP setting, so
+# the interfaces of the zone that comes up second stay down and nothing reports it. Zones that
+# never meet on a node never share a socket.
+sub check_vxlan_gbp_agrees {
+    my ($zones_cfg) = @_;
+
+    my $seen_by_port = {};
+    for my $zoneid (sort keys $zones_cfg->{ids}->%*) {
+        my $zone = $zones_cfg->{ids}->{$zoneid};
+        next if $zone->{type} ne 'vxlan' && $zone->{type} ne 'evpn';
+        my $port = $zone->{'vxlan-port'} // 4789;
+        my $gbp = $zone->{'vxlan-gbp'} ? 1 : 0;
+        for my $other ($seen_by_port->{$port}->@*) {
+            next if $other->{gbp} == $gbp || !zones_share_a_node($other->{nodes}, $zone->{nodes});
+            die "zones '$other->{id}' and '$zoneid' share VXLAN port $port"
+                . " and disagree on vxlan-gbp\n";
+        }
+        push $seen_by_port->{$port}->@*, { id => $zoneid, gbp => $gbp, nodes => $zone->{nodes} };
+    }
+}
+
 sub get_plugin_config {
     my ($vnet) = @_;
     my $zoneid = $vnet->{zone};
diff --git a/src/PVE/Network/SDN/Zones/EvpnPlugin.pm b/src/PVE/Network/SDN/Zones/EvpnPlugin.pm
index c31b6c6a2bfd..b27af45cfca7 100644
--- a/src/PVE/Network/SDN/Zones/EvpnPlugin.pm
+++ b/src/PVE/Network/SDN/Zones/EvpnPlugin.pm
@@ -123,6 +123,7 @@ sub options {
         'bridge-disable-mac-learning' => { optional => 1 },
         'rt-import' => { optional => 1 },
         'vxlan-port' => { optional => 1 },
+        'vxlan-gbp' => { optional => 1 },
         mtu => { optional => 1 },
         mac => { optional => 1 },
         dns => { optional => 1 },
@@ -155,6 +156,11 @@ sub generate_sdn_config {
     my $vrf_iface = "vrf_$zoneid";
     my $vrfvxlan = $plugin_config->{'vrf-vxlan'};
     my $local_node = PVE::INotify::nodename();
+    # ifupdown2 would otherwise turn VXLAN learning off to match bridge learning in a later link
+    # change, and the kernel then treats the GBP bits as reserved again
+    # TODO: drop once the kernel keeps the reserved bits across a changelink, see
+    # https://lore.kernel.org/all/20261023-b4-vxlan-changelink-gbp-v1-0-f0d13951c09b@proxmox.com/
+    my @gbp = $plugin_config->{'vxlan-gbp'} ? ("vxlan-gbp on", "vxlan-learning off") : ();
 
     die "missing vxlan tag" if !$tag;
     die "missing controller" if !$controller;
@@ -230,6 +236,7 @@ sub generate_sdn_config {
     push @iface_config, "vxlan-id $tag";
     push @iface_config, "vxlan-local-tunnelip $ifaceip" if $ifaceip;
     push @iface_config, "vxlan-port $vxlanport" if $vxlanport;
+    push @iface_config, @gbp;
     push @iface_config, "bridge-learning off";
     push @iface_config, "bridge-arp-nd-suppress on"
         if !$plugin_config->{'disable-arp-nd-suppression'};
@@ -334,6 +341,7 @@ sub generate_sdn_config {
             push @iface_config, "vxlan-id $vrfvxlan";
             push @iface_config, "vxlan-local-tunnelip $ifaceip" if $ifaceip;
             push @iface_config, "vxlan-port $vxlanport" if $vxlanport;
+            push @iface_config, @gbp;
             push @iface_config, "bridge-learning off";
             push @iface_config, "bridge-arp-nd-suppress on"
                 if !$plugin_config->{'disable-arp-nd-suppression'};
diff --git a/src/PVE/Network/SDN/Zones/VxlanPlugin.pm b/src/PVE/Network/SDN/Zones/VxlanPlugin.pm
index b8b1ef0eeb7f..ea2c81e88539 100644
--- a/src/PVE/Network/SDN/Zones/VxlanPlugin.pm
+++ b/src/PVE/Network/SDN/Zones/VxlanPlugin.pm
@@ -45,6 +45,16 @@ sub properties {
             type => 'string',
             format => 'pve-sdn-fabric-id',
         },
+        'vxlan-gbp' => {
+            description => "Enable VXLAN Group Based Policy (GBP) on the zone's VXLAN"
+                . " interfaces. Carries the 16-bit group ID of the packet mark"
+                . " across hosts. Every VTEP in the zone must have it enabled. Zones that"
+                . " share a VXLAN port on a node must agree on it, and an SDN apply refuses a"
+                . " config where they do not. Changing it on a zone in use recreates its VXLAN"
+                . " interfaces on the next apply.",
+            type => 'boolean',
+            optional => 1,
+        },
     };
 }
 
@@ -59,6 +69,7 @@ sub options {
         dnszone => { optional => 1 },
         ipam => { optional => 1 },
         fabric => { optional => 1 },
+        'vxlan-gbp' => { optional => 1 },
     };
 }
 
@@ -136,6 +147,7 @@ sub generate_sdn_config {
         push @iface_config, "vxlan_remoteip $address";
     }
     push @iface_config, "vxlan-port $vxlanport" if $vxlanport;
+    push @iface_config, "vxlan-gbp on" if $plugin_config->{'vxlan-gbp'};
 
     push @iface_config, "mtu $mtu" if $mtu;
     push(@{ $config->{$vxlan_iface} }, @iface_config) if !$config->{$vxlan_iface};
diff --git a/src/test/zones/evpn/gbp/expected_controller_config b/src/test/zones/evpn/gbp/expected_controller_config
new file mode 100644
index 000000000000..e7923027f01e
--- /dev/null
+++ b/src/test/zones/evpn/gbp/expected_controller_config
@@ -0,0 +1,77 @@
+frr version 10.6.1
+frr defaults datacenter
+hostname localhost
+log syslog informational
+service integrated-vtysh-config
+!
+vrf vrf_myzone
+ vni 1000
+exit-vrf
+!
+router bgp 65000
+ bgp router-id 192.168.0.1
+ no bgp hard-administrative-reset
+ no bgp default ipv4-unicast
+ coalesce-time 1000
+ no bgp graceful-restart notification
+ neighbor VTEP peer-group
+ neighbor VTEP remote-as 65000
+ neighbor VTEP bfd
+ neighbor 192.168.0.2 peer-group VTEP
+ neighbor 192.168.0.3 peer-group VTEP
+ !
+ address-family ipv4 unicast
+  import vrf vrf_myzone
+ exit-address-family
+ !
+ address-family ipv6 unicast
+  import vrf vrf_myzone
+ exit-address-family
+ !
+ address-family l2vpn evpn
+  neighbor VTEP activate
+  neighbor VTEP route-map MAP_VTEP_IN in
+  neighbor VTEP route-map MAP_VTEP_OUT out
+  advertise-all-vni
+ exit-address-family
+exit
+!
+router bgp 65000 vrf vrf_myzone
+ bgp router-id 192.168.0.1
+ no bgp hard-administrative-reset
+ no bgp graceful-restart notification
+ !
+ address-family ipv4 unicast
+  redistribute connected
+ exit-address-family
+ !
+ address-family ipv6 unicast
+  redistribute connected
+ exit-address-family
+ !
+ address-family l2vpn evpn
+  default-originate ipv4
+  default-originate ipv6
+ exit-address-family
+exit
+!
+ip prefix-list only_default seq 1 permit 0.0.0.0/0
+!
+ipv6 prefix-list only_default_v6 seq 1 permit ::/0
+!
+route-map MAP_VTEP_IN deny 1
+ match ip address prefix-list only_default
+exit
+!
+route-map MAP_VTEP_IN deny 2
+ match ipv6 address prefix-list only_default_v6
+exit
+!
+route-map MAP_VTEP_IN permit 3
+exit
+!
+route-map MAP_VTEP_OUT permit 1
+exit
+!
+line vty
+!
diff --git a/src/test/zones/evpn/gbp/expected_sdn_interfaces b/src/test/zones/evpn/gbp/expected_sdn_interfaces
new file mode 100644
index 000000000000..5480b2a1b225
--- /dev/null
+++ b/src/test/zones/evpn/gbp/expected_sdn_interfaces
@@ -0,0 +1,45 @@
+#version:1
+
+auto myvnet
+iface myvnet
+	address 10.0.0.1/24
+	bridge_ports vxlan_myvnet
+	bridge_stp off
+	bridge_fd 0
+	mtu 1450
+	ip-forward on
+	arp-accept on
+	vrf vrf_myzone
+
+auto vrf_myzone
+iface vrf_myzone
+	vrf-table auto
+	post-up ip route del vrf vrf_myzone unreachable default metric 4278198272
+
+auto vrfbr_myzone
+iface vrfbr_myzone
+	bridge-ports vrfvx_myzone
+	bridge_stp off
+	bridge_fd 0
+	mtu 1450
+	vrf vrf_myzone
+
+auto vrfvx_myzone
+iface vrfvx_myzone
+	vxlan-id 1000
+	vxlan-local-tunnelip 192.168.0.1
+	vxlan-gbp on
+	vxlan-learning off
+	bridge-learning off
+	bridge-arp-nd-suppress on
+	mtu 1450
+
+auto vxlan_myvnet
+iface vxlan_myvnet
+	vxlan-id 100
+	vxlan-local-tunnelip 192.168.0.1
+	vxlan-gbp on
+	vxlan-learning off
+	bridge-learning off
+	bridge-arp-nd-suppress on
+	mtu 1450
diff --git a/src/test/zones/evpn/gbp/interfaces b/src/test/zones/evpn/gbp/interfaces
new file mode 100644
index 000000000000..66bb826a44b3
--- /dev/null
+++ b/src/test/zones/evpn/gbp/interfaces
@@ -0,0 +1,7 @@
+auto vmbr0
+iface vmbr0 inet static
+	address 192.168.0.1/24
+	gateway 192.168.0.254
+        bridge-ports eth0
+        bridge-stp off
+        bridge-fd 0
diff --git a/src/test/zones/evpn/gbp/sdn_config b/src/test/zones/evpn/gbp/sdn_config
new file mode 100644
index 000000000000..9b3cbb6f893d
--- /dev/null
+++ b/src/test/zones/evpn/gbp/sdn_config
@@ -0,0 +1,26 @@
+{
+  version => 1,
+  vnets   => {
+               ids => {
+                        myvnet => { tag => "100", type => "vnet", zone => "myzone" },
+                      },
+             },
+
+  zones   => {
+               ids => { myzone => { ipam => "pve", type => "evpn", controller => "evpnctl", 'vrf-vxlan' => 1000, 'vxlan-gbp' => 1, exitnodes => { 'localhost' => 1 } } },
+             },
+  controllers  => {
+               ids => { evpnctl => { type => "evpn", 'peers' => '192.168.0.1,192.168.0.2,192.168.0.3', asn => "65000" } },
+             },
+
+  subnets => {
+              ids => { 'myzone-10.0.0.0-24' => {
+							'type' => 'subnet',
+							'vnet' => 'myvnet',
+							'gateway' => '10.0.0.1',
+						  }
+		     }
+	     }
+}
+
+
diff --git a/src/test/zones/vxlan/gbp/expected_sdn_interfaces b/src/test/zones/vxlan/gbp/expected_sdn_interfaces
new file mode 100644
index 000000000000..c66cff3f0c85
--- /dev/null
+++ b/src/test/zones/vxlan/gbp/expected_sdn_interfaces
@@ -0,0 +1,17 @@
+#version:1
+
+auto myvnet
+iface myvnet
+	bridge_ports vxlan_myvnet
+	bridge_stp off
+	bridge_fd 0
+	mtu 1450
+
+auto vxlan_myvnet
+iface vxlan_myvnet
+	vxlan-id 100
+	vxlan-local-tunnelip 192.168.0.1
+	vxlan_remoteip 192.168.0.2
+	vxlan_remoteip 192.168.0.3
+	vxlan-gbp on
+	mtu 1450
diff --git a/src/test/zones/vxlan/gbp/interfaces b/src/test/zones/vxlan/gbp/interfaces
new file mode 100644
index 000000000000..66bb826a44b3
--- /dev/null
+++ b/src/test/zones/vxlan/gbp/interfaces
@@ -0,0 +1,7 @@
+auto vmbr0
+iface vmbr0 inet static
+	address 192.168.0.1/24
+	gateway 192.168.0.254
+        bridge-ports eth0
+        bridge-stp off
+        bridge-fd 0
diff --git a/src/test/zones/vxlan/gbp/sdn_config b/src/test/zones/vxlan/gbp/sdn_config
new file mode 100644
index 000000000000..c61a7de5f371
--- /dev/null
+++ b/src/test/zones/vxlan/gbp/sdn_config
@@ -0,0 +1,11 @@
+{
+  version => 1,
+  vnets   => {
+               ids => {
+                        myvnet => { tag => 100, type => "vnet", zone => "myzone" },
+                      },
+             },
+  zones   => {
+               ids => { myzone => { ipam => "pve", type => "vxlan", 'vxlan-gbp' => 1, peers => "192.168.0.1,192.168.0.2,192.168.0.3" } },
+             },
+}
-- 
2.47.3