From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 9EAB31FF0DA for ; Thu, 19 Nov 2026 20:45:10 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 2BC842181A; Thu, 19 Nov 2026 20:42:18 +0100 (CET) From: Gabriel Goller To: pve-devel@lists.proxmox.com Subject: [PATCH pve-network v3 15/28] sdn: allow fabrics to use simple zone VRFs Date: Thu, 19 Nov 2026 20:41:01 +0100 Message-ID: <20261119194116.575520-16-g.goller@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261119194116.575520-1-g.goller@proxmox.com> References: <20261119194116.575520-1-g.goller@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795117282342 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.789 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: NLFLI6VDWGX45FDLLLGKAQTBZETCPEAX X-Message-ID-Hash: NLFLI6VDWGX45FDLLLGKAQTBZETCPEAX X-MailFrom: g.goller@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Let BGP and OSPF fabrics use a simple zone with dedicated-vrf enabled to keep their routes separate from the default VRF. The zone must be enabled on every fabric node. Prevent zone changes or deletion that would leave a fabric without its VRF. A fabric's zone cannot be changed after creation, since that resets all its sessions anyway and other objects rely on the fabric staying in its VRF. Fail to generate the network config when a fabric interface also has a VRF set in /etc/network/interfaces. ifupdown2 would apply only one of the two VRFs, usually the host one, so the fabric would silently lose the link. The fabric API cannot check this, because it does not see the interfaces file of each node. Signed-off-by: Gabriel Goller --- src/PVE/API2/Network/SDN/Fabrics/Fabric.pm | 56 +++++++++-- .../API2/Network/SDN/Fabrics/FabricNode.pm | 20 ++++ src/PVE/API2/Network/SDN/Zones.pm | 28 ++++++ src/PVE/Network/SDN/Controllers/BgpPlugin.pm | 2 +- src/PVE/Network/SDN/Fabrics.pm | 39 ++++++++ src/test/Makefile | 5 +- src/test/run_test_fabric_host_vrf.pl | 93 +++++++++++++++++++ 7 files changed, 232 insertions(+), 11 deletions(-) create mode 100755 src/test/run_test_fabric_host_vrf.pl diff --git a/src/PVE/API2/Network/SDN/Fabrics/Fabric.pm b/src/PVE/API2/Network/SDN/Fabrics/Fabric.pm index bbc33e047154..453fc0e20a69 100644 --- a/src/PVE/API2/Network/SDN/Fabrics/Fabric.pm +++ b/src/PVE/API2/Network/SDN/Fabrics/Fabric.pm @@ -6,13 +6,51 @@ use warnings; use PVE::Network::SDN; use PVE::Network::SDN::Controllers; use PVE::Network::SDN::Fabrics; +use PVE::Network::SDN::Zones; +use PVE::Exception qw(raise_param_exc); use PVE::JSONSchema qw(get_standard_option); use PVE::Tools qw(extract_param); use PVE::RESTHandler; use base qw(PVE::RESTHandler); +sub validate_fabric_zone { + my ($fabric, $config) = @_; + + my $protocol = $fabric->{protocol} // ''; + if ($protocol eq 'bgp' && !$fabric->{zone}) { + my $controller_cfg = PVE::Network::SDN::Controllers::config(); + for my $controller_id (keys %{ $controller_cfg->{ids} // {} }) { + my $controller = $controller_cfg->{ids}->{$controller_id}; + if ($controller->{type} eq 'bgp') { + die "cannot configure a BGP fabric while BGP controller '$controller_id' exists:" + . " both target the default-VRF BGP router\n"; + } + } + } + + my $zone_id = $fabric->{zone}; + return if !$zone_id; + + raise_param_exc({ zone => 'VRF zones are only supported for BGP and OSPF fabrics' }) + if $protocol !~ /^(?:bgp|ospf)$/; + + my $zone = PVE::Network::SDN::Zones::get_zone($zone_id); + + raise_param_exc({ zone => "zone '$zone_id' does not exist" }) if !$zone; + raise_param_exc({ zone => "zone '$zone_id' is not a simple zone" }) + if $zone->{type} ne 'simple'; + raise_param_exc({ zone => "zone '$zone_id' is configured to use the default VRF" }) + if !$zone->{'dedicated-vrf'}; + + if ($config) { + PVE::Network::SDN::Fabrics::assert_fabric_nodes_in_zone( + $config, $fabric->{id}, $zone_id, $zone, + ); + } +} + __PACKAGE__->register_method({ name => 'index', path => '', @@ -129,6 +167,8 @@ __PACKAGE__->register_method({ protected => 1, permissions => { check => ['perm', '/sdn/fabrics', ['SDN.Allocate']], + description => "Assigning a VRF zone additionally requires 'SDN.Use' or 'SDN.Allocate'" + . " on '/sdn/zones/'.", }, parameters => { properties => PVE::Network::SDN::Fabrics::fabric_properties(0), @@ -148,18 +188,11 @@ __PACKAGE__->register_method({ my $digest = extract_param($param, 'digest'); PVE::Tools::assert_if_modified($config->digest(), $digest) if $digest; - if (($param->{protocol} // '') eq 'bgp') { - my $controller_cfg = PVE::Network::SDN::Controllers::config(); - for my $id (keys %{ $controller_cfg->{ids} // {} }) { - die "cannot add a BGP fabric while BGP controller '$id' exists:" - . " both target the default-VRF BGP router\n" - if $controller_cfg->{ids}->{$id}->{type} eq 'bgp'; - } - } - my $privs = ['SDN.Use', 'SDN.Allocate']; PVE::Network::SDN::Fabrics::api_validate_property_references($param, $privs); + validate_fabric_zone($param); + $config->add_fabric($param); PVE::Network::SDN::Fabrics::write_config($config); }, @@ -200,7 +233,12 @@ __PACKAGE__->register_method({ my $privs = ['SDN.Use', 'SDN.Allocate']; PVE::Network::SDN::Fabrics::api_validate_property_references($param, $privs); + my $old_zone = $config->get_fabric($id)->{zone} // ''; + raise_param_exc({ zone => "cannot change the VRF zone of an existing fabric" }) + if defined($param->{zone}) && $param->{zone} ne $old_zone; + $config->update_fabric($id, $param); + validate_fabric_zone($config->get_fabric($id), $config); PVE::Network::SDN::Fabrics::write_config($config); }, "updating fabric failed", diff --git a/src/PVE/API2/Network/SDN/Fabrics/FabricNode.pm b/src/PVE/API2/Network/SDN/Fabrics/FabricNode.pm index 7a148b550f7d..e5272bb82f68 100644 --- a/src/PVE/API2/Network/SDN/Fabrics/FabricNode.pm +++ b/src/PVE/API2/Network/SDN/Fabrics/FabricNode.pm @@ -9,6 +9,7 @@ use PVE::Tools qw(extract_param run_command); use PVE::Network::SDN; use PVE::Network::SDN::Fabrics; use PVE::Network::SDN::WireGuard; +use PVE::Network::SDN::Zones; use PVE::RS::SDN::Fabrics; use PVE::RESTHandler; @@ -138,6 +139,21 @@ my sub is_internal_wireguard_node { return $node->{protocol} eq 'wireguard' && $node->{role} eq 'internal'; } +my sub validate_node_fabric_zone { + my ($config, $fabric_id, $node_id) = @_; + + my $fabric = $config->get_fabric($fabric_id); + my $zone_id = $fabric->{zone}; + return if !$zone_id; + + PVE::Network::SDN::Zones::api_get_zone_checked($zone_id, ['SDN.Use', 'SDN.Allocate']); + my $zone = PVE::Network::SDN::Zones::get_zone($zone_id); + + PVE::Network::SDN::Fabrics::assert_fabric_node_in_zone( + $fabric_id, $node_id, $zone_id, $zone, + ); +} + __PACKAGE__->register_method({ name => 'add_node', path => '', @@ -150,6 +166,8 @@ __PACKAGE__->register_method({ ['perm', '/sdn/fabrics/{fabric_id}', ['SDN.Allocate']], ['perm', '/nodes/{node_id}', ['Sys.Modify']], ], + description => "If the fabric uses a VRF zone, also requires 'SDN.Use' or" + . " 'SDN.Allocate' on '/sdn/zones/'.", }, parameters => { properties => PVE::Network::SDN::Fabrics::node_properties(0), @@ -169,6 +187,8 @@ __PACKAGE__->register_method({ my $digest = extract_param($param, 'digest'); PVE::Tools::assert_if_modified($config->digest(), $digest) if $digest; + validate_node_fabric_zone($config, $param->{fabric_id}, $param->{node_id}); + if (is_internal_wireguard_node($param) && $param->{interfaces}) { my $private_keys = PVE::Network::SDN::WireGuard::private_keys(); diff --git a/src/PVE/API2/Network/SDN/Zones.pm b/src/PVE/API2/Network/SDN/Zones.pm index 38d950fda076..e4f4b174466e 100644 --- a/src/PVE/API2/Network/SDN/Zones.pm +++ b/src/PVE/API2/Network/SDN/Zones.pm @@ -11,6 +11,7 @@ use PVE::SafeSyslog; use PVE::Tools qw(extract_param); use PVE::Network::SDN::Dns; +use PVE::Network::SDN::Fabrics; use PVE::Network::SDN::Subnets; use PVE::Network::SDN::Vnets; use PVE::Network::SDN; @@ -363,6 +364,13 @@ sub create_etc_interfaces_sdn_dir { mkdir("/etc/pve/sdn"); } +sub fabrics_using_zone { + my ($zone_id, $fabric_config) = @_; + + my $fabrics = $fabric_config->list_fabrics(); + return grep { ($fabrics->{$_}->{zone} // '') eq $zone_id } sort keys $fabrics->%*; +} + __PACKAGE__->register_method({ name => 'create', protected => 1, @@ -475,6 +483,22 @@ __PACKAGE__->register_method({ $scfg->{$_} = $opts->{$_} for keys $opts->%*; + my $fabric_config = PVE::Network::SDN::Fabrics::config(); + my @fabric_ids = fabrics_using_zone($id, $fabric_config); + + if (@fabric_ids && $scfg->{type} eq 'simple' && !$scfg->{'dedicated-vrf'}) { + raise_param_exc({ + 'dedicated-vrf' => + "zone is still used as VRF by fabric '$fabric_ids[0]'", + }); + } + + for my $fabric_id (@fabric_ids) { + PVE::Network::SDN::Fabrics::assert_fabric_nodes_in_zone( + $fabric_config, $fabric_id, $id, $scfg, + ); + } + my $new_ipam = $scfg->{ipam}; if (!$new_ipam != !$old_ipam || (($new_ipam // '') ne ($old_ipam // ''))) { # don't allow ipam change if subnet are defined for now, need to implement resync ipam content @@ -547,6 +571,10 @@ __PACKAGE__->register_method({ my $plugin = PVE::Network::SDN::Zones::Plugin->lookup($scfg->{type}); my $vnet_cfg = PVE::Network::SDN::Vnets::config(); + my $fabric_config = PVE::Network::SDN::Fabrics::config(); + my @fabric_ids = fabrics_using_zone($id, $fabric_config); + die "zone is still used as VRF by fabric '$fabric_ids[0]'\n" if @fabric_ids; + $plugin->on_delete_hook($id, $vnet_cfg); delete $cfg->{ids}->{$id}; diff --git a/src/PVE/Network/SDN/Controllers/BgpPlugin.pm b/src/PVE/Network/SDN/Controllers/BgpPlugin.pm index 537b46229e58..328246ce7d77 100644 --- a/src/PVE/Network/SDN/Controllers/BgpPlugin.pm +++ b/src/PVE/Network/SDN/Controllers/BgpPlugin.pm @@ -209,7 +209,7 @@ sub on_update_hook { for my $id (keys %$fabrics) { die "cannot configure a BGP controller while BGP fabric '$id' exists:" . " both target the default-VRF BGP router\n" - if $fabrics->{$id}->{protocol} eq 'bgp'; + if $fabrics->{$id}->{protocol} eq 'bgp' && !$fabrics->{$id}->{zone}; } my $controller = $controller_cfg->{ids}->{$controllerid}; diff --git a/src/PVE/Network/SDN/Fabrics.pm b/src/PVE/Network/SDN/Fabrics.pm index 41119b2ec782..5202ad62e520 100644 --- a/src/PVE/Network/SDN/Fabrics.pm +++ b/src/PVE/Network/SDN/Fabrics.pm @@ -13,6 +13,7 @@ use PVE::ParseUtils qw($IPV4RE); use PVE::Network::SDN::PrefixLists; use PVE::Network::SDN::RouteMaps; +use PVE::Network::SDN::Zones; use PVE::RS::SDN; use PVE::RS::SDN::Fabrics; @@ -229,6 +230,23 @@ sub write_config { cfs_write_file("sdn/fabrics.cfg", $config->to_raw(), 1); } +sub assert_fabric_node_in_zone { + my ($fabric_id, $node_id, $zone_id, $zone) = @_; + + return if !defined($zone->{nodes}) || $zone->{nodes}->{$node_id}; + + die "fabric '$fabric_id' contains node '$node_id', which is not enabled for zone '$zone_id'\n"; +} + +sub assert_fabric_nodes_in_zone { + my ($config, $fabric_id, $zone_id, $zone) = @_; + + my $nodes = $config->list_nodes_fabric($fabric_id); + for my $node_id (sort keys $nodes->%*) { + assert_fabric_node_in_zone($fabric_id, $node_id, $zone_id, $zone); + } +} + sub get_frr_daemon_status { my ($fabric_config) = @_; @@ -254,6 +272,14 @@ sub generate_etc_network_config { my $fabrics_config = $running_cfg->{fabrics}->{ids} // {}; my $fabric_object = PVE::RS::SDN::Fabrics->running_config($fabrics_config); + my $ifaces = PVE::INotify::read_file('interfaces')->{ifaces}; + for my $iface (sort keys %$ifaces) { + next if !$ifaces->{$iface}->{vrf}; + die "interface '$iface' - cannot set VRF '$ifaces->{$iface}->{vrf}' on an interface that" + . " is part of a fabric, remove the VRF from the host interface\n" + if $fabric_object->is_fabric_interface($nodename, $iface); + } + return $fabric_object->get_interfaces_etc_network_config($nodename); } @@ -512,6 +538,15 @@ sub fabric_properties { protocol => get_standard_option('pve-sdn-fabric-protocol'), digest => get_standard_option('pve-config-digest'), 'lock-token' => get_standard_option('pve-sdn-lock-token'), + zone => get_standard_option( + 'pve-sdn-zone-id', + { + description => 'Simple zone whose VRF contains this fabric.', + 'type-property' => 'protocol', + 'instance-types' => ['bgp', 'ospf'], + optional => 1, + }, + ), ip_prefix => { type => 'string', format => 'CIDR', @@ -716,6 +751,10 @@ sub api_validate_property_references { my $prefix_list_config = PVE::Network::SDN::PrefixLists::config(); my $route_map_config = PVE::Network::SDN::RouteMaps::config(); + if (my $zone = $fabric->{zone}) { + PVE::Network::SDN::Zones::api_get_zone_checked($zone, $privs); + } + if (my $route_filter = $fabric->{'route_filter'}) { raise_param_exc({ route_filter => "could not find prefix list $route_filter" }) if !$prefix_list_config->get($route_filter); diff --git a/src/test/Makefile b/src/test/Makefile index 2e35a3d0ae2a..3df96852f27f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,10 +1,13 @@ all: test -test: test_zones test_ipams test_dns test_subnets test_vnets_blackbox test_running_config test_api_index +test: test_zones test_fabric_host_vrf test_ipams test_dns test_subnets test_vnets_blackbox test_running_config test_api_index test_zones: run_test_zones.pl ./run_test_zones.pl +test_fabric_host_vrf: run_test_fabric_host_vrf.pl + ./run_test_fabric_host_vrf.pl + test_ipams: run_test_ipams.pl ./run_test_ipams.pl diff --git a/src/test/run_test_fabric_host_vrf.pl b/src/test/run_test_fabric_host_vrf.pl new file mode 100755 index 000000000000..9ed3f2579d34 --- /dev/null +++ b/src/test/run_test_fabric_host_vrf.pl @@ -0,0 +1,93 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib qw(..); + +use Test::More; +use Test::MockModule; + +use PVE::Network::SDN::Fabrics; + +my $running_cfg = { + fabrics => { + ids => { + test => { + id => 'test', + type => 'ospf_fabric', + area => '0', + ip_prefix => '172.20.30.0/24', + }, + test_localhost => { + id => 'test_localhost', + type => 'ospf_node', + ip => '172.20.30.1', + interfaces => ['name=ens19,ip=172.16.3.10/31'], + }, + }, + }, +}; + +my $interfaces; +my $inotify = Test::MockModule->new('PVE::INotify'); +$inotify->mock( + nodename => sub { return 'localhost' }, + read_file => sub { + my ($file) = @_; + die "unexpected read of '$file'\n" if $file ne 'interfaces'; + return { ifaces => $interfaces }; + }, +); + +$interfaces = { ens19 => { vrf => 'vrf_blue' } }; +eval { PVE::Network::SDN::Fabrics::generate_etc_network_config($running_cfg) }; +like( + $@, + qr/^interface 'ens19' - cannot set VRF 'vrf_blue' on an interface that is part of a fabric/, + 'VRF on a fabric interface fails', +); + +$interfaces = { ens20 => { vrf => 'vrf_blue' }, ens19 => {} }; +my $config = eval { PVE::Network::SDN::Fabrics::generate_etc_network_config($running_cfg) }; +is($@, '', 'VRF on another interface is allowed'); +like($config, qr/^iface ens19 inet static$/m, 'fabric interface config is generated'); + +my $zone_cfg = { + fabrics => { + ids => { + red => { + id => 'red', + type => 'ospf_fabric', + area => '0', + ip_prefix => '172.20.30.0/24', + zone => 'red', + }, + red_localhost => { + id => 'red_localhost', + type => 'ospf_node', + ip => '172.20.30.1', + interfaces => ['name=ens19,ip=172.16.3.10/31'], + }, + }, + }, +}; + +$interfaces = { ens19 => {} }; +$config = eval { PVE::Network::SDN::Fabrics::generate_etc_network_config($zone_cfg) }; +is($@, '', 'OSPF fabric in a zone VRF is allowed'); +like( + $config, + qr/^iface ens19 inet static\n(?:\t.*\n)*\tvrf vrf_red$/m, + 'fabric interface is put into the zone VRF', +); + +$interfaces = { ens19 => { vrf => 'vrf_red' } }; +eval { PVE::Network::SDN::Fabrics::generate_etc_network_config($zone_cfg) }; +like( + $@, + qr/^interface 'ens19' - cannot set VRF 'vrf_red'/, + 'host VRF on a zone fabric interface fails', +); + +done_testing(); -- 2.47.3