[RFC PATCH network 05/16] sdn: ovn: configure the chassis on boot and on network reload

["Rémi PAETA" <remi.paeta@comaite.com>]

ovn-controller needs the southbound database, the encapsulation type
and the local tunnel IP, which are stored in the external_ids of the
local Open_vSwitch table. They are now set whenever the network
configuration of a node gets generated, using the local IP of the route
towards the southbound database as tunnel IP. A chassis attached by PVE
gets detached again once the node is not part of any OVN zone anymore.
A node can only be attached to a single OVN deployment.

pvenetcommit.service removes the Open vSwitch database on every boot,
which drops these settings. The new pve-sdn-ovn-chassis.service sets
them again before the guests get started. The postinst needs the
debhelper token for the unit to get enabled.

ovn-controller needs a few seconds to connect to the southbound
database, so wait for up to ten seconds before warning that it is not
connected. The zone status also reports an ovn-controller that is not
connected.

Signed-off-by: Rémi PAETA <remi.paeta@comaite.com>
---
 debian/libpve-network-perl.install       |   1 +
 debian/libpve-network-perl.postinst      |   2 +
 src/PVE/Network/SDN.pm                   |  19 +++
 src/PVE/Network/SDN/Ovn.pm               | 179 +++++++++++++++++++++++
 src/PVE/Network/SDN/Zones/OvnPlugin.pm   |   2 +
 src/services/Makefile                    |   1 +
 src/services/pve-sdn-ovn-chassis.service |  16 ++
 src/test/run_test_ovn.pl                 | 145 ++++++++++++++++++
 8 files changed, 365 insertions(+)
 create mode 100644 src/services/pve-sdn-ovn-chassis.service

diff --git a/debian/libpve-network-perl.install b/debian/libpve-network-perl.install
index 4e63c1f..ad5fc2d 100644
--- a/debian/libpve-network-perl.install
+++ b/debian/libpve-network-perl.install
@@ -1,2 +1,3 @@
 lib/systemd/system/dnsmasq@.service.d/00-dnsmasq-after-networking.conf /usr/lib/systemd/system/dnsmasq@.service.d/
+lib/systemd/system/pve-sdn-ovn-chassis.service /usr/lib/systemd/system/
 usr/share/perl5/PVE/Network
diff --git a/debian/libpve-network-perl.postinst b/debian/libpve-network-perl.postinst
index 99faedf..360863e 100644
--- a/debian/libpve-network-perl.postinst
+++ b/debian/libpve-network-perl.postinst
@@ -32,4 +32,6 @@ case "$1" in
   ;;
 esac
 
+#DEBHELPER#
+
 exit 0
diff --git a/src/PVE/Network/SDN.pm b/src/PVE/Network/SDN.pm
index d601cd0..e2cdf3d 100644
--- a/src/PVE/Network/SDN.pm
+++ b/src/PVE/Network/SDN.pm
@@ -421,12 +421,18 @@ sub write_raw_etc_network_config {
 Generates the network configuration for all SDN plugins and writes it to the SDN
 interfaces files (/etc/network/interfaces.d/sdn).
 
+Also configures the local OVN chassis, as this is called on every node when the
+SDN configuration gets applied.
+
 =cut
 
 sub generate_etc_network_config {
     my $running_cfg = PVE::Network::SDN::running_config();
     my $raw_config = PVE::Network::SDN::generate_raw_etc_network_config($running_cfg);
     PVE::Network::SDN::write_raw_etc_network_config($raw_config);
+
+    eval { PVE::Network::SDN::Ovn::configure_chassis($running_cfg) };
+    log_warn("failed to configure the OVN chassis - $@") if $@;
 }
 
 =head3 generate_frr_raw_config(\%running_config, \%fabric_config)
@@ -519,6 +525,19 @@ sub sync_ovn_northbound {
     PVE::Network::SDN::Ovn::sync_northbound($previous_running_config, $running_config);
 }
 
+=head3 generate_ovn_chassis_config()
+
+Configures the local ovn-controller according to the OVN zones of this node.
+Called on boot by pve-sdn-ovn-chassis.service, as the Open vSwitch database
+gets recreated on every boot.
+
+=cut
+
+sub generate_ovn_chassis_config {
+    my $running_config = PVE::Network::SDN::running_config();
+    PVE::Network::SDN::Ovn::configure_chassis($running_config);
+}
+
 sub encode_value {
     my ($type, $key, $value) = @_;
 
diff --git a/src/PVE/Network/SDN/Ovn.pm b/src/PVE/Network/SDN/Ovn.pm
index 142477b..a3fbe44 100644
--- a/src/PVE/Network/SDN/Ovn.pm
+++ b/src/PVE/Network/SDN/Ovn.pm
@@ -5,12 +5,19 @@ use v5.36;
 use JSON;
 
 use PVE::Cluster;
+use PVE::INotify;
 use PVE::Network;
+use PVE::RESTEnvironment qw(log_warn);
 use PVE::Tools qw(run_command);
 
+use PVE::Network::SDN::Zones::Plugin;
+
 our $NBCTL = '/usr/bin/ovn-nbctl';
 our $VSCTL = '/usr/bin/ovs-vsctl';
+our $APPCTL = '/usr/bin/ovn-appctl';
 my $NBCTL_TIMEOUT = 10;
+# seconds to wait for ovn-controller to connect to the southbound database
+our $CONNECT_TIMEOUT = 10;
 
 our $INTEGRATION_BRIDGE = 'br-int';
 
@@ -22,6 +29,9 @@ my $EXT_ZONE = 'pve-sdn-zone';
 my $EXT_VNET = 'pve-sdn-vnet';
 my $EXT_VMID = 'pve-sdn-vmid';
 
+# chassis settings written to the local Open_vSwitch table
+my @CHASSIS_KEYS = ('ovn-remote', 'ovn-encap-type', 'ovn-encap-ip', $EXT_CONTROLLER);
+
 =head3 ovn_controllers(\%running_cfg)
 
 Returns a hash of all controllers of type C<ovn> in C<\%running_cfg>, keyed by
@@ -448,4 +458,173 @@ sub plug_port {
     run_command([$VSCTL, 'set', 'Interface', $port, "external_ids:iface-id=$iface"]);
 }
 
+=head3 local_controllers(\%running_cfg, $nodename)
+
+Returns the OVN controllers used by the zones that are active on C<$nodename>.
+
+=cut
+
+sub local_controllers {
+    my ($running_cfg, $nodename) = @_;
+
+    my $controllers = ovn_controllers($running_cfg);
+    my $zones = $running_cfg->{zones}->{ids} // {};
+
+    my $local = {};
+    for my $zone (values %$zones) {
+        next if $zone->{type} ne 'ovn';
+        next if $zone->{nodes} && !$zone->{nodes}->{$nodename};
+
+        my $controllerid = $zone->{controller};
+        $local->{$controllerid} = $controllers->{$controllerid}
+            if $controllerid && $controllers->{$controllerid};
+    }
+
+    return $local;
+}
+
+=head3 parse_remote_ip($remotes)
+
+Returns the IP address of the first entry of an OVN database connection string
+such as C<tcp:10.0.0.1:6642,tcp:[fd00::1]:6642>.
+
+=cut
+
+sub parse_remote_ip {
+    my ($remotes) = @_;
+
+    my ($first) = split(/,/, $remotes);
+    return $1 if $first =~ m/^[a-z]+:\[([^\]]+)\]:\d+$/;
+    return $1 if $first =~ m/^[a-z]+:([^:]+):\d+$/;
+
+    die "unable to parse OVN database connection '$first'\n";
+}
+
+=head3 chassis_settings($controllerid, $controller, $encap_ip)
+
+Returns the external_ids to set in the local Open_vSwitch table so that
+ovn-controller joins the OVN deployment of C<$controller>.
+
+=cut
+
+sub chassis_settings {
+    my ($controllerid, $controller, $encap_ip) = @_;
+
+    return {
+        'ovn-remote' => $controller->{'sb-connection'},
+        'ovn-encap-type' => $controller->{'encap-type'} // 'geneve',
+        'ovn-encap-ip' => $encap_ip,
+        $EXT_CONTROLLER => $controllerid,
+    };
+}
+
+=head3 configure_chassis(\%running_cfg)
+
+Configures the local ovn-controller for the OVN controller used by the zones of
+this node, or detaches it if no zone uses OVN anymore. Called on every node when
+the SDN configuration gets applied.
+
+=cut
+
+sub configure_chassis {
+    my ($running_cfg) = @_;
+
+    my $nodename = PVE::INotify::nodename();
+    my $controllers = local_controllers($running_cfg, $nodename);
+    my @ids = sort keys %$controllers;
+
+    if (!-x $VSCTL) {
+        die "Open vSwitch is not installed, but node '$nodename' is part of an OVN zone\n"
+            if @ids;
+        return;
+    }
+
+    if (@ids > 1) {
+        die "node '$nodename' is part of zones using different OVN controllers (@ids), but a"
+            . " node can only be attached to one OVN deployment\n";
+    }
+
+    if (!@ids) {
+        # only detach a chassis that was attached by us
+        my $attached = '';
+        eval {
+            run_command(
+                [
+                    $VSCTL,
+                    '--if-exists',
+                    'get',
+                    'Open_vSwitch',
+                    '.',
+                    "external_ids:$EXT_CONTROLLER",
+                ],
+                outfunc => sub { $attached .= $_[0]; },
+            );
+        };
+        if ($attached) {
+            run_command([$VSCTL, 'remove', 'Open_vSwitch', '.', 'external_ids', @CHASSIS_KEYS]);
+        }
+        return;
+    }
+
+    my $id = $ids[0];
+    my $controller = $controllers->{$id};
+
+    my $remote_ip = parse_remote_ip($controller->{'sb-connection'});
+    my ($encap_ip) = PVE::Network::SDN::Zones::Plugin::get_local_route_ip($remote_ip);
+    die "unable to find the local IP to reach the OVN southbound database at $remote_ip\n"
+        if !$encap_ip;
+
+    my $settings = chassis_settings($id, $controller, $encap_ip);
+    run_command([
+        $VSCTL,
+        'set',
+        'Open_vSwitch',
+        '.',
+        map { "external_ids:$_=$settings->{$_}" } sort keys %$settings,
+    ]);
+
+    if (!wait_controller_connected($CONNECT_TIMEOUT)) {
+        log_warn(
+            "ovn-controller is not connected to the OVN southbound database: running guests keep"
+                . " their connectivity, but new guest interfaces and OVN configuration changes are"
+                . " only applied once it is",
+        );
+    }
+}
+
+# ovn-controller needs a few seconds to connect, especially after a restart
+sub wait_controller_connected {
+    my ($timeout) = @_;
+
+    for (my $i = 0; $i < $timeout; $i++) {
+        return 1 if controller_connected();
+        sleep(1);
+    }
+
+    return controller_connected();
+}
+
+=head3 controller_connected()
+
+Returns whether the local ovn-controller is connected to the southbound
+database.
+
+=cut
+
+sub controller_connected {
+    return 0 if !-x $APPCTL;
+
+    my $status = '';
+    eval {
+        run_command(
+            [$APPCTL, '-t', 'ovn-controller', 'connection-status'],
+            outfunc => sub { $status .= $_[0]; },
+            errfunc => sub { },
+            timeout => 5,
+        );
+    };
+
+    return $status eq 'connected';
+}
+
 1;
diff --git a/src/PVE/Network/SDN/Zones/OvnPlugin.pm b/src/PVE/Network/SDN/Zones/OvnPlugin.pm
index 26557e7..fc15e20 100644
--- a/src/PVE/Network/SDN/Zones/OvnPlugin.pm
+++ b/src/PVE/Network/SDN/Zones/OvnPlugin.pm
@@ -66,6 +66,8 @@ sub status {
 
     if (!-d "/sys/class/net/$PVE::Network::SDN::Ovn::INTEGRATION_BRIDGE") {
         push @$err_msg, "missing $PVE::Network::SDN::Ovn::INTEGRATION_BRIDGE";
+    } elsif (!PVE::Network::SDN::Ovn::controller_connected()) {
+        push @$err_msg, "ovn-controller not connected to the southbound database";
     }
 
     return $err_msg;
diff --git a/src/services/Makefile b/src/services/Makefile
index 818c106..b6cd56d 100644
--- a/src/services/Makefile
+++ b/src/services/Makefile
@@ -8,6 +8,7 @@ install:
 	install -d $(SERVICEDIR)
 	install -d $(SERVICEDIR)/dnsmasq@.service.d
 	install -t $(SERVICEDIR)/dnsmasq@.service.d -m 0644 00-dnsmasq-after-networking.conf
+	install -t $(SERVICEDIR) -m 0644 pve-sdn-ovn-chassis.service
 
 .PHONY: clean
 clean:
diff --git a/src/services/pve-sdn-ovn-chassis.service b/src/services/pve-sdn-ovn-chassis.service
new file mode 100644
index 0000000..13775cc
--- /dev/null
+++ b/src/services/pve-sdn-ovn-chassis.service
@@ -0,0 +1,16 @@
+# The Open vSwitch database gets recreated on every boot (see
+# pvenetcommit.service), so the OVN chassis settings need to be set again
+# before the guests get started.
+[Unit]
+Description=Configure the OVN chassis of the Proxmox VE SDN
+Wants=pve-cluster.service
+After=pve-cluster.service ovs-vswitchd.service networking.service
+Before=pve-guests.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/perl -e 'use PVE::Network::SDN; PVE::Network::SDN::generate_ovn_chassis_config()'
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
diff --git a/src/test/run_test_ovn.pl b/src/test/run_test_ovn.pl
index b12ca0b..16b7eef 100755
--- a/src/test/run_test_ovn.pl
+++ b/src/test/run_test_ovn.pl
@@ -347,6 +347,151 @@ eval {
 };
 like($@, qr/isolate-ports is not supported/, 'vnet_update_hook: isolate-ports refused');
 
+# chassis
+
+is(
+    PVE::Network::SDN::Ovn::parse_remote_ip('tcp:10.0.0.1:6642,tcp:10.0.0.2:6642'),
+    '10.0.0.1',
+    'parse_remote_ip: ipv4',
+);
+is(
+    PVE::Network::SDN::Ovn::parse_remote_ip('tcp:[fd00::1]:6642'),
+    'fd00::1',
+    'parse_remote_ip: ipv6',
+);
+
+is_deeply(
+    [sort keys PVE::Network::SDN::Ovn::local_controllers($running_cfg, 'node1')->%*],
+    ['ovnctl'],
+    'local_controllers: zone restricted to other nodes is ignored',
+);
+
+is_deeply(
+    PVE::Network::SDN::Ovn::chassis_settings('ovnctl', $controller, '192.168.0.10'),
+    {
+        'ovn-remote' => $controller->{'sb-connection'},
+        'ovn-encap-type' => 'geneve',
+        'ovn-encap-ip' => '192.168.0.10',
+        'pve-sdn-controller' => 'ovnctl',
+    },
+    'chassis_settings',
+);
+
+my $vsctl_calls = [];
+my $attached = '';
+$PVE::Network::SDN::Ovn::VSCTL = '/bin/true';
+$ovn_module->mock(
+    run_command => sub {
+        my ($cmd, %param) = @_;
+        if (grep { $_ eq 'get' } @$cmd) {
+            $param{outfunc}->($attached) if $attached;
+            return;
+        }
+        push @$vsctl_calls, [$cmd->@[1 .. $#$cmd]];
+    },
+    controller_connected => sub { return 1 },
+);
+my $inotify_module = Test::MockModule->new('PVE::INotify');
+$inotify_module->mock(nodename => sub { return 'node1' });
+my $zones_plugin_module = Test::MockModule->new('PVE::Network::SDN::Zones::Plugin');
+$zones_plugin_module->mock(get_local_route_ip => sub { return ('192.168.0.10', 'vmbr0') });
+
+PVE::Network::SDN::Ovn::configure_chassis($running_cfg);
+is_deeply(
+    $vsctl_calls,
+    [[
+        'set',
+        'Open_vSwitch',
+        '.',
+        'external_ids:ovn-encap-ip=192.168.0.10',
+        'external_ids:ovn-encap-type=geneve',
+        "external_ids:ovn-remote=$controller->{'sb-connection'}",
+        'external_ids:pve-sdn-controller=ovnctl',
+    ]],
+    'configure_chassis: attach chassis',
+);
+
+$vsctl_calls = [];
+PVE::Network::SDN::Ovn::configure_chassis({});
+is_deeply($vsctl_calls, [], 'configure_chassis: leave chassis not attached by PVE alone');
+
+$attached = '"ovnctl"';
+PVE::Network::SDN::Ovn::configure_chassis({});
+is_deeply(
+    $vsctl_calls,
+    [[
+        'remove',
+        'Open_vSwitch',
+        '.',
+        'external_ids',
+        'ovn-remote',
+        'ovn-encap-type',
+        'ovn-encap-ip',
+        'pve-sdn-controller',
+    ]],
+    'configure_chassis: detach chassis',
+);
+
+my $two_controllers = {
+    controllers => { ids => { a => $controller, b => $controller } },
+    zones => {
+        ids => {
+            za => { type => 'ovn', controller => 'a' },
+            zb => { type => 'ovn', controller => 'b' },
+        },
+    },
+};
+eval { PVE::Network::SDN::Ovn::configure_chassis($two_controllers) };
+like(
+    $@,
+    qr/can only be attached to one OVN deployment/,
+    'configure_chassis: one deployment per node',
+);
+
+# chassis configured when the SDN configuration gets applied
+
+my $sdn_module = Test::MockModule->new('PVE::Network::SDN');
+my $written;
+$sdn_module->mock(
+    running_config => sub { return $running_cfg },
+    generate_raw_etc_network_config => sub { return "raw" },
+    write_raw_etc_network_config => sub { $written = $_[0] },
+);
+my $chassis_cfg;
+$ovn_module->mock(configure_chassis => sub { $chassis_cfg = $_[0] });
+PVE::Network::SDN::generate_etc_network_config();
+is($written, 'raw', 'generate_etc_network_config: network config written');
+is($chassis_cfg, $running_cfg, 'generate_etc_network_config: chassis configured');
+
+$written = undef;
+$ovn_module->mock(configure_chassis => sub { die "ovs down\n" });
+my @warnings;
+$sdn_module->mock(log_warn => sub { push @warnings, $_[0] });
+eval { PVE::Network::SDN::generate_etc_network_config() };
+is($@, '', 'generate_etc_network_config: chassis failure does not abort');
+is($written, 'raw', 'generate_etc_network_config: network config written despite chassis failure');
+like(
+    join('', @warnings),
+    qr/failed to configure the OVN chassis - ovs down/,
+    'generate_etc_network_config: chassis failure is logged',
+);
+$ovn_module->unmock('configure_chassis');
+$sdn_module->unmock_all();
+
+# ovn-controller connection
+
+my $connection_checks = 0;
+my $connected_after = 2;
+$ovn_module->mock(
+    controller_connected =>
+        sub { $connection_checks++; return $connection_checks >= $connected_after },
+);
+ok(PVE::Network::SDN::Ovn::wait_controller_connected(10), 'wait_controller_connected: connects');
+is($connection_checks, 2, 'wait_controller_connected: waits until connected');
+($connection_checks, $connected_after) = (0, 100);
+ok(!PVE::Network::SDN::Ovn::wait_controller_connected(1), 'wait_controller_connected: timeout');
+$ovn_module->mock(controller_connected => sub { return 1 });
+
 # controller schema
 
 ok(
-- 
2.47.3