[RFC PATCH network 09/16] sdn: ovn: support ssl connections to the databases

["Rémi PAETA" <remi.paeta@comaite.com>]

Allow ssl:<ip>:<port> connections to the OVN databases, so that the
northbound and southbound traffic gets authenticated and encrypted.
Without it, any host reaching the southbound database can register as
a chassis and claim the ports of guests.

As for the OVN deployment itself, the certificates are provided by the
administrator, in /etc/pve/priv/sdn/ovn/<controller>/: the CA
certificate ca.pem, and a certificate and key per node, <node>.pem and
<node>.key. Certificates per node allow restricting each chassis to
its own data with the southbound role based access control.

ovn-nbctl gets them as options. ovn-controller only takes them on its
command line, so the node certificate gets copied locally and a runtime
drop-in adds the options to its unit, written along with the chassis
settings on boot and on network reload. ovn-controller only gets
restarted when this setup changes.

Signed-off-by: Rémi PAETA <remi.paeta@comaite.com>
---
 src/PVE/Network/SDN/Controllers/OvnPlugin.pm |  14 +-
 src/PVE/Network/SDN/Ovn.pm                   | 168 ++++++++++++++++++-
 src/PVE/Network/SDN/Zones/OvnPlugin.pm       |   9 +-
 src/test/run_test_ovn.pl                     | 103 +++++++++++-
 4 files changed, 281 insertions(+), 13 deletions(-)

diff --git a/src/PVE/Network/SDN/Controllers/OvnPlugin.pm b/src/PVE/Network/SDN/Controllers/OvnPlugin.pm
index f7a9b96..9e31eca 100644
--- a/src/PVE/Network/SDN/Controllers/OvnPlugin.pm
+++ b/src/PVE/Network/SDN/Controllers/OvnPlugin.pm
@@ -15,11 +15,12 @@ sub pve_verify_sdn_ovn_remotes {
     my ($remotes, $noerr) = @_;
 
     for my $remote (split(/,/, $remotes)) {
-        next if $remote =~ m/^tcp:$PVE::Tools::IPV4RE:\d+$/;
-        next if $remote =~ m/^tcp:\[$PVE::Tools::IPV6RE\]:\d+$/;
+        next if $remote =~ m/^(?:tcp|ssl):$PVE::Tools::IPV4RE:\d+$/;
+        next if $remote =~ m/^(?:tcp|ssl):\[$PVE::Tools::IPV6RE\]:\d+$/;
 
         return undef if $noerr;
-        die "invalid OVN database connection '$remote', expected tcp:<ip>:<port>\n";
+        die "invalid OVN database connection '$remote', expected tcp:<ip>:<port> or"
+            . " ssl:<ip>:<port>\n";
     }
 
     return $remotes;
@@ -33,13 +34,16 @@ sub properties {
     return {
         'nb-connection' => {
             description => "Comma separated list of the OVN northbound database servers,"
-                . " for example tcp:10.0.0.1:6641,tcp:10.0.0.2:6641.",
+                . " for example tcp:10.0.0.1:6641,tcp:10.0.0.2:6641. With ssl:<ip>:<port>,"
+                . " every node needs its certificate in /etc/pve/priv/sdn/ovn/<controller>/"
+                . "<node>.pem and .key, next to the CA certificate ca.pem.",
             type => 'string',
             format => 'pve-sdn-ovn-remotes',
         },
         'sb-connection' => {
             description => "Comma separated list of the OVN southbound database servers,"
-                . " for example tcp:10.0.0.1:6642,tcp:10.0.0.2:6642.",
+                . " for example tcp:10.0.0.1:6642,tcp:10.0.0.2:6642. SSL connections use the"
+                . " same certificates as for the northbound database.",
             type => 'string',
             format => 'pve-sdn-ovn-remotes',
         },
diff --git a/src/PVE/Network/SDN/Ovn.pm b/src/PVE/Network/SDN/Ovn.pm
index cdedc4a..ad9dda3 100644
--- a/src/PVE/Network/SDN/Ovn.pm
+++ b/src/PVE/Network/SDN/Ovn.pm
@@ -2,6 +2,7 @@ package PVE::Network::SDN::Ovn;
 
 use v5.36;
 
+use File::Path qw(make_path);
 use JSON;
 
 use PVE::Cluster;
@@ -21,6 +22,17 @@ our $CONNECT_TIMEOUT = 10;
 
 our $INTEGRATION_BRIDGE = 'br-int';
 
+# certificates provided by the administrator for SSL connections to the OVN
+# databases: <controller>/ca.pem, and <controller>/<node>.pem and .key
+our $SSL_DIR = '/etc/pve/priv/sdn/ovn';
+
+# local copies of the certificates of this node for ovn-controller, which only
+# takes them as command line options, through a runtime drop-in for its unit
+our $CHASSIS_SSL_DIR = '/etc/ovn/pve-sdn';
+our $CONTROLLER_UNIT_FILE = '/usr/lib/systemd/system/ovn-controller.service';
+our $CONTROLLER_DROPIN = '/run/systemd/system/ovn-controller.service.d/pve-sdn-ssl.conf';
+our $SYSTEMCTL = '/usr/bin/systemctl';
+
 # external_ids keys tagging the objects managed by PVE. Northbound objects
 # without them are never touched, so the OVN deployment can be shared with
 # other consumers.
@@ -36,7 +48,7 @@ my @CHASSIS_KEYS = ('ovn-remote', 'ovn-encap-type', 'ovn-encap-ip', $EXT_CONTROL
 =head3 ovn_controllers(\%running_cfg)
 
 Returns a hash of all controllers of type C<ovn> in C<\%running_cfg>, keyed by
-their ID.
+their ID, as returned by C<controller_config>.
 
 =cut
 
@@ -46,11 +58,63 @@ sub ovn_controllers {
     my $controllers = $running_cfg->{controllers}->{ids} // {};
 
     return {
-        map { $_ => $controllers->{$_} }
+        map { $_ => controller_config($running_cfg, $_) }
         grep { $controllers->{$_}->{type} eq 'ovn' } keys %$controllers
     };
 }
 
+=head3 controller_config(\%running_cfg, $controllerid)
+
+Returns a copy of the configuration of the controller C<$controllerid>, with its
+ID under C<id>, which is needed to find its certificates.
+
+=cut
+
+sub controller_config {
+    my ($running_cfg, $controllerid) = @_;
+
+    my $controller = $running_cfg->{controllers}->{ids}->{$controllerid} or return;
+
+    return { $controller->%*, id => $controllerid };
+}
+
+=head3 uses_ssl($connection)
+
+Returns whether the OVN database connection string C<$connection> uses SSL.
+
+=cut
+
+sub uses_ssl {
+    my ($connection) = @_;
+
+    return (grep { m/^ssl:/ } split(/,/, $connection // '')) ? 1 : 0;
+}
+
+=head3 ssl_files($controllerid, $nodename)
+
+Returns the paths of the private key, certificate and CA certificate that node
+C<$nodename> uses for the SSL connections to the databases of the OVN
+controller C<$controllerid>. Dies if one of them is missing.
+
+=cut
+
+sub ssl_files {
+    my ($controllerid, $nodename) = @_;
+
+    my $dir = "$SSL_DIR/$controllerid";
+    my $files = {
+        'private-key' => "$dir/$nodename.key",
+        certificate => "$dir/$nodename.pem",
+        'ca-cert' => "$dir/ca.pem",
+    };
+
+    my @missing = grep { !-f $_ } sort values %$files;
+    die "missing SSL certificates for the OVN controller '$controllerid': @missing\n"
+        if @missing;
+
+    return $files;
+}
+
 =head3 desired_state(\%running_cfg)
 
 Computes the northbound objects that should exist for C<\%running_cfg>, keyed by
@@ -240,10 +304,22 @@ sub guest_nics {
 sub nbctl {
     my ($controller, @args) = @_;
 
+    my @ssl_args = ();
+    if (uses_ssl($controller->{'nb-connection'})) {
+        my $files = ssl_files($controller->{id}, PVE::INotify::nodename());
+        @ssl_args = map { "--$_=$files->{$_}" } sort keys %$files;
+    }
+
     my ($output, $errors) = ('', '');
     eval {
         run_command(
-            [$NBCTL, "--db=$controller->{'nb-connection'}", "--timeout=$NBCTL_TIMEOUT", @args],
+            [
+                $NBCTL,
+                "--db=$controller->{'nb-connection'}",
+                "--timeout=$NBCTL_TIMEOUT",
+                @ssl_args,
+                @args,
+            ],
             outfunc => sub { $output .= "$_[0]\n"; },
             errfunc => sub { $errors .= "$_[0]\n"; },
         );
@@ -752,6 +828,7 @@ sub configure_chassis {
         if ($attached) {
             run_command([$VSCTL, 'remove', 'Open_vSwitch', '.', 'external_ids', @CHASSIS_KEYS]);
         }
+        configure_chassis_ssl(undef, $nodename);
         return;
     }
 
@@ -763,6 +840,10 @@ sub configure_chassis {
     die "unable to find the local IP to reach the OVN southbound database at $remote_ip\n"
         if !$encap_ip;
 
+    configure_chassis_ssl(
+        uses_ssl($controller->{'sb-connection'}) ? $id : undef, $nodename,
+    );
+
     my $settings = chassis_settings($id, $controller, $encap_ip);
     run_command([
         $VSCTL,
@@ -793,6 +874,87 @@ sub wait_controller_connected {
     return controller_connected();
 }
 
+=head3 chassis_ssl_dropin($unit_content)
+
+Returns the systemd drop-in for the ovn-controller unit with the content
+C<$unit_content>, which adds the options for the local copies of the
+certificates to its command line.
+
+=cut
+
+sub chassis_ssl_dropin {
+    my ($unit_content) = @_;
+
+    my @exec_start = $unit_content =~ m/^ExecStart=(.+)$/mg;
+    die "unable to find the command line of ovn-controller in $CONTROLLER_UNIT_FILE\n"
+        if scalar(@exec_start) != 1;
+
+    my $options = join(
+        ' ',
+        "--ovn-controller-ssl-key=$CHASSIS_SSL_DIR/chassis.key",
+        "--ovn-controller-ssl-cert=$CHASSIS_SSL_DIR/chassis.pem",
+        "--ovn-controller-ssl-ca-cert=$CHASSIS_SSL_DIR/ca.pem",
+    );
+
+    return "# generated by the Proxmox VE SDN, do not edit\n"
+        . "[Service]\nExecStart=\nExecStart=$exec_start[0] $options\n";
+}
+
+# writes $content to $path if it changed, returns whether it did
+sub update_file {
+    my ($path, $content, $perm) = @_;
+
+    my $current = -f $path ? PVE::Tools::file_get_contents($path) : undef;
+    return 0 if defined($current) && $current eq $content;
+
+    PVE::Tools::file_set_contents($path, $content, $perm);
+    return 1;
+}
+
+=head3 configure_chassis_ssl($controllerid, $nodename)
+
+Sets up the certificates of node C<$nodename> for the SSL connection of
+ovn-controller to the southbound database of the OVN controller
+C<$controllerid>, or removes that setup if C<$controllerid> is undefined.
+ovn-controller gets restarted if anything changed.
+
+=cut
+
+sub configure_chassis_ssl {
+    my ($controllerid, $nodename) = @_;
+
+    my $changed = 0;
+
+    if (defined($controllerid)) {
+        my $files = ssl_files($controllerid, $nodename);
+
+        make_path($CHASSIS_SSL_DIR, { mode => 0700 });
+        my $copies = {
+            'private-key' => ['chassis.key', 0600],
+            certificate => ['chassis.pem', 0644],
+            'ca-cert' => ['ca.pem', 0644],
+        };
+        for my $type (sort keys %$copies) {
+            my ($name, $perm) = $copies->{$type}->@*;
+            my $content = PVE::Tools::file_get_contents($files->{$type});
+            $changed = 1 if update_file("$CHASSIS_SSL_DIR/$name", $content, $perm);
+        }
+
+        my $dropin = chassis_ssl_dropin(PVE::Tools::file_get_contents($CONTROLLER_UNIT_FILE));
+        make_path($CONTROLLER_DROPIN =~ s|/[^/]+$||r);
+        $changed = 1 if update_file($CONTROLLER_DROPIN, $dropin, 0644);
+    } elsif (-e $CONTROLLER_DROPIN) {
+        unlink($CONTROLLER_DROPIN) or die "unable to remove $CONTROLLER_DROPIN - $!\n";
+        unlink(glob("$CHASSIS_SSL_DIR/*"));
+        $changed = 1;
+    }
+
+    return if !$changed;
+
+    run_command([$SYSTEMCTL, 'daemon-reload']);
+    run_command([$SYSTEMCTL, 'try-restart', 'ovn-controller.service']);
+}
+
 =head3 controller_connected()
 
 Returns whether the local ovn-controller is connected to the southbound
diff --git a/src/PVE/Network/SDN/Zones/OvnPlugin.pm b/src/PVE/Network/SDN/Zones/OvnPlugin.pm
index 83c2c51..a474dce 100644
--- a/src/PVE/Network/SDN/Zones/OvnPlugin.pm
+++ b/src/PVE/Network/SDN/Zones/OvnPlugin.pm
@@ -92,8 +92,10 @@ sub tap_plug {
     my ($class, $plugin_config, $vnet, $tag, $iface, $vnetid, $firewall, $trunks, $rate) = @_;
 
     my $controllerid = $plugin_config->{controller};
-    my $controller =
-        PVE::Network::SDN::running_config()->{controllers}->{ids}->{$controllerid};
+    my $controller = PVE::Network::SDN::Ovn::controller_config(
+        PVE::Network::SDN::running_config(),
+        $controllerid,
+    );
     die "OVN controller '$controllerid' of vnet '$vnetid' is not applied yet\n"
         if !$controller;
 
@@ -119,7 +121,8 @@ sub add_bridge_fdb {
 
     my $running_config = PVE::Network::SDN::running_config();
     my $controllerid = $plugin_config->{controller};
-    my $controller = $running_config->{controllers}->{ids}->{$controllerid} or return;
+    my $controller = PVE::Network::SDN::Ovn::controller_config($running_config, $controllerid)
+        or return;
 
     eval {
         my $vnetid = PVE::Network::SDN::Ovn::port_vnet($controller, $iface)
diff --git a/src/test/run_test_ovn.pl b/src/test/run_test_ovn.pl
index cb36a0f..83bfd80 100755
--- a/src/test/run_test_ovn.pl
+++ b/src/test/run_test_ovn.pl
@@ -5,6 +5,7 @@ use warnings;
 
 use lib qw(..);
 
+use File::Temp qw(tempdir);
 use JSON;
 use Test::More;
 use Test::MockModule;
@@ -15,6 +16,10 @@ use PVE::Network::SDN::Controllers::OvnPlugin;
 use PVE::Network::SDN::SubnetPlugin;
 use PVE::Network::SDN::Zones::OvnPlugin;
 
+# keep the tests away from the systemd configuration of the host
+my $tmpdir = tempdir(CLEANUP => 1);
+$PVE::Network::SDN::Ovn::CONTROLLER_DROPIN = "$tmpdir/dropin/pve-sdn-ssl.conf";
+
 my $controller = {
     type => 'ovn',
     'nb-connection' => 'tcp:192.168.0.1:6641,tcp:192.168.0.2:6641',
@@ -668,15 +673,109 @@ is($connection_checks, 2, 'wait_controller_connected: waits until connected');
 ok(!PVE::Network::SDN::Ovn::wait_controller_connected(1), 'wait_controller_connected: timeout');
 $ovn_module->mock(controller_connected => sub { return 1 });
 
+# SSL
+
+ok(PVE::Network::SDN::Ovn::uses_ssl('ssl:10.0.0.1:6641,ssl:10.0.0.2:6641'), 'uses_ssl: ssl');
+ok(!PVE::Network::SDN::Ovn::uses_ssl('tcp:10.0.0.1:6641'), 'uses_ssl: tcp');
+
+$PVE::Network::SDN::Ovn::SSL_DIR = "$tmpdir/ssl";
+eval { PVE::Network::SDN::Ovn::ssl_files('ovnctl', 'node1') };
+like($@, qr/missing SSL certificates .*ca\.pem .*node1\.key .*node1\.pem/, 'ssl_files: missing');
+
+mkdir("$tmpdir/ssl");
+mkdir("$tmpdir/ssl/ovnctl");
+PVE::Tools::file_set_contents("$tmpdir/ssl/ovnctl/$_", "$_\n") for qw(ca.pem node1.pem node1.key);
+is_deeply(
+    PVE::Network::SDN::Ovn::ssl_files('ovnctl', 'node1'),
+    {
+        'private-key' => "$tmpdir/ssl/ovnctl/node1.key",
+        certificate => "$tmpdir/ssl/ovnctl/node1.pem",
+        'ca-cert' => "$tmpdir/ssl/ovnctl/ca.pem",
+    },
+    'ssl_files: present',
+);
+
+my $ssl_controller = {
+    id => 'ovnctl',
+    'nb-connection' => 'ssl:192.168.0.1:6641',
+    'sb-connection' => 'ssl:192.168.0.1:6642',
+};
+my $nbctl_cmd;
+$ovn_module->mock(run_command => sub { $nbctl_cmd = $_[0] });
+PVE::Network::SDN::Ovn::nbctl($ssl_controller, 'ls-list');
+is_deeply(
+    [$nbctl_cmd->@[1 .. $#$nbctl_cmd]],
+    [
+        '--db=ssl:192.168.0.1:6641',
+        '--timeout=10',
+        "--ca-cert=$tmpdir/ssl/ovnctl/ca.pem",
+        "--certificate=$tmpdir/ssl/ovnctl/node1.pem",
+        "--private-key=$tmpdir/ssl/ovnctl/node1.key",
+        'ls-list',
+    ],
+    'nbctl: certificates of the node for ssl connections',
+);
+
+is_deeply(
+    PVE::Network::SDN::Ovn::controller_config($running_cfg, 'ovnctl'),
+    { $controller->%*, id => 'ovnctl' },
+    'controller_config: with its ID',
+);
+
+my $unit = "[Service]\nExecStart=/usr/share/ovn/scripts/ovn-ctl start_controller \$OVN_CTL_OPTS\n";
+like(
+    PVE::Network::SDN::Ovn::chassis_ssl_dropin($unit),
+    qr{^ExecStart=\nExecStart=/usr/share/ovn/scripts/ovn-ctl start_controller \$OVN_CTL_OPTS --ovn-controller-ssl-key=\S+/chassis\.key --ovn-controller-ssl-cert=\S+/chassis\.pem --ovn-controller-ssl-ca-cert=\S+/ca\.pem$}m,
+    'chassis_ssl_dropin: ssl options appended to the command line of the unit',
+);
+eval { PVE::Network::SDN::Ovn::chassis_ssl_dropin("[Service]\n") };
+like($@, qr/unable to find the command line/, 'chassis_ssl_dropin: unexpected unit');
+
+$PVE::Network::SDN::Ovn::CHASSIS_SSL_DIR = "$tmpdir/chassis";
+$PVE::Network::SDN::Ovn::CONTROLLER_UNIT_FILE = "$tmpdir/ovn-controller.service";
+PVE::Tools::file_set_contents("$tmpdir/ovn-controller.service", $unit);
+my $systemctl_calls = [];
+$ovn_module->mock(run_command => sub { push @$systemctl_calls, [$_[0]->@[1 .. $_[0]->$#*]] });
+my $restart = [['daemon-reload'], ['try-restart', 'ovn-controller.service']];
+
+PVE::Network::SDN::Ovn::configure_chassis_ssl('ovnctl', 'node1');
+is_deeply($systemctl_calls, $restart, 'configure_chassis_ssl: restart on setup');
+is(
+    PVE::Tools::file_get_contents("$tmpdir/chassis/chassis.key"),
+    "node1.key\n",
+    'configure_chassis_ssl: key copied',
+);
+is((stat("$tmpdir/chassis/chassis.key"))[2] & 07777, 0600, 'configure_chassis_ssl: key mode');
+ok(-f $PVE::Network::SDN::Ovn::CONTROLLER_DROPIN, 'configure_chassis_ssl: drop-in written');
+
+$systemctl_calls = [];
+PVE::Network::SDN::Ovn::configure_chassis_ssl('ovnctl', 'node1');
+is_deeply($systemctl_calls, [], 'configure_chassis_ssl: no restart without changes');
+
+PVE::Tools::file_set_contents("$tmpdir/ssl/ovnctl/node1.pem", "renewed\n");
+PVE::Network::SDN::Ovn::configure_chassis_ssl('ovnctl', 'node1');
+is_deeply($systemctl_calls, $restart, 'configure_chassis_ssl: restart on renewed certificate');
+
+$systemctl_calls = [];
+PVE::Network::SDN::Ovn::configure_chassis_ssl(undef, 'node1');
+is_deeply($systemctl_calls, $restart, 'configure_chassis_ssl: restart on removal');
+ok(!-e $PVE::Network::SDN::Ovn::CONTROLLER_DROPIN, 'configure_chassis_ssl: drop-in removed');
+ok(!-e "$tmpdir/chassis/chassis.key", 'configure_chassis_ssl: key removed');
+
+$systemctl_calls = [];
+PVE::Network::SDN::Ovn::configure_chassis_ssl(undef, 'node1');
+is_deeply($systemctl_calls, [], 'configure_chassis_ssl: nothing to remove');
+$ovn_module->unmock('run_command');
+
 # controller schema
 
 ok(
     PVE::Network::SDN::Controllers::OvnPlugin::pve_verify_sdn_ovn_remotes(
-        'tcp:10.0.0.1:6641,tcp:[fd00::1]:6641', 1,
+        'tcp:10.0.0.1:6641,tcp:[fd00::1]:6641,ssl:10.0.0.2:6641,ssl:[fd00::2]:6641', 1,
     ),
     'remotes format: valid',
 );
-for my $invalid ('ssl:10.0.0.1:6641', 'tcp:10.0.0.1', 'tcp:ovn.example.com:6641') {
+for my $invalid ('unix:/run/ovn/ovnnb_db.sock', 'tcp:10.0.0.1', 'ssl:ovn.example.com:6641') {
     ok(
         !PVE::Network::SDN::Controllers::OvnPlugin::pve_verify_sdn_ovn_remotes($invalid, 1),
         "remotes format: invalid '$invalid'",
-- 
2.47.3