[PATCH e2e-tests 26/38] tests: ceph: verify setting up a hyper-converged cluster

["Michael Köppl" <m.koeppl@proxmox.com>]

Set up Ceph on the main cluster through the API with dedicated
public and cluster networks, as the setup wizard requires a public
network and the documentation recommends separating Ceph traffic.

Check that the configuration on pmxcfs has the networks and is linked
to from /etc/ceph on every node, that the monitors form a quorum on
the public network, and that the OSDs use the public network for
clients and the cluster network for replication. Also check the
managers and creating a pool with an RBD storage.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/ceph/test_ceph_cluster_setup.pl | 206 ++++++++++++++++++++++++++
 tests/testcases.ron                   |  11 ++
 2 files changed, 217 insertions(+)
 create mode 100755 tests/ceph/test_ceph_cluster_setup.pl

diff --git a/tests/ceph/test_ceph_cluster_setup.pl b/tests/ceph/test_ceph_cluster_setup.pl
new file mode 100755
index 0000000..5bd47d3
--- /dev/null
+++ b/tests/ceph/test_ceph_cluster_setup.pl
@@ -0,0 +1,206 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<test_ceph_cluster_setup> - Set up a hyper-converged Ceph cluster on 3 nodes.
+
+=head1 DESCRIPTION
+
+This testcase sets up Ceph on the cluster nodes. It gives every node an address
+in a dedicated Ceph public and cluster network and writes the initial
+configuration with these networks. It creates a monitor and a manager on every
+node and an OSD on every Ceph disk.
+
+It verifies that the configuration on pmxcfs has the networks and is linked to
+from /etc/ceph on every node, that the monitors form a quorum and use the public
+network, that one manager is active with the others on standby, and that all
+OSDs are up and in and use the public network for clients and the cluster
+network for replication. Finally, it creates a pool together with an RBD
+storage and verifies that the storage is active on every node and the cluster
+reports HEALTH_OK.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use JSON;
+
+use lib '.';
+use Proxmox::Test::Ceph qw(
+    CEPH_CLUSTER_NETWORK
+    CEPH_PUBLIC_NETWORK
+    ceph_disks
+    ceph_osds
+    ceph_status
+    check_storage_active
+    create_managers
+    create_monitors
+    create_osds
+    create_pool
+    init_ceph
+    setup_ceph_network
+    wait_for_ceph
+);
+use Proxmox::Test::Helper qw(get_pve_instances);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_instances = get_pve_instances();
+die "not enough PVE instances" if scalar($pve_instances->@*) < 3;
+
+my $first = $pve_instances->[0];
+my $monitors = scalar($pve_instances->@*);
+my $pool = 'e2e-setup';
+my $osds = 0;
+$osds += scalar(ceph_disks($_)->@*) for $pve_instances->@*;
+
+my $public_network = CEPH_PUBLIC_NETWORK;
+my $cluster_network = CEPH_CLUSTER_NETWORK;
+my $addresses;
+
+# Commands like 'ceph' find the configuration on pmxcfs through this link.
+my $check_config_link = sub {
+    my ($instance) = @_;
+
+    my ($target) = eval { $instance->run_command('readlink /etc/ceph/ceph.conf') };
+    is(
+        $target,
+        "/etc/pve/ceph.conf\n",
+        "/etc/ceph/ceph.conf links to the config on pmxcfs on '"
+            . $instance->get_nodename() . "'",
+    );
+};
+
+Proxmox::Test::Testcase->new('ceph_cluster_setup')->with([{}])->step(
+    'set up Ceph networks',
+    sub {
+        $addresses = setup_ceph_network($pve_instances);
+        pass('nodes reach each other in the Ceph public and cluster network');
+    },
+)->step(
+    'initialize Ceph',
+    sub {
+        init_ceph(
+            $pve_instances,
+            network => $public_network,
+            'cluster-network' => $cluster_network,
+        );
+
+        my $node = $first->get_nodename();
+        my $conf = $first->client()->get("/nodes/$node/ceph/cfg/raw", {});
+        like($conf, qr/^\s*fsid\s*=\s*[0-9a-f-]{36}\s*$/m, 'ceph.conf has a cluster fsid');
+        like(
+            $conf,
+            qr/^\s*public_network\s*=\s*\Q$public_network\E\s*$/m,
+            'ceph.conf has the public network',
+        );
+        like(
+            $conf,
+            qr/^\s*cluster_network\s*=\s*\Q$cluster_network\E\s*$/m,
+            'ceph.conf has the cluster network',
+        );
+
+        $check_config_link->($first);
+    },
+)->step(
+    'create monitors',
+    sub {
+        create_monitors($pve_instances);
+
+        my @quorum = sort(ceph_status($first)->{quorum_names}->@*);
+        my @nodes = sort map { $_->get_nodename() } $pve_instances->@*;
+        is_deeply(\@quorum, \@nodes, 'every node has a monitor in the quorum');
+
+        my $node = $first->get_nodename();
+        my $monitor_list = $first->client()->get("/nodes/$node/ceph/mon", {});
+        for my $instance ($pve_instances->@*) {
+            my $name = $instance->get_nodename();
+            my ($monitor) = grep { $_->{name} eq $name } $monitor_list->@*;
+            like(
+                ($monitor // {})->{addr} // '',
+                qr/^\Q$addresses->{$name}->{public}\E:/,
+                "monitor on '$name' uses its address in the public network",
+            );
+
+            # The nodes other than the one that initialized Ceph should by now also have the link.
+            $check_config_link->($instance);
+        }
+    },
+)->step(
+    'create managers',
+    sub {
+        create_managers($pve_instances);
+
+        # A new manager is not immediately registered as standby
+        my $mgrmap;
+        eval {
+            poll_until(
+                sub {
+                    $mgrmap = ceph_status($first)->{mgrmap};
+                    return $mgrmap->{available} && $mgrmap->{num_standbys} == $monitors - 1;
+                },
+                timeout => 60,
+                interval => 2,
+                description => 'one active and ' . ($monitors - 1) . ' standby managers',
+            );
+        };
+        diag("manager map: " . JSON::encode_json($mgrmap // {})) if $@;
+        ok($mgrmap->{available}, 'a manager is active');
+        is($mgrmap->{num_standbys}, $monitors - 1, 'the other managers are on standby');
+    },
+)->step(
+    'create OSDs',
+    sub {
+        is(create_osds($pve_instances), $osds, "created $osds OSDs");
+        wait_for_ceph($first, monitors => $monitors, osds => $osds);
+        pass("all $osds OSDs are up and in");
+
+        for my $instance ($pve_instances->@*) {
+            my $name = $instance->get_nodename();
+            my $node_osds = ceph_osds($instance);
+            is(
+                scalar($node_osds->@*),
+                scalar(ceph_disks($instance)->@*),
+                "OSD tree lists an OSD for every Ceph disk under '$name'",
+            );
+
+            for my $osd ($node_osds->@*) {
+                my $metadata =
+                    $instance->client()->get("/nodes/$name/ceph/osd/$osd->{id}/metadata", {})
+                    ->{osd};
+                like(
+                    $metadata->{front_addr} // '',
+                    qr/\b\Q$addresses->{$name}->{public}\E:/,
+                    "osd.$osd->{id} serves clients in the public network",
+                );
+                like(
+                    $metadata->{back_addr} // '',
+                    qr/\b\Q$addresses->{$name}->{cluster}\E:/,
+                    "osd.$osd->{id} replicates in the cluster network",
+                );
+            }
+        }
+    },
+)->step(
+    'create pool with RBD storage',
+    sub {
+        create_pool($first, $pool, add_storages => 1);
+
+        my $storage = $first->client()->get("/storage/$pool", {});
+        is($storage->{type}, 'rbd', "RBD storage '$pool' was added");
+        is($storage->{pool}, $pool, "storage '$pool' uses the new pool");
+
+        check_storage_active($_, $pool) for $pve_instances->@*;
+    },
+)->step(
+    'verify cluster health',
+    sub {
+        wait_for_ceph($first, monitors => $monitors, osds => $osds, health_ok => 1);
+        pass('the Ceph cluster reports HEALTH_OK');
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index fba3e06..47c06b0 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -392,6 +392,17 @@ TestcaseConfig(
             ],
             tag: "sdn",
         ),
+        Testcase(
+            name: "ceph_cluster_setup",
+            run: "./tests/ceph/test_ceph_cluster_setup.pl",
+            timeout: 1800,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "ceph",
+        ),
     ],
     rollback_between_tests: Some("pre-test"),
 )
-- 
2.47.3