[PATCH e2e-tests 29/38] tests: ceph: verify replacing an OSD

["Michael Köppl" <m.koeppl@proxmox.com>]

Replace an OSD as the documentation describes, while a container
runs on an RBD storage. Mark the OSD out, wait until its placement
groups have moved, check that it is safe to stop and stop it, then
destroy it with cleanup and create a new OSD on the same disk.

Destroying with cleanup must remove the OSD from the OSD tree, leave
neither an enabled service nor a data directory behind and leave the
disk unused, so that it can be reused right away. Afterwards, the
cluster must be healthy with all OSDs up and in, and the container
must have kept running with its data intact.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/ceph/test_ceph_osd_replace.pl | 237 ++++++++++++++++++++++++++++
 tests/testcases.ron                 |  11 ++
 2 files changed, 248 insertions(+)
 create mode 100755 tests/ceph/test_ceph_osd_replace.pl

diff --git a/tests/ceph/test_ceph_osd_replace.pl b/tests/ceph/test_ceph_osd_replace.pl
new file mode 100755
index 0000000..51857ec
--- /dev/null
+++ b/tests/ceph/test_ceph_osd_replace.pl
@@ -0,0 +1,237 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<test_ceph_osd_replace> - Test replacing an OSD of a hyper-converged Ceph cluster.
+
+=head1 DESCRIPTION
+
+This testcase sets up Ceph on the cluster nodes with a pool and an RBD storage
+and starts a container on that storage. It then replaces an OSD on the second
+node as the documentation describes. It marks the OSD out, waits until Ceph has
+moved all placement groups off it, checks that it is safe to stop and stops it.
+It destroys the OSD with cleanup, which must remove the OSD from the OSD tree,
+leave neither an enabled service nor a data directory of it behind and leave
+its disk unused, so that a new OSD can be created on the same disk right away.
+
+Afterwards, the cluster must report HEALTH_OK with all OSDs up and in again.
+The container must have kept running with its data intact.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use lib '.';
+use Proxmox::Test::Ceph qw(
+    ceph_disks
+    ceph_osds
+    ceph_status
+    create_ceph_cluster
+    create_pool
+    wait_for_ceph
+);
+use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instances);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_instances = get_pve_instances();
+die "not enough PVE instances" if scalar($pve_instances->@*) < 3;
+
+my $first = $pve_instances->[0];
+my $second = $pve_instances->[1];
+my $node = $first->get_nodename();
+my $osd_node = $second->get_nodename();
+my $client = $first->client();
+my $osd_client = $second->client();
+
+my $pool = 'e2e-replace';
+my $marker = 'e2e-osd-replace';
+
+my $osds;
+my $ctid;
+my $osd = {};
+
+# The entry of osd.$id in the OSD tree of the second (!) node, or undef if it is not listed.
+my $osd_entry = sub {
+    my ($id) = @_;
+    my ($entry) = grep { $_->{id} == $id } ceph_osds($second)->@*;
+    return $entry;
+};
+
+# The entry of the disk $devpath in the disk list of the second (!) node.
+my $disk_entry = sub {
+    my ($devpath) = @_;
+    my ($entry) =
+        grep { $_->{devpath} eq $devpath } $osd_client->get("/nodes/$osd_node/disks/list", {})->@*;
+    return $entry // {};
+};
+
+Proxmox::Test::Testcase->new('ceph_osd_replace')->before_all(sub {
+    $osds = create_ceph_cluster($pve_instances);
+    create_pool($first, $pool, add_storages => 1);
+})->with([{}])->step(
+        'start container on RBD storage',
+        sub {
+            $ctid = $client->get('/cluster/nextid');
+            my $upid = $client->post(
+                "/nodes/$node/lxc",
+                {
+                    vmid => $ctid,
+                    ostemplate => CONTAINER_TEMPLATE,
+                    rootfs => "$pool:2",
+                    memory => 256,
+                    unprivileged => 1,
+                    features => 'nesting=1',
+                    hostname => 'ct-osd-replace',
+                },
+            );
+            $first->task_ok($upid, "created container $ctid on '$pool'")
+            or die "creating the container failed\n";
+
+            $first->start_guest('lxc', $ctid);
+            $first->ct_exec($ctid, "sh -c 'echo $marker > /root/marker'");
+        },
+        sub {
+            $first->destroy_guest('lxc', $ctid) if defined($ctid);
+        },
+)->step(
+    'mark OSD out',
+    sub {
+        my ($entry) = ceph_osds($second)->@*;
+        die "no OSD on '$osd_node'\n" if !$entry;
+        my $id = $osd->{id} = $entry->{id};
+
+        my ($disk) = grep { ($_->{osdid} // -1) == $id }
+            $osd_client->get("/nodes/$osd_node/disks/list", {})->@*;
+        $osd->{disk} = ($disk // {})->{devpath};
+        ok($osd->{disk}, "disk list shows the disk of osd.$id") or die "disk not found\n";
+
+        $osd_client->post("/nodes/$osd_node/ceph/osd/$id/out", {});
+
+        poll_until(
+            sub { !($osd_entry->($id) // {})->{in} },
+            timeout => 30,
+            interval => 2,
+            description => "osd.$id to be out",
+        );
+        pass("osd.$id is out");
+    },
+)->step(
+    'wait for the data to move off the OSD',
+    sub {
+        my $id = $osd->{id};
+
+        poll_until(
+            sub {
+                return 0 if (($osd_entry->($id) // {})->{pgs} // -1) != 0;
+
+                my $pgmap = ceph_status($first)->{pgmap} // {};
+                my ($clean) = map { $_->{count} }
+                    grep { $_->{state_name} eq 'active+clean' } ($pgmap->{pgs_by_state} // [])->@*;
+                return ($clean // 0) == ($pgmap->{num_pgs} // -1);
+            },
+            timeout => 600,
+            interval => 5,
+            description => "osd.$id to hold no placement groups",
+        );
+        pass("Ceph moved all placement groups off osd.$id");
+    },
+)->step(
+    'stop OSD',
+    sub {
+        my $id = $osd->{id};
+
+        my $safety = $osd_client->get(
+            "/nodes/$osd_node/ceph/cmd-safety",
+            { service => 'osd', id => $id, action => 'stop' },
+        );
+        ok($safety->{safe}, "Ceph reports that stopping osd.$id is safe")
+            or diag($safety->{status} // 'no status');
+
+        my $upid = $osd_client->post("/nodes/$osd_node/ceph/stop", { service => "osd.$id" });
+        $second->task_ok($upid, "stopped osd.$id") or die "stopping osd.$id failed\n";
+
+        poll_until(
+            sub { (($osd_entry->($id) // {})->{status} // '') eq 'down' },
+            timeout => 60,
+            interval => 2,
+            description => "osd.$id to be down",
+        );
+        pass("osd.$id is down");
+    },
+)->step(
+    'destroy OSD with cleanup',
+    sub {
+        my $id = $osd->{id};
+
+        my $upid = $osd_client->delete("/nodes/$osd_node/ceph/osd/$id", { cleanup => 1 });
+        $second->task_ok($upid, "destroyed osd.$id") or die "destroying osd.$id failed\n";
+
+        ok(!$osd_entry->($id), "osd.$id is gone from the OSD tree");
+
+        my ($enabled) =
+            $second->run_command("systemctl is-enabled ceph-osd\@$id.service || true");
+        unlike($enabled // '', qr/^enabled/, "service of osd.$id is no longer enabled");
+        ok(
+            eval { $second->run_command("test ! -e /var/lib/ceph/osd/ceph-$id"); 1 },
+            "data directory of osd.$id is removed",
+        );
+
+        # The change in the disk list is not immediately reflected.
+        my $disk;
+        eval {
+            poll_until(
+                sub {
+                    $disk = $disk_entry->($osd->{disk});
+                    return !$disk->{used} && ($disk->{osdid} // -1) == -1;
+                },
+                timeout => 60,
+                interval => 2,
+                description => "disk '$osd->{disk}' to be unused",
+            );
+        };
+        ok(!$disk->{used}, "disk '$osd->{disk}' is unused")
+            or diag("disk is used by: $disk->{used}");
+        is($disk->{osdid} // -1, -1, "disk '$osd->{disk}' no longer belongs to an OSD");
+    },
+)->step(
+    'create OSD on the same disk',
+    sub {
+        my $upid = $osd_client->post("/nodes/$osd_node/ceph/osd", { dev => $osd->{disk} });
+        $second->task_ok($upid, "created OSD on '$osd->{disk}' again")
+            or die "creating the OSD failed\n";
+
+        wait_for_ceph(
+            $first,
+            monitors => scalar($pve_instances->@*),
+            osds => $osds,
+            health_ok => 1,
+        );
+        pass("the Ceph cluster reports HEALTH_OK with all $osds OSDs up and in");
+
+        is(
+            scalar(ceph_osds($second)->@*),
+            scalar(ceph_disks($second)->@*),
+            "'$osd_node' has an OSD on every Ceph disk again",
+        );
+    },
+)->step(
+    'container kept running',
+    sub {
+        is(
+            $client->get("/nodes/$node/lxc/$ctid/status/current", {})->{status},
+            'running',
+            "container $ctid kept running",
+        );
+        is($first->ct_exec($ctid, 'cat /root/marker'), "$marker\n", 'container data is intact');
+        ok(
+            eval { $first->ct_exec($ctid, "sh -c 'echo written > /root/after-replace'"); 1 },
+            'container can still write to its volume',
+        );
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index 9a94d11..340ac81 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -425,6 +425,17 @@ TestcaseConfig(
             ],
             tag: "ceph",
         ),
+        Testcase(
+            name: "ceph_osd_replace",
+            run: "./tests/ceph/test_ceph_osd_replace.pl",
+            timeout: 1800,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "ceph",
+        ),
     ],
     rollback_between_tests: Some("pre-test"),
 )
-- 
2.47.3