[PATCH e2e-tests 28/38] tests: ceph: verify setting up, using and destroying a CephFS

["Michael Köppl" <m.koeppl@proxmox.com>]

Create a metadata server on every node and a CephFS with a storage
for it. Check that the CephFS gets its own pools, that one metadata
server becomes active with the others on standby, and that the
storage is mounted on every node. Backup a container to the storage
and restore it on another node, which lists the backup, as the
storage is shared.

Destroy the CephFS as the documentation describes. Destroying it
together with its storage is refused while the storage is enabled
(based on `man pveceph(1)`). Once the storage is disabled and unmounted
and the metadata servers are stopped, it succeeds and removes the
storage and the pools too.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/ceph/test_ceph_cephfs.pl | 311 +++++++++++++++++++++++++++++++++
 tests/testcases.ron            |  11 ++
 2 files changed, 322 insertions(+)
 create mode 100755 tests/ceph/test_ceph_cephfs.pl

diff --git a/tests/ceph/test_ceph_cephfs.pl b/tests/ceph/test_ceph_cephfs.pl
new file mode 100755
index 0000000..40fdb3c
--- /dev/null
+++ b/tests/ceph/test_ceph_cephfs.pl
@@ -0,0 +1,311 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<test_ceph_cephfs> - Test setting up, using and destroying a CephFS.
+
+=head1 DESCRIPTION
+
+This testcase sets up Ceph on the cluster nodes and creates an MDS on every
+node, which are all on standby while there is no CephFS. It then creates a
+CephFS together with a storage for it and verifies that the CephFS has its
+own data and metadata pool, that one MDS became active with the others
+still on standby (only one can be active at a time), and that the storage is
+active and mounted on every node.
+
+It backs up a container on the first node to the CephFS storage and restores
+the backup on the second node, which lists it as the storage is shared.
+
+Finally, it destroys the CephFS as the documentation describes. Destroying it
+together with its storage must be refused while the storage is enabled. Once
+the storage is disabled and unmounted on every node and all MDS are stopped,
+destroying the CephFS must remove it together with its storage and its pools.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use JSON;
+
+use lib '.';
+use Proxmox::Test::Ceph qw(check_storage_active create_ceph_cluster wait_for_ceph);
+use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instances);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_instances = get_pve_instances();
+die "not enough PVE instances" if scalar($pve_instances->@*) < 3;
+
+my $first = $pve_instances->[0];
+my $second = $pve_instances->[1];
+my $node = $first->get_nodename();
+my $target = $second->get_nodename();
+my $client = $first->client();
+
+my $fs = 'e2e-cephfs';
+my $mountpoint = "/mnt/pve/$fs";
+my $marker = 'e2e-cephfs-backup';
+
+my $ctid;
+my $restored_ctid;
+my $backup_volid;
+
+# The state of every MDS by name, as reported by the API.
+my $mds_states = sub {
+    my $list = $client->get("/nodes/$node/ceph/mds", {});
+    return { map { $_->{name} => $_->{state} // '' } $list->@* };
+};
+
+my $wait_for_mds_states = sub {
+    my ($active, $standby) = @_;
+
+    my $states;
+    eval {
+        poll_until(
+            sub {
+                $states = $mds_states->();
+                my @values = values $states->%*;
+                return scalar(grep { $_ eq 'up:active' } @values) == $active
+                    && scalar(grep { $_ eq 'up:standby' } @values) == $standby;
+            },
+            timeout => 120,
+            interval => 2,
+            description => "$active active and $standby standby MDS",
+        );
+    };
+    diag('MDS states: ' . encode_json($states // {})) if $@;
+    return $states;
+};
+
+my $is_mounted = sub {
+    my ($instance) = @_;
+    my ($fstype) = eval { $instance->run_command("findmnt -n -o FSTYPE $mountpoint") };
+    return ($fstype // '') eq "ceph\n";
+};
+
+my $cephfs_entry = sub {
+    my ($entry) = grep { $_->{name} eq $fs } $client->get("/nodes/$node/ceph/fs", {})->@*;
+    return $entry;
+};
+
+Proxmox::Test::Testcase->new('ceph_cephfs')->before_all(sub {
+    create_ceph_cluster($pve_instances);
+})->with([{}])->step(
+        'create metadata servers',
+        sub {
+            for my $instance ($pve_instances->@*) {
+                my $name = $instance->get_nodename();
+                my $upid = $instance->client()->post("/nodes/$name/ceph/mds/$name", {});
+                $instance->task_ok($upid, "created MDS on '$name'")
+                or die "creating the MDS on '$name' failed\n";
+            }
+
+            my $states = $wait_for_mds_states->(0, scalar($pve_instances->@*));
+            is_deeply(
+                [sort keys $states->%*],
+                [sort map { $_->get_nodename() } $pve_instances->@*],
+                'every node has an MDS',
+            );
+            ok(
+                !(grep { $_ ne 'up:standby' } values $states->%*),
+                'all MDS are on standby without a CephFS',
+            );
+        },
+)->step(
+    'create CephFS with storage',
+    sub {
+        my $upid =
+            $client->post("/nodes/$node/ceph/fs/$fs", { pg_num => 32, 'add-storage' => 1 });
+        $first->task_ok($upid, "created CephFS '$fs'") or die "creating the CephFS failed\n";
+
+        my $entry = $cephfs_entry->();
+        ok($entry, "CephFS '$fs' is listed") or die "CephFS '$fs' not found\n";
+        is($entry->{metadata_pool}, "${fs}_metadata", 'CephFS has its own metadata pool');
+        is_deeply($entry->{data_pools}, ["${fs}_data"], 'CephFS has its own data pool');
+
+        my $states = $wait_for_mds_states->(1, scalar($pve_instances->@*) - 1);
+        my ($active) = grep { $states->{$_} eq 'up:active' } keys $states->%*;
+        ok($active, "MDS on '" . ($active // '') . "' became active for the CephFS");
+
+        my $storage = $client->get("/storage/$fs", {});
+        is($storage->{type}, 'cephfs', "CephFS storage '$fs' was added");
+        is($storage->{'fs-name'}, $fs, "storage '$fs' uses the new CephFS");
+        like($storage->{content} // '', qr/\bbackup\b/, "storage '$fs' can hold backups");
+
+        # The fs is not immediately available on all nodes. It is mounted on the first and mounted
+        # shortly after on the other nodes by pvestatd. When querying this too early, the storage
+        # is reported ad inactive.
+        for my $instance ($pve_instances->@*) {
+            my $name = $instance->get_nodename();
+            eval {
+                poll_until(
+                    sub { $is_mounted->($instance) },
+                    timeout => 60,
+                    interval => 2,
+                    description => "CephFS to be mounted on '$name'",
+                );
+            };
+            ok($is_mounted->($instance), "CephFS is mounted at $mountpoint on '$name'");
+            check_storage_active($instance, $fs);
+        }
+
+        wait_for_ceph($first);
+    },
+)->step(
+    'back up container to CephFS',
+    sub {
+        $ctid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node/lxc",
+            {
+                vmid => $ctid,
+                ostemplate => CONTAINER_TEMPLATE,
+                rootfs => 'local-lvm:2',
+                memory => 256,
+                unprivileged => 1,
+                features => 'nesting=1',
+                hostname => 'ct-cephfs-backup',
+            },
+        );
+        $first->task_ok($upid, "created container $ctid") or die "creating container failed\n";
+
+        $first->start_guest('lxc', $ctid);
+        $first->ct_exec($ctid, "sh -c 'echo $marker > /root/marker'");
+
+        $upid = $client->post(
+            "/nodes/$node/vzdump",
+            { vmid => $ctid, storage => $fs, mode => 'snapshot' },
+        );
+        $first->task_ok($upid, 'backed up the container to CephFS') or die "backup failed\n";
+
+        my $content = $client->get("/nodes/$node/storage/$fs/content", { content => 'backup' });
+        ($backup_volid) = map { $_->{volid} } grep { ($_->{vmid} // 0) == $ctid } $content->@*;
+        ok($backup_volid, 'backup is listed on the CephFS storage')
+            or die "backup not found\n";
+    },
+    sub {
+        $first->destroy_guest('lxc', $ctid) if defined($ctid);
+    },
+)->step(
+    'restore backup on another node',
+    sub {
+        my $target_client = $second->client();
+
+        my $content =
+            $target_client->get("/nodes/$target/storage/$fs/content", { content => 'backup' });
+        ok(
+            (grep { $_->{volid} eq $backup_volid } $content->@*),
+            "node '$target' lists the backup on the shared storage",
+        );
+
+        $restored_ctid = $client->get('/cluster/nextid');
+        my $upid = $target_client->post(
+            "/nodes/$target/lxc",
+            {
+                vmid => $restored_ctid,
+                ostemplate => $backup_volid,
+                restore => 1,
+                storage => 'local-lvm',
+            },
+        );
+        $second->task_ok($upid, "restored the backup as container $restored_ctid on '$target'")
+            or die "restoring the backup failed\n";
+
+        $second->start_guest('lxc', $restored_ctid);
+        is(
+            $second->ct_exec($restored_ctid, 'cat /root/marker'),
+            "$marker\n",
+            'restored container has the data from the backup',
+        );
+        $second->stop_guest('lxc', $restored_ctid);
+    },
+    sub {
+        $second->destroy_guest('lxc', $restored_ctid) if defined($restored_ctid);
+    },
+)->step(
+    'destroying CephFS with an enabled storage is refused',
+    sub {
+        my $upid = eval {
+            $client->delete(
+                "/nodes/$node/ceph/fs/$fs",
+                { 'remove-storages' => 1, 'remove-pools' => 1 },
+            );
+        };
+        my $status = defined($upid) ? $first->wait_for_task($upid, 1) : "API error: $@";
+        ok(!$first->task_successful($status), 'destroying the CephFS is refused');
+        like($status, qr/not disabled/, 'destroying is refused because the storage is enabled');
+        ok($cephfs_entry->(), 'CephFS still exists');
+    },
+)->step(
+    'destroy CephFS',
+    sub {
+        $client->put("/storage/$fs", { disable => 1 });
+
+        # Done manually according to the admin guide.
+        for my $instance ($pve_instances->@*) {
+            $instance->run_command("umount $mountpoint") if $is_mounted->($instance);
+            ok(
+                !$is_mounted->($instance),
+                "CephFS is unmounted on '" . $instance->get_nodename() . "'",
+            );
+        }
+
+        # Stop the standby MDS first, so that none of them takes over from the active one.
+        my $states = $mds_states->();
+        my @standby = grep { $states->{$_} ne 'up:active' } sort keys $states->%*;
+        my @active = grep { $states->{$_} eq 'up:active' } sort keys $states->%*;
+        for my $name (@standby, @active) {
+            my ($instance) = grep { $_->get_nodename() eq $name } $pve_instances->@*;
+            my $upid =
+                $instance->client()->post("/nodes/$name/ceph/stop", { service => "mds.$name" });
+            $instance->task_ok($upid, "stopped MDS on '$name'")
+                or die "stopping the MDS on '$name' failed\n";
+        }
+
+        poll_until(
+            sub {
+                my ($out) = $first->run_command("ceph fs get $fs --format json");
+                return !(keys(decode_json($out)->{mdsmap}->{up}->%*));
+            },
+            timeout => 120,
+            interval => 2,
+            description => "CephFS '$fs' to have no MDS up",
+        );
+
+        my $upid = $client->delete(
+            "/nodes/$node/ceph/fs/$fs",
+            { 'remove-storages' => 1, 'remove-pools' => 1 },
+        );
+        $first->task_ok($upid, "destroyed CephFS '$fs'")
+            or die "destroying the CephFS failed\n";
+
+        ok(!$cephfs_entry->(), 'CephFS is gone');
+
+        my $storages = $client->get('/storage', {});
+        ok(!(grep { $_->{storage} eq $fs } $storages->@*), "storage '$fs' is removed");
+
+        # Pool deletion runs in the background.
+        my %pools;
+        eval {
+            poll_until(
+                sub {
+                    %pools =
+                        map { $_->{pool_name} => 1 } $client->get("/nodes/$node/ceph/pool", {})->@*;
+                    return !$pools{"${fs}_data"} && !$pools{"${fs}_metadata"};
+                },
+                timeout => 60,
+                interval => 2,
+                description => 'the pools of the CephFS to be removed',
+            );
+        };
+        ok(
+            !$pools{"${fs}_data"} && !$pools{"${fs}_metadata"},
+            'pools of the CephFS are removed',
+        );
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index fec8667..9a94d11 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -414,6 +414,17 @@ TestcaseConfig(
             ],
             tag: "ceph",
         ),
+        Testcase(
+            name: "ceph_cephfs",
+            run: "./tests/ceph/test_ceph_cephfs.pl",
+            timeout: 1800,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "ceph",
+        ),
     ],
     rollback_between_tests: Some("pre-test"),
 )
-- 
2.47.3