From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 45D3B1FF0C0 for ; Wed, 04 Nov 2026 20:08:31 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 4A356219D0; Wed, 04 Nov 2026 20:05:29 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pve-devel@lists.proxmox.com Subject: [PATCH e2e-tests 28/38] tests: ceph: verify setting up, using and destroying a CephFS Date: Wed, 4 Nov 2026 20:03:35 +0100 Message-ID: <20261104190345.2573464-29-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261104190345.2573464-1-m.koeppl@proxmox.com> References: <20261104190345.2573464-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793819035989 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.220 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: UYQX4R6C7PBDILETGBYH7JVWQ6VBYABR X-Message-ID-Hash: UYQX4R6C7PBDILETGBYH7JVWQ6VBYABR X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Create a metadata server on every node and a CephFS with a storage for it. Check that the CephFS gets its own pools, that one metadata server becomes active with the others on standby, and that the storage is mounted on every node. Backup a container to the storage and restore it on another node, which lists the backup, as the storage is shared. Destroy the CephFS as the documentation describes. Destroying it together with its storage is refused while the storage is enabled (based on `man pveceph(1)`). Once the storage is disabled and unmounted and the metadata servers are stopped, it succeeds and removes the storage and the pools too. Signed-off-by: Michael Köppl --- tests/ceph/test_ceph_cephfs.pl | 311 +++++++++++++++++++++++++++++++++ tests/testcases.ron | 11 ++ 2 files changed, 322 insertions(+) create mode 100755 tests/ceph/test_ceph_cephfs.pl diff --git a/tests/ceph/test_ceph_cephfs.pl b/tests/ceph/test_ceph_cephfs.pl new file mode 100755 index 0000000..40fdb3c --- /dev/null +++ b/tests/ceph/test_ceph_cephfs.pl @@ -0,0 +1,311 @@ +#!/usr/bin/perl + +=head1 NAME + +C - Test setting up, using and destroying a CephFS. + +=head1 DESCRIPTION + +This testcase sets up Ceph on the cluster nodes and creates an MDS on every +node, which are all on standby while there is no CephFS. It then creates a +CephFS together with a storage for it and verifies that the CephFS has its +own data and metadata pool, that one MDS became active with the others +still on standby (only one can be active at a time), and that the storage is +active and mounted on every node. + +It backs up a container on the first node to the CephFS storage and restores +the backup on the second node, which lists it as the storage is shared. + +Finally, it destroys the CephFS as the documentation describes. Destroying it +together with its storage must be refused while the storage is enabled. Once +the storage is disabled and unmounted on every node and all MDS are stopped, +destroying the CephFS must remove it together with its storage and its pools. + +=cut + +use strict; +use warnings; + +use Test::More; + +use JSON; + +use lib '.'; +use Proxmox::Test::Ceph qw(check_storage_active create_ceph_cluster wait_for_ceph); +use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instances); +use Proxmox::Test::Testcase; +use Proxmox::Test::Util qw(poll_until); + +my $pve_instances = get_pve_instances(); +die "not enough PVE instances" if scalar($pve_instances->@*) < 3; + +my $first = $pve_instances->[0]; +my $second = $pve_instances->[1]; +my $node = $first->get_nodename(); +my $target = $second->get_nodename(); +my $client = $first->client(); + +my $fs = 'e2e-cephfs'; +my $mountpoint = "/mnt/pve/$fs"; +my $marker = 'e2e-cephfs-backup'; + +my $ctid; +my $restored_ctid; +my $backup_volid; + +# The state of every MDS by name, as reported by the API. +my $mds_states = sub { + my $list = $client->get("/nodes/$node/ceph/mds", {}); + return { map { $_->{name} => $_->{state} // '' } $list->@* }; +}; + +my $wait_for_mds_states = sub { + my ($active, $standby) = @_; + + my $states; + eval { + poll_until( + sub { + $states = $mds_states->(); + my @values = values $states->%*; + return scalar(grep { $_ eq 'up:active' } @values) == $active + && scalar(grep { $_ eq 'up:standby' } @values) == $standby; + }, + timeout => 120, + interval => 2, + description => "$active active and $standby standby MDS", + ); + }; + diag('MDS states: ' . encode_json($states // {})) if $@; + return $states; +}; + +my $is_mounted = sub { + my ($instance) = @_; + my ($fstype) = eval { $instance->run_command("findmnt -n -o FSTYPE $mountpoint") }; + return ($fstype // '') eq "ceph\n"; +}; + +my $cephfs_entry = sub { + my ($entry) = grep { $_->{name} eq $fs } $client->get("/nodes/$node/ceph/fs", {})->@*; + return $entry; +}; + +Proxmox::Test::Testcase->new('ceph_cephfs')->before_all(sub { + create_ceph_cluster($pve_instances); +})->with([{}])->step( + 'create metadata servers', + sub { + for my $instance ($pve_instances->@*) { + my $name = $instance->get_nodename(); + my $upid = $instance->client()->post("/nodes/$name/ceph/mds/$name", {}); + $instance->task_ok($upid, "created MDS on '$name'") + or die "creating the MDS on '$name' failed\n"; + } + + my $states = $wait_for_mds_states->(0, scalar($pve_instances->@*)); + is_deeply( + [sort keys $states->%*], + [sort map { $_->get_nodename() } $pve_instances->@*], + 'every node has an MDS', + ); + ok( + !(grep { $_ ne 'up:standby' } values $states->%*), + 'all MDS are on standby without a CephFS', + ); + }, +)->step( + 'create CephFS with storage', + sub { + my $upid = + $client->post("/nodes/$node/ceph/fs/$fs", { pg_num => 32, 'add-storage' => 1 }); + $first->task_ok($upid, "created CephFS '$fs'") or die "creating the CephFS failed\n"; + + my $entry = $cephfs_entry->(); + ok($entry, "CephFS '$fs' is listed") or die "CephFS '$fs' not found\n"; + is($entry->{metadata_pool}, "${fs}_metadata", 'CephFS has its own metadata pool'); + is_deeply($entry->{data_pools}, ["${fs}_data"], 'CephFS has its own data pool'); + + my $states = $wait_for_mds_states->(1, scalar($pve_instances->@*) - 1); + my ($active) = grep { $states->{$_} eq 'up:active' } keys $states->%*; + ok($active, "MDS on '" . ($active // '') . "' became active for the CephFS"); + + my $storage = $client->get("/storage/$fs", {}); + is($storage->{type}, 'cephfs', "CephFS storage '$fs' was added"); + is($storage->{'fs-name'}, $fs, "storage '$fs' uses the new CephFS"); + like($storage->{content} // '', qr/\bbackup\b/, "storage '$fs' can hold backups"); + + # The fs is not immediately available on all nodes. It is mounted on the first and mounted + # shortly after on the other nodes by pvestatd. When querying this too early, the storage + # is reported ad inactive. + for my $instance ($pve_instances->@*) { + my $name = $instance->get_nodename(); + eval { + poll_until( + sub { $is_mounted->($instance) }, + timeout => 60, + interval => 2, + description => "CephFS to be mounted on '$name'", + ); + }; + ok($is_mounted->($instance), "CephFS is mounted at $mountpoint on '$name'"); + check_storage_active($instance, $fs); + } + + wait_for_ceph($first); + }, +)->step( + 'back up container to CephFS', + sub { + $ctid = $client->get('/cluster/nextid'); + my $upid = $client->post( + "/nodes/$node/lxc", + { + vmid => $ctid, + ostemplate => CONTAINER_TEMPLATE, + rootfs => 'local-lvm:2', + memory => 256, + unprivileged => 1, + features => 'nesting=1', + hostname => 'ct-cephfs-backup', + }, + ); + $first->task_ok($upid, "created container $ctid") or die "creating container failed\n"; + + $first->start_guest('lxc', $ctid); + $first->ct_exec($ctid, "sh -c 'echo $marker > /root/marker'"); + + $upid = $client->post( + "/nodes/$node/vzdump", + { vmid => $ctid, storage => $fs, mode => 'snapshot' }, + ); + $first->task_ok($upid, 'backed up the container to CephFS') or die "backup failed\n"; + + my $content = $client->get("/nodes/$node/storage/$fs/content", { content => 'backup' }); + ($backup_volid) = map { $_->{volid} } grep { ($_->{vmid} // 0) == $ctid } $content->@*; + ok($backup_volid, 'backup is listed on the CephFS storage') + or die "backup not found\n"; + }, + sub { + $first->destroy_guest('lxc', $ctid) if defined($ctid); + }, +)->step( + 'restore backup on another node', + sub { + my $target_client = $second->client(); + + my $content = + $target_client->get("/nodes/$target/storage/$fs/content", { content => 'backup' }); + ok( + (grep { $_->{volid} eq $backup_volid } $content->@*), + "node '$target' lists the backup on the shared storage", + ); + + $restored_ctid = $client->get('/cluster/nextid'); + my $upid = $target_client->post( + "/nodes/$target/lxc", + { + vmid => $restored_ctid, + ostemplate => $backup_volid, + restore => 1, + storage => 'local-lvm', + }, + ); + $second->task_ok($upid, "restored the backup as container $restored_ctid on '$target'") + or die "restoring the backup failed\n"; + + $second->start_guest('lxc', $restored_ctid); + is( + $second->ct_exec($restored_ctid, 'cat /root/marker'), + "$marker\n", + 'restored container has the data from the backup', + ); + $second->stop_guest('lxc', $restored_ctid); + }, + sub { + $second->destroy_guest('lxc', $restored_ctid) if defined($restored_ctid); + }, +)->step( + 'destroying CephFS with an enabled storage is refused', + sub { + my $upid = eval { + $client->delete( + "/nodes/$node/ceph/fs/$fs", + { 'remove-storages' => 1, 'remove-pools' => 1 }, + ); + }; + my $status = defined($upid) ? $first->wait_for_task($upid, 1) : "API error: $@"; + ok(!$first->task_successful($status), 'destroying the CephFS is refused'); + like($status, qr/not disabled/, 'destroying is refused because the storage is enabled'); + ok($cephfs_entry->(), 'CephFS still exists'); + }, +)->step( + 'destroy CephFS', + sub { + $client->put("/storage/$fs", { disable => 1 }); + + # Done manually according to the admin guide. + for my $instance ($pve_instances->@*) { + $instance->run_command("umount $mountpoint") if $is_mounted->($instance); + ok( + !$is_mounted->($instance), + "CephFS is unmounted on '" . $instance->get_nodename() . "'", + ); + } + + # Stop the standby MDS first, so that none of them takes over from the active one. + my $states = $mds_states->(); + my @standby = grep { $states->{$_} ne 'up:active' } sort keys $states->%*; + my @active = grep { $states->{$_} eq 'up:active' } sort keys $states->%*; + for my $name (@standby, @active) { + my ($instance) = grep { $_->get_nodename() eq $name } $pve_instances->@*; + my $upid = + $instance->client()->post("/nodes/$name/ceph/stop", { service => "mds.$name" }); + $instance->task_ok($upid, "stopped MDS on '$name'") + or die "stopping the MDS on '$name' failed\n"; + } + + poll_until( + sub { + my ($out) = $first->run_command("ceph fs get $fs --format json"); + return !(keys(decode_json($out)->{mdsmap}->{up}->%*)); + }, + timeout => 120, + interval => 2, + description => "CephFS '$fs' to have no MDS up", + ); + + my $upid = $client->delete( + "/nodes/$node/ceph/fs/$fs", + { 'remove-storages' => 1, 'remove-pools' => 1 }, + ); + $first->task_ok($upid, "destroyed CephFS '$fs'") + or die "destroying the CephFS failed\n"; + + ok(!$cephfs_entry->(), 'CephFS is gone'); + + my $storages = $client->get('/storage', {}); + ok(!(grep { $_->{storage} eq $fs } $storages->@*), "storage '$fs' is removed"); + + # Pool deletion runs in the background. + my %pools; + eval { + poll_until( + sub { + %pools = + map { $_->{pool_name} => 1 } $client->get("/nodes/$node/ceph/pool", {})->@*; + return !$pools{"${fs}_data"} && !$pools{"${fs}_metadata"}; + }, + timeout => 60, + interval => 2, + description => 'the pools of the CephFS to be removed', + ); + }; + ok( + !$pools{"${fs}_data"} && !$pools{"${fs}_metadata"}, + 'pools of the CephFS are removed', + ); + }, +)->run(); + +done_testing(); diff --git a/tests/testcases.ron b/tests/testcases.ron index fec8667..9a94d11 100644 --- a/tests/testcases.ron +++ b/tests/testcases.ron @@ -414,6 +414,17 @@ TestcaseConfig( ], tag: "ceph", ), + Testcase( + name: "ceph_cephfs", + run: "./tests/ceph/test_ceph_cephfs.pl", + timeout: 1800, + instances: [ + "pve1-cluster", + "pve2-cluster", + "pve3-cluster", + ], + tag: "ceph", + ), ], rollback_between_tests: Some("pre-test"), ) -- 2.47.3