From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 28DE41FF0C0 for ; Wed, 04 Nov 2026 20:07:45 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id EFA0E2190C; Wed, 04 Nov 2026 20:05:26 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pve-devel@lists.proxmox.com Subject: [PATCH e2e-tests 26/38] tests: ceph: verify setting up a hyper-converged cluster Date: Wed, 4 Nov 2026 20:03:33 +0100 Message-ID: <20261104190345.2573464-27-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261104190345.2573464-1-m.koeppl@proxmox.com> References: <20261104190345.2573464-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793819035824 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.783 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: YBCC6RD3Y2QIABXRRLKBBMTUYDWFWMZH X-Message-ID-Hash: YBCC6RD3Y2QIABXRRLKBBMTUYDWFWMZH X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Set up Ceph on the main cluster through the API with dedicated public and cluster networks, as the setup wizard requires a public network and the documentation recommends separating Ceph traffic. Check that the configuration on pmxcfs has the networks and is linked to from /etc/ceph on every node, that the monitors form a quorum on the public network, and that the OSDs use the public network for clients and the cluster network for replication. Also check the managers and creating a pool with an RBD storage. Signed-off-by: Michael Köppl --- tests/ceph/test_ceph_cluster_setup.pl | 206 ++++++++++++++++++++++++++ tests/testcases.ron | 11 ++ 2 files changed, 217 insertions(+) create mode 100755 tests/ceph/test_ceph_cluster_setup.pl diff --git a/tests/ceph/test_ceph_cluster_setup.pl b/tests/ceph/test_ceph_cluster_setup.pl new file mode 100755 index 0000000..5bd47d3 --- /dev/null +++ b/tests/ceph/test_ceph_cluster_setup.pl @@ -0,0 +1,206 @@ +#!/usr/bin/perl + +=head1 NAME + +C - Set up a hyper-converged Ceph cluster on 3 nodes. + +=head1 DESCRIPTION + +This testcase sets up Ceph on the cluster nodes. It gives every node an address +in a dedicated Ceph public and cluster network and writes the initial +configuration with these networks. It creates a monitor and a manager on every +node and an OSD on every Ceph disk. + +It verifies that the configuration on pmxcfs has the networks and is linked to +from /etc/ceph on every node, that the monitors form a quorum and use the public +network, that one manager is active with the others on standby, and that all +OSDs are up and in and use the public network for clients and the cluster +network for replication. Finally, it creates a pool together with an RBD +storage and verifies that the storage is active on every node and the cluster +reports HEALTH_OK. + +=cut + +use strict; +use warnings; + +use Test::More; + +use JSON; + +use lib '.'; +use Proxmox::Test::Ceph qw( + CEPH_CLUSTER_NETWORK + CEPH_PUBLIC_NETWORK + ceph_disks + ceph_osds + ceph_status + check_storage_active + create_managers + create_monitors + create_osds + create_pool + init_ceph + setup_ceph_network + wait_for_ceph +); +use Proxmox::Test::Helper qw(get_pve_instances); +use Proxmox::Test::Testcase; +use Proxmox::Test::Util qw(poll_until); + +my $pve_instances = get_pve_instances(); +die "not enough PVE instances" if scalar($pve_instances->@*) < 3; + +my $first = $pve_instances->[0]; +my $monitors = scalar($pve_instances->@*); +my $pool = 'e2e-setup'; +my $osds = 0; +$osds += scalar(ceph_disks($_)->@*) for $pve_instances->@*; + +my $public_network = CEPH_PUBLIC_NETWORK; +my $cluster_network = CEPH_CLUSTER_NETWORK; +my $addresses; + +# Commands like 'ceph' find the configuration on pmxcfs through this link. +my $check_config_link = sub { + my ($instance) = @_; + + my ($target) = eval { $instance->run_command('readlink /etc/ceph/ceph.conf') }; + is( + $target, + "/etc/pve/ceph.conf\n", + "/etc/ceph/ceph.conf links to the config on pmxcfs on '" + . $instance->get_nodename() . "'", + ); +}; + +Proxmox::Test::Testcase->new('ceph_cluster_setup')->with([{}])->step( + 'set up Ceph networks', + sub { + $addresses = setup_ceph_network($pve_instances); + pass('nodes reach each other in the Ceph public and cluster network'); + }, +)->step( + 'initialize Ceph', + sub { + init_ceph( + $pve_instances, + network => $public_network, + 'cluster-network' => $cluster_network, + ); + + my $node = $first->get_nodename(); + my $conf = $first->client()->get("/nodes/$node/ceph/cfg/raw", {}); + like($conf, qr/^\s*fsid\s*=\s*[0-9a-f-]{36}\s*$/m, 'ceph.conf has a cluster fsid'); + like( + $conf, + qr/^\s*public_network\s*=\s*\Q$public_network\E\s*$/m, + 'ceph.conf has the public network', + ); + like( + $conf, + qr/^\s*cluster_network\s*=\s*\Q$cluster_network\E\s*$/m, + 'ceph.conf has the cluster network', + ); + + $check_config_link->($first); + }, +)->step( + 'create monitors', + sub { + create_monitors($pve_instances); + + my @quorum = sort(ceph_status($first)->{quorum_names}->@*); + my @nodes = sort map { $_->get_nodename() } $pve_instances->@*; + is_deeply(\@quorum, \@nodes, 'every node has a monitor in the quorum'); + + my $node = $first->get_nodename(); + my $monitor_list = $first->client()->get("/nodes/$node/ceph/mon", {}); + for my $instance ($pve_instances->@*) { + my $name = $instance->get_nodename(); + my ($monitor) = grep { $_->{name} eq $name } $monitor_list->@*; + like( + ($monitor // {})->{addr} // '', + qr/^\Q$addresses->{$name}->{public}\E:/, + "monitor on '$name' uses its address in the public network", + ); + + # The nodes other than the one that initialized Ceph should by now also have the link. + $check_config_link->($instance); + } + }, +)->step( + 'create managers', + sub { + create_managers($pve_instances); + + # A new manager is not immediately registered as standby + my $mgrmap; + eval { + poll_until( + sub { + $mgrmap = ceph_status($first)->{mgrmap}; + return $mgrmap->{available} && $mgrmap->{num_standbys} == $monitors - 1; + }, + timeout => 60, + interval => 2, + description => 'one active and ' . ($monitors - 1) . ' standby managers', + ); + }; + diag("manager map: " . JSON::encode_json($mgrmap // {})) if $@; + ok($mgrmap->{available}, 'a manager is active'); + is($mgrmap->{num_standbys}, $monitors - 1, 'the other managers are on standby'); + }, +)->step( + 'create OSDs', + sub { + is(create_osds($pve_instances), $osds, "created $osds OSDs"); + wait_for_ceph($first, monitors => $monitors, osds => $osds); + pass("all $osds OSDs are up and in"); + + for my $instance ($pve_instances->@*) { + my $name = $instance->get_nodename(); + my $node_osds = ceph_osds($instance); + is( + scalar($node_osds->@*), + scalar(ceph_disks($instance)->@*), + "OSD tree lists an OSD for every Ceph disk under '$name'", + ); + + for my $osd ($node_osds->@*) { + my $metadata = + $instance->client()->get("/nodes/$name/ceph/osd/$osd->{id}/metadata", {}) + ->{osd}; + like( + $metadata->{front_addr} // '', + qr/\b\Q$addresses->{$name}->{public}\E:/, + "osd.$osd->{id} serves clients in the public network", + ); + like( + $metadata->{back_addr} // '', + qr/\b\Q$addresses->{$name}->{cluster}\E:/, + "osd.$osd->{id} replicates in the cluster network", + ); + } + } + }, +)->step( + 'create pool with RBD storage', + sub { + create_pool($first, $pool, add_storages => 1); + + my $storage = $first->client()->get("/storage/$pool", {}); + is($storage->{type}, 'rbd', "RBD storage '$pool' was added"); + is($storage->{pool}, $pool, "storage '$pool' uses the new pool"); + + check_storage_active($_, $pool) for $pve_instances->@*; + }, +)->step( + 'verify cluster health', + sub { + wait_for_ceph($first, monitors => $monitors, osds => $osds, health_ok => 1); + pass('the Ceph cluster reports HEALTH_OK'); + }, +)->run(); + +done_testing(); diff --git a/tests/testcases.ron b/tests/testcases.ron index fba3e06..47c06b0 100644 --- a/tests/testcases.ron +++ b/tests/testcases.ron @@ -392,6 +392,17 @@ TestcaseConfig( ], tag: "sdn", ), + Testcase( + name: "ceph_cluster_setup", + run: "./tests/ceph/test_ceph_cluster_setup.pl", + timeout: 1800, + instances: [ + "pve1-cluster", + "pve2-cluster", + "pve3-cluster", + ], + tag: "ceph", + ), ], rollback_between_tests: Some("pre-test"), ) -- 2.47.3