From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 27FAA1FF0C0 for ; Wed, 04 Nov 2026 20:07:53 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 628EB21911; Wed, 04 Nov 2026 20:05:27 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pve-devel@lists.proxmox.com Subject: [PATCH e2e-tests 29/38] tests: ceph: verify replacing an OSD Date: Wed, 4 Nov 2026 20:03:36 +0100 Message-ID: <20261104190345.2573464-30-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261104190345.2573464-1-m.koeppl@proxmox.com> References: <20261104190345.2573464-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793819036113 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.780 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: NMJ5KJKBXNNVKBRGLYIDSLKHZVYQVK5D X-Message-ID-Hash: NMJ5KJKBXNNVKBRGLYIDSLKHZVYQVK5D X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Replace an OSD as the documentation describes, while a container runs on an RBD storage. Mark the OSD out, wait until its placement groups have moved, check that it is safe to stop and stop it, then destroy it with cleanup and create a new OSD on the same disk. Destroying with cleanup must remove the OSD from the OSD tree, leave neither an enabled service nor a data directory behind and leave the disk unused, so that it can be reused right away. Afterwards, the cluster must be healthy with all OSDs up and in, and the container must have kept running with its data intact. Signed-off-by: Michael Köppl --- tests/ceph/test_ceph_osd_replace.pl | 237 ++++++++++++++++++++++++++++ tests/testcases.ron | 11 ++ 2 files changed, 248 insertions(+) create mode 100755 tests/ceph/test_ceph_osd_replace.pl diff --git a/tests/ceph/test_ceph_osd_replace.pl b/tests/ceph/test_ceph_osd_replace.pl new file mode 100755 index 0000000..51857ec --- /dev/null +++ b/tests/ceph/test_ceph_osd_replace.pl @@ -0,0 +1,237 @@ +#!/usr/bin/perl + +=head1 NAME + +C - Test replacing an OSD of a hyper-converged Ceph cluster. + +=head1 DESCRIPTION + +This testcase sets up Ceph on the cluster nodes with a pool and an RBD storage +and starts a container on that storage. It then replaces an OSD on the second +node as the documentation describes. It marks the OSD out, waits until Ceph has +moved all placement groups off it, checks that it is safe to stop and stops it. +It destroys the OSD with cleanup, which must remove the OSD from the OSD tree, +leave neither an enabled service nor a data directory of it behind and leave +its disk unused, so that a new OSD can be created on the same disk right away. + +Afterwards, the cluster must report HEALTH_OK with all OSDs up and in again. +The container must have kept running with its data intact. + +=cut + +use strict; +use warnings; + +use Test::More; + +use lib '.'; +use Proxmox::Test::Ceph qw( + ceph_disks + ceph_osds + ceph_status + create_ceph_cluster + create_pool + wait_for_ceph +); +use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instances); +use Proxmox::Test::Testcase; +use Proxmox::Test::Util qw(poll_until); + +my $pve_instances = get_pve_instances(); +die "not enough PVE instances" if scalar($pve_instances->@*) < 3; + +my $first = $pve_instances->[0]; +my $second = $pve_instances->[1]; +my $node = $first->get_nodename(); +my $osd_node = $second->get_nodename(); +my $client = $first->client(); +my $osd_client = $second->client(); + +my $pool = 'e2e-replace'; +my $marker = 'e2e-osd-replace'; + +my $osds; +my $ctid; +my $osd = {}; + +# The entry of osd.$id in the OSD tree of the second (!) node, or undef if it is not listed. +my $osd_entry = sub { + my ($id) = @_; + my ($entry) = grep { $_->{id} == $id } ceph_osds($second)->@*; + return $entry; +}; + +# The entry of the disk $devpath in the disk list of the second (!) node. +my $disk_entry = sub { + my ($devpath) = @_; + my ($entry) = + grep { $_->{devpath} eq $devpath } $osd_client->get("/nodes/$osd_node/disks/list", {})->@*; + return $entry // {}; +}; + +Proxmox::Test::Testcase->new('ceph_osd_replace')->before_all(sub { + $osds = create_ceph_cluster($pve_instances); + create_pool($first, $pool, add_storages => 1); +})->with([{}])->step( + 'start container on RBD storage', + sub { + $ctid = $client->get('/cluster/nextid'); + my $upid = $client->post( + "/nodes/$node/lxc", + { + vmid => $ctid, + ostemplate => CONTAINER_TEMPLATE, + rootfs => "$pool:2", + memory => 256, + unprivileged => 1, + features => 'nesting=1', + hostname => 'ct-osd-replace', + }, + ); + $first->task_ok($upid, "created container $ctid on '$pool'") + or die "creating the container failed\n"; + + $first->start_guest('lxc', $ctid); + $first->ct_exec($ctid, "sh -c 'echo $marker > /root/marker'"); + }, + sub { + $first->destroy_guest('lxc', $ctid) if defined($ctid); + }, +)->step( + 'mark OSD out', + sub { + my ($entry) = ceph_osds($second)->@*; + die "no OSD on '$osd_node'\n" if !$entry; + my $id = $osd->{id} = $entry->{id}; + + my ($disk) = grep { ($_->{osdid} // -1) == $id } + $osd_client->get("/nodes/$osd_node/disks/list", {})->@*; + $osd->{disk} = ($disk // {})->{devpath}; + ok($osd->{disk}, "disk list shows the disk of osd.$id") or die "disk not found\n"; + + $osd_client->post("/nodes/$osd_node/ceph/osd/$id/out", {}); + + poll_until( + sub { !($osd_entry->($id) // {})->{in} }, + timeout => 30, + interval => 2, + description => "osd.$id to be out", + ); + pass("osd.$id is out"); + }, +)->step( + 'wait for the data to move off the OSD', + sub { + my $id = $osd->{id}; + + poll_until( + sub { + return 0 if (($osd_entry->($id) // {})->{pgs} // -1) != 0; + + my $pgmap = ceph_status($first)->{pgmap} // {}; + my ($clean) = map { $_->{count} } + grep { $_->{state_name} eq 'active+clean' } ($pgmap->{pgs_by_state} // [])->@*; + return ($clean // 0) == ($pgmap->{num_pgs} // -1); + }, + timeout => 600, + interval => 5, + description => "osd.$id to hold no placement groups", + ); + pass("Ceph moved all placement groups off osd.$id"); + }, +)->step( + 'stop OSD', + sub { + my $id = $osd->{id}; + + my $safety = $osd_client->get( + "/nodes/$osd_node/ceph/cmd-safety", + { service => 'osd', id => $id, action => 'stop' }, + ); + ok($safety->{safe}, "Ceph reports that stopping osd.$id is safe") + or diag($safety->{status} // 'no status'); + + my $upid = $osd_client->post("/nodes/$osd_node/ceph/stop", { service => "osd.$id" }); + $second->task_ok($upid, "stopped osd.$id") or die "stopping osd.$id failed\n"; + + poll_until( + sub { (($osd_entry->($id) // {})->{status} // '') eq 'down' }, + timeout => 60, + interval => 2, + description => "osd.$id to be down", + ); + pass("osd.$id is down"); + }, +)->step( + 'destroy OSD with cleanup', + sub { + my $id = $osd->{id}; + + my $upid = $osd_client->delete("/nodes/$osd_node/ceph/osd/$id", { cleanup => 1 }); + $second->task_ok($upid, "destroyed osd.$id") or die "destroying osd.$id failed\n"; + + ok(!$osd_entry->($id), "osd.$id is gone from the OSD tree"); + + my ($enabled) = + $second->run_command("systemctl is-enabled ceph-osd\@$id.service || true"); + unlike($enabled // '', qr/^enabled/, "service of osd.$id is no longer enabled"); + ok( + eval { $second->run_command("test ! -e /var/lib/ceph/osd/ceph-$id"); 1 }, + "data directory of osd.$id is removed", + ); + + # The change in the disk list is not immediately reflected. + my $disk; + eval { + poll_until( + sub { + $disk = $disk_entry->($osd->{disk}); + return !$disk->{used} && ($disk->{osdid} // -1) == -1; + }, + timeout => 60, + interval => 2, + description => "disk '$osd->{disk}' to be unused", + ); + }; + ok(!$disk->{used}, "disk '$osd->{disk}' is unused") + or diag("disk is used by: $disk->{used}"); + is($disk->{osdid} // -1, -1, "disk '$osd->{disk}' no longer belongs to an OSD"); + }, +)->step( + 'create OSD on the same disk', + sub { + my $upid = $osd_client->post("/nodes/$osd_node/ceph/osd", { dev => $osd->{disk} }); + $second->task_ok($upid, "created OSD on '$osd->{disk}' again") + or die "creating the OSD failed\n"; + + wait_for_ceph( + $first, + monitors => scalar($pve_instances->@*), + osds => $osds, + health_ok => 1, + ); + pass("the Ceph cluster reports HEALTH_OK with all $osds OSDs up and in"); + + is( + scalar(ceph_osds($second)->@*), + scalar(ceph_disks($second)->@*), + "'$osd_node' has an OSD on every Ceph disk again", + ); + }, +)->step( + 'container kept running', + sub { + is( + $client->get("/nodes/$node/lxc/$ctid/status/current", {})->{status}, + 'running', + "container $ctid kept running", + ); + is($first->ct_exec($ctid, 'cat /root/marker'), "$marker\n", 'container data is intact'); + ok( + eval { $first->ct_exec($ctid, "sh -c 'echo written > /root/after-replace'"); 1 }, + 'container can still write to its volume', + ); + }, +)->run(); + +done_testing(); diff --git a/tests/testcases.ron b/tests/testcases.ron index 9a94d11..340ac81 100644 --- a/tests/testcases.ron +++ b/tests/testcases.ron @@ -425,6 +425,17 @@ TestcaseConfig( ], tag: "ceph", ), + Testcase( + name: "ceph_osd_replace", + run: "./tests/ceph/test_ceph_osd_replace.pl", + timeout: 1800, + instances: [ + "pve1-cluster", + "pve2-cluster", + "pve3-cluster", + ], + tag: "ceph", + ), ], rollback_between_tests: Some("pre-test"), ) -- 2.47.3