From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 86FF91FF09C for ; Mon, 19 Oct 2026 10:24:41 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 21F9121643; Mon, 19 Oct 2026 10:24:26 +0200 (CEST) From: Kefu Chai To: pve-devel@lists.proxmox.com Subject: [PATCH manager v2 4/4] migrations: cephx: name the hosts keeping key copies we cannot write Date: Mon, 19 Oct 2026 16:23:46 +0800 Message-ID: <20261019082346.1781960-5-k.chai@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261019082346.1781960-1-k.chai@proxmox.com> References: <20261019082346.1781960-1-k.chai@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1792398235969 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.327 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: ARIQ6FO6B364MLYVFNSEWXNUFAC7C3JR X-Message-ID-Hash: ARIQ6FO6B364MLYVFNSEWXNUFAC7C3JR X-MailFrom: k.chai@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Thomas Lamprecht , Jonas Theisen X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A host outside this cluster that runs a Ceph daemon usually keeps its own copy of the 'client.admin' keyring. We do not write that copy, so a key rotation leaves it stale. Name these hosts when the key is staged and when it is committed, and print commands to refresh the copy: - While staged, fetch the new key, usually as 'client.admin'. A user with only 'profile rbd' caps cannot read its own key. Take the key from the JSON output and write it with ceph-authtool, because a keyring saved by 'auth get' has a 'pending key' line that Ceph cannot read. - After the commit, fetch it with the monitor's own keyring, if the host runs a monitor. Otherwise copy it over from one of our nodes. The commands name their keyring files, because a ceph.conf copied from here points to /etc/pve. Signed-off-by: Kefu Chai --- bin/pve-cephx-rotate-service-keys | 53 ++++++++++++++++- test/CephKeyMigrationScript_test.pl | 89 ++++++++++++++++++++++++++++- 2 files changed, 140 insertions(+), 2 deletions(-) diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys index c7565e9e..ea953f76 100755 --- a/bin/pve-cephx-rotate-service-keys +++ b/bin/pve-cephx-rotate-service-keys @@ -197,6 +197,51 @@ my sub unmanaged_monitors($info) { return grep { !$_->{managed} } $info->{daemons}->{mon}->@*; } +# hosts that run Ceph daemons outside this cluster. '$DAEMON_TYPES' has no monitors, so add them. +my sub unmanaged_hosts_of($info) { + my $hosts = {}; + for my $type ('mon', @$DAEMON_TYPES) { + $hosts->{ $_->{node} } = 1 + for grep { !$_->{managed} && defined($_->{node}) } $info->{daemons}->{$type}->@*; + } + return sort keys %$hosts; +} + +# Such a host keeps its own key copies, usually of 'client.admin'. While the key is staged, its +# admin can fetch the new key with the old one, as a user allowed to read keys. The key comes from +# the JSON output, as Ceph cannot read the 'pending key' line that 'auth get' writes. After the +# commit, a monitor host can use its 'mon.' keyring instead. +my sub unmanaged_copy_note($info, $committed = 0) { + my @hosts = unmanaged_hosts_of($info); + return if !@hosts; + + my $where = join(', ', map { "'$_'" } @hosts); + if (!$committed) { + log_step("Proxmox VE rewrites the key copies it manages. These hosts run Ceph daemons" + . " without being nodes of this cluster, so a keyring they keep for their own use is" + . " not among them: $where. Their admin can refresh one while both keys still work," + . " as a user allowed to read keys, usually 'client.admin'. The commands name both" + . " keyrings, as a configuration copied from here looks for keys under '/etc/pve'," + . " which such a host does not have:"); + log_step(" KEY=\$(ceph -n client.admin -k auth get USER -f json" + . " | python3 -c 'import json, sys; print(json.load(sys.stdin)[0][\"pending_key\"])')" + ); + log_step(" ceph-authtool -n USER --add-key \"\$KEY\""); + return; + } + + log_warn("The previous key no longer works. A keyring these hosts keep for their own use was" + . " not rewritten, as they are not nodes of this cluster: $where. On a monitor host, and" + . " while its keyring still holds the current 'mon.' key, its admin can refresh one with" + . " the monitor's own credential:"); + log_step(" ceph --name mon. --keyring " + . mon_keyring_path('') + . " auth get USER -o "); + log_step("Otherwise fetch the key on a node of this cluster and copy the file over."); + + return; +} + my sub assert_located($node, $entity) { die "Cannot manage '$entity': no valid host name was reported\n" if !defined($node) || ref($node) || !length($node); @@ -2029,6 +2074,7 @@ my sub preflight_cluster( }; my $confirmation_refused = 0; + my $committed_copies = 0; my $confirm_all = $opts->{'confirm-all-clients-refreshed'}; my $confirmation_seen = {}; my @confirm = @@ -2256,7 +2302,7 @@ my sub preflight_cluster( } # The record remains open if the commit fails, so the confirmation is retryable. - commit_staged_key($info->{rados}, $state, $entity) if $staged; + $committed_copies = 1 if $staged && commit_staged_key($info->{rados}, $state, $entity); my $mark = $state->{client_refresh}->{$entity}; $mark->{cleared} = time(); $mark->{acknowledged} = time(); @@ -2265,6 +2311,9 @@ my sub preflight_cluster( } } + # copies on hosts we do not manage are stale from here on + unmanaged_copy_note($info, 1) if $committed_copies; + # Confirmation can promote staged keys. Recollect before any remaining preflight decision and # update the caller's shared cluster picture before it builds the migration plan. if (!$confirmation_refused && scalar(@confirm)) { @@ -3045,6 +3094,7 @@ my sub print_plan($info, $plan, $state, $opts, $storage_entities) { log_step("For each staged Ceph user key, both the current and new keys authenticate" . " until the new key is committed with '--confirm-clients-refreshed USER' or," . " once every open record is ready, '--confirm-all-clients-refreshed'."); + unmanaged_copy_note($info); log_step("the monitors stop automatic pending-key promotion until every staged key is" . " committed or aborted") if $opts->{verbose}; @@ -6344,6 +6394,7 @@ sub key_migration_test_hooks { check_client_kernels => \&check_client_kernels, manual_promotion_with_retries => \&manual_promotion_with_retries, mon_admin_run => \&mon_admin_run, + unmanaged_copy_note => \&unmanaged_copy_note, describe_live => \&describe_live, }; } diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl index 5d251596..5c5a9bf8 100755 --- a/test/CephKeyMigrationScript_test.pl +++ b/test/CephKeyMigrationScript_test.pl @@ -3619,7 +3619,15 @@ sub run_aggregate_confirmation { mon => [], mgr => [], mds => [], - osd => [{ type => 'osd', id => 7, entity => 'osd.7', node => 'node-daemon' }], + osd => [ + { + type => 'osd', + id => 7, + entity => 'osd.7', + node => 'node-daemon', + managed => 1, + }, + ], }, }; my $plan = { @@ -3717,6 +3725,41 @@ sub run_aggregate_confirmation { qr/For each staged Ceph user key, both the current and new keys authenticate.*committed with '--confirm-clients-refreshed USER' or, once every open record is ready, '--confirm-all-clients-refreshed'/s, 'the default names both staged completion paths without offering the aggregate early', ); + unlike( + $concise, + qr/without being nodes of this cluster/, + 'a cluster that manages every daemon host is told nothing about copies elsewhere', + ); + + # while the key is staged, a host outside this cluster can fetch it with the old key + { + my $stretched = { %$info, daemons => { %{ $info->{daemons} } } }; + $stretched->{daemons}->{mon} = [ + { type => 'mon', id => 'a', node => 'node-a', managed => 1 }, + { type => 'mon', id => 'tiebreaker', node => 'witness.example', managed => 0 }, + ]; + my $output = $render->(0, undef, undef, undef, $stretched); + like( + $output, + qr/without being nodes of this cluster.*'witness\.example'.*while both keys still work/s, + 'the host is named while the old key still authenticates', + ); + like( + $output, + qr/KEY=\$\(ceph -n client\.admin -k auth get USER -f json \|.*"pending_key".*\n.*ceph-authtool -n USER --add-key "\$KEY"/, + 'as a user allowed to read keys, writing the staged key itself, naming both keyrings', + ); + unlike( + $output, + qr/'node-a'/, + 'and a managed node is not, as Proxmox VE rewrites its copies', + ); + unlike( + $output, + qr/--name mon\./, + 'the monitor credential is not offered while the simpler one still works', + ); + } like( $concise, qr/For staged keys.*live-migrate affected\s+VMs, including those with kernel RBD disks/s, @@ -7902,6 +7945,50 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) { ); } +# After the commit the old copy no longer works, so the note names another credential. +{ + my $stretched = { + daemons => { + mon => [ + { type => 'mon', id => 'a', node => 'node-a', managed => 1 }, + { type => 'mon', id => 'witness', node => 'witness.example', managed => 0 }, + ], + mgr => [], + mds => [], + osd => [], + }, + }; + my $note = sub { + my ($committed) = @_; + my $out = ''; + open(my $stdout, '>', \$out) or die $!; + local *STDOUT = $stdout; + $HOOKS->{unmanaged_copy_note}->($stretched, $committed); + return $out; + }; + + my $committed = $note->(1); + like($committed, qr/WARN.*previous key no longer works/, 'the note says the old key is gone'); + like( + $committed, + qr{ceph --name mon\. --keyring /var/lib/ceph/mon/\S+-/keyring auth get USER}, + "and gives the monitor's own credential as the way back", + ); + like($committed, qr/copy the file over/, 'with the fallback for a host running no monitor'); + like($committed, qr/'witness\.example'/, 'naming the unmanaged host'); + unlike($committed, qr/'node-a'/, 'and not the one Proxmox VE rewrites itself'); + like($note->(0), qr/while both keys still work/, 'before the commit it is the simpler route'); + + my $managed = { daemons => { map { $_ => [] } qw(mon mgr mds osd) } }; + my $out = ''; + { + open(my $stdout, '>', \$out) or die $!; + local *STDOUT = $stdout; + $HOOKS->{unmanaged_copy_note}->($managed, 1); + } + is($out, '', 'a cluster that manages every daemon host hears nothing about copies elsewhere'); +} + # A monitor on one of our nodes is asked over its admin socket, any other one with 'ceph tell'. { no warnings qw(once redefine); -- 2.47.3