* [PATCH manager v2 1/4] migrations: cephx: ask a monitor about itself through one route
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
@ 2026-10-19 8:23 ` Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 2/4] migrations: cephx: judge cipher support by the version a daemon runs Kefu Chai
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Kefu Chai @ 2026-10-19 8:23 UTC (permalink / raw)
To: pve-devel; +Cc: Thomas Lamprecht, Jonas Theisen
The helper queries each monitor over its admin socket, which needs a
shell on the monitor's host. For a monitor on a host outside this
cluster, such as a stretch cluster's tiebreaker, these queries fail. The
helper then treats the monitor as unreachable, refuses to stage client
keys, and reports the session list as incomplete.
Record whether a daemon runs on one of our nodes. For a monitor that
does not, use 'ceph tell' as 'mon.' instead, with a short timeout.
'ceph tell' shows up in the monitor's own session list as a client named
'mon.'. Ignore 'mon.' sessions, since 'mon.' is the monitors' own key
and not a client key to refresh. Otherwise, if 'mon.' is not in the auth
database, the run would stop on a client without an auth entry.
Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
PVE/Ceph/KeyMigration.pm | 21 ++++----
bin/pve-cephx-rotate-service-keys | 75 ++++++++++++++---------------
test/CephKeyMigrationScript_test.pl | 61 +++++++++++++++++++++--
test/CephKeyMigration_test.pl | 12 +++++
4 files changed, 117 insertions(+), 52 deletions(-)
diff --git a/PVE/Ceph/KeyMigration.pm b/PVE/Ceph/KeyMigration.pm
index d0613e8b..c24c1c3b 100644
--- a/PVE/Ceph/KeyMigration.pm
+++ b/PVE/Ceph/KeyMigration.pm
@@ -561,6 +561,8 @@ sub summarize_sessions($per_mon, $expected_mons = undef) {
next;
}
next if $session->{con_type} ne 'client';
+ # skip the monitors' own key, which a 'ceph tell' sent as 'mon.' also shows up as
+ next if ($session->{entity_name} // '') eq 'mon.';
my $gid = $session->{global_id};
my $entity = $session->{entity_name};
@@ -1101,15 +1103,16 @@ sub merge_configured_daemons($daemons, $type, $configured, $existing = undef) {
push @$ghosts, { type => $type, id => "$id", node => $configured->{$id} };
next;
}
- push @$daemons,
- {
- type => $type,
- id => "$id",
- entity => $type eq 'mon' ? 'mon.' : "$type.$id",
- node => $configured->{$id},
- down => 1,
- $type eq 'osd' && $existing ? ('osd-uuid' => $existing->{$id}) : (),
- };
+ push @$daemons, {
+ type => $type,
+ id => "$id",
+ entity => $type eq 'mon' ? 'mon.' : "$type.$id",
+ node => $configured->{$id},
+ # pvestatd only reports daemons on nodes of this cluster
+ managed => 1,
+ down => 1,
+ $type eq 'osd' && $existing ? ('osd-uuid' => $existing->{$id}) : (),
+ };
}
return wantarray ? ($daemons, $ghosts) : $daemons;
diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys
index ad425f96..84daff6d 100755
--- a/bin/pve-cephx-rotate-service-keys
+++ b/bin/pve-cephx-rotate-service-keys
@@ -175,11 +175,18 @@ my sub cluster_nodes() {
return { map { $_ => 1 } PVE::Cluster::get_nodelist()->@* };
}
+# 'managed': the host is a node of this cluster, so we can reach its files, units and packages.
+# Other daemons, like the tiebreaker monitor of a stretch cluster, are reachable only via Ceph.
my sub daemon_location($type, $id, $host) {
- return { node => undef } if !defined($host) || (!ref($host) && !length($host));
+ return { node => undef, managed => 0 } if !defined($host) || (!ref($host) && !length($host));
die "Cannot locate '$type.$id': no valid host name was reported\n" if ref($host);
- my $node = PVE::Ceph::Services::metadata_host_node($host, cluster_nodes());
- return { node => $node, $node ne $host ? ('metadata-host' => $host) : () };
+ my $nodes = cluster_nodes();
+ my $node = PVE::Ceph::Services::metadata_host_node($host, $nodes);
+ return {
+ node => $node,
+ managed => $nodes->{$node} ? 1 : 0,
+ $node ne $host ? ('metadata-host' => $host) : (),
+ };
}
my sub assert_member($node, $entity) {
@@ -559,13 +566,14 @@ my sub auth_entry($rados, $entity) {
}
# the monitor identity keeps working while a client.admin rotation invalidates the default keyring
-my sub monitor_command($args) {
+my sub monitor_command($args, $timeout = undef) {
return node_run(
$nodename,
[
'ceph', '--cluster', $ccname, '--name', 'mon.', '--keyring', $pve_mon_keyring,
@$args,
],
+ defined($timeout) ? (timeout => $timeout) : (),
);
}
@@ -579,19 +587,26 @@ my sub monitor_auth_entry($entity) {
return $res->[0];
}
+# Ask a monitor about itself over its admin socket, which works even when cephx does not. For a
+# monitor outside this cluster, use 'ceph tell' to reach the same socket. That needs cephx and a
+# quorum. It runs as 'mon.', so it still works while 'client.admin' is rotated.
+my sub mon_admin_run($node, $id, $words, $run_node = undef) {
+ if (defined($node) && cluster_nodes()->{$node}) {
+ my $cmd = ['ceph', 'daemon', "mon.$id", @$words];
+ return $run_node ? $run_node->($node, $cmd) : node_run($node, $cmd, entity => "mon.$id");
+ }
+
+ # do not let an unreachable monitor hold the run for the 300 second default
+ return monitor_command(['--connect-timeout', '10', 'tell', "mon.$id", @$words], 20);
+}
+
my sub poll_monitor_authentication($monitors, $nodes, $run_node = undef) {
- $run_node //= sub($node, $command) { node_run($node, $command, entity => $command->[2]) };
my ($reports, $errors) = ({}, {});
for my $mon (@$monitors) {
my $option = 'auth_service_required';
my $reply = eval {
- assert_member($nodes->{$mon}, "mon.$mon");
- decode_json(
- $run_node->(
- $nodes->{$mon},
- ['ceph', 'daemon', "mon.$mon", 'config', 'get', $option],
- ),
- );
+ decode_json(mon_admin_run(
+ $nodes->{$mon}, $mon, ['config', 'get', $option], $run_node));
};
my $error = $@;
$reports->{$mon}->{$option} = ref($reply) eq 'HASH' ? $reply->{$option} : undef;
@@ -789,20 +804,12 @@ for my $fsid (@fsids) {
}
PERL
-# Every monitor is asked over its admin socket, which needs no cephx and answers even while the
-# authentication layer itself is in trouble. One that does not answer marks the result incomplete,
-# so the checks built on it stay honest.
+# A monitor that does not answer marks the result incomplete.
my sub poll_client_sessions($info) {
my $per_mon = [];
for my $mon ($info->{daemons}->{mon}->@*) {
next if $mon->{down};
- my $sessions = eval {
- decode_json(node_run(
- $mon->{node},
- ['ceph', 'daemon', "mon.$mon->{id}", 'sessions'],
- entity => "mon.$mon->{id}",
- ));
- };
+ my $sessions = eval { decode_json(mon_admin_run($mon->{node}, $mon->{id}, ['sessions'])) };
push @$per_mon, { mon => $mon->{id}, sessions => $sessions };
}
return summarize_sessions($per_mon, $info->{monmap_mons});
@@ -811,15 +818,14 @@ my sub poll_client_sessions($info) {
# Whether a monitor can keep two client keys valid is told by its admin socket, which needs no
# cephx. One that answers but does not report the option is old; one that does not answer at all
# cannot be told apart from a broken connection, and rules staging out just the same.
-my sub probe_manual_promotion($node, $id, $run_node) {
- my $out =
- eval { $run_node->($node, ['ceph', 'daemon', "mon.$id", 'config', 'get', $GRACE_OPTION]) };
+my sub probe_manual_promotion($node, $id, $run_node = undef) {
+ my $out = eval { mon_admin_run($node, $id, ['config', 'get', $GRACE_OPTION], $run_node) };
if (!$@) {
my $res = eval { decode_json($out) };
my $value = ref($res) eq 'HASH' ? $res->{$GRACE_OPTION} : undef;
return { reached => 1, value => defined($value) && !ref($value) ? "$value" : undef };
}
- my $reached = eval { $run_node->($node, ['ceph', 'daemon', "mon.$id", 'version']); 1 } ? 1 : 0;
+ my $reached = eval { mon_admin_run($node, $id, ['version'], $run_node); 1 } ? 1 : 0;
return { reached => $reached, value => undef };
}
@@ -827,11 +833,7 @@ my sub poll_manual_promotion($info) {
my $reports = {};
for my $mon ($info->{daemons}->{mon}->@*) {
next if $mon->{down};
- $reports->{ $mon->{id} } = probe_manual_promotion(
- $mon->{node},
- $mon->{id},
- sub($node, $cmd) { node_run($node, $cmd, entity => "mon.$mon->{id}") },
- );
+ $reports->{ $mon->{id} } = probe_manual_promotion($mon->{node}, $mon->{id});
}
return manual_promotion_support($reports, $info->{monmap_mons});
}
@@ -882,8 +884,6 @@ my sub session_snapshot_with_retries($collect, $attempts = 10) {
# metadata, missing quorum membership, and a failed admin-socket query therefore make the summary
# incomplete rather than making that monitor disappear from it.
my sub collect_current_monitor_state($rados, $run_node = undef) {
- $run_node //= sub($node, $command) { node_run($node, $command) };
-
my $mon_dump = $rados->mon_command({ prefix => 'mon dump', format => 'json' });
die "could not refresh the monitor map\n"
if ref($mon_dump) ne 'HASH'
@@ -936,13 +936,9 @@ my sub collect_current_monitor_state($rados, $run_node = undef) {
$errors->{$id} = 'not in quorum';
} elsif (!$nodes->{$id}) {
$errors->{$id} = 'no hostname in monitor metadata';
- } elsif (!eval { assert_member($nodes->{$id}, "mon.$id"); 1 }) {
- $errors->{$id} = $@;
} else {
- $sessions = eval {
- decode_json($run_node->(
- $nodes->{$id}, ['ceph', 'daemon', "mon.$id", 'sessions']));
- };
+ $sessions =
+ eval { decode_json(mon_admin_run($nodes->{$id}, $id, ['sessions'], $run_node)) };
my $error = $@;
if ($error) {
$error =~ s/\s+/ /g;
@@ -6258,6 +6254,7 @@ sub key_migration_test_hooks {
client_key_files => \&client_key_files,
check_client_kernels => \&check_client_kernels,
manual_promotion_with_retries => \&manual_promotion_with_retries,
+ mon_admin_run => \&mon_admin_run,
describe_live => \&describe_live,
};
}
diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl
index ef13d8dd..fc9811d7 100755
--- a/test/CephKeyMigrationScript_test.pl
+++ b/test/CephKeyMigrationScript_test.pl
@@ -6600,7 +6600,16 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
$rados->{host} = 'foreign.invalid';
@commands = ();
$info = $HOOKS->{collect_cluster_info}->($rados, { 'rotate-lockbox-keys' => 1 }, {});
- is_deeply(\@commands, [], 'non-member gets no commands during pre-preflight collection');
+ is(
+ scalar(grep { /foreign\.invalid/ } map { join(' ', @$_) } @commands),
+ 0,
+ 'pre-preflight collection sends nothing to the non-member host',
+ );
+ is(
+ scalar(grep { /tell mon\.a config get/ } map { join(' ', @$_) } @commands),
+ 1,
+ 'the monitor outside the cluster is asked with tell instead of its socket',
+ );
like(
$info->{lockbox}->{'client.osd-lockbox.osd-uuid'}->{missing},
qr/osd\.7.*foreign\.invalid.*not a node/,
@@ -6609,10 +6618,14 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
@targets = ();
$monitor = $HOOKS->{collect_monitor_state}->($rados, sub { push @targets, $_[0]; return '[]' });
is_deeply(\@targets, [], 'fresh monitor collection never queries a non-member');
- like(
+ is(
$monitor->{sessions}->{errors}->{a},
- qr/mon\.a.*not a node/,
- 'monitor diagnostic names membership',
+ undef,
+ 'and its sessions come over the network, so nothing is left unanswered',
+ );
+ ok(
+ $monitor->{manual_promotion}->{supported},
+ 'so a client key can still be staged with a monitor outside the cluster',
);
for my $version (1, 2) {
@@ -7842,6 +7855,46 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
);
}
+# A monitor on one of our nodes is asked over its admin socket, any other one with 'ceph tell'.
+{
+ no warnings qw(once redefine);
+ local *PVE::SSHInfo::get_ssh_info = sub { return { node => $_[0] } };
+ local *PVE::SSHInfo::ssh_info_to_command = sub { return ['ssh', $_[0]->{node}, '--'] };
+
+ my @commands;
+ local *main::run_command = sub {
+ my ($cmd, %args) = @_;
+ push @commands, join(' ', @$cmd);
+ $args{outfunc}->('answer');
+ };
+
+ my $run = sub {
+ @commands = ();
+ return $HOOKS->{mon_admin_run}->(@_);
+ };
+
+ is($run->('node-a', 'a', ['sessions']), "answer\n", 'a member answers over its admin socket');
+ is($commands[0], 'ssh node-a -- ceph daemon mon.a sessions', 'which is asked through ssh');
+
+ is($run->('witness.invalid', 'tiebreaker', ['sessions']), "answer\n", 'so does an outsider');
+ like(
+ $commands[0],
+ qr/^ceph --cluster \S+ --name mon\. --keyring \S+ --connect-timeout 10 tell mon\.tiebreaker sessions$/,
+ 'asked locally with tell, as mon., and with a connect timeout instead of a long default',
+ );
+
+ $run->(undef, 'nameless', ['version']);
+ like($commands[0], qr/tell mon\.nameless version/, 'a monitor without a known host too');
+
+ my @routed;
+ $run->('node-a', 'a', ['config', 'get', 'opt'], sub { push @routed, $_[1]; return '{}' });
+ is_deeply(
+ \@routed,
+ [['ceph', 'daemon', 'mon.a', 'config', 'get', 'opt']],
+ 'a caller-supplied runner keeps the member route, which the tests rely on',
+ );
+}
+
{
my $out = '';
{
diff --git a/test/CephKeyMigration_test.pl b/test/CephKeyMigration_test.pl
index 79fccedf..d22ac24b 100755
--- a/test/CephKeyMigration_test.pl
+++ b/test/CephKeyMigration_test.pl
@@ -771,6 +771,18 @@ is(
sessions => [{ con_type => 'osd', entity_name => '', global_id => [] }],
}]);
ok($non_client->{complete}, 'a valid non-client session remains irrelevant');
+
+ my $own_tell = summarize_sessions([{
+ mon => 'a',
+ sessions => [{
+ con_type => 'client',
+ entity_name => 'mon.',
+ global_id => 140,
+ socket_addr => { addr => '10.0.0.3:0' },
+ }],
+ }]);
+ ok($own_tell->{complete}, "a 'mon.' session leaves the picture complete");
+ is_deeply($own_tell->{clients}, {}, "a 'mon.' session is no client of a key to refresh");
}
# --- the final step is offered once nothing else is open ---------------------------------------
--
2.47.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH manager v2 2/4] migrations: cephx: judge cipher support by the version a daemon runs
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 1/4] migrations: cephx: ask a monitor about itself through one route Kefu Chai
@ 2026-10-19 8:23 ` Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 3/4] migrations: cephx: rotate the monitor key with monitors we do not manage Kefu Chai
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Kefu Chai @ 2026-10-19 8:23 UTC (permalink / raw)
To: pve-devel; +Cc: Thomas Lamprecht, Jonas Theisen
The cipher check needs both the running and the installed version of a
daemon. Only pvestatd reports the installed version, and only on our
nodes. So a daemon on another host always failed the check, and a
stretch cluster with an external tiebreaker could not migrate.
For such a daemon, check only the running version and warn that Ceph on
that host must not be downgraded. A stopped daemon still fails the
check, as nothing reports its version.
Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
bin/pve-cephx-rotate-service-keys | 47 ++++++++++++++++----
test/CephKeyMigrationScript_test.pl | 66 +++++++++++++++++++++++++++++
2 files changed, 104 insertions(+), 9 deletions(-)
diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys
index 84daff6d..8048944b 100755
--- a/bin/pve-cephx-rotate-service-keys
+++ b/bin/pve-cephx-rotate-service-keys
@@ -189,9 +189,13 @@ my sub daemon_location($type, $id, $host) {
};
}
-my sub assert_member($node, $entity) {
+my sub assert_located($node, $entity) {
die "Cannot manage '$entity': no valid host name was reported\n"
if !defined($node) || ref($node) || !length($node);
+}
+
+my sub assert_member($node, $entity) {
+ assert_located($node, $entity);
die "Cannot manage '$entity' on '$node', which is not a node of this Proxmox VE cluster\n"
if !cluster_nodes()->{$node};
}
@@ -2545,31 +2549,49 @@ my sub preflight_nodes($info, $plan, $opts) {
(@touched, grep { defined($_->{node}) && $lockbox_nodes->{ $_->{node} } } @all);
}
- my (@outdated, @unknown_versions);
+ # The running version tells whether a daemon can use the new cipher now, and Ceph reports it for
+ # every daemon. The installed version tells whether it still can after a restart, and only
+ # pvestatd reports it, for our nodes. A daemon elsewhere is checked by its running version only,
+ # so it has to be running.
+ my (@outdated, @unknown_versions, @running_only);
for my $daemon (@judged) {
- if (!eval { assert_member($daemon->{node}, "$daemon->{type}.$daemon->{id}"); 1 }) {
+ my $label = "$daemon->{type}.$daemon->{id}";
+ my $stopped = ($daemon->{recovered} || $daemon->{down}) ? 1 : 0;
+
+ if (!eval { assert_located($daemon->{node}, $label); 1 }) {
push @unknown_versions, $@;
next;
}
- if (!$daemon->{recovered} && !$daemon->{down}) {
+ if (!$daemon->{managed} && $stopped) {
+ push @unknown_versions,
+ "cannot verify the Ceph version of $label: it does not run, and its host"
+ . " '$daemon->{node}' is not a node of this Proxmox VE cluster";
+ next;
+ }
+
+ if (!$stopped) {
if (!defined($daemon->{version}) || $daemon->{version} eq '') {
push @unknown_versions,
- "could not verify the running Ceph version of $daemon->{entity} on node"
+ "could not verify the running Ceph version of $label on node"
. " '$daemon->{node}'";
} elsif (!version_has_cipher($daemon->{version})) {
push @outdated,
- "$daemon->{entity} on node '$daemon->{node}' runs "
- . short_version($daemon->{version});
+ "$label on node '$daemon->{node}' runs " . short_version($daemon->{version});
+ } elsif (!$daemon->{managed}) {
+ push @running_only,
+ "$label on '$daemon->{node}' runs " . short_version($daemon->{version});
}
}
+ next if !$daemon->{managed};
+
if (!defined($daemon->{binary}) || $daemon->{binary} eq '') {
push @unknown_versions,
- "could not verify the installed Ceph version of $daemon->{entity} on node"
+ "could not verify the installed Ceph version of $label on node"
. " '$daemon->{node}'; check that 'pvestatd' runs there";
} elsif (!version_has_cipher($daemon->{binary})) {
push @outdated,
- "$daemon->{entity} on node '$daemon->{node}' would restart into "
+ "$label on node '$daemon->{node}' would restart into "
. short_version($daemon->{binary});
}
}
@@ -2636,6 +2658,13 @@ my sub preflight_nodes($info, $plan, $opts) {
}
}
+ if (@running_only) {
+ log_warn("These daemons run on hosts this cluster does not manage, so only the version they"
+ . " run was checked, not the one installed there. Keep Ceph on those hosts at this"
+ . " version or newer, or they lose the new cipher on their next restart:");
+ log_steps(\@running_only);
+ }
+
if (@unknown_versions) {
log_fail("Could not verify '$CIPHER' support for every service daemon:");
log_steps(\@unknown_versions);
diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl
index fc9811d7..6ccadbc4 100755
--- a/test/CephKeyMigrationScript_test.pl
+++ b/test/CephKeyMigrationScript_test.pl
@@ -6171,6 +6171,7 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
id => "$_",
entity => "mon.$_",
node => "node$_",
+ managed => 1,
store => 'file',
version => '20.2.4-pve4',
binary => '20.2.4-pve4',
@@ -8030,4 +8031,69 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
);
}
+# a daemon outside the cluster has no installed version, so only its running version counts
+{
+ my $info = sub {
+ my (%tiebreaker) = @_;
+ my $mons = [
+ (
+ map { {
+ type => 'mon',
+ id => "$_",
+ entity => "mon.$_",
+ node => "node$_",
+ managed => 1,
+ store => 'file',
+ version => '20.2.4-pve4',
+ binary => '20.2.4-pve4',
+ } } 1 .. 2
+ ),
+ {
+ type => 'mon',
+ id => 'tiebreaker',
+ # all monitors share the 'mon.' key, as the inventory records
+ entity => 'mon.',
+ node => 'tiebreaker.invalid',
+ managed => 0,
+ store => 'file',
+ version => '20.2.4-pve4',
+ %tiebreaker,
+ },
+ ];
+ return {
+ daemons => { mon => $mons, mgr => [], mds => [], osd => [] },
+ monmap_mons => [map { $_->{id} } @$mons],
+ rados => MonCountRados->new(),
+ };
+ };
+ my $preflight = sub {
+ my ($info) = @_;
+ my ($output, $verdict) = ('');
+ {
+ local *STDOUT;
+ open(STDOUT, '>', \$output) or die $!;
+ $verdict = $HOOKS->{preflight_nodes}
+ ->($info, { daemons => [], lockbox_keys => [], service_cipher => 1 }, {});
+ }
+ return ($verdict, $output // '');
+ };
+
+ my ($verdict, $output) = $preflight->($info->());
+ is($verdict, 1, 'a capable monitor outside the cluster does not block the cipher switch');
+ like(
+ $output,
+ qr/does not manage.*mon\.tiebreaker on 'tiebreaker\.invalid' runs 20\.2\.4/s,
+ 'naming it with the version it was judged by',
+ );
+
+ ($verdict, $output) = $preflight->($info->(version => '19.2.5'));
+ is($verdict, -1, 'an outdated monitor outside the cluster still blocks');
+ like($output, qr/mon\.tiebreaker on node 'tiebreaker\.invalid' runs 19\.2\.5/, 'as outdated');
+
+ ($verdict, $output) = $preflight->($info->(down => 1));
+ is($verdict, -1, 'a stopped monitor outside the cluster cannot be judged');
+ like($output, qr/mon\.tiebreaker.*not a node/, 'which names the membership');
+
+}
+
done_testing();
--
2.47.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH manager v2 3/4] migrations: cephx: rotate the monitor key with monitors we do not manage
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 1/4] migrations: cephx: ask a monitor about itself through one route Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 2/4] migrations: cephx: judge cipher support by the version a daemon runs Kefu Chai
@ 2026-10-19 8:23 ` Kefu Chai
2026-10-19 8:23 ` [PATCH manager v2 4/4] migrations: cephx: name the hosts keeping key copies we cannot write Kefu Chai
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Kefu Chai @ 2026-10-19 8:23 UTC (permalink / raw)
To: pve-devel; +Cc: Thomas Lamprecht, Jonas Theisen
Rotating the 'mon.' key writes each monitor's keyring and restarts the
monitors, which needs a shell on their hosts. A monitor outside this
cluster made the rotation fail.
Such a monitor does not need either step. It gets the new key from the
auth database like every other monitor. Only the keyring in its data
directory keeps the old key, and a monitor uses that keyring only if a
peer rejects it.
So write and restart only the monitors we manage. For the others, print
the commands to refresh their keyring and check that they are still in
the quorum. Skip the keyring checks for them in the preflight.
Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
bin/pve-cephx-rotate-service-keys | 76 +++++++++++++--
test/CephKeyMigrationScript_test.pl | 142 +++++++++++++++++++++++++++-
test/CephKeyMigration_test.pl | 16 ++++
3 files changed, 225 insertions(+), 9 deletions(-)
diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys
index 8048944b..c7565e9e 100755
--- a/bin/pve-cephx-rotate-service-keys
+++ b/bin/pve-cephx-rotate-service-keys
@@ -189,6 +189,14 @@ my sub daemon_location($type, $id, $host) {
};
}
+my sub mon_keyring_path($id) {
+ return "/var/lib/ceph/mon/$ccname-$id/keyring";
+}
+
+my sub unmanaged_monitors($info) {
+ return grep { !$_->{managed} } $info->{daemons}->{mon}->@*;
+}
+
my sub assert_located($node, $entity) {
die "Cannot manage '$entity': no valid host name was reported\n"
if !defined($node) || ref($node) || !length($node);
@@ -1589,7 +1597,9 @@ my sub probe_nodes($info, $plan, $opts = {}) {
my $installed = installed_versions();
my $specs = {};
for my $daemon (touched_daemons($info, $plan)) {
- assert_member($daemon->{node}, "$daemon->{type}.$daemon->{id}");
+ assert_located($daemon->{node}, "$daemon->{type}.$daemon->{id}");
+ # a host we do not manage has nothing to probe, so its running version is checked instead
+ next if !$daemon->{managed};
push $specs->{ $daemon->{node} }->@*, "$daemon->{type}:$daemon->{id}";
}
@@ -1612,6 +1622,7 @@ my sub probe_nodes($info, $plan, $opts = {}) {
for $info->{daemons}->{$type}->@*;
}
for my $daemon (touched_daemons($info, $plan)) {
+ next if !$daemon->{managed};
my $probe = $by_node->{ $daemon->{node} }->{"$daemon->{type}:$daemon->{id}"} // {};
$daemon->{store} = $probe->{store};
$daemon->{error} = $probe->{error};
@@ -2598,6 +2609,10 @@ my sub preflight_nodes($info, $plan, $opts) {
my @unusable;
for my $daemon (@touched) {
+ # a monitor gets the 'mon.' key from the auth database, so it needs no keyring on its host.
+ # Other daemons read their key from a file there.
+ next if !$daemon->{managed} && $daemon->{type} eq 'mon';
+
eval { assert_daemon_location($info->{rados}, $daemon, $info->{fsid}, $daemon) };
push @unusable, $@ if $@;
my $type = $daemon->{type};
@@ -2893,14 +2908,25 @@ my sub print_plan($info, $plan, $state, $opts, $storage_entities) {
$step++;
log_text("Step $step: rotate the shared 'mon.' key. Every keyring is written first, then"
. " the monitors restart one at a time.");
+ my @unmanaged = unmanaged_monitors($info);
+ log_step("Not managed by this cluster, so their emergency keyring keeps the old key: "
+ . join(', ', map { "mon.$_->{id} on '$_->{node}'" } @unmanaged)
+ . ". The commands for their admin follow at the end of the run.")
+ if @unmanaged;
if ($opts->{verbose}) {
log_step(
$info->{mon_key_in_auth_db}
? "Ceph lists the key in its health checks."
: "Ceph's health checks cannot see the key."
);
- log_step("monitors, restarted one at a time: "
- . join(', ', map { "$_->{id} (node $_->{node})" } $info->{daemons}->{mon}->@*));
+ log_step(
+ "monitors, restarted one at a time: "
+ . join(
+ ', ',
+ map { "$_->{id} (node $_->{node})" }
+ grep { $_->{managed} } $info->{daemons}->{mon}->@*,
+ ),
+ );
}
}
@@ -3253,6 +3279,37 @@ my sub merge_pve_mon_keyring($entry) {
return;
}
+# Such a monitor already uses the new key from the auth database. Only the keyring in its data
+# directory, which it uses if a peer rejects it, keeps the old key, and only its admin can fix that.
+my sub report_unmanaged_monitors($rados, $unmanaged) {
+ log_warn("These monitors run on hosts outside this Proxmox VE cluster. They use the new key"
+ . " already, while their emergency keyring still holds the old one. To bring that copy up"
+ . " to date, their admin can run:");
+ for my $mon (@$unmanaged) {
+ my $path = mon_keyring_path($mon->{id});
+ log_step("on '$mon->{node}', as root:");
+ log_step(" ceph auth get mon. -o $path");
+ log_step(" chown ceph:ceph $path && chmod 600 $path");
+ log_step(" systemctl restart ceph-mon\@$mon->{id}");
+ }
+ log_text("The first command needs an admin keyring on that host. Without one, run it on a node"
+ . " of this cluster and copy the file over. Nothing is broken until then, but such a"
+ . " monitor cannot authenticate from its keyring alone, which is what it falls back on"
+ . " after losing its store.");
+
+ my @lost = map { "mon.$_->{id} on '$_->{node}'" }
+ grep { !daemon_is_running($rados, 'mon', $_->{id}) } @$unmanaged;
+ if (@lost) {
+ log_warn("These are not in the quorum right now, so check them before anything else: "
+ . join(', ', @lost));
+ return;
+ }
+
+ log_pass("every monitor outside this cluster is still in the quorum");
+
+ return;
+}
+
my sub migrate_mon_key($rados, $state, $info, $opts, $plan) {
log_heading(
$plan->{mon_repair_only}
@@ -3263,8 +3320,9 @@ my sub migrate_mon_key($rados, $state, $info, $opts, $plan) {
# a stale-copy repair is not gated behind the opt-in, so it must not rotate and restart the
# quorum unasked. Finishing a started rotation is the exception
my $rotate = !$plan->{mon_repair_only};
- assert_daemon_location($rados, $_, $info->{fsid})
- for $rotate ? $info->{daemons}->{mon}->@* : ();
+ my @managed = grep { $_->{managed} } $info->{daemons}->{mon}->@*;
+ my @unmanaged = unmanaged_monitors($info);
+ assert_daemon_location($rados, $_, $info->{fsid}) for $rotate ? @managed : ();
my $entry = $rotate ? rotate_entity($rados, $state, 'mon.') : auth_entry($rados, 'mon.');
my $keyring = keyring_text($entry);
my $target = key_fingerprint($entry->{key});
@@ -3272,11 +3330,11 @@ my sub migrate_mon_key($rados, $state, $info, $opts, $plan) {
merge_pve_mon_keyring($entry) if ($info->{pve_mon_key} // '') ne $entry->{key};
# all keyrings first, so a monitor going down in between still finds the new key locally
- for my $mon ($info->{daemons}->{mon}->@*) {
+ for my $mon (@managed) {
next if $plan->{mon_repair_only};
next if ($state->{mon_keyring}->{ $mon->{id} } // '') eq $target;
- my $path = "/var/lib/ceph/mon/$ccname-$mon->{id}/keyring";
+ my $path = mon_keyring_path($mon->{id});
log_info("writing the new key to '$path' on node '$mon->{node}'");
write_node_file($mon->{node}, $path, $keyring);
@@ -3285,7 +3343,7 @@ my sub migrate_mon_key($rados, $state, $info, $opts, $plan) {
}
# only monitors holding the superseded key restart, so a keyring repair leaves the quorum alone
- for my $mon ($info->{daemons}->{mon}->@*) {
+ for my $mon (@managed) {
next if $plan->{mon_repair_only};
next if ($state->{mon_restarted}->{ $mon->{id} } // '') eq $target;
@@ -3321,6 +3379,8 @@ my sub migrate_mon_key($rados, $state, $info, $opts, $plan) {
log_pass("the shared monitor key now uses the '$CIPHER' cipher");
+ report_unmanaged_monitors($rados, \@unmanaged) if @unmanaged;
+
return;
}
diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl
index 6ccadbc4..5d251596 100755
--- a/test/CephKeyMigrationScript_test.pl
+++ b/test/CephKeyMigrationScript_test.pl
@@ -6193,6 +6193,49 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
return ($verdict, $output // '');
};
+ # an unmanaged monitor gets the key from the auth database, so no keyring is required there
+ {
+ my $mons = [
+ (
+ map { {
+ type => 'mon',
+ id => "$_",
+ entity => 'mon.',
+ node => "node$_",
+ managed => 1,
+ store => 'file',
+ version => '20.2.4',
+ binary => '20.2.4',
+ } } 1 .. 2
+ ),
+ {
+ type => 'mon',
+ id => 'tiebreaker',
+ entity => 'mon.',
+ node => 'witness.example',
+ managed => 0,
+ version => '20.2.4',
+ },
+ ];
+ my $info = {
+ daemons => { mon => $mons, mgr => [], mds => [], osd => [] },
+ monmap_mons => [map { $_->{id} } @$mons],
+ fsid => 'cluster-fsid',
+ rados => MonCountRados->new(),
+ };
+ my ($output, $verdict) = ('');
+ {
+ local *STDOUT;
+ open(STDOUT, '>', \$output) or die $!;
+ $verdict = $HOOKS->{preflight_nodes}->(
+ $info, { daemons => [], lockbox_keys => [], mon_key => 1 }, {},
+ );
+ }
+ is($verdict, 1, 'a monitor we do not manage does not block the monitor key rotation');
+ unlike($output, qr/neither a keyring file nor a bluestore device/, 'nothing is read there');
+ unlike($output, qr/Cannot manage 'mon\.tiebreaker'/, 'and nothing is verified there');
+ }
+
my ($verdict, $output) = $preflight->(2, { mon_key => 1 });
is($verdict, -1, 'a two-monitor cluster refuses the shared monitor key rotation');
like($output, qr/monitor map holds 2 monitors/, 'naming what it found');
@@ -7827,7 +7870,10 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
fsid => 'cluster-fsid',
pve_mon_key => $OLD,
exported => { 'mon.' => { key => $OLD } },
- daemons => { mon => [map { { type => 'mon', id => $_, node => "node-$_" } } qw(a b)] },
+ daemons => {
+ mon =>
+ [map { { type => 'mon', id => $_, node => "node-$_", managed => 1 } } qw(a b)],
+ },
};
my $plan = { mon_key => 1 };
my $state = {};
@@ -7896,6 +7942,100 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
);
}
+# A monitor outside this cluster gets the rotated key from the auth database. Its keyring and
+# restart are left to its admin.
+{
+ no warnings qw(once redefine);
+ local *PVE::Ceph::Services::get_blocking_health_errors = sub { return [] };
+ local *PVE::Ceph::Services::wait_for_safe_to_stop = sub { return (1, '') };
+ local *PVE::Ceph::Services::wait_for_daemon_up = sub { };
+ local *PVE::SSHInfo::get_ssh_info = sub { return { node => $_[0] } };
+ local *PVE::SSHInfo::ssh_info_to_command = sub { return ['ssh', $_[0]->{node}, '--'] };
+ local *main::file_set_contents = sub { };
+
+ my @commands;
+ local *main::run_command = sub { push @commands, join(' ', $_[0]->@*) };
+
+ my $run = sub {
+ my ($quorum) = @_;
+ my $rados = ClientRotationRados->new($OLD);
+ my $mon_command = ClientRotationRados->can('mon_command');
+ local *ClientRotationRados::mon_command = sub {
+ my ($self, $args) = @_;
+ die "the cluster cannot be read\n"
+ if !defined($quorum) && $args->{prefix} =~ m/^(?:quorum_status|health)$/;
+ return $quorum if $args->{prefix} eq 'quorum_status';
+ return {} if $args->{prefix} eq 'health';
+ return $mon_command->(@_);
+ };
+ my $info = {
+ fsid => 'cluster-fsid',
+ pve_mon_key => $OLD,
+ exported => { 'mon.' => { key => $OLD } },
+ daemons => {
+ mon => [
+ (
+ map { { type => 'mon', id => $_, node => "node-$_", managed => 1 } }
+ qw(a b)
+ ),
+ {
+ type => 'mon',
+ id => 'tiebreaker',
+ node => 'witness.example',
+ managed => 0,
+ },
+ ],
+ },
+ };
+ my $state = {};
+ my $out = '';
+ {
+ open(my $stdout, '>', \$out) or die $!;
+ local *STDOUT = $stdout;
+ eval {
+ $HOOKS->{migrate_mon_key}
+ ->($rados, $state, $info, { timeout => 30 }, { mon_key => 1 });
+ };
+ main::log_fail($@) if $@;
+ }
+ return ($state, $out, $rados);
+ };
+
+ @commands = ();
+ my ($state, $out, $rados) = $run->({ quorum_names => [qw(a b tiebreaker)] });
+ is($rados->{key}, $NEW, 'the shared key is rotated, not refused over the outside monitor');
+ is_deeply(
+ $state->{mon_restarted},
+ { map { $_ => key_fingerprint($NEW) } qw(a b) },
+ 'the monitors this cluster manages are restarted',
+ );
+ is(
+ scalar(grep { /ssh witness\.example/ } @commands),
+ 0,
+ 'and nothing is sent to the host outside it',
+ );
+ like(
+ $out,
+ qr{ceph auth get mon\. -o /var/lib/ceph/mon/\S+-tiebreaker/keyring},
+ 'the admin is handed the keyring command for that monitor',
+ );
+ like($out, qr/systemctl restart ceph-mon\@tiebreaker/, 'and the restart to run there');
+ like($out, qr/chmod 600/, 'with the mode that keeps the key private');
+ like($out, qr/needs an admin keyring on that host/, 'and what to do without admin access');
+ like($out, qr/PASS.*outside this cluster is still in the quorum/, 'the quorum is confirmed');
+
+ (undef, $out) = $run->({ quorum_names => [qw(a b)] });
+ like(
+ $out,
+ qr/WARN.*not in the quorum right now.*mon\.tiebreaker on 'witness\.example'/,
+ 'a monitor that left the quorum is named instead',
+ );
+
+ # an unreadable cluster says nothing about that monitor, and the rotation is done by then
+ (undef, $out) = $run->(undef);
+ unlike($out, qr/not in the quorum right now/, 'an unreadable cluster accuses nobody');
+}
+
{
my $out = '';
{
diff --git a/test/CephKeyMigration_test.pl b/test/CephKeyMigration_test.pl
index d22ac24b..d087c76a 100755
--- a/test/CephKeyMigration_test.pl
+++ b/test/CephKeyMigration_test.pl
@@ -566,6 +566,22 @@ is(
1,
'repairing only the stored copy writes to none of them, so none is validated either',
);
+
+ # its key is rotated too, so its version is checked. Only the work on its node is skipped.
+ my $stretch = {
+ daemons => {
+ mon => [
+ { entity => 'mon.', id => 'a', managed => 1 },
+ { entity => 'mon.', id => 'b', managed => 1 },
+ { entity => 'mon.', id => 'tiebreaker', managed => 0 },
+ ],
+ },
+ };
+ is(
+ scalar(touched_daemons($stretch, { daemons => [$osd], mon_key => 1 })),
+ 4,
+ 'a monitor outside the cluster counts as changed as well',
+ );
}
# --- daemons ceph cannot see ------------------------------------------------------------------
--
2.47.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH manager v2 4/4] migrations: cephx: name the hosts keeping key copies we cannot write
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
` (2 preceding siblings ...)
2026-10-19 8:23 ` [PATCH manager v2 3/4] migrations: cephx: rotate the monitor key with monitors we do not manage Kefu Chai
@ 2026-10-19 8:23 ` Kefu Chai
2026-10-19 14:47 ` [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Jonas Theisen
2026-10-19 18:23 ` applied: " Thomas Lamprecht
5 siblings, 0 replies; 7+ messages in thread
From: Kefu Chai @ 2026-10-19 8:23 UTC (permalink / raw)
To: pve-devel; +Cc: Thomas Lamprecht, Jonas Theisen
A host outside this cluster that runs a Ceph daemon usually keeps its
own copy of the 'client.admin' keyring. We do not write that copy, so a
key rotation leaves it stale.
Name these hosts when the key is staged and when it is committed, and
print commands to refresh the copy:
- While staged, fetch the new key, usually as 'client.admin'. A user
with only 'profile rbd' caps cannot read its own key. Take the key
from the JSON output and write it with ceph-authtool, because a
keyring saved by 'auth get' has a 'pending key' line that Ceph cannot
read.
- After the commit, fetch it with the monitor's own keyring, if the
host runs a monitor. Otherwise copy it over from one of our nodes.
The commands name their keyring files, because a ceph.conf copied from
here points to /etc/pve.
Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
bin/pve-cephx-rotate-service-keys | 53 ++++++++++++++++-
test/CephKeyMigrationScript_test.pl | 89 ++++++++++++++++++++++++++++-
2 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys
index c7565e9e..ea953f76 100755
--- a/bin/pve-cephx-rotate-service-keys
+++ b/bin/pve-cephx-rotate-service-keys
@@ -197,6 +197,51 @@ my sub unmanaged_monitors($info) {
return grep { !$_->{managed} } $info->{daemons}->{mon}->@*;
}
+# hosts that run Ceph daemons outside this cluster. '$DAEMON_TYPES' has no monitors, so add them.
+my sub unmanaged_hosts_of($info) {
+ my $hosts = {};
+ for my $type ('mon', @$DAEMON_TYPES) {
+ $hosts->{ $_->{node} } = 1
+ for grep { !$_->{managed} && defined($_->{node}) } $info->{daemons}->{$type}->@*;
+ }
+ return sort keys %$hosts;
+}
+
+# Such a host keeps its own key copies, usually of 'client.admin'. While the key is staged, its
+# admin can fetch the new key with the old one, as a user allowed to read keys. The key comes from
+# the JSON output, as Ceph cannot read the 'pending key' line that 'auth get' writes. After the
+# commit, a monitor host can use its 'mon.' keyring instead.
+my sub unmanaged_copy_note($info, $committed = 0) {
+ my @hosts = unmanaged_hosts_of($info);
+ return if !@hosts;
+
+ my $where = join(', ', map { "'$_'" } @hosts);
+ if (!$committed) {
+ log_step("Proxmox VE rewrites the key copies it manages. These hosts run Ceph daemons"
+ . " without being nodes of this cluster, so a keyring they keep for their own use is"
+ . " not among them: $where. Their admin can refresh one while both keys still work,"
+ . " as a user allowed to read keys, usually 'client.admin'. The commands name both"
+ . " keyrings, as a configuration copied from here looks for keys under '/etc/pve',"
+ . " which such a host does not have:");
+ log_step(" KEY=\$(ceph -n client.admin -k <admin keyring file> auth get USER -f json"
+ . " | python3 -c 'import json, sys; print(json.load(sys.stdin)[0][\"pending_key\"])')"
+ );
+ log_step(" ceph-authtool <keyring file> -n USER --add-key \"\$KEY\"");
+ return;
+ }
+
+ log_warn("The previous key no longer works. A keyring these hosts keep for their own use was"
+ . " not rewritten, as they are not nodes of this cluster: $where. On a monitor host, and"
+ . " while its keyring still holds the current 'mon.' key, its admin can refresh one with"
+ . " the monitor's own credential:");
+ log_step(" ceph --name mon. --keyring "
+ . mon_keyring_path('<id>')
+ . " auth get USER -o <keyring file>");
+ log_step("Otherwise fetch the key on a node of this cluster and copy the file over.");
+
+ return;
+}
+
my sub assert_located($node, $entity) {
die "Cannot manage '$entity': no valid host name was reported\n"
if !defined($node) || ref($node) || !length($node);
@@ -2029,6 +2074,7 @@ my sub preflight_cluster(
};
my $confirmation_refused = 0;
+ my $committed_copies = 0;
my $confirm_all = $opts->{'confirm-all-clients-refreshed'};
my $confirmation_seen = {};
my @confirm =
@@ -2256,7 +2302,7 @@ my sub preflight_cluster(
}
# The record remains open if the commit fails, so the confirmation is retryable.
- commit_staged_key($info->{rados}, $state, $entity) if $staged;
+ $committed_copies = 1 if $staged && commit_staged_key($info->{rados}, $state, $entity);
my $mark = $state->{client_refresh}->{$entity};
$mark->{cleared} = time();
$mark->{acknowledged} = time();
@@ -2265,6 +2311,9 @@ my sub preflight_cluster(
}
}
+ # copies on hosts we do not manage are stale from here on
+ unmanaged_copy_note($info, 1) if $committed_copies;
+
# Confirmation can promote staged keys. Recollect before any remaining preflight decision and
# update the caller's shared cluster picture before it builds the migration plan.
if (!$confirmation_refused && scalar(@confirm)) {
@@ -3045,6 +3094,7 @@ my sub print_plan($info, $plan, $state, $opts, $storage_entities) {
log_step("For each staged Ceph user key, both the current and new keys authenticate"
. " until the new key is committed with '--confirm-clients-refreshed USER' or,"
. " once every open record is ready, '--confirm-all-clients-refreshed'.");
+ unmanaged_copy_note($info);
log_step("the monitors stop automatic pending-key promotion until every staged key is"
. " committed or aborted")
if $opts->{verbose};
@@ -6344,6 +6394,7 @@ sub key_migration_test_hooks {
check_client_kernels => \&check_client_kernels,
manual_promotion_with_retries => \&manual_promotion_with_retries,
mon_admin_run => \&mon_admin_run,
+ unmanaged_copy_note => \&unmanaged_copy_note,
describe_live => \&describe_live,
};
}
diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl
index 5d251596..5c5a9bf8 100755
--- a/test/CephKeyMigrationScript_test.pl
+++ b/test/CephKeyMigrationScript_test.pl
@@ -3619,7 +3619,15 @@ sub run_aggregate_confirmation {
mon => [],
mgr => [],
mds => [],
- osd => [{ type => 'osd', id => 7, entity => 'osd.7', node => 'node-daemon' }],
+ osd => [
+ {
+ type => 'osd',
+ id => 7,
+ entity => 'osd.7',
+ node => 'node-daemon',
+ managed => 1,
+ },
+ ],
},
};
my $plan = {
@@ -3717,6 +3725,41 @@ sub run_aggregate_confirmation {
qr/For each staged Ceph user key, both the current and new keys authenticate.*committed with '--confirm-clients-refreshed USER' or, once every open record is ready, '--confirm-all-clients-refreshed'/s,
'the default names both staged completion paths without offering the aggregate early',
);
+ unlike(
+ $concise,
+ qr/without being nodes of this cluster/,
+ 'a cluster that manages every daemon host is told nothing about copies elsewhere',
+ );
+
+ # while the key is staged, a host outside this cluster can fetch it with the old key
+ {
+ my $stretched = { %$info, daemons => { %{ $info->{daemons} } } };
+ $stretched->{daemons}->{mon} = [
+ { type => 'mon', id => 'a', node => 'node-a', managed => 1 },
+ { type => 'mon', id => 'tiebreaker', node => 'witness.example', managed => 0 },
+ ];
+ my $output = $render->(0, undef, undef, undef, $stretched);
+ like(
+ $output,
+ qr/without being nodes of this cluster.*'witness\.example'.*while both keys still work/s,
+ 'the host is named while the old key still authenticates',
+ );
+ like(
+ $output,
+ qr/KEY=\$\(ceph -n client\.admin -k <admin keyring file> auth get USER -f json \|.*"pending_key".*\n.*ceph-authtool <keyring file> -n USER --add-key "\$KEY"/,
+ 'as a user allowed to read keys, writing the staged key itself, naming both keyrings',
+ );
+ unlike(
+ $output,
+ qr/'node-a'/,
+ 'and a managed node is not, as Proxmox VE rewrites its copies',
+ );
+ unlike(
+ $output,
+ qr/--name mon\./,
+ 'the monitor credential is not offered while the simpler one still works',
+ );
+ }
like(
$concise,
qr/For staged keys.*live-migrate affected\s+VMs, including those with kernel RBD disks/s,
@@ -7902,6 +7945,50 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
);
}
+# After the commit the old copy no longer works, so the note names another credential.
+{
+ my $stretched = {
+ daemons => {
+ mon => [
+ { type => 'mon', id => 'a', node => 'node-a', managed => 1 },
+ { type => 'mon', id => 'witness', node => 'witness.example', managed => 0 },
+ ],
+ mgr => [],
+ mds => [],
+ osd => [],
+ },
+ };
+ my $note = sub {
+ my ($committed) = @_;
+ my $out = '';
+ open(my $stdout, '>', \$out) or die $!;
+ local *STDOUT = $stdout;
+ $HOOKS->{unmanaged_copy_note}->($stretched, $committed);
+ return $out;
+ };
+
+ my $committed = $note->(1);
+ like($committed, qr/WARN.*previous key no longer works/, 'the note says the old key is gone');
+ like(
+ $committed,
+ qr{ceph --name mon\. --keyring /var/lib/ceph/mon/\S+-<id>/keyring auth get USER},
+ "and gives the monitor's own credential as the way back",
+ );
+ like($committed, qr/copy the file over/, 'with the fallback for a host running no monitor');
+ like($committed, qr/'witness\.example'/, 'naming the unmanaged host');
+ unlike($committed, qr/'node-a'/, 'and not the one Proxmox VE rewrites itself');
+ like($note->(0), qr/while both keys still work/, 'before the commit it is the simpler route');
+
+ my $managed = { daemons => { map { $_ => [] } qw(mon mgr mds osd) } };
+ my $out = '';
+ {
+ open(my $stdout, '>', \$out) or die $!;
+ local *STDOUT = $stdout;
+ $HOOKS->{unmanaged_copy_note}->($managed, 1);
+ }
+ is($out, '', 'a cluster that manages every daemon host hears nothing about copies elsewhere');
+}
+
# A monitor on one of our nodes is asked over its admin socket, any other one with 'ceph tell'.
{
no warnings qw(once redefine);
--
2.47.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
` (3 preceding siblings ...)
2026-10-19 8:23 ` [PATCH manager v2 4/4] migrations: cephx: name the hosts keeping key copies we cannot write Kefu Chai
@ 2026-10-19 14:47 ` Jonas Theisen
2026-10-19 18:23 ` applied: " Thomas Lamprecht
5 siblings, 0 replies; 7+ messages in thread
From: Jonas Theisen @ 2026-10-19 14:47 UTC (permalink / raw)
To: Kefu Chai, pve-devel; +Cc: Thomas Lamprecht
On 10/19/26 10:23, Kefu Chai wrote:
> Hi everyone,
>
> A user reported an interesting stretch-cluster setup that
> pve-cephx-rotate-service-keys wasn't ready for: a tiebreaker monitor
> running on a standalone Proxmox VE host that isn't a node of the
> cluster itself, joined to the corosync quorum only through a QDevice
> and never listed under /etc/pve/nodes. All five monitors were in
> quorum and on the same, capable Ceph version, but the dry run still
> failed twice over:
>
> <snip>
This issue is currently also present in a support ticket.
Tested this against a virtual 4-node PVE cluster with a fifth
PVE node outside the cluster as tiebreaker.
Without the patches as expected the migration helper fails.
With the patches and the instructed commands from the migration
helper manually run on the separate node, the migration was successful.
Small knit: The message during the admin keyring migration telling
the operator to manually update the key on the external node
is not highlighted in any way and such i nearly overlooked it.
Maybe this can be rendered as a warning.
Apart from that, consider this:
Tested-by: Jonas Theisen <j.theisen@proxmox.com>
^ permalink raw reply [flat|nested] 7+ messages in thread* applied: [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage
2026-10-19 8:23 [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Kefu Chai
` (4 preceding siblings ...)
2026-10-19 14:47 ` [PATCH manager v2 0/4] cephx: migrate keys with a monitor this cluster does not manage Jonas Theisen
@ 2026-10-19 18:23 ` Thomas Lamprecht
5 siblings, 0 replies; 7+ messages in thread
From: Thomas Lamprecht @ 2026-10-19 18:23 UTC (permalink / raw)
To: pve-devel, Kefu Chai; +Cc: Jonas Theisen
On Mon, 19 Oct 2026 16:23:42 +0800, Kefu Chai wrote:
> A user reported an interesting stretch-cluster setup that
> pve-cephx-rotate-service-keys wasn't ready for: a tiebreaker monitor
> running on a standalone Proxmox VE host that isn't a node of the
> cluster itself, joined to the corosync quorum only through a QDevice
> and never listed under /etc/pve/nodes. All five monitors were in
> quorum and on the same, capable Ceph version, but the dry run still
> failed twice over:
>
> [...]
Applied, thanks!
[1/4] migrations: cephx: ask a monitor about itself through one route
https://git.proxmox.com/?p=pve-manager.git;a=commitdiff;h=9b21b8d0e94cf4faa56ee3849180290e8999b00d
[2/4] migrations: cephx: judge cipher support by the version a daemon runs
https://git.proxmox.com/?p=pve-manager.git;a=commitdiff;h=05d23944d932c5736a8a80e855ccda56f5f4d100
[3/4] migrations: cephx: rotate the monitor key with monitors we do not manage
https://git.proxmox.com/?p=pve-manager.git;a=commitdiff;h=17b16f4f7a73636e03849d4005601be6f93b7bc5
[4/4] migrations: cephx: name the hosts keeping key copies we cannot write
https://git.proxmox.com/?p=pve-manager.git;a=commitdiff;h=ef9dcc649cc86c14dd502bcd8c0313ba6333dc48
^ permalink raw reply [flat|nested] 7+ messages in thread