From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 5328D1FF0B2 for ; Sun, 25 Oct 2026 10:58:11 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 652CF2125D; Sun, 25 Oct 2026 10:58:00 +0100 (CET) From: Kefu Chai To: pve-devel@lists.proxmox.com Subject: [PATCH manager v2 2/3] migrations: cephx: report daemons whose key on disk differs Date: Sun, 25 Oct 2026 17:57:29 +0800 Message-ID: <20261025095730.4027539-3-k.chai@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261025095730.4027539-1-k.chai@proxmox.com> References: <20261025095730.4027539-1-k.chai@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1792922257567 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.715 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: WDBMTCOCI3DCA5CXSPSAOPRVWTPB3JIY X-Message-ID-Hash: WDBMTCOCI3DCA5CXSPSAOPRVWTPB3JIY X-MailFrom: k.chai@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The helper judged a daemon migrated by the cipher of its key in the auth database alone. After a 'ceph auth rotate' by hand, the dry run reported PASS, while the daemon kept the old key on disk and failed with handle_auth_bad_method at its next restart. A user lost CephFS this way when all its MDS daemons restarted. Before any verdict, have the node probe report a fingerprint of the key each manager, metadata server and OSD in scope reads at its next start: the one in its keyring and, for an OSD, also the one in its bluestore label. Warn about every daemon whose copy differs from the key the monitors hold, naming the copy. List a daemon whose copy cannot be compared, for example one on an unreachable node, and in both cases turn the verdicts that report the cluster as migrated into a warning instead of a PASS. This probes every daemon in scope on every run, dry runs included, with one SSH session per node and one 'ceph-bluestore-tool show-label' per OSD, where only the daemons of a plan were probed before. Signed-off-by: Kefu Chai --- PVE/Ceph/KeyMigration.pm | 57 ++++++- bin/pve-cephx-rotate-service-keys | 172 ++++++++++++++++++--- test/CephKeyMigrationScript_test.pl | 227 ++++++++++++++++++++++++++++ test/CephKeyMigration_test.pl | 75 +++++++++ 4 files changed, 502 insertions(+), 29 deletions(-) diff --git a/PVE/Ceph/KeyMigration.pm b/PVE/Ceph/KeyMigration.pm index c24c1c3b..0cf2fc0e 100644 --- a/PVE/Ceph/KeyMigration.pm +++ b/PVE/Ceph/KeyMigration.pm @@ -21,7 +21,8 @@ our @EXPORT_OK = qw( key_cipher key_fingerprint keyring_text short_version version_has_cipher parse_storage_key storage_config_problem storage_key_copy_problems storage_key_copy_warnings parse_auth_methods summarize_monitor_authentication - parse_probe_output osd_label_identity needs_rotation mon_key_needs_rotation mon_keyring_stale + parse_probe_output osd_label_identity daemon_in_scope disk_key_candidates disk_key_verdict + needs_rotation mon_key_needs_rotation mon_keyring_stale mon_key_rotation_wanted client_keys_requested migration_unfinished unfinished_entities bulk_storage_staging_needed client_staging_needed touched_daemons plan_client_keys plan_lockbox_keys build_plan configured_daemon_locations @@ -262,7 +263,7 @@ sub version_has_cipher($version) { sub parse_probe_output($output) { my $probes = {}; for my $line (split(/\n/, $output)) { - next if $line !~ m/^(label|keyring|store|error) (\S+)(?: (.*))?$/; + next if $line !~ m/^(label|keyring|keyfp|store|error) (\S+)(?: (.*))?$/; my ($kind, $spec, $payload) = ($1, $2, $3 // ''); my $probe = $probes->{$spec} //= {}; @@ -279,6 +280,9 @@ sub parse_probe_output($output) { } elsif ($kind eq 'keyring') { $probe->{store} = 'file'; $probe->{sections} = [$payload =~ m/\[([^\]]+)\]/g]; + } elsif ($kind eq 'keyfp') { + # empty for a keyring without a key under the daemon's own section + $probe->{'keyring-fingerprint'} = $payload; } elsif ($kind eq 'store') { $probe->{store} = $payload; } elsif ($kind eq 'label') { @@ -326,6 +330,51 @@ sub osd_label_identity($daemon, $cluster_fsid) { return 'ok'; } +# '--only' names daemon types or single daemons +sub daemon_in_scope($only, $type, $entity) { + return 1 if !$only || $only->{$type}; + return $only->{$entity} ? 1 : 0; +} + +# daemons in scope whose key on disk is compared; the plan rewrites the keys of resumed ones anyway +sub disk_key_candidates($info, $state, $only = undef) { + return grep { + daemon_in_scope($only, $_->{type}, $_->{entity}) + && !$_->{recovered} + && !migration_unfinished($state, $_->{entity}) + } map { + $info->{daemons}->{$_}->@* + } @$DAEMON_TYPES; +} + +# Compare the key a daemon reads at its next start with the monitors'. An OSD reads its keyring at a +# restart, which ceph-volume rebuilds from the bluestore label at activation, so both count. +sub disk_key_verdict($info, $daemon, $probe) { + my $active = ($info->{exported}->{ $daemon->{entity} } // {})->{key}; + return { verdict => 'match' } if !length($active // ''); + + my $store = ($probe // {})->{store} // 'missing'; + my $copies; + if ($store eq 'file') { + $copies = { keyring => $probe->{'keyring-fingerprint'} }; + } elsif ($store eq 'block') { + return { verdict => 'unknown', why => 'identity' } + if osd_label_identity({ %$daemon, %$probe }, $info->{fsid}) ne 'ok'; + $copies = { label => key_fingerprint($probe->{'label-key'}) }; + # only an activated OSD has a keyring + $copies->{keyring} = $probe->{'keyring-fingerprint'} + if exists($probe->{'keyring-fingerprint'}); + } else { + return { verdict => 'unknown', why => $store }; + } + + my $want = key_fingerprint($active); + my @wrong = sort grep { ($copies->{$_} // '') ne $want } keys %$copies; + return { verdict => 'match' } if !@wrong; + + return { verdict => 'stale', copies => \@wrong }; +} + sub needs_rotation($info, $entity) { my $entry = $info->{exported}->{$entity}; @@ -1750,8 +1799,6 @@ sub build_plan($info, $state, $opts, $files) { } for my $type (@$DAEMON_TYPES) { - next if $only && !$only->{$type} && !grep { m/^\Q$type\E\./ } keys %$only; - # Restarting a standby costs no failover, so the active manager goes last: a fallback # restart there fails over onto one already migrated. my $daemons = [$info->{daemons}->{$type}->@*]; @@ -1759,7 +1806,7 @@ sub build_plan($info, $state, $opts, $files) { if $type eq 'mgr'; for my $daemon (@$daemons) { - next if $only && !$only->{$type} && !$only->{ $daemon->{entity} }; + next if !daemon_in_scope($only, $type, $daemon->{entity}); my $unfinished = migration_unfinished($state, $daemon->{entity}); # Not the marker alone: a reused OSD id gets a fresh key that needs migrating like any # other. diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys index b0202313..7b6bd536 100755 --- a/bin/pve-cephx-rotate-service-keys +++ b/bin/pve-cephx-rotate-service-keys @@ -31,7 +31,8 @@ use PVE::Ceph::KeyMigration qw( key_cipher key_fingerprint keyring_text short_version version_has_cipher parse_storage_key storage_config_problem storage_key_copy_problems storage_key_copy_warnings parse_auth_methods summarize_monitor_authentication - parse_probe_output osd_label_identity needs_rotation mon_key_needs_rotation mon_keyring_stale + parse_probe_output osd_label_identity disk_key_candidates disk_key_verdict + needs_rotation mon_key_needs_rotation mon_keyring_stale mon_key_rotation_wanted client_keys_requested migration_unfinished unfinished_entities client_staging_needed touched_daemons plan_client_keys plan_lockbox_keys build_plan configured_daemon_locations @@ -471,6 +472,7 @@ my $PROBE_SCRIPT = <<'PERL_EOF'; use strict; use warnings; +use Digest::SHA qw(sha256_hex); use JSON; use File::Temp; use POSIX (); @@ -514,10 +516,27 @@ sub command_output { return ($out, undef); } +# the sections of a keyring, and the fingerprint of the key under the daemon's own section +sub read_keyring { + my ($path, $entity) = @_; + open(my $fh, '<', $path) or return; + my ($sections, $section, $key) = ('', '', undef); + for my $line (<$fh>) { + if ($line =~ m/^(\[([^\]]*)\])/) { + $sections .= $1; + $section = $2; + } elsif ($section eq $entity && $line =~ m/^\s*key\s*=\s*(\S+)/) { + $key //= $1; + } + } + return ($sections, defined($key) ? substr(sha256_hex($key), 0, 16) : ''); +} + my ($cluster, @specs) = @ARGV; for my $spec (@specs) { my ($type, $id) = split(/:/, $spec, 2); my $dir = "/var/lib/ceph/$type/$cluster-$id"; + my $entity = "$type.$id"; if (-e "$dir/block") { my ($label, $err, $details) = @@ -527,10 +546,15 @@ for my $spec (@specs) { } else { $label =~ s/\n//g; print "label $spec $label\n"; + # what a restart reads, until the next activation rebuilds it from the label + my (undef, $fingerprint) = read_keyring("$dir/keyring", $entity); + print "keyfp $spec $fingerprint\n" if defined($fingerprint); } - } elsif (-f "$dir/keyring" && open(my $fh, '<', "$dir/keyring")) { - my $sections = join('', map { m/^(\[[^\]]*\])/ ? $1 : () } <$fh>); + } elsif (-f "$dir/keyring" + && (my ($sections, $fingerprint) = read_keyring("$dir/keyring", $entity)) + ) { print "keyring $spec $sections\n"; + print "keyfp $spec $fingerprint\n"; } else { print "store $spec missing\n"; } @@ -1641,27 +1665,39 @@ my sub installed_versions() { return { map { $_ => $broadcast->{$_}->{version}->{str} } keys %$broadcast }; } -my sub probe_nodes($info, $plan, $opts = {}) { - my $installed = installed_versions(); +# one SSH session per node; a node that cannot be reached gets its error instead of probes +my sub probe_daemons($opts, @daemons) { my $specs = {}; - for my $daemon (touched_daemons($info, $plan)) { - assert_located($daemon->{node}, "$daemon->{type}.$daemon->{id}"); - # a host we do not manage has nothing to probe, so its running version is checked instead - next if !$daemon->{managed}; - push $specs->{ $daemon->{node} }->@*, "$daemon->{type}:$daemon->{id}"; - } + push $specs->{ $_->{node} }->@*, "$_->{type}:$_->{id}" for @daemons; my $by_node = {}; log_info( "collecting daemon keyrings and bluestore labels from " . scalar(keys %$specs) . " node(s)") if scalar(keys %$specs) && ($opts->{apply} || $opts->{verbose}); for my $node (sort keys %$specs) { - my $output = eval { node_perl($node, $PROBE_SCRIPT, args => [$ccname, sort $specs->{$node}->@*]); }; - die "could not reach node '$node': $@" if $@; + if (my $err = $@) { + # an abort has to travel, not read as a node that cannot be reached + die $err if $err =~ m/aborting (?:on signal|bulk-restart)/; + $by_node->{$node} = { error => "could not reach node '$node': $err" }; + next; + } + $by_node->{$node} = { probes => parse_probe_output($output) }; + } - $by_node->{$node} = parse_probe_output($output); + return $by_node; +} + +my sub probe_nodes($info, $plan, $opts = {}) { + my $installed = installed_versions(); + my @touched = touched_daemons($info, $plan); + assert_located($_->{node}, "$_->{type}.$_->{id}") for @touched; + # a host we do not manage has nothing to probe, so its running version is checked instead + my @managed = grep { $_->{managed} } @touched; + my $by_node = probe_daemons($opts, @managed); + for my $node (sort keys %$by_node) { + die $by_node->{$node}->{error} if $by_node->{$node}->{error}; } # every daemon is judged on its Ceph version, only the touched ones on their data directory @@ -1669,9 +1705,9 @@ my sub probe_nodes($info, $plan, $opts = {}) { $_->{binary} = defined($_->{node}) ? $installed->{ $_->{node} } : undef for $info->{daemons}->{$type}->@*; } - for my $daemon (touched_daemons($info, $plan)) { - next if !$daemon->{managed}; - my $probe = $by_node->{ $daemon->{node} }->{"$daemon->{type}:$daemon->{id}"} // {}; + for my $daemon (@managed) { + my $probe = $by_node->{ $daemon->{node} }->{probes}->{"$daemon->{type}:$daemon->{id}"} + // {}; $daemon->{store} = $probe->{store}; $daemon->{error} = $probe->{error}; $daemon->{'error-details'} = $probe->{'error-details'}; @@ -1684,6 +1720,89 @@ my sub probe_nodes($info, $plan, $opts = {}) { return; } +my sub describe_disk_key_doubt($daemon, $why) { + my $text = { + 'probe-error' => $daemon->{error} // 'its inspection failed', + 'identity' => 'its bluestore label does not identify it as this OSD', + }->{$why} + // "no key it reads was found under /var/lib/ceph/$daemon->{type}/$ccname-$daemon->{id}"; + return "$daemon->{entity} on node '$daemon->{node}': $text"; +} + +# daemons whose key on disk this run could not compare with the monitors' +our $disk_key_doubts = []; + +# A key changed by hand reads as migrated in the auth database, while the daemon keeps the old one +# on disk and fails at its next start. Record such daemons in $info->{stale_keys}. +my sub check_disk_keys($info, $state, $opts) { + my (@probed, @doubts); + for my $daemon (disk_key_candidates($info, $state, $opts->{only})) { + my ($entity, $node) = ($daemon->{entity}, $daemon->{node}); + if ($daemon->{managed}) { + push @probed, $daemon; + } elsif (defined($node)) { + push @doubts, + [$entity, "$entity on node '$node': not a node of this Proxmox VE cluster"]; + } else { + push @doubts, [$entity, "$entity: no valid host name was reported"]; + } + } + my $by_node = probe_daemons($opts, @probed); + + $info->{stale_keys} = {}; + for my $daemon (@probed) { + my ($entity, $node) = ($daemon->{entity}, $daemon->{node}); + if (my $err = $by_node->{$node}->{error}) { + chomp($err); + push @doubts, [$entity, "$entity on node '$node': $err"]; + next; + } + my $probe = $by_node->{$node}->{probes}->{"$daemon->{type}:$daemon->{id}"} // {}; + my $verdict = disk_key_verdict($info, $daemon, $probe); + if ($verdict->{verdict} eq 'stale') { + $info->{stale_keys}->{$entity} = $verdict->{copies}; + } elsif ($verdict->{verdict} eq 'unknown') { + chomp(my $text = describe_disk_key_doubt({ %$daemon, %$probe }, $verdict->{why})); + push @doubts, [$entity, $text]; + } + } + # a stale copy is not planned for repair yet, so it also keeps the verdict from passing + $disk_key_doubts = [(map { $_->[0] } @doubts), sort keys $info->{stale_keys}->%*]; + if (@doubts) { + log_warn("Could not compare the key on disk with the monitors' for these daemons, so a" + . " key changed outside this script would go unnoticed there:"); + log_steps([map { $_->[1] } @doubts]); + } + # a keyring is read at the next start, an OSD's label at its next activation + my $stale = $info->{stale_keys}; + if (%$stale) { + log_warn( + "These keep a key on disk that the monitors do not accept, as after a" + . " 'ceph auth rotate', and would fail to authenticate once they read it: " + . join( + ', ', + map { "$_ (" . join(' and ', $stale->{$_}->@*) . ")" } sort keys %$stale, + ) + . "." + ); + } + + return; +} + +# a daemon whose key on disk could not be compared may still fail at its next start +my sub log_verdict($message) { + if (@$disk_key_doubts) { + log_warn("$message But the key on disk could not be confirmed for " + . join(', ', @$disk_key_doubts) + . ", so a daemon listed may still fail at its next start."); + return; + } + log_pass($message); + + return; +} + # A client key rotated outside this script leaves clients behind the same way as one this # script rotates, and only the key fingerprints recorded at earlier apply runs make that # visible. Shared by the preflight and by the destructive actions, which reconcile again right @@ -2442,7 +2561,7 @@ my sub preflight_cluster( && !$recovered_count && !@unfinished ) { - log_pass("Every service key uses '$CIPHER', and so do the service tickets."); + log_verdict("Every service key uses '$CIPHER', and so do the service tickets."); mon_key_hint($info, $opts); return 0; } @@ -5113,7 +5232,7 @@ my sub print_open_options( } if (!$planned && !$opts->{only} && $open->{complete}) { - log_pass("Cephx migration is complete. Authentication and new keys use only '$CIPHER'."); + log_verdict("Cephx migration is complete. Authentication and new keys use only '$CIPHER'."); log_text("AUTH_INSECURE_ROTATING_SERVICE_KEY_TYPE clears on its own within a few hours." . " You do not need to wait for it.") if ($snapshot->{health_checks} // {})->{AUTH_INSECURE_ROTATING_SERVICE_KEY_TYPE}; @@ -5880,6 +5999,7 @@ my sub collect_run_info($rados, $opts, $state, $previous = undef) { my sub run_migration($opts, $confirm = \&confirm_apply_run) { local $storage_copy_warnings_seen = {}; + local $disk_key_doubts = []; $monmap_changed = 0; PVE::RPCEnvironment->setup_default_cli_env(); PVE::Ceph::Tools::check_ceph_inited(); @@ -6078,6 +6198,9 @@ my sub run_migration($opts, $confirm = \&confirm_apply_run) { . " saved plan because Ceph no longer lists it"); } + # before any verdict + check_disk_keys($info, $state, $opts); + # Refresh storage mappings after any lock wait, then make sure they still agree with the # options before settling a staged key. $client_files = client_key_files(); @@ -6170,9 +6293,9 @@ my sub run_migration($opts, $confirm = \&confirm_apply_run) { } if ($opts->{only}) { - log_pass("There is nothing left to migrate in the scope given with '--only'."); + log_verdict("There is nothing left to migrate in the scope given with '--only'."); } else { - log_pass("Every service key uses '$CIPHER', and so do the service tickets."); + log_verdict("Every service key uses '$CIPHER', and so do the service tickets."); } mon_key_hint($info, $opts); print_open_options( @@ -6338,12 +6461,12 @@ my sub run_migration($opts, $confirm = \&confirm_apply_run) { log_heading("Done"); if ($plan->{scoped}) { - log_pass("The keys covered by '--only' now use the '$CIPHER' cipher. Run this without" + log_verdict("The keys covered by '--only' now use the '$CIPHER' cipher. Run this without" . " '--only' to migrate the rest and to switch the service cipher over."); mon_key_hint($info, $opts); } elsif (scalar($plan->{daemons}->@*)) { - log_pass("The manager, metadata server, and OSD keys of this cluster now use the '$CIPHER'" - . " cipher."); + log_verdict("The manager, metadata server, and OSD keys of this cluster now use the" + . " '$CIPHER' cipher."); mon_key_hint($info, $opts); } else { mon_key_hint($info, $opts); @@ -6428,6 +6551,7 @@ sub key_migration_test_hooks { print_closing_notes => \&print_closing_notes, resume_live_swap => \&resume_live_swap, probe_nodes => \&probe_nodes, + check_disk_keys => \&check_disk_keys, probe_script => $PROBE_SCRIPT, preflight_nodes => \&preflight_nodes, preflight_mon_key => \&preflight_mon_key, diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl index b03c2e29..1d473dd6 100755 --- a/test/CephKeyMigrationScript_test.pl +++ b/test/CephKeyMigrationScript_test.pl @@ -8411,4 +8411,231 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) { } +# a key changed in the auth database alone must not read as a finished migration +{ + my $root = tempdir(CLEANUP => 1); + my $node = PVE::INotify::nodename(); + mkdir("$root/mds"); + mkdir("$root/mds/ceph-a"); + my $write_keyring = sub { + open(my $fh, '>', "$root/mds/ceph-a/keyring") or die $!; + print {$fh} join('', map { "[$_->[0]]\n\tkey = $_->[1]\n" } @_); + close($fh); + }; + + my $mds = { type => 'mds', id => 'a', entity => 'mds.a', node => $node, managed => 1 }; + my $info = { + fsid => 'cluster-fsid', + daemons => { mon => [], mgr => [], mds => [$mds], osd => [] }, + exported => { 'mds.a' => { key => $NEW } }, + }; + my $check = sub { + my ($fail) = @_; + my $output; + no warnings qw(once redefine); + # the real probe, reading a data directory under $root instead of /var/lib/ceph. Through + # a pipe, as run_command() loses a child's output while STDOUT is captured in memory. + local *main::run_command = sub { + my ($cmd, %param) = @_; + die "$fail\n" if $fail; + my ($perl, undef, undef, @specs) = @$cmd; + my $script = File::Temp->new(); + print {$script} $param{input} =~ s{/var/lib/ceph/}{$root/}gr; + close($script); + open(my $pipe, '-|', $perl, $script->filename, 'ceph', @specs) or die $!; + while (my $line = <$pipe>) { + chomp $line; + $param{outfunc}->($line); + } + close($pipe) or die "probe failed\n"; + }; + local *STDOUT; + open(STDOUT, '>', \$output) or die $!; + $HOOKS->{check_disk_keys}->($info, {}, {}); + return ($info->{stale_keys}, $output // ''); + }; + + $write_keyring->(['mds.a', $OLD]); + my ($stale, $output) = $check->(); + is_deeply($stale, { 'mds.a' => ['keyring'] }, 'a keyring holding an old key is found'); + like( + $output, + qr/^WARN: These keep a key on disk that the monitors do not accept.*: mds\.a \(keyring\)\.$/m, + 'and reported, naming the copy that differs', + ); + $write_keyring->(['client.admin', $OLD], ['mds.a', $NEW]); + ($stale) = $check->(); + is_deeply($stale, {}, 'a keyring is judged by the key under the daemon\'s own section'); + + ($stale, $output) = $check->("ssh: connect to host $node: No route to host"); + is_deeply($stale, {}, 'an unreachable node judges nothing'); + like( + $output, + qr/Could not compare the key on disk.*mds\.a on node '\Q$node\E': could not reach node .*No route to host/s, + 'but says which daemons it could not compare', + ); + eval { $check->('aborting on signal, run this again to resume') }; + like($@, qr/aborting on signal/, 'an abort while probing ends the check, not as a dead node'); + + $info->{daemons}->{mds} = [{ %$mds, node => 'elsewhere', managed => 0 }]; + (undef, $output) = $check->(); + like( + $output, + qr/mds\.a on node 'elsewhere': not a node of this Proxmox VE cluster$/m, + 'a daemon outside the cluster is reported too', + ); +} + +# The real orchestration, on a cluster that reads as migrated in the auth database while a +# daemon kept an old key on disk. +{ + no warnings qw(once redefine); + my $dir = tempdir(CLEANUP => 1); + my $source = file_get_contents($SCRIPT); + $source =~ s{/etc/pve/priv/cephx-key-migration.json}{$dir/state.json}g; + $source =~ s{/var/lock/pve-ceph-bulk-restart.lck}{$dir/operation.lock}g; + $source =~ + s/package PVE::Ceph::KeyMigration::ClusterLock;/package MigrationDiskKeyTest::ClusterLock;/; + local *PVE::Ceph::Tools::get_config = sub { return $_[0] eq 'ccname' ? 'ceph' : "$dir/$_[0]" }; + file_set_contents("$dir/pve_mon_key_path", "[mon.]\n key = $NEW\n"); + # a package of its own, as the run above loaded the script into MigrationRunTest already + eval "package MigrationDiskKeyTest; $source" or die $@; + my $migrate = MigrationDiskKeyTest::key_migration_test_hooks()->{run_migration}; + + local *PVE::RPCEnvironment::setup_default_cli_env = sub { }; + local *PVE::Ceph::Tools::check_ceph_inited = sub { }; + local *PVE::Ceph::KeyMigration::ClusterLock::take = sub { return {} }; + local *PVE::Ceph::Services::get_cluster_service = sub { return {} }; + local *PVE::Ceph::Services::get_ceph_versions = sub { + return { 'node-a' => { version => { str => '20.2.4' } } }; + }; + # 'ceph tell' runs on this node + local *PVE::Cluster::get_nodelist = sub { return ['node-a', PVE::INotify::nodename()] }; + local *PVE::SSHInfo::get_ssh_info = sub { return { node => $_[0] } }; + local *PVE::SSHInfo::ssh_info_to_command = sub { return ['ssh', $_[0]->{node}, '--'] }; + local *PVE::Storage::config = sub { return { ids => {} } }; + local *PVE::Ceph::Services::with_cluster_bulk_restart_lock = sub { $_[3]->() }; + local *PVE::Ceph::Services::acquire_cluster_bulk_restart_lock = sub { }; + local *PVE::Ceph::Services::get_blocking_health_errors = sub { return [] }; + local *PVE::Ceph::Services::daemon_is_up = sub { return 1 }; + local *PVE::Ceph::Services::wait_for_safe_to_stop = sub { return (1, '') }; + local *PVE::Ceph::Services::wait_for_daemon_up = sub { }; + + my $entries; + my $fresh = 'AgCk941qku/sDSAAIjO5RhRv/ogXhuxccNS4DZxlXS1LUgzEGFIiY/U7ImI='; + my $rados = CurrentMonitorRados->new( + mons => ['a'], + quorum => ['a'], + osds => [], + metadata => [{ name => 'a', hostname => 'node-a', ceph_version_short => '20.2.4' }], + ); + local *PVE::Ceph::Services::ResilientRados::new = sub { return $rados }; + my $mon_command = CurrentMonitorRados->can('mon_command'); + my @auth; + local *CurrentMonitorRados::mon_command = sub { + my ($self, $args) = @_; + my $prefix = $args->{prefix}; + return 'cluster-fsid' if $prefix eq 'fsid'; + return { quorum_names => ['a'], features => { quorum_mon => ['cephx_auth_aes256k'] } } + if $prefix eq 'quorum_status'; + return {} if $prefix eq 'mgr dump'; + return [{ name => 'a', hostname => 'node-a', ceph_version => 'ceph version 20.2.4' }] + if $prefix eq 'mds metadata'; + return [] if $prefix =~ /^(?:mgr|osd) metadata$/; + return [ + map { + { %$_ } + } values %$entries + ] + if $prefix eq 'auth export'; + return [{ %{ $entries->{ $args->{entity} } } }] if $prefix eq 'auth get'; + if ($prefix =~ m/^auth (rotate|get-or-create-pending|commit-pending)$/) { + push @auth, "$1 $args->{entity}"; + my $entry = $entries->{ $args->{entity} }; + $entry->{pending_key} = $fresh if $1 eq 'get-or-create-pending'; + $entry->{key} = delete($entry->{pending_key}) if $1 eq 'commit-pending'; + return [{%$entry}]; + } + return $mon_command->(@_); + }; + my ($on_disk, $unreachable, @probes, @written, @told, @units); + local *MigrationDiskKeyTest::file_set_contents = sub { }; + local *MigrationDiskKeyTest::run_command = sub { + my ($cmd, %args) = @_; + return 0 if $cmd->[0] eq 'dpkg' && $cmd->[1] eq '--compare-versions'; + my $out = ''; + if (grep { $_ eq "'mds:a'" || $_ eq 'mds:a' } @$cmd) { + push @probes, join(' ', @$cmd); + die "ssh: connect to host node-a: No route to host\n" if $unreachable; + $out = "keyring mds:a [mds.a]\nkeyfp mds:a " . key_fingerprint($on_disk); + } elsif ($cmd->[-1] =~ m{/var/lib/ceph/mds/ceph-a/keyring'?$}) { + my ($key) = $args{input} =~ m/__END__\n\[mds\.a\]\n\tkey = (\S+)\n/; + push @written, $key; + $on_disk = $key; + } elsif (grep { $_ eq 'tell' } @$cmd) { + push @told, + join(' ', grep { $_ ne '--cluster' && $_ ne 'ceph' } @$cmd[1 .. $#$cmd]); + } elsif (grep { m/^'?systemctl'?$/ } @$cmd) { + push @units, join(' ', map { s/'//gr } @$cmd[-2, -1]); + } elsif ($cmd->[-1] eq 'sessions') { + $out = '[]'; + } elsif ($cmd->[-1] eq 'auth_service_required') { + $out = encode_json({ auth_service_required => 'cephx' }); + } elsif ($cmd->[-1] eq 'mon_auth_client_pending_key_auto_promote') { + $out = encode_json({ mon_auth_client_pending_key_auto_promote => 'false' }); + } else { + die 'unexpected node command: ' . join(' ', @$cmd) . "\n"; + } + $args{outfunc}->($_) for split(/\n/, $out); + return 0; + }; + my $run = sub { + my ($disk_key, $opts, $node_down) = @_; + ($on_disk, $unreachable) = ($disk_key, $node_down); + (@probes, @written, @told, @units, @auth) = (); + $entries = { + 'mon.' => { entity => 'mon.', key => $NEW }, + 'mds.a' => + { entity => 'mds.a', key => $NEW, caps => { mon => 'allow profile mds' } }, + }; + my ($result, $out) = (undef, ''); + open(my $stdout, '>', \$out) or die $!; + local *STDOUT = $stdout; + $result = eval { + $migrate->($opts, sub { return 1 }); + }; + is($@, '', 'the run itself does not die'); + return ($result, $out); + }; + + my ($result, $out) = $run->($NEW, {}); + is($result, 0, 'with the key on disk matching, the migrated cluster passes'); + like($out, qr/^PASS: Every service key uses 'aes256k'/m, 'as before'); + is(scalar(@probes), 1, 'after inspecting the keyring of the metadata server once'); + + ($result, $out) = $run->($OLD, {}); + is($result, 0, 'a daemon holding another key than the monitors does not stop the run'); + like( + $out, + qr/^WARN: These keep a key on disk that the monitors do not accept, as after a 'ceph auth rotate', and would fail to authenticate once they read it: mds\.a \(keyring\)\.$/m, + 'but is reported, naming the copy that differs', + ); + like( + $out, + qr/^WARN: Every service key uses 'aes256k', and so do the service tickets\. But the key on disk could not be confirmed for mds\.a/m, + 'and the verdict names it', + ); + unlike($out, qr/^PASS: Every service key/m, 'without reporting the cluster as migrated'); + is_deeply([@written, @told, @units, @auth], [], 'and a dry run changes nothing'); + + ($result, $out) = $run->($NEW, {}, 1); + is($result, 0, 'a cluster whose daemon could not be checked still reads as migrated'); + like( + $out, + qr/^WARN: Every service key uses 'aes256k', and so do the service tickets\. But the key on disk could not be confirmed for mds\.a/m, + 'but the verdict says which daemon could not be checked', + ); + unlike($out, qr/^PASS:/m, 'and passes nothing'); +} + done_testing(); diff --git a/test/CephKeyMigration_test.pl b/test/CephKeyMigration_test.pl index d087c76a..dfa3ec91 100755 --- a/test/CephKeyMigration_test.pl +++ b/test/CephKeyMigration_test.pl @@ -15,6 +15,7 @@ use PVE::Ceph::KeyMigration qw( parse_auth_methods summarize_monitor_authentication $CIPHER $LEGACY_CIPHER key_cipher key_fingerprint parse_probe_output osd_label_identity needs_rotation mon_keyring_stale + daemon_in_scope disk_key_candidates disk_key_verdict mon_key_rotation_wanted migration_unfinished unfinished_entities touched_daemons plan_client_keys build_plan bulk_storage_staging_needed client_staging_needed client_keys_requested @@ -427,6 +428,80 @@ isnt(key_fingerprint($NEW), key_fingerprint($OLD), 'different keys get different is($broken->{'osd:5'}->{store}, 'block-without-key', 'so is one that does not decode'); } +{ + # the fingerprint of the key under the daemon's own section, empty if that section has none + my $old = key_fingerprint($OLD); + my $probes = parse_probe_output( + "keyring mds:a [mds.a]\nkeyfp mds:a $old\nkeyring mds:b [mds.b]\nkeyfp mds:b\n" + . "label osd:3 $LABEL\n", + ); + is($probes->{'mds:a'}->{'keyring-fingerprint'}, $old, 'a keyring reports its key fingerprint'); + is($probes->{'mds:b'}->{'keyring-fingerprint'}, '', 'one without a key reports none'); + ok(!exists($probes->{'osd:3'}->{'keyring-fingerprint'}), 'an OSD not activated has no keyring'); +} + +# --- whether the key a daemon reads at its next start is one the monitors accept ------------- +{ + my ($old, $new) = (key_fingerprint($OLD), key_fingerprint($NEW)); + my $info = { + fsid => 'e3bcf831-39e8-4db3-a458-bbfa643a139d', + exported => { map { $_ => { key => $NEW } } qw(mds.a osd.2) }, + }; + my $mds = { type => 'mds', id => 'a', entity => 'mds.a' }; + my $osd = { type => 'osd', id => '2', entity => 'osd.2' }; + my $label = parse_probe_output("label osd:2 $LABEL\n")->{'osd:2'}; + + is_deeply( + disk_key_verdict($info, $mds, { store => 'file', 'keyring-fingerprint' => $old }), + { verdict => 'stale', copies => ['keyring'] }, + 'a keyring holding another key than the monitors is stale', + ); + is_deeply( + disk_key_verdict($info, $osd, { %$label, 'keyring-fingerprint' => $new }), + { verdict => 'match' }, + 'an OSD whose label and keyring hold the active key matches', + ); + is_deeply( + disk_key_verdict($info, $osd, { %$label, 'keyring-fingerprint' => $old }), + { verdict => 'stale', copies => ['keyring'] }, + 'a restart reads the keyring, so it counts apart from the label', + ); + is_deeply( + disk_key_verdict($info, $osd, { %$label, 'label-whoami' => '9' }), + { verdict => 'unknown', why => 'identity' }, + 'a label left behind by another OSD says nothing about this one', + ); + is_deeply( + disk_key_verdict($info, $mds, { store => 'missing' }), + { verdict => 'unknown', why => 'missing' }, + 'a daemon without a key on disk cannot be judged', + ); + is_deeply( + disk_key_verdict({ %$info, exported => {} }, $mds, { store => 'file' }), + { verdict => 'match' }, + 'without an auth entry there is nothing to differ from', + ); +} +{ + ok(daemon_in_scope(undef, 'mds', 'mds.a'), 'without --only every daemon is in scope'); + ok(daemon_in_scope({ 'mds.a' => 1 }, 'mds', 'mds.a'), 'a name selects its daemon'); + ok(!daemon_in_scope({ osd => 1 }, 'mds', 'mds.a'), 'and others are out of scope'); + + # the plan rewrites the keys of a resumed daemon anyway, so its copies are not compared + my $info = { + daemons => { + mgr => [{ type => 'mgr', id => 'x', entity => 'mgr.x' }], + mds => [{ type => 'mds', id => 'a', entity => 'mds.a', recovered => 1 }], + osd => [{ type => 'osd', id => '2', entity => 'osd.2' }], + }, + }; + is_deeply( + [map { $_->{entity} } disk_key_candidates($info, { rotated => { 'osd.2' => 1 } })], + ['mgr.x'], + 'a daemon resumed from the saved plan or with an unfinished rotation is not compared', + ); +} + # --- which keys still need work ------------------------------------------------------------- { my $info = { exported => { 'osd.1' => { key => $OLD }, 'osd.2' => { key => $NEW } } }; -- 2.47.3