[PATCH manager v2 4/4] migrations: cephx: name the hosts keeping key copies we cannot write

[Kefu Chai <k.chai@proxmox.com>]

A host outside this cluster that runs a Ceph daemon usually keeps its
own copy of the 'client.admin' keyring. We do not write that copy, so a
key rotation leaves it stale.

Name these hosts when the key is staged and when it is committed, and
print commands to refresh the copy:

- While staged, fetch the new key, usually as 'client.admin'. A user
  with only 'profile rbd' caps cannot read its own key. Take the key
  from the JSON output and write it with ceph-authtool, because a
  keyring saved by 'auth get' has a 'pending key' line that Ceph cannot
  read.
- After the commit, fetch it with the monitor's own keyring, if the
  host runs a monitor. Otherwise copy it over from one of our nodes.

The commands name their keyring files, because a ceph.conf copied from
here points to /etc/pve.

Signed-off-by: Kefu Chai <k.chai@proxmox.com>
---
 bin/pve-cephx-rotate-service-keys   | 53 ++++++++++++++++-
 test/CephKeyMigrationScript_test.pl | 89 ++++++++++++++++++++++++++++-
 2 files changed, 140 insertions(+), 2 deletions(-)

diff --git a/bin/pve-cephx-rotate-service-keys b/bin/pve-cephx-rotate-service-keys
index c7565e9e..ea953f76 100755
--- a/bin/pve-cephx-rotate-service-keys
+++ b/bin/pve-cephx-rotate-service-keys
@@ -197,6 +197,51 @@ my sub unmanaged_monitors($info) {
     return grep { !$_->{managed} } $info->{daemons}->{mon}->@*;
 }
 
+# hosts that run Ceph daemons outside this cluster. '$DAEMON_TYPES' has no monitors, so add them.
+my sub unmanaged_hosts_of($info) {
+    my $hosts = {};
+    for my $type ('mon', @$DAEMON_TYPES) {
+        $hosts->{ $_->{node} } = 1
+            for grep { !$_->{managed} && defined($_->{node}) } $info->{daemons}->{$type}->@*;
+    }
+    return sort keys %$hosts;
+}
+
+# Such a host keeps its own key copies, usually of 'client.admin'. While the key is staged, its
+# admin can fetch the new key with the old one, as a user allowed to read keys. The key comes from
+# the JSON output, as Ceph cannot read the 'pending key' line that 'auth get' writes. After the
+# commit, a monitor host can use its 'mon.' keyring instead.
+my sub unmanaged_copy_note($info, $committed = 0) {
+    my @hosts = unmanaged_hosts_of($info);
+    return if !@hosts;
+
+    my $where = join(', ', map { "'$_'" } @hosts);
+    if (!$committed) {
+        log_step("Proxmox VE rewrites the key copies it manages. These hosts run Ceph daemons"
+            . " without being nodes of this cluster, so a keyring they keep for their own use is"
+            . " not among them: $where. Their admin can refresh one while both keys still work,"
+            . " as a user allowed to read keys, usually 'client.admin'. The commands name both"
+            . " keyrings, as a configuration copied from here looks for keys under '/etc/pve',"
+            . " which such a host does not have:");
+        log_step("  KEY=\$(ceph -n client.admin -k <admin keyring file> auth get USER -f json"
+            . " | python3 -c 'import json, sys; print(json.load(sys.stdin)[0][\"pending_key\"])')"
+        );
+        log_step("  ceph-authtool <keyring file> -n USER --add-key \"\$KEY\"");
+        return;
+    }
+
+    log_warn("The previous key no longer works. A keyring these hosts keep for their own use was"
+        . " not rewritten, as they are not nodes of this cluster: $where. On a monitor host, and"
+        . " while its keyring still holds the current 'mon.' key, its admin can refresh one with"
+        . " the monitor's own credential:");
+    log_step("  ceph --name mon. --keyring "
+        . mon_keyring_path('<id>')
+        . " auth get USER -o <keyring file>");
+    log_step("Otherwise fetch the key on a node of this cluster and copy the file over.");
+
+    return;
+}
+
 my sub assert_located($node, $entity) {
     die "Cannot manage '$entity': no valid host name was reported\n"
         if !defined($node) || ref($node) || !length($node);
@@ -2029,6 +2074,7 @@ my sub preflight_cluster(
     };
 
     my $confirmation_refused = 0;
+    my $committed_copies = 0;
     my $confirm_all = $opts->{'confirm-all-clients-refreshed'};
     my $confirmation_seen = {};
     my @confirm =
@@ -2256,7 +2302,7 @@ my sub preflight_cluster(
             }
 
             # The record remains open if the commit fails, so the confirmation is retryable.
-            commit_staged_key($info->{rados}, $state, $entity) if $staged;
+            $committed_copies = 1 if $staged && commit_staged_key($info->{rados}, $state, $entity);
             my $mark = $state->{client_refresh}->{$entity};
             $mark->{cleared} = time();
             $mark->{acknowledged} = time();
@@ -2265,6 +2311,9 @@ my sub preflight_cluster(
         }
     }
 
+    # copies on hosts we do not manage are stale from here on
+    unmanaged_copy_note($info, 1) if $committed_copies;
+
     # Confirmation can promote staged keys. Recollect before any remaining preflight decision and
     # update the caller's shared cluster picture before it builds the migration plan.
     if (!$confirmation_refused && scalar(@confirm)) {
@@ -3045,6 +3094,7 @@ my sub print_plan($info, $plan, $state, $opts, $storage_entities) {
             log_step("For each staged Ceph user key, both the current and new keys authenticate"
                 . " until the new key is committed with '--confirm-clients-refreshed USER' or,"
                 . " once every open record is ready, '--confirm-all-clients-refreshed'.");
+            unmanaged_copy_note($info);
             log_step("the monitors stop automatic pending-key promotion until every staged key is"
                     . " committed or aborted")
                 if $opts->{verbose};
@@ -6344,6 +6394,7 @@ sub key_migration_test_hooks {
         check_client_kernels => \&check_client_kernels,
         manual_promotion_with_retries => \&manual_promotion_with_retries,
         mon_admin_run => \&mon_admin_run,
+        unmanaged_copy_note => \&unmanaged_copy_note,
         describe_live => \&describe_live,
     };
 }
diff --git a/test/CephKeyMigrationScript_test.pl b/test/CephKeyMigrationScript_test.pl
index 5d251596..5c5a9bf8 100755
--- a/test/CephKeyMigrationScript_test.pl
+++ b/test/CephKeyMigrationScript_test.pl
@@ -3619,7 +3619,15 @@ sub run_aggregate_confirmation {
             mon => [],
             mgr => [],
             mds => [],
-            osd => [{ type => 'osd', id => 7, entity => 'osd.7', node => 'node-daemon' }],
+            osd => [
+                {
+                    type => 'osd',
+                    id => 7,
+                    entity => 'osd.7',
+                    node => 'node-daemon',
+                    managed => 1,
+                },
+            ],
         },
     };
     my $plan = {
@@ -3717,6 +3725,41 @@ sub run_aggregate_confirmation {
         qr/For each staged Ceph user key, both the current and new keys authenticate.*committed with '--confirm-clients-refreshed USER' or, once every open record is ready, '--confirm-all-clients-refreshed'/s,
         'the default names both staged completion paths without offering the aggregate early',
     );
+    unlike(
+        $concise,
+        qr/without being nodes of this cluster/,
+        'a cluster that manages every daemon host is told nothing about copies elsewhere',
+    );
+
+    # while the key is staged, a host outside this cluster can fetch it with the old key
+    {
+        my $stretched = { %$info, daemons => { %{ $info->{daemons} } } };
+        $stretched->{daemons}->{mon} = [
+            { type => 'mon', id => 'a', node => 'node-a', managed => 1 },
+            { type => 'mon', id => 'tiebreaker', node => 'witness.example', managed => 0 },
+        ];
+        my $output = $render->(0, undef, undef, undef, $stretched);
+        like(
+            $output,
+            qr/without being nodes of this cluster.*'witness\.example'.*while both keys still work/s,
+            'the host is named while the old key still authenticates',
+        );
+        like(
+            $output,
+            qr/KEY=\$\(ceph -n client\.admin -k <admin keyring file> auth get USER -f json \|.*"pending_key".*\n.*ceph-authtool <keyring file> -n USER --add-key "\$KEY"/,
+            'as a user allowed to read keys, writing the staged key itself, naming both keyrings',
+        );
+        unlike(
+            $output,
+            qr/'node-a'/,
+            'and a managed node is not, as Proxmox VE rewrites its copies',
+        );
+        unlike(
+            $output,
+            qr/--name mon\./,
+            'the monitor credential is not offered while the simpler one still works',
+        );
+    }
     like(
         $concise,
         qr/For staged keys.*live-migrate affected\s+VMs, including those with kernel RBD disks/s,
@@ -7902,6 +7945,50 @@ for my $case ([0, 0], [1, 0], [0, 1], [1, 1]) {
     );
 }
 
+# After the commit the old copy no longer works, so the note names another credential.
+{
+    my $stretched = {
+        daemons => {
+            mon => [
+                { type => 'mon', id => 'a', node => 'node-a', managed => 1 },
+                { type => 'mon', id => 'witness', node => 'witness.example', managed => 0 },
+            ],
+            mgr => [],
+            mds => [],
+            osd => [],
+        },
+    };
+    my $note = sub {
+        my ($committed) = @_;
+        my $out = '';
+        open(my $stdout, '>', \$out) or die $!;
+        local *STDOUT = $stdout;
+        $HOOKS->{unmanaged_copy_note}->($stretched, $committed);
+        return $out;
+    };
+
+    my $committed = $note->(1);
+    like($committed, qr/WARN.*previous key no longer works/, 'the note says the old key is gone');
+    like(
+        $committed,
+        qr{ceph --name mon\. --keyring /var/lib/ceph/mon/\S+-<id>/keyring auth get USER},
+        "and gives the monitor's own credential as the way back",
+    );
+    like($committed, qr/copy the file over/, 'with the fallback for a host running no monitor');
+    like($committed, qr/'witness\.example'/, 'naming the unmanaged host');
+    unlike($committed, qr/'node-a'/, 'and not the one Proxmox VE rewrites itself');
+    like($note->(0), qr/while both keys still work/, 'before the commit it is the simpler route');
+
+    my $managed = { daemons => { map { $_ => [] } qw(mon mgr mds osd) } };
+    my $out = '';
+    {
+        open(my $stdout, '>', \$out) or die $!;
+        local *STDOUT = $stdout;
+        $HOOKS->{unmanaged_copy_note}->($managed, 1);
+    }
+    is($out, '', 'a cluster that manages every daemon host hears nothing about copies elsewhere');
+}
+
 # A monitor on one of our nodes is asked over its admin socket, any other one with 'ceph tell'.
 {
     no warnings qw(once redefine);
-- 
2.47.3