[PATCH datacenter-manager 09/17] server: connection: report mismatching fingerprint as untrusted on probe

[Shannon Sterz <s.sterz@proxmox.com>]

instead of erroring out. previously this function returned a
connection error if the provided fingerprint did not match the
remote's fingerprint. instead, report the certificate as untrusted,
giving client's more appropriate information in such cases.

note that while the documentation for this function was technically
correct, the probe_tls endpoints for pve and pbs remotes stated:

> If the certificate is not trusted with the given parameters, returns
> the certificate information.

however, that was incorrect, since the endpoints returned an error if
the fingerprint did not match. since those two endpoints are currently
the only users that could actually provide a fingerprint (all other
callers explicitly provide `None`), this is more of a bug fix than a
public api break.

Signed-off-by: Shannon Sterz <s.sterz@proxmox.com>
---
 server/src/connection.rs | 62 ++++++++++++++++++++++++++++------------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/server/src/connection.rs b/server/src/connection.rs
index 7ad1a5b9..f8b277da 100644
--- a/server/src/connection.rs
+++ b/server/src/connection.rs
@@ -15,12 +15,14 @@ use std::time::{Duration, SystemTime};
 use anyhow::{Error, bail, format_err};
 use http::Method;
 use http::uri::Authority;
+use openssl::hash::MessageDigest;
 use openssl::x509::X509StoreContextRef;
 use serde::Serialize;
 
 use proxmox_acme_api::CertificateInfo;
 use proxmox_client::{Client, HttpApiClient, HttpApiResponse, HttpApiResponseStream, TlsOptions};
 
+use pdm_api_types::Fingerprint;
 use pdm_api_types::remotes::{NodeUrl, Remote, RemoteType, TlsProbeOutcome};
 use pve_api_types::client::PveClientImpl;
 
@@ -867,7 +869,7 @@ impl HttpApiClient for MultiClient {
 /// Checks TLS connection to the given remote
 ///
 /// Returns `Ok(TlsProbeOutcome::TrustedCertificate)` if connecting with the given parameters works
-/// Returns `Ok(TlsProbeOutcome::UntrustedCertificate)` if no fingerprint was given and some certificate could not be validated
+/// Returns `Ok(TlsProbeOutcome::UntrustedCertificate)` if the provided fingerprint does not match or a certificate could not be validated
 /// Returns `Err(err)` if some other error occurred
 ///
 /// # Example
@@ -902,25 +904,47 @@ pub async fn probe_tls_connection(
     // to save the invalid cert we find
     let invalid_cert = Arc::new(StdMutex::new(None));
 
-    let options = if let Some(fp) = &fingerprint {
-        TlsOptions::parse_fingerprint(fp)?
-    } else {
-        TlsOptions::Callback(Box::new({
-            let invalid_cert = invalid_cert.clone();
-            move |valid: bool, chain: &mut X509StoreContextRef| {
-                if let Some(cert) = chain.current_cert() {
-                    if !valid {
-                        let cert = cert
-                            .to_pem()
-                            .map_err(Error::from)
-                            .and_then(|pem| CertificateInfo::from_pem("", &pem));
-                        *invalid_cert.lock().unwrap() = Some(cert);
-                    }
-                }
-                true
+    let fingerprint = fingerprint
+        .map(|fp| fp.parse::<Fingerprint>())
+        .transpose()?;
+
+    let options = TlsOptions::Callback(Box::new({
+        let invalid_cert = invalid_cert.clone();
+        move |valid: bool, chain: &mut X509StoreContextRef| {
+            // If no fingerprint was provided and the trust store trusts the certificate, the
+            // connection is valid.
+            if fingerprint.is_none() && valid {
+                return true;
             }
-        }))
-    };
+
+            let Some(cert) = chain.current_cert() else {
+                return true;
+            };
+
+            // If a fingerprint was provided and the certificate matches it, the connection is
+            // valid.
+            if let Some(provided_fp) = &fingerprint {
+                if cert
+                    .digest(MessageDigest::sha256())
+                    .map(|fp| *fp == **provided_fp)
+                    .unwrap_or(false)
+                {
+                    return true;
+                }
+            }
+
+            // Otherwise, the certificate is not trusted.
+            let cert = cert
+                .to_pem()
+                .map_err(Error::from)
+                .and_then(|pem| CertificateInfo::from_pem("", &pem));
+
+            *invalid_cert.lock().unwrap() = Some(cert);
+
+            true
+        }
+    }));
+
     let client = proxmox_client::Client::with_options(uri, options, Default::default())?;
 
     // set fake auth info. we don't need any, but the proxmox client will return unauthenticated if
-- 
2.47.3