[PATCH proxmox v4 1/8] http: factor out openssl verification callback

[Dominik Csapak <d.csapak@proxmox.com>]

with the 'tls' feature offers a callback method that can be used within
openssl's `set_verify_callback` with a given expected fingerprint.

The logic is inspired by our perl and proxmox-websocket-tunnel
verification logic:

Use openssl's verification if no fingerprint is pinned. If a fingerprint
is given, ignore openssl's verification and check if the leafs
certificate is a match.

Introduce a Fingerprint struct and FingerprintError to be able to
represent fingerprints easily (without strings) and match on the
different error behaviors. This is mostly copied from PDMs CLI client
and can be removed there and reused from here.

This introduces a custom SslVerifyError type for this, since we need to
handle errors differently for different users, e.g. pbs-client wants to
be able to use a fingerprint cache and let the user accept it in
interactive cli sessions.

Co-developed-by: Shannon Sterz <s.sterz@proxmox.com>
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 proxmox-http/Cargo.toml |  11 +++
 proxmox-http/src/lib.rs |   5 +
 proxmox-http/src/tls.rs | 202 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 218 insertions(+)
 create mode 100644 proxmox-http/src/tls.rs

diff --git a/proxmox-http/Cargo.toml b/proxmox-http/Cargo.toml
index 66b11650..cdc2861b 100644
--- a/proxmox-http/Cargo.toml
+++ b/proxmox-http/Cargo.toml
@@ -15,6 +15,7 @@ rust-version.workspace = true
 anyhow.workspace = true
 bytes = { workspace = true, optional = true }
 futures = { workspace = true, optional = true }
+hex = { workspace = true, optional = true }
 http = { workspace = true, optional = true }
 http-body = { workspace = true, optional = true }
 http-body-util = { workspace = true, optional = true }
@@ -22,8 +23,11 @@ hyper = { workspace = true, optional = true }
 hyper-util = { workspace = true, optional = true, features = ["http2"] }
 native-tls = { workspace = true, optional = true }
 openssl =  { version = "0.10", optional = true }
+serde = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }
+serde_plain = { workspace = true, optional = true }
 sync_wrapper = { workspace = true, optional = true }
+thiserror = { workspace = true, optional = true }
 tokio = { workspace = true, features = [], optional = true }
 tokio-openssl = { workspace = true, optional = true }
 tower-service = { workspace = true, optional = true }
@@ -105,3 +109,10 @@ websocket = [
     "tokio?/sync",
     "body",
 ]
+tls = [
+    "dep:openssl",
+    "dep:hex",
+    "dep:thiserror",
+    "dep:serde",
+    "dep:serde_plain",
+]
diff --git a/proxmox-http/src/lib.rs b/proxmox-http/src/lib.rs
index ae3301e8..d27df7bc 100644
--- a/proxmox-http/src/lib.rs
+++ b/proxmox-http/src/lib.rs
@@ -39,3 +39,8 @@ pub use rate_limited_stream::{
 mod body;
 #[cfg(feature = "body")]
 pub use body::Body;
+
+#[cfg(feature = "tls")]
+mod tls;
+#[cfg(feature = "tls")]
+pub use tls::*;
diff --git a/proxmox-http/src/tls.rs b/proxmox-http/src/tls.rs
new file mode 100644
index 00000000..5c2b1743
--- /dev/null
+++ b/proxmox-http/src/tls.rs
@@ -0,0 +1,202 @@
+use anyhow::Error;
+use openssl::x509::{X509Ref, X509StoreContextRef, X509VerifyResult};
+
+///
+/// Error type returned when trying to get a fingerprint from openssl.
+///
+#[derive(Debug, thiserror::Error)]
+pub enum FingerprintError {
+    /// Cannot calculate fingerprint from connection
+    #[error("failed to calculate fingerprint - {0}")]
+    CannotCalculate(openssl::error::ErrorStack),
+
+    /// Fingerprint has an invalid length
+    #[error("Unexpected fingerprint length")]
+    InvalidLength,
+}
+
+/// A sha256 fingerprint.
+// NOTE: The difference to ConfigDigest is that this also allows colons between bytes when parsing.
+// Also the API type's description is different.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Fingerprint([u8; 32]);
+serde_plain::derive_deserialize_from_fromstr!(Fingerprint, "valid sha256 fingerprint");
+serde_plain::derive_serialize_from_display!(Fingerprint);
+
+impl From<[u8; 32]> for Fingerprint {
+    #[inline]
+    fn from(fp: [u8; 32]) -> Self {
+        Self(fp)
+    }
+}
+
+impl From<Fingerprint> for [u8; 32] {
+    #[inline]
+    fn from(fp: Fingerprint) -> Self {
+        fp.0
+    }
+}
+
+impl TryFrom<&[u8]> for Fingerprint {
+    type Error = std::array::TryFromSliceError;
+
+    fn try_from(slice: &[u8]) -> Result<Self, Self::Error> {
+        Ok(Self(slice.try_into()?))
+    }
+}
+
+impl TryFrom<&X509Ref> for Fingerprint {
+    type Error = FingerprintError;
+
+    fn try_from(value: &X509Ref) -> Result<Self, Self::Error> {
+        let digest = value
+            .digest(openssl::hash::MessageDigest::sha256())
+            .map_err(FingerprintError::CannotCalculate)?;
+        Fingerprint::try_from(&*digest).map_err(|_| FingerprintError::InvalidLength)
+    }
+}
+
+impl AsRef<[u8]> for Fingerprint {
+    fn as_ref(&self) -> &[u8] {
+        &self.0
+    }
+}
+
+impl AsRef<[u8; 32]> for Fingerprint {
+    fn as_ref(&self) -> &[u8; 32] {
+        &self.0
+    }
+}
+
+impl std::ops::Deref for Fingerprint {
+    type Target = [u8; 32];
+
+    fn deref(&self) -> &[u8; 32] {
+        &self.0
+    }
+}
+
+impl std::ops::DerefMut for Fingerprint {
+    fn deref_mut(&mut self) -> &mut [u8; 32] {
+        &mut self.0
+    }
+}
+
+impl std::fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:02x}", self[0])?;
+        for b in &self[1..] {
+            write!(f, ":{b:02x}")?;
+        }
+        Ok(())
+    }
+}
+
+impl std::str::FromStr for Fingerprint {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Error> {
+        let s = s.replace(':', "");
+        let mut fp = [0u8; 32];
+        hex::decode_to_slice(s, &mut fp)?;
+        Ok(Fingerprint(fp))
+    }
+}
+
+///
+/// Error type returned by failed [`openssl_verify_callback`].
+///
+#[derive(Debug, thiserror::Error)]
+pub enum SslVerifyError {
+    /// Occurs if no certificate is found in the current part of the chain. Should never happen!
+    #[error("SSL context lacks current certificate")]
+    NoCertificate,
+
+    /// Cannot get the fingerprint from openssl.
+    #[error("error getting the fingerprint from openssl - {0}")]
+    InvalidFingerprint(FingerprintError),
+
+    /// Fingerprint match error
+    #[error("found fingerprint ({fingerprint}) does not match expected fingerprint ({expected})")]
+    FingerprintMismatch {
+        fingerprint: Fingerprint,
+        expected: Fingerprint,
+    },
+
+    /// Untrusted certificate with fingerprint information
+    #[error("certificate validation failed")]
+    UntrustedCertificate { fingerprint: Fingerprint },
+}
+
+// Useful for testing. Can't hide it behind `#[cfg(test)]` for integration tests, though.
+impl PartialEq for SslVerifyError {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (SslVerifyError::NoCertificate, SslVerifyError::NoCertificate) => true,
+            (SslVerifyError::InvalidFingerprint(_), SslVerifyError::InvalidFingerprint(_)) => true,
+            (
+                SslVerifyError::FingerprintMismatch {
+                    fingerprint: a_fingerprint,
+                    expected: a_expected,
+                },
+                SslVerifyError::FingerprintMismatch {
+                    fingerprint: b_fingerprint,
+                    expected: b_expected,
+                },
+            ) => a_fingerprint == b_fingerprint && a_expected == b_expected,
+            (
+                SslVerifyError::UntrustedCertificate {
+                    fingerprint: a_fingerprint,
+                },
+                SslVerifyError::UntrustedCertificate {
+                    fingerprint: b_fingerprint,
+                },
+            ) => a_fingerprint == b_fingerprint,
+            _ => false,
+        }
+    }
+}
+
+/// Intended as an openssl verification callback.
+///
+/// The following things are checked:
+///
+/// * If no fingerprint is given, return the openssl verification result
+/// * If a fingerprint is given get the leaf fp and check that against the given
+pub fn openssl_verify_callback(
+    openssl_valid: bool,
+    ctx: &mut X509StoreContextRef,
+    expected_fp: Option<Fingerprint>,
+) -> Result<(), SslVerifyError> {
+    match expected_fp {
+        Some(expected_fp) => {
+            let fingerprint = get_leaf_fp(ctx)?;
+            if expected_fp == fingerprint {
+                ctx.set_error(X509VerifyResult::OK);
+                Ok(())
+            } else {
+                Err(SslVerifyError::FingerprintMismatch {
+                    fingerprint,
+                    expected: expected_fp,
+                })
+            }
+        }
+        None if openssl_valid => Ok(()),
+        None => {
+            let fingerprint = get_leaf_fp(ctx)?;
+            Err(SslVerifyError::UntrustedCertificate { fingerprint })
+        }
+    }
+}
+
+fn get_leaf_cert(ctx: &X509StoreContextRef) -> Option<&X509Ref> {
+    let chain = ctx.chain()?;
+    chain.get(0)
+}
+
+fn get_leaf_fp(ctx: &X509StoreContextRef) -> Result<Fingerprint, SslVerifyError> {
+    let leaf_cert = get_leaf_cert(ctx).ok_or_else(|| SslVerifyError::NoCertificate)?;
+    leaf_cert
+        .try_into()
+        .map_err(SslVerifyError::InvalidFingerprint)
+}
-- 
2.47.3