[PATCH proxmox-backup 04/10] config/server/api: add certificate renewal logic including notifications

[Shannon Sterz <s.sterz@proxmox.com>]

the daily-update service is used to check whether a self-signed
certificate is in use and renews it if it would expire within the next
15 days. it will also send out reminder notifications starting 15 days
before a certificate could be renewed.

a self-signed certificate is detected by parsing the issuer field of
the certificate. if it aligns with how self-signed certificates are
created by this instance of proxmox backup server, it will be deemed
self-signed.

15 days was chosen as that should give a reasonable trade-off between
not failing if the daily-update service can't run on the specific day
that the certificate would expire and not refreshing the certificate
too often. note that 15 days before expiry corresponds to let's
encrypt's recommendation for when to update new certificates that are
valid for 45 days:

> Acceptable behavior includes renewing certificates at approximately
> two thirds of the way through the current certificate’s lifetime.
>
> -  https://letsencrypt.org/2025/12/02/from-90-to-45#action-required

15 days before expiry is about two thirds of the lifetime of a
certificate that lasts for 45 days.

Signed-off-by: Shannon Sterz <s.sterz@proxmox.com>
---

Notes:
    currently this will send out a notification every time the service is
    run in the 15 days between a renewal being imminent and not being
    renewed yet. this is probably excessive and we should limit the
    notifications here to only go out once per week (or similar).
    
    i chose 15 days instead of two thirds of ten years, because for a long
    lasting certificate like that, trying to refresh it starting from 3.3
    years before it expires seems excessive to me.

 debian/proxmox-backup-server.install          |  4 ++
 src/api2/node/certificates.rs                 | 44 +++++++++++++
 src/bin/proxmox-daily-update.rs               | 28 +++++++++
 src/config/mod.rs                             |  4 +-
 src/server/notifications/mod.rs               | 50 +++++++++++++++
 templates/Makefile                            | 62 ++++++++++---------
 templates/default/cert-refresh-body.txt.hbs   |  8 +++
 .../default/cert-refresh-subject.txt.hbs      |  1 +
 .../cert-upcoming-refresh-body.txt.hbs        |  9 +++
 .../cert-upcoming-refresh-subject.txt.hbs     |  1 +
 10 files changed, 181 insertions(+), 30 deletions(-)
 create mode 100644 templates/default/cert-refresh-body.txt.hbs
 create mode 100644 templates/default/cert-refresh-subject.txt.hbs
 create mode 100644 templates/default/cert-upcoming-refresh-body.txt.hbs
 create mode 100644 templates/default/cert-upcoming-refresh-subject.txt.hbs

diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install
index 254a4b4b..674c9bf0 100644
--- a/debian/proxmox-backup-server.install
+++ b/debian/proxmox-backup-server.install
@@ -45,6 +45,10 @@ usr/share/man/man5/user.cfg.5
 usr/share/man/man5/verification.cfg.5
 usr/share/proxmox-backup/templates/default/acme-err-body.txt.hbs
 usr/share/proxmox-backup/templates/default/acme-err-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-refresh-body.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-refresh-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-upcoming-refresh-body.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-upcoming-refresh-subject.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-err-body.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-err-subject.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-ok-body.txt.hbs
diff --git a/src/api2/node/certificates.rs b/src/api2/node/certificates.rs
index a69f6511..5812c4fa 100644
--- a/src/api2/node/certificates.rs
+++ b/src/api2/node/certificates.rs
@@ -1,4 +1,6 @@
 use anyhow::{bail, format_err, Error};
+use const_format::concatcp;
+use openssl::asn1::Asn1Time;
 use openssl::pkey::PKey;
 use openssl::x509::X509;
 use serde::{Deserialize, Serialize};
@@ -17,6 +19,14 @@ use pbs_tools::cert;
 
 use crate::server::send_certificate_renewal_mail;
 
+proxmox_schema::const_regex! {
+    pub SELF_SIGNED_REGEX = concatcp!(
+        r#"^O\s?=\s?"#,
+        crate::config::PRODUCT_NAME,
+        r#", OU\s?=\s?[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}, CN\s?=\s?(?<fqdn>.*)$"#
+    );
+}
+
 pub const ROUTER: Router = Router::new()
     .get(&list_subdirs_api_method!(SUBDIRS))
     .subdirs(SUBDIRS);
@@ -319,6 +329,40 @@ pub fn cert_expires_soon() -> Result<bool, Error> {
         .map_err(|err| format_err!("Failed to check certificate expiration date: {}", err))
 }
 
+/// Check whether the current certificate is self-signed and returns the remaining days the
+/// certificate is valid for.
+pub fn self_signed_cert_expires_in() -> Result<Option<i32>, Error> {
+    let cert = pem_to_cert_info(get_certificate_pem()?.as_bytes())?;
+    let cert_issuer = cert.issuer_name()?;
+
+    let Some(captures) = (SELF_SIGNED_REGEX.regex_obj)().captures(&cert_issuer) else {
+        return Ok(None);
+    };
+
+    let mut fqdn = proxmox_sys::nodename().to_owned();
+    let resolv_conf = crate::api2::node::dns::read_etc_resolv_conf()?;
+
+    if let Some(domain) = resolv_conf["search"].as_str() {
+        fqdn.push('.');
+        fqdn.push_str(domain);
+    }
+
+    if captures["fqdn"] != fqdn {
+        return Ok(None);
+    }
+
+    let now = Asn1Time::from_unix(proxmox_time::epoch_i64())?;
+    let diff = now.diff(cert.not_after())?;
+    Ok(Some(diff.days))
+}
+
+/// Renews the self signed certificate. The caller needs to make sure the current certificate is
+/// really a self-signed certificate and not an ACME or custom certificate.
+pub async fn renew_self_signed_cert() -> Result<(), Error> {
+    crate::config::update_self_signed_cert(true)?;
+    crate::server::reload_proxy_certificate().await
+}
+
 fn spawn_certificate_worker(
     name: &'static str,
     force: bool,
diff --git a/src/bin/proxmox-daily-update.rs b/src/bin/proxmox-daily-update.rs
index c4d68e30..49159d24 100644
--- a/src/bin/proxmox-daily-update.rs
+++ b/src/bin/proxmox-daily-update.rs
@@ -1,6 +1,9 @@
 use anyhow::Error;
 use serde_json::json;
 
+use proxmox_backup::server::notifications::{
+    send_self_signed_renewal_notification, send_upcoming_self_signed_renewal_notification,
+};
 use proxmox_notify::context::pbs::PBS_CONTEXT;
 use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
 use proxmox_subscription::SubscriptionStatus;
@@ -61,6 +64,10 @@ async fn do_update(rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> {
         log::error!("error checking certificates: {err}");
     }
 
+    if let Err(err) = renew_self_signed_certificate().await {
+        log::error!("error checking self-signed certificate renewal: {err:#}");
+    }
+
     // TODO: cleanup tasks like in PVE?
 
     Ok(())
@@ -89,6 +96,27 @@ async fn check_acme_certificates(rpcenv: &mut dyn RpcEnvironment) -> Result<(),
     Ok(())
 }
 
+async fn renew_self_signed_certificate() -> Result<(), Error> {
+    let days = match api2::node::certificates::self_signed_cert_expires_in()? {
+        None => {
+            log::debug!("Certificate is not self-signed, nothing to do.");
+            return Ok(());
+        }
+        Some(days) => days,
+    };
+
+    if days <= 15 {
+        log::info!("Certificate expires within 15 days, renewing certificate...");
+        let err = &api2::node::certificates::renew_self_signed_cert().await;
+        send_self_signed_renewal_notification(&err)?;
+    } else if days <= 30 {
+        log::info!("Certificate expires within 30 days, notify about renewal.");
+        send_upcoming_self_signed_renewal_notification()?;
+    }
+
+    Ok(())
+}
+
 async fn run(rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> {
     let backup_user = pbs_config::backup_user()?;
     let file_opts = CreateOptions::new()
diff --git a/src/config/mod.rs b/src/config/mod.rs
index 56907ae9..4712fb1d 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -14,6 +14,8 @@ use pbs_buildcfg::{self, configdir};
 
 pub mod tfa;
 
+pub(crate) const PRODUCT_NAME: &str = "Proxmox Backup Server";
+
 /// Check configuration directory permissions
 ///
 /// For security reasons, we want to make sure they are set correctly:
@@ -89,7 +91,7 @@ pub fn update_self_signed_cert(force: bool) -> Result<(), Error> {
     let resolv_conf = crate::api2::node::dns::read_etc_resolv_conf()?;
 
     let (priv_key, cert) = proxmox_acme_api::create_self_signed_cert(
-        "Proxmox Backup Server",
+        PRODUCT_NAME,
         proxmox_sys::nodename(),
         resolv_conf["search"].as_str(),
         None,
diff --git a/src/server/notifications/mod.rs b/src/server/notifications/mod.rs
index 95ff9ef1..2cbe3aee 100644
--- a/src/server/notifications/mod.rs
+++ b/src/server/notifications/mod.rs
@@ -575,6 +575,56 @@ pub fn send_certificate_renewal_mail(result: &Result<(), Error>) -> Result<(), E
     Ok(())
 }
 
+/// send email for upcoming self signed renewal.
+pub fn send_upcoming_self_signed_renewal_notification() -> Result<(), Error> {
+    let metadata = HashMap::from([
+        ("hostname".into(), proxmox_sys::nodename().into()),
+        ("type".into(), "cert".into()),
+    ]);
+
+    let notification = Notification::from_template(
+        Severity::Info,
+        "cert-upcoming-refresh",
+        serde_json::to_value(CommonData::new())?,
+        metadata,
+    );
+
+    send_notification(notification)?;
+    Ok(())
+}
+
+/// send email renewed self-signed certificate
+pub fn send_self_signed_renewal_notification(result: &Result<(), Error>) -> Result<(), Error> {
+    let metadata = HashMap::from([
+        ("hostname".into(), proxmox_sys::nodename().into()),
+        ("type".into(), "acme".into()),
+    ]);
+
+    let notification = match result {
+        Err(e) => {
+            let template_data = AcmeErrTemplateData {
+                common: CommonData::new(),
+                error: format!("{e:#}"),
+            };
+
+            Notification::from_template(
+                Severity::Info,
+                "acme-err",
+                serde_json::to_value(template_data)?,
+                metadata,
+            )
+        }
+        _ => Notification::from_template(
+            Severity::Info,
+            "cert-refresh",
+            serde_json::to_value(CommonData::new())?,
+            metadata,
+        ),
+    };
+
+    send_notification(notification)
+}
+
 /// Lookup users email address
 pub fn lookup_user_email(userid: &Userid) -> Option<String> {
     if let Ok(user_config) = pbs_config::user::cached_config() {
diff --git a/templates/Makefile b/templates/Makefile
index 0539902e..45929f72 100644
--- a/templates/Makefile
+++ b/templates/Makefile
@@ -1,34 +1,38 @@
 include ../defines.mk
 
-NOTIFICATION_TEMPLATES=						\
-	default/acme-err-body.txt.hbs			\
-	default/acme-err-subject.txt.hbs		\
-	default/gc-err-body.txt.hbs				\
-	default/gc-ok-body.txt.hbs				\
-	default/gc-err-subject.txt.hbs			\
-	default/gc-ok-subject.txt.hbs			\
-	default/package-updates-body.txt.hbs	\
-	default/package-updates-subject.txt.hbs	\
-	default/prune-err-body.txt.hbs			\
-	default/prune-ok-body.txt.hbs			\
-	default/prune-err-subject.txt.hbs		\
-	default/prune-ok-subject.txt.hbs		\
-	default/sync-err-body.txt.hbs			\
-	default/sync-ok-body.txt.hbs			\
-	default/sync-err-subject.txt.hbs		\
-	default/sync-ok-subject.txt.hbs			\
-	default/tape-backup-err-body.txt.hbs	\
-	default/tape-backup-err-subject.txt.hbs	\
-	default/tape-backup-ok-body.txt.hbs		\
-	default/tape-backup-ok-subject.txt.hbs	\
-	default/tape-load-body.txt.hbs			\
-	default/tape-load-subject.txt.hbs		\
-	default/test-body.txt.hbs				\
-	default/test-subject.txt.hbs			\
-	default/verify-err-body.txt.hbs			\
-	default/verify-ok-body.txt.hbs			\
-	default/verify-err-subject.txt.hbs		\
-	default/verify-ok-subject.txt.hbs		\
+NOTIFICATION_TEMPLATES=								\
+	default/acme-err-body.txt.hbs					\
+	default/acme-err-subject.txt.hbs				\
+	default/cert-refresh-body.txt.hbs				\
+	default/cert-refresh-subject.txt.hbs			\
+	default/cert-upcoming-refresh-body.txt.hbs		\
+	default/cert-upcoming-refresh-subject.txt.hbs	\
+	default/gc-err-body.txt.hbs						\
+	default/gc-ok-body.txt.hbs						\
+	default/gc-err-subject.txt.hbs					\
+	default/gc-ok-subject.txt.hbs					\
+	default/package-updates-body.txt.hbs			\
+	default/package-updates-subject.txt.hbs			\
+	default/prune-err-body.txt.hbs					\
+	default/prune-ok-body.txt.hbs					\
+	default/prune-err-subject.txt.hbs				\
+	default/prune-ok-subject.txt.hbs				\
+	default/sync-err-body.txt.hbs					\
+	default/sync-ok-body.txt.hbs					\
+	default/sync-err-subject.txt.hbs				\
+	default/sync-ok-subject.txt.hbs					\
+	default/tape-backup-err-body.txt.hbs			\
+	default/tape-backup-err-subject.txt.hbs			\
+	default/tape-backup-ok-body.txt.hbs				\
+	default/tape-backup-ok-subject.txt.hbs			\
+	default/tape-load-body.txt.hbs					\
+	default/tape-load-subject.txt.hbs				\
+	default/test-body.txt.hbs						\
+	default/test-subject.txt.hbs					\
+	default/verify-err-body.txt.hbs					\
+	default/verify-ok-body.txt.hbs					\
+	default/verify-err-subject.txt.hbs				\
+	default/verify-ok-subject.txt.hbs				\
 
 all:
 
diff --git a/templates/default/cert-refresh-body.txt.hbs b/templates/default/cert-refresh-body.txt.hbs
new file mode 100644
index 00000000..608ba30c
--- /dev/null
+++ b/templates/default/cert-refresh-body.txt.hbs
@@ -0,0 +1,8 @@
+Proxmox Backup Server has refreshed its self-signed TLS certificate.
+
+The new certificate is now active. Please update any clients relying on the
+certificate fingerprint to verify their connection to the server.
+
+Please visit the web interface for further details:
+
+<{{base-url}}/#pbsCertificateConfiguration>
diff --git a/templates/default/cert-refresh-subject.txt.hbs b/templates/default/cert-refresh-subject.txt.hbs
new file mode 100644
index 00000000..e57f5cd4
--- /dev/null
+++ b/templates/default/cert-refresh-subject.txt.hbs
@@ -0,0 +1 @@
+Self-Signed Certificate Has Been Refreshed
diff --git a/templates/default/cert-upcoming-refresh-body.txt.hbs b/templates/default/cert-upcoming-refresh-body.txt.hbs
new file mode 100644
index 00000000..8d199c9f
--- /dev/null
+++ b/templates/default/cert-upcoming-refresh-body.txt.hbs
@@ -0,0 +1,9 @@
+Proxmox Backup Server will refresh its TLS certificate within the next 30 days.
+
+If you rely on the certificate's fingerprint to verify TLS sessions between the
+server and a client, please update the fingerprint once the certificate was
+updated. Otherwise, no action is required.
+
+Please visit the web interface for further details:
+
+<{{base-url}}/#pbsCertificateConfiguration>
diff --git a/templates/default/cert-upcoming-refresh-subject.txt.hbs b/templates/default/cert-upcoming-refresh-subject.txt.hbs
new file mode 100644
index 00000000..f188e391
--- /dev/null
+++ b/templates/default/cert-upcoming-refresh-subject.txt.hbs
@@ -0,0 +1 @@
+Self-signed Certificate is About to be Refreshed
-- 
2.47.3