[PATCH proxmox-backup 06/16] config/server/api: add certificate renewal logic including notifications

[Shannon Sterz <s.sterz@proxmox.com>]

the daily-update service is used to check whether a self-signed
certificate is in use and renews it if it would expire within the next
15 days. it will also send out reminder notifications starting 15 days
before a certificate could be renewed.

a self-signed certificate is detected by parsing the issuer field of
the certificate. if it aligns with how self-signed certificates are
created by this instance of proxmox backup server, it will be deemed
self-signed.

15 days was chosen as that should give a reasonable trade-off between
not failing if the daily-update service can't run on the specific day
that the certificate would expire and not refreshing the certificate
too often. note that 15 days before expiry corresponds to let's
encrypt's recommendation for when to update new certificates that are
valid for 45 days:

> Acceptable behavior includes renewing certificates at approximately
> two thirds of the way through the current certificate’s lifetime.
>
> -  https://letsencrypt.org/2025/12/02/from-90-to-45#action-required

15 days before expiry is about two thirds of the lifetime of a
certificate that lasts for 45 days.

Signed-off-by: Shannon Sterz <s.sterz@proxmox.com>
---

Notes:
    currently this will send out a notification every time the service is
    run in the 15 days between a renewal being imminent and not being
    renewed yet. this is probably excessive and we should limit the
    notifications here to only go out once per week (or similar).
    
    i chose 15 days instead of two thirds of ten years, because for a long
    lasting certificate like that, trying to refresh it starting from 3.3
    years before it expires seems excessive to me.

 debian/proxmox-backup-server.install          |  6 ++
 src/api2/node/certificates.rs                 |  9 ++-
 src/bin/proxmox-daily-update.rs               | 39 ++++++++++-
 src/config/mod.rs                             |  4 +-
 src/server/notifications/mod.rs               | 56 +++++++++++++++
 src/server/notifications/template_data.rs     | 11 +++
 templates/Makefile                            | 68 ++++++++++---------
 templates/default/cert-err-body.txt.hbs       |  7 ++
 templates/default/cert-err-subject.txt.hbs    |  1 +
 templates/default/cert-refresh-body.txt.hbs   |  8 +++
 .../default/cert-refresh-subject.txt.hbs      |  1 +
 .../cert-upcoming-refresh-body.txt.hbs        | 10 +++
 .../cert-upcoming-refresh-subject.txt.hbs     |  1 +
 13 files changed, 187 insertions(+), 34 deletions(-)
 create mode 100644 templates/default/cert-err-body.txt.hbs
 create mode 100644 templates/default/cert-err-subject.txt.hbs
 create mode 100644 templates/default/cert-refresh-body.txt.hbs
 create mode 100644 templates/default/cert-refresh-subject.txt.hbs
 create mode 100644 templates/default/cert-upcoming-refresh-body.txt.hbs
 create mode 100644 templates/default/cert-upcoming-refresh-subject.txt.hbs

diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install
index 1f1d9b601..c485b28a9 100644
--- a/debian/proxmox-backup-server.install
+++ b/debian/proxmox-backup-server.install
@@ -45,6 +45,12 @@ usr/share/man/man5/user.cfg.5
 usr/share/man/man5/verification.cfg.5
 usr/share/proxmox-backup/templates/default/acme-err-body.txt.hbs
 usr/share/proxmox-backup/templates/default/acme-err-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-err-body.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-err-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-refresh-body.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-refresh-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-upcoming-refresh-body.txt.hbs
+usr/share/proxmox-backup/templates/default/cert-upcoming-refresh-subject.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-err-body.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-err-subject.txt.hbs
 usr/share/proxmox-backup/templates/default/gc-ok-body.txt.hbs
diff --git a/src/api2/node/certificates.rs b/src/api2/node/certificates.rs
index 3df05b020..7d19334f5 100644
--- a/src/api2/node/certificates.rs
+++ b/src/api2/node/certificates.rs
@@ -96,7 +96,7 @@ pub struct CertificateInfo {
     pub fingerprint: Option<String>,
 }
 
-fn get_certificate_pem() -> Result<String, Error> {
+pub fn get_certificate_pem() -> Result<String, Error> {
     let cert_path = configdir!("/proxy.pem");
     let cert_pem = proxmox_sys::fs::file_get_contents(cert_path)?;
     String::from_utf8(cert_pem)
@@ -354,6 +354,13 @@ pub fn check_renewal_needed() -> Result<(bool, i64), Error> {
     Ok((expires_soon, lead / SECONDS_PER_DAY))
 }
 
+/// Renews the self signed certificate. The caller needs to make sure the current certificate is
+/// really a self-signed certificate and not an ACME or custom certificate.
+pub async fn renew_self_signed_cert() -> Result<(), Error> {
+    crate::config::update_self_signed_cert(true)?;
+    crate::server::reload_proxy_certificate().await
+}
+
 fn spawn_certificate_worker(
     name: &'static str,
     force: bool,
diff --git a/src/bin/proxmox-daily-update.rs b/src/bin/proxmox-daily-update.rs
index 597722497..e1483488f 100644
--- a/src/bin/proxmox-daily-update.rs
+++ b/src/bin/proxmox-daily-update.rs
@@ -1,12 +1,15 @@
 use anyhow::Error;
 use serde_json::json;
 
+use proxmox_backup::server::notifications::{
+    send_self_signed_renewal_notification, send_upcoming_self_signed_renewal_notification,
+};
 use proxmox_notify::context::pbs::PBS_CONTEXT;
 use proxmox_router::{ApiHandler, RpcEnvironment, cli::*};
 use proxmox_subscription::SubscriptionStatus;
 
 use pbs_buildcfg::configdir;
-use proxmox_backup::api2;
+use proxmox_backup::{api2, config};
 
 async fn wait_for_local_worker(upid_str: &str) -> Result<(), Error> {
     let upid: pbs_api_types::UPID = upid_str.parse()?;
@@ -60,6 +63,10 @@ async fn do_update(rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> {
         log::error!("error checking certificates: {err}");
     }
 
+    if let Err(err) = renew_self_signed_certificate().await {
+        log::error!("error checking self-signed certificate renewal: {err:#}");
+    }
+
     // TODO: cleanup tasks like in PVE?
 
     Ok(())
@@ -89,6 +96,36 @@ async fn check_acme_certificates(rpcenv: &mut dyn RpcEnvironment) -> Result<(),
     Ok(())
 }
 
+async fn renew_self_signed_certificate() -> Result<(), Error> {
+    let resolv_conf = crate::api2::node::dns::read_etc_resolv_conf()?;
+    let pem = api2::node::certificates::get_certificate_pem()?;
+
+    let days = match proxmox_tls_certificates::self_signed_cert_expires_in(
+        config::PRODUCT_NAME,
+        proxmox_sys::nodename(),
+        resolv_conf["search"].as_str(),
+        proxmox_tls_certificates::CertificateInfo::from_pem("proxy.pem", pem.as_bytes())?,
+    )? {
+        None => {
+            log::debug!("Certificate is not self-signed, nothing to do.");
+            return Ok(());
+        }
+        Some(days) => days as i64,
+    };
+
+    if days <= 15 {
+        log::info!("Certificate expires within 15 days, renewing certificate...");
+        let err = &api2::node::certificates::renew_self_signed_cert().await;
+        send_self_signed_renewal_notification(&err)?;
+    } else if days <= 30 {
+        log::info!("Certificate expires within 30 days, notify about renewal.");
+        let earliest_renewal = proxmox_time::epoch_i64() + (days - 15) * 24 * 60 * 60;
+        send_upcoming_self_signed_renewal_notification(earliest_renewal)?;
+    }
+
+    Ok(())
+}
+
 async fn run(rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> {
     proxmox_product_config::init(pbs_config::backup_user()?, pbs_config::priv_user()?);
     proxmox_rest_server::init_worker_tasks(
diff --git a/src/config/mod.rs b/src/config/mod.rs
index bbca5a9af..57a5c7a06 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -15,6 +15,8 @@ use pbs_buildcfg::{self, configdir};
 
 pub mod tfa;
 
+pub const PRODUCT_NAME: &str = "Proxmox Backup Server";
+
 /// Check configuration directory permissions
 ///
 /// For security reasons, we want to make sure they are set correctly:
@@ -90,7 +92,7 @@ pub fn update_self_signed_cert(force: bool) -> Result<(), Error> {
     let resolv_conf = crate::api2::node::dns::read_etc_resolv_conf()?;
 
     let (priv_key, cert) = proxmox_tls_certificates::create_self_signed_cert(
-        "Proxmox Backup Server",
+        PRODUCT_NAME,
         proxmox_sys::nodename(),
         resolv_conf["search"].as_str(),
         None,
diff --git a/src/server/notifications/mod.rs b/src/server/notifications/mod.rs
index 181f12cbc..89ba0a209 100644
--- a/src/server/notifications/mod.rs
+++ b/src/server/notifications/mod.rs
@@ -10,6 +10,7 @@ use proxmox_notify::context::pbs::PBS_CONTEXT;
 use proxmox_schema::ApiType;
 use proxmox_sys::fs::{CreateOptions, create_path};
 
+use crate::server::notifications::template_data::CertUpcomingRefreshTemplateData;
 use crate::tape::TapeNotificationMode;
 use pbs_api_types::{
     APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, NotificationMode,
@@ -575,6 +576,61 @@ pub fn send_certificate_renewal_mail(result: &Result<(), Error>) -> Result<(), E
     Ok(())
 }
 
+/// Send email for upcoming self signed renewal.
+///
+/// * `earliest_renewal`: A Unix timestamp specifying the earliest a certificate may be renewed.
+pub fn send_upcoming_self_signed_renewal_notification(earliest_renewal: i64) -> Result<(), Error> {
+    let metadata = HashMap::from([
+        ("hostname".into(), proxmox_sys::nodename().into()),
+        ("type".into(), "cert".into()),
+    ]);
+
+    let notification = Notification::from_template(
+        Severity::Info,
+        "cert-upcoming-refresh",
+        serde_json::to_value(CertUpcomingRefreshTemplateData {
+            common: CommonData::new(),
+            earliest_renewal,
+        })?,
+        metadata,
+    );
+
+    send_notification(notification)?;
+    Ok(())
+}
+
+/// Send email renewed self-signed certificate
+pub fn send_self_signed_renewal_notification(result: &Result<(), Error>) -> Result<(), Error> {
+    let metadata = HashMap::from([
+        ("hostname".into(), proxmox_sys::nodename().into()),
+        ("type".into(), "cert-renewal".into()),
+    ]);
+
+    let notification = match result {
+        Err(e) => {
+            let template_data = AcmeErrTemplateData {
+                common: CommonData::new(),
+                error: format!("{e:#}"),
+            };
+
+            Notification::from_template(
+                Severity::Error,
+                "cert-err",
+                serde_json::to_value(template_data)?,
+                metadata,
+            )
+        }
+        _ => Notification::from_template(
+            Severity::Notice,
+            "cert-refresh",
+            serde_json::to_value(CommonData::new())?,
+            metadata,
+        ),
+    };
+
+    send_notification(notification)
+}
+
 /// Send notification if datastore values are exceeding the set threshold limit.
 pub fn send_datastore_threshold_exceeded(
     datastore: &str,
diff --git a/src/server/notifications/template_data.rs b/src/server/notifications/template_data.rs
index e10215bdc..a869a2bbf 100644
--- a/src/server/notifications/template_data.rs
+++ b/src/server/notifications/template_data.rs
@@ -145,6 +145,17 @@ pub struct AcmeErrTemplateData {
     pub error: String,
 }
 
+/// Template data for the cert-upcoming-refresh template.
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct CertUpcomingRefreshTemplateData {
+    /// Common properties.
+    #[serde(flatten)]
+    pub common: CommonData,
+    /// A Unix timestamp representing the earliest point in time a certificate may be renewed.
+    pub earliest_renewal: i64,
+}
+
 #[derive(Serialize)]
 #[serde(rename_all = "kebab-case")]
 /// A single package which can be upgraded.
diff --git a/templates/Makefile b/templates/Makefile
index 8a4586d78..f086b927b 100644
--- a/templates/Makefile
+++ b/templates/Makefile
@@ -1,36 +1,42 @@
 include ../defines.mk
 
-NOTIFICATION_TEMPLATES=						\
-	default/acme-err-body.txt.hbs			\
-	default/acme-err-subject.txt.hbs		\
-	default/gc-err-body.txt.hbs				\
-	default/gc-ok-body.txt.hbs				\
-	default/gc-err-subject.txt.hbs			\
-	default/gc-ok-subject.txt.hbs			\
-	default/package-updates-body.txt.hbs	\
-	default/package-updates-subject.txt.hbs	\
-	default/prune-err-body.txt.hbs			\
-	default/prune-ok-body.txt.hbs			\
-	default/prune-err-subject.txt.hbs		\
-	default/prune-ok-subject.txt.hbs		\
-	default/sync-err-body.txt.hbs			\
-	default/sync-ok-body.txt.hbs			\
-	default/sync-err-subject.txt.hbs		\
-	default/sync-ok-subject.txt.hbs			\
-	default/tape-backup-err-body.txt.hbs	\
-	default/tape-backup-err-subject.txt.hbs	\
-	default/tape-backup-ok-body.txt.hbs		\
-	default/tape-backup-ok-subject.txt.hbs	\
-	default/tape-load-body.txt.hbs			\
-	default/tape-load-subject.txt.hbs		\
-	default/test-body.txt.hbs				\
-	default/test-subject.txt.hbs			\
-	default/thresholds-exceeded-body.txt.hbs	\
-	default/thresholds-exceeded-subject.txt.hbs	\
-	default/verify-err-body.txt.hbs			\
-	default/verify-ok-body.txt.hbs			\
-	default/verify-err-subject.txt.hbs		\
-	default/verify-ok-subject.txt.hbs		\
+NOTIFICATION_TEMPLATES=								\
+	default/acme-err-body.txt.hbs					\
+	default/acme-err-subject.txt.hbs				\
+	default/cert-err-body.txt.hbs					\
+	default/cert-err-subject.txt.hbs				\
+	default/cert-refresh-body.txt.hbs				\
+	default/cert-refresh-subject.txt.hbs			\
+	default/cert-upcoming-refresh-body.txt.hbs		\
+	default/cert-upcoming-refresh-subject.txt.hbs	\
+	default/gc-err-body.txt.hbs						\
+	default/gc-ok-body.txt.hbs						\
+	default/gc-err-subject.txt.hbs					\
+	default/gc-ok-subject.txt.hbs					\
+	default/package-updates-body.txt.hbs			\
+	default/package-updates-subject.txt.hbs			\
+	default/prune-err-body.txt.hbs					\
+	default/prune-ok-body.txt.hbs					\
+	default/prune-err-subject.txt.hbs				\
+	default/prune-ok-subject.txt.hbs				\
+	default/sync-err-body.txt.hbs					\
+	default/sync-ok-body.txt.hbs					\
+	default/sync-err-subject.txt.hbs				\
+	default/sync-ok-subject.txt.hbs					\
+	default/tape-backup-err-body.txt.hbs			\
+	default/tape-backup-err-subject.txt.hbs			\
+	default/tape-backup-ok-body.txt.hbs				\
+	default/tape-backup-ok-subject.txt.hbs			\
+	default/tape-load-body.txt.hbs					\
+	default/tape-load-subject.txt.hbs				\
+	default/test-body.txt.hbs						\
+	default/test-subject.txt.hbs					\
+	default/thresholds-exceeded-body.txt.hbs		\
+	default/thresholds-exceeded-subject.txt.hbs		\
+	default/verify-err-body.txt.hbs					\
+	default/verify-ok-body.txt.hbs					\
+	default/verify-err-subject.txt.hbs				\
+	default/verify-ok-subject.txt.hbs				\
 
 all:
 
diff --git a/templates/default/cert-err-body.txt.hbs b/templates/default/cert-err-body.txt.hbs
new file mode 100644
index 000000000..27f444517
--- /dev/null
+++ b/templates/default/cert-err-body.txt.hbs
@@ -0,0 +1,7 @@
+Proxmox Backup Server was not able to renew a self-signed TLS certificate.
+
+Error: {{error}}
+
+Please visit the web interface for further details:
+
+<{{base-url}}/#pbsCertificateConfiguration>
diff --git a/templates/default/cert-err-subject.txt.hbs b/templates/default/cert-err-subject.txt.hbs
new file mode 100644
index 000000000..7b8f2a20e
--- /dev/null
+++ b/templates/default/cert-err-subject.txt.hbs
@@ -0,0 +1 @@
+Could not renew self-signed certificate
diff --git a/templates/default/cert-refresh-body.txt.hbs b/templates/default/cert-refresh-body.txt.hbs
new file mode 100644
index 000000000..608ba30c0
--- /dev/null
+++ b/templates/default/cert-refresh-body.txt.hbs
@@ -0,0 +1,8 @@
+Proxmox Backup Server has refreshed its self-signed TLS certificate.
+
+The new certificate is now active. Please update any clients relying on the
+certificate fingerprint to verify their connection to the server.
+
+Please visit the web interface for further details:
+
+<{{base-url}}/#pbsCertificateConfiguration>
diff --git a/templates/default/cert-refresh-subject.txt.hbs b/templates/default/cert-refresh-subject.txt.hbs
new file mode 100644
index 000000000..e57f5cd4c
--- /dev/null
+++ b/templates/default/cert-refresh-subject.txt.hbs
@@ -0,0 +1 @@
+Self-Signed Certificate Has Been Refreshed
diff --git a/templates/default/cert-upcoming-refresh-body.txt.hbs b/templates/default/cert-upcoming-refresh-body.txt.hbs
new file mode 100644
index 000000000..a79db0299
--- /dev/null
+++ b/templates/default/cert-upcoming-refresh-body.txt.hbs
@@ -0,0 +1,10 @@
+Proxmox Backup Server will refresh its TLS certificate within the next 30 days.
+
+The earliest the certificate may be renewed is {{ timstamp earliest-renewal }}.
+If you rely on the certificate's fingerprint to verify TLS sessions between the
+server and a client, please update the fingerprint once the certificate was
+updated. Otherwise, no action is required.
+
+Please visit the web interface for further details:
+
+<{{base-url}}/#pbsCertificateConfiguration>
diff --git a/templates/default/cert-upcoming-refresh-subject.txt.hbs b/templates/default/cert-upcoming-refresh-subject.txt.hbs
new file mode 100644
index 000000000..f188e391c
--- /dev/null
+++ b/templates/default/cert-upcoming-refresh-subject.txt.hbs
@@ -0,0 +1 @@
+Self-signed Certificate is About to be Refreshed
-- 
2.47.3