[pbs-devel] [PATCH proxmox-backup 14/33] server: notifications: send GC notifications via notification system

[Lukas Wagner <l.wagner@proxmox.com>]

If the `notification-mode` parameter is set to `legacy-sendmail`, then
we still use the new infrastructure, but don't consider the
notification config and use a hard-coded sendmail endpoint directly.

Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
---
 debian/proxmox-backup-server.install     |   4 +
 src/api2/pull.rs                         |   2 +-
 src/server/gc_job.rs                     |  10 +-
 src/server/notifications.rs              | 184 +++++++----------------
 src/server/verify_job.rs                 |   3 +-
 templates/Makefile                       |   4 +
 templates/default/gc-err-body.txt.hbs    |   8 +
 templates/default/gc-err-subject.txt.hbs |   1 +
 templates/default/gc-ok-body.txt.hbs     |  23 +++
 templates/default/gc-ok-subject.txt.hbs  |   1 +
 10 files changed, 102 insertions(+), 138 deletions(-)
 create mode 100644 templates/default/gc-err-body.txt.hbs
 create mode 100644 templates/default/gc-err-subject.txt.hbs
 create mode 100644 templates/default/gc-ok-body.txt.hbs
 create mode 100644 templates/default/gc-ok-subject.txt.hbs

diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install
index 6aff594d..197c070f 100644
--- a/debian/proxmox-backup-server.install
+++ b/debian/proxmox-backup-server.install
@@ -41,6 +41,10 @@ usr/share/zsh/vendor-completions/_pmtx
 usr/share/zsh/vendor-completions/_proxmox-backup-debug
 usr/share/zsh/vendor-completions/_proxmox-backup-manager
 usr/share/zsh/vendor-completions/_proxmox-tape
+usr/share/proxmox-backup/templates/default/gc-err-body.txt.hbs
+usr/share/proxmox-backup/templates/default/gc-ok-body.txt.hbs
+usr/share/proxmox-backup/templates/default/gc-err-subject.txt.hbs
+usr/share/proxmox-backup/templates/default/gc-ok-subject.txt.hbs
 usr/share/proxmox-backup/templates/default/test-body.txt.hbs
 usr/share/proxmox-backup/templates/default/test-body.html.hbs
 usr/share/proxmox-backup/templates/default/test-subject.txt.hbs
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index 59db3660..7fe2267a 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -114,7 +114,7 @@ pub fn do_sync_job(
         bail!("can't sync to same datastore");
     }
 
-    let (email, notify) = crate::server::lookup_datastore_notify_settings(&sync_job.store);
+    let (email, notify, _) = crate::server::lookup_datastore_notify_settings(&sync_job.store);
 
     let upid_str = WorkerTask::spawn(
         &worker_type,
diff --git a/src/server/gc_job.rs b/src/server/gc_job.rs
index 41375d72..ff5bdccf 100644
--- a/src/server/gc_job.rs
+++ b/src/server/gc_job.rs
@@ -19,8 +19,6 @@ pub fn do_garbage_collection_job(
 ) -> Result<String, Error> {
     let store = datastore.name().to_string();
 
-    let (email, notify) = crate::server::lookup_datastore_notify_settings(&store);
-
     let worker_type = job.jobtype().to_string();
     let upid_str = WorkerTask::new_thread(
         &worker_type,
@@ -43,11 +41,9 @@ pub fn do_garbage_collection_job(
                 eprintln!("could not finish job state for {}: {err}", job.jobtype());
             }
 
-            if let Some(email) = email {
-                let gc_status = datastore.last_gc_status();
-                if let Err(err) = send_gc_status(&email, notify, &store, &gc_status, &result) {
-                    eprintln!("send gc notification failed: {err}");
-                }
+            let gc_status = datastore.last_gc_status();
+            if let Err(err) = send_gc_status(&store, &gc_status, &result) {
+                eprintln!("send gc notification failed: {err}");
             }
 
             result
diff --git a/src/server/notifications.rs b/src/server/notifications.rs
index 8dde2eea..720ae5ff 100644
--- a/src/server/notifications.rs
+++ b/src/server/notifications.rs
@@ -1,16 +1,13 @@
 use anyhow::Error;
 use const_format::concatcp;
-use serde_json::{json, Value};
+use serde_json::json;
 use std::collections::HashMap;
 use std::path::Path;
 use std::time::{Duration, Instant};
 
-use handlebars::{
-    Context, Handlebars, Helper, HelperResult, Output, RenderContext, RenderError, TemplateError,
-};
+use handlebars::{Handlebars, TemplateError};
 use nix::unistd::Uid;
 
-use proxmox_human_byte::HumanByte;
 use proxmox_lang::try_block;
 use proxmox_notify::context::pbs::PBS_CONTEXT;
 use proxmox_schema::ApiType;
@@ -18,52 +15,13 @@ use proxmox_sys::email::sendmail;
 use proxmox_sys::fs::{create_path, CreateOptions};
 
 use pbs_api_types::{
-    APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, Notify,
-    SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig,
+    APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, NotificationMode,
+    Notify, SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig,
 };
-use proxmox_notify::{Notification, Severity};
+use proxmox_notify::endpoints::sendmail::{SendmailConfig, SendmailEndpoint};
+use proxmox_notify::{Endpoint, Notification, Severity};
 
 const SPOOL_DIR: &str = concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications");
-const GC_OK_TEMPLATE: &str = r###"
-
-Datastore:            {{datastore}}
-Task ID:              {{status.upid}}
-Index file count:     {{status.index-file-count}}
-
-Removed garbage:      {{human-bytes status.removed-bytes}}
-Removed chunks:       {{status.removed-chunks}}
-Removed bad chunks:   {{status.removed-bad}}
-
-Leftover bad chunks:  {{status.still-bad}}
-Pending removals:     {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks)
-
-Original Data usage:  {{human-bytes status.index-data-bytes}}
-On-Disk usage:        {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}})
-On-Disk chunks:       {{status.disk-chunks}}
-
-Deduplication Factor: {{deduplication-factor}}
-
-Garbage collection successful.
-
-
-Please visit the web interface for further details:
-
-<https://{{fqdn}}:{{port}}/#DataStore-{{datastore}}>
-
-"###;
-
-const GC_ERR_TEMPLATE: &str = r###"
-
-Datastore: {{datastore}}
-
-Garbage collection failed: {{error}}
-
-
-Please visit the web interface for further details:
-
-<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
-
-"###;
 
 const VERIFY_OK_TEMPLATE: &str = r###"
 
@@ -259,12 +217,6 @@ lazy_static::lazy_static! {
             hb.set_strict_mode(true);
             hb.register_escape_fn(handlebars::no_escape);
 
-            hb.register_helper("human-bytes", Box::new(handlebars_humam_bytes_helper));
-            hb.register_helper("relative-percentage", Box::new(handlebars_relative_percentage_helper));
-
-            hb.register_template_string("gc_ok_template", GC_OK_TEMPLATE)?;
-            hb.register_template_string("gc_err_template", GC_ERR_TEMPLATE)?;
-
             hb.register_template_string("verify_ok_template", VERIFY_OK_TEMPLATE)?;
             hb.register_template_string("verify_err_template", VERIFY_ERR_TEMPLATE)?;
 
@@ -377,6 +329,19 @@ fn send_notification(notification: Notification) -> Result<(), Error> {
     Ok(())
 }
 
+fn send_sendmail_legacy_notification(notification: Notification, email: &str) -> Result<(), Error> {
+    let endpoint = SendmailEndpoint {
+        config: SendmailConfig {
+            mailto: vec![email.into()],
+            ..Default::default()
+        },
+    };
+
+    endpoint.send(&notification)?;
+
+    Ok(())
+}
+
 /// Summary of a successful Tape Job
 #[derive(Default)]
 pub struct TapeBackupJobSummary {
@@ -413,21 +378,10 @@ fn send_job_status_mail(email: &str, subject: &str, text: &str) -> Result<(), Er
 }
 
 pub fn send_gc_status(
-    email: &str,
-    notify: DatastoreNotify,
     datastore: &str,
     status: &GarbageCollectionStatus,
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
-    match notify.gc {
-        None => { /* send notifications by default */ }
-        Some(notify) => {
-            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
-                return Ok(());
-            }
-        }
-    }
-
     let (fqdn, port) = get_server_url();
     let mut data = json!({
         "datastore": datastore,
@@ -435,7 +389,7 @@ pub fn send_gc_status(
         "port": port,
     });
 
-    let text = match result {
+    let (severity, template) = match result {
         Ok(()) => {
             let deduplication_factor = if status.disk_bytes > 0 {
                 (status.index_data_bytes as f64) / (status.disk_bytes as f64)
@@ -446,20 +400,38 @@ pub fn send_gc_status(
             data["status"] = json!(status);
             data["deduplication-factor"] = format!("{:.2}", deduplication_factor).into();
 
-            HANDLEBARS.render("gc_ok_template", &data)?
+            (Severity::Info, "gc-ok")
         }
         Err(err) => {
             data["error"] = err.to_string().into();
-            HANDLEBARS.render("gc_err_template", &data)?
+            (Severity::Error, "gc-err")
         }
     };
+    let metadata = HashMap::from([
+        ("datastore".into(), datastore.into()),
+        ("hostname".into(), proxmox_sys::nodename().into()),
+        ("type".into(), "gc".into()),
+    ]);
 
-    let subject = match result {
-        Ok(()) => format!("Garbage Collect Datastore '{datastore}' successful"),
-        Err(_) => format!("Garbage Collect Datastore '{datastore}' failed"),
-    };
+    let notification = Notification::from_template(severity, template, data, metadata);
 
-    send_job_status_mail(email, &subject, &text)?;
+    let (email, notify, mode) = lookup_datastore_notify_settings(datastore);
+    match mode {
+        NotificationMode::LegacySendmail => {
+            let notify = notify.gc.unwrap_or(Notify::Always);
+
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+
+            if let Some(email) = email {
+                send_sendmail_legacy_notification(notification, &email)?;
+            }
+        }
+        NotificationMode::NotificationSystem => {
+            send_notification(notification)?;
+        }
+    }
 
     Ok(())
 }
@@ -519,8 +491,8 @@ pub fn send_prune_status(
     result: &Result<(), Error>,
 ) -> Result<(), Error> {
     let (email, notify) = match lookup_datastore_notify_settings(store) {
-        (Some(email), notify) => (email, notify),
-        (None, _) => return Ok(()),
+        (Some(email), notify, _) => (email, notify),
+        (None, _, _) => return Ok(()),
     };
 
     let notify_prune = notify.prune.unwrap_or(Notify::Error);
@@ -755,7 +727,9 @@ pub fn lookup_user_email(userid: &Userid) -> Option<String> {
 }
 
 /// Lookup Datastore notify settings
-pub fn lookup_datastore_notify_settings(store: &str) -> (Option<String>, DatastoreNotify) {
+pub fn lookup_datastore_notify_settings(
+    store: &str,
+) -> (Option<String>, DatastoreNotify, NotificationMode) {
     let mut email = None;
 
     let notify = DatastoreNotify {
@@ -767,12 +741,12 @@ pub fn lookup_datastore_notify_settings(store: &str) -> (Option<String>, Datasto
 
     let (config, _digest) = match pbs_config::datastore::config() {
         Ok(result) => result,
-        Err(_) => return (email, notify),
+        Err(_) => return (email, notify, NotificationMode::default()),
     };
 
     let config: DataStoreConfig = match config.lookup("datastore", store) {
         Ok(result) => result,
-        Err(_) => return (email, notify),
+        Err(_) => return (email, notify, NotificationMode::default()),
     };
 
     email = match config.notify_user {
@@ -780,68 +754,20 @@ pub fn lookup_datastore_notify_settings(store: &str) -> (Option<String>, Datasto
         None => lookup_user_email(Userid::root_userid()),
     };
 
+    let notification_mode = config.notification_mode.unwrap_or_default();
     let notify_str = config.notify.unwrap_or_default();
 
     if let Ok(value) = DatastoreNotify::API_SCHEMA.parse_property_string(&notify_str) {
         if let Ok(notify) = serde_json::from_value(value) {
-            return (email, notify);
+            return (email, notify, notification_mode);
         }
     }
 
-    (email, notify)
-}
-
-// Handlerbar helper functions
-
-fn handlebars_humam_bytes_helper(
-    h: &Helper,
-    _: &Handlebars,
-    _: &Context,
-    _rc: &mut RenderContext,
-    out: &mut dyn Output,
-) -> HelperResult {
-    let param = h
-        .param(0)
-        .and_then(|v| v.value().as_u64())
-        .ok_or_else(|| RenderError::new("human-bytes: param not found"))?;
-
-    out.write(&HumanByte::from(param).to_string())?;
-
-    Ok(())
-}
-
-fn handlebars_relative_percentage_helper(
-    h: &Helper,
-    _: &Handlebars,
-    _: &Context,
-    _rc: &mut RenderContext,
-    out: &mut dyn Output,
-) -> HelperResult {
-    let param0 = h
-        .param(0)
-        .and_then(|v| v.value().as_f64())
-        .ok_or_else(|| RenderError::new("relative-percentage: param0 not found"))?;
-    let param1 = h
-        .param(1)
-        .and_then(|v| v.value().as_f64())
-        .ok_or_else(|| RenderError::new("relative-percentage: param1 not found"))?;
-
-    if param1 == 0.0 {
-        out.write("-")?;
-    } else {
-        out.write(&format!("{:.2}%", (param0 * 100.0) / param1))?;
-    }
-    Ok(())
+    (email, notify, notification_mode)
 }
 
 #[test]
 fn test_template_register() {
-    HANDLEBARS.get_helper("human-bytes").unwrap();
-    HANDLEBARS.get_helper("relative-percentage").unwrap();
-
-    assert!(HANDLEBARS.has_template("gc_ok_template"));
-    assert!(HANDLEBARS.has_template("gc_err_template"));
-
     assert!(HANDLEBARS.has_template("verify_ok_template"));
     assert!(HANDLEBARS.has_template("verify_err_template"));
 
diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs
index 8bf2a0c9..fed80e5c 100644
--- a/src/server/verify_job.rs
+++ b/src/server/verify_job.rs
@@ -23,7 +23,8 @@ pub fn do_verification_job(
     let outdated_after = verification_job.outdated_after;
     let ignore_verified_snapshots = verification_job.ignore_verified.unwrap_or(true);
 
-    let (email, notify) = crate::server::lookup_datastore_notify_settings(&verification_job.store);
+    let (email, notify, _) =
+        crate::server::lookup_datastore_notify_settings(&verification_job.store);
 
     // FIXME encode namespace here for filter/ACL check?
     let job_id = format!("{}:{}", &verification_job.store, job.jobname());
diff --git a/templates/Makefile b/templates/Makefile
index b35a8bb3..7d4cb19f 100644
--- a/templates/Makefile
+++ b/templates/Makefile
@@ -1,6 +1,10 @@
 include ../defines.mk
 
 NOTIFICATION_TEMPLATES=					\
+	default/gc-err-body.txt.hbs			\
+	default/gc-ok-body.txt.hbs			\
+	default/gc-err-subject.txt.hbs		\
+	default/gc-ok-subject.txt.hbs		\
 	default/test-body.txt.hbs			\
 	default/test-body.html.hbs			\
 	default/test-subject.txt.hbs		\
diff --git a/templates/default/gc-err-body.txt.hbs b/templates/default/gc-err-body.txt.hbs
new file mode 100644
index 00000000..d6c2d0bc
--- /dev/null
+++ b/templates/default/gc-err-body.txt.hbs
@@ -0,0 +1,8 @@
+Datastore: {{datastore}}
+
+Garbage collection failed: {{error}}
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
diff --git a/templates/default/gc-err-subject.txt.hbs b/templates/default/gc-err-subject.txt.hbs
new file mode 100644
index 00000000..ebf49f3b
--- /dev/null
+++ b/templates/default/gc-err-subject.txt.hbs
@@ -0,0 +1 @@
+Garbage Collect Datastore '{{ datastore }}' failed
diff --git a/templates/default/gc-ok-body.txt.hbs b/templates/default/gc-ok-body.txt.hbs
new file mode 100644
index 00000000..d2f7cd81
--- /dev/null
+++ b/templates/default/gc-ok-body.txt.hbs
@@ -0,0 +1,23 @@
+Datastore:            {{datastore}}
+Task ID:              {{status.upid}}
+Index file count:     {{status.index-file-count}}
+
+Removed garbage:      {{human-bytes status.removed-bytes}}
+Removed chunks:       {{status.removed-chunks}}
+Removed bad chunks:   {{status.removed-bad}}
+
+Leftover bad chunks:  {{status.still-bad}}
+Pending removals:     {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks)
+
+Original Data usage:  {{human-bytes status.index-data-bytes}}
+On-Disk usage:        {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}})
+On-Disk chunks:       {{status.disk-chunks}}
+
+Deduplication Factor: {{deduplication-factor}}
+
+Garbage collection successful.
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#DataStore-{{datastore}}>
diff --git a/templates/default/gc-ok-subject.txt.hbs b/templates/default/gc-ok-subject.txt.hbs
new file mode 100644
index 00000000..538e3700
--- /dev/null
+++ b/templates/default/gc-ok-subject.txt.hbs
@@ -0,0 +1 @@
+Garbage Collect Datastore '{{ datastore }}' successful
-- 
2.39.2