From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 9DA0595766 for ; Fri, 12 Apr 2024 12:14:33 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 76AF681F2 for ; Fri, 12 Apr 2024 12:14:03 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 12 Apr 2024 12:14:00 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id D7B2A45142 for ; Fri, 12 Apr 2024 12:06:44 +0200 (CEST) From: Lukas Wagner To: pbs-devel@lists.proxmox.com Date: Fri, 12 Apr 2024 12:06:12 +0200 Message-Id: <20240412100631.94218-15-l.wagner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240412100631.94218-1-l.wagner@proxmox.com> References: <20240412100631.94218-1-l.wagner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.005 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 14/33] server: notifications: send GC notifications via notification system X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 12 Apr 2024 10:14:33 -0000 If the `notification-mode` parameter is set to `legacy-sendmail`, then we still use the new infrastructure, but don't consider the notification config and use a hard-coded sendmail endpoint directly. Signed-off-by: Lukas Wagner --- debian/proxmox-backup-server.install | 4 + src/api2/pull.rs | 2 +- src/server/gc_job.rs | 10 +- src/server/notifications.rs | 184 +++++++---------------- src/server/verify_job.rs | 3 +- templates/Makefile | 4 + templates/default/gc-err-body.txt.hbs | 8 + templates/default/gc-err-subject.txt.hbs | 1 + templates/default/gc-ok-body.txt.hbs | 23 +++ templates/default/gc-ok-subject.txt.hbs | 1 + 10 files changed, 102 insertions(+), 138 deletions(-) create mode 100644 templates/default/gc-err-body.txt.hbs create mode 100644 templates/default/gc-err-subject.txt.hbs create mode 100644 templates/default/gc-ok-body.txt.hbs create mode 100644 templates/default/gc-ok-subject.txt.hbs diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install index 6aff594d..197c070f 100644 --- a/debian/proxmox-backup-server.install +++ b/debian/proxmox-backup-server.install @@ -41,6 +41,10 @@ usr/share/zsh/vendor-completions/_pmtx usr/share/zsh/vendor-completions/_proxmox-backup-debug usr/share/zsh/vendor-completions/_proxmox-backup-manager usr/share/zsh/vendor-completions/_proxmox-tape +usr/share/proxmox-backup/templates/default/gc-err-body.txt.hbs +usr/share/proxmox-backup/templates/default/gc-ok-body.txt.hbs +usr/share/proxmox-backup/templates/default/gc-err-subject.txt.hbs +usr/share/proxmox-backup/templates/default/gc-ok-subject.txt.hbs usr/share/proxmox-backup/templates/default/test-body.txt.hbs usr/share/proxmox-backup/templates/default/test-body.html.hbs usr/share/proxmox-backup/templates/default/test-subject.txt.hbs diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 59db3660..7fe2267a 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -114,7 +114,7 @@ pub fn do_sync_job( bail!("can't sync to same datastore"); } - let (email, notify) = crate::server::lookup_datastore_notify_settings(&sync_job.store); + let (email, notify, _) = crate::server::lookup_datastore_notify_settings(&sync_job.store); let upid_str = WorkerTask::spawn( &worker_type, diff --git a/src/server/gc_job.rs b/src/server/gc_job.rs index 41375d72..ff5bdccf 100644 --- a/src/server/gc_job.rs +++ b/src/server/gc_job.rs @@ -19,8 +19,6 @@ pub fn do_garbage_collection_job( ) -> Result { let store = datastore.name().to_string(); - let (email, notify) = crate::server::lookup_datastore_notify_settings(&store); - let worker_type = job.jobtype().to_string(); let upid_str = WorkerTask::new_thread( &worker_type, @@ -43,11 +41,9 @@ pub fn do_garbage_collection_job( eprintln!("could not finish job state for {}: {err}", job.jobtype()); } - if let Some(email) = email { - let gc_status = datastore.last_gc_status(); - if let Err(err) = send_gc_status(&email, notify, &store, &gc_status, &result) { - eprintln!("send gc notification failed: {err}"); - } + let gc_status = datastore.last_gc_status(); + if let Err(err) = send_gc_status(&store, &gc_status, &result) { + eprintln!("send gc notification failed: {err}"); } result diff --git a/src/server/notifications.rs b/src/server/notifications.rs index 8dde2eea..720ae5ff 100644 --- a/src/server/notifications.rs +++ b/src/server/notifications.rs @@ -1,16 +1,13 @@ use anyhow::Error; use const_format::concatcp; -use serde_json::{json, Value}; +use serde_json::json; use std::collections::HashMap; use std::path::Path; use std::time::{Duration, Instant}; -use handlebars::{ - Context, Handlebars, Helper, HelperResult, Output, RenderContext, RenderError, TemplateError, -}; +use handlebars::{Handlebars, TemplateError}; use nix::unistd::Uid; -use proxmox_human_byte::HumanByte; use proxmox_lang::try_block; use proxmox_notify::context::pbs::PBS_CONTEXT; use proxmox_schema::ApiType; @@ -18,52 +15,13 @@ use proxmox_sys::email::sendmail; use proxmox_sys::fs::{create_path, CreateOptions}; use pbs_api_types::{ - APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, Notify, - SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig, + APTUpdateInfo, DataStoreConfig, DatastoreNotify, GarbageCollectionStatus, NotificationMode, + Notify, SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig, }; -use proxmox_notify::{Notification, Severity}; +use proxmox_notify::endpoints::sendmail::{SendmailConfig, SendmailEndpoint}; +use proxmox_notify::{Endpoint, Notification, Severity}; const SPOOL_DIR: &str = concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications"); -const GC_OK_TEMPLATE: &str = r###" - -Datastore: {{datastore}} -Task ID: {{status.upid}} -Index file count: {{status.index-file-count}} - -Removed garbage: {{human-bytes status.removed-bytes}} -Removed chunks: {{status.removed-chunks}} -Removed bad chunks: {{status.removed-bad}} - -Leftover bad chunks: {{status.still-bad}} -Pending removals: {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks) - -Original Data usage: {{human-bytes status.index-data-bytes}} -On-Disk usage: {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}}) -On-Disk chunks: {{status.disk-chunks}} - -Deduplication Factor: {{deduplication-factor}} - -Garbage collection successful. - - -Please visit the web interface for further details: - - - -"###; - -const GC_ERR_TEMPLATE: &str = r###" - -Datastore: {{datastore}} - -Garbage collection failed: {{error}} - - -Please visit the web interface for further details: - - - -"###; const VERIFY_OK_TEMPLATE: &str = r###" @@ -259,12 +217,6 @@ lazy_static::lazy_static! { hb.set_strict_mode(true); hb.register_escape_fn(handlebars::no_escape); - hb.register_helper("human-bytes", Box::new(handlebars_humam_bytes_helper)); - hb.register_helper("relative-percentage", Box::new(handlebars_relative_percentage_helper)); - - hb.register_template_string("gc_ok_template", GC_OK_TEMPLATE)?; - hb.register_template_string("gc_err_template", GC_ERR_TEMPLATE)?; - hb.register_template_string("verify_ok_template", VERIFY_OK_TEMPLATE)?; hb.register_template_string("verify_err_template", VERIFY_ERR_TEMPLATE)?; @@ -377,6 +329,19 @@ fn send_notification(notification: Notification) -> Result<(), Error> { Ok(()) } +fn send_sendmail_legacy_notification(notification: Notification, email: &str) -> Result<(), Error> { + let endpoint = SendmailEndpoint { + config: SendmailConfig { + mailto: vec![email.into()], + ..Default::default() + }, + }; + + endpoint.send(¬ification)?; + + Ok(()) +} + /// Summary of a successful Tape Job #[derive(Default)] pub struct TapeBackupJobSummary { @@ -413,21 +378,10 @@ fn send_job_status_mail(email: &str, subject: &str, text: &str) -> Result<(), Er } pub fn send_gc_status( - email: &str, - notify: DatastoreNotify, datastore: &str, status: &GarbageCollectionStatus, result: &Result<(), Error>, ) -> Result<(), Error> { - match notify.gc { - None => { /* send notifications by default */ } - Some(notify) => { - if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) { - return Ok(()); - } - } - } - let (fqdn, port) = get_server_url(); let mut data = json!({ "datastore": datastore, @@ -435,7 +389,7 @@ pub fn send_gc_status( "port": port, }); - let text = match result { + let (severity, template) = match result { Ok(()) => { let deduplication_factor = if status.disk_bytes > 0 { (status.index_data_bytes as f64) / (status.disk_bytes as f64) @@ -446,20 +400,38 @@ pub fn send_gc_status( data["status"] = json!(status); data["deduplication-factor"] = format!("{:.2}", deduplication_factor).into(); - HANDLEBARS.render("gc_ok_template", &data)? + (Severity::Info, "gc-ok") } Err(err) => { data["error"] = err.to_string().into(); - HANDLEBARS.render("gc_err_template", &data)? + (Severity::Error, "gc-err") } }; + let metadata = HashMap::from([ + ("datastore".into(), datastore.into()), + ("hostname".into(), proxmox_sys::nodename().into()), + ("type".into(), "gc".into()), + ]); - let subject = match result { - Ok(()) => format!("Garbage Collect Datastore '{datastore}' successful"), - Err(_) => format!("Garbage Collect Datastore '{datastore}' failed"), - }; + let notification = Notification::from_template(severity, template, data, metadata); - send_job_status_mail(email, &subject, &text)?; + let (email, notify, mode) = lookup_datastore_notify_settings(datastore); + match mode { + NotificationMode::LegacySendmail => { + let notify = notify.gc.unwrap_or(Notify::Always); + + if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) { + return Ok(()); + } + + if let Some(email) = email { + send_sendmail_legacy_notification(notification, &email)?; + } + } + NotificationMode::NotificationSystem => { + send_notification(notification)?; + } + } Ok(()) } @@ -519,8 +491,8 @@ pub fn send_prune_status( result: &Result<(), Error>, ) -> Result<(), Error> { let (email, notify) = match lookup_datastore_notify_settings(store) { - (Some(email), notify) => (email, notify), - (None, _) => return Ok(()), + (Some(email), notify, _) => (email, notify), + (None, _, _) => return Ok(()), }; let notify_prune = notify.prune.unwrap_or(Notify::Error); @@ -755,7 +727,9 @@ pub fn lookup_user_email(userid: &Userid) -> Option { } /// Lookup Datastore notify settings -pub fn lookup_datastore_notify_settings(store: &str) -> (Option, DatastoreNotify) { +pub fn lookup_datastore_notify_settings( + store: &str, +) -> (Option, DatastoreNotify, NotificationMode) { let mut email = None; let notify = DatastoreNotify { @@ -767,12 +741,12 @@ pub fn lookup_datastore_notify_settings(store: &str) -> (Option, Datasto let (config, _digest) = match pbs_config::datastore::config() { Ok(result) => result, - Err(_) => return (email, notify), + Err(_) => return (email, notify, NotificationMode::default()), }; let config: DataStoreConfig = match config.lookup("datastore", store) { Ok(result) => result, - Err(_) => return (email, notify), + Err(_) => return (email, notify, NotificationMode::default()), }; email = match config.notify_user { @@ -780,68 +754,20 @@ pub fn lookup_datastore_notify_settings(store: &str) -> (Option, Datasto None => lookup_user_email(Userid::root_userid()), }; + let notification_mode = config.notification_mode.unwrap_or_default(); let notify_str = config.notify.unwrap_or_default(); if let Ok(value) = DatastoreNotify::API_SCHEMA.parse_property_string(¬ify_str) { if let Ok(notify) = serde_json::from_value(value) { - return (email, notify); + return (email, notify, notification_mode); } } - (email, notify) -} - -// Handlerbar helper functions - -fn handlebars_humam_bytes_helper( - h: &Helper, - _: &Handlebars, - _: &Context, - _rc: &mut RenderContext, - out: &mut dyn Output, -) -> HelperResult { - let param = h - .param(0) - .and_then(|v| v.value().as_u64()) - .ok_or_else(|| RenderError::new("human-bytes: param not found"))?; - - out.write(&HumanByte::from(param).to_string())?; - - Ok(()) -} - -fn handlebars_relative_percentage_helper( - h: &Helper, - _: &Handlebars, - _: &Context, - _rc: &mut RenderContext, - out: &mut dyn Output, -) -> HelperResult { - let param0 = h - .param(0) - .and_then(|v| v.value().as_f64()) - .ok_or_else(|| RenderError::new("relative-percentage: param0 not found"))?; - let param1 = h - .param(1) - .and_then(|v| v.value().as_f64()) - .ok_or_else(|| RenderError::new("relative-percentage: param1 not found"))?; - - if param1 == 0.0 { - out.write("-")?; - } else { - out.write(&format!("{:.2}%", (param0 * 100.0) / param1))?; - } - Ok(()) + (email, notify, notification_mode) } #[test] fn test_template_register() { - HANDLEBARS.get_helper("human-bytes").unwrap(); - HANDLEBARS.get_helper("relative-percentage").unwrap(); - - assert!(HANDLEBARS.has_template("gc_ok_template")); - assert!(HANDLEBARS.has_template("gc_err_template")); - assert!(HANDLEBARS.has_template("verify_ok_template")); assert!(HANDLEBARS.has_template("verify_err_template")); diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs index 8bf2a0c9..fed80e5c 100644 --- a/src/server/verify_job.rs +++ b/src/server/verify_job.rs @@ -23,7 +23,8 @@ pub fn do_verification_job( let outdated_after = verification_job.outdated_after; let ignore_verified_snapshots = verification_job.ignore_verified.unwrap_or(true); - let (email, notify) = crate::server::lookup_datastore_notify_settings(&verification_job.store); + let (email, notify, _) = + crate::server::lookup_datastore_notify_settings(&verification_job.store); // FIXME encode namespace here for filter/ACL check? let job_id = format!("{}:{}", &verification_job.store, job.jobname()); diff --git a/templates/Makefile b/templates/Makefile index b35a8bb3..7d4cb19f 100644 --- a/templates/Makefile +++ b/templates/Makefile @@ -1,6 +1,10 @@ include ../defines.mk NOTIFICATION_TEMPLATES= \ + default/gc-err-body.txt.hbs \ + default/gc-ok-body.txt.hbs \ + default/gc-err-subject.txt.hbs \ + default/gc-ok-subject.txt.hbs \ default/test-body.txt.hbs \ default/test-body.html.hbs \ default/test-subject.txt.hbs \ diff --git a/templates/default/gc-err-body.txt.hbs b/templates/default/gc-err-body.txt.hbs new file mode 100644 index 00000000..d6c2d0bc --- /dev/null +++ b/templates/default/gc-err-body.txt.hbs @@ -0,0 +1,8 @@ +Datastore: {{datastore}} + +Garbage collection failed: {{error}} + + +Please visit the web interface for further details: + + diff --git a/templates/default/gc-err-subject.txt.hbs b/templates/default/gc-err-subject.txt.hbs new file mode 100644 index 00000000..ebf49f3b --- /dev/null +++ b/templates/default/gc-err-subject.txt.hbs @@ -0,0 +1 @@ +Garbage Collect Datastore '{{ datastore }}' failed diff --git a/templates/default/gc-ok-body.txt.hbs b/templates/default/gc-ok-body.txt.hbs new file mode 100644 index 00000000..d2f7cd81 --- /dev/null +++ b/templates/default/gc-ok-body.txt.hbs @@ -0,0 +1,23 @@ +Datastore: {{datastore}} +Task ID: {{status.upid}} +Index file count: {{status.index-file-count}} + +Removed garbage: {{human-bytes status.removed-bytes}} +Removed chunks: {{status.removed-chunks}} +Removed bad chunks: {{status.removed-bad}} + +Leftover bad chunks: {{status.still-bad}} +Pending removals: {{human-bytes status.pending-bytes}} (in {{status.pending-chunks}} chunks) + +Original Data usage: {{human-bytes status.index-data-bytes}} +On-Disk usage: {{human-bytes status.disk-bytes}} ({{relative-percentage status.disk-bytes status.index-data-bytes}}) +On-Disk chunks: {{status.disk-chunks}} + +Deduplication Factor: {{deduplication-factor}} + +Garbage collection successful. + + +Please visit the web interface for further details: + + diff --git a/templates/default/gc-ok-subject.txt.hbs b/templates/default/gc-ok-subject.txt.hbs new file mode 100644 index 00000000..538e3700 --- /dev/null +++ b/templates/default/gc-ok-subject.txt.hbs @@ -0,0 +1 @@ +Garbage Collect Datastore '{{ datastore }}' successful -- 2.39.2