From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 9A8C41FF0B7 for ; Fri, 13 Nov 2026 16:18:10 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 8383C21DAC; Fri, 13 Nov 2026 16:10:23 +0100 (CET) From: Lukas Wagner To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox-backup 46/55] notifications: send all notifications via proxmox-notify's queue Date: Fri, 13 Nov 2026 16:08:19 +0100 Message-ID: <20261113150828.756836-47-l.wagner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113150828.756836-1-l.wagner@proxmox.com> References: <20261113150828.756836-1-l.wagner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794582528867 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.800 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 6EFJJUZVORXKBSTL5PVAE2KQRAUZCBLK X-Message-ID-Hash: 6EFJJUZVORXKBSTL5PVAE2KQRAUZCBLK X-MailFrom: l.wagner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Previously, notifications were only queued when sent from an unprivileged process, which wrote them as JSON files to the spool directory for the worker in proxmox-backup-api to pick up. Each file was removed before sending, so a notification was lost if the daemon stopped while it was being sent. Privileged processes sent notifications synchronously. Now every notification is submitted to the queue provided by proxmox-notify, located in /var/lib/proxmox-backup/notifications/queue. The queue directory is owned by the privileged user and writable by the 'backup' group, so that both privileged and unprivileged processes can submit notifications. The worker in proxmox-backup-api is replaced by proxmox-notify's QueueWorker, which polls the queue every five seconds. If the worker is interrupted while sending a notification, it retries sending it up to two times after restart before dropping it. Notifications that are still in the old spool directory on upgrade are moved to the new queue when the worker starts. Signed-off-by: Lukas Wagner --- Cargo.toml | 13 ++- src/bin/proxmox-backup-api.rs | 2 +- src/server/notifications/mod.rs | 163 ++++++++++++++++---------------- 3 files changed, 93 insertions(+), 85 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d609e91fb..bdffd6ffd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -230,7 +230,18 @@ proxmox-ldap.workspace = true proxmox-metrics.workspace = true proxmox-network-api = { workspace = true, features = [ "impl" ] } proxmox-network-types.workspace = true -proxmox-notify = { workspace = true, features = [ "pbs-context", "legacy-matchers", "api-impl", "send-impl", "api-types" ]} +proxmox-notify = { + workspace = true, + features = [ + "pbs-context", + "legacy-matchers", + "api-impl", + "send-impl", + "api-types", + "queue-submitter", + "queue-worker" + ] +} proxmox-openid.workspace = true proxmox-product-config.workspace = true proxmox-parallel-handler.workspace = true diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs index aad3fabc3..7e38b0225 100644 --- a/src/bin/proxmox-backup-api.rs +++ b/src/bin/proxmox-backup-api.rs @@ -60,7 +60,7 @@ async fn run() -> Result<(), Error> { proxmox_backup::server::create_state_dir()?; proxmox_backup::server::create_active_operations_dir()?; proxmox_backup::server::jobstate::create_jobstate_dir()?; - proxmox_backup::server::notifications::create_spool_dir()?; + proxmox_backup::server::notifications::create_notification_dirs()?; proxmox_backup::tape::create_tape_status_dir()?; proxmox_backup::tape::create_drive_state_dir()?; proxmox_backup::tape::create_changer_state_dir()?; diff --git a/src/server/notifications/mod.rs b/src/server/notifications/mod.rs index 1e71cd39d..525d8e02b 100644 --- a/src/server/notifications/mod.rs +++ b/src/server/notifications/mod.rs @@ -1,15 +1,17 @@ use std::path::Path; use std::sync::OnceLock; -use std::time::{Duration, Instant}; +use std::time::Duration; use anyhow::Error; use const_format::concatcp; -use nix::unistd::Uid; +use nix::sys::stat::Mode; use proxmox_notify::context::pbs::PBSContext; +use proxmox_notify::queue::{QueueConsumer, QueueSubmitter, QueueWorker}; use proxmox_notify::{Context, EmailRecipient, Notification, Recipient, Severity}; use proxmox_schema::ApiType; use proxmox_sys::fs::{CreateOptions, create_path}; +use proxmox_uuid::Uuid; use crate::tape::TapeNotificationMode; use pbs_api_types::{ @@ -17,7 +19,9 @@ use pbs_api_types::{ Notify, SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig, }; -const SPOOL_DIR: &str = concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications"); +const NOTIFICATION_BASE_DIR: &str = + concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications"); +const QUEUE_DIR: &str = concatcp!(NOTIFICATION_BASE_DIR, "/queue"); mod template_data; @@ -35,101 +39,94 @@ pub fn context() -> &'static Context { CONTEXT.get_or_init(|| Context::new(PBSContext)) } -/// Create the directory which will be used to temporarily store notifications -/// which were sent from an unprivileged process. -pub fn create_spool_dir() -> Result<(), Error> { +/// Creates directories used for queuing notifications. +pub fn create_notification_dirs() -> Result<(), Error> { let backup_user = proxmox_product_config::get_api_user(); let opts = CreateOptions::new() .owner(backup_user.uid) .group(backup_user.gid); - create_path(SPOOL_DIR, None, Some(opts))?; + create_path(NOTIFICATION_BASE_DIR, None, Some(opts))?; + + // The queue is writable by anybody in the 'backup' group. + let opts = CreateOptions::new() + .owner(proxmox_product_config::get_priv_user().uid) + .group(backup_user.gid) + .perm(Mode::from_bits_truncate(0o0770)); + + proxmox_sys::fs::ensure_dir_exists(QUEUE_DIR, &opts, true)?; + Ok(()) } -async fn send_queued_notifications() -> Result<(), Error> { - let mut read_dir = tokio::fs::read_dir(SPOOL_DIR).await?; - - let mut notifications = Vec::new(); - - while let Some(entry) = read_dir.next_entry().await? { - let path = entry.path(); - - if let Some(ext) = path.extension() { - if ext == "json" { - let p = path.clone(); - - let bytes = tokio::fs::read(p).await?; - let notification: Notification = serde_json::from_slice(&bytes)?; - notifications.push(notification); - - // Currently, there is no retry-mechanism in case of failure... - // For retries, we'd have to keep track of which targets succeeded/failed - // to send, so we do not retry notifying a target which succeeded before. - tokio::fs::remove_file(path).await?; - } - } - } - - if notifications.is_empty() { - return Ok(()); - } - - // Make sure that we send the oldest notification first - notifications.sort_unstable_by_key(|n| n.timestamp()); - - let context = context(); - - let res = tokio::task::spawn_blocking(move || { - let config = pbs_config::notifications::config(context)?; - for notification in notifications { - if let Err(err) = proxmox_notify::send(context, &config, ¬ification) { - log::error!("failed to send notification: {err}"); - } - } - - Ok::<(), Error>(()) - }) - .await?; - - if let Err(e) = res { - log::error!("could not read notification config: {e}"); - } - - Ok::<(), Error>(()) -} - /// Worker task to periodically send any queued notifications. pub async fn notification_worker() { - loop { - let delay_target = Instant::now() + Duration::from_secs(5); - - if let Err(err) = send_queued_notifications().await { - log::error!("notification worker task error: {err}"); - } - - tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; + // At startup, migrate any left-over notifications that weren't delivered + // right before the switch to PBS with the newer notification queue/worker. + // + // FIXME: MAJOR VERSION: Remove + if let Err(err) = migrate_leftover_notifications() { + log::error!("failed to migrate notifications: {err:#}"); } + + let consumer = QueueConsumer::new(QUEUE_DIR); + + QueueWorker::builder(context().clone(), consumer) + .interval(Duration::from_secs(5)) + .retry_attempts(2) + .build() + .run() + .await; +} + +/// Move any left-over notifications from the old spool directory at +/// /var/lib/proxmox-backup/notifications to the new queue +/// at /var/lib/proxmox-backup/notifications/queue +fn migrate_leftover_notifications() -> Result<(), Error> { + let read_dir = std::fs::read_dir(NOTIFICATION_BASE_DIR)?; + + for entry in read_dir.flatten() { + if let Some((name, ext)) = entry.file_name().to_string_lossy().split_once(".") { + if ext == "json" && Uuid::parse_str(name).is_ok() { + let path = entry.path(); + + if let Err(err) = migrate_single_notification(&path) { + log::error!( + "failed to migrate notification at '{p}' to new queue: {err:#}", + p = path.display() + ); + } + } + } + } + + Ok(()) +} + +fn migrate_single_notification(path: &Path) -> Result<(), Error> { + let bytes = std::fs::read(path)?; + let notification: Notification = serde_json::from_slice(&bytes)?; + + // Removing the file from the old spool directory before putting it in the new queue is a + // deliberate decision: If, for some reason, the old directory is not writable, we don't + // want to continue to migrate the same notification over and over again. + std::fs::remove_file(path)?; + + let uuid = notification.id().clone(); + + // Put notification into new queue + send_notification(notification)?; + + log::info!("moved notification with ID '{uuid}' to new notification queue"); + + Ok(()) } fn send_notification(notification: Notification) -> Result<(), Error> { - let context = context(); - - if nix::unistd::ROOT == Uid::current() { - let config = pbs_config::notifications::config(context)?; - proxmox_notify::send(context, &config, ¬ification)?; - } else { - let ser = serde_json::to_vec(¬ification)?; - let path = Path::new(SPOOL_DIR).join(format!("{id}.json", id = notification.id())); - - let backup_user = proxmox_product_config::get_api_user(); - let opts = CreateOptions::new() - .owner(backup_user.uid) - .group(backup_user.gid); - proxmox_sys::fs::replace_file(path, &ser, opts, true)?; - log::info!("queued notification (id={id})", id = notification.id()) - } + let options = proxmox_product_config::default_create_options(); + let writer = QueueSubmitter::new(QUEUE_DIR, options); + writer.submit(¬ification)?; Ok(()) } -- 2.47.3