From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id BDC341FF0C0 for ; Wed, 18 Nov 2026 10:52:18 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id C7DA9215FD; Wed, 18 Nov 2026 10:45:13 +0100 (CET) From: Lukas Wagner To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v2 48/63] notifications: send all notifications via proxmox-notify's queue Date: Wed, 18 Nov 2026 10:43:32 +0100 Message-ID: <20261118094348.273646-49-l.wagner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118094348.273646-1-l.wagner@proxmox.com> References: <20261118094348.273646-1-l.wagner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794995041114 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.815 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_MAILER 2 Automated Mailer Tag Left in Email RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record WEIRD_PORT 0.001 Uses non-standard port number for HTTP Message-ID-Hash: RMMIXWJCS25LGH5IBN5A2HUF5ZWZPK2X X-Message-ID-Hash: RMMIXWJCS25LGH5IBN5A2HUF5ZWZPK2X X-MailFrom: l.wagner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Previously, notifications were only queued when sent from an unprivileged process, which wrote them as JSON files to the spool directory for the worker in proxmox-backup-api to pick up. Each file was removed before sending, so a notification was lost if the daemon stopped while it was being sent. Privileged processes sent notifications synchronously. Now every notification is submitted to the queue provided by proxmox-notify, located in /var/lib/proxmox-backup/notifications/queue. The queue directory is owned by the privileged user and writable by the 'backup' group, so that both privileged and unprivileged processes can submit notifications. The worker in proxmox-backup-api is replaced by proxmox-notify's QueueWorker, which polls the queue every five seconds. If the worker is interrupted while sending a notification, it retries sending it up to two times after restart before dropping it. Notifications left in the old spool directory are moved into the new queue by renaming them, so that they cannot be lost in between. This is repeated once a minute rather than only at startup: After an upgrade, the proxy process running the old binary could keep running until all worker tasks have finished. At move only on startup would delay these notifications by potentially a long time. Scanning the old queue directory once a minute is cheap enough to keep around until the next major update, where it can be dropped. Signed-off-by: Lukas Wagner --- Notes: Changes since v1: Keep scanning the old notification queue once a minute, as to ensure notifications sent by workers running in the proxy process *while* the package is updated are not delayed until the next daemon restart. Cargo.toml | 13 +- src/bin/proxmox-backup-api.rs | 2 +- src/server/notifications/mod.rs | 172 +++++++++++++----------- tests/data/serialized_notification.json | 1 + tests/notifications.rs | 11 ++ 5 files changed, 118 insertions(+), 81 deletions(-) create mode 100644 tests/data/serialized_notification.json create mode 100644 tests/notifications.rs diff --git a/Cargo.toml b/Cargo.toml index b83924a85..173437781 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -230,7 +230,18 @@ proxmox-ldap.workspace = true proxmox-metrics.workspace = true proxmox-network-api = { workspace = true, features = [ "impl" ] } proxmox-network-types.workspace = true -proxmox-notify = { workspace = true, features = [ "pbs-context", "legacy-matchers", "api-impl", "send-impl", "api-types" ]} +proxmox-notify = { + workspace = true, + features = [ + "pbs-context", + "legacy-matchers", + "api-impl", + "send-impl", + "api-types", + "queue-submitter", + "queue-worker", + ] +} proxmox-openid.workspace = true proxmox-product-config.workspace = true proxmox-parallel-handler.workspace = true diff --git a/src/bin/proxmox-backup-api.rs b/src/bin/proxmox-backup-api.rs index aad3fabc3..7e38b0225 100644 --- a/src/bin/proxmox-backup-api.rs +++ b/src/bin/proxmox-backup-api.rs @@ -60,7 +60,7 @@ async fn run() -> Result<(), Error> { proxmox_backup::server::create_state_dir()?; proxmox_backup::server::create_active_operations_dir()?; proxmox_backup::server::jobstate::create_jobstate_dir()?; - proxmox_backup::server::notifications::create_spool_dir()?; + proxmox_backup::server::notifications::create_notification_dirs()?; proxmox_backup::tape::create_tape_status_dir()?; proxmox_backup::tape::create_drive_state_dir()?; proxmox_backup::tape::create_changer_state_dir()?; diff --git a/src/server/notifications/mod.rs b/src/server/notifications/mod.rs index 1e71cd39d..2bbe45bce 100644 --- a/src/server/notifications/mod.rs +++ b/src/server/notifications/mod.rs @@ -1,15 +1,17 @@ use std::path::Path; use std::sync::OnceLock; -use std::time::{Duration, Instant}; +use std::time::Duration; use anyhow::Error; use const_format::concatcp; -use nix::unistd::Uid; +use nix::sys::stat::Mode; use proxmox_notify::context::pbs::PBSContext; +use proxmox_notify::queue::{QueueConsumer, QueueSubmitter, QueueWorker}; use proxmox_notify::{Context, EmailRecipient, Notification, Recipient, Severity}; use proxmox_schema::ApiType; use proxmox_sys::fs::{CreateOptions, create_path}; +use proxmox_uuid::Uuid; use crate::tape::TapeNotificationMode; use pbs_api_types::{ @@ -17,7 +19,9 @@ use pbs_api_types::{ Notify, SyncJobConfig, TapeBackupJobSetup, User, Userid, VerificationJobConfig, }; -const SPOOL_DIR: &str = concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications"); +const NOTIFICATION_BASE_DIR: &str = + concatcp!(pbs_buildcfg::PROXMOX_BACKUP_STATE_DIR, "/notifications"); +const QUEUE_DIR: &str = concatcp!(NOTIFICATION_BASE_DIR, "/queue"); mod template_data; @@ -35,104 +39,114 @@ pub fn context() -> &'static Context { CONTEXT.get_or_init(|| Context::new(PBSContext)) } -/// Create the directory which will be used to temporarily store notifications -/// which were sent from an unprivileged process. -pub fn create_spool_dir() -> Result<(), Error> { +/// Creates directories used for queuing notifications. +pub fn create_notification_dirs() -> Result<(), Error> { let backup_user = proxmox_product_config::get_api_user(); let opts = CreateOptions::new() .owner(backup_user.uid) .group(backup_user.gid); - create_path(SPOOL_DIR, None, Some(opts))?; + create_path(NOTIFICATION_BASE_DIR, None, Some(opts))?; + + // The queue is writable by anybody in the 'backup' group. + let opts = CreateOptions::new() + .owner(proxmox_product_config::get_priv_user().uid) + .group(backup_user.gid) + .perm(Mode::from_bits_truncate(0o0770)); + + proxmox_sys::fs::ensure_dir_exists(QUEUE_DIR, &opts, true)?; + Ok(()) } -async fn send_queued_notifications() -> Result<(), Error> { - let mut read_dir = tokio::fs::read_dir(SPOOL_DIR).await?; - - let mut notifications = Vec::new(); - - while let Some(entry) = read_dir.next_entry().await? { - let path = entry.path(); - - if let Some(ext) = path.extension() { - if ext == "json" { - let p = path.clone(); - - let bytes = tokio::fs::read(p).await?; - let notification: Notification = serde_json::from_slice(&bytes)?; - notifications.push(notification); - - // Currently, there is no retry-mechanism in case of failure... - // For retries, we'd have to keep track of which targets succeeded/failed - // to send, so we do not retry notifying a target which succeeded before. - tokio::fs::remove_file(path).await?; - } - } - } - - if notifications.is_empty() { - return Ok(()); - } - - // Make sure that we send the oldest notification first - notifications.sort_unstable_by_key(|n| n.timestamp()); - - let context = context(); - - let res = tokio::task::spawn_blocking(move || { - let config = pbs_config::notifications::config(context)?; - for notification in notifications { - if let Err(err) = proxmox_notify::send(context, &config, ¬ification) { - log::error!("failed to send notification: {err}"); - } - } - - Ok::<(), Error>(()) - }) - .await?; - - if let Err(e) = res { - log::error!("could not read notification config: {e}"); - } - - Ok::<(), Error>(()) -} - /// Worker task to periodically send any queued notifications. pub async fn notification_worker() { - loop { - let delay_target = Instant::now() + Duration::from_secs(5); + let consumer = QueueConsumer::new(QUEUE_DIR); - if let Err(err) = send_queued_notifications().await { - log::error!("notification worker task error: {err}"); - } + let worker = QueueWorker::builder(context().clone(), consumer) + .interval(Duration::from_secs(5)) + .retry_attempts(2) + .build(); - tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; + tokio::select! { + _ = worker.run() => {} + _ = migrate_leftover_notifications_loop() => {} } } -fn send_notification(notification: Notification) -> Result<(), Error> { - let context = context(); +/// Periodically move notifications from the old spool directory to the new queue. +/// +/// After an upgrade, proxy processes running the old binary keep spooling notifications there +/// until all of their worker tasks have finished, so checking only at startup is not enough. +/// +/// FIXME: MAJOR VERSION: Remove +async fn migrate_leftover_notifications_loop() { + loop { + match tokio::task::spawn_blocking(migrate_leftover_notifications).await { + Ok(Ok(())) => {} + Ok(Err(err)) => log::error!("failed to migrate notifications: {err:#}"), + Err(err) => log::error!("notification migration task failed: {err}"), + } - if nix::unistd::ROOT == Uid::current() { - let config = pbs_config::notifications::config(context)?; - proxmox_notify::send(context, &config, ¬ification)?; - } else { - let ser = serde_json::to_vec(¬ification)?; - let path = Path::new(SPOOL_DIR).join(format!("{id}.json", id = notification.id())); + tokio::time::sleep(Duration::from_secs(60)).await; + } +} - let backup_user = proxmox_product_config::get_api_user(); - let opts = CreateOptions::new() - .owner(backup_user.uid) - .group(backup_user.gid); - proxmox_sys::fs::replace_file(path, &ser, opts, true)?; - log::info!("queued notification (id={id})", id = notification.id()) +/// Move any left-over notifications from the old spool directory at +/// /var/lib/proxmox-backup/notifications to the new queue +/// at /var/lib/proxmox-backup/notifications/queue +fn migrate_leftover_notifications() -> Result<(), Error> { + let read_dir = std::fs::read_dir(NOTIFICATION_BASE_DIR)?; + + for entry in read_dir.flatten() { + if let Some((name, ext)) = entry.file_name().to_string_lossy().split_once(".") { + if ext == "json" && Uuid::parse_str(name).is_ok() { + let path = entry.path(); + + if let Err(err) = migrate_single_notification(&path) { + log::error!( + "failed to migrate notification at '{p}' to new queue: {err:#}", + p = path.display() + ); + } + } + } } Ok(()) } +fn migrate_single_notification(path: &Path) -> Result<(), Error> { + let bytes = std::fs::read(path)?; + + // Technically, we don't need to parse, but this is just to ensure that + // we don't migrate over something that is not understood by the + // notification worker. + let notification: Notification = serde_json::from_slice(&bytes)?; + + let uuid = notification.id().clone(); + + // Manually move the notification into the new queue directory. + // + // Use a manual rename instead of resubmitting to ensure atomicity. + std::fs::rename( + path, + Path::new(QUEUE_DIR).join(format!("notification-{uuid}")), + )?; + + log::info!("moved notification {uuid} to new notification queue"); + + Ok(()) +} + +fn send_notification(notification: Notification) -> Result<(), Error> { + let options = proxmox_product_config::default_create_options(); + let writer = QueueSubmitter::new(QUEUE_DIR, options); + + writer.submit(¬ification)?; + Ok(()) +} + /// Summary of a successful Tape Job #[derive(Default)] pub struct TapeBackupJobSummary { diff --git a/tests/data/serialized_notification.json b/tests/data/serialized_notification.json new file mode 100644 index 000000000..aedefd455 --- /dev/null +++ b/tests/data/serialized_notification.json @@ -0,0 +1 @@ +{"content":{"template":{"template-name":"gc-ok","data":{"base-url":"https://pbs.proxmox.com:8007","datastore":"store","deduplication-factor":"0.00","disk-bytes":159,"disk-chunks":1,"fqdn":"pbs.proxmox.com","hostname":"pbs","index-data-bytes":0,"index-file-count":0,"pending-bytes":0,"pending-chunks":0,"removed-bad":0,"removed-bytes":0,"removed-chunks":0,"still-bad":0,"upid":"UPID:pbs:00000C78:00189303:00000000:6ADB5D2E:garbage_collection:store:root@pam:"}}},"metadata":{"severity":"info","timestamp":1792761135,"additional-fields":{"type":"gc","hostname":"pbs","datastore":"store"}},"id":"46516f7e-7eb7-43e7-b776-6048f144a6fe"} diff --git a/tests/notifications.rs b/tests/notifications.rs new file mode 100644 index 000000000..a02d13405 --- /dev/null +++ b/tests/notifications.rs @@ -0,0 +1,11 @@ +use proxmox_notify::Notification; + +/// Ensures that a notification serialized before switching to the shared notification worker +/// continues to parse as the `Notification` type from proxmox-notify. +#[test] +fn test_parse_serialized_notification() { + let data = std::fs::read("tests/data/serialized_notification.json").unwrap(); + let notification: Notification = serde_json::from_slice(&data).unwrap(); + + assert_eq!(notification.timestamp(), 1792761135); +} -- 2.47.3