From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id A9CA11FF0C0 for ; Wed, 18 Nov 2026 10:53:14 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 49C5721F2D; Wed, 18 Nov 2026 10:45:16 +0100 (CET) From: Lukas Wagner To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox v2 43/63] notify: queue: add queue consumer Date: Wed, 18 Nov 2026 10:43:27 +0100 Message-ID: <20261118094348.273646-44-l.wagner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118094348.273646-1-l.wagner@proxmox.com> References: <20261118094348.273646-1-l.wagner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794995040451 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.187 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: U2EX2XYNUHNCOO4JQDOMTJHCZHKJIYME X-Message-ID-Hash: U2EX2XYNUHNCOO4JQDOMTJHCZHKJIYME X-MailFrom: l.wagner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a QueueConsumer type which allows one to consume queued notifications from the product-specific notification queue. When consuming a notification, it first goes to a 'claimed' state, which atomically renames the notification queue file to record the state change. If the worker is unexpectedly terminated or crashes while a notification is being sent, the renamed notification file allows us to detect this and attempt recovery by retrying the send. We don't ever want to get caught in a 'send-crash-recover' loop where the same notification is sent over and over again, so a retry counter is also encoded in the file name of the notification. This allows the worker to give up on retrying after a configurable number of attempts. The queue consumer is gated behind the 'queue-worker' flag for now, since the worker will the only consumer for now. Later, with a notification history that would maybe also show queued notifications, a separate feature could be in order. Signed-off-by: Lukas Wagner --- proxmox-notify/Cargo.toml | 6 + proxmox-notify/src/error.rs | 28 ++ proxmox-notify/src/lib.rs | 2 +- proxmox-notify/src/queue/consumer.rs | 513 +++++++++++++++++++++++++++ proxmox-notify/src/queue/mod.rs | 12 + 5 files changed, 560 insertions(+), 1 deletion(-) create mode 100644 proxmox-notify/src/queue/consumer.rs diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml index 4cfb1f2eb..fbdd5e601 100644 --- a/proxmox-notify/Cargo.toml +++ b/proxmox-notify/Cargo.toml @@ -16,6 +16,7 @@ const_format.workspace = true handlebars = { workspace = true, optional = true } http = { workspace = true, optional = true } lettre = { workspace = true, optional = true } +libc = { workspace = true, optional = true } tracing.workspace = true mail-parser = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } @@ -41,6 +42,7 @@ proxmox-time.workspace = true proxmox-uuid = { workspace = true, features = ["serde"] } [dev-dependencies] +nix = { workspace = true, features = ["fs"] } pretty_assertions.workspace = true tempfile.workspace = true @@ -80,3 +82,7 @@ pve-context = ["dep:proxmox-sys", "legacy-matchers"] pbs-context = ["dep:proxmox-sys", "legacy-matchers"] legacy-matchers = [] queue-submitter = ["dep:proxmox-sys"] +queue-worker = [ + "dep:libc", + "proxmox-sys/timer", +] diff --git a/proxmox-notify/src/error.rs b/proxmox-notify/src/error.rs index bab240506..98e08a898 100644 --- a/proxmox-notify/src/error.rs +++ b/proxmox-notify/src/error.rs @@ -56,6 +56,15 @@ pub enum Error { source: Option, }, + /// A queued notification is invalid. + #[error("invalid file in notification queue: {err}", err = print_optional_error(.message, .source))] + InvalidQueuedNotification { + /// Description of the error. + message: String, + /// Underlying error, if any. + source: Option, + }, + #[error("notification queue error: {err}", err = print_optional_error(.message, .source))] Queue { /// Description of the error. @@ -163,6 +172,25 @@ impl Error { } } + /// Create an [`Error::InvalidQueuedNotification`] without an underlying error. + pub(crate) fn invalid_queued_notification(s: impl Into) -> Self { + Self::InvalidQueuedNotification { + message: s.into(), + source: None, + } + } + + /// Create an [`Error::InvalidQueuedNotification`] with an underlying error. + pub(crate) fn invalid_queued_notification_with_source( + s: impl Into, + source: impl Into, + ) -> Self { + Self::InvalidQueuedNotification { + message: s.into(), + source: Some(source.into()), + } + } + /// Create an [`Error::Queue`] with an underlying error. pub(crate) fn queue_with_source(s: impl Into, source: impl Into) -> Self { Self::Queue { diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs index ab5a929c1..6460e69ed 100644 --- a/proxmox-notify/src/lib.rs +++ b/proxmox-notify/src/lib.rs @@ -28,7 +28,7 @@ mod send; #[cfg(feature = "send-impl")] pub use send::send; -#[cfg(feature = "queue-submitter")] +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] pub mod queue; mod error; diff --git a/proxmox-notify/src/queue/consumer.rs b/proxmox-notify/src/queue/consumer.rs new file mode 100644 index 000000000..2ee8a9b8c --- /dev/null +++ b/proxmox-notify/src/queue/consumer.rs @@ -0,0 +1,513 @@ +//! Consumer side of the notification queue. + +use std::fs::File; +use std::io::{self, Read}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::PathBuf; +use std::time::Duration; + +use crate::{Error, Notification}; + +use super::MAX_NOTIFICATION_SIZE; + +use proxmox_uuid::Uuid; + +/// Reads and manages queued notifications in the queue directory. +/// +/// All state changes are done by renaming or removing the notification files, followed by an fsync +/// of the queue directory. +pub struct QueueConsumer { + queue_dir: PathBuf, +} + +impl QueueConsumer { + /// Create a new consumer for the queue directory `base_dir`. + pub fn new>(base_dir: P) -> Self { + Self { + queue_dir: base_dir.into(), + } + } + + /// Try to acquire an exclusive lock on the queue directory without blocking. + /// + /// The lock is held until the returned guard is dropped. Returns `None` if the lock is + /// currently held by someone else. + pub fn try_lock(&self) -> Result, Error> { + let mut dir = File::open(&self.queue_dir).map_err(|err| { + Error::queue_with_source("could not open queue directory for locking", err) + })?; + + match proxmox_sys::fs::lock_file(&mut dir, true, Some(Duration::ZERO)) { + Ok(()) => Ok(Some(QueueLockGuard { _dir: dir })), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => Ok(None), + Err(err) => Err(Error::queue_with_source( + "could not lock queue directory", + err, + )), + } + } + + /// Claim a notification for sending by incrementing its attempt counter. + /// + /// Returns the updated handle, which must be used for further operations. + pub fn claim(&self, queued: QueuedNotification) -> Result { + let claimed = queued.with_attempt(queued.attempt() + 1); + + let old_path = self.queue_dir.join(queued.filename()); + let new_path = self.queue_dir.join(claimed.filename()); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::queue_with_source("could not rename notification file when claiming it", err) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(claimed) + } + + /// Put a claimed notification back into the queue by decrementing its attempt counter. + /// + /// Used when sending failed without notifying any target, so that the attempt does not count + /// towards the retry limit. Returns the updated handle. + pub fn requeue(&self, queued: QueuedNotification) -> Result { + let requeued = queued.with_attempt(queued.attempt().saturating_sub(1)); + + let old_path = self.queue_dir.join(queued.filename()); + let new_path = self.queue_dir.join(requeued.filename()); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::queue_with_source("could not rename notification file when requeuing it", err) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(requeued) + } + + /// Remove a notification from the queue. + pub fn resolve(&self, queued: QueuedNotification) -> Result<(), Error> { + let path = self.queue_dir.join(queued.filename()); + + std::fs::remove_file(path) + .map_err(|err| Error::queue_with_source("could not remove notification file", err))?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(()) + } + + /// Mark a notification as broken by appending `.broken` to its file name. + /// + /// Broken notifications are ignored by [`QueueConsumer::queued_notifications`], but kept for + /// later inspection. + pub fn mark_broken(&self, queued: QueuedNotification) -> Result<(), Error> { + let old_name = queued.filename(); + let old_path = self.queue_dir.join(old_name); + let new_name = format!("{old_name}.broken"); + let new_path = self.queue_dir.join(new_name); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::queue_with_source( + "could not rename notification file when marking it as broken", + err, + ) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(()) + } + + /// Return all queued notifications, oldest submission first. + /// + /// Files that are not valid queued notifications are skipped. + pub fn queued_notifications(&self) -> Result, Error> { + let read_dir = std::fs::read_dir(&self.queue_dir).map_err(|err| { + Error::queue_with_source("could not read notification queue directory", err) + })?; + + let mut queued = Vec::new(); + + for entry in read_dir.flatten() { + let filename = entry.file_name(); + let name = filename.to_string_lossy(); + + let Some(notification) = QueuedNotification::parse(&name) else { + continue; + }; + + // Renaming a file does not touch its mtime, so it reflects when the notification was + // submitted, regardless of how often it has been claimed or requeued since. Entries + // whose metadata cannot be read have most likely been removed concurrently. + if let Ok(mtime) = entry.metadata().and_then(|metadata| metadata.modified()) { + queued.push((mtime, notification)); + } + } + + queued.sort_by_key(|(mtime, _)| *mtime); + + Ok(queued + .into_iter() + .map(|(_, notification)| notification) + .collect()) + } + + /// Read and deserialize the contents of a queued notification. + /// + /// Fails if the file is a symlink, not a regular file, or larger than + /// [`MAX_NOTIFICATION_SIZE`]. + pub fn read_notification(&self, queued: &QueuedNotification) -> Result { + let path = self.queue_dir.join(queued.filename()); + + // Anybody allowed to submit can write to the queue directory directly, so neither follow + // symlinks nor block on opening a FIFO. The file type is checked on the opened file, as + // checking the path beforehand would race with the file being replaced. + let file = File::options() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .map_err(|err| Error::queue_with_source("could not open queued notification", err))?; + + let metadata = file + .metadata() + .map_err(|err| Error::queue_with_source("could not stat queued notification", err))?; + + if !metadata.file_type().is_file() { + return Err(Error::invalid_queued_notification("not a regular file")); + } + + let mut buf = Vec::new(); + // Read one additional byte to detect whether the file exceeds the limit. + file.take(MAX_NOTIFICATION_SIZE + 1) + .read_to_end(&mut buf) + .map_err(|err| Error::queue_with_source("could not read queued notification", err))?; + + if buf.len() as u64 > MAX_NOTIFICATION_SIZE { + return Err(Error::invalid_queued_notification(format!( + "exceeds maximum size of {MAX_NOTIFICATION_SIZE} bytes" + ))); + } + + let notification = serde_json::from_slice(&buf).map_err(|err| { + Error::invalid_queued_notification_with_source("could not deserialize JSON", err) + })?; + + Ok(notification) + } +} + +/// Exclusive lock on the queue directory, released on drop. +pub struct QueueLockGuard { + _dir: File, +} + +/// A notification in the queue directory, identified by its file name. +/// +/// The file name encodes the whole queue state of a notification. A freshly submitted notification +/// is stored as `notification-`. Claiming it for sending renames it to +/// `notification-.attempt-`. +/// +/// The parser accepts some variations of these names, such as an uppercase UUID or a counter with +/// leading zeros. The actual file name is therefore kept as found, so that it can always be +/// accessed, and only renames write the canonical form. +pub struct QueuedNotification { + id: Uuid, + attempt: Option, + filename: String, +} + +impl QueuedNotification { + fn parse(filename: &str) -> Option { + if let Some(s) = filename.strip_prefix("notification-") { + let (uuid, attempt) = if let Some((uuid_str, attempt_str)) = s.split_once('.') { + let uuid = uuid_str.parse::().ok()?; + let attempt = attempt_str.strip_prefix("attempt-")?.parse::().ok()?; + + (uuid, Some(attempt)) + } else { + let uuid = s.parse::().ok()?; + + (uuid, None) + }; + + Some(Self { + id: uuid, + attempt, + filename: filename.to_string(), + }) + } else { + None + } + } + + /// Return the same notification with the given attempt counter and the canonical file name + /// for it. + fn with_attempt(&self, attempt: u32) -> Self { + Self { + id: self.id.clone(), + attempt: Some(attempt), + filename: format!("notification-{}.attempt-{attempt}", self.id), + } + } + + /// Return the filename of this queued notification. + fn filename(&self) -> &str { + &self.filename + } + + /// Return the attempt counter. + pub fn attempt(&self) -> u32 { + self.attempt.unwrap_or(0) + } + + /// Return the ID of the notification. + /// + /// The ID is only derived from the filename of the notification, not from + /// the contents. + pub fn id(&self) -> &Uuid { + &self.id + } +} + +#[cfg(all(test, feature = "queue-submitter"))] +mod test { + use std::time::{Duration, SystemTime}; + + use tempfile::tempdir; + + use proxmox_sys::fs::CreateOptions; + + use crate::queue::QueueSubmitter; + + use super::*; + + #[test] + fn test_enqueue() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + let queued = consumer.queued_notifications().unwrap(); + assert!(queued.is_empty()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + + let n = queued.pop().unwrap(); + + assert_eq!(n.attempt(), 0); + let n = consumer.claim(n).unwrap(); + + assert_eq!(notification, consumer.read_notification(&n).unwrap()); + + consumer.resolve(n).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert!(queued.is_empty()); + } + + #[test] + fn test_requeue() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + + let n = queued.pop().unwrap(); + + let n = consumer.claim(n).unwrap(); + let n = consumer.requeue(n).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 0); + + // Requeueing again must cap the attempt counter at 0 + consumer.requeue(n).unwrap(); + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 0); + } + + #[test] + fn test_attempt_counter() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].attempt(), 0); + consumer.claim(queued.pop().unwrap()).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 1); + let _ = consumer.claim(queued.pop().unwrap()).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 2); + } + + #[test] + fn test_sorted_by_mtime() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + + let notifications: Vec<_> = (0..5) + .map(|_| Notification::template_builder("foo", ()).build().unwrap()) + .collect(); + + let now = SystemTime::now(); + + // Submit in order, but make earlier submissions appear newer + for (i, notification) in notifications.iter().enumerate() { + submitter.submit(notification).unwrap(); + + let path = base_dir + .path() + .join(format!("notification-{}", notification.id())); + let file = std::fs::File::options().write(true).open(path).unwrap(); + file.set_modified(now - Duration::from_secs(i as u64 * 60)) + .unwrap(); + } + + let queued = consumer.queued_notifications().unwrap(); + let queued_ids: Vec<_> = queued.iter().map(|n| n.id().clone()).collect(); + let expected_ids: Vec<_> = notifications.iter().rev().map(|n| n.id().clone()).collect(); + + assert_eq!(queued_ids, expected_ids); + + // Claiming renames the file, which must not affect the order + let oldest = queued.into_iter().next().unwrap(); + consumer.claim(oldest).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + let queued_ids: Vec<_> = queued.iter().map(|n| n.id().clone()).collect(); + + assert_eq!(queued_ids, expected_ids); + } + + #[test] + fn test_non_canonical_filenames() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let notification = Notification::template_builder("foo", ()).build().unwrap(); + let contents = serde_json::to_vec(¬ification).unwrap(); + + // Uppercase UUID without an attempt counter, and a counter with a leading zero. + let uppercase_id = Uuid::generate(); + let padded_id = Uuid::generate(); + let filenames = [ + format!("notification-{}", uppercase_id.to_string().to_uppercase()), + format!("notification-{padded_id}.attempt-01"), + ]; + for filename in &filenames { + std::fs::write(base_dir.path().join(filename), &contents).unwrap(); + } + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 2); + + for n in queued { + assert_eq!(consumer.read_notification(&n).unwrap(), notification); + + let n = consumer.claim(n).unwrap(); + assert_eq!(consumer.read_notification(&n).unwrap(), notification); + + let n = consumer.requeue(n).unwrap(); + consumer.resolve(n).unwrap(); + } + + assert!(consumer.queued_notifications().unwrap().is_empty()); + assert_eq!(std::fs::read_dir(base_dir.path()).unwrap().count(), 0); + } + + fn single_queued(consumer: &QueueConsumer) -> QueuedNotification { + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + queued.pop().unwrap() + } + + #[test] + fn test_read_symlink() { + let base_dir = tempdir().unwrap(); + let other_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let notification = Notification::template_builder("foo", ()).build().unwrap(); + let target = other_dir.path().join("target"); + std::fs::write(&target, serde_json::to_vec(¬ification).unwrap()).unwrap(); + + let link = base_dir + .path() + .join(format!("notification-{}", notification.id())); + std::os::unix::fs::symlink(&target, link).unwrap(); + + let n = single_queued(&consumer); + assert!(consumer.read_notification(&n).is_err()); + } + + #[test] + fn test_read_fifo() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let path = base_dir + .path() + .join(format!("notification-{}", Uuid::generate())); + nix::unistd::mkfifo( + &path, + nix::sys::stat::Mode::S_IRUSR | nix::sys::stat::Mode::S_IWUSR, + ) + .unwrap(); + + // Without a writer, a blocking open would hang here. + let n = single_queued(&consumer); + assert!(consumer.read_notification(&n).is_err()); + } + + #[test] + fn test_read_oversized() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let notification = Notification::template_builder("foo", ()).build().unwrap(); + let path = base_dir + .path() + .join(format!("notification-{}", notification.id())); + + // Trailing whitespace keeps the contents valid JSON, so only the size limit can reject it. + let mut contents = serde_json::to_vec(¬ification).unwrap(); + contents.resize(MAX_NOTIFICATION_SIZE as usize, b' '); + std::fs::write(&path, &contents).unwrap(); + + let n = single_queued(&consumer); + assert_eq!(consumer.read_notification(&n).unwrap(), notification); + + contents.push(b' '); + std::fs::write(&path, &contents).unwrap(); + + assert!(consumer.read_notification(&n).is_err()); + } +} diff --git a/proxmox-notify/src/queue/mod.rs b/proxmox-notify/src/queue/mod.rs index 25137e123..11317aa04 100644 --- a/proxmox-notify/src/queue/mod.rs +++ b/proxmox-notify/src/queue/mod.rs @@ -1,16 +1,28 @@ +//! File-based notification queue and accompanying worker task. + +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] use std::path::Path; +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] use crate::Error; +#[cfg(feature = "queue-submitter")] mod submitter; +#[cfg(feature = "queue-submitter")] pub use submitter::QueueSubmitter; +#[cfg(feature = "queue-worker")] +mod consumer; +#[cfg(feature = "queue-worker")] +pub use consumer::QueueConsumer; + /// Maximum size of a queued notification file. /// /// Well above what legitimate producers submit, e.g. forwarded mails or backup logs limited to /// 1 MiB, but bounds the memory a bogus file written directly to the queue can consume. const MAX_NOTIFICATION_SIZE: u64 = 16 * 1024 * 1024; +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] fn fsync_queue_dir(queue_dir: &Path) -> Result<(), Error> { std::fs::File::open(queue_dir) .and_then(|dir| dir.sync_all()) -- 2.47.3