From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id E48F11FF0B7 for ; Fri, 13 Nov 2026 16:17:28 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 5E69F21D09; Fri, 13 Nov 2026 16:10:20 +0100 (CET) From: Lukas Wagner To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox 41/55] notify: queue: add queue consumer Date: Fri, 13 Nov 2026 16:08:14 +0100 Message-ID: <20261113150828.756836-42-l.wagner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113150828.756836-1-l.wagner@proxmox.com> References: <20261113150828.756836-1-l.wagner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794582528219 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.200 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: GEHY7NUDDSZKMJ34JIQD2G2QOUSX2J5Q X-Message-ID-Hash: GEHY7NUDDSZKMJ34JIQD2G2QOUSX2J5Q X-MailFrom: l.wagner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add a QueueConsumer type which allows one to consume queued notifications from the product-specific notification queue. When consuming a notification, it first goes to a 'claimed' state, which atomically renames the notification queue file to record the state change. If the worker is unexpectedly terminated or crashes while sending a notification, the renamed notification file allows us to detect this and attempt recovery by retrying the send. We don't ever want to get caught in a 'send-crash-recover' loop where the same notification is sent over and over again, so a retry counter is also encoded in the file name of the notification. This allows the worker to give up on retrying after a configurable number of attempts. The queue consumer is gated behind the 'queue-worker' flag for now, since the worker will the only consumer for now. Later, with a notification history that would maybe also show queued notifications, a separate feature could be in order. Signed-off-by: Lukas Wagner --- Cargo.toml | 1 + proxmox-notify/Cargo.toml | 4 + proxmox-notify/src/lib.rs | 2 +- proxmox-notify/src/queue/consumer.rs | 418 +++++++++++++++++++++++++++ proxmox-notify/src/queue/mod.rs | 12 + 5 files changed, 436 insertions(+), 1 deletion(-) create mode 100644 proxmox-notify/src/queue/consumer.rs diff --git a/Cargo.toml b/Cargo.toml index ba62378ec..04b535f6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ serde_regex = "1.1" syn = { version = "2", features = [ "full", "visit-mut" ] } sync_wrapper = "1" tar = "0.4" +tempfile = "3.15" termcolor = "1.1.2" thiserror = "2" tokio = "1.6" diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml index a3de1a3fe..e08c0f512 100644 --- a/proxmox-notify/Cargo.toml +++ b/proxmox-notify/Cargo.toml @@ -42,6 +42,7 @@ proxmox-uuid = { workspace = true, features = ["serde"] } [dev-dependencies] pretty_assertions.workspace = true +tempfile.workspace = true [features] default = [] @@ -79,3 +80,6 @@ pve-context = ["dep:proxmox-sys", "legacy-matchers"] pbs-context = ["dep:proxmox-sys", "legacy-matchers"] legacy-matchers = [] queue-submitter = ["dep:proxmox-sys"] +queue-worker = [ + "proxmox-sys/timer", +] diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs index ab5a929c1..6460e69ed 100644 --- a/proxmox-notify/src/lib.rs +++ b/proxmox-notify/src/lib.rs @@ -28,7 +28,7 @@ mod send; #[cfg(feature = "send-impl")] pub use send::send; -#[cfg(feature = "queue-submitter")] +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] pub mod queue; mod error; diff --git a/proxmox-notify/src/queue/consumer.rs b/proxmox-notify/src/queue/consumer.rs new file mode 100644 index 000000000..2defc0c6d --- /dev/null +++ b/proxmox-notify/src/queue/consumer.rs @@ -0,0 +1,418 @@ +//! Consumer side of the notification queue. + +use std::fs::File; +use std::io; +use std::path::PathBuf; +use std::time::Duration; + +use crate::{Error, Notification}; + +use proxmox_uuid::Uuid; + +/// Reads and manages queued notifications in the queue directory. +/// +/// All state changes are done by renaming or removing the notification files, followed by an fsync +/// of the queue directory. +pub struct QueueConsumer { + queue_dir: PathBuf, +} + +impl QueueConsumer { + /// Create a new consumer for the queue directory `base_dir`. + pub fn new>(base_dir: P) -> Self { + Self { + queue_dir: base_dir.into(), + } + } + + /// Try to acquire an exclusive lock on the queue directory without blocking. + /// + /// The lock is held until the returned guard is dropped. Returns `None` if the lock is + /// currently held by someone else. + pub fn try_lock(&self) -> Result, Error> { + let mut dir = File::open(&self.queue_dir).map_err(|err| { + Error::other_with_source("could not open queue directory for locking", err) + })?; + + match proxmox_sys::fs::lock_file(&mut dir, true, Some(Duration::ZERO)) { + Ok(()) => Ok(Some(QueueLockGuard { _dir: dir })), + Err(err) if err.kind() == io::ErrorKind::WouldBlock => Ok(None), + Err(err) => Err(Error::other_with_source( + "could not lock queue directory", + err, + )), + } + } + + /// Claim a notification for sending by incrementing its attempt counter. + /// + /// Returns the updated handle, which must be used for further operations. + pub fn claim(&self, queued: QueuedNotification) -> Result { + let claimed = queued.with_attempt(queued.attempt() + 1); + + let old_path = self.queue_dir.join(queued.filename()); + let new_path = self.queue_dir.join(claimed.filename()); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::other_with_source("could not rename notification file when claiming it", err) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(claimed) + } + + /// Put a claimed notification back into the queue by decrementing its attempt counter. + /// + /// Used when sending failed without notifying any target, so that the attempt does not count + /// towards the retry limit. Returns the updated handle. + pub fn requeue(&self, queued: QueuedNotification) -> Result { + let requeued = queued.with_attempt(queued.attempt().saturating_sub(1)); + + let old_path = self.queue_dir.join(queued.filename()); + let new_path = self.queue_dir.join(requeued.filename()); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::other_with_source("could not rename notification file when requeuing it", err) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(requeued) + } + + /// Remove a notification from the queue. + pub fn resolve(&self, queued: QueuedNotification) -> Result<(), Error> { + let path = self.queue_dir.join(queued.filename()); + + std::fs::remove_file(path) + .map_err(|err| Error::other_with_source("could not remove notification file", err))?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(()) + } + + /// Mark a notification as broken by appending `.broken` to its file name. + /// + /// Broken notifications are ignored by [`QueueConsumer::queued_notifications`], but kept for + /// later inspection. + pub fn mark_broken(&self, queued: QueuedNotification) -> Result<(), Error> { + let old_name = queued.filename(); + let old_path = self.queue_dir.join(old_name); + let new_name = format!("{old_name}.broken"); + let new_path = self.queue_dir.join(new_name); + + std::fs::rename(old_path, new_path).map_err(|err| { + Error::other_with_source( + "could not rename notification file when marking it as broken", + err, + ) + })?; + + super::fsync_queue_dir(&self.queue_dir)?; + + Ok(()) + } + + /// Return all queued notifications, oldest submission first. + /// + /// Files that are not valid queued notifications are skipped. + pub fn queued_notifications(&self) -> Result, Error> { + let read_dir = std::fs::read_dir(&self.queue_dir).map_err(|err| { + Error::other_with_source("could not read notification queue directory", err) + })?; + + let mut queued = Vec::new(); + + for entry in read_dir.flatten() { + let filename = entry.file_name(); + let name = filename.to_string_lossy(); + + let Ok(notification) = QueuedNotification::parse(&name) else { + continue; + }; + + // Renaming a file does not touch its mtime, so it reflects when the notification was + // submitted, regardless of how often it has been claimed or requeued since. Entries + // whose metadata cannot be read have most likely been removed concurrently. + if let Ok(mtime) = entry.metadata().and_then(|metadata| metadata.modified()) { + queued.push((mtime, notification)); + } + } + + queued.sort_by_key(|(mtime, _)| *mtime); + + Ok(queued + .into_iter() + .map(|(_, notification)| notification) + .collect()) + } + + /// Read and deserialize the contents of a queued notification. + pub fn read_notification(&self, queued: &QueuedNotification) -> Result { + let path = self.queue_dir.join(queued.filename()); + + let buf = std::fs::read(path) + .map_err(|err| Error::other_with_source("could not read queued notification", err))?; + + let notification = serde_json::from_slice(&buf).map_err(|err| { + Error::other_with_source("could not deserialize queued notification", err) + })?; + + Ok(notification) + } +} + +/// Exclusive lock on the queue directory, released on drop. +pub struct QueueLockGuard { + _dir: File, +} + +/// A notification in the queue directory, identified by its file name. +/// +/// The file name encodes the whole queue state of a notification. A freshly submitted notification +/// is stored as `notification-`. Claiming it for sending renames it to +/// `notification-.attempt-`. +/// +/// The parser accepts some variations of these names, such as an uppercase UUID or a counter with +/// leading zeros. The actual file name is therefore kept as found, so that it can always be +/// accessed, and only renames write the canonical form. +pub struct QueuedNotification { + id: Uuid, + attempt: Option, + filename: String, +} + +impl QueuedNotification { + fn parse(filename: &str) -> Result { + if let Some(s) = filename.strip_prefix("notification-") { + let (uuid, attempt) = if let Some((uuid_str, attempt_str)) = s.split_once('.') { + let uuid = uuid_str + .parse::() + .map_err(|err| Error::other_with_source("invalid UUID", err))?; + let attempt = attempt_str + .strip_prefix("attempt-") + .ok_or_else(|| Error::other("missing 'attempt-'"))? + .parse::() + .map_err(|err| Error::other_with_source("invalid attempt counter", err))?; + + (uuid, Some(attempt)) + } else { + let uuid = s + .parse::() + .map_err(|err| Error::other_with_source("invalid UUID", err))?; + + (uuid, None) + }; + + Ok(Self { + id: uuid, + attempt, + filename: filename.to_string(), + }) + } else { + Err(Error::other("not a queued notification")) + } + } + + /// Return the same notification with the given attempt counter and the canonical file name + /// for it. + fn with_attempt(&self, attempt: u32) -> Self { + Self { + id: self.id.clone(), + attempt: Some(attempt), + filename: format!("notification-{}.attempt-{attempt}", self.id), + } + } + + /// Return the filename of this queued notification. + fn filename(&self) -> &str { + &self.filename + } + + /// Return the attempt counter. + pub fn attempt(&self) -> u32 { + self.attempt.unwrap_or(0) + } + + /// Return the ID of the notification. + /// + /// The ID is only derived from the filename of the notification, not from + /// the contents. + pub fn id(&self) -> &Uuid { + &self.id + } +} + +#[cfg(all(test, feature = "queue-submitter"))] +mod test { + use std::time::{Duration, SystemTime}; + + use tempfile::tempdir; + + use proxmox_sys::fs::CreateOptions; + + use crate::queue::QueueSubmitter; + + use super::*; + + #[test] + fn test_enqueue() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + let queued = consumer.queued_notifications().unwrap(); + assert!(queued.is_empty()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + + let n = queued.pop().unwrap(); + + assert_eq!(n.attempt(), 0); + let n = consumer.claim(n).unwrap(); + + assert_eq!(notification, consumer.read_notification(&n).unwrap()); + + consumer.resolve(n).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert!(queued.is_empty()); + } + + #[test] + fn test_requeue() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + + let n = queued.pop().unwrap(); + + let n = consumer.claim(n).unwrap(); + let n = consumer.requeue(n).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 0); + + // Requeueing again must cap the attempt counter at 0 + consumer.requeue(n).unwrap(); + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 0); + } + + #[test] + fn test_attempt_counter() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + let notification = Notification::template_builder("foo", ()).build().unwrap(); + + submitter.submit(¬ification).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].attempt(), 0); + consumer.claim(queued.pop().unwrap()).unwrap(); + + let mut queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 1); + let _ = consumer.claim(queued.pop().unwrap()).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued[0].attempt(), 2); + } + + #[test] + fn test_sorted_by_mtime() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new()); + + let notifications: Vec<_> = (0..5) + .map(|_| Notification::template_builder("foo", ()).build().unwrap()) + .collect(); + + let now = SystemTime::now(); + + // Submit in order, but make earlier submissions appear newer + for (i, notification) in notifications.iter().enumerate() { + submitter.submit(notification).unwrap(); + + let path = base_dir + .path() + .join(format!("notification-{}", notification.id())); + let file = std::fs::File::options().write(true).open(path).unwrap(); + file.set_modified(now - Duration::from_secs(i as u64 * 60)) + .unwrap(); + } + + let queued = consumer.queued_notifications().unwrap(); + let queued_ids: Vec<_> = queued.iter().map(|n| n.id().clone()).collect(); + let expected_ids: Vec<_> = notifications.iter().rev().map(|n| n.id().clone()).collect(); + + assert_eq!(queued_ids, expected_ids); + + // Claiming renames the file, which must not affect the order + let oldest = queued.into_iter().next().unwrap(); + consumer.claim(oldest).unwrap(); + + let queued = consumer.queued_notifications().unwrap(); + let queued_ids: Vec<_> = queued.iter().map(|n| n.id().clone()).collect(); + + assert_eq!(queued_ids, expected_ids); + } + + #[test] + fn test_non_canonical_filenames() { + let base_dir = tempdir().unwrap(); + + let consumer = QueueConsumer::new(base_dir.path()); + + let notification = Notification::template_builder("foo", ()).build().unwrap(); + let contents = serde_json::to_vec(¬ification).unwrap(); + + // Uppercase UUID without an attempt counter, and a counter with a leading zero. + let uppercase_id = Uuid::generate(); + let padded_id = Uuid::generate(); + let filenames = [ + format!("notification-{}", uppercase_id.to_string().to_uppercase()), + format!("notification-{padded_id}.attempt-01"), + ]; + for filename in &filenames { + std::fs::write(base_dir.path().join(filename), &contents).unwrap(); + } + + let queued = consumer.queued_notifications().unwrap(); + assert_eq!(queued.len(), 2); + + for n in queued { + assert_eq!(consumer.read_notification(&n).unwrap(), notification); + + let n = consumer.claim(n).unwrap(); + assert_eq!(consumer.read_notification(&n).unwrap(), notification); + + let n = consumer.requeue(n).unwrap(); + consumer.resolve(n).unwrap(); + } + + assert!(consumer.queued_notifications().unwrap().is_empty()); + assert_eq!(std::fs::read_dir(base_dir.path()).unwrap().count(), 0); + } +} diff --git a/proxmox-notify/src/queue/mod.rs b/proxmox-notify/src/queue/mod.rs index 59d5e12cd..7d20bd85e 100644 --- a/proxmox-notify/src/queue/mod.rs +++ b/proxmox-notify/src/queue/mod.rs @@ -1,10 +1,22 @@ +//! File-based notification queue and accompanying worker task. + +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] use std::path::Path; +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] use crate::Error; +#[cfg(feature = "queue-submitter")] mod submitter; +#[cfg(feature = "queue-submitter")] pub use submitter::QueueSubmitter; +#[cfg(feature = "queue-worker")] +mod consumer; +#[cfg(feature = "queue-worker")] +pub use consumer::QueueConsumer; + +#[cfg(any(feature = "queue-submitter", feature = "queue-worker"))] fn fsync_queue_dir(queue_dir: &Path) -> Result<(), Error> { std::fs::File::open(queue_dir) .and_then(|dir| dir.sync_all()) -- 2.47.3