From: Lukas Wagner <l.wagner@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH proxmox v2 41/63] notify: implement notification queue submitter
Date: Wed, 18 Nov 2026 10:43:25 +0100 [thread overview]
Message-ID: <20261118094348.273646-42-l.wagner@proxmox.com> (raw)
In-Reply-To: <20261118094348.273646-1-l.wagner@proxmox.com>
Each product will be switched to a queue/worker approach, where each
sender simply drops the notification in serialized form into a queue
directory. Each notification has a unique UUID which is used as the
filename, making filename collisions not an issue in practice.
The new QueueSubmitter has a single method `submit`, which serialized
the notification and atomically places it in the pre-existing queue
directory.
Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
---
Cargo.toml | 1 +
proxmox-notify/Cargo.toml | 2 +
proxmox-notify/src/error.rs | 16 +++
proxmox-notify/src/lib.rs | 3 +
proxmox-notify/src/queue/mod.rs | 18 +++
proxmox-notify/src/queue/submitter.rs | 171 ++++++++++++++++++++++++++
6 files changed, 211 insertions(+)
create mode 100644 proxmox-notify/src/queue/mod.rs
create mode 100644 proxmox-notify/src/queue/submitter.rs
diff --git a/Cargo.toml b/Cargo.toml
index 091f585d1..c1ce30ee0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -146,6 +146,7 @@ signal-hook-registry = "1"
syn = { version = "2", features = [ "full", "visit-mut" ] }
sync_wrapper = "1"
tar = "0.4"
+tempfile = "3.15"
termcolor = "1.1.2"
thiserror = "2"
tokio = "1.6"
diff --git a/proxmox-notify/Cargo.toml b/proxmox-notify/Cargo.toml
index e47912d11..4cfb1f2eb 100644
--- a/proxmox-notify/Cargo.toml
+++ b/proxmox-notify/Cargo.toml
@@ -42,6 +42,7 @@ proxmox-uuid = { workspace = true, features = ["serde"] }
[dev-dependencies]
pretty_assertions.workspace = true
+tempfile.workspace = true
[features]
default = []
@@ -78,3 +79,4 @@ send-impl = [
pve-context = ["dep:proxmox-sys", "legacy-matchers"]
pbs-context = ["dep:proxmox-sys", "legacy-matchers"]
legacy-matchers = []
+queue-submitter = ["dep:proxmox-sys"]
diff --git a/proxmox-notify/src/error.rs b/proxmox-notify/src/error.rs
index c8747f39f..bab240506 100644
--- a/proxmox-notify/src/error.rs
+++ b/proxmox-notify/src/error.rs
@@ -56,6 +56,14 @@ pub enum Error {
source: Option<BoxedError>,
},
+ #[error("notification queue error: {err}", err = print_optional_error(.message, .source))]
+ Queue {
+ /// Description of the error.
+ message: String,
+ /// Underlying error, if any.
+ source: Option<BoxedError>,
+ },
+
/// The notification configuration could not be serialized.
#[error("could not serialize config: {0}")]
ConfigSerialization(#[source] BoxedError),
@@ -155,6 +163,14 @@ impl Error {
}
}
+ /// Create an [`Error::Queue`] with an underlying error.
+ pub(crate) fn queue_with_source(s: impl Into<String>, source: impl Into<BoxedError>) -> Self {
+ Self::Queue {
+ message: s.into(),
+ source: Some(source.into()),
+ }
+ }
+
/// Create an [`Error::ConfigDeserialization`] from the given error.
pub(crate) fn config_deserialization(err: impl Into<BoxedError>) -> Self {
Self::ConfigDeserialization(err.into())
diff --git a/proxmox-notify/src/lib.rs b/proxmox-notify/src/lib.rs
index 4304e2977..ab5a929c1 100644
--- a/proxmox-notify/src/lib.rs
+++ b/proxmox-notify/src/lib.rs
@@ -28,6 +28,9 @@ mod send;
#[cfg(feature = "send-impl")]
pub use send::send;
+#[cfg(feature = "queue-submitter")]
+pub mod queue;
+
mod error;
pub use error::Error;
diff --git a/proxmox-notify/src/queue/mod.rs b/proxmox-notify/src/queue/mod.rs
new file mode 100644
index 000000000..25137e123
--- /dev/null
+++ b/proxmox-notify/src/queue/mod.rs
@@ -0,0 +1,18 @@
+use std::path::Path;
+
+use crate::Error;
+
+mod submitter;
+pub use submitter::QueueSubmitter;
+
+/// Maximum size of a queued notification file.
+///
+/// Well above what legitimate producers submit, e.g. forwarded mails or backup logs limited to
+/// 1 MiB, but bounds the memory a bogus file written directly to the queue can consume.
+const MAX_NOTIFICATION_SIZE: u64 = 16 * 1024 * 1024;
+
+fn fsync_queue_dir(queue_dir: &Path) -> Result<(), Error> {
+ std::fs::File::open(queue_dir)
+ .and_then(|dir| dir.sync_all())
+ .map_err(|err| Error::queue_with_source("fsync of queue directory failed", err))
+}
diff --git a/proxmox-notify/src/queue/submitter.rs b/proxmox-notify/src/queue/submitter.rs
new file mode 100644
index 000000000..3f96a2152
--- /dev/null
+++ b/proxmox-notify/src/queue/submitter.rs
@@ -0,0 +1,171 @@
+//! Submission of notifications to the queue.
+
+use std::path::PathBuf;
+
+use proxmox_sys::fs::CreateOptions;
+
+use crate::{Error, Notification};
+
+use super::MAX_NOTIFICATION_SIZE;
+
+/// Writes notifications to the queue directory.
+pub struct QueueSubmitter {
+ queue_dir: PathBuf,
+ create_options: CreateOptions,
+}
+
+impl QueueSubmitter {
+ /// Create a new submitter for the queue directory `base_dir`.
+ ///
+ /// Queued notification files are created with `create_options`.
+ pub fn new<P: Into<PathBuf>>(base_dir: P, create_options: CreateOptions) -> Self {
+ Self {
+ queue_dir: base_dir.into(),
+ create_options,
+ }
+ }
+
+ /// Add a notification to the queue.
+ ///
+ /// The file is written atomically and the queue directory is synced before returning.
+ pub fn submit(&self, notification: &Notification) -> Result<(), Error> {
+ let filename = format!("notification-{}", notification.id());
+ let path = self.queue_dir.join(filename);
+
+ let bytes = serde_json::to_vec(notification).map_err(|err| {
+ Error::other_with_source(
+ format!("failed to serialize notification {}", notification.id()),
+ err,
+ )
+ })?;
+
+ let serialized_size = bytes.len();
+ if serialized_size as u64 > MAX_NOTIFICATION_SIZE {
+ return Err(Error::invalid_notification(format!(
+ "serialized notification too large ({serialized_size} > {MAX_NOTIFICATION_SIZE} bytes)"
+ )));
+ }
+
+ proxmox_sys::fs::replace_file(&path, &bytes, self.create_options, true).map_err(|err| {
+ Error::queue_with_source(
+ format!(
+ "failed to store serialized notification {}",
+ notification.id()
+ ),
+ err,
+ )
+ })?;
+
+ super::fsync_queue_dir(&self.queue_dir)?;
+
+ tracing::info!(
+ "submitted notification {} to notification queue",
+ notification.id()
+ );
+
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use proxmox_uuid::Uuid;
+ use serde_json::json;
+ use tempfile::tempdir;
+
+ use crate::notification::Content;
+
+ use super::*;
+
+ /// Build a notification that serializes to exactly `size` bytes.
+ fn notification_with_size(size: u64) -> Notification {
+ let mut notification = Notification::template_builder("foo", json!({ "pad": "" }))
+ .build()
+ .unwrap();
+
+ let base_size = serde_json::to_vec(¬ification).unwrap().len() as u64;
+ let Content::Template { data, .. } = &mut notification.content else {
+ unreachable!();
+ };
+ data["pad"] = "a".repeat((size - base_size) as usize).into();
+
+ assert_eq!(
+ serde_json::to_vec(¬ification).unwrap().len() as u64,
+ size
+ );
+
+ notification
+ }
+
+ #[test]
+ fn test_submit_multiple() {
+ let base_dir = tempdir().unwrap();
+ let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new());
+
+ let notifications: Vec<_> = (0..3)
+ .map(|_| Notification::template_builder("foo", ()).build().unwrap())
+ .collect();
+
+ for notification in ¬ifications {
+ submitter.submit(notification).unwrap();
+ }
+
+ let entries: Vec<_> = std::fs::read_dir(base_dir.path())
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ assert_eq!(entries.len(), notifications.len());
+
+ for entry in entries {
+ let filename = entry.file_name().into_string().unwrap();
+ let id: Uuid = filename
+ .strip_prefix("notification-")
+ .unwrap()
+ .parse()
+ .unwrap();
+
+ let submitted = notifications.iter().find(|n| n.id() == &id).unwrap();
+
+ let content = std::fs::read(entry.path()).unwrap();
+ let queued: Notification = serde_json::from_slice(&content).unwrap();
+ assert_eq!(&queued, submitted);
+ }
+ }
+
+ #[test]
+ fn test_submit_missing_queue_dir() {
+ let base_dir = tempdir().unwrap();
+ let submitter = QueueSubmitter::new(base_dir.path().join("missing"), CreateOptions::new());
+
+ let notification = Notification::template_builder("foo", ()).build().unwrap();
+ assert!(submitter.submit(¬ification).is_err());
+ }
+
+ #[test]
+ fn test_submit_max_size() {
+ let base_dir = tempdir().unwrap();
+ let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new());
+
+ let notification = notification_with_size(MAX_NOTIFICATION_SIZE);
+ submitter.submit(¬ification).unwrap();
+
+ let path = base_dir
+ .path()
+ .join(format!("notification-{}", notification.id()));
+ assert_eq!(
+ std::fs::metadata(path).unwrap().len(),
+ MAX_NOTIFICATION_SIZE
+ );
+ }
+
+ #[test]
+ fn test_submit_oversized() {
+ let base_dir = tempdir().unwrap();
+ let submitter = QueueSubmitter::new(base_dir.path(), CreateOptions::new());
+
+ let notification = notification_with_size(MAX_NOTIFICATION_SIZE + 1);
+ assert!(submitter.submit(¬ification).is_err());
+
+ assert_eq!(std::fs::read_dir(base_dir.path()).unwrap().count(), 0);
+ }
+}
--
2.47.3
next prev parent reply other threads:[~2026-11-18 9:49 UTC|newest]
Thread overview: 66+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 9:42 [PATCH many v2 00/63] notify: maintenance, PDM preparations, queue + worker Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 01/63] notify: disallow `unsafe` code Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 02/63] notify: refactor error type Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 03/63] notify: move `Endpoint` trait to endpoint module as pub(crate) Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 04/63] notify: use ConfigDigest instead of raw byte arrays Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 05/63] notify: remove static context and pass context as parameter Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 06/63] notify: matcher: drop unused field matcher verifiers Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 07/63] notify: introduce 'send-impl', 'api-impl' and 'api-types' features Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 08/63] notify: matcher: make matcher validation methods public Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 09/63] notify: matcher: stub CalendarMatcher::matches on wasm32 Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 10/63] notify: make SendmailEndpoint and SmtpEndpoint private types Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 11/63] notify: move `send`, `test_target` and `Bus` to new `send` module Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 12/63] notify: move Origin type to api_types module Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 13/63] notify: move Target " Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 14/63] notify: move Config to config module Lukas Wagner
2026-11-18 9:42 ` [PATCH proxmox v2 15/63] notify: move Notification and sub-types to `notification` module Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 16/63] notify: move schema module to api_types Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 17/63] notify: move check_matches from api_types::matcher to send::Bus Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 18/63] notify: move TemplateSource to context module Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 19/63] notify: config: cleanup helper names and visibility Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 20/63] notify: config: rename section config parser/writer methods Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 21/63] notify: config: pass builtins explicitly when parsing section config Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 22/63] notify: implement PartialEq for all Config types Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 23/63] notify: use typed SectionConfigData Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 24/63] notify: api: api types: rename Deleteable* -> Deletable* Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 25/63] notify: context: parse builtin config only once Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 26/63] notify: context: parse proxy config in trait impl, not at the callsite Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 27/63] notify: gate sub-modules with the appropriate feature flags Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 28/63] notify: fold mail-forwarder feature into send-impl Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 29/63] notify: forwarded mails: don't hardcode severity and metadata fields Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 30/63] notify: don't pass uid when instantiating forwarded mail notifications Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 31/63] notify: derive PartialEq for Notification type Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 32/63] notify: add getters for severity and fields on " Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 33/63] notify: add test suite for " Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 34/63] notify: serialize raw, forwarded mail content as base64 Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 35/63] notify: unify code paths for legacy-sendmail and regular notifications Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 36/63] notify: add backwards-compat note to Notification type Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 37/63] notify: remove 'test_target' API Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 38/63] notify: replace notification instantiation with builder pattern Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 39/63] notify: add send receipts Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 40/63] notify: make sure that template name matches SAFE_ID_REGEX Lukas Wagner
2026-11-18 9:43 ` Lukas Wagner [this message]
2026-11-18 9:43 ` [PATCH proxmox v2 42/63] notify: context: add read_config Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 43/63] notify: queue: add queue consumer Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox v2 44/63] notify: queue: add worker task Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-backup v2 45/63] notifications: don't ignore verification errors in legacy-sendmail mode Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-backup v2 46/63] notifications: adapt to proxmox-notify API changes Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-backup v2 47/63] api: notification: send test notification via proxmox_notify::send Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-backup v2 48/63] notifications: send all notifications via proxmox-notify's queue Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-perl-rs v2 49/63] pve: notify: adapt to changes in the public API of proxmox-notify Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-perl-rs v2 50/63] pve: notify: add `PVE::RS::Notify::enqueue` method Lukas Wagner
2026-11-18 9:43 ` [PATCH pve-cluster v2 51/63] notify: send notifications via notification queue Lukas Wagner
2026-11-18 9:43 ` [PATCH manager v2 52/63] vzdump: send legacy-sendmail notifications via simplified API Lukas Wagner
2026-11-18 9:43 ` [PATCH manager v2 53/63] debian: create notification queue directory via tmpfiles Lukas Wagner
2026-11-18 9:43 ` [PATCH manager v2 54/63] debian: add 'pve-notify-worker' as dependecy Lukas Wagner
2026-11-18 9:43 ` [PATCH manager v2 55/63] api: services: add 'pve-notify-worker' as known service Lukas Wagner
2026-11-18 9:43 ` [PATCH manager v2 56/63] services: request pve-notify-worker.service via Wants Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 57/63] forward mails via notification queue Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 58/63] use return code to signal permanent or temporary failure Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 59/63] limit size of mail read from stdin Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 60/63] cargo: move mail forwarder into a workspace member crate Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 61/63] add crate for code shared between the mail-forwarder and the worker Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 62/63] add pve-notify-worker crate and package Lukas Wagner
2026-11-18 9:43 ` [PATCH proxmox-mail-forward v2 63/63] debian: add versioned break for pve-manager and proxmox-backup-server Lukas Wagner
2026-11-18 21:44 ` [PATCH many v2 00/63] notify: maintenance, PDM preparations, queue + worker Thomas Lamprecht
2026-11-19 10:45 ` Lukas Wagner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261118094348.273646-42-l.wagner@proxmox.com \
--to=l.wagner@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox