From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id E5CB21FF0B7 for ; Fri, 13 Nov 2026 10:06:28 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3B3F121820; Fri, 13 Nov 2026 10:06:07 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=stuttgart.de; s=s1022; t=1794560285; bh=n9X7YlRNiPD395fGuY4m6Kq20GDb0EBs/FACS8JkeCk=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=k/6cJb6G9IRzDursB+CAGcAeriot5TngxsbdGu1U6+U2AwBvd+ggqQdnVYXxk4jSX URRsSjR+mEbXl3CjuU0WF+eeD3XWu5WHSHHEQBOHmi2kpZzDKrH0LOYbbfMptVzMZH 1fOuqC44+So73aq+maEVzNvoy7wMFTKmnxfL1clEgxBS01G9UeAKFeImWqIoXQRrxa TwlOHK9GH9RXTo7THspTwuowbIaNajG6qzXH7afuBfKfWO3V3klhOold1R2n0tBb5H OtWDJryWDfkqn0Wk2KGrDPTkNyxizXVhDzKab6PpFQjilj8cRawifRiekvTbN2xMXQ swTN2K/tqeaww== From: Jakob-Tobias Winter To: pbs-devel@lists.proxmox.com Subject: [RFC PATCH proxmox 02/12] pbs-api-types: tape: add attribute filter to backup jobs Date: Fri, 13 Nov 2026 09:56:29 +0100 Message-ID: <20261113085637.57360-5-tobias@stuttgart.de> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20261113085637.57360-3-tobias@stuttgart.de> References: <20261113085637.57360-3-tobias@stuttgart.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.022 Adjusted score from AWL reputation of From: address DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_PASS -0.001 SPF: HELO matches SPF record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 6ODMSHQT52CIWUPD2B3MZTP4COQ2PPDN X-Message-ID-Hash: 6ODMSHQT52CIWUPD2B3MZTP4COQ2PPDN X-MailFrom: prvs=d19b210678=tobias@stuttgart.de X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Jakob-Tobias Winter X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Tape backup jobs get a list of attribute filters to select backup groups by the attributes of their snapshots, for example to write the guests tagged as production and as test to different media pools. A filter only matches a whole value of its key. Several filters are combined like group filters: one of the include filters has to match, if there are any, and none of the exclude filters. The job option is optional. Existing job configurations stay valid and select the same groups as before. Signed-off-by: Jakob-Tobias Winter --- pbs-api-types/src/jobs.rs | 98 ++++++++++- pbs-api-types/tests/attribute_filter_tests.rs | 155 ++++++++++++++++++ 2 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 pbs-api-types/tests/attribute_filter_tests.rs diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs index 9a5d2b77..8e558e4a 100644 --- a/pbs-api-types/src/jobs.rs +++ b/pbs-api-types/src/jobs.rs @@ -15,7 +15,7 @@ use crate::{ BackupType, CRYPT_KEY_ID_SCHEMA, DATASTORE_SCHEMA, DRIVE_NAME_SCHEMA, MEDIA_POOL_NAME_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, NotificationMode, PROXMOX_SAFE_ID_FORMAT, PROXMOX_SAFE_ID_REGEX_STR, REMOTE_ID_SCHEMA, RateLimitConfig, SINGLE_LINE_COMMENT_SCHEMA, - Userid, + SNAPSHOT_ATTRIBUTE_REGEX, SnapshotAttributes, Userid, }; const_regex! { @@ -345,6 +345,10 @@ pub struct VerificationJobStatus { schema: GROUP_FILTER_LIST_SCHEMA, optional: true, }, + "attribute-filter": { + schema: ATTRIBUTE_FILTER_LIST_SCHEMA, + optional: true, + }, ns: { type: BackupNamespace, optional: true, @@ -379,6 +383,8 @@ pub struct TapeBackupJobSetup { pub notification_mode: Option, #[serde(skip_serializing_if = "Option::is_none")] pub group_filter: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub attribute_filter: Option>, #[serde(skip_serializing_if = "Option::is_none", default)] pub ns: Option, #[serde(skip_serializing_if = "Option::is_none", default)] @@ -550,6 +556,96 @@ pub const GROUP_FILTER_SCHEMA: Schema = StringSchema::new( pub const GROUP_FILTER_LIST_SCHEMA: Schema = ArraySchema::new("List of group filters.", &GROUP_FILTER_SCHEMA).schema(); +#[derive(Clone, Debug, PartialEq, Eq)] +/// Filter for matching the attributes of a snapshot. +pub struct AttributeFilter { + pub is_exclude: bool, + pub key: String, + pub value: String, +} + +impl AttributeFilter { + /// Returns whether the value of the filter is one of the values the attributes have for its + /// key. + pub fn matches(&self, attributes: &SnapshotAttributes) -> bool { + attributes + .get(&self.key) + .is_some_and(|values| values.contains(&self.value)) + } + + /// Returns whether the attributes pass the filters. + /// + /// If there are include filters, at least one of them has to match. No exclude filter may + /// match. + pub fn apply_filters(filters: &[AttributeFilter], attributes: &SnapshotAttributes) -> bool { + let is_included = if filters.iter().filter(|f| !f.is_exclude).count() == 0 { + true + } else { + filters + .iter() + .filter(|f| !f.is_exclude) + .any(|filter| filter.matches(attributes)) + }; + + is_included + && !filters + .iter() + .filter(|f| f.is_exclude) + .any(|filter| filter.matches(attributes)) + } +} + +impl std::str::FromStr for AttributeFilter { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let (is_exclude, attribute) = match s.split_once(':') { + Some(("include", value)) => (false, value), + Some(("exclude", value)) => (true, value), + _ => (false, s), + }; + + match attribute.split_once('=') { + Some((key, value)) if SNAPSHOT_ATTRIBUTE_REGEX.is_match(attribute) => { + Ok(AttributeFilter { + is_exclude, + key: key.to_string(), + value: value.to_string(), + }) + } + _ => bail!("input doesn't match expected format '[]KEY=VALUE'"), + } + } +} + +// used for serializing below, caution! +impl std::fmt::Display for AttributeFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.is_exclude { + f.write_str("exclude:")?; + } + write!(f, "{}={}", self.key, self.value) + } +} + +proxmox_serde::forward_deserialize_to_from_str!(AttributeFilter); +proxmox_serde::forward_serialize_to_display!(AttributeFilter); + +fn verify_attribute_filter(input: &str) -> Result<(), anyhow::Error> { + AttributeFilter::from_str(input).map(|_| ()) +} + +pub const ATTRIBUTE_FILTER_SCHEMA: Schema = StringSchema::new( + "Attribute filter, matches if the given value is one of the values of the key. Can be \ + inverted by prepending 'exclude:'.", +) +.format(&ApiStringFormat::VerifyFn(verify_attribute_filter)) +.type_text("[]KEY=VALUE") +.schema(); + +pub const ATTRIBUTE_FILTER_LIST_SCHEMA: Schema = + ArraySchema::new("List of attribute filters.", &ATTRIBUTE_FILTER_SCHEMA).schema(); + pub const TRANSFER_LAST_SCHEMA: Schema = IntegerSchema::new("Limit transfer to last N snapshots (per group), skipping others") .minimum(1) diff --git a/pbs-api-types/tests/attribute_filter_tests.rs b/pbs-api-types/tests/attribute_filter_tests.rs new file mode 100644 index 00000000..c355e6b4 --- /dev/null +++ b/pbs-api-types/tests/attribute_filter_tests.rs @@ -0,0 +1,155 @@ +use pbs_api_types::{AttributeFilter, SnapshotAttributes}; +use std::str::FromStr; + +fn attributes(list: &[(&str, &[&str])]) -> SnapshotAttributes { + list.iter() + .map(|(key, values)| { + ( + key.to_string(), + values.iter().map(|value| value.to_string()).collect(), + ) + }) + .collect() +} + +fn filters(list: &[&str]) -> Vec { + list.iter() + .map(|filter| AttributeFilter::from_str(filter).unwrap()) + .collect() +} + +#[test] +fn test_parse_filter() { + let filter = AttributeFilter::from_str("pve-tag=prod").unwrap(); + assert!(!filter.is_exclude); + assert_eq!(filter.key, "pve-tag"); + assert_eq!(filter.value, "prod"); + assert_eq!(filter.to_string(), "pve-tag=prod"); + + let filter = AttributeFilter::from_str("include:pve-tag=prod").unwrap(); + assert!(!filter.is_exclude); + assert_eq!(filter.to_string(), "pve-tag=prod"); + + let filter = AttributeFilter::from_str("exclude:pve-pool=customers/a_1.test").unwrap(); + assert!(filter.is_exclude); + assert_eq!(filter.key, "pve-pool"); + assert_eq!(filter.value, "customers/a_1.test"); + assert_eq!(filter.to_string(), "exclude:pve-pool=customers/a_1.test"); + + let invalid = [ + "", + "pve-tag", + "pve-tag=", + "=prod", + "Pve-Tag=prod", + "1tag=prod", + "pve-tag=prod;web", + "pve-tag=prod,web", + "pve-tag=prod web", + "pve-tag=prod=web", + "pve-tag=prod:web", + "other:pve-tag=prod", + "exclude:exclude:pve-tag=prod", + ]; + + for filter in invalid { + assert!( + AttributeFilter::from_str(filter).is_err(), + "'{filter}' must not parse" + ); + } +} + +#[test] +fn test_no_filters() { + let filters = []; + + assert!(AttributeFilter::apply_filters(&filters, &attributes(&[]))); + assert!(AttributeFilter::apply_filters( + &filters, + &attributes(&[("pve-tag", &["prod", "web"])]) + )); +} + +#[test] +fn test_include_filters() { + let filters = filters(&["pve-tag=prod", "pve-pool=customer1"]); + + let do_backup = [ + attributes(&[("pve-tag", &["prod"])]), + attributes(&[("pve-tag", &["db", "prod", "web"])]), + attributes(&[("pve-pool", &["customer1"])]), + attributes(&[("pve-pool", &["customer1"]), ("pve-tag", &["test"])]), + ]; + + let dont_backup = [ + attributes(&[]), + attributes(&[("pve-tag", &[])]), + attributes(&[("pve-tag", &["nonprod", "prod-eu", "pro", "Prod"])]), + attributes(&[("pve-pool", &["prod"])]), + attributes(&[("pve-pool", &["customer1/sub", "customer10"])]), + attributes(&[("other", &["prod", "customer1"])]), + ]; + + for attributes in do_backup { + assert!(AttributeFilter::apply_filters(&filters, &attributes)); + } + + for attributes in dont_backup { + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); + } +} + +#[test] +fn test_exclude_filters() { + let filters = filters(&["exclude:pve-tag=prod", "exclude:pve-tag=test"]); + + let do_backup = [ + attributes(&[]), + attributes(&[("pve-tag", &["web"])]), + attributes(&[("pve-pool", &["prod"])]), + ]; + + let dont_backup = [ + attributes(&[("pve-tag", &["prod"])]), + attributes(&[("pve-tag", &["test", "web"])]), + attributes(&[("pve-tag", &["prod", "test"])]), + ]; + + for attributes in do_backup { + assert!(AttributeFilter::apply_filters(&filters, &attributes)); + } + + for attributes in dont_backup { + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); + } +} + +#[test] +fn test_include_and_exclude_filters() { + let filters = filters(&[ + "exclude:pve-tag=test", + "pve-pool=customer1", + "pve-pool=customer2", + ]); + + let do_backup = [ + attributes(&[("pve-pool", &["customer1"])]), + attributes(&[("pve-pool", &["customer2"]), ("pve-tag", &["prod"])]), + ]; + + let dont_backup = [ + attributes(&[]), + attributes(&[("pve-tag", &["prod"])]), + attributes(&[("pve-pool", &["customer3"])]), + attributes(&[("pve-pool", &["customer1"]), ("pve-tag", &["test"])]), + ]; + + for attributes in do_backup { + assert!(AttributeFilter::apply_filters(&filters, &attributes)); + } + + for attributes in dont_backup { + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); + } +} -- 2.53.0