From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id D1DAF1FF0BC for ; Mon, 16 Nov 2026 15:57:55 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 95EAC21709; Mon, 16 Nov 2026 15:57:55 +0100 (CET) Message-ID: <7802620f-4d59-41d7-8807-a8cfc34fe007@proxmox.com> Date: Mon, 16 Nov 2026 15:57:50 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird From: Christian Ebner Subject: Re: [RFC PATCH proxmox 02/12] pbs-api-types: tape: add attribute filter to backup jobs To: Jakob-Tobias Winter , pbs-devel@lists.proxmox.com References: <20261113085637.57360-3-tobias@stuttgart.de> <20261113085637.57360-5-tobias@stuttgart.de> Content-Language: en-US, de-DE In-Reply-To: <20261113085637.57360-5-tobias@stuttgart.de> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 7bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794841070490 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.376 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: UXYBFV2TJKUU7XU472IYG24UCPK5QUGE X-Message-ID-Hash: UXYBFV2TJKUU7XU472IYG24UCPK5QUGE X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On 11/13/26 10:06 AM, Jakob-Tobias Winter wrote: > Tape backup jobs get a list of attribute filters to select backup > groups by the attributes of their snapshots, for example to write > the guests tagged as production and as test to different media > pools. A filter only matches a whole value of its key. Several > filters are combined like group filters: one of the include filters > has to match, if there are any, and none of the exclude filters. > > The job option is optional. Existing job configurations stay valid > and select the same groups as before. > > Signed-off-by: Jakob-Tobias Winter > --- > pbs-api-types/src/jobs.rs | 98 ++++++++++- > pbs-api-types/tests/attribute_filter_tests.rs | 155 ++++++++++++++++++ > 2 files changed, 252 insertions(+), 1 deletion(-) > create mode 100644 pbs-api-types/tests/attribute_filter_tests.rs > > diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs > index 9a5d2b77..8e558e4a 100644 > --- a/pbs-api-types/src/jobs.rs > +++ b/pbs-api-types/src/jobs.rs > @@ -15,7 +15,7 @@ use crate::{ > BackupType, CRYPT_KEY_ID_SCHEMA, DATASTORE_SCHEMA, DRIVE_NAME_SCHEMA, MEDIA_POOL_NAME_SCHEMA, > NS_MAX_DEPTH_REDUCED_SCHEMA, NotificationMode, PROXMOX_SAFE_ID_FORMAT, > PROXMOX_SAFE_ID_REGEX_STR, REMOTE_ID_SCHEMA, RateLimitConfig, SINGLE_LINE_COMMENT_SCHEMA, > - Userid, > + SNAPSHOT_ATTRIBUTE_REGEX, SnapshotAttributes, Userid, > }; > > const_regex! { > @@ -345,6 +345,10 @@ pub struct VerificationJobStatus { > schema: GROUP_FILTER_LIST_SCHEMA, > optional: true, > }, > + "attribute-filter": { > + schema: ATTRIBUTE_FILTER_LIST_SCHEMA, > + optional: true, > + }, > ns: { > type: BackupNamespace, > optional: true, > @@ -379,6 +383,8 @@ pub struct TapeBackupJobSetup { > pub notification_mode: Option, > #[serde(skip_serializing_if = "Option::is_none")] > pub group_filter: Option>, > + #[serde(skip_serializing_if = "Option::is_none")] > + pub attribute_filter: Option>, > #[serde(skip_serializing_if = "Option::is_none", default)] > pub ns: Option, > #[serde(skip_serializing_if = "Option::is_none", default)] > @@ -550,6 +556,96 @@ pub const GROUP_FILTER_SCHEMA: Schema = StringSchema::new( > pub const GROUP_FILTER_LIST_SCHEMA: Schema = > ArraySchema::new("List of group filters.", &GROUP_FILTER_SCHEMA).schema(); > > +#[derive(Clone, Debug, PartialEq, Eq)] > +/// Filter for matching the attributes of a snapshot. > +pub struct AttributeFilter { > + pub is_exclude: bool, > + pub key: String, as mentioned on patch 0001, key ... > + pub value: String, ... and value should preferably have dedicated types ... > +} > + > +impl AttributeFilter { > + /// Returns whether the value of the filter is one of the values the attributes have for its > + /// key. > + pub fn matches(&self, attributes: &SnapshotAttributes) -> bool { > + attributes > + .get(&self.key) > + .is_some_and(|values| values.contains(&self.value)) > + } > + > + /// Returns whether the attributes pass the filters. > + /// > + /// If there are include filters, at least one of them has to match. No exclude filter may > + /// match. > + pub fn apply_filters(filters: &[AttributeFilter], attributes: &SnapshotAttributes) -> bool { > + let is_included = if filters.iter().filter(|f| !f.is_exclude).count() == 0 { > + true > + } else { > + filters > + .iter() > + .filter(|f| !f.is_exclude) > + .any(|filter| filter.matches(attributes)) > + }; > + > + is_included > + && !filters > + .iter() > + .filter(|f| f.is_exclude) > + .any(|filter| filter.matches(attributes)) > + } > +} > + > +impl std::str::FromStr for AttributeFilter { > + type Err = anyhow::Error; > + > + fn from_str(s: &str) -> Result { > + let (is_exclude, attribute) = match s.split_once(':') { > + Some(("include", value)) => (false, value), > + Some(("exclude", value)) => (true, value), > + _ => (false, s), > + }; > + > + match attribute.split_once('=') { > + Some((key, value)) if SNAPSHOT_ATTRIBUTE_REGEX.is_match(attribute) => { ... this can then use the dedicate types for parsing > + Ok(AttributeFilter { > + is_exclude, > + key: key.to_string(), > + value: value.to_string(), > + }) > + } > + _ => bail!("input doesn't match expected format '[]KEY=VALUE'"), > + } > + } > +} > + > +// used for serializing below, caution! > +impl std::fmt::Display for AttributeFilter { > + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { > + if self.is_exclude { > + f.write_str("exclude:")?; > + } > + write!(f, "{}={}", self.key, self.value) > + } > +} > + > +proxmox_serde::forward_deserialize_to_from_str!(AttributeFilter); > +proxmox_serde::forward_serialize_to_display!(AttributeFilter); > + > +fn verify_attribute_filter(input: &str) -> Result<(), anyhow::Error> { > + AttributeFilter::from_str(input).map(|_| ()) > +} > + > +pub const ATTRIBUTE_FILTER_SCHEMA: Schema = StringSchema::new( > + "Attribute filter, matches if the given value is one of the values of the key. Can be \ > + inverted by prepending 'exclude:'.", > +) > +.format(&ApiStringFormat::VerifyFn(verify_attribute_filter)) > +.type_text("[]KEY=VALUE") > +.schema(); > + > +pub const ATTRIBUTE_FILTER_LIST_SCHEMA: Schema = > + ArraySchema::new("List of attribute filters.", &ATTRIBUTE_FILTER_SCHEMA).schema(); > + > pub const TRANSFER_LAST_SCHEMA: Schema = > IntegerSchema::new("Limit transfer to last N snapshots (per group), skipping others") > .minimum(1) > diff --git a/pbs-api-types/tests/attribute_filter_tests.rs b/pbs-api-types/tests/attribute_filter_tests.rs > new file mode 100644 > index 00000000..c355e6b4 > --- /dev/null > +++ b/pbs-api-types/tests/attribute_filter_tests.rs > @@ -0,0 +1,155 @@ > +use pbs_api_types::{AttributeFilter, SnapshotAttributes}; > +use std::str::FromStr; > + > +fn attributes(list: &[(&str, &[&str])]) -> SnapshotAttributes { > + list.iter() > + .map(|(key, values)| { > + ( > + key.to_string(), > + values.iter().map(|value| value.to_string()).collect(), > + ) > + }) > + .collect() > +} > + > +fn filters(list: &[&str]) -> Vec { > + list.iter() > + .map(|filter| AttributeFilter::from_str(filter).unwrap()) > + .collect() > +} > + > +#[test] > +fn test_parse_filter() { > + let filter = AttributeFilter::from_str("pve-tag=prod").unwrap(); > + assert!(!filter.is_exclude); > + assert_eq!(filter.key, "pve-tag"); > + assert_eq!(filter.value, "prod"); > + assert_eq!(filter.to_string(), "pve-tag=prod"); > + > + let filter = AttributeFilter::from_str("include:pve-tag=prod").unwrap(); > + assert!(!filter.is_exclude); > + assert_eq!(filter.to_string(), "pve-tag=prod"); > + > + let filter = AttributeFilter::from_str("exclude:pve-pool=customers/a_1.test").unwrap(); > + assert!(filter.is_exclude); > + assert_eq!(filter.key, "pve-pool"); > + assert_eq!(filter.value, "customers/a_1.test"); > + assert_eq!(filter.to_string(), "exclude:pve-pool=customers/a_1.test"); > + > + let invalid = [ > + "", > + "pve-tag", > + "pve-tag=", > + "=prod", > + "Pve-Tag=prod", > + "1tag=prod", > + "pve-tag=prod;web", > + "pve-tag=prod,web", > + "pve-tag=prod web", > + "pve-tag=prod=web", > + "pve-tag=prod:web", > + "other:pve-tag=prod", > + "exclude:exclude:pve-tag=prod", > + ]; > + > + for filter in invalid { > + assert!( > + AttributeFilter::from_str(filter).is_err(), > + "'{filter}' must not parse" > + ); > + } > +} > + > +#[test] > +fn test_no_filters() { > + let filters = []; > + > + assert!(AttributeFilter::apply_filters(&filters, &attributes(&[]))); > + assert!(AttributeFilter::apply_filters( > + &filters, > + &attributes(&[("pve-tag", &["prod", "web"])]) > + )); > +} > + > +#[test] > +fn test_include_filters() { > + let filters = filters(&["pve-tag=prod", "pve-pool=customer1"]); > + > + let do_backup = [ > + attributes(&[("pve-tag", &["prod"])]), > + attributes(&[("pve-tag", &["db", "prod", "web"])]), > + attributes(&[("pve-pool", &["customer1"])]), > + attributes(&[("pve-pool", &["customer1"]), ("pve-tag", &["test"])]), > + ]; > + > + let dont_backup = [ > + attributes(&[]), > + attributes(&[("pve-tag", &[])]), > + attributes(&[("pve-tag", &["nonprod", "prod-eu", "pro", "Prod"])]), > + attributes(&[("pve-pool", &["prod"])]), > + attributes(&[("pve-pool", &["customer1/sub", "customer10"])]), > + attributes(&[("other", &["prod", "customer1"])]), > + ]; > + > + for attributes in do_backup { > + assert!(AttributeFilter::apply_filters(&filters, &attributes)); > + } > + > + for attributes in dont_backup { > + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); > + } > +} > + > +#[test] > +fn test_exclude_filters() { > + let filters = filters(&["exclude:pve-tag=prod", "exclude:pve-tag=test"]); > + > + let do_backup = [ > + attributes(&[]), > + attributes(&[("pve-tag", &["web"])]), > + attributes(&[("pve-pool", &["prod"])]), > + ]; > + > + let dont_backup = [ > + attributes(&[("pve-tag", &["prod"])]), > + attributes(&[("pve-tag", &["test", "web"])]), > + attributes(&[("pve-tag", &["prod", "test"])]), > + ]; > + > + for attributes in do_backup { > + assert!(AttributeFilter::apply_filters(&filters, &attributes)); > + } > + > + for attributes in dont_backup { > + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); > + } > +} > + > +#[test] > +fn test_include_and_exclude_filters() { > + let filters = filters(&[ > + "exclude:pve-tag=test", > + "pve-pool=customer1", > + "pve-pool=customer2", > + ]); > + > + let do_backup = [ > + attributes(&[("pve-pool", &["customer1"])]), > + attributes(&[("pve-pool", &["customer2"]), ("pve-tag", &["prod"])]), > + ]; > + > + let dont_backup = [ > + attributes(&[]), > + attributes(&[("pve-tag", &["prod"])]), > + attributes(&[("pve-pool", &["customer3"])]), > + attributes(&[("pve-pool", &["customer1"]), ("pve-tag", &["test"])]), > + ]; > + > + for attributes in do_backup { > + assert!(AttributeFilter::apply_filters(&filters, &attributes)); > + } > + > + for attributes in dont_backup { > + assert!(!AttributeFilter::apply_filters(&filters, &attributes)); > + } > +}