From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DFB13A0A59 for ; Thu, 9 Nov 2023 15:44:06 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B850516654 for ; Thu, 9 Nov 2023 15:43:36 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 9 Nov 2023 15:43:35 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 6C903476DB for ; Thu, 9 Nov 2023 15:43:35 +0100 (CET) From: Philipp Hufnagl To: pbs-devel@lists.proxmox.com Date: Thu, 9 Nov 2023 15:43:27 +0100 Message-Id: <20231109144329.1830522-2-p.hufnagl@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20231109144329.1830522-1-p.hufnagl@proxmox.com> References: <20231109144329.1830522-1-p.hufnagl@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.062 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [backup.rs, pull.rs, jobs.rs, datastore.rs] Subject: [pbs-devel] [PATCH proxmox-backup v2 1/3] fix #4315: jobs: modify GroupFilter so include/exclude is tracked X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Nov 2023 14:44:06 -0000 After some discussion I canged the include/exclude behavior to first run all include filter and after that all exclude filter (rather then allowing to alternate inbetween). This is simply done by sorting the list (include first) before executing it. Since a GroupFilter now also features an behavior, the Struct has been renamed To GroupType (since simply type is a keyword). The new GroupFilter now has a behaviour as a flag 'is_exclude'. I considered calling it 'is_include' but a reader later then might not know what the opposite of 'include' is (do not include? deactivate?). I also considered making a new enum 'behaviour' but since there are only 2 values I considered it over engeneered. Matching a filter will now iterate with a forech loop in order to also exclude matches. Signed-off-by: Philipp Hufnagl --- pbs-api-types/src/datastore.rs | 11 +++--- pbs-api-types/src/jobs.rs | 64 +++++++++++++++++++++++++++------- src/api2/pull.rs | 11 +++++- src/api2/tape/backup.rs | 8 ++++- src/server/pull.rs | 8 ++++- 5 files changed, 80 insertions(+), 22 deletions(-) diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index 73c4890e..c3beedaf 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -841,17 +841,16 @@ impl BackupGroup { } pub fn matches(&self, filter: &crate::GroupFilter) -> bool { - use crate::GroupFilter; - - match filter { - GroupFilter::Group(backup_group) => { + use crate::FilterType; + match &filter.filter_type { + FilterType::Group(backup_group) => { match backup_group.parse::() { Ok(group) => *self == group, Err(_) => false, // shouldn't happen if value is schema-checked } } - GroupFilter::BackupType(ty) => self.ty == *ty, - GroupFilter::Regex(regex) => regex.is_match(&self.to_string()), + FilterType::BackupType(ty) => self.ty == *ty, + FilterType::Regex(regex) => regex.is_match(&self.to_string()), } } } diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs index 23e19b7b..ef182795 100644 --- a/pbs-api-types/src/jobs.rs +++ b/pbs-api-types/src/jobs.rs @@ -3,6 +3,7 @@ use std::str::FromStr; use regex::Regex; use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; use proxmox_schema::*; @@ -383,7 +384,7 @@ pub struct TapeBackupJobStatus { #[derive(Clone, Debug)] /// Filter for matching `BackupGroup`s, for use with `BackupGroup::filter`. -pub enum GroupFilter { +pub enum FilterType { /// BackupGroup type - either `vm`, `ct`, or `host`. BackupType(BackupType), /// Full identifier of BackupGroup, including type @@ -392,7 +393,7 @@ pub enum GroupFilter { Regex(Regex), } -impl PartialEq for GroupFilter { +impl PartialEq for FilterType { fn eq(&self, other: &Self) -> bool { match (self, other) { (Self::BackupType(a), Self::BackupType(b)) => a == b, @@ -403,27 +404,64 @@ impl PartialEq for GroupFilter { } } +#[derive(Clone, Debug)] +pub struct GroupFilter { + pub is_exclude: bool, + pub filter_type: FilterType, +} + +impl PartialEq for GroupFilter { + fn eq(&self, other: &Self) -> bool { + self.filter_type == other.filter_type && self.is_exclude == other.is_exclude + } +} + +impl Eq for GroupFilter {} + +impl PartialOrd for GroupFilter { + fn partial_cmp(&self, other: &Self) -> Option { + self.is_exclude.partial_cmp(&other.is_exclude) + } +} + +impl Ord for GroupFilter { + fn cmp(&self, other: &Self) -> Ordering { + self.is_exclude.cmp(&other.is_exclude) + } +} + impl std::str::FromStr for GroupFilter { type Err = anyhow::Error; fn from_str(s: &str) -> Result { - match s.split_once(':') { - Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| GroupFilter::Group(value.to_string())), - Some(("type", value)) => Ok(GroupFilter::BackupType(value.parse()?)), - Some(("regex", value)) => Ok(GroupFilter::Regex(Regex::new(value)?)), + let (is_exclude, type_str) = match s.split_once(':') { + Some(("include", value)) => (false, value), + Some(("exclude", value)) => (true, value), + _ => (false, s), + }; + + let filter_type = match type_str.split_once(':') { + Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| FilterType::Group(value.to_string())), + Some(("type", value)) => Ok(FilterType::BackupType(value.parse()?)), + Some(("regex", value)) => Ok(FilterType::Regex(Regex::new(value)?)), Some((ty, _value)) => Err(format_err!("expected 'group', 'type' or 'regex' prefix, got '{}'", ty)), None => Err(format_err!("input doesn't match expected format '|regex:REGEX>'")), - }.map_err(|err| format_err!("'{}' - {}", s, err)) + }?; + Ok(GroupFilter { + is_exclude, + filter_type, + }) } } // used for serializing below, caution! impl std::fmt::Display for GroupFilter { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - GroupFilter::BackupType(backup_type) => write!(f, "type:{}", backup_type), - GroupFilter::Group(backup_group) => write!(f, "group:{}", backup_group), - GroupFilter::Regex(regex) => write!(f, "regex:{}", regex.as_str()), + let exclude = if self.is_exclude { "exclude:" } else { "" }; + match &self.filter_type { + FilterType::BackupType(backup_type) => write!(f, "{}type:{}", exclude, backup_type), + FilterType::Group(backup_group) => write!(f, "{}group:{}", exclude, backup_group), + FilterType::Regex(regex) => write!(f, "{}regex:{}", exclude, regex.as_str()), } } } @@ -436,9 +474,9 @@ fn verify_group_filter(input: &str) -> Result<(), anyhow::Error> { } pub const GROUP_FILTER_SCHEMA: Schema = StringSchema::new( - "Group filter based on group identifier ('group:GROUP'), group type ('type:'), or regex ('regex:RE').") + "Group filter based on group identifier ('group:GROUP'), group type ('type:'), or regex ('regex:RE'). Can be inverted by adding 'exclude:' before.") .format(&ApiStringFormat::VerifyFn(verify_group_filter)) - .type_text("|group:GROUP|regex:RE>") + .type_text("[]|group:GROUP|regex:RE>") .schema(); pub const GROUP_FILTER_LIST_SCHEMA: Schema = diff --git a/src/api2/pull.rs b/src/api2/pull.rs index daeba7cf..20571345 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -62,6 +62,15 @@ impl TryFrom<&SyncJobConfig> for PullParameters { type Error = Error; fn try_from(sync_job: &SyncJobConfig) -> Result { + let filters = match &sync_job.group_filter { + Some(v) => { + let mut f = v.clone(); + f.sort(); + Some(f) + } + None => None, + }; + PullParameters::new( &sync_job.store, sync_job.ns.clone().unwrap_or_default(), @@ -75,7 +84,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters { .clone(), sync_job.remove_vanished, sync_job.max_depth, - sync_job.group_filter.clone(), + filters, sync_job.limit.clone(), sync_job.transfer_last, ) diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs index 2f9385a7..04abdb71 100644 --- a/src/api2/tape/backup.rs +++ b/src/api2/tape/backup.rs @@ -413,7 +413,13 @@ fn backup_worker( let (group_list, group_count) = if let Some(group_filters) = &setup.group_filter { let filter_fn = |group: &BackupGroup, group_filters: &[GroupFilter]| { - group_filters.iter().any(|filter| group.matches(filter)) + let mut is_match = false; + for filter in group_filters.iter() { + if group.matches(filter) { + is_match = !filter.is_exclude; + } + } + is_match }; let group_count_full = group_list.len(); diff --git a/src/server/pull.rs b/src/server/pull.rs index a973a10e..0124bd3f 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -1084,7 +1084,13 @@ pub(crate) async fn pull_ns( }); let apply_filters = |group: &pbs_api_types::BackupGroup, filters: &[GroupFilter]| -> bool { - filters.iter().any(|filter| group.matches(filter)) + let mut is_match = false; + for filter in filters.iter() { + if group.matches(filter) { + is_match = !filter.is_exclude; + } + } + is_match }; // Get groups with target NS set -- 2.39.2