* [pbs-devel] [PATCH proxmox-backup v4 1/3] fix #4315: jobs: modify GroupFilter so include/exclude is tracked
2023-11-28 14:34 [pbs-devel] [PATCH proxmox-backup v4 0/3] fix #4315: datastore: Exclude entries from sync Philipp Hufnagl
@ 2023-11-28 14:34 ` Philipp Hufnagl
2023-11-28 14:34 ` [pbs-devel] [PATCH proxmox-backup v4 2/3] ui: Show if Filter includes or excludes Philipp Hufnagl
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Philipp Hufnagl @ 2023-11-28 14:34 UTC (permalink / raw)
To: pbs-devel
After some discussion I canged the include/exclude behavior to first run
all include filter and after that all exclude filter (rather then
allowing to alternate inbetween). This is simply done by sorting the
list (include first) before executing it.
Since a GroupFilter now also features an behavior, the Struct has been
renamed To GroupType (since simply type is a keyword). The new
GroupFilter now has a behaviour as a flag 'is_exclude'.
I considered calling it 'is_include' but a reader later then might not
know what the opposite of 'include' is (do not include? deactivate?). I
also considered making a new enum 'behaviour' but since there are only 2
values I considered it over engeneered.
Matching a filter will now iterate with a forech loop in order to also
exclude matches.
If only include filter are set, every group starts included.
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
pbs-api-types/src/datastore.rs | 11 +++---
pbs-api-types/src/jobs.rs | 64 +++++++++++++++++++++++++++-------
src/api2/pull.rs | 11 +++++-
src/api2/tape/backup.rs | 17 +++++++--
src/server/pull.rs | 23 +++++++++---
5 files changed, 99 insertions(+), 27 deletions(-)
diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs
index 54251897..48f17356 100644
--- a/pbs-api-types/src/datastore.rs
+++ b/pbs-api-types/src/datastore.rs
@@ -879,17 +879,16 @@ impl BackupGroup {
}
pub fn matches(&self, filter: &crate::GroupFilter) -> bool {
- use crate::GroupFilter;
-
- match filter {
- GroupFilter::Group(backup_group) => {
+ use crate::FilterType;
+ match &filter.filter_type {
+ FilterType::Group(backup_group) => {
match backup_group.parse::<BackupGroup>() {
Ok(group) => *self == group,
Err(_) => false, // shouldn't happen if value is schema-checked
}
}
- GroupFilter::BackupType(ty) => self.ty == *ty,
- GroupFilter::Regex(regex) => regex.is_match(&self.to_string()),
+ FilterType::BackupType(ty) => self.ty == *ty,
+ FilterType::Regex(regex) => regex.is_match(&self.to_string()),
}
}
}
diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs
index 1f5b3cf1..dff02395 100644
--- a/pbs-api-types/src/jobs.rs
+++ b/pbs-api-types/src/jobs.rs
@@ -3,6 +3,7 @@ use std::str::FromStr;
use regex::Regex;
use serde::{Deserialize, Serialize};
+use std::cmp::Ordering;
use proxmox_schema::*;
@@ -388,7 +389,7 @@ pub struct TapeBackupJobStatus {
#[derive(Clone, Debug)]
/// Filter for matching `BackupGroup`s, for use with `BackupGroup::filter`.
-pub enum GroupFilter {
+pub enum FilterType {
/// BackupGroup type - either `vm`, `ct`, or `host`.
BackupType(BackupType),
/// Full identifier of BackupGroup, including type
@@ -397,7 +398,7 @@ pub enum GroupFilter {
Regex(Regex),
}
-impl PartialEq for GroupFilter {
+impl PartialEq for FilterType {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::BackupType(a), Self::BackupType(b)) => a == b,
@@ -408,27 +409,64 @@ impl PartialEq for GroupFilter {
}
}
+#[derive(Clone, Debug)]
+pub struct GroupFilter {
+ pub is_exclude: bool,
+ pub filter_type: FilterType,
+}
+
+impl PartialEq for GroupFilter {
+ fn eq(&self, other: &Self) -> bool {
+ self.filter_type == other.filter_type && self.is_exclude == other.is_exclude
+ }
+}
+
+impl Eq for GroupFilter {}
+
+impl PartialOrd for GroupFilter {
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+ self.is_exclude.partial_cmp(&other.is_exclude)
+ }
+}
+
+impl Ord for GroupFilter {
+ fn cmp(&self, other: &Self) -> Ordering {
+ self.is_exclude.cmp(&other.is_exclude)
+ }
+}
+
impl std::str::FromStr for GroupFilter {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
- match s.split_once(':') {
- Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| GroupFilter::Group(value.to_string())),
- Some(("type", value)) => Ok(GroupFilter::BackupType(value.parse()?)),
- Some(("regex", value)) => Ok(GroupFilter::Regex(Regex::new(value)?)),
+ let (is_exclude, type_str) = match s.split_once(':') {
+ Some(("include", value)) => (false, value),
+ Some(("exclude", value)) => (true, value),
+ _ => (false, s),
+ };
+
+ let filter_type = match type_str.split_once(':') {
+ Some(("group", value)) => BACKUP_GROUP_SCHEMA.parse_simple_value(value).map(|_| FilterType::Group(value.to_string())),
+ Some(("type", value)) => Ok(FilterType::BackupType(value.parse()?)),
+ Some(("regex", value)) => Ok(FilterType::Regex(Regex::new(value)?)),
Some((ty, _value)) => Err(format_err!("expected 'group', 'type' or 'regex' prefix, got '{}'", ty)),
None => Err(format_err!("input doesn't match expected format '<group:GROUP||type:<vm|ct|host>|regex:REGEX>'")),
- }.map_err(|err| format_err!("'{}' - {}", s, err))
+ }?;
+ Ok(GroupFilter {
+ is_exclude,
+ filter_type,
+ })
}
}
// used for serializing below, caution!
impl std::fmt::Display for GroupFilter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- match self {
- GroupFilter::BackupType(backup_type) => write!(f, "type:{}", backup_type),
- GroupFilter::Group(backup_group) => write!(f, "group:{}", backup_group),
- GroupFilter::Regex(regex) => write!(f, "regex:{}", regex.as_str()),
+ let exclude = if self.is_exclude { "exclude:" } else { "" };
+ match &self.filter_type {
+ FilterType::BackupType(backup_type) => write!(f, "{}type:{}", exclude, backup_type),
+ FilterType::Group(backup_group) => write!(f, "{}group:{}", exclude, backup_group),
+ FilterType::Regex(regex) => write!(f, "{}regex:{}", exclude, regex.as_str()),
}
}
}
@@ -441,9 +479,9 @@ fn verify_group_filter(input: &str) -> Result<(), anyhow::Error> {
}
pub const GROUP_FILTER_SCHEMA: Schema = StringSchema::new(
- "Group filter based on group identifier ('group:GROUP'), group type ('type:<vm|ct|host>'), or regex ('regex:RE').")
+ "Group filter based on group identifier ('group:GROUP'), group type ('type:<vm|ct|host>'), or regex ('regex:RE'). Can be inverted by adding 'exclude:' before.")
.format(&ApiStringFormat::VerifyFn(verify_group_filter))
- .type_text("<type:<vm|ct|host>|group:GROUP|regex:RE>")
+ .type_text("[<exclude:|include:>]<type:<vm|ct|host>|group:GROUP|regex:RE>")
.schema();
pub const GROUP_FILTER_LIST_SCHEMA: Schema =
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index eb9a2199..f174926c 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -72,6 +72,15 @@ impl TryFrom<&SyncJobConfig> for PullParameters {
type Error = Error;
fn try_from(sync_job: &SyncJobConfig) -> Result<Self, Self::Error> {
+ let filters = match &sync_job.group_filter {
+ Some(v) => {
+ let mut f = v.clone();
+ f.sort();
+ Some(f)
+ }
+ None => None,
+ };
+
PullParameters::new(
&sync_job.store,
sync_job.ns.clone().unwrap_or_default(),
@@ -85,7 +94,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters {
.clone(),
sync_job.remove_vanished,
sync_job.max_depth,
- sync_job.group_filter.clone(),
+ filters,
sync_job.limit.clone(),
sync_job.transfer_last,
)
diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index 2f9385a7..80dcdd1d 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -412,14 +412,25 @@ fn backup_worker(
group_list.sort_unstable_by(|a, b| a.group().cmp(b.group()));
let (group_list, group_count) = if let Some(group_filters) = &setup.group_filter {
- let filter_fn = |group: &BackupGroup, group_filters: &[GroupFilter]| {
- group_filters.iter().any(|filter| group.matches(filter))
+ let filter_fn = |group: &BackupGroup, group_filters: &[GroupFilter], start_with: bool| {
+ let mut is_match = start_with;
+ for filter in group_filters.iter() {
+ if group.matches(filter) {
+ is_match = !filter.is_exclude;
+ }
+ }
+ is_match
};
let group_count_full = group_list.len();
+ // if there are only exclude filter, inculude everything
+ let mut include_all = false;
+ if !group_filters.is_empty() || group_filters.first().unwrap().is_exclude {
+ include_all = true;
+ }
let list: Vec<BackupGroup> = group_list
.into_iter()
- .filter(|group| filter_fn(group, group_filters))
+ .filter(|group| filter_fn(group, group_filters, include_all))
.collect();
let group_count = list.len();
task_log!(
diff --git a/src/server/pull.rs b/src/server/pull.rs
index 1403c7a7..10190cca 100644
--- a/src/server/pull.rs
+++ b/src/server/pull.rs
@@ -1368,15 +1368,26 @@ pub(crate) async fn pull_ns(
}
});
- let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool {
- filters.iter().any(|filter| group.matches(filter))
+ let apply_filters = |group: &BackupGroup, filters: &[GroupFilter], start_with: bool| -> bool {
+ let mut is_match = start_with;
+ for filter in filters.iter() {
+ if group.matches(filter) {
+ is_match = !filter.is_exclude;
+ }
+ }
+ is_match
};
let list = if let Some(ref group_filter) = ¶ms.group_filter {
+ // if there are only exclude filter, inculude everything
+ let mut include_all = false;
+ if !group_filter.is_empty() || group_filter.first().unwrap().is_exclude {
+ include_all = true;
+ }
let unfiltered_count = list.len();
let list: Vec<BackupGroup> = list
.into_iter()
- .filter(|group| apply_filters(group, group_filter))
+ .filter(|group| apply_filters(group, group_filter, include_all))
.collect();
task_log!(
worker,
@@ -1458,7 +1469,11 @@ pub(crate) async fn pull_ns(
continue;
}
if let Some(ref group_filter) = ¶ms.group_filter {
- if !apply_filters(local_group, group_filter) {
+ let mut include_all = false;
+ if !group_filter.is_empty() || group_filter.first().unwrap().is_exclude {
+ include_all = true;
+ }
+ if !apply_filters(local_group, group_filter, include_all) {
continue;
}
}
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread* [pbs-devel] [PATCH proxmox-backup v4 2/3] ui: Show if Filter includes or excludes
2023-11-28 14:34 [pbs-devel] [PATCH proxmox-backup v4 0/3] fix #4315: datastore: Exclude entries from sync Philipp Hufnagl
2023-11-28 14:34 ` [pbs-devel] [PATCH proxmox-backup v4 1/3] fix #4315: jobs: modify GroupFilter so include/exclude is tracked Philipp Hufnagl
@ 2023-11-28 14:34 ` Philipp Hufnagl
2023-11-28 14:34 ` [pbs-devel] [PATCH proxmox-backup v4 3/3] docs: document new include/exclude paramenter Philipp Hufnagl
2023-11-28 14:48 ` [pbs-devel] [PATCH proxmox-backup v4 0/3] fix #4315: datastore: Exclude entries from sync Philipp Hufnagl
3 siblings, 0 replies; 6+ messages in thread
From: Philipp Hufnagl @ 2023-11-28 14:34 UTC (permalink / raw)
To: pbs-devel
To make the UI compatible, the Group Filter dialogue has been extended
by a second list, so it now features a list for all include filter and
one for all exclude filters.
Internally, all include as well as exclude filter are managed into one
list. The 2 list view is just for a cleaner representation in the UI.
Signed-off-by: Philipp Hufnagl <p.hufnagl@proxmox.com>
---
www/form/GroupFilter.js | 233 +++++++++++++++++++++++++++++-----------
1 file changed, 169 insertions(+), 64 deletions(-)
diff --git a/www/form/GroupFilter.js b/www/form/GroupFilter.js
index dee37b0b..70bd1a56 100644
--- a/www/form/GroupFilter.js
+++ b/www/form/GroupFilter.js
@@ -35,13 +35,36 @@ Ext.define('PBS.form.GroupFilter', {
// break cyclic reference
me.removeReferences(record);
- me.lookup('grid').getStore().remove(record);
+ me.lookup('grid-include').getStore().remove(record);
+ me.lookup('grid-exclude').getStore().remove(record);
me.updateRealField();
},
- addFilter: function() {
+ addIncludeFilter: function() {
let me = this;
- me.lookup('grid').getStore().add({});
+ me.lookup('grid-include').getStore().add({ behavior: 'include' });
+ me.updateRealField();
+ },
+
+ addExcludeFilter: function() {
+ let me = this;
+ me.lookup('grid-exclude').getStore().add({ behavior: 'exclude' });
+ me.updateRealField();
+ },
+
+
+ onBehaviorChange: function(field, value) {
+ let me = this;
+ let record = field.getWidgetRecord();
+ if (record === undefined) {
+ return;
+ }
+
+ record.set('behavior', value);
+ record.commit();
+ if (record.widgets) {
+ me.setInputValue(record.widgets, record);
+ }
me.updateRealField();
},
@@ -77,8 +100,12 @@ Ext.define('PBS.form.GroupFilter', {
},
parseGroupFilter: function(filter) {
- let [, type, input] = filter.match(/^(type|group|regex):(.*)$/);
+ let [, behavior, type, input] = filter.match(/^(?:(exclude|include):)?(type|group|regex):(.*)$/);
+ if (behavior === undefined) {
+ behavior = "include";
+ }
return {
+ behavior,
type,
input,
};
@@ -86,13 +113,16 @@ Ext.define('PBS.form.GroupFilter', {
onValueChange: function(field, values) {
let me = this;
- let grid = me.lookup('grid');
+ let grid_include = me.lookup('grid-include');
+ let grid_exclude = me.lookup('grid-exclude');
if (!values || values.length === 0) {
- grid.getStore().removeAll();
+ grid_include.getStore().removeAll();
+ grid_exclude.getStore().removeAll();
return;
}
let records = values.map((filter) => me.parseGroupFilter(filter));
- grid.getStore().setData(records);
+ grid_include.getStore().setData(records);
+ grid_exclude.getStore().setData(records);
},
setInputValue: function(widgets, rec) {
@@ -162,9 +192,18 @@ Ext.define('PBS.form.GroupFilter', {
let me = this;
let filter = [];
- me.lookup('grid').getStore().each((rec) => {
- if (rec.data.type && rec.data.input) {
- filter.push(`${rec.data.type}:${rec.data.input}`);
+ me.lookup('grid-include').getStore().each((rec) => {
+ if (rec.data.type && rec.data.input) {
+ filter.push(`${rec.data.type}:${rec.data.input}`);
+ }
+ });
+ me.lookup('grid-exclude').getStore().each((rec) => {
+ if (rec.data.type && rec.data.input && rec.data.behavior) {
+ let behavior_string = '';
+ if (rec.data.behavior === 'exclude') {
+ behavior_string = 'exclude:';
+ }
+ filter.push(`${behavior_string}${rec.data.type}:${rec.data.input}`);
}
});
@@ -175,6 +214,9 @@ Ext.define('PBS.form.GroupFilter', {
},
control: {
+ 'grid pbsGroupBehaviorSelector': {
+ change: 'onBehaviorChange',
+ },
'grid pbsGroupFilterTypeSelector': {
change: 'onTypeChange',
},
@@ -264,72 +306,59 @@ Ext.define('PBS.form.GroupFilter', {
items: [
{
- xtype: 'grid',
- reference: 'grid',
+ xtype: 'pbsGroupFilterGrid',
+ title: 'Include filters',
margin: '0 0 5 0',
- scrollable: true,
- height: 300,
+ reference: 'grid-include',
store: {
- fields: ['type', 'input'],
- },
+ filters: [
+ function(item) {
+ return item.data.behavior === "include";
+ },
+ ],
+ },
emptyText: gettext('Include all groups'),
viewConfig: {
deferEmptyText: false,
},
- columns: [
+ },
+ {
+ xtype: 'container',
+ layout: {
+ type: 'hbox',
+ },
+ items: [
{
- text: gettext('Filter Type'),
- xtype: 'widgetcolumn',
- dataIndex: 'type',
- flex: 1,
- widget: {
- xtype: 'pbsGroupFilterTypeSelector',
- isFormField: false,
- },
+ xtype: 'button',
+ text: gettext('Add include'),
+ iconCls: 'fa fa-plus-circle',
+ handler: 'addIncludeFilter',
},
{
- text: gettext('Filter Value'),
- xtype: 'widgetcolumn',
+ xtype: 'box',
flex: 1,
- onWidgetAttach: 'newInputColumn',
- widget: {
- padding: 0,
- bodyPadding: 0,
- xtype: 'fieldcontainer',
- layout: 'fit',
- defaults: {
- margin: 0,
- },
- items: [
- {
- hidden: true,
- xtype: 'pbsGroupTypeSelector',
- isFormField: false,
- },
- {
- hidden: true,
- xtype: 'textfield',
- type: 'regex',
- isFormField: false,
- },
- {
- hidden: true,
- xtype: 'pbsGroupSelector',
- isFormField: false,
- },
- ],
- },
},
{
- xtype: 'widgetcolumn',
- width: 40,
- widget: {
- xtype: 'button',
- iconCls: 'fa fa-trash-o',
- },
+ xtype: 'box',
+ style: 'margin: 3px 0px;',
+ html: `<span class="pmx-hint">${gettext('Note')}</span>: `
+ + gettext('Filters are additive'),
},
],
},
+ {
+ xtype: 'pbsGroupFilterGrid',
+ title: 'Exclude filters',
+ margin: '10 0 5 0',
+ reference: 'grid-exclude',
+ store: {
+ filters: [
+ function(item) {
+ return item.data.behavior === "exclude";
+ },
+ ],
+ },
+ },
{
xtype: 'hiddenfield',
reference: 'realfield',
@@ -356,9 +385,9 @@ Ext.define('PBS.form.GroupFilter', {
items: [
{
xtype: 'button',
- text: gettext('Add'),
+ text: gettext('Add exclude'),
iconCls: 'fa fa-plus-circle',
- handler: 'addFilter',
+ handler: 'addExcludeFilter',
},
{
xtype: 'box',
@@ -368,7 +397,7 @@ Ext.define('PBS.form.GroupFilter', {
xtype: 'box',
style: 'margin: 3px 0px;',
html: `<span class="pmx-hint">${gettext('Note')}</span>: `
- + gettext('Filters are additive (OR-like)'),
+ + gettext('Exclude filters will be applied after include filters'),
},
],
},
@@ -384,6 +413,82 @@ Ext.define('PBS.form.GroupFilter', {
},
});
+Ext.define('PBS.form.pbsGroupBehaviorSelector', {
+ extend: 'Proxmox.form.KVComboBox',
+ alias: 'widget.pbsGroupBehaviorSelector',
+
+ allowBlank: false,
+
+ comboItems: [
+ ['include', gettext('Include')],
+ ['exclude', gettext('Exclude')],
+ ],
+});
+Ext.define('PBS.form.GroupFilterGrid', {
+ extend: 'Ext.grid.Panel',
+ alias: 'widget.pbsGroupFilterGrid',
+
+ scrollable: true,
+ height: 200,
+ store: {
+ fields: ['type', 'input'],
+ },
+ columns: [
+ {
+ text: gettext('Filter Type'),
+ xtype: 'widgetcolumn',
+ dataIndex: 'type',
+ flex: 1,
+ widget: {
+ xtype: 'pbsGroupFilterTypeSelector',
+ isFormField: false,
+ },
+ },
+ {
+ text: gettext('Filter Value'),
+ xtype: 'widgetcolumn',
+ flex: 1,
+ onWidgetAttach: 'newInputColumn',
+ widget: {
+ padding: 0,
+ bodyPadding: 0,
+ xtype: 'fieldcontainer',
+ layout: 'fit',
+ defaults:
+ {
+ margin: 0,
+ },
+ items: [
+ {
+ hidden: true,
+ xtype: 'pbsGroupTypeSelector',
+ isFormField: false,
+ },
+ {
+ hidden: true,
+ xtype: 'textfield',
+ type: 'regex',
+ isFormField: false,
+ },
+ {
+ hidden: true,
+ xtype: 'pbsGroupSelector',
+ isFormField: false,
+ },
+ ],
+ },
+ },
+ {
+ xtype: 'widgetcolumn',
+ width: 30,
+ widget: {
+ xtype: 'button',
+ iconCls: 'fa fa-trash-o',
+ },
+ },
+ ],
+});
+
Ext.define('PBS.form.GroupFilterTypeSelector', {
extend: 'Proxmox.form.KVComboBox',
alias: 'widget.pbsGroupFilterTypeSelector',
--
2.39.2
^ permalink raw reply [flat|nested] 6+ messages in thread