[pbs-devel] [PATCH proxmox-backup 0/5] add group filters to tape backup (jobs)

[pbs-devel] [PATCH proxmox-backup 0/5] add group filters to tape backup (jobs)

[Dominik Csapak]

requires fabians pull/sync groupfilter patches to work[0]

no gui support for setting filters yet, but i'm working on it
(for sync+tape)

0: https://lists.proxmox.com/pipermail/pbs-devel/2021-October/004265.html

Dominik Csapak (5):
  proxmox-tape: add missing 'notify-user' option to backup command
  tape backup jobs: add group filters to config/api
  fix #3533: tape backup: filter groups according to config
  ui: tape: show configred group filters
  proxmox-tape: add groups filter to backup command

 pbs-api-types/src/jobs.rs          |  6 ++++
 src/api2/config/tape_backup_job.rs |  4 +++
 src/api2/tape/backup.rs            | 20 +++++++++++--
 src/bin/proxmox-tape.rs            | 47 ++++++++++++++++++++++++++++--
 www/tape/BackupJobs.js             |  8 ++++-
 www/tape/window/TapeBackupJob.js   |  9 ++++++
 6 files changed, 88 insertions(+), 6 deletions(-)

-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 1/5] proxmox-tape: add missing 'notify-user' option to backup command

[Dominik Csapak]

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 src/bin/proxmox-tape.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/bin/proxmox-tape.rs b/src/bin/proxmox-tape.rs
index 5de727c1..b7979026 100644
--- a/src/bin/proxmox-tape.rs
+++ b/src/bin/proxmox-tape.rs
@@ -830,6 +830,10 @@ async fn clean_drive(mut param: Value) -> Result<(), Error> {
                 type: bool,
                 optional: true,
             },
+            "notify-user": {
+                optional: true,
+                type: Userid,
+            },
             "force-media-set": {
                 description: "Ignore the allocation policy and start a new media-set.",
                 optional: true,
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 2/5] tape backup jobs: add group filters to config/api

[Dominik Csapak]

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 pbs-api-types/src/jobs.rs          | 6 ++++++
 src/api2/config/tape_backup_job.rs | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs
index 18c55dad..320c879e 100644
--- a/pbs-api-types/src/jobs.rs
+++ b/pbs-api-types/src/jobs.rs
@@ -247,6 +247,10 @@ pub struct VerificationJobStatus {
             optional: true,
             type: Userid,
         },
+        groups: {
+            schema: GROUP_FILTER_LIST_SCHEMA,
+            optional: true,
+        },
     }
 )]
 #[derive(Serialize,Deserialize,Clone,Updater)]
@@ -265,6 +269,8 @@ pub struct TapeBackupJobSetup {
     /// Send job email notification to this user
     #[serde(skip_serializing_if="Option::is_none")]
     pub notify_user: Option<Userid>,
+    #[serde(skip_serializing_if="Option::is_none")]
+    pub groups: Option<Vec<GroupFilter>>,
 }
 
 #[api(
diff --git a/src/api2/config/tape_backup_job.rs b/src/api2/config/tape_backup_job.rs
index 6c5680d3..84dda075 100644
--- a/src/api2/config/tape_backup_job.rs
+++ b/src/api2/config/tape_backup_job.rs
@@ -133,6 +133,8 @@ pub enum DeletableProperty {
     LatestOnly,
     /// Delete the 'notify-user' property
     NotifyUser,
+    /// Delete the 'groups' property
+    Groups,
 }
 
 #[api(
@@ -191,6 +193,7 @@ pub fn update_tape_backup_job(
                 DeletableProperty::NotifyUser => { data.setup.notify_user = None; },
                 DeletableProperty::Schedule => { data.schedule = None; },
                 DeletableProperty::Comment => { data.comment = None; },
+                DeletableProperty::Groups => { data.setup.groups = None; },
             }
         }
     }
@@ -203,6 +206,7 @@ pub fn update_tape_backup_job(
     if update.setup.export_media_set.is_some() { data.setup.export_media_set = update.setup.export_media_set; }
     if update.setup.latest_only.is_some() { data.setup.latest_only = update.setup.latest_only; }
     if update.setup.notify_user.is_some() { data.setup.notify_user = update.setup.notify_user; }
+    if update.setup.groups.is_some() { data.setup.groups = update.setup.groups; }
 
     let schedule_changed = data.schedule != update.schedule;
     if update.schedule.is_some() { data.schedule = update.schedule; }
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 3/5] fix #3533: tape backup: filter groups according to config

[Dominik Csapak]

this fixes bug #3533, since now a user can backup a single datastore
on multiple tape media pools in parallel, e.g. vms on one pool, ct on
another.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 src/api2/tape/backup.rs | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/api2/tape/backup.rs b/src/api2/tape/backup.rs
index a0578391..1d775459 100644
--- a/src/api2/tape/backup.rs
+++ b/src/api2/tape/backup.rs
@@ -11,10 +11,11 @@ use proxmox_schema::api;
 use pbs_api_types::{
     Authid, Userid, TapeBackupJobConfig, TapeBackupJobSetup, TapeBackupJobStatus, MediaPoolConfig,
     UPID_SCHEMA, JOB_ID_SCHEMA, PRIV_DATASTORE_READ, PRIV_TAPE_AUDIT, PRIV_TAPE_WRITE,
+    GroupFilter,
 };
 
 use pbs_datastore::{DataStore, StoreProgress, SnapshotReader};
-use pbs_datastore::backup_info::{BackupDir, BackupInfo};
+use pbs_datastore::backup_info::{BackupDir, BackupInfo, BackupGroup};
 use pbs_tools::{task_log, task_warn, task::WorkerTaskContext};
 use pbs_config::CachedUserInfo;
 use proxmox_rest_server::WorkerTask;
@@ -436,8 +437,21 @@ fn backup_worker(
 
     group_list.sort_unstable();
 
-    let group_count = group_list.len();
-    task_log!(worker, "found {} groups", group_count);
+    let (group_list, group_count) = if let Some(group_filters) = &setup.groups {
+        let filter_fn = |group: &BackupGroup, group_filters: &[GroupFilter]| {
+            group_filters.iter().any(|filter| group.matches(filter))
+        };
+
+        let group_count_full = group_list.len();
+        let list: Vec<BackupGroup> = group_list.into_iter().filter(|group| filter_fn(group, &group_filters)).collect();
+        let group_count = list.len();
+        task_log!(worker, "found {} groups (out of {} total)", group_count, group_count_full);
+        (list, group_count)
+    } else {
+        let group_count = group_list.len();
+        task_log!(worker, "found {} groups", group_count);
+        (group_list, group_count)
+    };
 
     let mut progress = StoreProgress::new(group_count as u64);
 
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 4/5] ui: tape: show configred group filters

[Dominik Csapak]

in the grid and in the edit window

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 www/tape/BackupJobs.js           | 8 +++++++-
 www/tape/window/TapeBackupJob.js | 9 +++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/www/tape/BackupJobs.js b/www/tape/BackupJobs.js
index 6a80c97e..b6ff8b14 100644
--- a/www/tape/BackupJobs.js
+++ b/www/tape/BackupJobs.js
@@ -1,7 +1,7 @@
 Ext.define('pbs-tape-backup-job-status', {
     extend: 'Ext.data.Model',
     fields: [
-	'id', 'store', 'pool', 'drive', 'store', 'schedule', 'comment',
+	'id', 'store', 'pool', 'drive', 'store', 'schedule', 'comment', 'groups',
 	{ name: 'eject-media', type: 'boolean' },
 	{ name: 'export-media-set', type: 'boolean' },
 	{ name: 'latest-only', type: 'boolean' },
@@ -221,6 +221,12 @@ Ext.define('PBS.config.TapeBackupJobView', {
 	    renderer: Proxmox.Utils.format_boolean,
 	    sortable: false,
 	},
+	{
+	    header: gettext('Backup Groups'),
+	    dataIndex: 'groups',
+	    renderer: v => v ? Ext.String.htmlEncode(v) : gettext('All'),
+	    width: 80,
+	},
 	{
 	    header: gettext('Schedule'),
 	    dataIndex: 'schedule',
diff --git a/www/tape/window/TapeBackupJob.js b/www/tape/window/TapeBackupJob.js
index c5541d87..f77c18ec 100644
--- a/www/tape/window/TapeBackupJob.js
+++ b/www/tape/window/TapeBackupJob.js
@@ -123,6 +123,15 @@ Ext.define('PBS.TapeManagement.BackupJobEdit', {
 	],
 
 	columnB: [
+	    {
+		fieldLabel: gettext('Backup Groups'),
+		xtype: 'displayfield',
+		name: 'groups',
+		renderer: v => v ? Ext.String.htmlEncode(v) : gettext('All'),
+		cbind: {
+		    hidden: '{isCreate}',
+		},
+	    },
 	    {
 		fieldLabel: gettext('Comment'),
 		xtype: 'proxmoxtextfield',
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 5/5] proxmox-tape: add groups filter to backup command

[Dominik Csapak]

and add a completion handler to complete the backup groups

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 src/bin/proxmox-tape.rs | 43 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/src/bin/proxmox-tape.rs b/src/bin/proxmox-tape.rs
index b7979026..f49857dd 100644
--- a/src/bin/proxmox-tape.rs
+++ b/src/bin/proxmox-tape.rs
@@ -1,4 +1,6 @@
-use anyhow::{format_err, Error};
+use std::collections::HashMap;
+
+use anyhow::{bail, format_err, Error};
 use serde_json::{json, Value};
 
 use proxmox_io::ReadExt;
@@ -22,7 +24,7 @@ use pbs_config::datastore::complete_datastore_name;
 use pbs_api_types::{
     Userid, Authid, DATASTORE_SCHEMA, DATASTORE_MAP_LIST_SCHEMA,
     DRIVE_NAME_SCHEMA, MEDIA_LABEL_SCHEMA, MEDIA_POOL_NAME_SCHEMA,
-    TAPE_RESTORE_SNAPSHOT_SCHEMA,
+    TAPE_RESTORE_SNAPSHOT_SCHEMA, GROUP_FILTER_LIST_SCHEMA, GroupListItem,
 };
 use pbs_tape::{
     PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0, BlockReadError, MediaContentHeader,
@@ -49,6 +51,38 @@ use proxmox_backup::{
 mod proxmox_tape;
 use proxmox_tape::*;
 
+async fn get_backup_groups(store: &str) -> Result<Vec<GroupListItem>, Error> {
+    let client = connect_to_localhost()?;
+    let api_res = client
+        .get(&format!("api2/json/admin/datastore/{}/groups", store), None)
+        .await?;
+
+    match api_res.get("data") {
+        Some(data) => Ok(serde_json::from_value::<Vec<GroupListItem>>(data.to_owned())?),
+        None => bail!("could not get group list"),
+    }
+}
+
+// shell completion helper
+pub fn complete_datastore_group_filter(_arg: &str, param: &HashMap<String, String>) -> Vec<String> {
+
+    let mut list = Vec::new();
+
+    list.push("regex:".to_string());
+    list.push("type:ct".to_string());
+    list.push("type:host".to_string());
+    list.push("type:vm".to_string());
+
+    if let Some(store) =  param.get("store") {
+        let groups = pbs_runtime::block_on(async { get_backup_groups(store).await });
+        if let Ok(groups) = groups {
+            list.extend(groups.iter().map(|group| format!("group:{}/{}", group.backup_type, group.backup_id)));
+        }
+    }
+
+    list
+}
+
 pub fn extract_drive_name(
     param: &mut Value,
     config: &SectionConfigData,
@@ -834,6 +868,10 @@ async fn clean_drive(mut param: Value) -> Result<(), Error> {
                 optional: true,
                 type: Userid,
             },
+            groups: {
+                schema: GROUP_FILTER_LIST_SCHEMA,
+                optional: true,
+            },
             "force-media-set": {
                 description: "Ignore the allocation policy and start a new media-set.",
                 optional: true,
@@ -978,6 +1016,7 @@ fn main() {
                 .completion_cb("drive", complete_drive_name)
                 .completion_cb("store", complete_datastore_name)
                 .completion_cb("pool", complete_pool_name)
+                .completion_cb("groups", complete_datastore_group_filter)
         )
         .insert(
             "restore",
-- 
2.30.2

[pbs-devel] applied-series: [PATCH proxmox-backup 0/5] add group filters to tape backup (jobs)

[Thomas Lamprecht]

On 04.11.21 10:56, Dominik Csapak wrote:
> requires fabians pull/sync groupfilter patches to work[0]
> 
> no gui support for setting filters yet, but i'm working on it
> (for sync+tape)
> 
> 0: https://lists.proxmox.com/pipermail/pbs-devel/2021-October/004265.html
> 
> Dominik Csapak (5):
>   proxmox-tape: add missing 'notify-user' option to backup command
>   tape backup jobs: add group filters to config/api
>   fix #3533: tape backup: filter groups according to config
>   ui: tape: show configred group filters
>   proxmox-tape: add groups filter to backup command
> 
>  pbs-api-types/src/jobs.rs          |  6 ++++
>  src/api2/config/tape_backup_job.rs |  4 +++
>  src/api2/tape/backup.rs            | 20 +++++++++++--
>  src/bin/proxmox-tape.rs            | 47 ++++++++++++++++++++++++++++--
>  www/tape/BackupJobs.js             |  8 ++++-
>  www/tape/window/TapeBackupJob.js   |  9 ++++++
>  6 files changed, 88 insertions(+), 6 deletions(-)
> 

applied series, thanks!

renamed `groups` to `group-filter` in a follow up.