From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH v3 proxmox-backup 06/11] pull: allow pulling groups selectively
Date: Thu, 28 Oct 2021 15:00:53 +0200 [thread overview]
Message-ID: <20211028130058.1308810-8-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20211028130058.1308810-1-f.gruenbichler@proxmox.com>
without requiring workarounds based on ownership and limited
visibility/access.
if a group filter is set, remove_vanished will only consider filtered
groups for removal to prevent concurrent disjunct filters from trashing
eachother's synced groups.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
Notes:
v2->v3: use Vec<GroupFilter> and GROUP_FILTER_LIST_SCHEMA
src/api2/pull.rs | 9 +++++++-
src/bin/proxmox-backup-manager.rs | 14 ++++++++++--
src/server/pull.rs | 38 +++++++++++++++++++++++++++----
3 files changed, 53 insertions(+), 8 deletions(-)
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index 5ae916ed..d4a14f10 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -8,7 +8,7 @@ use proxmox_schema::api;
use proxmox_router::{ApiMethod, Router, RpcEnvironment, Permission};
use pbs_api_types::{
- Authid, SyncJobConfig,
+ Authid, SyncJobConfig, GroupFilter, GROUP_FILTER_LIST_SCHEMA,
DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ,
};
@@ -50,6 +50,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters {
&sync_job.remote_store,
sync_job.owner.as_ref().unwrap_or_else(|| Authid::root_auth_id()).clone(),
sync_job.remove_vanished,
+ None,
)
}
}
@@ -151,6 +152,10 @@ pub fn do_sync_job(
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ schema: GROUP_FILTER_LIST_SCHEMA,
+ optional: true,
+ },
},
},
access: {
@@ -168,6 +173,7 @@ async fn pull (
remote: String,
remote_store: String,
remove_vanished: Option<bool>,
+ groups: Option<Vec<GroupFilter>>,
_info: &ApiMethod,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
@@ -183,6 +189,7 @@ async fn pull (
&remote_store,
auth_id.clone(),
remove_vanished,
+ groups,
)?;
let client = pull_params.client().await?;
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 92e6bb2a..9e52a474 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -12,8 +12,9 @@ use pbs_client::{display_task_log, view_task_result};
use pbs_tools::percent_encoding::percent_encode_component;
use pbs_tools::json::required_string_param;
use pbs_api_types::{
- DATASTORE_SCHEMA, UPID_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
- IGNORE_VERIFIED_BACKUPS_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA,
+ GroupFilter,
+ DATASTORE_SCHEMA, GROUP_FILTER_LIST_SCHEMA, IGNORE_VERIFIED_BACKUPS_SCHEMA, REMOTE_ID_SCHEMA,
+ REMOVE_VANISHED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA,
};
use proxmox_rest_server::wait_for_local_worker;
@@ -238,6 +239,10 @@ fn task_mgmt_cli() -> CommandLineInterface {
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ schema: GROUP_FILTER_LIST_SCHEMA,
+ optional: true,
+ },
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
@@ -251,6 +256,7 @@ async fn pull_datastore(
remote_store: String,
local_store: String,
remove_vanished: Option<bool>,
+ groups: Option<Vec<GroupFilter>>,
param: Value,
) -> Result<Value, Error> {
@@ -264,6 +270,10 @@ async fn pull_datastore(
"remote-store": remote_store,
});
+ if groups.is_some() {
+ args["groups"] = json!(groups);
+ }
+
if let Some(remove_vanished) = remove_vanished {
args["remove-vanished"] = Value::from(remove_vanished);
}
diff --git a/src/server/pull.rs b/src/server/pull.rs
index 2c454e2d..63bf92b4 100644
--- a/src/server/pull.rs
+++ b/src/server/pull.rs
@@ -13,8 +13,9 @@ use http::StatusCode;
use proxmox_router::HttpError;
-use pbs_api_types::{Authid, GroupListItem, Remote, SnapshotListItem};
-use pbs_datastore::{DataStore, BackupInfo, BackupDir, BackupGroup, StoreProgress};
+use pbs_api_types::{Authid, GroupFilter, GroupListItem, Remote, SnapshotListItem};
+
+use pbs_datastore::{BackupDir, BackupInfo, BackupGroup, DataStore, StoreProgress};
use pbs_datastore::data_blob::DataBlob;
use pbs_datastore::dynamic_index::DynamicIndexReader;
use pbs_datastore::fixed_index::FixedIndexReader;
@@ -39,6 +40,7 @@ pub struct PullParameters {
store: Arc<DataStore>,
owner: Authid,
remove_vanished: bool,
+ group_filter: Option<Vec<GroupFilter>>,
}
impl PullParameters {
@@ -48,6 +50,7 @@ impl PullParameters {
remote_store: &str,
owner: Authid,
remove_vanished: Option<bool>,
+ group_filter: Option<Vec<GroupFilter>>,
) -> Result<Self, Error> {
let store = DataStore::lookup_datastore(store)?;
@@ -63,7 +66,7 @@ impl PullParameters {
remote_store.to_string(),
);
- Ok(Self { remote, source, store, owner, remove_vanished })
+ Ok(Self { remote, source, store, owner, remove_vanished, group_filter })
}
pub async fn client(&self) -> Result<HttpClient, Error> {
@@ -678,8 +681,7 @@ pub async fn pull_store(
let mut list: Vec<GroupListItem> = serde_json::from_value(result["data"].take())?;
- task_log!(worker, "found {} groups to sync", list.len());
-
+ let total_count = list.len();
list.sort_unstable_by(|a, b| {
let type_order = a.backup_type.cmp(&b.backup_type);
if type_order == std::cmp::Ordering::Equal {
@@ -689,11 +691,32 @@ pub async fn pull_store(
}
});
+ let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool {
+ filters
+ .iter()
+ .any(|filter| group.matches(filter))
+ };
+
let list:Vec<BackupGroup> = list
.into_iter()
.map(|item| BackupGroup::new(item.backup_type, item.backup_id))
.collect();
+ let list = if let Some(ref group_filter) = ¶ms.group_filter {
+ let unfiltered_count = list.len();
+ let list:Vec<BackupGroup> = list
+ .into_iter()
+ .filter(|group| {
+ apply_filters(&group, group_filter)
+ })
+ .collect();
+ task_log!(worker, "found {} groups to sync (out of {} total)", list.len(), unfiltered_count);
+ list
+ } else {
+ task_log!(worker, "found {} groups to sync", total_count);
+ list
+ };
+
let mut errors = false;
let mut new_groups = std::collections::HashSet::new();
@@ -755,6 +778,11 @@ pub async fn pull_store(
if new_groups.contains(&local_group) {
continue;
}
+ if let Some(ref group_filter) = ¶ms.group_filter {
+ if !apply_filters(&local_group, group_filter) {
+ continue;
+ }
+ }
task_log!(
worker,
"delete vanished group '{}/{}'",
--
2.30.2
next prev parent reply other threads:[~2021-10-28 13:02 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-10-28 13:00 [pbs-devel] [PATCH v3 proxmox-backup 0/12] pull/sync group filter Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox 1/1] updater: impl UpdaterType for Vec Fabian Grünbichler
2021-11-09 8:31 ` [pbs-devel] applied: " Dietmar Maurer
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 01/11] api-types: add schema for backup group Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 02/11] api: add GroupFilter(List) type Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 03/11] BackupGroup: add filter helper Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 04/11] pull: use BackupGroup consistently Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 05/11] pull/sync: extract passed along vars into struct Fabian Grünbichler
2021-10-28 13:00 ` Fabian Grünbichler [this message]
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 07/11] sync: add group filtering Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 08/11] remote: add backup group scanning Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 09/11] manager: render group filter properly Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [PATCH v3 proxmox-backup 10/11] docs: mention group filter in sync docs Fabian Grünbichler
2021-10-28 13:00 ` [pbs-devel] [RFC v3 proxmox-backup 11/11] fix #sync.cfg/pull: don't remove by default Fabian Grünbichler
2021-11-04 9:57 ` [pbs-devel] [PATCH v3 proxmox-backup 0/12] pull/sync group filter Dominik Csapak
2021-11-18 10:11 ` [pbs-devel] applied: " Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211028130058.1308810-8-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox