From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 8DC3069FD9 for ; Wed, 15 Sep 2021 15:42:49 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8534119B70 for ; Wed, 15 Sep 2021 15:42:19 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id B71F019B65 for ; Wed, 15 Sep 2021 15:42:18 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8EA324489C for ; Wed, 15 Sep 2021 15:42:18 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Wed, 15 Sep 2021 15:41:52 +0200 Message-Id: <20210915134157.19762-6-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210915134157.19762-1-f.gruenbichler@proxmox.com> References: <20210915134157.19762-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.386 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [pull.rs, proxmox-backup-manager.rs] Subject: [pve-devel] [PATCH v2 proxmox-backup 05/10] pull: allow pulling groups selectively X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Sep 2021 13:42:49 -0000 without requiring workarounds based on ownership and limited visibility/access. if a group filter is set, remove_vanished will only consider filtered groups for removal to prevent concurrent disjunct filters from trashing eachother's synced groups. Signed-off-by: Fabian Grünbichler --- other name options: - `group-filter` (with `exclude` boolean as extension?) - `include-groups` (with `exclude-groups` filter list as extension?) src/api2/pull.rs | 13 ++++++++++--- src/bin/proxmox-backup-manager.rs | 10 ++++++++++ src/server/pull.rs | 32 ++++++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/api2/pull.rs b/src/api2/pull.rs index d7b155a1..dfa6a938 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission}; use pbs_client::{HttpClient, BackupRepository}; use pbs_api_types::{ - Remote, Authid, SyncJobConfig, + Remote, Authid, SyncJobConfig, GroupFilterList, DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, }; @@ -102,7 +102,7 @@ pub fn do_sync_job( worker.log(format!("Sync datastore '{}' from '{}/{}'", sync_job.store, sync_job.remote, sync_job.remote_store)); - pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?; + pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner, None).await?; worker.log(format!("sync job '{}' end", &job_id)); @@ -153,6 +153,10 @@ pub fn do_sync_job( schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + type: GroupFilterList, + optional: true, + }, }, }, access: { @@ -170,6 +174,7 @@ async fn pull ( remote: String, remote_store: String, remove_vanished: Option, + groups: Option, _info: &ApiMethod, rpcenv: &mut dyn RpcEnvironment, ) -> Result { @@ -177,6 +182,8 @@ async fn pull ( let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; let delete = remove_vanished.unwrap_or(true); + let groups = groups.map(GroupFilterList::filters); + check_pull_privs(&auth_id, &store, &remote, &remote_store, delete)?; let (client, src_repo, tgt_store) = get_pull_parameters(&store, &remote, &remote_store).await?; @@ -186,7 +193,7 @@ async fn pull ( worker.log(format!("sync datastore '{}' start", store)); - let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id); + let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id, groups); let future = select!{ success = pull_future.fuse() => success, abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort, diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 689f44db..479ecf03 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -12,6 +12,7 @@ use pbs_tools::json::required_string_param; use pbs_api_types::{ DATASTORE_SCHEMA, UPID_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, IGNORE_VERIFIED_BACKUPS_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA, + GroupFilterList, }; use proxmox_backup::config; @@ -234,6 +235,10 @@ fn task_mgmt_cli() -> CommandLineInterface { schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + type: GroupFilterList, + optional: true, + }, "output-format": { schema: OUTPUT_FORMAT, optional: true, @@ -247,6 +252,7 @@ async fn pull_datastore( remote_store: String, local_store: String, remove_vanished: Option, + groups: Option, param: Value, ) -> Result { @@ -260,6 +266,10 @@ async fn pull_datastore( "remote-store": remote_store, }); + if groups.is_some() { + args["groups"] = json!(groups); + } + if let Some(remove_vanished) = remove_vanished { args["remove-vanished"] = Value::from(remove_vanished); } diff --git a/src/server/pull.rs b/src/server/pull.rs index 4e82df1a..2910d280 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -12,7 +12,7 @@ use serde_json::json; use proxmox::api::error::{HttpError, StatusCode}; -use pbs_api_types::{Authid, SnapshotListItem, GroupListItem}; +use pbs_api_types::{Authid, SnapshotListItem, GroupListItem, GroupFilter}; use pbs_datastore::{task_log, BackupInfo, BackupDir, BackupGroup, StoreProgress}; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::dynamic_index::DynamicIndexReader; @@ -631,6 +631,7 @@ pub async fn pull_store( tgt_store: Arc, delete: bool, auth_id: Authid, + group_filter: Option>, ) -> Result<(), Error> { // explicit create shared lock to prevent GC on newly created chunks let _shared_store_lock = tgt_store.try_shared_chunk_store_lock()?; @@ -644,8 +645,7 @@ pub async fn pull_store( let mut list: Vec = serde_json::from_value(result["data"].take())?; - worker.log(format!("found {} groups to sync", list.len())); - + let total_count = list.len(); list.sort_unstable_by(|a, b| { let type_order = a.backup_type.cmp(&b.backup_type); if type_order == std::cmp::Ordering::Equal { @@ -655,11 +655,32 @@ pub async fn pull_store( } }); + let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool { + filters + .iter() + .any(|filter| group.filter(filter)) + }; + let list:Vec = list .into_iter() .map(|item| BackupGroup::new(item.backup_type, item.backup_id)) .collect(); + let list = if let Some(ref group_filter) = &group_filter { + let unfiltered_count = list.len(); + let list:Vec = list + .into_iter() + .filter(|group| { + apply_filters(&group, group_filter) + }) + .collect(); + worker.log(format!("found {} groups to sync (out of {} total)", list.len(), unfiltered_count)); + list + } else { + worker.log(format!("found {} groups to sync", total_count)); + list + }; + let mut errors = false; let mut new_groups = std::collections::HashSet::new(); @@ -720,6 +741,11 @@ pub async fn pull_store( if new_groups.contains(&local_group) { continue; } + if let Some(ref group_filter) = &group_filter { + if !apply_filters(&local_group, group_filter) { + continue; + } + } worker.log(format!( "delete vanished group '{}/{}'", local_group.backup_type(), -- 2.30.2