From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 6232F7A10C for ; Thu, 28 Oct 2021 15:02:16 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 58E4E20DB6 for ; Thu, 28 Oct 2021 15:01:46 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 6662D20DAB for ; Thu, 28 Oct 2021 15:01:45 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4007545FDF for ; Thu, 28 Oct 2021 15:01:45 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Thu, 28 Oct 2021 15:00:53 +0200 Message-Id: <20211028130058.1308810-8-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211028130058.1308810-1-f.gruenbichler@proxmox.com> References: <20211028130058.1308810-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.331 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox-backup-manager.rs, pull.rs] Subject: [pbs-devel] [PATCH v3 proxmox-backup 06/11] pull: allow pulling groups selectively X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Oct 2021 13:02:16 -0000 without requiring workarounds based on ownership and limited visibility/access. if a group filter is set, remove_vanished will only consider filtered groups for removal to prevent concurrent disjunct filters from trashing eachother's synced groups. Signed-off-by: Fabian Grünbichler --- Notes: v2->v3: use Vec and GROUP_FILTER_LIST_SCHEMA src/api2/pull.rs | 9 +++++++- src/bin/proxmox-backup-manager.rs | 14 ++++++++++-- src/server/pull.rs | 38 +++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 5ae916ed..d4a14f10 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -8,7 +8,7 @@ use proxmox_schema::api; use proxmox_router::{ApiMethod, Router, RpcEnvironment, Permission}; use pbs_api_types::{ - Authid, SyncJobConfig, + Authid, SyncJobConfig, GroupFilter, GROUP_FILTER_LIST_SCHEMA, DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, }; @@ -50,6 +50,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters { &sync_job.remote_store, sync_job.owner.as_ref().unwrap_or_else(|| Authid::root_auth_id()).clone(), sync_job.remove_vanished, + None, ) } } @@ -151,6 +152,10 @@ pub fn do_sync_job( schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + schema: GROUP_FILTER_LIST_SCHEMA, + optional: true, + }, }, }, access: { @@ -168,6 +173,7 @@ async fn pull ( remote: String, remote_store: String, remove_vanished: Option, + groups: Option>, _info: &ApiMethod, rpcenv: &mut dyn RpcEnvironment, ) -> Result { @@ -183,6 +189,7 @@ async fn pull ( &remote_store, auth_id.clone(), remove_vanished, + groups, )?; let client = pull_params.client().await?; diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 92e6bb2a..9e52a474 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -12,8 +12,9 @@ use pbs_client::{display_task_log, view_task_result}; use pbs_tools::percent_encoding::percent_encode_component; use pbs_tools::json::required_string_param; use pbs_api_types::{ - DATASTORE_SCHEMA, UPID_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, - IGNORE_VERIFIED_BACKUPS_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA, + GroupFilter, + DATASTORE_SCHEMA, GROUP_FILTER_LIST_SCHEMA, IGNORE_VERIFIED_BACKUPS_SCHEMA, REMOTE_ID_SCHEMA, + REMOVE_VANISHED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA, }; use proxmox_rest_server::wait_for_local_worker; @@ -238,6 +239,10 @@ fn task_mgmt_cli() -> CommandLineInterface { schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + schema: GROUP_FILTER_LIST_SCHEMA, + optional: true, + }, "output-format": { schema: OUTPUT_FORMAT, optional: true, @@ -251,6 +256,7 @@ async fn pull_datastore( remote_store: String, local_store: String, remove_vanished: Option, + groups: Option>, param: Value, ) -> Result { @@ -264,6 +270,10 @@ async fn pull_datastore( "remote-store": remote_store, }); + if groups.is_some() { + args["groups"] = json!(groups); + } + if let Some(remove_vanished) = remove_vanished { args["remove-vanished"] = Value::from(remove_vanished); } diff --git a/src/server/pull.rs b/src/server/pull.rs index 2c454e2d..63bf92b4 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -13,8 +13,9 @@ use http::StatusCode; use proxmox_router::HttpError; -use pbs_api_types::{Authid, GroupListItem, Remote, SnapshotListItem}; -use pbs_datastore::{DataStore, BackupInfo, BackupDir, BackupGroup, StoreProgress}; +use pbs_api_types::{Authid, GroupFilter, GroupListItem, Remote, SnapshotListItem}; + +use pbs_datastore::{BackupDir, BackupInfo, BackupGroup, DataStore, StoreProgress}; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::dynamic_index::DynamicIndexReader; use pbs_datastore::fixed_index::FixedIndexReader; @@ -39,6 +40,7 @@ pub struct PullParameters { store: Arc, owner: Authid, remove_vanished: bool, + group_filter: Option>, } impl PullParameters { @@ -48,6 +50,7 @@ impl PullParameters { remote_store: &str, owner: Authid, remove_vanished: Option, + group_filter: Option>, ) -> Result { let store = DataStore::lookup_datastore(store)?; @@ -63,7 +66,7 @@ impl PullParameters { remote_store.to_string(), ); - Ok(Self { remote, source, store, owner, remove_vanished }) + Ok(Self { remote, source, store, owner, remove_vanished, group_filter }) } pub async fn client(&self) -> Result { @@ -678,8 +681,7 @@ pub async fn pull_store( let mut list: Vec = serde_json::from_value(result["data"].take())?; - task_log!(worker, "found {} groups to sync", list.len()); - + let total_count = list.len(); list.sort_unstable_by(|a, b| { let type_order = a.backup_type.cmp(&b.backup_type); if type_order == std::cmp::Ordering::Equal { @@ -689,11 +691,32 @@ pub async fn pull_store( } }); + let apply_filters = |group: &BackupGroup, filters: &[GroupFilter]| -> bool { + filters + .iter() + .any(|filter| group.matches(filter)) + }; + let list:Vec = list .into_iter() .map(|item| BackupGroup::new(item.backup_type, item.backup_id)) .collect(); + let list = if let Some(ref group_filter) = ¶ms.group_filter { + let unfiltered_count = list.len(); + let list:Vec = list + .into_iter() + .filter(|group| { + apply_filters(&group, group_filter) + }) + .collect(); + task_log!(worker, "found {} groups to sync (out of {} total)", list.len(), unfiltered_count); + list + } else { + task_log!(worker, "found {} groups to sync", total_count); + list + }; + let mut errors = false; let mut new_groups = std::collections::HashSet::new(); @@ -755,6 +778,11 @@ pub async fn pull_store( if new_groups.contains(&local_group) { continue; } + if let Some(ref group_filter) = ¶ms.group_filter { + if !apply_filters(&local_group, group_filter) { + continue; + } + } task_log!( worker, "delete vanished group '{}/{}'", -- 2.30.2