From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 34E7A68C19 for ; Thu, 22 Jul 2021 16:35:57 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2761012FF6 for ; Thu, 22 Jul 2021 16:35:27 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 1E17512FEB for ; Thu, 22 Jul 2021 16:35:26 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E8EEF42587 for ; Thu, 22 Jul 2021 16:35:25 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Thu, 22 Jul 2021 16:35:05 +0200 Message-Id: <20210722143510.238967-3-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210722143510.238967-1-f.gruenbichler@proxmox.com> References: <20210722143510.238967-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.460 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 2/7] pull: allow pulling groups selectively X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 22 Jul 2021 14:35:57 -0000 without requiring workarounds based on ownership and limited visibility/access. if a group filter is set, remove_vanished will only consider filtered groups for removal to prevent concurrent disjunct filters from trashing eachother's synced groups. Signed-off-by: Fabian Grünbichler --- src/api2/pull.rs | 29 +++++++++++++++++++++++++---- src/bin/proxmox-backup-manager.rs | 14 ++++++++++++++ src/server/pull.rs | 24 ++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 6 deletions(-) diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 4893c9fb..36149761 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -1,5 +1,6 @@ //! Sync datastore from remote server use std::sync::{Arc}; +use std::str::FromStr; use anyhow::{format_err, Error}; use futures::{select, future::FutureExt}; @@ -10,9 +11,9 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission}; use pbs_client::{HttpClient, BackupRepository}; use crate::server::{WorkerTask, jobstate::Job, pull::pull_store}; -use crate::backup::DataStore; +use crate::backup::{BackupGroup, DataStore}; use crate::api2::types::{ - DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid, + BACKUP_GROUP_SCHEMA, DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid, }; use crate::config::{ remote, @@ -101,7 +102,7 @@ pub fn do_sync_job( worker.log(format!("Sync datastore '{}' from '{}/{}'", sync_job.store, sync_job.remote, sync_job.remote_store)); - pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?; + pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner, None).await?; worker.log(format!("sync job '{}' end", &job_id)); @@ -152,6 +153,14 @@ pub fn do_sync_job( schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + type: Array, + description: "List of group identifiers to filter for. All if unspecified.", + items: { + schema: BACKUP_GROUP_SCHEMA, + }, + optional: true, + }, }, }, access: { @@ -169,6 +178,7 @@ async fn pull ( remote: String, remote_store: String, remove_vanished: Option, + groups: Option>, _info: &ApiMethod, rpcenv: &mut dyn RpcEnvironment, ) -> Result { @@ -185,7 +195,18 @@ async fn pull ( worker.log(format!("sync datastore '{}' start", store)); - let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id); + let groups = match groups { + Some(filter) => { + let mut groups = std::collections::HashSet::with_capacity(filter.len()); + for group in filter { + let group = BackupGroup::from_str(&group)?; + groups.insert(group); + } + Some(groups) + }, + None => None, + }; + let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id, groups); let future = select!{ success = pull_future.fuse() => success, abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort, diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 93d6de57..6844a1ab 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -230,6 +230,15 @@ fn task_mgmt_cli() -> CommandLineInterface { schema: REMOVE_VANISHED_BACKUPS_SCHEMA, optional: true, }, + "groups": { + type: Array, + description: "List of group identifiers to filter for. All if unspecified.", + items: { + type: String, + description: "Backup group identifier", + }, + optional: true, + }, "output-format": { schema: OUTPUT_FORMAT, optional: true, @@ -243,6 +252,7 @@ async fn pull_datastore( remote_store: String, local_store: String, remove_vanished: Option, + groups: Option>, param: Value, ) -> Result { @@ -256,6 +266,10 @@ async fn pull_datastore( "remote-store": remote_store, }); + if groups.is_some() { + args["groups"] = json!(groups); + } + if let Some(remove_vanished) = remove_vanished { args["remove-vanished"] = Value::from(remove_vanished); } diff --git a/src/server/pull.rs b/src/server/pull.rs index 5214a218..2904c37f 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -631,6 +631,7 @@ pub async fn pull_store( tgt_store: Arc, delete: bool, auth_id: Authid, + group_filter: Option>, ) -> Result<(), Error> { // explicit create shared lock to prevent GC on newly created chunks let _shared_store_lock = tgt_store.try_shared_chunk_store_lock()?; @@ -644,8 +645,7 @@ pub async fn pull_store( let mut list: Vec = serde_json::from_value(result["data"].take())?; - worker.log(format!("found {} groups to sync", list.len())); - + let total_count = list.len(); list.sort_unstable_by(|a, b| { let type_order = a.backup_type.cmp(&b.backup_type); if type_order == std::cmp::Ordering::Equal { @@ -655,6 +655,21 @@ pub async fn pull_store( } }); + + let list = if let Some(ref group_filter) = &group_filter { + let list:Vec = list + .into_iter() + .filter(|group| { + group_filter.contains(&BackupGroup::new(&group.backup_type, &group.backup_id)) + }) + .collect(); + worker.log(format!("found {} groups to sync (out of {} requested by filter)", list.len(), group_filter.len())); + list + } else { + worker.log(format!("found {} groups to sync", total_count)); + list + }; + let mut errors = false; let mut new_groups = std::collections::HashSet::new(); @@ -717,6 +732,11 @@ pub async fn pull_store( if new_groups.contains(&local_group) { continue; } + if let Some(ref group_filter) = &group_filter { + if !group_filter.contains(&local_group) { + continue; + } + } worker.log(format!( "delete vanished group '{}/{}'", local_group.backup_type(), -- 2.30.2