From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 2/7] pull: allow pulling groups selectively
Date: Thu, 22 Jul 2021 16:35:05 +0200 [thread overview]
Message-ID: <20210722143510.238967-3-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20210722143510.238967-1-f.gruenbichler@proxmox.com>
without requiring workarounds based on ownership and limited
visibility/access.
if a group filter is set, remove_vanished will only consider filtered
groups for removal to prevent concurrent disjunct filters from trashing
eachother's synced groups.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/api2/pull.rs | 29 +++++++++++++++++++++++++----
src/bin/proxmox-backup-manager.rs | 14 ++++++++++++++
src/server/pull.rs | 24 ++++++++++++++++++++++--
3 files changed, 61 insertions(+), 6 deletions(-)
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index 4893c9fb..36149761 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -1,5 +1,6 @@
//! Sync datastore from remote server
use std::sync::{Arc};
+use std::str::FromStr;
use anyhow::{format_err, Error};
use futures::{select, future::FutureExt};
@@ -10,9 +11,9 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
use pbs_client::{HttpClient, BackupRepository};
use crate::server::{WorkerTask, jobstate::Job, pull::pull_store};
-use crate::backup::DataStore;
+use crate::backup::{BackupGroup, DataStore};
use crate::api2::types::{
- DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid,
+ BACKUP_GROUP_SCHEMA, DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, Authid,
};
use crate::config::{
remote,
@@ -101,7 +102,7 @@ pub fn do_sync_job(
worker.log(format!("Sync datastore '{}' from '{}/{}'",
sync_job.store, sync_job.remote, sync_job.remote_store));
- pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner).await?;
+ pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, sync_owner, None).await?;
worker.log(format!("sync job '{}' end", &job_id));
@@ -152,6 +153,14 @@ pub fn do_sync_job(
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ type: Array,
+ description: "List of group identifiers to filter for. All if unspecified.",
+ items: {
+ schema: BACKUP_GROUP_SCHEMA,
+ },
+ optional: true,
+ },
},
},
access: {
@@ -169,6 +178,7 @@ async fn pull (
remote: String,
remote_store: String,
remove_vanished: Option<bool>,
+ groups: Option<Vec<String>>,
_info: &ApiMethod,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
@@ -185,7 +195,18 @@ async fn pull (
worker.log(format!("sync datastore '{}' start", store));
- let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id);
+ let groups = match groups {
+ Some(filter) => {
+ let mut groups = std::collections::HashSet::with_capacity(filter.len());
+ for group in filter {
+ let group = BackupGroup::from_str(&group)?;
+ groups.insert(group);
+ }
+ Some(groups)
+ },
+ None => None,
+ };
+ let pull_future = pull_store(&worker, &client, &src_repo, tgt_store.clone(), delete, auth_id, groups);
let future = select!{
success = pull_future.fuse() => success,
abort = worker.abort_future().map(|_| Err(format_err!("pull aborted"))) => abort,
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 93d6de57..6844a1ab 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -230,6 +230,15 @@ fn task_mgmt_cli() -> CommandLineInterface {
schema: REMOVE_VANISHED_BACKUPS_SCHEMA,
optional: true,
},
+ "groups": {
+ type: Array,
+ description: "List of group identifiers to filter for. All if unspecified.",
+ items: {
+ type: String,
+ description: "Backup group identifier",
+ },
+ optional: true,
+ },
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
@@ -243,6 +252,7 @@ async fn pull_datastore(
remote_store: String,
local_store: String,
remove_vanished: Option<bool>,
+ groups: Option<Vec<String>>,
param: Value,
) -> Result<Value, Error> {
@@ -256,6 +266,10 @@ async fn pull_datastore(
"remote-store": remote_store,
});
+ if groups.is_some() {
+ args["groups"] = json!(groups);
+ }
+
if let Some(remove_vanished) = remove_vanished {
args["remove-vanished"] = Value::from(remove_vanished);
}
diff --git a/src/server/pull.rs b/src/server/pull.rs
index 5214a218..2904c37f 100644
--- a/src/server/pull.rs
+++ b/src/server/pull.rs
@@ -631,6 +631,7 @@ pub async fn pull_store(
tgt_store: Arc<DataStore>,
delete: bool,
auth_id: Authid,
+ group_filter: Option<HashSet<BackupGroup>>,
) -> Result<(), Error> {
// explicit create shared lock to prevent GC on newly created chunks
let _shared_store_lock = tgt_store.try_shared_chunk_store_lock()?;
@@ -644,8 +645,7 @@ pub async fn pull_store(
let mut list: Vec<GroupListItem> = serde_json::from_value(result["data"].take())?;
- worker.log(format!("found {} groups to sync", list.len()));
-
+ let total_count = list.len();
list.sort_unstable_by(|a, b| {
let type_order = a.backup_type.cmp(&b.backup_type);
if type_order == std::cmp::Ordering::Equal {
@@ -655,6 +655,21 @@ pub async fn pull_store(
}
});
+
+ let list = if let Some(ref group_filter) = &group_filter {
+ let list:Vec<GroupListItem> = list
+ .into_iter()
+ .filter(|group| {
+ group_filter.contains(&BackupGroup::new(&group.backup_type, &group.backup_id))
+ })
+ .collect();
+ worker.log(format!("found {} groups to sync (out of {} requested by filter)", list.len(), group_filter.len()));
+ list
+ } else {
+ worker.log(format!("found {} groups to sync", total_count));
+ list
+ };
+
let mut errors = false;
let mut new_groups = std::collections::HashSet::new();
@@ -717,6 +732,11 @@ pub async fn pull_store(
if new_groups.contains(&local_group) {
continue;
}
+ if let Some(ref group_filter) = &group_filter {
+ if !group_filter.contains(&local_group) {
+ continue;
+ }
+ }
worker.log(format!(
"delete vanished group '{}/{}'",
local_group.backup_type(),
--
2.30.2
next prev parent reply other threads:[~2021-07-22 14:35 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-22 14:35 [pbs-devel] [PATCH proxmox-backup 0/7] pull/sync group filter Fabian Grünbichler
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 1/7] api-types: add schema for backup group Fabian Grünbichler
2021-07-22 14:35 ` Fabian Grünbichler [this message]
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 3/7] sync: add group filtering Fabian Grünbichler
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 4/7] remote: add backup group scanning Fabian Grünbichler
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 5/7] manager: extend sync/pull completion Fabian Grünbichler
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 6/7] manager: render group filter properly Fabian Grünbichler
2021-07-22 14:35 ` [pbs-devel] [PATCH proxmox-backup 7/7] manager: don't complete sync job ID on creation Fabian Grünbichler
2021-07-26 8:01 ` [pbs-devel] [PATCH proxmox-backup 0/7] pull/sync group filter Dietmar Maurer
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210722143510.238967-3-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox