From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id CA8CF1FF168
for <inbox@lore.proxmox.com>; Tue, 18 Mar 2025 13:25:23 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id CE57A1E9A6;
Tue, 18 Mar 2025 13:25:13 +0100 (CET)
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Tue, 18 Mar 2025 13:24:19 +0100
Message-Id: <20250318122423.385684-4-c.ebner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250318122423.385684-1-c.ebner@proxmox.com>
References: <20250318122423.385684-1-c.ebner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.031 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [PATCH v3 proxmox-backup 3/7] api: config/sync: add
optional `parallel-groups` property
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
Allow to configure from 1 up to 8 concurrent futures to perform
multiple group syncs in parallel.
The property is exposed via the sync job config and passed to
the pull/push parameters for the sync job to setup and execute the
futures accordingly.
Implements the schema definitions and includes the new property to
the `SyncJobConfig`, `PullParameters` and `PushParameters`.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- Split off pbs api types into own patch, as it has been moved to
proxmox repo since.
src/api2/config/sync.rs | 10 ++++++++++
src/api2/pull.rs | 9 ++++++++-
src/api2/push.rs | 9 ++++++++-
src/server/pull.rs | 4 ++++
src/server/push.rs | 4 ++++
src/server/sync.rs | 1 +
6 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/src/api2/config/sync.rs b/src/api2/config/sync.rs
index a8ea93465..517bcd868 100644
--- a/src/api2/config/sync.rs
+++ b/src/api2/config/sync.rs
@@ -337,6 +337,8 @@ pub enum DeletableProperty {
TransferLast,
/// Delete the sync_direction property,
SyncDirection,
+ /// Delete the parallel_groups property,
+ ParallelGroups,
}
#[api(
@@ -451,6 +453,9 @@ pub fn update_sync_job(
DeletableProperty::SyncDirection => {
data.sync_direction = None;
}
+ DeletableProperty::ParallelGroups => {
+ data.parallel_groups = None;
+ }
}
}
}
@@ -495,6 +500,10 @@ pub fn update_sync_job(
data.sync_direction = Some(sync_direction);
}
+ if let Some(parallel_groups) = update.parallel_groups {
+ data.parallel_groups = Some(parallel_groups);
+ }
+
if update.limit.rate_in.is_some() {
data.limit.rate_in = update.limit.rate_in;
}
@@ -666,6 +675,7 @@ acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator
limit: pbs_api_types::RateLimitConfig::default(), // no limit
transfer_last: None,
sync_direction: None, // use default
+ parallel_groups: None,
};
// should work without ACLs
diff --git a/src/api2/pull.rs b/src/api2/pull.rs
index d8ed1a734..36f66d9b0 100644
--- a/src/api2/pull.rs
+++ b/src/api2/pull.rs
@@ -10,7 +10,7 @@ use pbs_api_types::{
Authid, BackupNamespace, GroupFilter, RateLimitConfig, SyncJobConfig, DATASTORE_SCHEMA,
GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, PRIV_DATASTORE_BACKUP,
PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
- RESYNC_CORRUPT_SCHEMA, TRANSFER_LAST_SCHEMA,
+ RESYNC_CORRUPT_SCHEMA, SYNC_PARALLEL_GROUPS_SCHEMA, TRANSFER_LAST_SCHEMA,
};
use pbs_config::CachedUserInfo;
use proxmox_rest_server::WorkerTask;
@@ -88,6 +88,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters {
sync_job.limit.clone(),
sync_job.transfer_last,
sync_job.resync_corrupt,
+ sync_job.parallel_groups,
)
}
}
@@ -137,6 +138,10 @@ impl TryFrom<&SyncJobConfig> for PullParameters {
schema: RESYNC_CORRUPT_SCHEMA,
optional: true,
},
+ "parallel-groups": {
+ schema: SYNC_PARALLEL_GROUPS_SCHEMA,
+ optional: true,
+ },
},
},
access: {
@@ -162,6 +167,7 @@ async fn pull(
limit: RateLimitConfig,
transfer_last: Option<usize>,
resync_corrupt: Option<bool>,
+ parallel_groups: Option<usize>,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -200,6 +206,7 @@ async fn pull(
limit,
transfer_last,
resync_corrupt,
+ parallel_groups,
)?;
// fixme: set to_stdout to false?
diff --git a/src/api2/push.rs b/src/api2/push.rs
index bf846bb37..4bb91ed61 100644
--- a/src/api2/push.rs
+++ b/src/api2/push.rs
@@ -5,7 +5,8 @@ use pbs_api_types::{
Authid, BackupNamespace, GroupFilter, RateLimitConfig, DATASTORE_SCHEMA,
GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, PRIV_DATASTORE_BACKUP,
PRIV_DATASTORE_READ, PRIV_REMOTE_DATASTORE_BACKUP, PRIV_REMOTE_DATASTORE_PRUNE,
- REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, TRANSFER_LAST_SCHEMA,
+ REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, SYNC_PARALLEL_GROUPS_SCHEMA,
+ TRANSFER_LAST_SCHEMA,
};
use proxmox_rest_server::WorkerTask;
use proxmox_router::{Permission, Router, RpcEnvironment};
@@ -99,6 +100,10 @@ fn check_push_privs(
schema: TRANSFER_LAST_SCHEMA,
optional: true,
},
+ "parallel-groups": {
+ schema: SYNC_PARALLEL_GROUPS_SCHEMA,
+ optional: true,
+ },
},
},
access: {
@@ -122,6 +127,7 @@ async fn push(
group_filter: Option<Vec<GroupFilter>>,
limit: RateLimitConfig,
transfer_last: Option<usize>,
+ parallel_groups: Option<usize>,
rpcenv: &mut dyn RpcEnvironment,
) -> Result<String, Error> {
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -151,6 +157,7 @@ async fn push(
group_filter,
limit,
transfer_last,
+ parallel_groups,
)
.await?;
diff --git a/src/server/pull.rs b/src/server/pull.rs
index 516abfe5d..0986bc5c8 100644
--- a/src/server/pull.rs
+++ b/src/server/pull.rs
@@ -57,6 +57,8 @@ pub(crate) struct PullParameters {
transfer_last: Option<usize>,
/// Whether to re-sync corrupted snapshots
resync_corrupt: bool,
+ /// Maximum number of parallel groups to pull during sync job
+ parallel_groups: Option<usize>,
}
impl PullParameters {
@@ -75,6 +77,7 @@ impl PullParameters {
limit: RateLimitConfig,
transfer_last: Option<usize>,
resync_corrupt: Option<bool>,
+ parallel_groups: Option<usize>,
) -> Result<Self, Error> {
if let Some(max_depth) = max_depth {
ns.check_max_depth(max_depth)?;
@@ -121,6 +124,7 @@ impl PullParameters {
group_filter,
transfer_last,
resync_corrupt,
+ parallel_groups,
})
}
}
diff --git a/src/server/push.rs b/src/server/push.rs
index f00a6b1e0..878b0f6d6 100644
--- a/src/server/push.rs
+++ b/src/server/push.rs
@@ -75,6 +75,8 @@ pub(crate) struct PushParameters {
group_filter: Vec<GroupFilter>,
/// How many snapshots should be transferred at most (taking the newest N snapshots)
transfer_last: Option<usize>,
+ /// Maximum number of parallel groups to push during sync job
+ parallel_groups: Option<usize>,
}
impl PushParameters {
@@ -92,6 +94,7 @@ impl PushParameters {
group_filter: Option<Vec<GroupFilter>>,
limit: RateLimitConfig,
transfer_last: Option<usize>,
+ parallel_groups: Option<usize>,
) -> Result<Self, Error> {
if let Some(max_depth) = max_depth {
ns.check_max_depth(max_depth)?;
@@ -150,6 +153,7 @@ impl PushParameters {
max_depth,
group_filter,
transfer_last,
+ parallel_groups,
})
}
diff --git a/src/server/sync.rs b/src/server/sync.rs
index 4dd46c5a0..db04dfaa5 100644
--- a/src/server/sync.rs
+++ b/src/server/sync.rs
@@ -674,6 +674,7 @@ pub fn do_sync_job(
sync_job.group_filter.clone(),
sync_job.limit.clone(),
sync_job.transfer_last,
+ sync_job.parallel_groups,
)
.await?;
push_store(push_params).await?
--
2.39.5
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel