From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id AFBC91FF136 for ; Mon, 09 Mar 2026 17:21:51 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6773A3C24; Mon, 9 Mar 2026 17:21:37 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v5 02/11] api: config/sync: add optional `worker-threads` property Date: Mon, 9 Mar 2026 17:20:41 +0100 Message-ID: <20260309162050.1047341-4-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260309162050.1047341-1-c.ebner@proxmox.com> References: <20260309162050.1047341-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1773073228369 X-SPAM-LEVEL: Spam detection results: 0 AWL -1.010 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.408 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.819 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.903 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: WHUO42E2S6NZ2R6VLIQ53JFOJ35DCWMS X-Message-ID-Hash: WHUO42E2S6NZ2R6VLIQ53JFOJ35DCWMS X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Allow to configure from 1 up to 32 worker threads to perform multiple group syncs in parallel. The property is exposed via the sync job config and passed to the pull/push parameters for the sync job to setup and execute the thread pool accordingly. Implements the schema definitions and includes the new property to the `SyncJobConfig`, `PullParameters` and `PushParameters`. Signed-off-by: Christian Ebner --- src/api2/config/sync.rs | 10 ++++++++++ src/api2/pull.rs | 9 ++++++++- src/api2/push.rs | 8 +++++++- src/server/pull.rs | 4 ++++ src/server/push.rs | 4 ++++ src/server/sync.rs | 1 + 6 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/api2/config/sync.rs b/src/api2/config/sync.rs index dff447cb6..e432c3db4 100644 --- a/src/api2/config/sync.rs +++ b/src/api2/config/sync.rs @@ -345,6 +345,8 @@ pub enum DeletableProperty { UnmountOnDone, /// Delete the sync_direction property, SyncDirection, + /// Delete the worker_threads property, + WorkerThreads, } #[api( @@ -471,6 +473,9 @@ pub fn update_sync_job( DeletableProperty::SyncDirection => { data.sync_direction = None; } + DeletableProperty::WorkerThreads => { + data.worker_threads = None; + } } } } @@ -530,6 +535,10 @@ pub fn update_sync_job( data.sync_direction = Some(sync_direction); } + if let Some(worker_threads) = update.worker_threads { + data.worker_threads = Some(worker_threads); + } + if update.limit.rate_in.is_some() { data.limit.rate_in = update.limit.rate_in; } @@ -705,6 +714,7 @@ acl:1:/remote/remote1/remotestore1:write@pbs:RemoteSyncOperator run_on_mount: None, unmount_on_done: None, sync_direction: None, // use default + worker_threads: None, }; // should work without ACLs diff --git a/src/api2/pull.rs b/src/api2/pull.rs index 4b1fd5e60..7cf165f91 100644 --- a/src/api2/pull.rs +++ b/src/api2/pull.rs @@ -11,7 +11,7 @@ use pbs_api_types::{ GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, RESYNC_CORRUPT_SCHEMA, SYNC_ENCRYPTED_ONLY_SCHEMA, SYNC_VERIFIED_ONLY_SCHEMA, - TRANSFER_LAST_SCHEMA, + SYNC_WORKER_THREADS_SCHEMA, TRANSFER_LAST_SCHEMA, }; use pbs_config::CachedUserInfo; use proxmox_rest_server::WorkerTask; @@ -91,6 +91,7 @@ impl TryFrom<&SyncJobConfig> for PullParameters { sync_job.encrypted_only, sync_job.verified_only, sync_job.resync_corrupt, + sync_job.worker_threads, ) } } @@ -148,6 +149,10 @@ impl TryFrom<&SyncJobConfig> for PullParameters { schema: RESYNC_CORRUPT_SCHEMA, optional: true, }, + "worker-threads": { + schema: SYNC_WORKER_THREADS_SCHEMA, + optional: true, + }, }, }, access: { @@ -175,6 +180,7 @@ async fn pull( encrypted_only: Option, verified_only: Option, resync_corrupt: Option, + worker_threads: Option, rpcenv: &mut dyn RpcEnvironment, ) -> Result { let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; @@ -215,6 +221,7 @@ async fn pull( encrypted_only, verified_only, resync_corrupt, + worker_threads, )?; // fixme: set to_stdout to false? diff --git a/src/api2/push.rs b/src/api2/push.rs index e5edc13e0..f27f4ea1a 100644 --- a/src/api2/push.rs +++ b/src/api2/push.rs @@ -6,7 +6,7 @@ use pbs_api_types::{ GROUP_FILTER_LIST_SCHEMA, NS_MAX_DEPTH_REDUCED_SCHEMA, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_READ, PRIV_REMOTE_DATASTORE_BACKUP, PRIV_REMOTE_DATASTORE_PRUNE, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, SYNC_ENCRYPTED_ONLY_SCHEMA, - SYNC_VERIFIED_ONLY_SCHEMA, TRANSFER_LAST_SCHEMA, + SYNC_VERIFIED_ONLY_SCHEMA, SYNC_WORKER_THREADS_SCHEMA, TRANSFER_LAST_SCHEMA, }; use proxmox_rest_server::WorkerTask; use proxmox_router::{Permission, Router, RpcEnvironment}; @@ -108,6 +108,10 @@ fn check_push_privs( schema: TRANSFER_LAST_SCHEMA, optional: true, }, + "worker-threads": { + schema: SYNC_WORKER_THREADS_SCHEMA, + optional: true, + }, }, }, access: { @@ -133,6 +137,7 @@ async fn push( verified_only: Option, limit: RateLimitConfig, transfer_last: Option, + worker_threads: Option, rpcenv: &mut dyn RpcEnvironment, ) -> Result { let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; @@ -164,6 +169,7 @@ async fn push( verified_only, limit, transfer_last, + worker_threads, ) .await?; diff --git a/src/server/pull.rs b/src/server/pull.rs index 57c5ef323..ddb59db54 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -65,6 +65,8 @@ pub(crate) struct PullParameters { verified_only: bool, /// Whether to re-sync corrupted snapshots resync_corrupt: bool, + /// Maximum number of worker threads to pull during sync job + worker_threads: Option, } impl PullParameters { @@ -85,6 +87,7 @@ impl PullParameters { encrypted_only: Option, verified_only: Option, resync_corrupt: Option, + worker_threads: Option, ) -> Result { if let Some(max_depth) = max_depth { ns.check_max_depth(max_depth)?; @@ -134,6 +137,7 @@ impl PullParameters { encrypted_only, verified_only, resync_corrupt, + worker_threads, }) } } diff --git a/src/server/push.rs b/src/server/push.rs index b7eeeffae..a0484ef62 100644 --- a/src/server/push.rs +++ b/src/server/push.rs @@ -83,6 +83,8 @@ pub(crate) struct PushParameters { verified_only: bool, /// How many snapshots should be transferred at most (taking the newest N snapshots) transfer_last: Option, + /// Maximum number of worker threads for push during sync job + worker_threads: Option, } impl PushParameters { @@ -102,6 +104,7 @@ impl PushParameters { verified_only: Option, limit: RateLimitConfig, transfer_last: Option, + worker_threads: Option, ) -> Result { if let Some(max_depth) = max_depth { ns.check_max_depth(max_depth)?; @@ -164,6 +167,7 @@ impl PushParameters { encrypted_only, verified_only, transfer_last, + worker_threads, }) } diff --git a/src/server/sync.rs b/src/server/sync.rs index aedf4a271..9e6aeb9b0 100644 --- a/src/server/sync.rs +++ b/src/server/sync.rs @@ -675,6 +675,7 @@ pub fn do_sync_job( sync_job.verified_only, sync_job.limit.clone(), sync_job.transfer_last, + sync_job.worker_threads, ) .await?; push_store(push_params).await? -- 2.47.3