From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id DFD341FF141 for ; Tue, 05 May 2026 10:12:16 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E587619A2B; Tue, 5 May 2026 10:12:08 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup 4/4] api/datastore: use maintenance-mode lock to protect against changes Date: Tue, 5 May 2026 10:11:36 +0200 Message-ID: <20260505081137.227901-5-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260505081137.227901-1-c.ebner@proxmox.com> References: <20260505081137.227901-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1777968614137 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.070 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com,lib.rs,datastore.rs] Message-ID-Hash: X4CXADKXICKSTI5T4IUJQFLOALGKMXLF X-Message-ID-Hash: X4CXADKXICKSTI5T4IUJQFLOALGKMXLF X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Instead of locking the whole config, which causes also unrelated datastores to not be available, use a per-datastore maintenance mode lock to protect against changes during potentially longer running operations such as s3-refresh or removable datastore mount/unmount. However keep the current semantic of first setting the maintenance mode, allow to still opt out from the mode while waiting for active operations to finish, and only then enforce the maintenance mode during the s3 refresh or removable datastore mount operation. unset_maintenance() is adapted to either reacquire the datastore config lock or use the optional lock guard. Fixes: https://forum.proxmox.com/threads/183244/ Signed-off-by: Christian Ebner --- pbs-datastore/src/datastore.rs | 2 ++ pbs-datastore/src/lib.rs | 11 ++++++++++- src/api2/admin/datastore.rs | 26 ++++++++++++++++++++------ src/api2/config/datastore.rs | 6 +++++- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 34efcd398..aa372cd73 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -3036,6 +3036,7 @@ impl DataStore { let (mut config, _digest) = pbs_config::datastore::config()?; let mut datastore_config: DataStoreConfig = config.lookup("datastore", name)?; + let maintenance_mode_lock = crate::maintenance_mode_lock(name)?; datastore_config.set_maintenance_mode(Some(MaintenanceMode { ty: MaintenanceType::Delete, message: None, @@ -3043,6 +3044,7 @@ impl DataStore { config.set_data(name, "datastore", &datastore_config)?; pbs_config::datastore::save_config(&config)?; + drop(maintenance_mode_lock); drop(config_lock); let (operations, _lock) = task_tracking::get_active_operations_locked(name)?; diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 48acba4c8..823f3bf2f 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -159,8 +159,9 @@ use std::os::unix::io::AsRawFd; use std::path::Path; +use std::time::Duration; -use anyhow::{bail, Error}; +use anyhow::{bail, Context, Error}; use pbs_config::BackupLockGuard; @@ -271,3 +272,11 @@ where Ok(lock) } + +/// Acquire an exclusive lock for the datastore's maintenance-mode +pub fn maintenance_mode_lock(store: &str) -> Result { + lock_helper(store, Path::new("maintenance-mode.lck"), |p| { + pbs_config::open_backup_lockfile(p, Some(Duration::from_secs(0)), true) + .context("unable to acquire exclusive datastore's maintenance-mode lock") + }) +} diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 1e0a78f3a..883091e6f 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -61,8 +61,8 @@ use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::BackupManifest; use pbs_datastore::prune::compute_prune_info; use pbs_datastore::{ - check_backup_owner, ensure_datastore_is_mounted, task_tracking, BackupDir, DataStore, - LocalChunkReader, StoreProgress, + check_backup_owner, ensure_datastore_is_mounted, maintenance_mode_lock, task_tracking, + BackupDir, DataStore, LocalChunkReader, StoreProgress, }; use pbs_tools::json::required_string_param; use proxmox_rest_server::{formatter, worker_is_active, WorkerTask}; @@ -2705,9 +2705,13 @@ fn expect_maintenance_type( } fn unset_maintenance( - _lock: pbs_config::BackupLockGuard, + lock: Option, mut config: DataStoreConfig, ) -> Result<(), Error> { + let _lock = match lock { + Some(lock) => lock, + None => pbs_config::datastore::lock_config()?, + }; let (mut section_config, _digest) = pbs_config::datastore::config()?; config.maintenance_mode = None; section_config.set_data(&config.name, "datastore", &config)?; @@ -2763,6 +2767,7 @@ async fn do_unmount(store: String, auth_id: Authid, to_stdout: bool) -> Result Result Result Result Resu } /// Wait for no more active operations on the given datastore and run the provided callback in -/// a datastore locked context, protecting against maintenance mode changes. +/// a maintenance mode locked context, protecting against leaving the mode for the functions +/// lifetime. fn run_maintenance_locked( store: &str, maintenance_expected: MaintenanceType, @@ -2908,15 +2917,20 @@ fn run_maintenance_locked( let _ = expect_maintenance_type(store, maintenance_expected) .inspect_err(|e| warn!("maintenance mode was not as expected: {e}")) .and_then(|(lock, config)| { - unset_maintenance(lock, config) + unset_maintenance(Some(lock), config) .inspect_err(|e| warn!("could not reset maintenance mode: {e}")) }); bail!("aborted, due to user request"); } let (lock, config) = expect_maintenance_type(store, maintenance_expected)?; + let _maintenance_mode_lock = maintenance_mode_lock(&config.name)?; + // avoid blocking config access/updates, but keep maintenance-mode locked + // until done. + drop(lock); + callback()?; - unset_maintenance(lock, config) + unset_maintenance(None, config) .map_err(|e| format_err!("could not reset maintenance mode: {e}"))?; Ok(()) diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs index 16e85a636..b07e7fc7e 100644 --- a/src/api2/config/datastore.rs +++ b/src/api2/config/datastore.rs @@ -31,7 +31,9 @@ use crate::api2::config::tape_backup_job::{delete_tape_backup_job, list_tape_bac use crate::api2::config::verify::delete_verification_job; use pbs_config::CachedUserInfo; -use pbs_datastore::{get_datastore_mount_status, DataStore, S3_DATASTORE_IN_USE_MARKER}; +use pbs_datastore::{ + get_datastore_mount_status, maintenance_mode_lock, DataStore, S3_DATASTORE_IN_USE_MARKER, +}; use proxmox_rest_server::WorkerTask; use proxmox_s3_client::{S3ObjectKey, S3_HTTP_REQUEST_TIMEOUT}; @@ -540,6 +542,7 @@ pub fn update_datastore( data.tuning = None; } DeletableProperty::MaintenanceMode => { + let _maintenance_mode_lock = maintenance_mode_lock(&name)?; data.set_maintenance_mode(None)?; } DeletableProperty::NotificationThresholds => { @@ -638,6 +641,7 @@ pub fn update_datastore( )?), None => None, }; + let _maintenance_mode_lock = maintenance_mode_lock(&name)?; data.set_maintenance_mode(maintenance_mode)?; } -- 2.47.3