From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id BA2FB1FF165 for ; Thu, 3 Jul 2025 15:20:11 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6BAD814DB9; Thu, 3 Jul 2025 15:20:19 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 3 Jul 2025 15:18:34 +0200 Message-ID: <20250703131837.786811-47-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com> References: <20250703131837.786811-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v5 43/46] api/datastore: implement refresh endpoint for stores with s3 backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Allows to easily refresh the contents on the local cache store for datastores backed by an S3 object store. In order to guarantee that no read or write operations are ongoing, the store is first set into the maintenance mode `S3Refresh`. Objects are then fetched into a temporary directory to avoid loosing contents and consistency in case of an error. Once all objects have been fetched, clears out existing contents and moves the newly fetched contents in place. Signed-off-by: Christian Ebner --- pbs-datastore/src/datastore.rs | 183 ++++++++++++++++++++++++++++++++- src/api2/admin/datastore.rs | 34 ++++++ 2 files changed, 216 insertions(+), 1 deletion(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 7d3042b54..395977961 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -1,5 +1,6 @@ use std::collections::{HashMap, HashSet}; use std::io::{self, Write}; +use std::ops::Deref; use std::os::unix::ffi::OsStrExt; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; @@ -9,8 +10,10 @@ use std::time::{Duration, SystemTime}; use anyhow::{bail, format_err, Context, Error}; use http_body_util::BodyExt; use nix::unistd::{unlinkat, UnlinkatFlags}; -use pbs_s3_client::{S3Client, S3ClientOptions}; +use pbs_s3_client::{RelS3ObjectKey, S3Client, S3ClientOptions}; use pbs_tools::lru_cache::LruCache; +use proxmox_lang::try_block; +use tokio::io::AsyncWriteExt; use tracing::{info, warn}; use proxmox_human_byte::HumanByte; @@ -2122,4 +2125,182 @@ impl DataStore { pub fn old_locking(&self) -> bool { *OLD_LOCKING } + + /// Set the datastore's maintenance mode to `S3Refresh`, fetch from S3 object store, clear and + /// replace the local cache store contents. Once finished disable the maintenance mode again. + /// Returns with error for other datastore backends without setting the maintenance mode. + pub async fn s3_refresh(self: &Arc) -> Result<(), Error> { + match self.backend()? { + DatastoreBackend::Filesystem => bail!("store '{}' not backed by S3", self.name()), + DatastoreBackend::S3(s3_client) => { + try_block!({ + let _lock = pbs_config::datastore::lock_config()?; + let (mut section_config, _digest) = pbs_config::datastore::config()?; + let mut datastore: DataStoreConfig = + section_config.lookup("datastore", self.name())?; + datastore.set_maintenance_mode(Some(MaintenanceMode { + ty: MaintenanceType::S3Refresh, + message: None, + }))?; + section_config.set_data(self.name(), "datastore", &datastore)?; + pbs_config::datastore::save_config(§ion_config)?; + drop(_lock); + Ok::<(), Error>(()) + }) + .context("failed to set maintenance mode")?; + + let store_base = self.base_path(); + + let tmp_base = proxmox_sys::fs::make_tmp_dir(&store_base, None) + .context("failed to create temporary content folder")?; + + let backup_user = pbs_config::backup_user().context("failed to get backup user")?; + let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); + let file_create_options = CreateOptions::new() + .perm(mode) + .owner(backup_user.uid) + .group(backup_user.gid); + let mode = nix::sys::stat::Mode::from_bits_truncate(0o0755); + let dir_create_options = CreateOptions::new() + .perm(mode) + .owner(backup_user.uid) + .group(backup_user.gid); + + let list_prefix = S3PathPrefix::Some(S3_CONTENT_PREFIX.to_string()); + let store_prefix = format!("{}/{S3_CONTENT_PREFIX}/", self.name()); + let content_prefix = format!("{S3_CONTENT_PREFIX}/"); + let mut next_continuation_token: Option = None; + loop { + let list_objects_result = s3_client + .list_objects_v2(&list_prefix, next_continuation_token.as_deref()) + .await + .context("failed to list object")?; + + let mut objects_to_fetch: Vec = + Vec::with_capacity(list_objects_result.contents.len()); + for item in &list_objects_result.contents { + let object_key = item + .key + .strip_prefix(&store_prefix) + .ok_or_else(|| { + format_err!( + "failed to strip store context prefix {store_prefix} for {}", + item.key + ) + })? + .into(); + objects_to_fetch.push(object_key); + } + + for object in objects_to_fetch { + let object_path = object.deref().strip_prefix(&content_prefix).unwrap(); + let object_path = pbs_s3_client::uri_decode(object_path) + .context("failed to decode object path")?; + if object_path.ends_with(NAMESPACE_MARKER_FILENAME) { + continue; + } + + info!("Fetching object {object_path}"); + + let file_path = tmp_base.join(&object_path); + if let Some(parent) = file_path.parent() { + proxmox_sys::fs::create_path( + parent, + Some(dir_create_options), + Some(dir_create_options), + )?; + } + + let mut target_file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .read(true) + .open(&file_path) + .await + .with_context(|| { + format!("failed to create target file {file_path:?}") + })?; + + if let Some(response) = s3_client + .get_object(object.clone()) + .await + .with_context(|| format!("failed to fetch object {object_path}"))? + { + let data = response + .content + .collect() + .await + .context("failed to collect object contents")?; + target_file + .write_all(&data.to_bytes()) + .await + .context("failed to write to target file")?; + file_create_options + .apply_to(&mut target_file, &file_path) + .context("failed to set target file create options")?; + target_file + .flush() + .await + .context("failed to flush target file")?; + } else { + bail!("failed to download {object_path}, not found"); + } + } + + if list_objects_result.is_truncated { + next_continuation_token = list_objects_result + .next_continuation_token + .as_ref() + .cloned(); + continue; + } + break; + } + + for ty in ["vm", "ct", "host", "ns"] { + let store_base_clone = store_base.clone(); + let tmp_base_clone = tmp_base.clone(); + tokio::task::spawn_blocking(move || { + let type_dir = store_base_clone.join(ty); + if let Err(err) = std::fs::remove_dir_all(&type_dir) { + if err.kind() != io::ErrorKind::NotFound { + return Err(err).with_context(|| { + format!("failed to remove old contents in {type_dir:?}") + }); + } + } + let tmp_type_dir = tmp_base_clone.join(ty); + if let Err(err) = std::fs::rename(&tmp_type_dir, &type_dir) { + if err.kind() != io::ErrorKind::NotFound { + return Err(err) + .with_context(|| format!("failed to rename {tmp_type_dir:?}")); + } + } + Ok::<(), Error>(()) + }) + .await? + .with_context(|| format!("failed to refresh {store_base:?}"))?; + } + + std::fs::remove_dir_all(&tmp_base).with_context(|| { + format!("failed to cleanup temporary content in {tmp_base:?}") + })?; + + try_block!({ + let _lock = pbs_config::datastore::lock_config()?; + let (mut section_config, _digest) = pbs_config::datastore::config()?; + let mut datastore: DataStoreConfig = + section_config.lookup("datastore", self.name())?; + datastore.set_maintenance_mode(None)?; + section_config.set_data(self.name(), "datastore", &datastore)?; + pbs_config::datastore::save_config(§ion_config)?; + drop(_lock); + Ok::<(), Error>(()) + }) + .context("failed to clear maintenance mode")?; + } + } + Ok(()) + } } diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index e82014fbb..b94f9f104 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -2701,6 +2701,39 @@ pub async fn unmount(store: String, rpcenv: &mut dyn RpcEnvironment) -> Result Result { + let datastore = DataStore::lookup_datastore(&store, Some(Operation::Lookup))?; + let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; + let to_stdout = rpcenv.env_type() == RpcEnvironmentType::CLI; + + let upid = WorkerTask::spawn( + "s3-refresh", + Some(store), + auth_id.to_string(), + to_stdout, + move |_worker| async move { datastore.s3_refresh().await }, + )?; + + Ok(json!(upid)) +} + #[sortable] const DATASTORE_INFO_SUBDIRS: SubdirMap = &[ ( @@ -2767,6 +2800,7 @@ const DATASTORE_INFO_SUBDIRS: SubdirMap = &[ &Router::new().download(&API_METHOD_PXAR_FILE_DOWNLOAD), ), ("rrd", &Router::new().get(&API_METHOD_GET_RRD_STATS)), + ("s3-refresh", &Router::new().put(&API_METHOD_S3_REFRESH)), ( "snapshots", &Router::new() -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel