From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 9DB5C1FF17C for ; Wed, 3 Sep 2025 14:09:06 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B2A4CBB40; Wed, 3 Sep 2025 14:09:21 +0200 (CEST) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Wed, 3 Sep 2025 14:08:43 +0200 Message-ID: <20250903120844.211292-2-h.laimer@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250903120844.211292-1-h.laimer@proxmox.com> References: <20250903120844.211292-1-h.laimer@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1756901311814 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.125 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes RCVD_IN_MSPIKE_H2 0.001 Average reputation (+2) SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v2 1/2] fix #6195: api: datastore: add endpoint for moving namespaces X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Signed-off-by: Hannes Laimer --- since v2, thanks @Shannon: - check for running operations and lock operations tracking file. Since we are not on a the privilged api process setting a maintenance mode is not possible. And given that moving is pretty much instant, this is IMHO even more fitting than setting a maintenance mode as that would involve reading and write the whole config. pbs-datastore/src/datastore.rs | 121 +++++++++++++++++++++++++++++++++ src/api2/admin/namespace.rs | 50 +++++++++++++- 2 files changed, 169 insertions(+), 2 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 7cf020fc..531927ff 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -706,6 +706,127 @@ impl DataStore { Ok(ns) } + pub fn namespace_move( + self: &Arc, + ns: BackupNamespace, + mut target: BackupNamespace, + ) -> Result<(), Error> { + log::info!( + "moving ns '{}' to ns '{}' in datastore {}", + ns, + target, + self.name() + ); + if ns.is_root() { + bail!("can't move root"); + } + if !self.namespace_exists(&ns) { + bail!("cannot move a namespace that does not exist"); + } + if !self.namespace_exists(&target) { + bail!("cannot move to a namespace that does not exist"); + } + let Some(base_name) = ns.clone().pop() else { + bail!("source namespace can't be root"); + }; + target.push(base_name.clone())?; + if self.namespace_exists(&target) { + bail!("'{}' already exists", target); + } + + let source_path = self.namespace_path(&ns); + let target_path = self.namespace_path(&target); + if target_path.starts_with(&source_path) { + bail!("cannot move namespace into one of its sub-namespaces"); + } + + let iter = self.recursive_iter_backup_ns_ok(ns.to_owned(), None)?; + let source_depth = ns.depth(); + let source_height = iter.map(|ns| ns.depth() - source_depth).max().unwrap_or(1); + target.check_max_depth(source_height - 1)?; + + if let DatastoreBackend::S3(s3_client) = self.backend()? { + let src_ns_rel = ns.path(); + let src_ns_rel_str = src_ns_rel + .to_str() + .ok_or_else(|| format_err!("invalid source namespace path"))?; + + let dst_ns_rel = target.path(); + let dst_ns_rel_str = dst_ns_rel + .to_str() + .ok_or_else(|| format_err!("invalid destination namespace path"))?; + + let list_prefix = S3PathPrefix::Some(format!("{S3_CONTENT_PREFIX}/{src_ns_rel_str}")); + let mut next_continuation_token: Option = None; + let store_prefix = format!("{}/{S3_CONTENT_PREFIX}/", self.name()); + + loop { + let list_objects_result = proxmox_async::runtime::block_on( + s3_client.list_objects_v2(&list_prefix, next_continuation_token.as_deref()), + )?; + + let objects_to_move: Vec = list_objects_result + .contents + .iter() + .map(|c| c.key.clone()) + .collect(); + + for object_key in objects_to_move { + let object_key_str = object_key.to_string(); + let object_path = + object_key_str.strip_prefix(&store_prefix).ok_or_else(|| { + format_err!( + "failed to strip store prefix '{}' from object key '{}'", + store_prefix, + object_key_str + ) + })?; + + let src_key_rel = S3ObjectKey::try_from( + format!("{S3_CONTENT_PREFIX}/{}", object_path).as_str(), + )?; + + let src_rel_with_sep = format!("{}/", src_ns_rel_str); + let rest = object_path + .strip_prefix(&src_rel_with_sep) + .ok_or_else(|| format_err!("unexpected object path: {}", object_path))?; + + let dest_rel_path = format!("{}/{}", dst_ns_rel_str, rest); + let dest_rel_path = std::path::Path::new(&dest_rel_path); + let file_name = dest_rel_path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| format_err!("invalid destination object file name"))?; + let parent = dest_rel_path + .parent() + .unwrap_or_else(|| std::path::Path::new("")); + let dest_key = crate::s3::object_key_from_path(parent, file_name)?; + + proxmox_async::runtime::block_on( + s3_client.copy_object(src_key_rel.clone(), dest_key), + )?; + } + + if list_objects_result.is_truncated { + next_continuation_token = list_objects_result.next_continuation_token; + } else { + break; + } + } + + // delete objects under the old namespace prefix + let delete_error = + proxmox_async::runtime::block_on(s3_client.delete_objects_by_prefix(&list_prefix))?; + if delete_error { + bail!("deleting objects under old namespace prefix failed"); + } + } + + std::fs::create_dir_all(&target_path)?; + std::fs::rename(source_path, target_path)?; + Ok(()) + } + /// Returns if the given namespace exists on the datastore pub fn namespace_exists(&self, ns: &BackupNamespace) -> bool { let mut path = self.base_path(); diff --git a/src/api2/admin/namespace.rs b/src/api2/admin/namespace.rs index 6cf88d89..cf741ead 100644 --- a/src/api2/admin/namespace.rs +++ b/src/api2/admin/namespace.rs @@ -6,7 +6,7 @@ use proxmox_schema::*; use pbs_api_types::{ Authid, BackupGroupDeleteStats, BackupNamespace, NamespaceListItem, Operation, - DATASTORE_SCHEMA, NS_MAX_DEPTH_SCHEMA, PROXMOX_SAFE_ID_FORMAT, + DATASTORE_SCHEMA, NS_MAX_DEPTH_SCHEMA, PRIV_DATASTORE_MODIFY, PROXMOX_SAFE_ID_FORMAT, }; use pbs_datastore::DataStore; @@ -123,6 +123,51 @@ pub fn list_namespaces( Ok(namespace_list) } +#[api( + input: { + properties: { + store: { schema: DATASTORE_SCHEMA }, + ns: { + type: BackupNamespace, + }, + "target-ns": { + type: BackupNamespace, + optional: true, + }, + }, + }, + access: { + permission: &Permission::Anybody, + description: "Requires DATASTORE_MODIFY on both the source /datastore/{store}/{ns} and\ + the target /datastore/{store}/[{target-ns}]", + }, +)] +/// Move a namespace into a different one. No target means the root namespace is the target. +pub fn move_namespace( + store: String, + ns: BackupNamespace, + target_ns: Option, + _info: &ApiMethod, + rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; + let target = target_ns.unwrap_or(BackupNamespace::root()); + + check_ns_modification_privs(&store, &ns, &auth_id)?; + check_ns_privs(&store, &target, &auth_id, PRIV_DATASTORE_MODIFY)?; + + let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?; + + // This lock on the active operations tracking file prevents updates on it, so no new + // read/write tasks can be started while we hold this. We are the one writing operation we + // expect here. + let (operations, _lock) = pbs_datastore::task_tracking::get_active_operations_locked(&store)?; + if operations.read > 0 || operations.write > 1 { + bail!("can't move namespace while tasks are running"); + }; + datastore.namespace_move(ns, target) +} + #[api( input: { properties: { @@ -192,4 +237,5 @@ pub fn delete_namespace( pub const ROUTER: Router = Router::new() .get(&API_METHOD_LIST_NAMESPACES) .post(&API_METHOD_CREATE_NAMESPACE) - .delete(&API_METHOD_DELETE_NAMESPACE); + .delete(&API_METHOD_DELETE_NAMESPACE) + .subdirs(&[("move", &Router::new().post(&API_METHOD_MOVE_NAMESPACE))]); -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel