From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 5B5DA1FF16F for ; Tue, 22 Jul 2025 12:11:09 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8AA3835D43; Tue, 22 Jul 2025 12:12:15 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 22 Jul 2025 12:10:23 +0200 Message-ID: <20250722101106.526438-8-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250722101106.526438-1-c.ebner@proxmox.com> References: <20250722101106.526438-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753179079496 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.955 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_MAILER 2 Automated Mailer Tag Left in Email SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v11 03/46] api: config: implement endpoints to manipulate and list s3 configs X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Allows to create, list, modify and delete configurations for s3 clients via the api. Signed-off-by: Christian Ebner --- changes since version 10: - merge secrets into client config - use S3 config type constant Cargo.toml | 1 + src/api2/config/mod.rs | 2 + src/api2/config/s3.rs | 280 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 283 insertions(+) create mode 100644 src/api2/config/s3.rs diff --git a/Cargo.toml b/Cargo.toml index 46e3a737c..28c78cc1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -225,6 +225,7 @@ proxmox-notify = { workspace = true, features = [ "pbs-context" ] } proxmox-openid.workspace = true proxmox-rest-server = { workspace = true, features = [ "rate-limited-stream" ] } proxmox-router = { workspace = true, features = [ "cli", "server"] } +proxmox-s3-client.workspace = true proxmox-schema = { workspace = true, features = [ "api-macro" ] } proxmox-section-config.workspace = true proxmox-serde = { workspace = true, features = [ "serde_json" ] } diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs index 15dc5db92..1cd9ead76 100644 --- a/src/api2/config/mod.rs +++ b/src/api2/config/mod.rs @@ -14,6 +14,7 @@ pub mod metrics; pub mod notifications; pub mod prune; pub mod remote; +pub mod s3; pub mod sync; pub mod tape_backup_job; pub mod tape_encryption_keys; @@ -32,6 +33,7 @@ const SUBDIRS: SubdirMap = &sorted!([ ("notifications", ¬ifications::ROUTER), ("prune", &prune::ROUTER), ("remote", &remote::ROUTER), + ("s3", &s3::ROUTER), ("sync", &sync::ROUTER), ("tape-backup-job", &tape_backup_job::ROUTER), ("tape-encryption-keys", &tape_encryption_keys::ROUTER), diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs new file mode 100644 index 000000000..891c017c7 --- /dev/null +++ b/src/api2/config/s3.rs @@ -0,0 +1,280 @@ +use ::serde::{Deserialize, Serialize}; +use anyhow::{bail, Context, Error}; +use hex::FromHex; +use serde_json::Value; + +use proxmox_router::{http_bail, Permission, Router, RpcEnvironment}; +use proxmox_s3_client::{S3ClientConfig, S3ClientConfigUpdater}; +use proxmox_schema::{api, param_bail, ApiType}; + +use pbs_api_types::{ + DataStoreConfig, DatastoreBackendConfig, DatastoreBackendType, JOB_ID_SCHEMA, PRIV_SYS_AUDIT, + PRIV_SYS_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA, +}; +use pbs_config::s3::{self, S3_CFG_TYPE_ID}; + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List configured s3 clients.", + type: Array, + items: { type: S3ClientConfig }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// List all s3 client configurations. +pub fn list_s3_client_config( + _param: Value, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let (config, digest) = s3::config()?; + let list = config.convert_to_typed_array(S3_CFG_TYPE_ID)?; + rpcenv["digest"] = hex::encode(digest).into(); + + Ok(list) +} + +#[api( + protected: true, + input: { + properties: { + config: { + type: S3ClientConfig, + flatten: true, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Create a new s3 client configuration. +pub fn create_s3_client_config( + config: S3ClientConfig, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = s3::lock_config()?; + let (mut section_config, _digest) = s3::config()?; + if section_config.sections.contains_key(&config.id) { + param_bail!("id", "s3 client config '{}' already exists.", config.id); + } + + section_config.set_data(&config.id, S3_CFG_TYPE_ID, &config)?; + s3::save_config(§ion_config)?; + + Ok(()) +} + +#[api( + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + }, + }, + returns: { type: S3ClientConfig }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false), + }, +)] +/// Read an s3 client configuration. +pub fn read_s3_client_config( + id: String, + rpcenv: &mut dyn RpcEnvironment, +) -> Result { + let (config, digest) = s3::config()?; + let s3_client_config: S3ClientConfig = config.lookup(S3_CFG_TYPE_ID, &id)?; + rpcenv["digest"] = hex::encode(digest).into(); + + Ok(s3_client_config) +} + +#[api()] +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +/// Deletable property name +pub enum DeletableProperty { + /// Delete the port property. + Port, + /// Delete the region property. + Region, + /// Delete the fingerprint property. + Fingerprint, + /// Delete the path-style property. + PathStyle, +} + +#[api( + protected: true, + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + update: { + type: S3ClientConfigUpdater, + flatten: true, + }, + delete: { + description: "List of properties to delete.", + type: Array, + optional: true, + items: { + type: DeletableProperty, + } + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Update an s3 client configuration. +#[allow(clippy::too_many_arguments)] +pub fn update_s3_client_config( + id: String, + update: S3ClientConfigUpdater, + delete: Option>, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = s3::lock_config()?; + let (mut config, expected_digest) = s3::config()?; + + // Secrets are not included in digest concurrent changes therefore not detected. + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + let mut data: S3ClientConfig = config.lookup(S3_CFG_TYPE_ID, &id)?; + + if let Some(delete) = delete { + for delete_prop in delete { + match delete_prop { + DeletableProperty::Port => { + data.port = None; + } + DeletableProperty::Region => { + data.region = None; + } + DeletableProperty::Fingerprint => { + data.fingerprint = None; + } + DeletableProperty::PathStyle => { + data.path_style = None; + } + } + } + } + + if let Some(endpoint) = update.endpoint { + data.endpoint = endpoint; + } + if let Some(port) = update.port { + data.port = Some(port); + } + if let Some(region) = update.region { + data.region = Some(region); + } + if let Some(access_key) = update.access_key { + data.access_key = access_key; + } + if let Some(fingerprint) = update.fingerprint { + data.fingerprint = Some(fingerprint); + } + if let Some(path_style) = update.path_style { + data.path_style = Some(path_style); + } + if let Some(secret_key) = update.secret_key { + data.secret_key = secret_key; + } + + config.set_data(&id, S3_CFG_TYPE_ID, &data)?; + s3::save_config(&config)?; + + Ok(()) +} + +#[api( + protected: true, + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, + access: { + permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false), + }, +)] +/// Remove an s3 client configuration. +pub fn delete_s3_client_config( + id: String, + digest: Option, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let _lock = s3::lock_config()?; + let (mut config, expected_digest) = s3::config()?; + + if let Some(ref digest) = digest { + let digest = <[u8; 32]>::from_hex(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + if let Some(datastore) = + s3_client_in_use(&id).context("failed to check if s3 client is in-use")? + { + bail!("in-use by datastore {datastore}"); + } + + if config.sections.remove(&id).is_none() { + http_bail!(NOT_FOUND, "s3 client config '{id}' do not exist.") + } + s3::save_config(&config) +} + +// Check if the configured s3 client is still in-use by a datastore backend. +// +// If so, return the first datastore name with the configured client. +fn s3_client_in_use(id: &str) -> Result, Error> { + let (config, _digest) = pbs_config::datastore::config()?; + let list: Vec = config.convert_to_typed_array("datastore")?; + for datastore in list { + let backend_config: DatastoreBackendConfig = serde_json::from_value( + DatastoreBackendConfig::API_SCHEMA + .parse_property_string(datastore.backend.as_deref().unwrap_or(""))?, + )?; + match (backend_config.ty, backend_config.client) { + (Some(DatastoreBackendType::S3), Some(client)) if client == id => { + return Ok(Some(datastore.name.to_owned())) + } + _ => (), + } + } + Ok(None) +} + +const ITEM_ROUTER: Router = Router::new() + .get(&API_METHOD_READ_S3_CLIENT_CONFIG) + .put(&API_METHOD_UPDATE_S3_CLIENT_CONFIG) + .delete(&API_METHOD_DELETE_S3_CLIENT_CONFIG); + +pub const ROUTER: Router = Router::new() + .get(&API_METHOD_LIST_S3_CLIENT_CONFIG) + .post(&API_METHOD_CREATE_S3_CLIENT_CONFIG) + .match_all("id", &ITEM_ROUTER); -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel