[pbs-devel] [PATCH proxmox-backup v6 03/37] api: config: implement endpoints to manipulate and list s3 configs

From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup v6 03/37] api: config: implement endpoints to manipulate and list s3 configs
Date: Tue,  8 Jul 2025 19:00:40 +0200	[thread overview]
Message-ID: <20250708170114.1556057-13-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250708170114.1556057-1-c.ebner@proxmox.com>

Allows to create, list, modify and delete configurations for s3
clients via the api.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 Cargo.toml             |   1 +
 src/api2/config/mod.rs |   2 +
 src/api2/config/s3.rs  | 310 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 313 insertions(+)
 create mode 100644 src/api2/config/s3.rs

diff --git a/Cargo.toml b/Cargo.toml
index d2fbbb2b7..064878fe1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -225,6 +225,7 @@ proxmox-notify = { workspace = true, features = [ "pbs-context" ] }
 proxmox-openid.workspace = true
 proxmox-rest-server = { workspace = true, features = [ "rate-limited-stream" ] }
 proxmox-router = { workspace = true, features = [ "cli", "server"] }
+proxmox-s3-client.workspace = true
 proxmox-schema = { workspace = true, features = [ "api-macro" ] }
 proxmox-section-config.workspace = true
 proxmox-serde = { workspace = true, features = [ "serde_json" ] }
diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 15dc5db92..1cd9ead76 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -14,6 +14,7 @@ pub mod metrics;
 pub mod notifications;
 pub mod prune;
 pub mod remote;
+pub mod s3;
 pub mod sync;
 pub mod tape_backup_job;
 pub mod tape_encryption_keys;
@@ -32,6 +33,7 @@ const SUBDIRS: SubdirMap = &sorted!([
     ("notifications", &notifications::ROUTER),
     ("prune", &prune::ROUTER),
     ("remote", &remote::ROUTER),
+    ("s3", &s3::ROUTER),
     ("sync", &sync::ROUTER),
     ("tape-backup-job", &tape_backup_job::ROUTER),
     ("tape-encryption-keys", &tape_encryption_keys::ROUTER),
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
new file mode 100644
index 000000000..c76704f5a
--- /dev/null
+++ b/src/api2/config/s3.rs
@@ -0,0 +1,310 @@
+use ::serde::{Deserialize, Serialize};
+use anyhow::Error;
+use hex::FromHex;
+use serde_json::Value;
+
+use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
+use proxmox_s3_client::{
+    S3ClientConfig, S3ClientConfigUpdater, S3ClientSecretsConfig, S3ClientSecretsConfigUpdater,
+};
+use proxmox_schema::{api, param_bail};
+
+use pbs_api_types::{JOB_ID_SCHEMA, PRIV_SYS_AUDIT, PRIV_SYS_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA};
+use pbs_config::s3;
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured s3 clients.",
+        type: Array,
+        items: { type: S3ClientConfig },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// List all s3 client configurations.
+pub fn list_s3_client_config(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<S3ClientConfig>, Error> {
+    let (config, digest) = s3::config()?;
+    let list = config.convert_to_typed_array("s3client")?;
+
+    let (_secrets, secrets_digest) = s3::secrets_config()?;
+    let digest = digest_with_secrets(&digest, &secrets_digest);
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            config: {
+                type: S3ClientConfig,
+                flatten: true,
+            },
+            secrets: {
+                type: S3ClientSecretsConfig,
+                flatten: true,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Create a new s3 client configuration.
+pub fn create_s3_client_config(
+    config: S3ClientConfig,
+    secrets: S3ClientSecretsConfig,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    // Asssure both, config and secrets are referenced by the same `id`
+    if config.id != secrets.secrets_id {
+        param_bail!(
+            "id",
+            "config and secrets must use the same id ({} != {})",
+            config.id,
+            secrets.secrets_id
+        );
+    }
+
+    let _lock = s3::lock_config()?;
+    let (mut section_config, _digest) = s3::config()?;
+    if section_config.sections.contains_key(&config.id) {
+        param_bail!("id", "s3 client config '{}' already exists.", config.id);
+    }
+
+    let (mut section_secrets, _secrets_digest) = s3::secrets_config()?;
+    if section_secrets.sections.contains_key(&config.id) {
+        param_bail!("id", "s3 secrets config '{}' already exists.", config.id);
+    }
+
+    section_config.set_data(&config.id, "s3client", &config)?;
+    section_secrets.set_data(&config.id, "s3secrets", &secrets)?;
+    s3::save_config(&section_config, &section_secrets)?;
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+    returns: { type: S3ClientConfig },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+    },
+)]
+/// Read an s3 client configuration.
+pub fn read_s3_client_config(
+    id: String,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<S3ClientConfig, Error> {
+    let (config, digest) = s3::config()?;
+    let s3_client_config: S3ClientConfig = config.lookup("s3client", &id)?;
+
+    let (_secrets, secrets_digest) = s3::secrets_config()?;
+    let digest = digest_with_secrets(&digest, &secrets_digest);
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(s3_client_config)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the port property.
+    Port,
+    /// Delete the region property.
+    Region,
+    /// Delete the fingerprint property.
+    Fingerprint,
+    /// Delete the path-style property.
+    PathStyle,
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            update: {
+                type: S3ClientConfigUpdater,
+                flatten: true,
+            },
+            "update-secrets": {
+                type: S3ClientSecretsConfigUpdater,
+                flatten: true,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Update an s3 client configuration.
+#[allow(clippy::too_many_arguments)]
+pub fn update_s3_client_config(
+    id: String,
+    update: S3ClientConfigUpdater,
+    update_secrets: S3ClientSecretsConfigUpdater,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let _lock = s3::lock_config()?;
+    let (mut config, expected_digest) = s3::config()?;
+    let (mut secrets, secrets_digest) = s3::secrets_config()?;
+    let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+    // Secrets are not included in digest concurrent changes therefore not detected.
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: S3ClientConfig = config.lookup("s3client", &id)?;
+
+    if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::Port => {
+                    data.port = None;
+                }
+                DeletableProperty::Region => {
+                    data.region = None;
+                }
+                DeletableProperty::Fingerprint => {
+                    data.fingerprint = None;
+                }
+                DeletableProperty::PathStyle => {
+                    data.path_style = None;
+                }
+            }
+        }
+    }
+
+    if let Some(endpoint) = update.endpoint {
+        data.endpoint = endpoint;
+    }
+    if let Some(port) = update.port {
+        data.port = Some(port);
+    }
+    if let Some(region) = update.region {
+        data.region = Some(region);
+    }
+    if let Some(access_key) = update.access_key {
+        data.access_key = access_key;
+    }
+    if let Some(fingerprint) = update.fingerprint {
+        data.fingerprint = Some(fingerprint);
+    }
+    if let Some(path_style) = update.path_style {
+        data.path_style = Some(path_style);
+    }
+
+    let mut secrets_data: S3ClientSecretsConfig = secrets.lookup("s3secrets", &id)?;
+    if let Some(secret_key) = update_secrets.secret_key {
+        secrets_data.secret_key = secret_key;
+    }
+
+    config.set_data(&id, "s3client", &data)?;
+    secrets.set_data(&id, "s3secrets", &secrets_data)?;
+    s3::save_config(&config, &secrets)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Privilege(&[], PRIV_SYS_MODIFY, false),
+    },
+)]
+/// Remove an s3 client configuration.
+pub fn delete_s3_client_config(
+    id: String,
+    digest: Option<String>,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let _lock = s3::lock_config()?;
+    let (mut config, expected_digest) = s3::config()?;
+    let (mut secrets, secrets_digest) = s3::secrets_config()?;
+    let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    match (config.sections.remove(&id), secrets.sections.remove(&id)) {
+        (Some(_), Some(_)) => {}
+        (None, None) => http_bail!(
+            NOT_FOUND,
+            "s3 client config and secrets '{id}' do not exist."
+        ),
+        (Some(_), None) => http_bail!(
+            NOT_FOUND,
+            "removed s3 client config, but no secrets for '{id}' found."
+        ),
+        (None, Some(_)) => http_bail!(
+            NOT_FOUND,
+            "removed s3 client secrets, but no config for '{id}' found."
+        ),
+    }
+    s3::save_config(&config, &secrets)
+}
+
+// Calculate the digest based on the digest of config and secrets to detect changes for both
+fn digest_with_secrets(digest: &[u8; 32], secrets_digest: &[u8; 32]) -> [u8; 32] {
+    let mut digest = digest.to_vec();
+    digest.append(&mut secrets_digest.to_vec());
+    openssl::sha::sha256(&digest)
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_S3_CLIENT_CONFIG)
+    .put(&API_METHOD_UPDATE_S3_CLIENT_CONFIG)
+    .delete(&API_METHOD_DELETE_S3_CLIENT_CONFIG);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_S3_CLIENT_CONFIG)
+    .post(&API_METHOD_CREATE_S3_CLIENT_CONFIG)
+    .match_all("id", &ITEM_ROUTER);
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

