[pbs-devel] [RFC proxmox-backup 12/39] api: config: implement endpoints to manipulate and list s3 configs

From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [RFC proxmox-backup 12/39] api: config: implement endpoints to manipulate and list s3 configs
Date: Mon, 19 May 2025 13:46:13 +0200	[thread overview]
Message-ID: <20250519114640.303640-13-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250519114640.303640-1-c.ebner@proxmox.com>

Allows to create, list, modify and delete configurations for s3
clients via the api.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/api2/config/mod.rs |   2 +
 src/api2/config/s3.rs  | 349 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 351 insertions(+)
 create mode 100644 src/api2/config/s3.rs

diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 15dc5db92..1cd9ead76 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -14,6 +14,7 @@ pub mod metrics;
 pub mod notifications;
 pub mod prune;
 pub mod remote;
+pub mod s3;
 pub mod sync;
 pub mod tape_backup_job;
 pub mod tape_encryption_keys;
@@ -32,6 +33,7 @@ const SUBDIRS: SubdirMap = &sorted!([
     ("notifications", &notifications::ROUTER),
     ("prune", &prune::ROUTER),
     ("remote", &remote::ROUTER),
+    ("s3", &s3::ROUTER),
     ("sync", &sync::ROUTER),
     ("tape-backup-job", &tape_backup_job::ROUTER),
     ("tape-encryption-keys", &tape_encryption_keys::ROUTER),
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
new file mode 100644
index 000000000..11cf16411
--- /dev/null
+++ b/src/api2/config/s3.rs
@@ -0,0 +1,349 @@
+use ::serde::{Deserialize, Serialize};
+use anyhow::Error;
+use hex::FromHex;
+use serde_json::Value;
+
+use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
+use proxmox_schema::{api, param_bail};
+
+use pbs_api_types::{
+    Authid, S3ClientConfig, S3ClientConfigUpdater, S3ClientSecretsConfig,
+    S3ClientSecretsConfigUpdater, JOB_ID_SCHEMA, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY,
+    PROXMOX_CONFIG_DIGEST_SCHEMA,
+};
+use pbs_config::s3;
+
+use pbs_config::CachedUserInfo;
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured s3 clients.",
+        type: Array,
+        items: { type: S3ClientConfig },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Modify on datastore.",
+    },
+)]
+/// List all s3 client configurations.
+pub fn list_s3_client_config(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<S3ClientConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_MODIFY;
+
+    let (config, digest) = s3::config()?;
+    let list = config.convert_to_typed_array("s3client")?;
+    let list = list
+        .into_iter()
+        .filter(|s3_client_config: &S3ClientConfig| {
+            let privs = user_info.lookup_privs(&auth_id, &s3_client_config.acl_path());
+            privs & required_privs != 00
+        })
+        .collect();
+
+    let (_secrets, secrets_digest) = s3::secrets_config()?;
+    let digest = digest_with_secrets(&digest, &secrets_digest);
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            config: {
+                type: S3ClientConfig,
+                flatten: true,
+            },
+            secrets: {
+                type: S3ClientSecretsConfig,
+                flatten: true,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Modify on datastore.",
+    },
+)]
+/// Create a new s3 client configuration.
+pub fn create_s3_client_config(
+    config: S3ClientConfig,
+    secrets: S3ClientSecretsConfig,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    // Asssure both, config and secrets are referenced by the same `id`
+    if config.id != secrets.secrets_id {
+        param_bail!(
+            "id",
+            "config and secrets must use the same id ({} != {})",
+            config.id,
+            secrets.secrets_id
+        );
+    }
+
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &config.acl_path(), PRIV_DATASTORE_MODIFY, false)?;
+
+    let _lock = s3::lock_config()?;
+    let (mut section_config, _digest) = s3::config()?;
+    if section_config.sections.contains_key(&config.id) {
+        param_bail!("id", "s3 client config '{}' already exists.", config.id);
+    }
+
+    let (mut section_secrets, _secrets_digest) = s3::secrets_config()?;
+    if section_secrets.sections.contains_key(&config.id) {
+        param_bail!("id", "s3 secrets config '{}' already exists.", config.id);
+    }
+
+    section_config.set_data(&config.id, "s3client", &config)?;
+    section_secrets.set_data(&config.id, "s3secrets", &secrets)?;
+    s3::save_config(&section_config, &section_secrets)?;
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+    returns: { type: S3ClientConfig },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Audit or Datastore.Modify on datastore.",
+    },
+)]
+/// Read an s3 client configuration.
+pub fn read_s3_client_config(
+    id: String,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<S3ClientConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let (config, digest) = s3::config()?;
+    let s3_client_config: S3ClientConfig = config.lookup("s3client", &id)?;
+
+    let required_privs = PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_MODIFY;
+    user_info.check_privs(&auth_id, &s3_client_config.acl_path(), required_privs, true)?;
+
+    let (_secrets, secrets_digest) = s3::secrets_config()?;
+    let digest = digest_with_secrets(&digest, &secrets_digest);
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(s3_client_config)
+}
+
+#[api()]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the port property.
+    Port,
+    /// Delete the region property.
+    Region,
+    /// Delete the fingerprint property.
+    Fingerprint,
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            update: {
+                type: S3ClientConfigUpdater,
+                flatten: true,
+            },
+            "update-secrets": {
+                type: S3ClientSecretsConfigUpdater,
+                flatten: true,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Verify on job's datastore.",
+    },
+)]
+/// Update an s3 client configuration.
+#[allow(clippy::too_many_arguments)]
+pub fn update_s3_client_config(
+    id: String,
+    update: S3ClientConfigUpdater,
+    update_secrets: S3ClientSecretsConfigUpdater,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let _lock = s3::lock_config()?;
+    let (mut config, expected_digest) = s3::config()?;
+    let (mut secrets, secrets_digest) = s3::secrets_config()?;
+    let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+    // Secrets are not included in digest concurrent changes therefore not detected.
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: S3ClientConfig = config.lookup("s3client", &id)?;
+    user_info.check_privs(&auth_id, &data.acl_path(), PRIV_DATASTORE_MODIFY, true)?;
+
+    if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::Port => {
+                    data.port = None;
+                }
+                DeletableProperty::Region => {
+                    data.region = None;
+                }
+                DeletableProperty::Fingerprint => {
+                    data.fingerprint = None;
+                }
+            }
+        }
+    }
+
+    if let Some(host) = update.host {
+        data.host = host;
+    }
+    if let Some(bucket) = update.bucket {
+        data.bucket = bucket;
+    }
+    if let Some(port) = update.port {
+        data.port = Some(port);
+    }
+    if let Some(region) = update.region {
+        data.region = Some(region);
+    }
+    if let Some(access_key) = update.access_key {
+        data.access_key = access_key;
+    }
+    if let Some(fingerprint) = update.fingerprint {
+        data.fingerprint = Some(fingerprint);
+    }
+
+    let mut secrets_data: S3ClientSecretsConfig = secrets.lookup("s3secrets", &id)?;
+    if let Some(secret_key) = update_secrets.secret_key {
+        secrets_data.secret_key = secret_key;
+    }
+
+    config.set_data(&id, "s3client", &data)?;
+    secrets.set_data(&id, "s3secrets", &secrets_data)?;
+    s3::save_config(&config, &secrets)?;
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Datastore.Modify on job's datastore.",
+    },
+)]
+/// Remove an s3 client configuration.
+pub fn delete_s3_client_config(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+
+    let _lock = s3::lock_config()?;
+    let (mut config, expected_digest) = s3::config()?;
+    let s3_client_config: S3ClientConfig = config.lookup("s3client", &id)?;
+    user_info.check_privs(
+        &auth_id,
+        &s3_client_config.acl_path(),
+        PRIV_DATASTORE_MODIFY,
+        true,
+    )?;
+
+    let (mut secrets, secrets_digest) = s3::secrets_config()?;
+    let expected_digest = digest_with_secrets(&expected_digest, &secrets_digest);
+
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    match (config.sections.remove(&id), secrets.sections.remove(&id)) {
+        (Some(_), Some(_)) => {}
+        (None, None) => http_bail!(
+            NOT_FOUND,
+            "s3 client config and secrets '{id}' do not exist."
+        ),
+        (Some(_), None) => http_bail!(
+            NOT_FOUND,
+            "removed s3 client config, but no secrets for '{id}' found."
+        ),
+        (None, Some(_)) => http_bail!(
+            NOT_FOUND,
+            "removed s3 client secrets, but no config for '{id}' found."
+        ),
+    }
+    s3::save_config(&config, &secrets)
+}
+
+// Calculate the digest based on the digest of config and secrets to detect changes for both
+fn digest_with_secrets(digest: &[u8; 32], secrets_digest: &[u8; 32]) -> [u8; 32] {
+    let mut digest = digest.to_vec();
+    digest.append(&mut secrets_digest.to_vec());
+    openssl::sha::sha256(&digest)
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_S3_CLIENT_CONFIG)
+    .put(&API_METHOD_UPDATE_S3_CLIENT_CONFIG)
+    .delete(&API_METHOD_DELETE_S3_CLIENT_CONFIG);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_S3_CLIENT_CONFIG)
+    .post(&API_METHOD_CREATE_S3_CLIENT_CONFIG)
+    .match_all("id", &ITEM_ROUTER);
-- 
2.39.5



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

