[pbs-devel] [PATCH proxmox v5 1/3] pbs-api-types: add types for S3 client configs and secrets

From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox v5 1/3] pbs-api-types: add types for S3 client configs and secrets
Date: Thu,  3 Jul 2025 15:17:49 +0200	[thread overview]
Message-ID: <20250703131837.786811-2-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com>

Adds the new config types `S3ClientConfig` and `S3ClientSecret` to
configure datastore backends using an S3 compatible object store.

Secrets are stored as different config to never be returned on api
calls, only allowing to set/update the values.

Use a different name (`secrets_id`) for the unique identifier in case
of the secrets type, although the same id should be used for storing
and lookup. By this, clashing of property names when using flattened
types as api parameters is avoided.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-api-types/src/lib.rs |   3 +
 pbs-api-types/src/s3.rs  | 161 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 164 insertions(+)
 create mode 100644 pbs-api-types/src/s3.rs

diff --git a/pbs-api-types/src/lib.rs b/pbs-api-types/src/lib.rs
index 99ec7961..7a5ea11d 100644
--- a/pbs-api-types/src/lib.rs
+++ b/pbs-api-types/src/lib.rs
@@ -147,6 +147,9 @@ pub use remote::*;
 mod pathpatterns;
 pub use pathpatterns::*;
 
+mod s3;
+pub use s3::*;
+
 mod tape;
 pub use tape::*;
 
diff --git a/pbs-api-types/src/s3.rs b/pbs-api-types/src/s3.rs
new file mode 100644
index 00000000..a64a2ddf
--- /dev/null
+++ b/pbs-api-types/src/s3.rs
@@ -0,0 +1,161 @@
+use anyhow::bail;
+use const_format::concatcp;
+use serde::{Deserialize, Serialize};
+
+use proxmox_schema::api_types::{
+    CERT_FINGERPRINT_SHA256_SCHEMA, DNS_LABEL_STR, IPRE_STR, SAFE_ID_FORMAT,
+};
+use proxmox_schema::{api, const_regex, ApiStringFormat, Schema, StringSchema, Updater};
+
+#[rustfmt::skip]
+pub const S3_ENDPOINT_NAME_STR: &str = concatcp!(
+    r"(?:(?:(", DNS_LABEL_STR, r"|\{\{bucket\}\}|\{\{region\}\})\.)*", DNS_LABEL_STR, ")"
+);
+#[rustfmt::skip]
+pub const S3_BUCKET_NAME_REGEX_STR: &str = r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$";
+
+const_regex! {
+    /// Regex to match S3 bucket names.
+    ///
+    /// Be as strict as possible following the rules as described here:
+    /// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names
+    pub S3_BUCKET_NAME_REGEX = r"^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$";
+    /// Regex to match S3 endpoints including template patterns.
+    pub S3_ENDPOINT_REGEX = concatcp!(r"^(?:", S3_ENDPOINT_NAME_STR, "|",  IPRE_STR, r")$");
+    /// Regex to match S3 regions.
+    pub S3_REGION_REGEX = r"(^auto$)|(^[a-z]{2,}(?:-[a-z\d]+)+$)";
+}
+
+pub const S3_ENDPOINT_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&S3_ENDPOINT_REGEX);
+pub const S3_REGION_FORMAT: ApiStringFormat = ApiStringFormat::Pattern(&S3_REGION_REGEX);
+
+pub const S3_CLIENT_ID_SCHEMA: Schema =
+    StringSchema::new("Unique ID to identify s3 client config.")
+        .format(&SAFE_ID_FORMAT)
+        .min_length(3)
+        .max_length(32)
+        .schema();
+
+pub const S3_ENDPOINT_SCHEMA: Schema = StringSchema::new("Endpoint to access S3 object store.")
+    .format(&S3_ENDPOINT_FORMAT)
+    .schema();
+
+pub const S3_REGION_SCHEMA: Schema = StringSchema::new("Region to access S3 object store.")
+    .format(&S3_REGION_FORMAT)
+    .min_length(3)
+    .max_length(32)
+    .schema();
+
+pub const S3_BUCKET_NAME_SCHEMA: Schema = StringSchema::new("Bucket name for S3 object store.")
+    .format(&ApiStringFormat::VerifyFn(|bucket_name| {
+        if !(S3_BUCKET_NAME_REGEX.regex_obj)().is_match(bucket_name) {
+            bail!("Bucket name does not match the regex pattern");
+        }
+
+        // Exclude pre- and postfixes described here:
+        // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names
+        let forbidden_prefixes = ["xn--", "sthree-", "amzn-s3-demo-"];
+        for prefix in forbidden_prefixes {
+            if bucket_name.starts_with(prefix) {
+                bail!("Bucket name cannot start with '{prefix}'");
+            }
+        }
+
+        let forbidden_postfixes = ["--ol-s3", ".mrap", "--x-s3"];
+        for postfix in forbidden_postfixes {
+            if bucket_name.ends_with(postfix) {
+                bail!("Bucket name cannot end with '{postfix}'");
+            }
+        }
+
+        Ok(())
+    }))
+    .min_length(3)
+    .max_length(63)
+    .schema();
+
+#[api(
+    properties: {
+        id: {
+            schema: S3_CLIENT_ID_SCHEMA,
+        },
+        endpoint: {
+            schema: S3_ENDPOINT_SCHEMA,
+        },
+        port: {
+            type: u16,
+            description: "Port to access S3 object store.",
+            optional: true,
+        },
+        region: {
+            schema: S3_REGION_SCHEMA,
+            optional: true,
+        },
+        fingerprint: {
+            schema: CERT_FINGERPRINT_SHA256_SCHEMA,
+            optional: true,
+        },
+        "access-key": {
+            type: String,
+            description: "Access key for S3 object store.",
+        },
+        "path-style": {
+            type: bool,
+            optional: true,
+            default: false,
+            description: "Use path style bucket addressing over vhost style.",
+        },
+        "put-rate-limit": {
+            type: u64,
+            optional: true,
+            description: "Rate limit for put requests given as #reqest/s"
+        },
+    }
+)]
+#[derive(Serialize, Deserialize, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// S3 client configuration properties.
+pub struct S3ClientConfig {
+    #[updater(skip)]
+    pub id: String,
+    pub endpoint: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub port: Option<u16>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub region: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fingerprint: Option<String>,
+    pub access_key: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub path_style: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub put_rate_limit: Option<u64>,
+}
+
+impl S3ClientConfig {
+    pub fn acl_path(&self) -> Vec<&str> {
+        // Needs permissions on root path
+        Vec::new()
+    }
+}
+
+#[api(
+    properties: {
+        "secrets-id": {
+            type: String,
+            description: "Unique ID to identify s3 client secret config.",
+        },
+        "secret-key": {
+            type: String,
+            description: "Secret key for S3 object store.",
+        },
+    }
+)]
+#[derive(Serialize, Deserialize, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// S3 client secrets configuration properties.
+pub struct S3ClientSecretsConfig {
+    #[updater(skip)]
+    pub secrets_id: String,
+    pub secret_key: String,
+}
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

