[pbs-devel] [PATCH proxmox v7 8/9] pbs-api-types: extend datastore config by backend config enum

From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox v7 8/9] pbs-api-types: extend datastore config by backend config enum
Date: Thu, 10 Jul 2025 19:06:49 +0200	[thread overview]
Message-ID: <20250710170728.102829-9-c.ebner@proxmox.com> (raw)
In-Reply-To: <20250710170728.102829-1-c.ebner@proxmox.com>

Allows to configure a backend config variant for a datastore on
creation. The current default `Filesystem` backend variant is
introduced to be compatible with existing storages. A new S3 backend
variant allows to create datastores backed by an S3 compatible object
store instead.

For S3 backends, the type, id of the corresponding S3 client
configuration as well as the bucket name are stored as property
string. A valid datastore backend configuration for S3 therefore
contains:
```
    ...
    backend bucket=<BUCKET_NAME>,client=<S3_CONFIG_ID>,type=s3
    ...
```

Further, a maximum cache size for the local store cache can be
assigned, for example to limit to max 1G:
```
    ...
    backend bucket=<BUCKET_NAME>,client=<S3_CONFIG_ID>,type=s3,max-cache-size=1G
    ...
```

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 6:
- add max-cache-size option to backend property string, allows to limit
  local datastore cache LRU capacity.

 Cargo.toml                     |   1 +
 pbs-api-types/Cargo.toml       |   1 +
 pbs-api-types/debian/control   |   2 +
 pbs-api-types/src/datastore.rs | 111 ++++++++++++++++++++++++++++++++-
 4 files changed, 114 insertions(+), 1 deletion(-)

diff --git a/Cargo.toml b/Cargo.toml
index d88d8383..f8221877 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -151,6 +151,7 @@ proxmox-product-config = { version = "1.0.0", path = "proxmox-product-config" }
 proxmox-config-digest = { version = "1.0.0", path = "proxmox-config-digest" }
 proxmox-rest-server = { version = "1.0.0", path = "proxmox-rest-server" }
 proxmox-router = { version = "3.2.2", path = "proxmox-router" }
+proxmox-s3-client = { version = "1.0.0", path = "proxmox-s3-client" }
 proxmox-schema = { version = "4.1.0", path = "proxmox-schema" }
 proxmox-section-config = { version = "3.1.0", path = "proxmox-section-config" }
 proxmox-sendmail = { version = "1.0.0", path = "proxmox-sendmail" }
diff --git a/pbs-api-types/Cargo.toml b/pbs-api-types/Cargo.toml
index 0e8ca379..4929f157 100644
--- a/pbs-api-types/Cargo.toml
+++ b/pbs-api-types/Cargo.toml
@@ -20,6 +20,7 @@ proxmox-auth-api = { workspace = true, features = [ "api-types" ] }
 proxmox-apt-api-types.workspace = true
 proxmox-human-byte.workspace = true
 proxmox-lang.workspace=true
+proxmox-s3-client.workspace = true
 proxmox-schema = { workspace = true, features = [ "api-macro" ] }
 proxmox-serde.workspace = true
 proxmox-time.workspace = true
diff --git a/pbs-api-types/debian/control b/pbs-api-types/debian/control
index de863c3b..0c5bffb0 100644
--- a/pbs-api-types/debian/control
+++ b/pbs-api-types/debian/control
@@ -15,6 +15,7 @@ Build-Depends-Arch: cargo:native <!nocheck>,
  librust-proxmox-auth-api-1+default-dev <!nocheck>,
  librust-proxmox-human-byte-1+default-dev <!nocheck>,
  librust-proxmox-lang-1+default-dev (>= 1.5-~~) <!nocheck>,
+ librust-proxmox-s3-client-1+default-dev <!nocheck>,
  librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~) <!nocheck>,
  librust-proxmox-schema-4+default-dev (>= 4.1.0-~~) <!nocheck>,
  librust-proxmox-serde-1+default-dev <!nocheck>,
@@ -46,6 +47,7 @@ Depends:
  librust-proxmox-auth-api-1+default-dev,
  librust-proxmox-human-byte-1+default-dev,
  librust-proxmox-lang-1+default-dev (>= 1.5-~~),
+ librust-proxmox-s3-client-1+default-dev,
  librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~),
  librust-proxmox-schema-4+default-dev (>= 4.1.0-~~),
  librust-proxmox-serde-1+default-dev,
diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs
index 5bd953ac..504d03f9 100644
--- a/pbs-api-types/src/datastore.rs
+++ b/pbs-api-types/src/datastore.rs
@@ -8,6 +8,7 @@ use anyhow::{bail, format_err, Error};
 use const_format::concatcp;
 use serde::{Deserialize, Serialize};
 
+use proxmox_human_byte::HumanByte;
 use proxmox_schema::{
     api, const_regex, ApiStringFormat, ApiType, ArraySchema, EnumEntry, IntegerSchema, ReturnType,
     Schema, StringSchema, Updater, UpdaterType,
@@ -286,6 +287,104 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore
     ))
     .schema();
 
+#[api]
+#[derive(Copy, Clone, Default, Deserialize, Serialize, Updater, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Datastore backend type
+pub enum DatastoreBackendType {
+    /// Local filesystem
+    #[default]
+    Filesystem,
+    /// S3 object store
+    S3,
+}
+serde_plain::derive_display_from_serialize!(DatastoreBackendType);
+serde_plain::derive_fromstr_from_deserialize!(DatastoreBackendType);
+
+#[api(
+    properties: {
+        type: {
+            type: DatastoreBackendType,
+            optional: true,
+        },
+        client: {
+            schema: proxmox_s3_client::S3_CLIENT_ID_SCHEMA,
+            optional: true,
+        },
+        bucket: {
+            schema: proxmox_s3_client::S3_BUCKET_NAME_SCHEMA,
+            optional: true,
+        },
+        "max-cache-size": {
+            type: HumanByte,
+            optional: true,
+        }
+    },
+    default_key: "type",
+)]
+#[derive(Default, Deserialize, Serialize)]
+#[serde(rename_all = "kebab-case")]
+/// Datastore backend config
+pub struct DatastoreBackendConfig {
+    /// backend type
+    #[serde(rename = "type")]
+    pub ty: Option<DatastoreBackendType>,
+    /// s3 client id
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub client: Option<String>,
+    /// s3 bucket name
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bucket: Option<String>,
+    /// maximum cache size for local datastore LRU cache
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub max_cache_size: Option<HumanByte>,
+}
+
+pub const DATASTORE_BACKEND_CONFIG_STRING_SCHEMA: Schema =
+    StringSchema::new("Datastore backend config")
+        .format(&ApiStringFormat::VerifyFn(verify_datastore_backend_config))
+        .type_text("<backend-config>")
+        .schema();
+
+fn verify_datastore_backend_config(input: &str) -> Result<(), Error> {
+    DatastoreBackendConfig::from_str(input).map(|_| ())
+}
+
+impl FromStr for DatastoreBackendConfig {
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let backend_config: DatastoreBackendConfig =
+            proxmox_schema::property_string::parse_with_schema(
+                s,
+                &DatastoreBackendConfig::API_SCHEMA,
+            )?;
+        let backend_type = backend_config.ty.unwrap_or_default();
+        match backend_type {
+            DatastoreBackendType::Filesystem => {
+                if backend_config.client.is_some() {
+                    bail!("additional option client, not allowed for backend type filesystem");
+                }
+                if backend_config.bucket.is_some() {
+                    bail!("additional option bucket, not allowed for backend type filesystem");
+                }
+                if backend_config.max_cache_size.is_some() {
+                    bail!("additional option max-cache-size, not allowed for backend type filesystem");
+                }
+            }
+            DatastoreBackendType::S3 => {
+                if backend_config.client.is_none() {
+                    bail!("missing option client, required for backend type s3");
+                }
+                if backend_config.bucket.is_none() {
+                    bail!("missing option bucket, required for backend type s3");
+                }
+            }
+        }
+        Ok(backend_config)
+    }
+}
+
 #[api(
     properties: {
         name: {
@@ -336,7 +435,11 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore
             optional: true,
             format: &proxmox_schema::api_types::UUID_FORMAT,
             type: String,
-        }
+        },
+        backend: {
+            schema: DATASTORE_BACKEND_CONFIG_STRING_SCHEMA,
+            optional: true,
+        },
     }
 )]
 #[derive(Serialize, Deserialize, Updater, Clone, PartialEq)]
@@ -389,6 +492,11 @@ pub struct DataStoreConfig {
     #[updater(skip)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub backing_device: Option<String>,
+
+    /// Backend configuration for datastore
+    #[updater(skip)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub backend: Option<String>,
 }
 
 #[api]
@@ -424,6 +532,7 @@ impl DataStoreConfig {
             tuning: None,
             maintenance_mode: None,
             backing_device: None,
+            backend: None,
         }
     }
 
-- 
2.47.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

