From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 53E081FF185 for ; Mon, 21 Jul 2025 18:44:14 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 5063C15B81; Mon, 21 Jul 2025 18:45:18 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 21 Jul 2025 18:44:19 +0200 Message-ID: <20250721164507.1045869-2-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250721164507.1045869-1-c.ebner@proxmox.com> References: <20250721164507.1045869-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753116303588 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.044 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox v10 1/3] pbs-api-types: extend datastore config by backend config enum X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Allows to configure a backend config variant for a datastore on creation. The current default `Filesystem` backend variant is introduced to be compatible with existing storages. A new S3 backend variant allows to create datastores backed by an S3 compatible object store instead. For S3 backends, the type, id of the corresponding S3 client configuration as well as the bucket name are stored as property string. A valid datastore backend configuration for S3 therefore contains: ``` ... backend bucket=,client=,type=s3 ... ``` Further, a maximum cache size for the local store cache can be assigned, for example to limit to max 1G: ``` ... backend bucket=,client=,type=s3,max-cache-size=1G ... ``` Signed-off-by: Christian Ebner --- changes since version 9: - no changes Cargo.toml | 1 + pbs-api-types/Cargo.toml | 1 + pbs-api-types/debian/control | 2 + pbs-api-types/src/datastore.rs | 113 ++++++++++++++++++++++++++++++++- 4 files changed, 116 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dbc389b9..fd7eba63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,6 +152,7 @@ proxmox-product-config = { version = "1.0.0", path = "proxmox-product-config" } proxmox-config-digest = { version = "1.0.0", path = "proxmox-config-digest" } proxmox-rest-server = { version = "1.0.0", path = "proxmox-rest-server" } proxmox-router = { version = "3.2.2", path = "proxmox-router" } +proxmox-s3-client = { version = "1.0.0", path = "proxmox-s3-client" } proxmox-schema = { version = "4.1.0", path = "proxmox-schema" } proxmox-section-config = { version = "3.1.0", path = "proxmox-section-config" } proxmox-sendmail = { version = "1.0.0", path = "proxmox-sendmail" } diff --git a/pbs-api-types/Cargo.toml b/pbs-api-types/Cargo.toml index e23bd137..4e8c72d9 100644 --- a/pbs-api-types/Cargo.toml +++ b/pbs-api-types/Cargo.toml @@ -20,6 +20,7 @@ proxmox-auth-api = { workspace = true, features = [ "api-types" ] } proxmox-apt-api-types.workspace = true proxmox-human-byte.workspace = true proxmox-lang.workspace=true +proxmox-s3-client.workspace = true proxmox-schema = { workspace = true, features = [ "api-macro" ] } proxmox-serde.workspace = true proxmox-time.workspace = true diff --git a/pbs-api-types/debian/control b/pbs-api-types/debian/control index 8cf785dd..f4bca53f 100644 --- a/pbs-api-types/debian/control +++ b/pbs-api-types/debian/control @@ -15,6 +15,7 @@ Build-Depends-Arch: cargo:native , librust-proxmox-auth-api-1+default-dev , librust-proxmox-human-byte-1+default-dev , librust-proxmox-lang-1+default-dev (>= 1.5-~~) , + librust-proxmox-s3-client-1+default-dev , librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~) , librust-proxmox-schema-4+default-dev (>= 4.1.0-~~) , librust-proxmox-serde-1+default-dev , @@ -46,6 +47,7 @@ Depends: librust-proxmox-auth-api-1+default-dev, librust-proxmox-human-byte-1+default-dev, librust-proxmox-lang-1+default-dev (>= 1.5-~~), + librust-proxmox-s3-client-1+default-dev, librust-proxmox-schema-4+api-macro-dev (>= 4.1.0-~~), librust-proxmox-schema-4+default-dev (>= 4.1.0-~~), librust-proxmox-serde-1+default-dev, diff --git a/pbs-api-types/src/datastore.rs b/pbs-api-types/src/datastore.rs index 62b20460..7110a329 100644 --- a/pbs-api-types/src/datastore.rs +++ b/pbs-api-types/src/datastore.rs @@ -8,6 +8,7 @@ use anyhow::{bail, format_err, Error}; use const_format::concatcp; use serde::{Deserialize, Serialize}; +use proxmox_human_byte::HumanByte; use proxmox_schema::{ api, const_regex, ApiStringFormat, ApiType, ArraySchema, EnumEntry, IntegerSchema, ReturnType, Schema, StringSchema, Updater, UpdaterType, @@ -286,6 +287,106 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore )) .schema(); +#[api] +#[derive(Copy, Clone, Default, Deserialize, Serialize, Updater, PartialEq)] +#[serde(rename_all = "kebab-case")] +/// Datastore backend type +pub enum DatastoreBackendType { + /// Local filesystem + #[default] + Filesystem, + /// S3 object store + S3, +} +serde_plain::derive_display_from_serialize!(DatastoreBackendType); +serde_plain::derive_fromstr_from_deserialize!(DatastoreBackendType); + +#[api( + properties: { + type: { + type: DatastoreBackendType, + optional: true, + }, + client: { + schema: proxmox_s3_client::S3_CLIENT_ID_SCHEMA, + optional: true, + }, + bucket: { + schema: proxmox_s3_client::S3_BUCKET_NAME_SCHEMA, + optional: true, + }, + "max-cache-size": { + type: HumanByte, + optional: true, + } + }, + default_key: "type", +)] +#[derive(Default, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +/// Datastore backend config +pub struct DatastoreBackendConfig { + /// backend type + #[serde(rename = "type")] + pub ty: Option, + /// s3 client id + #[serde(skip_serializing_if = "Option::is_none")] + pub client: Option, + /// s3 bucket name + #[serde(skip_serializing_if = "Option::is_none")] + pub bucket: Option, + /// maximum cache size for local datastore LRU cache + #[serde(skip_serializing_if = "Option::is_none")] + pub max_cache_size: Option, +} + +pub const DATASTORE_BACKEND_CONFIG_STRING_SCHEMA: Schema = + StringSchema::new("Datastore backend config") + .format(&ApiStringFormat::VerifyFn(verify_datastore_backend_config)) + .type_text("") + .schema(); + +fn verify_datastore_backend_config(input: &str) -> Result<(), Error> { + DatastoreBackendConfig::from_str(input).map(|_| ()) +} + +impl FromStr for DatastoreBackendConfig { + type Err = Error; + + fn from_str(s: &str) -> Result { + let backend_config: DatastoreBackendConfig = + proxmox_schema::property_string::parse_with_schema( + s, + &DatastoreBackendConfig::API_SCHEMA, + )?; + let backend_type = backend_config.ty.unwrap_or_default(); + match backend_type { + DatastoreBackendType::Filesystem => { + if backend_config.client.is_some() { + bail!("additional option client, not allowed for backend type filesystem"); + } + if backend_config.bucket.is_some() { + bail!("additional option bucket, not allowed for backend type filesystem"); + } + if backend_config.max_cache_size.is_some() { + bail!( + "additional option max-cache-size, not allowed for backend type filesystem" + ); + } + } + DatastoreBackendType::S3 => { + if backend_config.client.is_none() { + bail!("missing option client, required for backend type s3"); + } + if backend_config.bucket.is_none() { + bail!("missing option bucket, required for backend type s3"); + } + } + } + Ok(backend_config) + } +} + #[api( properties: { name: { @@ -336,7 +437,11 @@ pub const DATASTORE_TUNING_STRING_SCHEMA: Schema = StringSchema::new("Datastore optional: true, format: &proxmox_schema::api_types::UUID_FORMAT, type: String, - } + }, + backend: { + schema: DATASTORE_BACKEND_CONFIG_STRING_SCHEMA, + optional: true, + }, } )] #[derive(Serialize, Deserialize, Updater, Clone, PartialEq)] @@ -389,6 +494,11 @@ pub struct DataStoreConfig { #[updater(skip)] #[serde(skip_serializing_if = "Option::is_none")] pub backing_device: Option, + + /// Backend configuration for datastore + #[updater(skip)] + #[serde(skip_serializing_if = "Option::is_none")] + pub backend: Option, } #[api] @@ -424,6 +534,7 @@ impl DataStoreConfig { tuning: None, maintenance_mode: None, backing_device: None, + backend: None, } } -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel