From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 72F241FF16F for ; Tue, 5 Aug 2025 12:50:36 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CA6FFCAF1; Tue, 5 Aug 2025 12:52:07 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 5 Aug 2025 12:51:18 +0200 Message-ID: <20250805105120.623089-2-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250805105120.623089-1-c.ebner@proxmox.com> References: <20250805105120.623089-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1754391074438 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.044 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox v4 1/1] s3 client: extend client config and options by optional provider quirks X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Add optional provider specific quirks as array to the client configuration and client options, allowing to set a variant of the client to follow some custom implementation variants. As first variant, add the quirk to skip setting the `If-None-Match` header in put object requests, as some providers do not accept and implement this headers functionality. Signed-off-by: Christian Ebner --- changes since version 3: - Drop provider list to feature list mapping, use and expose provider quirks directly. - Squashed together with next patch, as this is now pretty much self contained proxmox-s3-client/examples/s3_client.rs | 1 + proxmox-s3-client/src/api_types.rs | 21 +++++++++++++++++++++ proxmox-s3-client/src/client.rs | 15 +++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/proxmox-s3-client/examples/s3_client.rs b/proxmox-s3-client/examples/s3_client.rs index 4a9625cb..61b88077 100644 --- a/proxmox-s3-client/examples/s3_client.rs +++ b/proxmox-s3-client/examples/s3_client.rs @@ -38,6 +38,7 @@ async fn run() -> Result<(), anyhow::Error> { // `openssl s_client -connect testbucket.s3.pve-c1.local:7480 < /dev/null | openssl x509 -fingerprint -sha256 -noout` fingerprint: Some("".to_string()), put_rate_limit: None, + provider_quirks: Vec::new(), }; // Creating a client instance and connect to api endpoint diff --git a/proxmox-s3-client/src/api_types.rs b/proxmox-s3-client/src/api_types.rs index 265fedb9..c68d85ef 100644 --- a/proxmox-s3-client/src/api_types.rs +++ b/proxmox-s3-client/src/api_types.rs @@ -80,6 +80,17 @@ pub const S3_BUCKET_NAME_SCHEMA: Schema = StringSchema::new("Bucket name for S3 .max_length(63) .schema(); +#[api] +#[derive(Copy, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +/// Provider specific feature implementation quirks. +pub enum ProviderQuirks { + /// Prvider does not support the If-None-Match http header + SkipIfNoneMatchHeader, +} +serde_plain::derive_display_from_serialize!(ProviderQuirks); +serde_plain::derive_fromstr_from_deserialize!(ProviderQuirks); + #[api( properties: { endpoint: { @@ -109,6 +120,13 @@ pub const S3_BUCKET_NAME_SCHEMA: Schema = StringSchema::new("Bucket name for S3 type: u64, optional: true, }, + "provider-quirks": { + type: Array, + optional: true, + items: { + type: ProviderQuirks, + }, + }, }, )] #[derive(Serialize, Deserialize, Updater, Clone, PartialEq)] @@ -134,6 +152,9 @@ pub struct S3ClientConfig { /// Rate limit for put requests given as #reqest/s. #[serde(skip_serializing_if = "Option::is_none")] pub put_rate_limit: Option, + /// List of provider specific feature implementation quirks. + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_quirks: Option>, } impl S3ClientConfig { diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index ae61c590..3bc2a672 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -22,7 +22,7 @@ use proxmox_http::client::HttpsConnector; use proxmox_http::{Body, RateLimit, RateLimiter}; use proxmox_schema::api_types::CERT_FINGERPRINT_SHA256_SCHEMA; -use crate::api_types::S3ClientConfig; +use crate::api_types::{ProviderQuirks, S3ClientConfig}; use crate::aws_sign_v4::AWS_SIGN_V4_DATETIME_FORMAT; use crate::aws_sign_v4::{aws_sign_v4_signature, aws_sign_v4_uri_encode}; use crate::object_key::S3ObjectKey; @@ -69,6 +69,8 @@ pub struct S3ClientOptions { pub fingerprint: Option, /// Rate limit for put requests given as #reqest/s. pub put_rate_limit: Option, + /// Provider implementation specific features and limitations + pub provider_quirks: Vec, } impl S3ClientOptions { @@ -90,6 +92,7 @@ impl S3ClientOptions { access_key: config.access_key, secret_key, put_rate_limit: config.put_rate_limit, + provider_quirks: config.provider_quirks.unwrap_or_default(), } } } @@ -411,7 +414,15 @@ impl S3Client { .header(header::CONTENT_TYPE, "binary/octet"); if !replace { - request = request.header(header::IF_NONE_MATCH, "*"); + // Some providers not implement this and fails with error if the header is set, + // see https://forum.proxmox.com/threads/168834/post-786278 + if !self + .options + .provider_quirks + .contains(&ProviderQuirks::SkipIfNoneMatchHeader) + { + request = request.header(header::IF_NONE_MATCH, "*"); + } } let request = request.body(object_data)?; -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel