From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id D86501FF16B for ; Tue, 29 Jul 2025 10:17:16 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id C6BD1A35C; Tue, 29 Jul 2025 10:18:41 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 29 Jul 2025 10:17:47 +0200 Message-ID: <20250729081750.203736-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250729081750.203736-1-c.ebner@proxmox.com> References: <20250729081750.203736-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753777079297 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.045 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [client.rs, proxmox.com] Subject: [pbs-devel] [PATCH proxmox v2 2/2] s3 client: extend client options by feature list X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Adds a feature list to the s3 client options, which allows to define a set of provider specific implementation features or limitations to be used by the client. As a first limitation, the SkipIfNoneMatchHeader is added for provider not accepting and implementing this headers functionality. Defines also an associated helper function on S3ClientOptions to map a given provider quirk to a feature list. Signed-off-by: Christian Ebner --- changes since version 1: - use feature list instead of provider for s3 client options, adding flexibility for further extension - add helper to map from provider quirk to feature list - set If-None-Match header based on feature instead of provider quirk proxmox-s3-client/examples/s3_client.rs | 1 + proxmox-s3-client/src/api_types.rs | 11 +++++++++++ proxmox-s3-client/src/client.rs | 26 +++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/proxmox-s3-client/examples/s3_client.rs b/proxmox-s3-client/examples/s3_client.rs index 1cbb3939..21b0e9ce 100644 --- a/proxmox-s3-client/examples/s3_client.rs +++ b/proxmox-s3-client/examples/s3_client.rs @@ -38,6 +38,7 @@ async fn run() -> Result<(), anyhow::Error> { // `openssl s_client -connect testbucket.s3.pve-c1.local:7480 < /dev/null | openssl x509 -fingerprint -sha256 -noout` fingerprint: Some("".to_string()), put_rate_limit: None, + features: Vec::new(), }; // Creating a client instance and connect to api endpoint diff --git a/proxmox-s3-client/src/api_types.rs b/proxmox-s3-client/src/api_types.rs index 7d1ad24e..a87e0804 100644 --- a/proxmox-s3-client/src/api_types.rs +++ b/proxmox-s3-client/src/api_types.rs @@ -211,3 +211,14 @@ pub struct S3ClientConfigWithoutSecret { #[serde(flatten)] pub config: S3ClientConfig, } + +#[api] +#[derive(Copy, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +/// Provider specific implementation feature or limitation. +pub enum S3ClientFeature { + /// If-None-Match http header not implemented and not accepted. + SkipIfNoneMatchHeader, +} +serde_plain::derive_display_from_serialize!(S3ClientFeature); +serde_plain::derive_fromstr_from_deserialize!(S3ClientFeature); diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index 3a981bf4..ce57722a 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -22,7 +22,7 @@ use proxmox_http::client::HttpsConnector; use proxmox_http::{Body, RateLimit, RateLimiter}; use proxmox_schema::api_types::CERT_FINGERPRINT_SHA256_SCHEMA; -use crate::api_types::S3ClientConfig; +use crate::api_types::{ProviderQuirks, S3ClientConfig, S3ClientFeature}; use crate::aws_sign_v4::AWS_SIGN_V4_DATETIME_FORMAT; use crate::aws_sign_v4::{aws_sign_v4_signature, aws_sign_v4_uri_encode}; use crate::object_key::S3ObjectKey; @@ -69,6 +69,8 @@ pub struct S3ClientOptions { pub fingerprint: Option, /// Rate limit for put requests given as #reqest/s. pub put_rate_limit: Option, + /// Provider implementation specific features and limitations + pub features: Vec, } impl S3ClientOptions { @@ -90,6 +92,18 @@ impl S3ClientOptions { access_key: config.access_key, secret_key, put_rate_limit: config.put_rate_limit, + features: Self::map_provider_quirks_to_features(config.provider_quirks), + } + } + + fn map_provider_quirks_to_features( + provider_quirks: Option, + ) -> Vec { + match provider_quirks { + Some(ProviderQuirks::Backblaze) | Some(ProviderQuirks::Infomaniak) => { + vec![S3ClientFeature::SkipIfNoneMatchHeader] + } + _ => Vec::new(), } } } @@ -393,7 +407,15 @@ impl S3Client { .header(header::CONTENT_TYPE, "binary/octet"); if !replace { - request = request.header(header::IF_NONE_MATCH, "*"); + // Some providers not implement this and fails with error if the header is set, + // see https://forum.proxmox.com/threads/168834/post-786278 + if !self + .options + .features + .contains(&S3ClientFeature::SkipIfNoneMatchHeader) + { + request = request.header(header::IF_NONE_MATCH, "*"); + } } let request = request.body(object_data)?; -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel