From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 3A4451FF185 for ; Mon, 4 Aug 2025 08:52:31 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4968218F18; Mon, 4 Aug 2025 08:53:59 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 4 Aug 2025 08:53:29 +0200 Message-ID: <20250804065331.45272-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250804065331.45272-1-c.ebner@proxmox.com> References: <20250804065331.45272-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1754290416837 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.043 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox v3 2/2] s3 client: extend client options by feature list X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Adds a feature list to the s3 client options, which allows to define a set of provider specific implementation features or limitations to be used by the client. As a first limitation, the SkipIfNoneMatchHeader is added for provider not accepting and implementing this headers functionality. Defines also an associated helper function on S3ClientOptions to map a given provider quirk to a feature list. Signed-off-by: Christian Ebner Reviewed-by: Lukas Wagner Tested-by: Lukas Wagner --- changes since version 2: - rebased onto current master proxmox-s3-client/examples/s3_client.rs | 1 + proxmox-s3-client/src/api_types.rs | 11 +++++++++++ proxmox-s3-client/src/client.rs | 26 +++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/proxmox-s3-client/examples/s3_client.rs b/proxmox-s3-client/examples/s3_client.rs index 4a9625cb..600e2586 100644 --- a/proxmox-s3-client/examples/s3_client.rs +++ b/proxmox-s3-client/examples/s3_client.rs @@ -38,6 +38,7 @@ async fn run() -> Result<(), anyhow::Error> { // `openssl s_client -connect testbucket.s3.pve-c1.local:7480 < /dev/null | openssl x509 -fingerprint -sha256 -noout` fingerprint: Some("".to_string()), put_rate_limit: None, + features: Vec::new(), }; // Creating a client instance and connect to api endpoint diff --git a/proxmox-s3-client/src/api_types.rs b/proxmox-s3-client/src/api_types.rs index 3db6f7a4..baac6909 100644 --- a/proxmox-s3-client/src/api_types.rs +++ b/proxmox-s3-client/src/api_types.rs @@ -226,3 +226,14 @@ pub struct S3BucketListItem { /// S3 bucket name. pub name: String, } + +#[api] +#[derive(Copy, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +/// Provider specific implementation feature or limitation. +pub enum S3ClientFeature { + /// If-None-Match http header not implemented and not accepted. + SkipIfNoneMatchHeader, +} +serde_plain::derive_display_from_serialize!(S3ClientFeature); +serde_plain::derive_fromstr_from_deserialize!(S3ClientFeature); diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index a2c62811..1c923d5f 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -22,7 +22,7 @@ use proxmox_http::client::HttpsConnector; use proxmox_http::{Body, RateLimit, RateLimiter}; use proxmox_schema::api_types::CERT_FINGERPRINT_SHA256_SCHEMA; -use crate::api_types::S3ClientConfig; +use crate::api_types::{ProviderQuirks, S3ClientConfig, S3ClientFeature}; use crate::aws_sign_v4::AWS_SIGN_V4_DATETIME_FORMAT; use crate::aws_sign_v4::{aws_sign_v4_signature, aws_sign_v4_uri_encode}; use crate::object_key::S3ObjectKey; @@ -69,6 +69,8 @@ pub struct S3ClientOptions { pub fingerprint: Option, /// Rate limit for put requests given as #reqest/s. pub put_rate_limit: Option, + /// Provider implementation specific features and limitations + pub features: Vec, } impl S3ClientOptions { @@ -90,6 +92,18 @@ impl S3ClientOptions { access_key: config.access_key, secret_key, put_rate_limit: config.put_rate_limit, + features: Self::map_provider_quirks_to_features(config.provider_quirks), + } + } + + fn map_provider_quirks_to_features( + provider_quirks: Option, + ) -> Vec { + match provider_quirks { + Some(ProviderQuirks::Backblaze) | Some(ProviderQuirks::Infomaniak) => { + vec![S3ClientFeature::SkipIfNoneMatchHeader] + } + _ => Vec::new(), } } } @@ -412,7 +426,15 @@ impl S3Client { .header(header::CONTENT_TYPE, "binary/octet"); if !replace { - request = request.header(header::IF_NONE_MATCH, "*"); + // Some providers not implement this and fails with error if the header is set, + // see https://forum.proxmox.com/threads/168834/post-786278 + if !self + .options + .features + .contains(&S3ClientFeature::SkipIfNoneMatchHeader) + { + request = request.header(header::IF_NONE_MATCH, "*"); + } } let request = request.body(object_data)?; -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel