From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 4F9611FF138 for ; Wed, 04 Mar 2026 14:58:41 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 32D89B08A; Wed, 4 Mar 2026 14:59:42 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox v2 3/4] s3-client: extend provider quirks by delete objects via delete object Date: Wed, 4 Mar 2026 14:59:19 +0100 Message-ID: <20260304135922.717714-4-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260304135922.717714-1-c.ebner@proxmox.com> References: <20260304135922.717714-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1772632750930 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.053 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: Z46AK5R4EMGXPT3PSL26CSACGLQF3PDP X-Message-ID-Hash: Z46AK5R4EMGXPT3PSL26CSACGLQF3PDP X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Provide a workaround for providers not implementing the delete objects S3 API method. If ProviderQuirks::DeleteObjectsViaDeleteObject is set, S3Client::delete_objects() performs individual S3Client::delete_object_impl() calls on each provided object key instead of deleting the keys via the deleteObjects API call. This can also be used to reduce POST calls in favor of multiple DELETE calls, which might be charged differently by some S3 object store providers. Signed-off-by: Christian Ebner --- changes since version 1: - factor out delete_object_impl and use its return type to better handle delete object errors proxmox-s3-client/src/api_types.rs | 2 + proxmox-s3-client/src/client.rs | 66 +++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/proxmox-s3-client/src/api_types.rs b/proxmox-s3-client/src/api_types.rs index 93573fbd..b4fc24e0 100644 --- a/proxmox-s3-client/src/api_types.rs +++ b/proxmox-s3-client/src/api_types.rs @@ -87,6 +87,8 @@ pub const S3_BUCKET_NAME_SCHEMA: Schema = StringSchema::new("Bucket name for S3 pub enum ProviderQuirks { /// Prvider does not support the If-None-Match http header SkipIfNoneMatchHeader, + /// Prvider does not support DeleteObjects API endpoint, use delete object calls instead + DeleteObjectsViaDeleteObject, } serde_plain::derive_display_from_serialize!(ProviderQuirks); serde_plain::derive_fromstr_from_deserialize!(ProviderQuirks); diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index e536614b..5eea4dd3 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -29,9 +29,9 @@ use crate::aws_sign_v4::AWS_SIGN_V4_DATETIME_FORMAT; use crate::aws_sign_v4::{aws_sign_v4_signature, aws_sign_v4_uri_encode}; use crate::object_key::S3ObjectKey; use crate::response_reader::{ - CopyObjectResponse, DeleteObjectsResponse, DeletedObject, GetObjectResponse, - HeadObjectResponse, ListBucketsResponse, ListObjectsV2Response, PutObjectResponse, - ResponseReader, + CopyObjectResponse, DeleteError, DeleteObjectError, DeleteObjectsResponse, DeletedObject, + GetObjectResponse, HeadObjectResponse, ListBucketsResponse, ListObjectsV2Response, + PutObjectResponse, ResponseReader, }; /// Default timeout for s3 api requests. @@ -546,18 +546,31 @@ impl S3Client { /// Removes an object from a bucket. /// See reference docs: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html pub async fn delete_object(&self, object_key: S3ObjectKey) -> Result { + self.delete_object_impl(object_key) + .await + .map_err(Into::into) + } + + async fn delete_object_impl( + &self, + object_key: S3ObjectKey, + ) -> Result { let object_key = object_key.to_full_key(&self.options.common_prefix); let request = Request::builder() .method(Method::DELETE) - .uri(self.build_uri(&object_key, &[])?) - .body(Body::empty())?; + .uri( + self.build_uri(&object_key, &[]) + .map_err(|err| DeleteError::Parsing(err))?, + ) + .body(Body::empty()) + .map_err(|err| DeleteError::Parsing(err.into()))?; - let response = self.send(request, None).await?; - let response_reader = ResponseReader::new(response); - response_reader - .delete_object_response(object_key) + let response = self + .send(request, None) .await - .map_err(Into::into) + .map_err(|err| DeleteError::Parsing(err))?; + let response_reader = ResponseReader::new(response); + response_reader.delete_object_response(object_key).await } /// Delete multiple objects from a bucket using a single HTTP request. @@ -570,6 +583,39 @@ impl S3Client { return Ok(DeleteObjectsResponse::default()); } + if self + .options + .provider_quirks + .contains(&ProviderQuirks::DeleteObjectsViaDeleteObject) + { + let mut response = DeleteObjectsResponse::default(); + response.deleted = Some(Vec::with_capacity(object_keys.len())); + + for object_key in object_keys { + match self.delete_object_impl(object_key.clone()).await { + Ok(deleted_object) => { + let deleted = response.deleted.get_or_insert(Vec::new()); + deleted.push(deleted_object); + } + Err(err) => { + let errors = response.error.get_or_insert(Vec::new()); + let err = match err { + DeleteError::Response(err) => err, + DeleteError::Parsing(err) => DeleteObjectError { + code: None, + key: Some(object_key.clone()), + message: Some(format!("{err}")), + version_id: None, + }, + }; + errors.push(err); + } + } + } + + return Ok(response); + } + let mut body = String::from(r#""#); for object_key in object_keys { body.push_str(""); -- 2.47.3