[PATCH proxmox v2 3/4] s3-client: extend provider quirks by delete objects via delete object

[Christian Ebner <c.ebner@proxmox.com>]

Provide a workaround for providers not implementing the delete
objects S3 API method.

If ProviderQuirks::DeleteObjectsViaDeleteObject is set,
S3Client::delete_objects() performs individual
S3Client::delete_object_impl() calls on each provided object key
instead of deleting the keys via the deleteObjects API call.

This can also be used to reduce POST calls in favor of multiple
DELETE calls, which might be charged differently by some S3 object
store providers.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 1:
- factor out delete_object_impl and use its return type to better
  handle delete object errors

 proxmox-s3-client/src/api_types.rs |  2 +
 proxmox-s3-client/src/client.rs    | 66 +++++++++++++++++++++++++-----
 2 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/proxmox-s3-client/src/api_types.rs b/proxmox-s3-client/src/api_types.rs
index 93573fbd..b4fc24e0 100644
--- a/proxmox-s3-client/src/api_types.rs
+++ b/proxmox-s3-client/src/api_types.rs
@@ -87,6 +87,8 @@ pub const S3_BUCKET_NAME_SCHEMA: Schema = StringSchema::new("Bucket name for S3
 pub enum ProviderQuirks {
     /// Prvider does not support the If-None-Match http header
     SkipIfNoneMatchHeader,
+    /// Prvider does not support DeleteObjects API endpoint, use delete object calls instead
+    DeleteObjectsViaDeleteObject,
 }
 serde_plain::derive_display_from_serialize!(ProviderQuirks);
 serde_plain::derive_fromstr_from_deserialize!(ProviderQuirks);
diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs
index e536614b..5eea4dd3 100644
--- a/proxmox-s3-client/src/client.rs
+++ b/proxmox-s3-client/src/client.rs
@@ -29,9 +29,9 @@ use crate::aws_sign_v4::AWS_SIGN_V4_DATETIME_FORMAT;
 use crate::aws_sign_v4::{aws_sign_v4_signature, aws_sign_v4_uri_encode};
 use crate::object_key::S3ObjectKey;
 use crate::response_reader::{
-    CopyObjectResponse, DeleteObjectsResponse, DeletedObject, GetObjectResponse,
-    HeadObjectResponse, ListBucketsResponse, ListObjectsV2Response, PutObjectResponse,
-    ResponseReader,
+    CopyObjectResponse, DeleteError, DeleteObjectError, DeleteObjectsResponse, DeletedObject,
+    GetObjectResponse, HeadObjectResponse, ListBucketsResponse, ListObjectsV2Response,
+    PutObjectResponse, ResponseReader,
 };
 
 /// Default timeout for s3 api requests.
@@ -546,18 +546,31 @@ impl S3Client {
     /// Removes an object from a bucket.
     /// See reference docs: https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html
     pub async fn delete_object(&self, object_key: S3ObjectKey) -> Result<DeletedObject, Error> {
+        self.delete_object_impl(object_key)
+            .await
+            .map_err(Into::into)
+    }
+
+    async fn delete_object_impl(
+        &self,
+        object_key: S3ObjectKey,
+    ) -> Result<DeletedObject, DeleteError> {
         let object_key = object_key.to_full_key(&self.options.common_prefix);
         let request = Request::builder()
             .method(Method::DELETE)
-            .uri(self.build_uri(&object_key, &[])?)
-            .body(Body::empty())?;
+            .uri(
+                self.build_uri(&object_key, &[])
+                    .map_err(|err| DeleteError::Parsing(err))?,
+            )
+            .body(Body::empty())
+            .map_err(|err| DeleteError::Parsing(err.into()))?;
 
-        let response = self.send(request, None).await?;
-        let response_reader = ResponseReader::new(response);
-        response_reader
-            .delete_object_response(object_key)
+        let response = self
+            .send(request, None)
             .await
-            .map_err(Into::into)
+            .map_err(|err| DeleteError::Parsing(err))?;
+        let response_reader = ResponseReader::new(response);
+        response_reader.delete_object_response(object_key).await
     }
 
     /// Delete multiple objects from a bucket using a single HTTP request.
@@ -570,6 +583,39 @@ impl S3Client {
             return Ok(DeleteObjectsResponse::default());
         }
 
+        if self
+            .options
+            .provider_quirks
+            .contains(&ProviderQuirks::DeleteObjectsViaDeleteObject)
+        {
+            let mut response = DeleteObjectsResponse::default();
+            response.deleted = Some(Vec::with_capacity(object_keys.len()));
+
+            for object_key in object_keys {
+                match self.delete_object_impl(object_key.clone()).await {
+                    Ok(deleted_object) => {
+                        let deleted = response.deleted.get_or_insert(Vec::new());
+                        deleted.push(deleted_object);
+                    }
+                    Err(err) => {
+                        let errors = response.error.get_or_insert(Vec::new());
+                        let err = match err {
+                            DeleteError::Response(err) => err,
+                            DeleteError::Parsing(err) => DeleteObjectError {
+                                code: None,
+                                key: Some(object_key.clone()),
+                                message: Some(format!("{err}")),
+                                version_id: None,
+                            },
+                        };
+                        errors.push(err);
+                    }
+                }
+            }
+
+            return Ok(response);
+        }
+
         let mut body = String::from(r#"<Delete xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#);
         for object_key in object_keys {
             body.push_str("<Object><Key>");
-- 
2.47.3