From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 096F71FF191 for ; Mon, 16 Jun 2025 16:22:08 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E1E89A131; Mon, 16 Jun 2025 16:22:24 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 16 Jun 2025 16:21:32 +0200 Message-Id: <20250616142156.413652-20-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250616142156.413652-1-c.ebner@proxmox.com> References: <20250616142156.413652-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.036 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v3 17/41] api: backup: conditionally upload chunks to S3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Upload fixed and dynamic sized chunks to either the filesystem or the S3 object store, depending on the configured backend. Signed-off-by: Christian Ebner --- src/api2/backup/upload_chunk.rs | 44 ++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs index 20259660a..838eec1fa 100644 --- a/src/api2/backup/upload_chunk.rs +++ b/src/api2/backup/upload_chunk.rs @@ -15,7 +15,8 @@ use proxmox_sortable_macro::sortable; use pbs_api_types::{BACKUP_ARCHIVE_NAME_SCHEMA, CHUNK_DIGEST_SCHEMA}; use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader}; -use pbs_datastore::{DataBlob, DataStore}; +use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; +use pbs_s3_client::PutObjectResponse; use pbs_tools::json::{required_integer_param, required_string_param}; use super::environment::*; @@ -153,16 +154,10 @@ fn upload_fixed_chunk( ) -> ApiResponseFuture { async move { let wid = required_integer_param(¶m, "wid")? as usize; - let size = required_integer_param(¶m, "size")? as u32; - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; - - let digest_str = required_string_param(¶m, "digest")?; - let digest = <[u8; 32]>::from_hex(digest_str)?; - let env: &BackupEnvironment = rpcenv.as_ref(); let (digest, size, compressed_size, is_duplicate) = - UploadChunk::new(req_body, env.datastore.clone(), digest, size, encoded_size).await?; + upload_to_backend(req_body, param, env).await?; env.register_fixed_chunk(wid, digest, size, compressed_size, is_duplicate)?; let digest_str = hex::encode(digest); @@ -222,16 +217,10 @@ fn upload_dynamic_chunk( ) -> ApiResponseFuture { async move { let wid = required_integer_param(¶m, "wid")? as usize; - let size = required_integer_param(¶m, "size")? as u32; - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; - - let digest_str = required_string_param(¶m, "digest")?; - let digest = <[u8; 32]>::from_hex(digest_str)?; - let env: &BackupEnvironment = rpcenv.as_ref(); let (digest, size, compressed_size, is_duplicate) = - UploadChunk::new(req_body, env.datastore.clone(), digest, size, encoded_size).await?; + upload_to_backend(req_body, param, env).await?; env.register_dynamic_chunk(wid, digest, size, compressed_size, is_duplicate)?; let digest_str = hex::encode(digest); @@ -243,6 +232,31 @@ fn upload_dynamic_chunk( .boxed() } +async fn upload_to_backend( + req_body: Body, + param: Value, + env: &BackupEnvironment, +) -> Result<([u8; 32], u32, u32, bool), Error> { + let size = required_integer_param(¶m, "size")? as u32; + let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; + let digest_str = required_string_param(¶m, "digest")?; + let digest = <[u8; 32]>::from_hex(digest_str)?; + + match &env.backend { + DatastoreBackend::Filesystem => { + UploadChunk::new(req_body, env.datastore.clone(), digest, size, encoded_size).await + } + DatastoreBackend::S3(s3_client) => { + let is_duplicate = match s3_client.put_object(digest.into(), req_body).await? { + PutObjectResponse::PreconditionFailed => true, + PutObjectResponse::NeedsRetry => bail!("concurrent operation, reupload required"), + PutObjectResponse::Success(_content) => false, + }; + Ok((digest, size, encoded_size, is_duplicate)) + } + } +} + pub const API_METHOD_UPLOAD_SPEEDTEST: ApiMethod = ApiMethod::new( &ApiHandler::AsyncHttp(&upload_speedtest), &ObjectSchema::new("Test upload speed.", &[]), -- 2.39.5 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel