From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id A4B521FF165 for ; Thu, 3 Jul 2025 15:19:13 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1057714343; Thu, 3 Jul 2025 15:19:47 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 3 Jul 2025 15:18:08 +0200 Message-ID: <20250703131837.786811-21-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com> References: <20250703131837.786811-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v5 17/46] api: backup: conditionally upload chunks to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Upload fixed and dynamic sized chunks to either the filesystem or the S3 object store, depending on the configured backend. Signed-off-by: Christian Ebner --- src/api2/backup/upload_chunk.rs | 72 ++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs index 2c66c2855..5dd35b435 100644 --- a/src/api2/backup/upload_chunk.rs +++ b/src/api2/backup/upload_chunk.rs @@ -16,7 +16,7 @@ use proxmox_sortable_macro::sortable; use pbs_api_types::{BACKUP_ARCHIVE_NAME_SCHEMA, CHUNK_DIGEST_SCHEMA}; use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader}; -use pbs_datastore::{DataBlob, DataStore}; +use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; use pbs_tools::json::{required_integer_param, required_string_param}; use super::environment::*; @@ -154,22 +154,10 @@ fn upload_fixed_chunk( ) -> ApiResponseFuture { async move { let wid = required_integer_param(¶m, "wid")? as usize; - let size = required_integer_param(¶m, "size")? as u32; - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; - - let digest_str = required_string_param(¶m, "digest")?; - let digest = <[u8; 32]>::from_hex(digest_str)?; - let env: &BackupEnvironment = rpcenv.as_ref(); - let (digest, size, compressed_size, is_duplicate) = UploadChunk::new( - BodyDataStream::new(req_body), - env.datastore.clone(), - digest, - size, - encoded_size, - ) - .await?; + let (digest, size, compressed_size, is_duplicate) = + upload_to_backend(req_body, param, env).await?; env.register_fixed_chunk(wid, digest, size, compressed_size, is_duplicate)?; let digest_str = hex::encode(digest); @@ -229,22 +217,10 @@ fn upload_dynamic_chunk( ) -> ApiResponseFuture { async move { let wid = required_integer_param(¶m, "wid")? as usize; - let size = required_integer_param(¶m, "size")? as u32; - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; - - let digest_str = required_string_param(¶m, "digest")?; - let digest = <[u8; 32]>::from_hex(digest_str)?; - let env: &BackupEnvironment = rpcenv.as_ref(); - let (digest, size, compressed_size, is_duplicate) = UploadChunk::new( - BodyDataStream::new(req_body), - env.datastore.clone(), - digest, - size, - encoded_size, - ) - .await?; + let (digest, size, compressed_size, is_duplicate) = + upload_to_backend(req_body, param, env).await?; env.register_dynamic_chunk(wid, digest, size, compressed_size, is_duplicate)?; let digest_str = hex::encode(digest); @@ -256,6 +232,44 @@ fn upload_dynamic_chunk( .boxed() } +async fn upload_to_backend( + req_body: Incoming, + param: Value, + env: &BackupEnvironment, +) -> Result<([u8; 32], u32, u32, bool), Error> { + let size = required_integer_param(¶m, "size")? as u32; + let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; + let digest_str = required_string_param(¶m, "digest")?; + let digest = <[u8; 32]>::from_hex(digest_str)?; + + match &env.backend { + DatastoreBackend::Filesystem => { + UploadChunk::new( + BodyDataStream::new(req_body), + env.datastore.clone(), + digest, + size, + encoded_size, + ) + .await + } + DatastoreBackend::S3(s3_client) => { + let data = req_body.collect().await?.to_bytes(); + if encoded_size != data.len() as u32 { + bail!( + "got blob with unexpected length ({encoded_size} != {})", + data.len() + ); + } + + let is_duplicate = s3_client + .upload_with_retry(digest.into(), data, false) + .await?; + Ok((digest, size, encoded_size, is_duplicate)) + } + } +} + pub const API_METHOD_UPLOAD_SPEEDTEST: ApiMethod = ApiMethod::new( &ApiHandler::AsyncHttp(&upload_speedtest), &ObjectSchema::new("Test upload speed.", &[]), -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel