From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id D914B1FF17A for ; Fri, 18 Jul 2025 10:10:24 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CA453158AE; Fri, 18 Jul 2025 10:11:32 +0200 (CEST) Message-ID: <8724b580-8e64-4f4c-a0ef-4650eaa61702@proxmox.com> Date: Fri, 18 Jul 2025 10:11:29 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Proxmox Backup Server development discussion , Christian Ebner References: <20250715125332.954494-1-c.ebner@proxmox.com> <20250715125332.954494-18-c.ebner@proxmox.com> Content-Language: de-AT, en-US From: Lukas Wagner In-Reply-To: <20250715125332.954494-18-c.ebner@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1752826286508 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.020 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v8 08/45] api: backup: conditionally upload chunks to s3 object store backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Reviewed-by: Lukas Wagner On 2025-07-15 14:52, Christian Ebner wrote: > Upload fixed and dynamic sized chunks to either the filesystem or > the S3 object store, depending on the configured backend. > > Signed-off-by: Christian Ebner > --- > changes since version 7: > - no changes > > src/api2/backup/upload_chunk.rs | 71 +++++++++++++++++++-------------- > 1 file changed, 42 insertions(+), 29 deletions(-) > > diff --git a/src/api2/backup/upload_chunk.rs b/src/api2/backup/upload_chunk.rs > index 2c66c2855..3ad8c3c75 100644 > --- a/src/api2/backup/upload_chunk.rs > +++ b/src/api2/backup/upload_chunk.rs > @@ -16,7 +16,7 @@ use proxmox_sortable_macro::sortable; > > use pbs_api_types::{BACKUP_ARCHIVE_NAME_SCHEMA, CHUNK_DIGEST_SCHEMA}; > use pbs_datastore::file_formats::{DataBlobHeader, EncryptedDataBlobHeader}; > -use pbs_datastore::{DataBlob, DataStore}; > +use pbs_datastore::{DataBlob, DataStore, DatastoreBackend}; > use pbs_tools::json::{required_integer_param, required_string_param}; > > use super::environment::*; > @@ -154,22 +154,10 @@ fn upload_fixed_chunk( > ) -> ApiResponseFuture { > async move { > let wid = required_integer_param(¶m, "wid")? as usize; > - let size = required_integer_param(¶m, "size")? as u32; > - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; > - > - let digest_str = required_string_param(¶m, "digest")?; > - let digest = <[u8; 32]>::from_hex(digest_str)?; > - > let env: &BackupEnvironment = rpcenv.as_ref(); > > - let (digest, size, compressed_size, is_duplicate) = UploadChunk::new( > - BodyDataStream::new(req_body), > - env.datastore.clone(), > - digest, > - size, > - encoded_size, > - ) > - .await?; > + let (digest, size, compressed_size, is_duplicate) = > + upload_to_backend(req_body, param, env).await?; > > env.register_fixed_chunk(wid, digest, size, compressed_size, is_duplicate)?; > let digest_str = hex::encode(digest); > @@ -229,22 +217,10 @@ fn upload_dynamic_chunk( > ) -> ApiResponseFuture { > async move { > let wid = required_integer_param(¶m, "wid")? as usize; > - let size = required_integer_param(¶m, "size")? as u32; > - let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; > - > - let digest_str = required_string_param(¶m, "digest")?; > - let digest = <[u8; 32]>::from_hex(digest_str)?; > - > let env: &BackupEnvironment = rpcenv.as_ref(); > > - let (digest, size, compressed_size, is_duplicate) = UploadChunk::new( > - BodyDataStream::new(req_body), > - env.datastore.clone(), > - digest, > - size, > - encoded_size, > - ) > - .await?; > + let (digest, size, compressed_size, is_duplicate) = > + upload_to_backend(req_body, param, env).await?; > > env.register_dynamic_chunk(wid, digest, size, compressed_size, is_duplicate)?; > let digest_str = hex::encode(digest); > @@ -256,6 +232,43 @@ fn upload_dynamic_chunk( > .boxed() > } > > +async fn upload_to_backend( > + req_body: Incoming, > + param: Value, > + env: &BackupEnvironment, > +) -> Result<([u8; 32], u32, u32, bool), Error> { > + let size = required_integer_param(¶m, "size")? as u32; > + let encoded_size = required_integer_param(¶m, "encoded-size")? as u32; > + let digest_str = required_string_param(¶m, "digest")?; > + let digest = <[u8; 32]>::from_hex(digest_str)?; > + > + match &env.backend { > + DatastoreBackend::Filesystem => { > + UploadChunk::new( > + BodyDataStream::new(req_body), > + env.datastore.clone(), > + digest, > + size, > + encoded_size, > + ) > + .await > + } > + DatastoreBackend::S3(s3_client) => { > + let data = req_body.collect().await?.to_bytes(); > + if encoded_size != data.len() as u32 { > + bail!( > + "got blob with unexpected length ({encoded_size} != {})", > + data.len() > + ); > + } > + > + let object_key = pbs_datastore::s3::object_key_from_digest(&digest)?; > + let is_duplicate = s3_client.upload_with_retry(object_key, data, false).await?; > + Ok((digest, size, encoded_size, is_duplicate)) > + } > + } > +} > + > pub const API_METHOD_UPLOAD_SPEEDTEST: ApiMethod = ApiMethod::new( > &ApiHandler::AsyncHttp(&upload_speedtest), > &ObjectSchema::new("Test upload speed.", &[]), -- - Lukas _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel