From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 1D26D1FF185 for ; Mon, 21 Jul 2025 15:12:22 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CD3ABF6D6; Mon, 21 Jul 2025 15:13:32 +0200 (CEST) Mime-Version: 1.0 Date: Mon, 21 Jul 2025 15:12:59 +0200 Message-Id: To: "Proxmox Backup Server development discussion" Cc: "pbs-devel" From: "Hannes Laimer" X-Mailer: aerc 0.20.1-112-gd31995f1e20b References: <20250719125035.9926-1-c.ebner@proxmox.com> <20250719125035.9926-19-c.ebner@proxmox.com> In-Reply-To: <20250719125035.9926-19-c.ebner@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753103572453 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.023 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v9 15/46] datastore: local chunk reader: read chunks based on backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On Sat Jul 19, 2025 at 2:50 PM CEST, Christian Ebner wrote: > Get and store the datastore's backend on local chunk reader > instantiantion and fetch chunks based on the variant from either the > filesystem or the s3 object store. > > By storing the backend variant, the s3 client is instantiated only > once and reused until the local chunk reader instance is dropped. > > Signed-off-by: Christian Ebner > --- > changes since version 8: > - use Arc::clone() over .clone() > > pbs-datastore/Cargo.toml | 1 + > pbs-datastore/src/local_chunk_reader.rs | 38 +++++++++++++++++++++---- > 2 files changed, 33 insertions(+), 6 deletions(-) > > diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml > index 7e56dbd31..8ce930a94 100644 > --- a/pbs-datastore/Cargo.toml > +++ b/pbs-datastore/Cargo.toml > @@ -13,6 +13,7 @@ crc32fast.workspace = true > endian_trait.workspace = true > futures.workspace = true > hex = { workspace = true, features = [ "serde" ] } > +http-body-util.workspace = true > hyper.workspace = true > libc.workspace = true > log.workspace = true > diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs > index 05a70c068..667c97206 100644 > --- a/pbs-datastore/src/local_chunk_reader.rs > +++ b/pbs-datastore/src/local_chunk_reader.rs > @@ -3,17 +3,21 @@ use std::pin::Pin; > use std::sync::Arc; > > use anyhow::{bail, Error}; > +use http_body_util::BodyExt; > > use pbs_api_types::CryptMode; > use pbs_tools::crypt_config::CryptConfig; > +use proxmox_s3_client::S3Client; > > use crate::data_blob::DataBlob; > +use crate::datastore::DatastoreBackend; > use crate::read_chunk::{AsyncReadChunk, ReadChunk}; > use crate::DataStore; > > #[derive(Clone)] > pub struct LocalChunkReader { > store: Arc, > + backend: DatastoreBackend, > crypt_config: Option>, > crypt_mode: CryptMode, > } > @@ -24,8 +28,11 @@ impl LocalChunkReader { > crypt_config: Option>, > crypt_mode: CryptMode, > ) -> Self { > + // TODO: Error handling! > + let backend = store.backend().unwrap(); Was this missed, or was this intentionally left in? I feel like we don't want to panic here :P (correct me if I'm wrong, but I think we would whenever anything goes wrong when connecting to s3?) > Self { > store, > + backend, > crypt_config, > crypt_mode, > } > @@ -47,10 +54,26 @@ impl LocalChunkReader { > } > } > > +async fn fetch(s3_client: Arc, digest: &[u8; 32]) -> Result { > + let object_key = crate::s3::object_key_from_digest(digest)?; > + if let Some(response) = s3_client.get_object(object_key).await? { > + let bytes = response.content.collect().await?.to_bytes(); > + DataBlob::from_raw(bytes.to_vec()) > + } else { > + bail!("no object with digest {}", hex::encode(digest)); > + } > +} > + > impl ReadChunk for LocalChunkReader { > fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result { > - let chunk = self.store.load_chunk(digest)?; > + let chunk = match &self.backend { > + DatastoreBackend::Filesystem => self.store.load_chunk(digest)?, > + DatastoreBackend::S3(s3_client) => { > + proxmox_async::runtime::block_on(fetch(Arc::clone(s3_client), digest))? > + } > + }; > self.ensure_crypt_mode(chunk.crypt_mode()?)?; > + > Ok(chunk) > } > > @@ -69,11 +92,14 @@ impl AsyncReadChunk for LocalChunkReader { > digest: &'a [u8; 32], > ) -> Pin> + Send + 'a>> { > Box::pin(async move { > - let (path, _) = self.store.chunk_path(digest); > - > - let raw_data = tokio::fs::read(&path).await?; > - > - let chunk = DataBlob::load_from_reader(&mut &raw_data[..])?; > + let chunk = match &self.backend { > + DatastoreBackend::Filesystem => { > + let (path, _) = self.store.chunk_path(digest); > + let raw_data = tokio::fs::read(&path).await?; > + DataBlob::load_from_reader(&mut &raw_data[..])? > + } > + DatastoreBackend::S3(s3_client) => fetch(Arc::clone(s3_client), digest).await?, > + }; > self.ensure_crypt_mode(chunk.crypt_mode()?)?; > > Ok(chunk) _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel