From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 7B9D71FF165 for ; Thu, 3 Jul 2025 15:19:56 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id EBC7A14A47; Thu, 3 Jul 2025 15:20:16 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 3 Jul 2025 15:18:14 +0200 Message-ID: <20250703131837.786811-27-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250703131837.786811-1-c.ebner@proxmox.com> References: <20250703131837.786811-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.040 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v5 23/46] datastore: local chunk reader: read chunks based on backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Get and store the datastore's backend on local chunk reader instantiantion and fetch chunks based on the variant from either the filesystem or the s3 object store. By storing the backend variant, the s3 client is instantiated only once and reused until the local chunk reader instance is dropped. Signed-off-by: Christian Ebner --- pbs-datastore/Cargo.toml | 1 + pbs-datastore/src/local_chunk_reader.rs | 37 +++++++++++++++++++++---- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml index 6ae6509b3..0513dbce2 100644 --- a/pbs-datastore/Cargo.toml +++ b/pbs-datastore/Cargo.toml @@ -13,6 +13,7 @@ crc32fast.workspace = true endian_trait.workspace = true futures.workspace = true hex = { workspace = true, features = [ "serde" ] } +http-body-util.workspace = true hyper.workspace = true libc.workspace = true log.workspace = true diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs index 05a70c068..f91e1d0f3 100644 --- a/pbs-datastore/src/local_chunk_reader.rs +++ b/pbs-datastore/src/local_chunk_reader.rs @@ -3,17 +3,21 @@ use std::pin::Pin; use std::sync::Arc; use anyhow::{bail, Error}; +use http_body_util::BodyExt; use pbs_api_types::CryptMode; +use pbs_s3_client::S3Client; use pbs_tools::crypt_config::CryptConfig; use crate::data_blob::DataBlob; +use crate::datastore::DatastoreBackend; use crate::read_chunk::{AsyncReadChunk, ReadChunk}; use crate::DataStore; #[derive(Clone)] pub struct LocalChunkReader { store: Arc, + backend: DatastoreBackend, crypt_config: Option>, crypt_mode: CryptMode, } @@ -24,8 +28,11 @@ impl LocalChunkReader { crypt_config: Option>, crypt_mode: CryptMode, ) -> Self { + // TODO: Error handling! + let backend = store.backend().unwrap(); Self { store, + backend, crypt_config, crypt_mode, } @@ -47,10 +54,25 @@ impl LocalChunkReader { } } +async fn fetch(s3_client: Arc, digest: &[u8; 32]) -> Result { + if let Some(response) = s3_client.get_object(digest.into()).await? { + let bytes = response.content.collect().await?.to_bytes(); + DataBlob::from_raw(bytes.to_vec()) + } else { + bail!("no object with digest {}", hex::encode(digest)); + } +} + impl ReadChunk for LocalChunkReader { fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result { - let chunk = self.store.load_chunk(digest)?; + let chunk = match &self.backend { + DatastoreBackend::Filesystem => self.store.load_chunk(digest)?, + DatastoreBackend::S3(s3_client) => { + proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))? + } + }; self.ensure_crypt_mode(chunk.crypt_mode()?)?; + Ok(chunk) } @@ -69,11 +91,14 @@ impl AsyncReadChunk for LocalChunkReader { digest: &'a [u8; 32], ) -> Pin> + Send + 'a>> { Box::pin(async move { - let (path, _) = self.store.chunk_path(digest); - - let raw_data = tokio::fs::read(&path).await?; - - let chunk = DataBlob::load_from_reader(&mut &raw_data[..])?; + let chunk = match &self.backend { + DatastoreBackend::Filesystem => { + let (path, _) = self.store.chunk_path(digest); + let raw_data = tokio::fs::read(&path).await?; + DataBlob::load_from_reader(&mut &raw_data[..])? + } + DatastoreBackend::S3(s3_client) => fetch(s3_client.clone(), digest).await?, + }; self.ensure_crypt_mode(chunk.crypt_mode()?)?; Ok(chunk) -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel