From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 156221FF17A for ; Fri, 18 Jul 2025 13:35:30 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 669F51C0FB; Fri, 18 Jul 2025 13:36:36 +0200 (CEST) Message-ID: Date: Fri, 18 Jul 2025 13:36:33 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Proxmox Backup Server development discussion , Christian Ebner References: <20250715125332.954494-1-c.ebner@proxmox.com> <20250715125332.954494-43-c.ebner@proxmox.com> Content-Language: de-AT, en-US From: Lukas Wagner In-Reply-To: <20250715125332.954494-43-c.ebner@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1752838590194 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.020 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v8 33/45] datastore: local chunk reader: get cached chunk from local cache store X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" With my nits addresses: Reviewed-by: Lukas Wagner On 2025-07-15 14:53, Christian Ebner wrote: > Check if a chunk is contained in the local cache and if so prefer > fetching it from the cache instead of pulling it via the S3 api. This > improves performance and reduces number of requests to the backend. > > Basic restore performance tests: > > Restored a snapshot containing the linux git repository (on-disk size > 5.069 GiB, compressed 3.718 GiB) from an AWS S3 backed datastore, with > and without cached contents: > non cached: 691.95 s > all cached: 74.89 s > > Signed-off-by: Christian Ebner > --- > changes since version 7: > - no changes > > pbs-datastore/src/local_chunk_reader.rs | 31 +++++++++++++++++++++---- > 1 file changed, 26 insertions(+), 5 deletions(-) > > diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs > index f5aa217ae..7ad44c4fa 100644 > --- a/pbs-datastore/src/local_chunk_reader.rs > +++ b/pbs-datastore/src/local_chunk_reader.rs > @@ -2,7 +2,7 @@ use std::future::Future; > use std::pin::Pin; > use std::sync::Arc; > > -use anyhow::{bail, Error}; > +use anyhow::{bail, format_err, Error}; > use http_body_util::BodyExt; > > use pbs_api_types::CryptMode; > @@ -68,9 +68,18 @@ impl ReadChunk for LocalChunkReader { > fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result { > let chunk = match &self.backend { > DatastoreBackend::Filesystem => self.store.load_chunk(digest)?, > - DatastoreBackend::S3(s3_client) => { > - proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))? better use Arc::clone here :) > - } > + DatastoreBackend::S3(s3_client) => match self.store.cache() { > + None => proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))?, > + Some(cache) => { > + let mut cacher = self > + .store > + .cacher()? > + .ok_or(format_err!("no cacher for datastore"))?; > + proxmox_async::runtime::block_on(cache.access(digest, &mut cacher))?.ok_or( > + format_err!("unable to access chunk with digest {}", hex::encode(digest)), > + )? > + } > + }, > }; > self.ensure_crypt_mode(chunk.crypt_mode()?)?; > > @@ -98,7 +107,19 @@ impl AsyncReadChunk for LocalChunkReader { > let raw_data = tokio::fs::read(&path).await?; > DataBlob::load_from_reader(&mut &raw_data[..])? > } > - DatastoreBackend::S3(s3_client) => fetch(s3_client.clone(), digest).await?, > + DatastoreBackend::S3(s3_client) => match self.store.cache() { > + None => fetch(s3_client.clone(), digest).await?, same here > + Some(cache) => { > + let mut cacher = self > + .store > + .cacher()? > + .ok_or(format_err!("no cacher for datastore"))?; > + cache.access(digest, &mut cacher).await?.ok_or(format_err!( > + "unable to access chunk with digest {}", > + hex::encode(digest) > + ))? > + } > + }, > }; > self.ensure_crypt_mode(chunk.crypt_mode()?)?; > -- - Lukas _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel