From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 45FE41FF17A for ; Fri, 18 Jul 2025 17:03:56 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 73BD7336C4; Fri, 18 Jul 2025 17:05:04 +0200 (CEST) Message-ID: <22a2cfc2-1531-4b37-b350-b95d55281f22@proxmox.com> Date: Fri, 18 Jul 2025 17:04:30 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Lukas Wagner , Proxmox Backup Server development discussion References: <20250715125332.954494-1-c.ebner@proxmox.com> <20250715125332.954494-43-c.ebner@proxmox.com> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1752851068014 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.046 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup v8 33/45] datastore: local chunk reader: get cached chunk from local cache store X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On 7/18/25 1:36 PM, Lukas Wagner wrote: > With my nits addresses: > > Reviewed-by: Lukas Wagner > > > On 2025-07-15 14:53, Christian Ebner wrote: >> Check if a chunk is contained in the local cache and if so prefer >> fetching it from the cache instead of pulling it via the S3 api. This >> improves performance and reduces number of requests to the backend. >> >> Basic restore performance tests: >> >> Restored a snapshot containing the linux git repository (on-disk size >> 5.069 GiB, compressed 3.718 GiB) from an AWS S3 backed datastore, with >> and without cached contents: >> non cached: 691.95 s >> all cached: 74.89 s >> >> Signed-off-by: Christian Ebner >> --- >> changes since version 7: >> - no changes >> >> pbs-datastore/src/local_chunk_reader.rs | 31 +++++++++++++++++++++---- >> 1 file changed, 26 insertions(+), 5 deletions(-) >> >> diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs >> index f5aa217ae..7ad44c4fa 100644 >> --- a/pbs-datastore/src/local_chunk_reader.rs >> +++ b/pbs-datastore/src/local_chunk_reader.rs >> @@ -2,7 +2,7 @@ use std::future::Future; >> use std::pin::Pin; >> use std::sync::Arc; >> >> -use anyhow::{bail, Error}; >> +use anyhow::{bail, format_err, Error}; >> use http_body_util::BodyExt; >> >> use pbs_api_types::CryptMode; >> @@ -68,9 +68,18 @@ impl ReadChunk for LocalChunkReader { >> fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result { >> let chunk = match &self.backend { >> DatastoreBackend::Filesystem => self.store.load_chunk(digest)?, >> - DatastoreBackend::S3(s3_client) => { >> - proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))? > > better use Arc::clone here :) adapted here ... > >> - } >> + DatastoreBackend::S3(s3_client) => match self.store.cache() { >> + None => proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))?, >> + Some(cache) => { >> + let mut cacher = self >> + .store >> + .cacher()? >> + .ok_or(format_err!("no cacher for datastore"))?; >> + proxmox_async::runtime::block_on(cache.access(digest, &mut cacher))?.ok_or( >> + format_err!("unable to access chunk with digest {}", hex::encode(digest)), >> + )? >> + } >> + }, >> }; >> self.ensure_crypt_mode(chunk.crypt_mode()?)?; >> >> @@ -98,7 +107,19 @@ impl AsyncReadChunk for LocalChunkReader { >> let raw_data = tokio::fs::read(&path).await?; >> DataBlob::load_from_reader(&mut &raw_data[..])? >> } >> - DatastoreBackend::S3(s3_client) => fetch(s3_client.clone(), digest).await?, >> + DatastoreBackend::S3(s3_client) => match self.store.cache() { >> + None => fetch(s3_client.clone(), digest).await?, > > same here ... and here > >> + Some(cache) => { >> + let mut cacher = self >> + .store >> + .cacher()? >> + .ok_or(format_err!("no cacher for datastore"))?; >> + cache.access(digest, &mut cacher).await?.ok_or(format_err!( >> + "unable to access chunk with digest {}", >> + hex::encode(digest) >> + ))? >> + } >> + }, >> }; >> self.ensure_crypt_mode(chunk.crypt_mode()?)?; >> > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel