From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id B16851FF170 for ; Thu, 10 Jul 2025 19:07:58 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6BD8A39868; Thu, 10 Jul 2025 19:08:31 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 10 Jul 2025 19:07:23 +0200 Message-ID: <20250710170728.102829-43-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250710170728.102829-1-c.ebner@proxmox.com> References: <20250710170728.102829-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.044 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v7 33/38] datastore: local chunk reader: get cached chunk from local cache store X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Check if a chunk is contained in the local cache and if so prefer fetching it from the cache instead of pulling it via the S3 api. This improves performance and reduces number of requests to the backend. Basic restore performance tests: Restored a snapshot containing the linux git repository (on-disk size 5.069 GiB, compressed 3.718 GiB) from an AWS S3 backed datastore, with and without cached contents: non cached: 691.95 s all cached: 74.89 s Signed-off-by: Christian Ebner --- changes since version 6: - added basic restore preformace results to the commit message pbs-datastore/src/local_chunk_reader.rs | 31 +++++++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs index f5aa217ae..7ad44c4fa 100644 --- a/pbs-datastore/src/local_chunk_reader.rs +++ b/pbs-datastore/src/local_chunk_reader.rs @@ -2,7 +2,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use anyhow::{bail, Error}; +use anyhow::{bail, format_err, Error}; use http_body_util::BodyExt; use pbs_api_types::CryptMode; @@ -68,9 +68,18 @@ impl ReadChunk for LocalChunkReader { fn read_raw_chunk(&self, digest: &[u8; 32]) -> Result { let chunk = match &self.backend { DatastoreBackend::Filesystem => self.store.load_chunk(digest)?, - DatastoreBackend::S3(s3_client) => { - proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))? - } + DatastoreBackend::S3(s3_client) => match self.store.cache() { + None => proxmox_async::runtime::block_on(fetch(s3_client.clone(), digest))?, + Some(cache) => { + let mut cacher = self + .store + .cacher()? + .ok_or(format_err!("no cacher for datastore"))?; + proxmox_async::runtime::block_on(cache.access(digest, &mut cacher))?.ok_or( + format_err!("unable to access chunk with digest {}", hex::encode(digest)), + )? + } + }, }; self.ensure_crypt_mode(chunk.crypt_mode()?)?; @@ -98,7 +107,19 @@ impl AsyncReadChunk for LocalChunkReader { let raw_data = tokio::fs::read(&path).await?; DataBlob::load_from_reader(&mut &raw_data[..])? } - DatastoreBackend::S3(s3_client) => fetch(s3_client.clone(), digest).await?, + DatastoreBackend::S3(s3_client) => match self.store.cache() { + None => fetch(s3_client.clone(), digest).await?, + Some(cache) => { + let mut cacher = self + .store + .cacher()? + .ok_or(format_err!("no cacher for datastore"))?; + cache.access(digest, &mut cacher).await?.ok_or(format_err!( + "unable to access chunk with digest {}", + hex::encode(digest) + ))? + } + }, }; self.ensure_crypt_mode(chunk.crypt_mode()?)?; -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel