From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 144031FF3A8 for ; Tue, 28 May 2024 11:52:19 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0E1331392F; Tue, 28 May 2024 11:52:43 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 28 May 2024 11:42:34 +0200 Message-Id: <20240528094303.309806-41-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240528094303.309806-1-c.ebner@proxmox.com> References: <20240528094303.309806-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.028 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v8 proxmox-backup 40/69] client: pxar: helper for lookup of reusable dynamic entries X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" The helper method allows to lookup the entries of a dynamic index which fully cover a given offset range. Further, the helper returns the start padding from the start offset of the dynamic index entry to the start offset of the given range and the end padding. This will be used to lookup size and digest for chunks covering the payload range of a regular file in order to re-use found chunks by indexing them in the archives index file instead of re-encoding the payload. Signed-off-by: Christian Ebner --- changes since version 7: - no changes changes since version 6: - no changes pbs-client/src/pxar/create.rs | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index cc75f0262..6dbd1e664 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet}; use std::ffi::{CStr, CString, OsStr}; use std::fmt; use std::io::{self, Read}; +use std::ops::Range; use std::os::unix::ffi::OsStrExt; use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; @@ -25,6 +26,8 @@ use proxmox_lang::c_str; use proxmox_sys::fs::{self, acl, xattr}; use pbs_datastore::catalog::BackupCatalogWriter; +use pbs_datastore::dynamic_index::DynamicIndexReader; +use pbs_datastore::index::IndexFile; use crate::pxar::metadata::errno_is_unsupported; use crate::pxar::tools::assert_single_path_component; @@ -780,6 +783,73 @@ impl Archiver { } } +/// Dynamic entry reusable by payload references +#[derive(Clone, Debug)] +#[repr(C)] +pub struct ReusableDynamicEntry { + size: u64, + padding: u64, + digest: [u8; 32], +} + +impl ReusableDynamicEntry { + #[inline] + pub fn size(&self) -> u64 { + self.size + } + + #[inline] + pub fn digest(&self) -> [u8; 32] { + self.digest + } +} + +/// List of dynamic entries containing the data given by an offset range +fn lookup_dynamic_entries( + index: &DynamicIndexReader, + range: Range, +) -> Result<(Vec, u64, u64), Error> { + let end_idx = index.index_count() - 1; + let chunk_end = index.chunk_end(end_idx); + let start = index.binary_search(0, 0, end_idx, chunk_end, range.start)?; + + let mut prev_end = if start == 0 { + 0 + } else { + index.chunk_end(start - 1) + }; + let padding_start = range.start - prev_end; + let mut padding_end = 0; + + let mut indices = Vec::new(); + for dynamic_entry in &index.index()[start..] { + let end = dynamic_entry.end(); + + let reusable_dynamic_entry = ReusableDynamicEntry { + size: (end - prev_end), + padding: 0, + digest: dynamic_entry.digest(), + }; + indices.push(reusable_dynamic_entry); + + if range.end < end { + padding_end = end - range.end; + break; + } + prev_end = end; + } + + if let Some(first) = indices.first_mut() { + first.padding += padding_start; + } + + if let Some(last) = indices.last_mut() { + last.padding += padding_end; + } + + Ok((indices, padding_start, padding_end)) +} + fn get_metadata( fd: RawFd, stat: &FileStat, -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel