From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id EE12398B82 for ; Wed, 15 Nov 2023 16:49:28 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B23719670 for ; Wed, 15 Nov 2023 16:48:35 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 15 Nov 2023 16:48:35 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id DD51C432A8 for ; Wed, 15 Nov 2023 16:48:34 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 15 Nov 2023 16:48:10 +0100 Message-Id: <20231115154813.281564-26-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20231115154813.281564-1-c.ebner@proxmox.com> References: <20231115154813.281564-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.057 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v5 proxmox-backup 25/28] catalog: fetch offset and size for files and refs X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Nov 2023 15:49:29 -0000 Allows to fetch the pxar archive offsets and file size for regular files and appendix referenced files. Signed-off-by: Christian Ebner --- Changes since version 4: - no changes Changes since version 3: - no present in version 3 pbs-datastore/src/catalog.rs | 70 ++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pbs-datastore/src/catalog.rs b/pbs-datastore/src/catalog.rs index 220313c6..fe076a94 100644 --- a/pbs-datastore/src/catalog.rs +++ b/pbs-datastore/src/catalog.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::ffi::{CStr, CString, OsStr}; use std::fmt; use std::io::{Read, Seek, SeekFrom, Write}; @@ -1118,6 +1119,75 @@ impl CatalogReader { Ok(res) } + + /// Get all File and AppendixRef entries with their pxar archive offset and size + pub fn fetch_offsets(&mut self) -> Result, Error> { + let root = self.root()?; + let mut list = BTreeMap::new(); + match root { + DirEntry { + attr: DirEntryAttribute::Directory { start }, + .. + } => self.fetch_offsets_from_dir(std::path::Path::new("./"), start, &mut list, None)?, + _ => bail!("unexpected root entry type, not a directory!"), + } + Ok(list) + } + + fn fetch_offsets_from_dir( + &mut self, + prefix: &std::path::Path, + start: u64, + list: &mut BTreeMap, + appendix_start: Option, + ) -> Result<(), Error> { + let data = self.read_raw_dirinfo_block(start)?; + + DirInfo::parse( + &data, + self.magic, + |etype, name_bytes, offset, size, _mtime, _ctime, link_offset| { + let mut path = std::path::PathBuf::from(prefix); + let name: &OsStr = OsStrExt::from_bytes(name_bytes); + path.push(name); + + match etype { + CatalogEntryType::Archive => { + if offset > start { + bail!("got wrong archive offset ({} > {})", offset, start); + } + let pos = start - offset; + let appendix_start = self.appendix_offset(name_bytes)?; + self.fetch_offsets_from_dir(&path, pos, list, appendix_start)?; + } + CatalogEntryType::Directory => { + if offset > start { + bail!("got wrong directory offset ({} > {})", offset, start); + } + let pos = start - offset; + self.fetch_offsets_from_dir(&path, pos, list, appendix_start)?; + } + CatalogEntryType::AppendixRef => { + if let Some(Offset::AppendixRefOffset { offset }) = link_offset { + if let Some(appendix_start) = appendix_start { + list.insert(appendix_start.raw() + offset, size); + } else { + bail!("missing required appendix start offset"); + } + } + } + CatalogEntryType::File => { + if let Some(Offset::FileOffset { offset }) = link_offset { + list.insert(offset, size); + } + } + _ => {} + } + Ok(true) + }, + )?; + Ok(()) + } } /// Serialize i64 as short, variable length byte sequence -- 2.39.2