From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D3BA298AFF for ; Wed, 15 Nov 2023 16:49:28 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id EA1CE964D for ; Wed, 15 Nov 2023 16:48:33 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 15 Nov 2023 16:48:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 243D6432CA for ; Wed, 15 Nov 2023 16:48:32 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 15 Nov 2023 16:47:58 +0100 Message-Id: <20231115154813.281564-14-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20231115154813.281564-1-c.ebner@proxmox.com> References: <20231115154813.281564-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.059 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v5 proxmox-backup 13/28] fix #3174: archiver/extractor: impl appendix ref X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Nov 2023 15:49:28 -0000 Implements the functionality to create and extract appendix references via the pbs client. This reuses the pxar encoders functionality to write appendix reference entries and adds the implementation to store and access them in the catalog. Signed-off-by: Christian Ebner --- Changes since version 4: - no changes Changes since version 3: - no changes Changes since version 2: - no changes Changes since version 1: - Use custom types for appendix ref archive offsets - Include catalog implementation in this patch pbs-client/src/catalog_shell.rs | 2 +- pbs-client/src/pxar/create.rs | 12 ++++ pbs-client/src/pxar/extract.rs | 16 +++++ pbs-client/src/pxar/tools.rs | 8 +++ pbs-datastore/src/catalog.rs | 103 ++++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 1 deletion(-) diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs index f53b3cc5..99416d2f 100644 --- a/pbs-client/src/catalog_shell.rs +++ b/pbs-client/src/catalog_shell.rs @@ -1147,7 +1147,7 @@ impl<'a> ExtractorState<'a> { (_, DirEntryAttribute::Directory { .. }) => { self.handle_new_directory(entry, match_result?).await?; } - (true, DirEntryAttribute::File { .. }) => { + (true, DirEntryAttribute::File { .. } | DirEntryAttribute::AppendixRef { .. }) => { self.dir_stack.push(PathStackEntry::new(entry)); let file = Shell::walk_pxar_archive(self.accessor, &mut self.dir_stack).await?; self.extract_file(file).await?; diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index a2338218..611d7421 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -735,6 +735,18 @@ impl Archiver { Ok(out.file_offset()) } + async fn add_appendix_ref( + &mut self, + encoder: &mut Encoder<'_, T>, + file_name: &Path, + appendix_offset: pxar::encoder::AppendixRefOffset, + file_size: u64, + ) -> Result<(), Error> { + Ok(encoder + .add_appendix_ref(file_name, appendix_offset, file_size) + .await?) + } + async fn add_symlink( &mut self, encoder: &mut Encoder<'_, T>, diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs index f78e06c2..d2d42749 100644 --- a/pbs-client/src/pxar/extract.rs +++ b/pbs-client/src/pxar/extract.rs @@ -74,6 +74,7 @@ struct ExtractorIterState { err_path_stack: Vec, current_match: bool, end_reached: bool, + appendix_list: Vec<(PathBuf, u64, u64)>, } /// An [`Iterator`] that encapsulates the process of extraction in [extract_archive]. @@ -98,6 +99,7 @@ impl ExtractorIterState { err_path_stack: Vec::new(), current_match: options.extract_match_default, end_reached: false, + appendix_list: Vec::new(), } } } @@ -373,6 +375,20 @@ where } .context(PxarExtractContext::ExtractFile) } + ( + true, + EntryKind::AppendixRef { + appendix_offset, + file_size, + }, + ) => { + self.state.appendix_list.push(( + entry.path().to_path_buf(), + *appendix_offset, + *file_size, + )); + Ok(()) + } (false, _) => Ok(()), // skip this }; diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs index 0cfbaf5b..aac5a1e7 100644 --- a/pbs-client/src/pxar/tools.rs +++ b/pbs-client/src/pxar/tools.rs @@ -156,6 +156,14 @@ pub fn format_multi_line_entry(entry: &Entry) -> String { let (size, link, type_name) = match entry.kind() { EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"), + EntryKind::AppendixRef { + appendix_offset, + file_size, + } => ( + format!("{} {}", appendix_offset, file_size), + String::new(), + "appendix ref", + ), EntryKind::Symlink(link) => ( "0".to_string(), format!(" -> {:?}", link.as_os_str()), diff --git a/pbs-datastore/src/catalog.rs b/pbs-datastore/src/catalog.rs index 746b493b..8ae7c661 100644 --- a/pbs-datastore/src/catalog.rs +++ b/pbs-datastore/src/catalog.rs @@ -28,6 +28,14 @@ pub trait BackupCatalogWriter { ctime: i64, file_offset: pxar::encoder::LinkOffset, ) -> Result<(), Error>; + fn add_appendix_ref( + &mut self, + name: &CStr, + size: u64, + mtime: i64, + ctime: i64, + appendix_ref_offset: pxar::encoder::AppendixRefOffset, + ) -> Result<(), Error>; fn add_symlink(&mut self, name: &CStr) -> Result<(), Error>; fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error>; fn add_block_device(&mut self, name: &CStr) -> Result<(), Error>; @@ -41,6 +49,7 @@ pub trait BackupCatalogWriter { pub enum CatalogEntryType { Directory = b'd', File = b'f', + AppendixRef = b'r', Symlink = b'l', Hardlink = b'h', BlockDevice = b'b', @@ -56,6 +65,7 @@ impl TryFrom for CatalogEntryType { Ok(match value { b'd' => CatalogEntryType::Directory, b'f' => CatalogEntryType::File, + b'r' => CatalogEntryType::AppendixRef, b'l' => CatalogEntryType::Symlink, b'h' => CatalogEntryType::Hardlink, b'b' => CatalogEntryType::BlockDevice, @@ -72,6 +82,7 @@ impl From<&DirEntryAttribute> for CatalogEntryType { match value { DirEntryAttribute::Directory { .. } => CatalogEntryType::Directory, DirEntryAttribute::File { .. } => CatalogEntryType::File, + DirEntryAttribute::AppendixRef { .. } => CatalogEntryType::AppendixRef, DirEntryAttribute::Symlink => CatalogEntryType::Symlink, DirEntryAttribute::Hardlink => CatalogEntryType::Hardlink, DirEntryAttribute::BlockDevice => CatalogEntryType::BlockDevice, @@ -98,9 +109,22 @@ impl FileOffset { self.offset } } + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +pub struct AppendixRefOffset { + offset: u64, +} + +impl AppendixRefOffset { + pub fn raw(&self) -> u64 { + self.offset + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] pub enum Offset { FileOffset { offset: u64 }, + AppendixRefOffset { offset: u64 }, } /// Represents a named directory entry @@ -130,6 +154,12 @@ pub enum DirEntryAttribute { mtime: i64, extension: Option, }, + AppendixRef { + size: u64, + mtime: i64, + ctime: i64, + appendix_ref_offset: AppendixRefOffset, + }, Symlink, Hardlink, BlockDevice, @@ -195,6 +225,17 @@ impl DirEntry { name, attr: DirEntryAttribute::Socket, }, + (CatalogEntryType::AppendixRef, Some(Offset::AppendixRefOffset { offset })) => { + DirEntry { + name, + attr: DirEntryAttribute::AppendixRef { + size, + mtime, + ctime, + appendix_ref_offset: AppendixRefOffset { offset }, + }, + } + } _ => panic!("unexpected parameters '{etype}' and '{offset:?}'"), } } @@ -204,6 +245,7 @@ impl DirEntry { Some(match self.attr { DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR, DirEntryAttribute::File { .. } => pxar::mode::IFREG, + DirEntryAttribute::AppendixRef { .. } => pxar::mode::IFREG, DirEntryAttribute::Symlink => pxar::mode::IFLNK, DirEntryAttribute::Hardlink => return None, DirEntryAttribute::BlockDevice => pxar::mode::IFBLK, @@ -271,6 +313,24 @@ impl DirInfo { catalog_encode_u64(writer, file_offset.raw())?; } } + DirEntry { + name, + attr: + DirEntryAttribute::AppendixRef { + size, + mtime, + ctime, + appendix_ref_offset, + }, + } => { + writer.write_all(&[CatalogEntryType::AppendixRef as u8])?; + catalog_encode_u64(writer, name.len() as u64)?; + writer.write_all(name)?; + catalog_encode_u64(writer, *size)?; + catalog_encode_i64(writer, *mtime)?; + catalog_encode_i64(writer, *ctime)?; + catalog_encode_u64(writer, appendix_ref_offset.raw())?; + } DirEntry { name, attr: DirEntryAttribute::Symlink, @@ -390,6 +450,21 @@ impl DirInfo { }; callback(etype, name, 0, size, mtime, ctime, offset)? } + CatalogEntryType::AppendixRef => { + let size = catalog_decode_u64(&mut cursor)?; + let mtime = catalog_decode_i64(&mut cursor)?; + let ctime = catalog_decode_i64(&mut cursor)?; + let offset = catalog_decode_u64(&mut cursor)?; + callback( + etype, + name, + 0, + size, + mtime, + ctime, + Some(Offset::AppendixRefOffset { offset }), + )? + } _ => callback(etype, name, 0, 0, 0, 0, None)?, }; if !cont { @@ -517,6 +592,34 @@ impl BackupCatalogWriter for CatalogWriter { Ok(()) } + fn add_appendix_ref( + &mut self, + name: &CStr, + size: u64, + mtime: i64, + ctime: i64, + appendix_ref_offset: pxar::encoder::AppendixRefOffset, + ) -> Result<(), Error> { + let dir = self + .dirstack + .last_mut() + .ok_or_else(|| format_err!("outside root"))?; + let name = name.to_bytes().to_vec(); + let appendix_ref_offset = AppendixRefOffset { + offset: appendix_ref_offset.raw(), + }; + dir.entries.push(DirEntry { + name, + attr: DirEntryAttribute::AppendixRef { + size, + mtime, + ctime, + appendix_ref_offset, + }, + }); + Ok(()) + } + fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> { let dir = self .dirstack -- 2.39.2