From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 40F1EDAB8 for ; Fri, 22 Sep 2023 09:16:56 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A10796BD2 for ; Fri, 22 Sep 2023 09:16:53 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 22 Sep 2023 09:16:50 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 5924848773 for ; Fri, 22 Sep 2023 09:16:50 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Fri, 22 Sep 2023 09:16:07 +0200 Message-Id: <20230922071621.12670-7-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20230922071621.12670-1-c.ebner@proxmox.com> References: <20230922071621.12670-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.114 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [RFC pxar 6/20] fix #3174: enc/dec: impl PXAR_APPENDIX entrytype X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Sep 2023 07:16:56 -0000 Add an additional entry type for marking the start of a pxar archive appendix section. The appendix is a concatenation of possibly uncorrelated chunks, therefore not following the pxar archive format anymore. The appendix is only used to access the file metadata and payloads when a PXAR_APPENDIX_REF entry is encountered in the archive before this point. Signed-off-by: Christian Ebner --- examples/mk-format-hashes.rs | 1 + src/decoder/mod.rs | 9 +++++++++ src/encoder/aio.rs | 7 +++++++ src/encoder/mod.rs | 20 ++++++++++++++++++++ src/encoder/sync.rs | 7 +++++++ src/format/mod.rs | 7 +++++++ src/lib.rs | 4 ++++ 7 files changed, 55 insertions(+) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index 8b4f5de..f068edd 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -12,6 +12,7 @@ const CONSTANTS: &[(&str, &str, &str)] = &[ "__PROXMOX_FORMAT_ENTRY__", ), ("", "PXAR_FILENAME", "__PROXMOX_FORMAT_FILENAME__"), + ("", "PXAR_APPENDIX", "__PROXMOX_FORMAT_APPENDIX__"), ("", "PXAR_SYMLINK", "__PROXMOX_FORMAT_SYMLINK__"), ("", "PXAR_DEVICE", "__PROXMOX_FORMAT_DEVICE__"), ("", "PXAR_XATTR", "__PROXMOX_FORMAT_XATTR__"), diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 143a01d..e54ab41 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -291,6 +291,7 @@ impl DecoderImpl { continue; } } + format::PXAR_APPENDIX => return Ok(Some(self.entry.take())), _ => io_bail!( "expected filename or directory-goodbye pxar entry, got: {}", self.current_header, @@ -536,6 +537,14 @@ impl DecoderImpl { }; return Ok(ItemResult::Entry); } + format::PXAR_APPENDIX => { + let bytes = self.read_entry_as_bytes().await?; + let total = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + self.entry.kind = EntryKind::Appendix { + total, + }; + return Ok(ItemResult::Entry); + } format::PXAR_PAYLOAD => { let offset = seq_read_position(&mut self.input).await.transpose()?; self.entry.kind = EntryKind::File { diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs index 1d8e635..3b6c900 100644 --- a/src/encoder/aio.rs +++ b/src/encoder/aio.rs @@ -130,6 +130,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { .await } + /// Add the appendix start entry marker + /// + /// Returns the LinkOffset pointing after the entry, the appendix start offset + pub async fn add_appendix(&mut self, full_size: u64) -> io::Result { + self.inner.add_appendix(full_size).await + } + /// Add a symbolic link to the archive. pub async fn add_symlink, PT: AsRef>( &mut self, diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index ddb0125..a513ce6 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -473,6 +473,26 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { Ok(()) } + /// Add the appendix start entry marker + /// + /// Returns the LinkOffset pointing after the entry, the appendix start offset + pub async fn add_appendix(&mut self, full_size: u64) -> io::Result { + self.check()?; + + let data = &full_size.to_le_bytes().to_vec(); + seq_write_pxar_entry( + self.output.as_mut(), + format::PXAR_APPENDIX, + &data, + &mut self.state.write_position, + ) + .await?; + + let offset = self.position(); + + Ok(LinkOffset(offset)) + } + /// Return a file offset usable with `add_hardlink`. pub async fn add_symlink( &mut self, diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs index 6cac7eb..372ca12 100644 --- a/src/encoder/sync.rs +++ b/src/encoder/sync.rs @@ -126,6 +126,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { )) } + /// Add the appendix start entry marker + /// + /// Returns the LinkOffset pointing after the entry, the appendix start offset + pub async fn add_appendix(&mut self, full_size: u64) -> io::Result { + poll_result_once(self.inner.add_appendix(full_size)) + } + /// Add a symbolic link to the archive. pub fn add_symlink, PT: AsRef>( &mut self, diff --git a/src/format/mod.rs b/src/format/mod.rs index 5eb7562..8254df9 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -35,6 +35,12 @@ //! * `` -- serialization of the second directory entry //! * ... //! * `GOODBYE` -- lookup table at the end of a list of directory entries +//! +//! For backups referencing previous backups to skip file payloads, the archive is followed by a +//! appendix maker after which the concatinated pxar archive fragments containing the file payloads +//! are appended. They are NOT guaranteed to follow the full pxar structure and should only be +//! used to extract the file payloads by given offset. +//! * `APPENDIX` -- pxar archive fragments containing file payloads use std::cmp::Ordering; use std::ffi::{CStr, OsStr}; @@ -85,6 +91,7 @@ pub const PXAR_ENTRY: u64 = 0xd5956474e588acef; /// Previous version of the entry struct pub const PXAR_ENTRY_V1: u64 = 0x11da850a1c1cceff; pub const PXAR_FILENAME: u64 = 0x16701121063917b3; +pub const PXAR_APPENDIX: u64 = 0x9ff6c9507864b38d; pub const PXAR_SYMLINK: u64 = 0x27f971e7dbf5dc5f; pub const PXAR_DEVICE: u64 = 0x9fc9e906586d5ce9; pub const PXAR_XATTR: u64 = 0x0dab0229b57dcd03; diff --git a/src/lib.rs b/src/lib.rs index 6a30fc3..086c7f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -416,6 +416,10 @@ pub enum EntryKind { file_size: u64, }, + Appendix { + total: u64, + }, + /// Directory entry. When iterating through an archive, the contents follow next. Directory, -- 2.39.2