From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D30A5DB04 for ; Fri, 22 Sep 2023 09:16:55 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 57B006BD0 for ; Fri, 22 Sep 2023 09:16:53 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 22 Sep 2023 09:16:50 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 57F414878F for ; Fri, 22 Sep 2023 09:16:50 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Fri, 22 Sep 2023 09:16:06 +0200 Message-Id: <20230922071621.12670-6-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20230922071621.12670-1-c.ebner@proxmox.com> References: <20230922071621.12670-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.115 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [RFC pxar 5/20] fix #3174: enc/dec: impl PXAR_APPENDIX_REF entrytype X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 22 Sep 2023 07:16:55 -0000 Add an additional entry type for regular files to store a reference to the appenidx section of the pxar archive, the position relative to the appendix start is stored, in order to be able to access the file payload within the appendix. This new entry type is used to reference the contents of existing file payload chunks for unchanged file payloads. Signed-off-by: Christian Ebner --- examples/mk-format-hashes.rs | 5 +++++ src/decoder/mod.rs | 10 ++++++++++ src/encoder/aio.rs | 18 ++++++++++++++++++ src/encoder/mod.rs | 36 ++++++++++++++++++++++++++++++++++++ src/encoder/sync.rs | 16 ++++++++++++++++ src/format/mod.rs | 5 +++++ src/lib.rs | 6 ++++++ 7 files changed, 96 insertions(+) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index 1ad606c..8b4f5de 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -41,6 +41,11 @@ const CONSTANTS: &[(&str, &str, &str)] = &[ "PXAR_PAYLOAD", "__PROXMOX_FORMAT_PXAR_PAYLOAD__", ), + ( + "Marks the beginnig of an appendix reference for regular files", + "PXAR_APPENDIX_REF", + "__PROXMOX_FORMAT_PXAR_APPENDIX_REF__", + ), ( "Marks item as entry of goodbye table", "PXAR_GOODBYE", diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 2ca263b..143a01d 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -526,6 +526,16 @@ impl DecoderImpl { self.entry.kind = EntryKind::Device(self.read_device().await?); return Ok(ItemResult::Entry); } + format::PXAR_APPENDIX_REF => { + let bytes = self.read_entry_as_bytes().await?; + let appendix_offset = u64::from_le_bytes(bytes[0..8].try_into().unwrap()); + let file_size = u64::from_le_bytes(bytes[8..16].try_into().unwrap()); + self.entry.kind = EntryKind::AppendixRef { + appendix_offset, + file_size, + }; + return Ok(ItemResult::Entry); + } format::PXAR_PAYLOAD => { let offset = seq_read_position(&mut self.input).await.transpose()?; self.entry.kind = EntryKind::File { diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs index ad25fea..1d8e635 100644 --- a/src/encoder/aio.rs +++ b/src/encoder/aio.rs @@ -112,6 +112,24 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { self.inner.finish().await } + /// Add reference to archive appendix + pub async fn add_appendix_ref>( + &mut self, + metadata: &Metadata, + file_name: PF, + appendix_offset: u64, + file_size: u64, + ) -> io::Result<()> { + self.inner + .add_appendix_ref( + metadata, + file_name.as_ref(), + appendix_offset, + file_size, + ) + .await + } + /// Add a symbolic link to the archive. pub async fn add_symlink, PT: AsRef>( &mut self, diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 710ed98..ddb0125 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -437,6 +437,42 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { Ok(offset) } + /// Add reference to pxar archive appendix + pub async fn add_appendix_ref( + &mut self, + metadata: &Metadata, + file_name: &Path, + appendix_offset: u64, + file_size: u64, + ) -> io::Result<()> { + self.check()?; + + let file_offset = self.position(); + let file_name = file_name.as_os_str().as_bytes(); + self.start_file_do(Some(metadata), file_name).await?; + + let mut data = Vec::with_capacity(2 * 8); + data.extend(&appendix_offset.to_le_bytes()); + data.extend(&file_size.to_le_bytes()); + seq_write_pxar_entry( + self.output.as_mut(), + format::PXAR_APPENDIX_REF, + &data, + &mut self.state.write_position, + ) + .await?; + + let end_offset = self.position(); + + self.state.items.push(GoodbyeItem { + hash: format::hash_filename(file_name), + offset: file_offset, + size: end_offset - file_offset, + }); + + Ok(()) + } + /// Return a file offset usable with `add_hardlink`. pub async fn add_symlink( &mut self, diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs index 1ec91b8..6cac7eb 100644 --- a/src/encoder/sync.rs +++ b/src/encoder/sync.rs @@ -110,6 +110,22 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { poll_result_once(self.inner.finish()) } + /// Add reference to archive appendix + pub async fn add_appendix_ref>( + &mut self, + metadata: &Metadata, + file_name: PF, + appendix_offset: u64, + file_size: u64, + ) -> io::Result<()> { + poll_result_once(self.inner.add_appendix_ref( + metadata, + file_name.as_ref(), + appendix_offset, + file_size, + )) + } + /// Add a symbolic link to the archive. pub fn add_symlink, PT: AsRef>( &mut self, diff --git a/src/format/mod.rs b/src/format/mod.rs index 72a193c..5eb7562 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -22,6 +22,7 @@ //! * `FCAPS` -- file capability in Linux disk format //! * `QUOTA_PROJECT_ID` -- the ext4/xfs quota project ID //! * `PAYLOAD` -- file contents, if it is one +//! * `APPENDIX_REF` -- start offset and size of a file entry relative to the appendix start //! * `SYMLINK` -- symlink target, if it is one //! * `DEVICE` -- device major/minor, if it is a block/char device //! @@ -99,6 +100,8 @@ pub const PXAR_QUOTA_PROJID: u64 = 0xe07540e82f7d1cbb; pub const PXAR_HARDLINK: u64 = 0x51269c8422bd7275; /// Marks the beginnig of the payload (actual content) of regular files pub const PXAR_PAYLOAD: u64 = 0x28147a1b0b7c1a25; +/// Marks the beginnig of an appendix reference for regular files +pub const PXAR_APPENDIX_REF: u64 = 0x849b4a17e0234f8e; /// Marks item as entry of goodbye table pub const PXAR_GOODBYE: u64 = 0x2fec4fa642d5731d; /// The end marker used in the GOODBYE object @@ -151,6 +154,7 @@ impl Header { PXAR_ACL_GROUP_OBJ => size_of::() as u64, PXAR_QUOTA_PROJID => size_of::() as u64, PXAR_ENTRY => size_of::() as u64, + PXAR_APPENDIX_REF => u64::MAX - (size_of::() as u64), PXAR_PAYLOAD | PXAR_GOODBYE => u64::MAX - (size_of::() as u64), _ => u64::MAX - (size_of::() as u64), } @@ -191,6 +195,7 @@ impl Display for Header { PXAR_ACL_GROUP_OBJ => "ACL_GROUP_OBJ", PXAR_QUOTA_PROJID => "QUOTA_PROJID", PXAR_ENTRY => "ENTRY", + PXAR_APPENDIX_REF => "APPENDIX_REF", PXAR_PAYLOAD => "PAYLOAD", PXAR_GOODBYE => "GOODBYE", _ => "UNKNOWN", diff --git a/src/lib.rs b/src/lib.rs index ed7ba40..6a30fc3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -410,6 +410,12 @@ pub enum EntryKind { offset: Option, }, + /// Reference to pxar archive appendix + AppendixRef { + appendix_offset: u64, + file_size: u64, + }, + /// Directory entry. When iterating through an archive, the contents follow next. Directory, -- 2.39.2