From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id AEB5D1FF0AA for ; Tue, 17 Nov 2026 16:28:34 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 551BF2164C; Tue, 17 Nov 2026 16:28:34 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files Date: Tue, 17 Nov 2026 16:28:05 +0100 Message-ID: <20261117152814.491323-2-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261117152814.491323-1-c.ebner@proxmox.com> References: <20261117152814.491323-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794929309985 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.371 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: DSVGLMRNMPDINXCQCSJ427UQLCR2VVUI X-Message-ID-Hash: DSVGLMRNMPDINXCQCSJ427UQLCR2VVUI X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: For regular files encoded with a separate payload output, only the payload reference is written to the metadata archive. The goodbye table size of these entries still included the payload size as for unified archives, so the entry range derived from it extends past the entry and possibly past the end of the metadata archive. Files encoded as references to reused payloads already recorded the actual metadata size. Users of the entry range therefore operated on bogus ranges and could fail with out of range offsets. Only record the size of the entry in the metadata archive. Archives with the inflated sizes stay readable, as decoding a file entry stops after its payload reference. Also, add regression tests to cover this. Signed-off-by: Christian Ebner --- Cargo.toml | 6 ++ src/encoder/mod.rs | 9 ++- tests/split.rs | 152 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 tests/split.rs diff --git a/Cargo.toml b/Cargo.toml index eb6dfb1..cc52b61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,12 @@ path = "tests/compat.rs" test = true required-features = [ "test-harness" ] +[[test]] +name = "split" +path = "tests/split.rs" +test = true +required-features = [ "test-harness" ] + [dependencies] endian_trait = { version = "0.6", features = ["arrays"] } siphasher = "1" diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index c0a885e..d07a833 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -494,6 +494,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { self.start_file_do(Some(metadata), file_name).await?; let (mut output, state) = self.output_state()?; + let split = output.payload().is_some(); if let Some(payload_output) = output.payload_mut() { // payload references must point to the position prior to the payload header, // separating payload entries in the payload stream @@ -520,13 +521,19 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { let payload_data_offset = state.position(); let meta_size = payload_data_offset - file_offset; + // With a separate payload output only the payload reference is part of this archive. + let size = if split { + meta_size + } else { + meta_size + file_size + }; Ok(FileImpl { output, goodbye_item: GoodbyeItem { hash: format::hash_filename(file_name), offset: file_offset, - size: file_size + meta_size, + size, }, remaining_size: file_size, parent: state, diff --git a/tests/split.rs b/tests/split.rs new file mode 100644 index 0000000..2dc4aa8 --- /dev/null +++ b/tests/split.rs @@ -0,0 +1,152 @@ +use std::future::Future; +use std::io; +use std::path::Path; +use std::pin::pin; +use std::task::{Context, Poll, Waker}; + +use pxar::accessor::sync::{Accessor, FileEntry}; +use pxar::encoder::PayloadOffset; +use pxar::encoder::sync::{Encoder, StandardWriter}; +use pxar::format::{GoodbyeItem, Header}; +use pxar::{EntryKind, Metadata, PxarVariant}; + +const NEW_FILE: &str = "a_new.txt"; +const NEW_FILE_SIZE: u64 = 4096; +const HARDLINK: &str = "b_link"; +const REUSED_FILE: &str = "c_reused.txt"; +const REUSED_FILE_SIZE: u64 = 1024; + +fn block_on(fut: F) -> F::Output { + let mut fut = pin!(fut); + match fut.as_mut().poll(&mut Context::from_waker(Waker::noop())) { + Poll::Ready(output) => output, + Poll::Pending => panic!("future unexpectedly returned pending"), + } +} + +/// Encode a root directory with a newly written file, a hardlink to it and a file referencing an +/// already existing payload. Returns the metadata and payload archive for split archives, or just +/// the archive otherwise. +fn encode_archive(split: bool) -> io::Result<(Vec, Vec)> { + let mut archive = Vec::new(); + let mut payload = Vec::new(); + + let output = if split { + PxarVariant::Split( + StandardWriter::new(&mut archive), + StandardWriter::new(&mut payload), + ) + } else { + PxarVariant::Unified(StandardWriter::new(&mut archive)) + }; + + let file_meta = Metadata::file_builder(0o644).build(); + let mut encoder = Encoder::new(output, &Metadata::dir_builder(0o755).build(), None)?; + + let content = vec![0xAA; NEW_FILE_SIZE as usize]; + let link_offset = encoder.add_file(&file_meta, NEW_FILE, NEW_FILE_SIZE, &mut &content[..])?; + encoder.add_hardlink(HARDLINK, NEW_FILE, link_offset)?; + + if split { + // Reference payload data injected by the caller, like when reusing unchanged files. + let payload_offset = encoder.payload_position()?; + block_on(encoder.add_payload_ref( + &file_meta, + Path::new(REUSED_FILE), + REUSED_FILE_SIZE, + payload_offset, + ))?; + let injected = size_of::
() as u64 + REUSED_FILE_SIZE; + encoder.advance(PayloadOffset::default().add(injected))?; + } + + encoder.finish()?; + encoder.close()?; + + Ok((archive, payload)) +} + +fn open_accessor<'a>(archive: &'a [u8], payload: &'a [u8], split: bool) -> Accessor<&'a [u8]> { + let input = if split { + PxarVariant::Split(archive, (payload, payload.len() as u64)) + } else { + PxarVariant::Unified(archive) + }; + Accessor::new(input, archive.len() as u64).expect("failed to create accessor") +} + +/// Start of the root directory's goodbye table, which directly follows its last entry. +fn root_goodbye_start(archive: &[u8], entry_count: u64) -> u64 { + let table_size = size_of::
() + (entry_count as usize + 1) * size_of::(); + (archive.len() - table_size) as u64 +} + +fn lookup_file<'a>(accessor: &Accessor<&'a [u8]>, name: &str) -> FileEntry<&'a [u8]> { + accessor + .open_root() + .expect("failed to open root") + .lookup(name) + .expect("lookup failed") + .unwrap_or_else(|| panic!("{name} not found")) +} + +/// Check that each entry's range ends exactly where the next one starts. +fn check_entry_ranges(accessor: &Accessor<&[u8]>, archive: &[u8], split: bool) { + let new_file = lookup_file(accessor, NEW_FILE); + let hardlink = lookup_file(accessor, HARDLINK); + assert!(matches!(hardlink.kind(), EntryKind::Hardlink(_))); + + assert_eq!( + new_file.entry_range_info().entry_range.end, + hardlink.entry_range_info().filename_header_offset.unwrap(), + "range of {NEW_FILE} does not end at the following entry", + ); + + if split { + let reused_file = lookup_file(accessor, REUSED_FILE); + assert_eq!( + hardlink.entry_range_info().entry_range.end, + reused_file + .entry_range_info() + .filename_header_offset + .unwrap(), + ); + assert_eq!( + reused_file.entry_range_info().entry_range.end, + root_goodbye_start(archive, 3), + "range of {REUSED_FILE} does not end at the goodbye table", + ); + } else { + assert_eq!( + hardlink.entry_range_info().entry_range.end, + root_goodbye_start(archive, 2), + ); + } +} + +#[test] +fn unified_entry_ranges() { + let (archive, _) = encode_archive(false).expect("failed to encode archive"); + let accessor = open_accessor(&archive, &[], false); + check_entry_ranges(&accessor, &archive, false); +} + +#[test] +fn split_entry_ranges() { + let (archive, payload) = encode_archive(true).expect("failed to encode archive"); + let accessor = open_accessor(&archive, &payload, true); + check_entry_ranges(&accessor, &archive, true); + + // The goodbye table itself must record the exact ranges, not only the decoded entries. + let root = accessor.open_root().expect("failed to open root"); + for dir_entry in root.read_dir() { + let dir_entry = dir_entry.expect("failed to read directory entry"); + let decoded = dir_entry.decode_entry().expect("failed to decode entry"); + assert_eq!( + dir_entry.entry_range_info().entry_range, + decoded.entry_range_info().entry_range, + "goodbye table range of {:?} differs from the decoded entry", + dir_entry.file_name(), + ); + } +} -- 2.47.3