public inbox for pbs-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files
Date: Tue, 17 Nov 2026 16:28:05 +0100	[thread overview]
Message-ID: <20261117152814.491323-2-c.ebner@proxmox.com> (raw)
In-Reply-To: <20261117152814.491323-1-c.ebner@proxmox.com>

For regular files encoded with a separate payload output, only the
payload reference is written to the metadata archive. The goodbye
table size of these entries still included the payload size as for
unified archives, so the entry range derived from it extends past
the entry and possibly past the end of the metadata archive. Files
encoded as references to reused payloads already recorded the actual
metadata size.

Users of the entry range therefore operated on bogus ranges and could
fail with out of range offsets.

Only record the size of the entry in the metadata archive. Archives
with the inflated sizes stay readable, as decoding a file entry stops
after its payload reference.

Also, add regression tests to cover this.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 Cargo.toml         |   6 ++
 src/encoder/mod.rs |   9 ++-
 tests/split.rs     | 152 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 166 insertions(+), 1 deletion(-)
 create mode 100644 tests/split.rs

diff --git a/Cargo.toml b/Cargo.toml
index eb6dfb1..cc52b61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,6 +39,12 @@ path = "tests/compat.rs"
 test = true
 required-features = [ "test-harness" ]
 
+[[test]]
+name = "split"
+path = "tests/split.rs"
+test = true
+required-features = [ "test-harness" ]
+
 [dependencies]
 endian_trait = { version = "0.6", features = ["arrays"] }
 siphasher = "1"
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index c0a885e..d07a833 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -494,6 +494,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         self.start_file_do(Some(metadata), file_name).await?;
 
         let (mut output, state) = self.output_state()?;
+        let split = output.payload().is_some();
         if let Some(payload_output) = output.payload_mut() {
             // payload references must point to the position prior to the payload header,
             // separating payload entries in the payload stream
@@ -520,13 +521,19 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         let payload_data_offset = state.position();
 
         let meta_size = payload_data_offset - file_offset;
+        // With a separate payload output only the payload reference is part of this archive.
+        let size = if split {
+            meta_size
+        } else {
+            meta_size + file_size
+        };
 
         Ok(FileImpl {
             output,
             goodbye_item: GoodbyeItem {
                 hash: format::hash_filename(file_name),
                 offset: file_offset,
-                size: file_size + meta_size,
+                size,
             },
             remaining_size: file_size,
             parent: state,
diff --git a/tests/split.rs b/tests/split.rs
new file mode 100644
index 0000000..2dc4aa8
--- /dev/null
+++ b/tests/split.rs
@@ -0,0 +1,152 @@
+use std::future::Future;
+use std::io;
+use std::path::Path;
+use std::pin::pin;
+use std::task::{Context, Poll, Waker};
+
+use pxar::accessor::sync::{Accessor, FileEntry};
+use pxar::encoder::PayloadOffset;
+use pxar::encoder::sync::{Encoder, StandardWriter};
+use pxar::format::{GoodbyeItem, Header};
+use pxar::{EntryKind, Metadata, PxarVariant};
+
+const NEW_FILE: &str = "a_new.txt";
+const NEW_FILE_SIZE: u64 = 4096;
+const HARDLINK: &str = "b_link";
+const REUSED_FILE: &str = "c_reused.txt";
+const REUSED_FILE_SIZE: u64 = 1024;
+
+fn block_on<F: Future>(fut: F) -> F::Output {
+    let mut fut = pin!(fut);
+    match fut.as_mut().poll(&mut Context::from_waker(Waker::noop())) {
+        Poll::Ready(output) => output,
+        Poll::Pending => panic!("future unexpectedly returned pending"),
+    }
+}
+
+/// Encode a root directory with a newly written file, a hardlink to it and a file referencing an
+/// already existing payload. Returns the metadata and payload archive for split archives, or just
+/// the archive otherwise.
+fn encode_archive(split: bool) -> io::Result<(Vec<u8>, Vec<u8>)> {
+    let mut archive = Vec::new();
+    let mut payload = Vec::new();
+
+    let output = if split {
+        PxarVariant::Split(
+            StandardWriter::new(&mut archive),
+            StandardWriter::new(&mut payload),
+        )
+    } else {
+        PxarVariant::Unified(StandardWriter::new(&mut archive))
+    };
+
+    let file_meta = Metadata::file_builder(0o644).build();
+    let mut encoder = Encoder::new(output, &Metadata::dir_builder(0o755).build(), None)?;
+
+    let content = vec![0xAA; NEW_FILE_SIZE as usize];
+    let link_offset = encoder.add_file(&file_meta, NEW_FILE, NEW_FILE_SIZE, &mut &content[..])?;
+    encoder.add_hardlink(HARDLINK, NEW_FILE, link_offset)?;
+
+    if split {
+        // Reference payload data injected by the caller, like when reusing unchanged files.
+        let payload_offset = encoder.payload_position()?;
+        block_on(encoder.add_payload_ref(
+            &file_meta,
+            Path::new(REUSED_FILE),
+            REUSED_FILE_SIZE,
+            payload_offset,
+        ))?;
+        let injected = size_of::<Header>() as u64 + REUSED_FILE_SIZE;
+        encoder.advance(PayloadOffset::default().add(injected))?;
+    }
+
+    encoder.finish()?;
+    encoder.close()?;
+
+    Ok((archive, payload))
+}
+
+fn open_accessor<'a>(archive: &'a [u8], payload: &'a [u8], split: bool) -> Accessor<&'a [u8]> {
+    let input = if split {
+        PxarVariant::Split(archive, (payload, payload.len() as u64))
+    } else {
+        PxarVariant::Unified(archive)
+    };
+    Accessor::new(input, archive.len() as u64).expect("failed to create accessor")
+}
+
+/// Start of the root directory's goodbye table, which directly follows its last entry.
+fn root_goodbye_start(archive: &[u8], entry_count: u64) -> u64 {
+    let table_size = size_of::<Header>() + (entry_count as usize + 1) * size_of::<GoodbyeItem>();
+    (archive.len() - table_size) as u64
+}
+
+fn lookup_file<'a>(accessor: &Accessor<&'a [u8]>, name: &str) -> FileEntry<&'a [u8]> {
+    accessor
+        .open_root()
+        .expect("failed to open root")
+        .lookup(name)
+        .expect("lookup failed")
+        .unwrap_or_else(|| panic!("{name} not found"))
+}
+
+/// Check that each entry's range ends exactly where the next one starts.
+fn check_entry_ranges(accessor: &Accessor<&[u8]>, archive: &[u8], split: bool) {
+    let new_file = lookup_file(accessor, NEW_FILE);
+    let hardlink = lookup_file(accessor, HARDLINK);
+    assert!(matches!(hardlink.kind(), EntryKind::Hardlink(_)));
+
+    assert_eq!(
+        new_file.entry_range_info().entry_range.end,
+        hardlink.entry_range_info().filename_header_offset.unwrap(),
+        "range of {NEW_FILE} does not end at the following entry",
+    );
+
+    if split {
+        let reused_file = lookup_file(accessor, REUSED_FILE);
+        assert_eq!(
+            hardlink.entry_range_info().entry_range.end,
+            reused_file
+                .entry_range_info()
+                .filename_header_offset
+                .unwrap(),
+        );
+        assert_eq!(
+            reused_file.entry_range_info().entry_range.end,
+            root_goodbye_start(archive, 3),
+            "range of {REUSED_FILE} does not end at the goodbye table",
+        );
+    } else {
+        assert_eq!(
+            hardlink.entry_range_info().entry_range.end,
+            root_goodbye_start(archive, 2),
+        );
+    }
+}
+
+#[test]
+fn unified_entry_ranges() {
+    let (archive, _) = encode_archive(false).expect("failed to encode archive");
+    let accessor = open_accessor(&archive, &[], false);
+    check_entry_ranges(&accessor, &archive, false);
+}
+
+#[test]
+fn split_entry_ranges() {
+    let (archive, payload) = encode_archive(true).expect("failed to encode archive");
+    let accessor = open_accessor(&archive, &payload, true);
+    check_entry_ranges(&accessor, &archive, true);
+
+    // The goodbye table itself must record the exact ranges, not only the decoded entries.
+    let root = accessor.open_root().expect("failed to open root");
+    for dir_entry in root.read_dir() {
+        let dir_entry = dir_entry.expect("failed to read directory entry");
+        let decoded = dir_entry.decode_entry().expect("failed to decode entry");
+        assert_eq!(
+            dir_entry.entry_range_info().entry_range,
+            decoded.entry_range_info().entry_range,
+            "goodbye table range of {:?} differs from the decoded entry",
+            dir_entry.file_name(),
+        );
+    }
+}
-- 
2.47.3





  reply	other threads:[~2026-11-17 15:28 UTC|newest]

Thread overview: 11+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
2026-11-17 15:28 ` Christian Ebner [this message]
2026-11-17 15:28 ` [PATCH pxar 02/10] accessor: limit split archive file entry ranges to payload reference Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 03/10] accessor: add convenience helper to get content's byte range Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 04/10] datastore: index: return dedicated type for chunk offset trait method Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 05/10] bin: debug: fix off-by-one error in last chunk lookup Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 06/10] bin: debug: refactore dynamic index reader opening logic into helper Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 07/10] bin: debug: refactor and deduplicate chunk range difference check Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 09/10] bin: debug: use pxar related archive name helpers for diff Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 10/10] docs: debug: mention split pxar archives for diff subcommand Christian Ebner

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117152814.491323-2-c.ebner@proxmox.com \
    --to=c.ebner@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal