* [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 02/10] accessor: limit split archive file entry ranges to payload reference Christian Ebner
` (8 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
For regular files encoded with a separate payload output, only the
payload reference is written to the metadata archive. The goodbye
table size of these entries still included the payload size as for
unified archives, so the entry range derived from it extends past
the entry and possibly past the end of the metadata archive. Files
encoded as references to reused payloads already recorded the actual
metadata size.
Users of the entry range therefore operated on bogus ranges and could
fail with out of range offsets.
Only record the size of the entry in the metadata archive. Archives
with the inflated sizes stay readable, as decoding a file entry stops
after its payload reference.
Also, add regression tests to cover this.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
Cargo.toml | 6 ++
src/encoder/mod.rs | 9 ++-
tests/split.rs | 152 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 166 insertions(+), 1 deletion(-)
create mode 100644 tests/split.rs
diff --git a/Cargo.toml b/Cargo.toml
index eb6dfb1..cc52b61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,6 +39,12 @@ path = "tests/compat.rs"
test = true
required-features = [ "test-harness" ]
+[[test]]
+name = "split"
+path = "tests/split.rs"
+test = true
+required-features = [ "test-harness" ]
+
[dependencies]
endian_trait = { version = "0.6", features = ["arrays"] }
siphasher = "1"
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index c0a885e..d07a833 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -494,6 +494,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
self.start_file_do(Some(metadata), file_name).await?;
let (mut output, state) = self.output_state()?;
+ let split = output.payload().is_some();
if let Some(payload_output) = output.payload_mut() {
// payload references must point to the position prior to the payload header,
// separating payload entries in the payload stream
@@ -520,13 +521,19 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
let payload_data_offset = state.position();
let meta_size = payload_data_offset - file_offset;
+ // With a separate payload output only the payload reference is part of this archive.
+ let size = if split {
+ meta_size
+ } else {
+ meta_size + file_size
+ };
Ok(FileImpl {
output,
goodbye_item: GoodbyeItem {
hash: format::hash_filename(file_name),
offset: file_offset,
- size: file_size + meta_size,
+ size,
},
remaining_size: file_size,
parent: state,
diff --git a/tests/split.rs b/tests/split.rs
new file mode 100644
index 0000000..2dc4aa8
--- /dev/null
+++ b/tests/split.rs
@@ -0,0 +1,152 @@
+use std::future::Future;
+use std::io;
+use std::path::Path;
+use std::pin::pin;
+use std::task::{Context, Poll, Waker};
+
+use pxar::accessor::sync::{Accessor, FileEntry};
+use pxar::encoder::PayloadOffset;
+use pxar::encoder::sync::{Encoder, StandardWriter};
+use pxar::format::{GoodbyeItem, Header};
+use pxar::{EntryKind, Metadata, PxarVariant};
+
+const NEW_FILE: &str = "a_new.txt";
+const NEW_FILE_SIZE: u64 = 4096;
+const HARDLINK: &str = "b_link";
+const REUSED_FILE: &str = "c_reused.txt";
+const REUSED_FILE_SIZE: u64 = 1024;
+
+fn block_on<F: Future>(fut: F) -> F::Output {
+ let mut fut = pin!(fut);
+ match fut.as_mut().poll(&mut Context::from_waker(Waker::noop())) {
+ Poll::Ready(output) => output,
+ Poll::Pending => panic!("future unexpectedly returned pending"),
+ }
+}
+
+/// Encode a root directory with a newly written file, a hardlink to it and a file referencing an
+/// already existing payload. Returns the metadata and payload archive for split archives, or just
+/// the archive otherwise.
+fn encode_archive(split: bool) -> io::Result<(Vec<u8>, Vec<u8>)> {
+ let mut archive = Vec::new();
+ let mut payload = Vec::new();
+
+ let output = if split {
+ PxarVariant::Split(
+ StandardWriter::new(&mut archive),
+ StandardWriter::new(&mut payload),
+ )
+ } else {
+ PxarVariant::Unified(StandardWriter::new(&mut archive))
+ };
+
+ let file_meta = Metadata::file_builder(0o644).build();
+ let mut encoder = Encoder::new(output, &Metadata::dir_builder(0o755).build(), None)?;
+
+ let content = vec![0xAA; NEW_FILE_SIZE as usize];
+ let link_offset = encoder.add_file(&file_meta, NEW_FILE, NEW_FILE_SIZE, &mut &content[..])?;
+ encoder.add_hardlink(HARDLINK, NEW_FILE, link_offset)?;
+
+ if split {
+ // Reference payload data injected by the caller, like when reusing unchanged files.
+ let payload_offset = encoder.payload_position()?;
+ block_on(encoder.add_payload_ref(
+ &file_meta,
+ Path::new(REUSED_FILE),
+ REUSED_FILE_SIZE,
+ payload_offset,
+ ))?;
+ let injected = size_of::<Header>() as u64 + REUSED_FILE_SIZE;
+ encoder.advance(PayloadOffset::default().add(injected))?;
+ }
+
+ encoder.finish()?;
+ encoder.close()?;
+
+ Ok((archive, payload))
+}
+
+fn open_accessor<'a>(archive: &'a [u8], payload: &'a [u8], split: bool) -> Accessor<&'a [u8]> {
+ let input = if split {
+ PxarVariant::Split(archive, (payload, payload.len() as u64))
+ } else {
+ PxarVariant::Unified(archive)
+ };
+ Accessor::new(input, archive.len() as u64).expect("failed to create accessor")
+}
+
+/// Start of the root directory's goodbye table, which directly follows its last entry.
+fn root_goodbye_start(archive: &[u8], entry_count: u64) -> u64 {
+ let table_size = size_of::<Header>() + (entry_count as usize + 1) * size_of::<GoodbyeItem>();
+ (archive.len() - table_size) as u64
+}
+
+fn lookup_file<'a>(accessor: &Accessor<&'a [u8]>, name: &str) -> FileEntry<&'a [u8]> {
+ accessor
+ .open_root()
+ .expect("failed to open root")
+ .lookup(name)
+ .expect("lookup failed")
+ .unwrap_or_else(|| panic!("{name} not found"))
+}
+
+/// Check that each entry's range ends exactly where the next one starts.
+fn check_entry_ranges(accessor: &Accessor<&[u8]>, archive: &[u8], split: bool) {
+ let new_file = lookup_file(accessor, NEW_FILE);
+ let hardlink = lookup_file(accessor, HARDLINK);
+ assert!(matches!(hardlink.kind(), EntryKind::Hardlink(_)));
+
+ assert_eq!(
+ new_file.entry_range_info().entry_range.end,
+ hardlink.entry_range_info().filename_header_offset.unwrap(),
+ "range of {NEW_FILE} does not end at the following entry",
+ );
+
+ if split {
+ let reused_file = lookup_file(accessor, REUSED_FILE);
+ assert_eq!(
+ hardlink.entry_range_info().entry_range.end,
+ reused_file
+ .entry_range_info()
+ .filename_header_offset
+ .unwrap(),
+ );
+ assert_eq!(
+ reused_file.entry_range_info().entry_range.end,
+ root_goodbye_start(archive, 3),
+ "range of {REUSED_FILE} does not end at the goodbye table",
+ );
+ } else {
+ assert_eq!(
+ hardlink.entry_range_info().entry_range.end,
+ root_goodbye_start(archive, 2),
+ );
+ }
+}
+
+#[test]
+fn unified_entry_ranges() {
+ let (archive, _) = encode_archive(false).expect("failed to encode archive");
+ let accessor = open_accessor(&archive, &[], false);
+ check_entry_ranges(&accessor, &archive, false);
+}
+
+#[test]
+fn split_entry_ranges() {
+ let (archive, payload) = encode_archive(true).expect("failed to encode archive");
+ let accessor = open_accessor(&archive, &payload, true);
+ check_entry_ranges(&accessor, &archive, true);
+
+ // The goodbye table itself must record the exact ranges, not only the decoded entries.
+ let root = accessor.open_root().expect("failed to open root");
+ for dir_entry in root.read_dir() {
+ let dir_entry = dir_entry.expect("failed to read directory entry");
+ let decoded = dir_entry.decode_entry().expect("failed to decode entry");
+ assert_eq!(
+ dir_entry.entry_range_info().entry_range,
+ decoded.entry_range_info().entry_range,
+ "goodbye table range of {:?} differs from the decoded entry",
+ dir_entry.file_name(),
+ );
+ }
+}
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH pxar 02/10] accessor: limit split archive file entry ranges to payload reference
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 03/10] accessor: add convenience helper to get content's byte range Christian Ebner
` (7 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Archives encoded by older versions record the payload size in the
goodbye table size of regular files also for split archives, so the
range of such file entries extends past the entry in the metadata
archive.
Once the payload reference is decoded, the end of the entry in the
metadata archive is known. Limit the range to it for decoded file
entries, hardlink targets and files opened at a given range. This
is backwards compatible with archives still including the file
payload size in the goodbye entry.
Undecoded directory entries still expose the range as encoded in
the goodbye table, as the entry kind is only known after decoding.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/accessor/mod.rs | 32 +++++++++++++++------
tests/split.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 93 insertions(+), 9 deletions(-)
diff --git a/src/accessor/mod.rs b/src/accessor/mod.rs
index 5e2169c..02c7a19 100644
--- a/src/accessor/mod.rs
+++ b/src/accessor/mod.rs
@@ -52,6 +52,22 @@ impl EntryRangeInfo {
entry_range,
}
}
+
+ /// Limit the range of a decoded regular file in a split archive to its payload reference.
+ ///
+ /// Older encoders included the payload size in the goodbye table size of such files, letting
+ /// the range extend past the entry, possibly even past the end of the metadata archive.
+ fn trim_to_payload_ref(mut self, entry: &Entry) -> Self {
+ if let EntryKind::File {
+ offset: Some(offset),
+ payload_offset: Some(_),
+ ..
+ } = entry.kind()
+ {
+ self.entry_range.end = offset + size_of::<PayloadRef>() as u64;
+ }
+ self
+ }
}
/// Stores a content range to be accessed via the `Accessor` as well as the payload reference to
@@ -339,8 +355,8 @@ impl<T: Clone + ReadAt> AccessorImpl<T> {
Ok(FileEntryImpl {
input: self.input.clone(),
+ entry_range_info: entry_range_info.clone().trim_to_payload_ref(&entry),
entry,
- entry_range_info: entry_range_info.clone(),
caches: Arc::clone(&self.caches),
})
}
@@ -391,15 +407,15 @@ impl<T: Clone + ReadAt> AccessorImpl<T> {
size,
..
} => {
- let meta_size = offset - link_offset;
- let entry_end = link_offset + meta_size + size;
+ let entry_range_info = EntryRangeInfo {
+ filename_header_offset: Some(link_offset),
+ entry_range: entry_offset..offset + size,
+ }
+ .trim_to_payload_ref(&entry);
Ok(FileEntryImpl {
input: self.input.clone(),
entry,
- entry_range_info: EntryRangeInfo {
- filename_header_offset: Some(link_offset),
- entry_range: entry_offset..entry_end,
- },
+ entry_range_info,
caches: Arc::clone(&self.caches),
})
}
@@ -892,8 +908,8 @@ impl<T: Clone + ReadAt> DirEntryImpl<'_, T> {
Ok(FileEntryImpl {
input: self.dir.input.clone(),
+ entry_range_info: self.entry_range_info.clone().trim_to_payload_ref(&entry),
entry,
- entry_range_info: self.entry_range_info.clone(),
caches: Arc::clone(&self.caches),
})
}
diff --git a/tests/split.rs b/tests/split.rs
index 2dc4aa8..2419958 100644
--- a/tests/split.rs
+++ b/tests/split.rs
@@ -1,5 +1,6 @@
use std::future::Future;
use std::io;
+use std::ops::Range;
use std::path::Path;
use std::pin::pin;
use std::task::{Context, Poll, Waker};
@@ -7,7 +8,7 @@ use std::task::{Context, Poll, Waker};
use pxar::accessor::sync::{Accessor, FileEntry};
use pxar::encoder::PayloadOffset;
use pxar::encoder::sync::{Encoder, StandardWriter};
-use pxar::format::{GoodbyeItem, Header};
+use pxar::format::{self, GoodbyeItem, Header};
use pxar::{EntryKind, Metadata, PxarVariant};
const NEW_FILE: &str = "a_new.txt";
@@ -66,6 +67,27 @@ fn encode_archive(split: bool) -> io::Result<(Vec<u8>, Vec<u8>)> {
Ok((archive, payload))
}
+/// Inflate the goodbye table size of the given file as done by encoders including the payload
+/// size also for split archives.
+fn inflate_goodbye_size(archive: &mut [u8], file_name: &str, payload_size: u64) {
+ let hash = format::hash_filename(file_name.as_bytes()).to_le_bytes();
+ let positions: Vec<usize> = archive
+ .windows(hash.len())
+ .enumerate()
+ .filter_map(|(pos, window)| (window == hash).then_some(pos))
+ .collect();
+ assert_eq!(
+ positions.len(),
+ 1,
+ "goodbye item for {file_name} not unique"
+ );
+
+ let size_pos = positions[0] + 2 * size_of::<u64>();
+ let size_bytes = &mut archive[size_pos..size_pos + size_of::<u64>()];
+ let size = u64::from_le_bytes(size_bytes.try_into().unwrap()) + payload_size;
+ size_bytes.copy_from_slice(&size.to_le_bytes());
+}
+
fn open_accessor<'a>(archive: &'a [u8], payload: &'a [u8], split: bool) -> Accessor<&'a [u8]> {
let input = if split {
PxarVariant::Split(archive, (payload, payload.len() as u64))
@@ -102,6 +124,15 @@ fn check_entry_ranges(accessor: &Accessor<&[u8]>, archive: &[u8], split: bool) {
"range of {NEW_FILE} does not end at the following entry",
);
+ let target = accessor
+ .follow_hardlink(&hardlink)
+ .expect("failed to follow hardlink");
+ assert_eq!(
+ target.entry_range_info().entry_range,
+ new_file.entry_range_info().entry_range,
+ "hardlink target range differs from the file's range",
+ );
+
if split {
let reused_file = lookup_file(accessor, REUSED_FILE);
assert_eq!(
@@ -124,6 +155,13 @@ fn check_entry_ranges(accessor: &Accessor<&[u8]>, archive: &[u8], split: bool) {
}
}
+fn file_range(accessor: &Accessor<&[u8]>, name: &str) -> Range<u64> {
+ lookup_file(accessor, name)
+ .entry_range_info()
+ .entry_range
+ .clone()
+}
+
#[test]
fn unified_entry_ranges() {
let (archive, _) = encode_archive(false).expect("failed to encode archive");
@@ -150,3 +188,33 @@ fn split_entry_ranges() {
);
}
}
+
+#[test]
+fn split_entry_ranges_with_inflated_goodbye_size() {
+ let (archive, payload) = encode_archive(true).expect("failed to encode archive");
+ let expected = file_range(&open_accessor(&archive, &payload, true), NEW_FILE);
+
+ let mut inflated = archive.clone();
+ inflate_goodbye_size(&mut inflated, NEW_FILE, NEW_FILE_SIZE);
+ let accessor = open_accessor(&inflated, &payload, true);
+
+ // The undecoded directory entry still exposes the range as recorded in the goodbye table.
+ let root = accessor.open_root().expect("failed to open root");
+ let dir_entry = root
+ .read_dir()
+ .map(|entry| entry.expect("failed to read directory entry"))
+ .find(|entry| entry.file_name() == Path::new(NEW_FILE))
+ .expect("directory entry not found");
+ let inflated_info = dir_entry.entry_range_info().clone();
+ assert_eq!(inflated_info.entry_range.end, expected.end + NEW_FILE_SIZE);
+ assert!(inflated_info.entry_range.end > root_goodbye_start(&inflated, 3));
+
+ let decoded = dir_entry.decode_entry().expect("failed to decode entry");
+ assert_eq!(decoded.entry_range_info().entry_range, expected);
+
+ check_entry_ranges(&accessor, &inflated, true);
+
+ let opened =
+ unsafe { accessor.open_file_at_range(&inflated_info) }.expect("failed to open file");
+ assert_eq!(opened.entry_range_info().entry_range, expected);
+}
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH pxar 03/10] accessor: add convenience helper to get content's byte range
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 02/10] accessor: limit split archive file entry ranges to payload reference Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 04/10] datastore: index: return dedicated type for chunk offset trait method Christian Ebner
` (6 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
The helper allows to get the inner raw byte range stored the
ContentRange instance, which is required for raw chunk range checks
as will be performed in proxmox-backup-debug diff.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/accessor/mod.rs | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/accessor/mod.rs b/src/accessor/mod.rs
index 02c7a19..0a877ad 100644
--- a/src/accessor/mod.rs
+++ b/src/accessor/mod.rs
@@ -80,6 +80,13 @@ pub struct ContentRange {
payload_ref: Option<PayloadRef>,
}
+impl ContentRange {
+ /// Get the inner content's byte range
+ pub fn as_byte_range(&self) -> &Range<u64> {
+ &self.content
+ }
+}
+
/// awaitable version of `ReadAt`.
async fn read_at<T>(input: &T, buf: &mut [u8], offset: u64) -> io::Result<usize>
where
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 04/10] datastore: index: return dedicated type for chunk offset trait method
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (2 preceding siblings ...)
2026-11-17 15:28 ` [PATCH pxar 03/10] accessor: add convenience helper to get content's byte range Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 05/10] bin: debug: fix off-by-one error in last chunk lookup Christian Ebner
` (5 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
The trait method chunk_from_offset() as declared on IndexFile returns
the corresponding chunk position within the index file and the
relative offset within that chunk for a given absolute byte offset.
These values were however returned as tuple only, making it harder to
follow when reading the code.
Improve readability and protect against tuple value reordering by
returning a dedicated IndexedChunkOffset struct with fields chunk_pos
and relative_offset, instead of the opaque tuple positions.
No functional changes.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-datastore/src/cached_chunk_reader.rs | 9 ++++++---
pbs-datastore/src/dynamic_index.rs | 9 ++++++---
pbs-datastore/src/fixed_index.rs | 12 ++++++------
pbs-datastore/src/index.rs | 11 ++++++++++-
src/bin/proxmox_backup_debug/diff.rs | 4 ++--
5 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/pbs-datastore/src/cached_chunk_reader.rs b/pbs-datastore/src/cached_chunk_reader.rs
index 95ac23a54..e2336d874 100644
--- a/pbs-datastore/src/cached_chunk_reader.rs
+++ b/pbs-datastore/src/cached_chunk_reader.rs
@@ -76,9 +76,12 @@ impl<I: IndexFile, R: AsyncReadChunk + Send + Sync + 'static> CachedChunkReader<
let mut read: usize = 0;
while read < size {
let cur_offset = offset + read as u64;
- if let Some(chunk) = self.index.chunk_from_offset(cur_offset) {
+ if let Some(indexed_chunk_offset) = self.index.chunk_from_offset(cur_offset) {
// chunk indices retrieved from chunk_from_offset always resolve to Some(_)
- let info = self.index.chunk_info(chunk.0).unwrap();
+ let info = self
+ .index
+ .chunk_info(indexed_chunk_offset.chunk_pos)
+ .unwrap();
// will never be None, see AsyncChunkCacher
let data = self
@@ -89,7 +92,7 @@ impl<I: IndexFile, R: AsyncReadChunk + Send + Sync + 'static> CachedChunkReader<
let want_bytes = ((info.range.end - cur_offset) as usize).min(size - read);
let slice = &mut buf[read..(read + want_bytes)];
- let intra_chunk = chunk.1 as usize;
+ let intra_chunk = indexed_chunk_offset.relative_offset as usize;
slice.copy_from_slice(&data[intra_chunk..(intra_chunk + want_bytes)]);
read += want_bytes;
} else {
diff --git a/pbs-datastore/src/dynamic_index.rs b/pbs-datastore/src/dynamic_index.rs
index a0d9369a5..e4e485e0e 100644
--- a/pbs-datastore/src/dynamic_index.rs
+++ b/pbs-datastore/src/dynamic_index.rs
@@ -17,7 +17,7 @@ use pxar::accessor::{MaybeReady, ReadAt, ReadAtOperation};
use pbs_tools::lru_cache::LruCache;
use crate::file_formats;
-use crate::index::{ChunkReadInfo, IndexFile};
+use crate::index::{ChunkReadInfo, IndexFile, IndexedChunkOffset};
use crate::read_chunk::ReadChunk;
/// Header format definition for dynamic index files (`.dixd`)
@@ -247,7 +247,7 @@ impl IndexFile for DynamicIndexReader {
self.size
}
- fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
+ fn chunk_from_offset(&self, offset: u64) -> Option<IndexedChunkOffset> {
let end_idx = self.index.len() - 1;
let end = self.chunk_end(end_idx);
let found_idx = self.binary_search(0, 0, end_idx, end, offset);
@@ -262,7 +262,10 @@ impl IndexFile for DynamicIndexReader {
self.chunk_end(found_idx - 1)
};
- Some((found_idx, offset - found_start))
+ Some(IndexedChunkOffset {
+ chunk_pos: found_idx,
+ relative_offset: offset - found_start,
+ })
}
}
diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs
index ee94eae9c..df9af7c47 100644
--- a/pbs-datastore/src/fixed_index.rs
+++ b/pbs-datastore/src/fixed_index.rs
@@ -10,7 +10,7 @@ use proxmox_io::ReadExt;
use proxmox_uuid::Uuid;
use crate::file_formats;
-use crate::index::{ChunkReadInfo, IndexFile};
+use crate::index::{ChunkReadInfo, IndexFile, IndexedChunkOffset};
/// Chunk size for fixed index files.
///
@@ -211,15 +211,15 @@ impl IndexFile for FixedIndexReader {
(csum, chunk_end)
}
- fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
+ fn chunk_from_offset(&self, offset: u64) -> Option<IndexedChunkOffset> {
if offset >= self.size {
return None;
}
- Some((
- (offset / self.chunk_size as u64) as usize,
- offset & (self.chunk_size - 1) as u64, // fast modulo, valid for 2^x chunk_size
- ))
+ Some(IndexedChunkOffset {
+ chunk_pos: (offset / self.chunk_size as u64) as usize,
+ relative_offset: offset & (self.chunk_size - 1) as u64, // fast modulo, valid for 2^x chunk_size
+ })
}
}
diff --git a/pbs-datastore/src/index.rs b/pbs-datastore/src/index.rs
index b5d20ad91..dd1bee970 100644
--- a/pbs-datastore/src/index.rs
+++ b/pbs-datastore/src/index.rs
@@ -14,6 +14,15 @@ impl ChunkReadInfo {
}
}
+/// Stores an absolute byte offset as the offset relative to the start of the
+/// containing chunk and the chunk position within the index file.
+pub struct IndexedChunkOffset {
+ /// Position of the chunk within the index file
+ pub chunk_pos: usize,
+ /// Byte offset relative to the start of the chunk
+ pub relative_offset: u64,
+}
+
/// Trait to get digest list from index files
///
/// To allow easy iteration over all used chunks.
@@ -26,7 +35,7 @@ pub trait IndexFile {
fn index_size(&self) -> usize;
/// Get the chunk index and the relative offset within it for a byte offset
- fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)>;
+ fn chunk_from_offset(&self, offset: u64) -> Option<IndexedChunkOffset>;
/// Compute index checksum and size
fn compute_csum(&self) -> ([u8; 32], u64);
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 7fef82252..9cfbcddb6 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -354,11 +354,11 @@ fn visit_directory<'f, 'c>(
let first_chunk = index
.chunk_from_offset(range.start)
.context("Invalid offset")?
- .0;
+ .chunk_pos;
let last_chunk = index
.chunk_from_offset(range.end)
.context("Invalid offset")?
- .0;
+ .chunk_pos;
if entry.is_dir() {
let new_dir = entry.enter_directory().await?;
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 05/10] bin: debug: fix off-by-one error in last chunk lookup
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (3 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 04/10] datastore: index: return dedicated type for chunk offset trait method Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 06/10] bin: debug: refactore dynamic index reader opening logic into helper Christian Ebner
` (4 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
The end of a range as stored in pxar's EntryRangeInfo is exclusive,
which means this can potentially point already to the start offset of
the next chunk if the underlying chunk boundary is aligned by
chance.
Therefore, assure to exclude that chunk when looking up the last
chunk needed to cover the range.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox_backup_debug/diff.rs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 9cfbcddb6..a6e12f3d8 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -356,7 +356,7 @@ fn visit_directory<'f, 'c>(
.context("Invalid offset")?
.chunk_pos;
let last_chunk = index
- .chunk_from_offset(range.end)
+ .chunk_from_offset(range.end - 1)
.context("Invalid offset")?
.chunk_pos;
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 06/10] bin: debug: refactore dynamic index reader opening logic into helper
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (4 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 05/10] bin: debug: fix off-by-one error in last chunk lookup Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 07/10] bin: debug: refactor and deduplicate chunk range difference check Christian Ebner
` (3 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Factor out the logic to open a buffered local dynamic index reader
into a reusable helper function.
In preparation for supporting diff'ing split pxar archives, where
both the metadata and payload archive index readers need to be
opened.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox_backup_debug/diff.rs | 37 +++++++++++++++++++---------
1 file changed, 25 insertions(+), 12 deletions(-)
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index a6e12f3d8..caacea21e 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -23,6 +23,7 @@ use pbs_client::tools::{
optional_ns_param,
};
use pbs_client::{BackupReader, BackupRepository, BackupTargetArgs, RemoteChunkReader};
+use pbs_datastore::BackupManifest;
use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader, LocalDynamicReadAt};
use pbs_datastore::index::IndexFile;
use pbs_key_config::decrypt_key;
@@ -246,25 +247,19 @@ struct OutputParams {
color: ColorMode,
}
-async fn open_dynamic_index(
- snapshot: &str,
+async fn open_dynamic_index_reader(
+ backup_reader: &Arc<BackupReader>,
+ manifest: &BackupManifest,
archive_name: &BackupArchiveName,
params: &RepoParams,
-) -> Result<(DynamicIndexReader, Accessor), Error> {
- let backup_reader = create_backup_reader(snapshot, params).await?;
-
- let (manifest, _) = backup_reader
- .download_manifest(params.ignore_missing_signature)
- .await?;
- manifest.check_fingerprint(params.crypt_config.as_ref().map(Arc::as_ref))?;
-
+) -> Result<(DynamicIndexReader, Arc<dyn ReadAt + Send + Sync>, u64), Error> {
let index = backup_reader
- .download_dynamic_index(&manifest, archive_name)
+ .download_dynamic_index(manifest, archive_name)
.await?;
let most_used = index.find_most_used_chunks(8);
let lookup_index = backup_reader
- .download_dynamic_index(&manifest, archive_name)
+ .download_dynamic_index(manifest, archive_name)
.await?;
let file_info = manifest.lookup_file_info(archive_name)?;
@@ -278,6 +273,24 @@ async fn open_dynamic_index(
let reader = BufferedDynamicReader::new(index, chunk_reader);
let archive_size = reader.archive_size();
let reader: Arc<dyn ReadAt + Send + Sync> = Arc::new(LocalDynamicReadAt::new(reader));
+
+ Ok((lookup_index, reader, archive_size))
+}
+
+async fn open_dynamic_index(
+ snapshot: &str,
+ archive_name: &BackupArchiveName,
+ params: &RepoParams,
+) -> Result<(DynamicIndexReader, Accessor), Error> {
+ let backup_reader = create_backup_reader(snapshot, params).await?;
+
+ let (manifest, _) = backup_reader
+ .download_manifest(params.ignore_missing_signature)
+ .await?;
+ manifest.check_fingerprint(params.crypt_config.as_ref().map(Arc::as_ref))?;
+
+ let (lookup_index, reader, archive_size) =
+ open_dynamic_index_reader(&backup_reader, &manifest, archive_name, params).await?;
let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?;
Ok((lookup_index, accessor))
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 07/10] bin: debug: refactor and deduplicate chunk range difference check
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (5 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 06/10] bin: debug: refactore dynamic index reader opening logic into helper Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff Christian Ebner
` (2 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Refactor the chunk range lookup logic into a dedicated helper function
and deduplicate the chunk difference check. It is currently performed
for directories as well as file entries, but can be performed upfront.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox_backup_debug/diff.rs | 76 +++++++++++++++-------------
1 file changed, 42 insertions(+), 34 deletions(-)
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index caacea21e..06f89f26b 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet};
use std::ffi::{OsStr, OsString};
use std::io::{IsTerminal, Write};
use std::iter::FromIterator;
+use std::ops::Range;
use std::path::{Path, PathBuf};
use std::sync::Arc;
@@ -346,6 +347,35 @@ async fn files_in_chunk_set<'c, 'f>(
visit_directory(&root, index, &path, chunk_list, chunk_set).await
}
+fn range_contains_changed_chunk(
+ range: &Range<u64>,
+ index: &dyn IndexFile,
+ chunk_list: &[&ChunkDigest],
+ chunk_diff: &HashSet<&ChunkDigest>,
+) -> Result<bool, Error> {
+ if range.is_empty() {
+ return Ok(false);
+ }
+
+ let first_chunk = index
+ .chunk_from_offset(range.start)
+ .context("Invalid offset")?
+ .chunk_pos;
+ let last_chunk = index
+ .chunk_from_offset(range.end - 1)
+ .context("Invalid offset")?
+ .chunk_pos;
+
+ for chunk_index in first_chunk..=last_chunk {
+ let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?;
+ if chunk_diff.contains(digest) {
+ return Ok(true);
+ }
+ }
+
+ Ok(false)
+}
+
/// Recursively visits directories in .pxar archive and create a
/// map "digest --> set of contained files"
fn visit_directory<'f, 'c>(
@@ -364,47 +394,25 @@ fn visit_directory<'f, 'c>(
let entry = entry?.decode_entry().await?;
let range = &entry.entry_range_info().entry_range;
- let first_chunk = index
- .chunk_from_offset(range.start)
- .context("Invalid offset")?
- .chunk_pos;
- let last_chunk = index
- .chunk_from_offset(range.end - 1)
- .context("Invalid offset")?
- .chunk_pos;
+ // Check if any chunk of the entry is in the set of modified chunks (symmetric
+ // difference). If not, we can skip directories entirely and save a lot of time.
+ let entry_changed = range_contains_changed_chunk(range, index, chunk_list, chunk_diff)?;
+
+ if !entry_changed {
+ continue;
+ }
if entry.is_dir() {
let new_dir = entry.enter_directory().await?;
+ let dir_path = path.join(entry.file_name());
- for chunk_index in first_chunk..=last_chunk {
- // Check if any chunk of the serialized directory is in
- // set off modified chunks (symmetric difference).
- // If not, we can skip the directory entirely and save a lot of time.
-
- let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?;
-
- if chunk_diff.get(digest).is_some() {
- let dir_path = path.join(entry.file_name());
-
- entries.extend(
- visit_directory(&new_dir, index, &dir_path, chunk_list, chunk_diff)
- .await?,
- );
- break;
- }
- }
+ entries.extend(
+ visit_directory(&new_dir, index, &dir_path, chunk_list, chunk_diff).await?,
+ );
}
let file_path = path.join(entry.file_name());
-
- for chunk_index in first_chunk..=last_chunk {
- let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?;
-
- if chunk_diff.get(digest).is_some() {
- entries.insert(file_path.into_os_string(), entry);
- break;
- }
- }
+ entries.insert(file_path.into_os_string(), entry);
}
Ok(entries)
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (6 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 07/10] bin: debug: refactor and deduplicate chunk range difference check Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 09/10] bin: debug: use pxar related archive name helpers for diff Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 10/10] docs: debug: mention split pxar archives for diff subcommand Christian Ebner
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Allows comparing split pxar archives via:
```
proxmox-backup-debug diff archive ...
```
Metadata and payload archives are opened via the split variant of the
accessor, so metadata as well as payload data can be read.
To detect changes, chunk differences for both metadata and payload
archives are tracked independently. With the compare-content flag
enabled, a payload chunk difference will lead to checking of payload
chunk ranges for files and descend into directories even if the
metadata archive chunks are unchanged. By this any change is assured
to be detected.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox_backup_debug/diff.rs | 272 +++++++++++++++++++--------
1 file changed, 198 insertions(+), 74 deletions(-)
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 06f89f26b..cef4af68d 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -21,7 +21,7 @@ use pbs_client::tools::key_source::{
};
use pbs_client::tools::{
complete_archive_name, complete_group_or_snapshot, connect, extract_repository_from_value,
- optional_ns_param,
+ get_pxar_archive_names, optional_ns_param,
};
use pbs_client::{BackupReader, BackupRepository, BackupTargetArgs, RemoteChunkReader};
use pbs_datastore::BackupManifest;
@@ -166,27 +166,44 @@ async fn diff_archive(
compare_contents: bool,
output_params: &OutputParams,
) -> Result<(), Error> {
- let (index_a, accessor_a) = open_dynamic_index(snapshot_a, file_name, repo_params).await?;
- let (index_b, accessor_b) = open_dynamic_index(snapshot_b, file_name, repo_params).await?;
+ let archive_a = open_archive(snapshot_a, file_name, repo_params).await?;
+ let archive_b = open_archive(snapshot_b, file_name, repo_params).await?;
// vecs of chunk digests, in their correct order
- let chunks_a = chunk_digests_for_index(&index_a);
- let chunks_b = chunk_digests_for_index(&index_b);
-
- // sets of chunk digests, 'cause we want to perform set operations
- let chunk_set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied());
- let chunk_set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied());
-
- // Symmetric difference between both sets -
- // content stored in those chunks was either added, modified or deleted
- let chunk_sym_diff: HashSet<&ChunkDigest> = chunk_set_a
- .symmetric_difference(&chunk_set_b)
- .copied()
- .collect();
+ let archive_chunks_a = chunk_digests_for_index(&archive_a.index);
+ let archive_chunks_b = chunk_digests_for_index(&archive_b.index);
+ let archive_chunk_diff = compute_chunk_difference(&archive_chunks_a, &archive_chunks_b);
+
+ let payload_chunks_a = archive_a
+ .payload_index
+ .as_ref()
+ .map_or_else(Vec::new, |index| chunk_digests_for_index(index));
+ let payload_chunks_b = archive_b
+ .payload_index
+ .as_ref()
+ .map_or_else(Vec::new, |index| chunk_digests_for_index(index));
+ let payload_chunk_diff = compute_chunk_difference(&payload_chunks_a, &payload_chunks_b);
// Figure out which files are stored in which chunks
- let files_in_a = files_in_chunk_set(&chunks_a, &accessor_a, &index_a, &chunk_sym_diff).await?;
- let files_in_b = files_in_chunk_set(&chunks_b, &accessor_b, &index_b, &chunk_sym_diff).await?;
+ let diff_a = ArchiveDiff::new(
+ ChunkDiff::new(&archive_a.index, &archive_chunks_a, &archive_chunk_diff),
+ archive_a
+ .payload_index
+ .as_ref()
+ .map(|index| ChunkDiff::new(index, &payload_chunks_a, &payload_chunk_diff)),
+ compare_contents,
+ );
+ let files_in_a = files_in_chunk_set(&archive_a.accessor, &diff_a).await?;
+
+ let diff_b = ArchiveDiff::new(
+ ChunkDiff::new(&archive_b.index, &archive_chunks_b, &archive_chunk_diff),
+ archive_b
+ .payload_index
+ .as_ref()
+ .map(|index| ChunkDiff::new(index, &payload_chunks_b, &payload_chunk_diff)),
+ compare_contents,
+ );
+ let files_in_b = files_in_chunk_set(&archive_b.accessor, &diff_b).await?;
// If file in A but not in B --> deleted
let deleted_files: HashMap<&OsStr, &FileEntry> = files_in_a
@@ -278,11 +295,17 @@ async fn open_dynamic_index_reader(
Ok((lookup_index, reader, archive_size))
}
-async fn open_dynamic_index(
+struct ArchiveReader {
+ index: DynamicIndexReader,
+ payload_index: Option<DynamicIndexReader>,
+ accessor: Accessor,
+}
+
+async fn open_archive(
snapshot: &str,
archive_name: &BackupArchiveName,
params: &RepoParams,
-) -> Result<(DynamicIndexReader, Accessor), Error> {
+) -> Result<ArchiveReader, Error> {
let backup_reader = create_backup_reader(snapshot, params).await?;
let (manifest, _) = backup_reader
@@ -290,11 +313,30 @@ async fn open_dynamic_index(
.await?;
manifest.check_fingerprint(params.crypt_config.as_ref().map(Arc::as_ref))?;
- let (lookup_index, reader, archive_size) =
- open_dynamic_index_reader(&backup_reader, &manifest, archive_name, params).await?;
- let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?;
+ let (archive_name, payload_archive_name) = get_pxar_archive_names(archive_name, &manifest)?;
+
+ let (index, reader, archive_size) =
+ open_dynamic_index_reader(&backup_reader, &manifest, &archive_name, params).await?;
- Ok((lookup_index, accessor))
+ let (payload_index, variant) = if let Some(payload_archive_name) = payload_archive_name {
+ let (payload_index, payload_reader, payload_size) =
+ open_dynamic_index_reader(&backup_reader, &manifest, &payload_archive_name, params)
+ .await?;
+ (
+ Some(payload_index),
+ pxar::PxarVariant::Split(reader, (payload_reader, payload_size)),
+ )
+ } else {
+ (None, pxar::PxarVariant::Unified(reader))
+ };
+
+ let accessor = Accessor::new(variant, archive_size).await?;
+
+ Ok(ArchiveReader {
+ index,
+ payload_index,
+ accessor,
+ })
}
async fn create_backup_reader(
@@ -334,85 +376,167 @@ fn chunk_digests_for_index(index: &dyn IndexFile) -> Vec<&ChunkDigest> {
all_chunks
}
-/// Compute which files are contained in a given chunk set.
-async fn files_in_chunk_set<'c, 'f>(
- chunk_list: &[&'c ChunkDigest],
- accessor: &'f Accessor,
- index: &'f DynamicIndexReader,
- chunk_set: &HashSet<&'c ChunkDigest>,
-) -> Result<HashMap<OsString, FileEntry>, Error> {
- let path = PathBuf::new();
- let root = accessor.open_root().await?;
+/// Compute the symmetric difference between two lists of chunk digests.
+/// Content stored in those chunks was either added, modified or deleted.
+fn compute_chunk_difference<'a>(
+ chunks_a: &[&'a ChunkDigest],
+ chunks_b: &[&'a ChunkDigest],
+) -> HashSet<&'a ChunkDigest> {
+ let set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied());
+ let set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied());
+
+ set_a.symmetric_difference(&set_b).copied().collect()
+}
+
+struct ChunkDiff<'a, 'c> {
+ index: &'a DynamicIndexReader,
+ chunk_list: &'a [&'c ChunkDigest],
+ chunk_diff: &'a HashSet<&'c ChunkDigest>,
+}
+
+impl<'a, 'c> ChunkDiff<'a, 'c> {
+ fn new(
+ index: &'a DynamicIndexReader,
+ chunk_list: &'a [&'c ChunkDigest],
+ chunk_diff: &'a HashSet<&'c ChunkDigest>,
+ ) -> Self {
+ Self {
+ index,
+ chunk_list,
+ chunk_diff,
+ }
+ }
+
+ fn contains_chunks_within_range(&self, range: &Range<u64>) -> Result<bool, Error> {
+ if range.is_empty() {
+ return Ok(false);
+ }
+
+ let first_chunk = self
+ .index
+ .chunk_from_offset(range.start)
+ .context("Invalid offset")?
+ .chunk_pos;
+ let last_chunk = self
+ .index
+ .chunk_from_offset(range.end - 1)
+ .context("Invalid offset")?
+ .chunk_pos;
+
+ for chunk_index in first_chunk..=last_chunk {
+ let digest = self
+ .chunk_list
+ .get(chunk_index)
+ .context("Invalid chunk index")?;
+ if self.chunk_diff.contains(digest) {
+ return Ok(true);
+ }
+ }
- visit_directory(&root, index, &path, chunk_list, chunk_set).await
+ Ok(false)
+ }
+
+ fn is_empty(&self) -> bool {
+ self.chunk_diff.is_empty()
+ }
}
-fn range_contains_changed_chunk(
- range: &Range<u64>,
- index: &dyn IndexFile,
- chunk_list: &[&ChunkDigest],
- chunk_diff: &HashSet<&ChunkDigest>,
-) -> Result<bool, Error> {
- if range.is_empty() {
- return Ok(false);
- }
-
- let first_chunk = index
- .chunk_from_offset(range.start)
- .context("Invalid offset")?
- .chunk_pos;
- let last_chunk = index
- .chunk_from_offset(range.end - 1)
- .context("Invalid offset")?
- .chunk_pos;
-
- for chunk_index in first_chunk..=last_chunk {
- let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?;
- if chunk_diff.contains(digest) {
- return Ok(true);
+struct ArchiveDiff<'a, 'c> {
+ metadata_diff: ChunkDiff<'a, 'c>,
+ payload_diff: Option<ChunkDiff<'a, 'c>>,
+ compare_contents: bool,
+}
+
+impl<'a, 'c> ArchiveDiff<'a, 'c> {
+ fn new(
+ metadata_diff: ChunkDiff<'a, 'c>,
+ payload_diff: Option<ChunkDiff<'a, 'c>>,
+ compare_contents: bool,
+ ) -> Self {
+ Self {
+ metadata_diff,
+ payload_diff,
+ compare_contents,
}
}
+}
+
+/// Compute which files are contained in a given chunk set.
+async fn files_in_chunk_set<'f, 'a, 'c>(
+ accessor: &'f Accessor,
+ archive_diff: &'f ArchiveDiff<'a, 'c>,
+) -> Result<HashMap<OsString, FileEntry>, Error> {
+ let path = PathBuf::new();
+ let root = accessor.open_root().await?;
- Ok(false)
+ visit_directory(&root, &path, archive_diff).await
}
/// Recursively visits directories in .pxar archive and create a
/// map "digest --> set of contained files"
-fn visit_directory<'f, 'c>(
+fn visit_directory<'f, 'a, 'c>(
directory: &'f Directory,
- index: &'f DynamicIndexReader,
path: &'f Path,
- chunk_list: &'f [&'c ChunkDigest],
- chunk_diff: &'f HashSet<&'c ChunkDigest>,
+ archive_diff: &'f ArchiveDiff<'a, 'c>,
) -> BoxFuture<'f, Result<HashMap<OsString, FileEntry>, Error>> {
async move {
let mut entries: HashMap<OsString, FileEntry> = HashMap::new();
let mut iter = directory.read_dir();
+ // In split pxar archives, payload changes with identical size and metadata do not
+ // necessarily lead to changes in metadata archive chunks, so directories cannot be skipped
+ // when comparing contents if no difference in metadata chunks has been detected.
+ let descend_unchanged_dirs = archive_diff.compare_contents
+ && archive_diff
+ .payload_diff
+ .as_ref()
+ .is_some_and(|payload_diff| !payload_diff.is_empty());
while let Some(entry) = iter.next().await {
let entry = entry?.decode_entry().await?;
let range = &entry.entry_range_info().entry_range;
// Check if any chunk of the entry is in the set of modified chunks (symmetric
- // difference). If not, we can skip directories entirely and save a lot of time.
- let entry_changed = range_contains_changed_chunk(range, index, chunk_list, chunk_diff)?;
-
- if !entry_changed {
- continue;
- }
+ // difference). If not, we can skip directories entirely unless payload diffs
+ // require checking file contents.
+ let archive_changed = archive_diff
+ .metadata_diff
+ .contains_chunks_within_range(range)
+ .with_context(|| {
+ format!(
+ "failed to check archive chunk range {range:?} on entry {:#?}",
+ entry.file_name(),
+ )
+ })?;
- if entry.is_dir() {
+ if entry.is_dir() && (archive_changed || descend_unchanged_dirs) {
let new_dir = entry.enter_directory().await?;
let dir_path = path.join(entry.file_name());
- entries.extend(
- visit_directory(&new_dir, index, &dir_path, chunk_list, chunk_diff).await?,
- );
+ entries.extend(visit_directory(&new_dir, &dir_path, archive_diff).await?);
+ }
+
+ let mut payload_changed = false;
+ if archive_diff.compare_contents {
+ if let Some(payload_diff) = archive_diff.payload_diff.as_ref() {
+ if let Some(content_range) = entry.content_range()? {
+ let range = content_range.as_byte_range();
+ payload_changed = payload_diff
+ .contains_chunks_within_range(range)
+ .with_context(|| {
+ format!(
+ "failed to check payload archive chunk range {range:?} on entry {:#?}",
+ entry.file_name(),
+ )
+ })?
+ }
+ }
}
- let file_path = path.join(entry.file_name());
- entries.insert(file_path.into_os_string(), entry);
+ if archive_changed || payload_changed {
+ let file_path = path.join(entry.file_name());
+ entries.insert(file_path.into_os_string(), entry);
+ }
}
Ok(entries)
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 09/10] bin: debug: use pxar related archive name helpers for diff
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (7 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 10/10] docs: debug: mention split pxar archives for diff subcommand Christian Ebner
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Reuses the helpers to check for archive name extensions and archive
name completions used by the client and server for keeping
implementation details in sync.
Also, mention mpxar/ppxar extensions being supported when bailing.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox_backup_debug/diff.rs | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index cef4af68d..a8fe73b4f 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -20,7 +20,7 @@ use pbs_client::tools::key_source::{
get_encryption_key_password,
};
use pbs_client::tools::{
- complete_archive_name, complete_group_or_snapshot, connect, extract_repository_from_value,
+ complete_group_or_snapshot, complete_pxar_archive_name, connect, extract_repository_from_value,
get_pxar_archive_names, optional_ns_param,
};
use pbs_client::{BackupReader, BackupRepository, BackupTargetArgs, RemoteChunkReader};
@@ -51,7 +51,7 @@ pub fn diff_commands() -> CommandLineInterface {
.arg_param(&["prev-snapshot", "snapshot", "archive-name"])
.completion_cb("prev-snapshot", complete_group_or_snapshot)
.completion_cb("snapshot", complete_group_or_snapshot)
- .completion_cb("archive-name", complete_archive_name),
+ .completion_cb("archive-name", complete_pxar_archive_name),
);
cmd_def.into()
@@ -141,7 +141,7 @@ async fn diff_archive_cmd(
let output_params = OutputParams { color };
- if archive_name.ends_with(".pxar.didx") {
+ if archive_name.has_pxar_filename_extension() {
diff_archive(
&prev_snapshot,
&snapshot,
@@ -152,7 +152,7 @@ async fn diff_archive_cmd(
)
.await?;
} else {
- bail!("Only .pxar files are supported");
+ bail!("Only .pxar, .mpxar and .ppxar files are supported");
}
Ok(())
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH proxmox-backup 10/10] docs: debug: mention split pxar archives for diff subcommand
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
` (8 preceding siblings ...)
2026-11-17 15:28 ` [PATCH proxmox-backup 09/10] bin: debug: use pxar related archive name helpers for diff Christian Ebner
@ 2026-11-17 15:28 ` Christian Ebner
9 siblings, 0 replies; 11+ messages in thread
From: Christian Ebner @ 2026-11-17 15:28 UTC (permalink / raw)
To: pbs-devel
Document that diff now also supports split pxar archives.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
docs/proxmox-backup-debug/description.rst | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/proxmox-backup-debug/description.rst b/docs/proxmox-backup-debug/description.rst
index 8b28957e0..fa6f49bc4 100644
--- a/docs/proxmox-backup-debug/description.rst
+++ b/docs/proxmox-backup-debug/description.rst
@@ -1,8 +1,9 @@
Implements debugging functionality to inspect Proxmox Backup datastore
files, verify the integrity of chunks.
-The 'diff' subcommand allows comparing .pxar archives for two
-arbitrary snapshots. A list of added/modified/deleted files will be displayed.
+The 'diff' subcommand allows comparing .pxar (and split .mpxar/.ppxar)
+archives for two arbitrary snapshots. A list of added/modified/deleted
+files will be displayed.
Also contains an 'api' subcommand where arbitrary api paths can be called
(get/create/set/delete) as well as display their parameters (usage) and
--
2.47.3
^ permalink raw reply related [flat|nested] 11+ messages in thread