public inbox for pbs-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff
Date: Tue, 17 Nov 2026 16:28:12 +0100	[thread overview]
Message-ID: <20261117152814.491323-9-c.ebner@proxmox.com> (raw)
In-Reply-To: <20261117152814.491323-1-c.ebner@proxmox.com>

Allows comparing split pxar archives via:
```
proxmox-backup-debug diff archive ...
```

Metadata and payload archives are opened via the split variant of the
accessor, so metadata as well as payload data can be read.

To detect changes, chunk differences for both metadata and payload
archives are tracked independently. With the compare-content flag
enabled, a payload chunk difference will lead to checking of payload
chunk ranges for files and descend into directories even if the
metadata archive chunks are unchanged. By this any change is assured
to be detected.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/bin/proxmox_backup_debug/diff.rs | 272 +++++++++++++++++++--------
 1 file changed, 198 insertions(+), 74 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 06f89f26b..cef4af68d 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -21,7 +21,7 @@ use pbs_client::tools::key_source::{
 };
 use pbs_client::tools::{
     complete_archive_name, complete_group_or_snapshot, connect, extract_repository_from_value,
-    optional_ns_param,
+    get_pxar_archive_names, optional_ns_param,
 };
 use pbs_client::{BackupReader, BackupRepository, BackupTargetArgs, RemoteChunkReader};
 use pbs_datastore::BackupManifest;
@@ -166,27 +166,44 @@ async fn diff_archive(
     compare_contents: bool,
     output_params: &OutputParams,
 ) -> Result<(), Error> {
-    let (index_a, accessor_a) = open_dynamic_index(snapshot_a, file_name, repo_params).await?;
-    let (index_b, accessor_b) = open_dynamic_index(snapshot_b, file_name, repo_params).await?;
+    let archive_a = open_archive(snapshot_a, file_name, repo_params).await?;
+    let archive_b = open_archive(snapshot_b, file_name, repo_params).await?;
 
     // vecs of chunk digests, in their correct order
-    let chunks_a = chunk_digests_for_index(&index_a);
-    let chunks_b = chunk_digests_for_index(&index_b);
-
-    // sets of chunk digests, 'cause we want to perform set operations
-    let chunk_set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied());
-    let chunk_set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied());
-
-    // Symmetric difference between both sets -
-    // content stored in those chunks was either added, modified or deleted
-    let chunk_sym_diff: HashSet<&ChunkDigest> = chunk_set_a
-        .symmetric_difference(&chunk_set_b)
-        .copied()
-        .collect();
+    let archive_chunks_a = chunk_digests_for_index(&archive_a.index);
+    let archive_chunks_b = chunk_digests_for_index(&archive_b.index);
+    let archive_chunk_diff = compute_chunk_difference(&archive_chunks_a, &archive_chunks_b);
+
+    let payload_chunks_a = archive_a
+        .payload_index
+        .as_ref()
+        .map_or_else(Vec::new, |index| chunk_digests_for_index(index));
+    let payload_chunks_b = archive_b
+        .payload_index
+        .as_ref()
+        .map_or_else(Vec::new, |index| chunk_digests_for_index(index));
+    let payload_chunk_diff = compute_chunk_difference(&payload_chunks_a, &payload_chunks_b);
 
     // Figure out which files are stored in which chunks
-    let files_in_a = files_in_chunk_set(&chunks_a, &accessor_a, &index_a, &chunk_sym_diff).await?;
-    let files_in_b = files_in_chunk_set(&chunks_b, &accessor_b, &index_b, &chunk_sym_diff).await?;
+    let diff_a = ArchiveDiff::new(
+        ChunkDiff::new(&archive_a.index, &archive_chunks_a, &archive_chunk_diff),
+        archive_a
+            .payload_index
+            .as_ref()
+            .map(|index| ChunkDiff::new(index, &payload_chunks_a, &payload_chunk_diff)),
+        compare_contents,
+    );
+    let files_in_a = files_in_chunk_set(&archive_a.accessor, &diff_a).await?;
+
+    let diff_b = ArchiveDiff::new(
+        ChunkDiff::new(&archive_b.index, &archive_chunks_b, &archive_chunk_diff),
+        archive_b
+            .payload_index
+            .as_ref()
+            .map(|index| ChunkDiff::new(index, &payload_chunks_b, &payload_chunk_diff)),
+        compare_contents,
+    );
+    let files_in_b = files_in_chunk_set(&archive_b.accessor, &diff_b).await?;
 
     // If file in A but not in B --> deleted
     let deleted_files: HashMap<&OsStr, &FileEntry> = files_in_a
@@ -278,11 +295,17 @@ async fn open_dynamic_index_reader(
     Ok((lookup_index, reader, archive_size))
 }
 
-async fn open_dynamic_index(
+struct ArchiveReader {
+    index: DynamicIndexReader,
+    payload_index: Option<DynamicIndexReader>,
+    accessor: Accessor,
+}
+
+async fn open_archive(
     snapshot: &str,
     archive_name: &BackupArchiveName,
     params: &RepoParams,
-) -> Result<(DynamicIndexReader, Accessor), Error> {
+) -> Result<ArchiveReader, Error> {
     let backup_reader = create_backup_reader(snapshot, params).await?;
 
     let (manifest, _) = backup_reader
@@ -290,11 +313,30 @@ async fn open_dynamic_index(
         .await?;
     manifest.check_fingerprint(params.crypt_config.as_ref().map(Arc::as_ref))?;
 
-    let (lookup_index, reader, archive_size) =
-        open_dynamic_index_reader(&backup_reader, &manifest, archive_name, params).await?;
-    let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?;
+    let (archive_name, payload_archive_name) = get_pxar_archive_names(archive_name, &manifest)?;
+
+    let (index, reader, archive_size) =
+        open_dynamic_index_reader(&backup_reader, &manifest, &archive_name, params).await?;
 
-    Ok((lookup_index, accessor))
+    let (payload_index, variant) = if let Some(payload_archive_name) = payload_archive_name {
+        let (payload_index, payload_reader, payload_size) =
+            open_dynamic_index_reader(&backup_reader, &manifest, &payload_archive_name, params)
+                .await?;
+        (
+            Some(payload_index),
+            pxar::PxarVariant::Split(reader, (payload_reader, payload_size)),
+        )
+    } else {
+        (None, pxar::PxarVariant::Unified(reader))
+    };
+
+    let accessor = Accessor::new(variant, archive_size).await?;
+
+    Ok(ArchiveReader {
+        index,
+        payload_index,
+        accessor,
+    })
 }
 
 async fn create_backup_reader(
@@ -334,85 +376,167 @@ fn chunk_digests_for_index(index: &dyn IndexFile) -> Vec<&ChunkDigest> {
     all_chunks
 }
 
-/// Compute which files are contained in a given chunk set.
-async fn files_in_chunk_set<'c, 'f>(
-    chunk_list: &[&'c ChunkDigest],
-    accessor: &'f Accessor,
-    index: &'f DynamicIndexReader,
-    chunk_set: &HashSet<&'c ChunkDigest>,
-) -> Result<HashMap<OsString, FileEntry>, Error> {
-    let path = PathBuf::new();
-    let root = accessor.open_root().await?;
+/// Compute the symmetric difference between two lists of chunk digests.
+/// Content stored in those chunks was either added, modified or deleted.
+fn compute_chunk_difference<'a>(
+    chunks_a: &[&'a ChunkDigest],
+    chunks_b: &[&'a ChunkDigest],
+) -> HashSet<&'a ChunkDigest> {
+    let set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied());
+    let set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied());
+
+    set_a.symmetric_difference(&set_b).copied().collect()
+}
+
+struct ChunkDiff<'a, 'c> {
+    index: &'a DynamicIndexReader,
+    chunk_list: &'a [&'c ChunkDigest],
+    chunk_diff: &'a HashSet<&'c ChunkDigest>,
+}
+
+impl<'a, 'c> ChunkDiff<'a, 'c> {
+    fn new(
+        index: &'a DynamicIndexReader,
+        chunk_list: &'a [&'c ChunkDigest],
+        chunk_diff: &'a HashSet<&'c ChunkDigest>,
+    ) -> Self {
+        Self {
+            index,
+            chunk_list,
+            chunk_diff,
+        }
+    }
+
+    fn contains_chunks_within_range(&self, range: &Range<u64>) -> Result<bool, Error> {
+        if range.is_empty() {
+            return Ok(false);
+        }
+
+        let first_chunk = self
+            .index
+            .chunk_from_offset(range.start)
+            .context("Invalid offset")?
+            .chunk_pos;
+        let last_chunk = self
+            .index
+            .chunk_from_offset(range.end - 1)
+            .context("Invalid offset")?
+            .chunk_pos;
+
+        for chunk_index in first_chunk..=last_chunk {
+            let digest = self
+                .chunk_list
+                .get(chunk_index)
+                .context("Invalid chunk index")?;
+            if self.chunk_diff.contains(digest) {
+                return Ok(true);
+            }
+        }
 
-    visit_directory(&root, index, &path, chunk_list, chunk_set).await
+        Ok(false)
+    }
+
+    fn is_empty(&self) -> bool {
+        self.chunk_diff.is_empty()
+    }
 }
 
-fn range_contains_changed_chunk(
-    range: &Range<u64>,
-    index: &dyn IndexFile,
-    chunk_list: &[&ChunkDigest],
-    chunk_diff: &HashSet<&ChunkDigest>,
-) -> Result<bool, Error> {
-    if range.is_empty() {
-        return Ok(false);
-    }
-
-    let first_chunk = index
-        .chunk_from_offset(range.start)
-        .context("Invalid offset")?
-        .chunk_pos;
-    let last_chunk = index
-        .chunk_from_offset(range.end - 1)
-        .context("Invalid offset")?
-        .chunk_pos;
-
-    for chunk_index in first_chunk..=last_chunk {
-        let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?;
-        if chunk_diff.contains(digest) {
-            return Ok(true);
+struct ArchiveDiff<'a, 'c> {
+    metadata_diff: ChunkDiff<'a, 'c>,
+    payload_diff: Option<ChunkDiff<'a, 'c>>,
+    compare_contents: bool,
+}
+
+impl<'a, 'c> ArchiveDiff<'a, 'c> {
+    fn new(
+        metadata_diff: ChunkDiff<'a, 'c>,
+        payload_diff: Option<ChunkDiff<'a, 'c>>,
+        compare_contents: bool,
+    ) -> Self {
+        Self {
+            metadata_diff,
+            payload_diff,
+            compare_contents,
         }
     }
+}
+
+/// Compute which files are contained in a given chunk set.
+async fn files_in_chunk_set<'f, 'a, 'c>(
+    accessor: &'f Accessor,
+    archive_diff: &'f ArchiveDiff<'a, 'c>,
+) -> Result<HashMap<OsString, FileEntry>, Error> {
+    let path = PathBuf::new();
+    let root = accessor.open_root().await?;
 
-    Ok(false)
+    visit_directory(&root, &path, archive_diff).await
 }
 
 /// Recursively visits directories in .pxar archive and create a
 /// map "digest --> set of contained files"
-fn visit_directory<'f, 'c>(
+fn visit_directory<'f, 'a, 'c>(
     directory: &'f Directory,
-    index: &'f DynamicIndexReader,
     path: &'f Path,
-    chunk_list: &'f [&'c ChunkDigest],
-    chunk_diff: &'f HashSet<&'c ChunkDigest>,
+    archive_diff: &'f ArchiveDiff<'a, 'c>,
 ) -> BoxFuture<'f, Result<HashMap<OsString, FileEntry>, Error>> {
     async move {
         let mut entries: HashMap<OsString, FileEntry> = HashMap::new();
 
         let mut iter = directory.read_dir();
+        // In split pxar archives, payload changes with identical size and metadata do not
+        // necessarily lead to changes in metadata archive chunks, so directories cannot be skipped
+        // when comparing contents if no difference in metadata chunks has been detected.
+        let descend_unchanged_dirs = archive_diff.compare_contents
+            && archive_diff
+                .payload_diff
+                .as_ref()
+                .is_some_and(|payload_diff| !payload_diff.is_empty());
 
         while let Some(entry) = iter.next().await {
             let entry = entry?.decode_entry().await?;
             let range = &entry.entry_range_info().entry_range;
 
             // Check if any chunk of the entry is in the set of modified chunks (symmetric
-            // difference). If not, we can skip directories entirely and save a lot of time.
-            let entry_changed = range_contains_changed_chunk(range, index, chunk_list, chunk_diff)?;
-
-            if !entry_changed {
-                continue;
-            }
+            // difference). If not, we can skip directories entirely unless payload diffs
+            // require checking file contents.
+            let archive_changed = archive_diff
+                .metadata_diff
+                .contains_chunks_within_range(range)
+                .with_context(|| {
+                    format!(
+                        "failed to check archive chunk range {range:?} on entry {:#?}",
+                        entry.file_name(),
+                    )
+                })?;
 
-            if entry.is_dir() {
+            if entry.is_dir() && (archive_changed || descend_unchanged_dirs) {
                 let new_dir = entry.enter_directory().await?;
                 let dir_path = path.join(entry.file_name());
 
-                entries.extend(
-                    visit_directory(&new_dir, index, &dir_path, chunk_list, chunk_diff).await?,
-                );
+                entries.extend(visit_directory(&new_dir, &dir_path, archive_diff).await?);
+            }
+
+            let mut payload_changed = false;
+            if archive_diff.compare_contents {
+                if let Some(payload_diff) = archive_diff.payload_diff.as_ref() {
+                    if let Some(content_range) = entry.content_range()? {
+                        let range = content_range.as_byte_range();
+                        payload_changed = payload_diff
+                            .contains_chunks_within_range(range)
+                            .with_context(|| {
+                                format!(
+                                    "failed to check payload archive chunk range {range:?} on entry {:#?}",
+                                    entry.file_name(),
+                                )
+                            })?
+                    }
+                }
             }
 
-            let file_path = path.join(entry.file_name());
-            entries.insert(file_path.into_os_string(), entry);
+            if archive_changed || payload_changed {
+                let file_path = path.join(entry.file_name());
+                entries.insert(file_path.into_os_string(), entry);
+            }
         }
 
         Ok(entries)
-- 
2.47.3





  parent reply	other threads:[~2026-11-17 15:29 UTC|newest]

Thread overview: 11+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17 15:28 [PATCH proxmox-backup/pxar 00/10] fix #8319: support split pxar archives in proxmox-backup-debug diff Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 01/10] encoder: record metadata-only goodbye size for split archive files Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 02/10] accessor: limit split archive file entry ranges to payload reference Christian Ebner
2026-11-17 15:28 ` [PATCH pxar 03/10] accessor: add convenience helper to get content's byte range Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 04/10] datastore: index: return dedicated type for chunk offset trait method Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 05/10] bin: debug: fix off-by-one error in last chunk lookup Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 06/10] bin: debug: refactore dynamic index reader opening logic into helper Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 07/10] bin: debug: refactor and deduplicate chunk range difference check Christian Ebner
2026-11-17 15:28 ` Christian Ebner [this message]
2026-11-17 15:28 ` [PATCH proxmox-backup 09/10] bin: debug: use pxar related archive name helpers for diff Christian Ebner
2026-11-17 15:28 ` [PATCH proxmox-backup 10/10] docs: debug: mention split pxar archives for diff subcommand Christian Ebner

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117152814.491323-9-c.ebner@proxmox.com \
    --to=c.ebner@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal