From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 2CF151FF0AA for ; Tue, 17 Nov 2026 16:29:02 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 107972165A; Tue, 17 Nov 2026 16:28:52 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup 08/10] fix #8319: bin: debug: support split pxar archives via diff Date: Tue, 17 Nov 2026 16:28:12 +0100 Message-ID: <20261117152814.491323-9-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261117152814.491323-1-c.ebner@proxmox.com> References: <20261117152814.491323-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794929312085 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.365 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: S5XQELV3UTLLWCC2W4EZ4URCOLLD33XI X-Message-ID-Hash: S5XQELV3UTLLWCC2W4EZ4URCOLLD33XI X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Allows comparing split pxar archives via: ``` proxmox-backup-debug diff archive ... ``` Metadata and payload archives are opened via the split variant of the accessor, so metadata as well as payload data can be read. To detect changes, chunk differences for both metadata and payload archives are tracked independently. With the compare-content flag enabled, a payload chunk difference will lead to checking of payload chunk ranges for files and descend into directories even if the metadata archive chunks are unchanged. By this any change is assured to be detected. Signed-off-by: Christian Ebner --- src/bin/proxmox_backup_debug/diff.rs | 272 +++++++++++++++++++-------- 1 file changed, 198 insertions(+), 74 deletions(-) diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs index 06f89f26b..cef4af68d 100644 --- a/src/bin/proxmox_backup_debug/diff.rs +++ b/src/bin/proxmox_backup_debug/diff.rs @@ -21,7 +21,7 @@ use pbs_client::tools::key_source::{ }; use pbs_client::tools::{ complete_archive_name, complete_group_or_snapshot, connect, extract_repository_from_value, - optional_ns_param, + get_pxar_archive_names, optional_ns_param, }; use pbs_client::{BackupReader, BackupRepository, BackupTargetArgs, RemoteChunkReader}; use pbs_datastore::BackupManifest; @@ -166,27 +166,44 @@ async fn diff_archive( compare_contents: bool, output_params: &OutputParams, ) -> Result<(), Error> { - let (index_a, accessor_a) = open_dynamic_index(snapshot_a, file_name, repo_params).await?; - let (index_b, accessor_b) = open_dynamic_index(snapshot_b, file_name, repo_params).await?; + let archive_a = open_archive(snapshot_a, file_name, repo_params).await?; + let archive_b = open_archive(snapshot_b, file_name, repo_params).await?; // vecs of chunk digests, in their correct order - let chunks_a = chunk_digests_for_index(&index_a); - let chunks_b = chunk_digests_for_index(&index_b); - - // sets of chunk digests, 'cause we want to perform set operations - let chunk_set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied()); - let chunk_set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied()); - - // Symmetric difference between both sets - - // content stored in those chunks was either added, modified or deleted - let chunk_sym_diff: HashSet<&ChunkDigest> = chunk_set_a - .symmetric_difference(&chunk_set_b) - .copied() - .collect(); + let archive_chunks_a = chunk_digests_for_index(&archive_a.index); + let archive_chunks_b = chunk_digests_for_index(&archive_b.index); + let archive_chunk_diff = compute_chunk_difference(&archive_chunks_a, &archive_chunks_b); + + let payload_chunks_a = archive_a + .payload_index + .as_ref() + .map_or_else(Vec::new, |index| chunk_digests_for_index(index)); + let payload_chunks_b = archive_b + .payload_index + .as_ref() + .map_or_else(Vec::new, |index| chunk_digests_for_index(index)); + let payload_chunk_diff = compute_chunk_difference(&payload_chunks_a, &payload_chunks_b); // Figure out which files are stored in which chunks - let files_in_a = files_in_chunk_set(&chunks_a, &accessor_a, &index_a, &chunk_sym_diff).await?; - let files_in_b = files_in_chunk_set(&chunks_b, &accessor_b, &index_b, &chunk_sym_diff).await?; + let diff_a = ArchiveDiff::new( + ChunkDiff::new(&archive_a.index, &archive_chunks_a, &archive_chunk_diff), + archive_a + .payload_index + .as_ref() + .map(|index| ChunkDiff::new(index, &payload_chunks_a, &payload_chunk_diff)), + compare_contents, + ); + let files_in_a = files_in_chunk_set(&archive_a.accessor, &diff_a).await?; + + let diff_b = ArchiveDiff::new( + ChunkDiff::new(&archive_b.index, &archive_chunks_b, &archive_chunk_diff), + archive_b + .payload_index + .as_ref() + .map(|index| ChunkDiff::new(index, &payload_chunks_b, &payload_chunk_diff)), + compare_contents, + ); + let files_in_b = files_in_chunk_set(&archive_b.accessor, &diff_b).await?; // If file in A but not in B --> deleted let deleted_files: HashMap<&OsStr, &FileEntry> = files_in_a @@ -278,11 +295,17 @@ async fn open_dynamic_index_reader( Ok((lookup_index, reader, archive_size)) } -async fn open_dynamic_index( +struct ArchiveReader { + index: DynamicIndexReader, + payload_index: Option, + accessor: Accessor, +} + +async fn open_archive( snapshot: &str, archive_name: &BackupArchiveName, params: &RepoParams, -) -> Result<(DynamicIndexReader, Accessor), Error> { +) -> Result { let backup_reader = create_backup_reader(snapshot, params).await?; let (manifest, _) = backup_reader @@ -290,11 +313,30 @@ async fn open_dynamic_index( .await?; manifest.check_fingerprint(params.crypt_config.as_ref().map(Arc::as_ref))?; - let (lookup_index, reader, archive_size) = - open_dynamic_index_reader(&backup_reader, &manifest, archive_name, params).await?; - let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; + let (archive_name, payload_archive_name) = get_pxar_archive_names(archive_name, &manifest)?; + + let (index, reader, archive_size) = + open_dynamic_index_reader(&backup_reader, &manifest, &archive_name, params).await?; - Ok((lookup_index, accessor)) + let (payload_index, variant) = if let Some(payload_archive_name) = payload_archive_name { + let (payload_index, payload_reader, payload_size) = + open_dynamic_index_reader(&backup_reader, &manifest, &payload_archive_name, params) + .await?; + ( + Some(payload_index), + pxar::PxarVariant::Split(reader, (payload_reader, payload_size)), + ) + } else { + (None, pxar::PxarVariant::Unified(reader)) + }; + + let accessor = Accessor::new(variant, archive_size).await?; + + Ok(ArchiveReader { + index, + payload_index, + accessor, + }) } async fn create_backup_reader( @@ -334,85 +376,167 @@ fn chunk_digests_for_index(index: &dyn IndexFile) -> Vec<&ChunkDigest> { all_chunks } -/// Compute which files are contained in a given chunk set. -async fn files_in_chunk_set<'c, 'f>( - chunk_list: &[&'c ChunkDigest], - accessor: &'f Accessor, - index: &'f DynamicIndexReader, - chunk_set: &HashSet<&'c ChunkDigest>, -) -> Result, Error> { - let path = PathBuf::new(); - let root = accessor.open_root().await?; +/// Compute the symmetric difference between two lists of chunk digests. +/// Content stored in those chunks was either added, modified or deleted. +fn compute_chunk_difference<'a>( + chunks_a: &[&'a ChunkDigest], + chunks_b: &[&'a ChunkDigest], +) -> HashSet<&'a ChunkDigest> { + let set_a: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_a.iter().copied()); + let set_b: HashSet<&ChunkDigest> = HashSet::from_iter(chunks_b.iter().copied()); + + set_a.symmetric_difference(&set_b).copied().collect() +} + +struct ChunkDiff<'a, 'c> { + index: &'a DynamicIndexReader, + chunk_list: &'a [&'c ChunkDigest], + chunk_diff: &'a HashSet<&'c ChunkDigest>, +} + +impl<'a, 'c> ChunkDiff<'a, 'c> { + fn new( + index: &'a DynamicIndexReader, + chunk_list: &'a [&'c ChunkDigest], + chunk_diff: &'a HashSet<&'c ChunkDigest>, + ) -> Self { + Self { + index, + chunk_list, + chunk_diff, + } + } + + fn contains_chunks_within_range(&self, range: &Range) -> Result { + if range.is_empty() { + return Ok(false); + } + + let first_chunk = self + .index + .chunk_from_offset(range.start) + .context("Invalid offset")? + .chunk_pos; + let last_chunk = self + .index + .chunk_from_offset(range.end - 1) + .context("Invalid offset")? + .chunk_pos; + + for chunk_index in first_chunk..=last_chunk { + let digest = self + .chunk_list + .get(chunk_index) + .context("Invalid chunk index")?; + if self.chunk_diff.contains(digest) { + return Ok(true); + } + } - visit_directory(&root, index, &path, chunk_list, chunk_set).await + Ok(false) + } + + fn is_empty(&self) -> bool { + self.chunk_diff.is_empty() + } } -fn range_contains_changed_chunk( - range: &Range, - index: &dyn IndexFile, - chunk_list: &[&ChunkDigest], - chunk_diff: &HashSet<&ChunkDigest>, -) -> Result { - if range.is_empty() { - return Ok(false); - } - - let first_chunk = index - .chunk_from_offset(range.start) - .context("Invalid offset")? - .chunk_pos; - let last_chunk = index - .chunk_from_offset(range.end - 1) - .context("Invalid offset")? - .chunk_pos; - - for chunk_index in first_chunk..=last_chunk { - let digest = chunk_list.get(chunk_index).context("Invalid chunk index")?; - if chunk_diff.contains(digest) { - return Ok(true); +struct ArchiveDiff<'a, 'c> { + metadata_diff: ChunkDiff<'a, 'c>, + payload_diff: Option>, + compare_contents: bool, +} + +impl<'a, 'c> ArchiveDiff<'a, 'c> { + fn new( + metadata_diff: ChunkDiff<'a, 'c>, + payload_diff: Option>, + compare_contents: bool, + ) -> Self { + Self { + metadata_diff, + payload_diff, + compare_contents, } } +} + +/// Compute which files are contained in a given chunk set. +async fn files_in_chunk_set<'f, 'a, 'c>( + accessor: &'f Accessor, + archive_diff: &'f ArchiveDiff<'a, 'c>, +) -> Result, Error> { + let path = PathBuf::new(); + let root = accessor.open_root().await?; - Ok(false) + visit_directory(&root, &path, archive_diff).await } /// Recursively visits directories in .pxar archive and create a /// map "digest --> set of contained files" -fn visit_directory<'f, 'c>( +fn visit_directory<'f, 'a, 'c>( directory: &'f Directory, - index: &'f DynamicIndexReader, path: &'f Path, - chunk_list: &'f [&'c ChunkDigest], - chunk_diff: &'f HashSet<&'c ChunkDigest>, + archive_diff: &'f ArchiveDiff<'a, 'c>, ) -> BoxFuture<'f, Result, Error>> { async move { let mut entries: HashMap = HashMap::new(); let mut iter = directory.read_dir(); + // In split pxar archives, payload changes with identical size and metadata do not + // necessarily lead to changes in metadata archive chunks, so directories cannot be skipped + // when comparing contents if no difference in metadata chunks has been detected. + let descend_unchanged_dirs = archive_diff.compare_contents + && archive_diff + .payload_diff + .as_ref() + .is_some_and(|payload_diff| !payload_diff.is_empty()); while let Some(entry) = iter.next().await { let entry = entry?.decode_entry().await?; let range = &entry.entry_range_info().entry_range; // Check if any chunk of the entry is in the set of modified chunks (symmetric - // difference). If not, we can skip directories entirely and save a lot of time. - let entry_changed = range_contains_changed_chunk(range, index, chunk_list, chunk_diff)?; - - if !entry_changed { - continue; - } + // difference). If not, we can skip directories entirely unless payload diffs + // require checking file contents. + let archive_changed = archive_diff + .metadata_diff + .contains_chunks_within_range(range) + .with_context(|| { + format!( + "failed to check archive chunk range {range:?} on entry {:#?}", + entry.file_name(), + ) + })?; - if entry.is_dir() { + if entry.is_dir() && (archive_changed || descend_unchanged_dirs) { let new_dir = entry.enter_directory().await?; let dir_path = path.join(entry.file_name()); - entries.extend( - visit_directory(&new_dir, index, &dir_path, chunk_list, chunk_diff).await?, - ); + entries.extend(visit_directory(&new_dir, &dir_path, archive_diff).await?); + } + + let mut payload_changed = false; + if archive_diff.compare_contents { + if let Some(payload_diff) = archive_diff.payload_diff.as_ref() { + if let Some(content_range) = entry.content_range()? { + let range = content_range.as_byte_range(); + payload_changed = payload_diff + .contains_chunks_within_range(range) + .with_context(|| { + format!( + "failed to check payload archive chunk range {range:?} on entry {:#?}", + entry.file_name(), + ) + })? + } + } } - let file_path = path.join(entry.file_name()); - entries.insert(file_path.into_os_string(), entry); + if archive_changed || payload_changed { + let file_path = path.join(entry.file_name()); + entries.insert(file_path.into_os_string(), entry); + } } Ok(entries) -- 2.47.3