From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id BBC651FF0B3 for ; Wed, 14 Oct 2026 16:44:28 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3D0B52141B; Wed, 14 Oct 2026 16:44:13 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v2 09/14] fix #5799: GC: track chunk digests and accumulate statistics Date: Wed, 14 Oct 2026 16:43:52 +0200 Message-ID: <20261014144357.771051-10-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261014144357.771051-1-c.ebner@proxmox.com> References: <20261014144357.771051-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1791989043076 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.522 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: IWBS7NMFHWSMD7WNVRKT544E6N3SVHFH X-Message-ID-Hash: IWBS7NMFHWSMD7WNVRKT544E6N3SVHFH X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Keep track of all digests referenced by snapshots index files encountered during phase 1 of garbage collection in the chunk map for accounting and fill in raw chunk size information during phase 2. Signed-off-by: Christian Ebner --- pbs-datastore/src/chunk_store.rs | 9 ++++++ pbs-datastore/src/datastore.rs | 54 ++++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/pbs-datastore/src/chunk_store.rs b/pbs-datastore/src/chunk_store.rs index ec657092e..a7dea91ee 100644 --- a/pbs-datastore/src/chunk_store.rs +++ b/pbs-datastore/src/chunk_store.rs @@ -19,6 +19,7 @@ use proxmox_sys::process_locker::{ use proxmox_worker_task::WorkerTaskContext; use crate::backup_info::DATASTORE_LOCKS_DIR; +use crate::chunk_map::ChunkMap; use crate::data_blob::DataChunkBuilder; use crate::file_formats::{ COMPRESSED_BLOB_MAGIC_1_0, ENCRYPTED_BLOB_MAGIC_1_0, UNCOMPRESSED_BLOB_MAGIC_1_0, @@ -441,6 +442,7 @@ impl ChunkStore { status: &mut GarbageCollectionStatus, worker: &dyn WorkerTaskContext, cache: Option<&LocalDatastoreLruCache>, + mut chunk_map: Option<&mut ChunkMap>, ) -> Result<(), Error> { // unwrap: only `None` in unit tests assert!(self.locker.is_some()); @@ -531,6 +533,13 @@ impl ChunkStore { }, )?; } + + if chunk_ext == ChunkExt::None { + if let Some(ref mut chunk_map) = chunk_map { + let digest = <[u8; 32]>::from_hex(filename.to_bytes())?; + chunk_map.set_raw_chunk_size(&digest, stat.st_size as u64); + } + } } drop(lock); } diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 941e74437..f568ea2b3 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -43,6 +43,7 @@ use proxmox_section_config::SectionConfigData; use crate::backup_info::{ BackupDir, BackupGroup, BackupInfo, OLD_LOCKING, PROTECTED_MARKER_FILENAME, }; +use crate::chunk_map::ChunkMap; use crate::chunk_store::{ChunkStore, is_bad_chunk_suffix}; use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; use crate::fixed_index::{FixedIndexReader, FixedIndexWriter}; @@ -2117,6 +2118,7 @@ impl DataStore { index: Box, file_name: &Path, // only used for error reporting chunk_lru_cache: &mut Option>, + mut chunk_map_info: Option, status: &mut GarbageCollectionStatus, worker: &dyn WorkerTaskContext, s3_client: Option>, @@ -2127,7 +2129,13 @@ impl DataStore { for pos in 0..index.index_count() { worker.check_abort()?; worker.fail_on_shutdown()?; - let digest = index.index_digest(pos).unwrap(); + let chunk_info = index.chunk_info(pos).unwrap(); + let digest = &chunk_info.digest; + + if let Some(map) = chunk_map_info.as_mut() { + map.digests + .insert(digest, map.namespace, map.snapshot, chunk_info.size()); + } // Avoid multiple expensive atime updates by utimensat if let Some(chunk_lru_cache) = chunk_lru_cache { @@ -2171,12 +2179,13 @@ impl DataStore { Ok(()) } - fn mark_used_chunks( + fn mark_used_chunks<'a>( &self, status: &mut GarbageCollectionStatus, worker: &dyn WorkerTaskContext, cache_capacity: usize, s3_client: Option>, + mut chunk_map: Option<&mut ChunkMap<'a>>, ) -> Result<(), Error> { // Iterate twice over the datastore to fetch index files, even if this comes with an // additional runtime cost: @@ -2206,7 +2215,7 @@ impl DataStore { .context("creating namespace iterator failed")? { let namespace = namespace.context("iterating namespaces failed")?; - for group in arc_self.iter_backup_groups(namespace)? { + for group in arc_self.iter_backup_groups(namespace.clone())? { let group = group.context("iterating backup groups failed")?; // Avoid race between listing/marking of snapshots by GC and pruning the last @@ -2264,10 +2273,21 @@ impl DataStore { } }; + let digest_map = if let Some(digests) = chunk_map.as_mut() { + Some(ChunkMapInfo { + digests, + namespace: &namespace, + snapshot: snapshot.backup_dir.dir(), + }) + } else { + None + }; + self.index_mark_used_chunks( index, &path, &mut chunk_lru_cache, + digest_map, status, worker, s3_client.as_ref().cloned(), @@ -2311,6 +2331,7 @@ impl DataStore { index, &path, &mut chunk_lru_cache, + None, status, worker, s3_client.as_ref().cloned(), @@ -2333,6 +2354,7 @@ impl DataStore { index, path, &mut chunk_lru_cache, + None, status, worker, s3_client.as_ref().cloned(), @@ -2461,6 +2483,13 @@ impl DataStore { 1024 * 1024 }; + let mut chunk_map = if gc_store_config.extended_usage_statistics { + info!("Enabled extended usage statistics gathering."); + Some(ChunkMap::with_datastore(self)) + } else { + None + }; + info!("Start GC phase1 (mark used chunks)"); self.mark_used_chunks( @@ -2468,6 +2497,7 @@ impl DataStore { worker, gc_cache_capacity, s3_client.as_ref().cloned(), + chunk_map.as_mut(), ) .context("marking used chunks failed")?; @@ -2493,6 +2523,10 @@ impl DataStore { None => continue, }; + if let Some(map) = chunk_map.as_mut() { + map.set_raw_chunk_size(&digest, content.size); + } + let timeout = std::time::Duration::from_secs(0); let _chunk_guard = match self.inner.chunk_store.lock_chunk(&digest, timeout) { Ok(guard) => guard, @@ -2589,6 +2623,7 @@ impl DataStore { &mut tmp_gc_status, worker, self.cache(), + None, )?; } else { self.inner.chunk_store.sweep_unused_chunks( @@ -2597,6 +2632,7 @@ impl DataStore { &mut gc_status, worker, None, + chunk_map.as_mut(), )?; } @@ -2610,6 +2646,12 @@ impl DataStore { ); } } + + if let Some(map) = chunk_map.take() { + map.accumulate_and_persist() + .context("failed to accumulate and persists usage statistics")?; + } + info!( "Removed garbage: {}", HumanByte::from(gc_status.removed_bytes), @@ -3577,3 +3619,9 @@ impl S3DeleteList { Ok(()) } } + +struct ChunkMapInfo<'a, 'b> { + digests: &'a mut ChunkMap<'b>, + namespace: &'a pbs_api_types::BackupNamespace, + snapshot: &'a pbs_api_types::BackupDir, +} -- 2.47.3