[PATCH proxmox-backup v2 09/14] fix #5799: GC: track chunk digests and accumulate statistics

[Christian Ebner <c.ebner@proxmox.com>]

Keep track of all digests referenced by snapshots index files
encountered during phase 1 of garbage collection in the chunk
map for accounting and fill in raw chunk size information
during phase 2.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-datastore/src/chunk_store.rs |  9 ++++++
 pbs-datastore/src/datastore.rs   | 54 ++++++++++++++++++++++++++++++--
 2 files changed, 60 insertions(+), 3 deletions(-)

diff --git a/pbs-datastore/src/chunk_store.rs b/pbs-datastore/src/chunk_store.rs
index ec657092e..a7dea91ee 100644
--- a/pbs-datastore/src/chunk_store.rs
+++ b/pbs-datastore/src/chunk_store.rs
@@ -19,6 +19,7 @@ use proxmox_sys::process_locker::{
 use proxmox_worker_task::WorkerTaskContext;
 
 use crate::backup_info::DATASTORE_LOCKS_DIR;
+use crate::chunk_map::ChunkMap;
 use crate::data_blob::DataChunkBuilder;
 use crate::file_formats::{
     COMPRESSED_BLOB_MAGIC_1_0, ENCRYPTED_BLOB_MAGIC_1_0, UNCOMPRESSED_BLOB_MAGIC_1_0,
@@ -441,6 +442,7 @@ impl ChunkStore {
         status: &mut GarbageCollectionStatus,
         worker: &dyn WorkerTaskContext,
         cache: Option<&LocalDatastoreLruCache>,
+        mut chunk_map: Option<&mut ChunkMap>,
     ) -> Result<(), Error> {
         // unwrap: only `None` in unit tests
         assert!(self.locker.is_some());
@@ -531,6 +533,13 @@ impl ChunkStore {
                         },
                     )?;
                 }
+
+                if chunk_ext == ChunkExt::None {
+                    if let Some(ref mut chunk_map) = chunk_map {
+                        let digest = <[u8; 32]>::from_hex(filename.to_bytes())?;
+                        chunk_map.set_raw_chunk_size(&digest, stat.st_size as u64);
+                    }
+                }
             }
             drop(lock);
         }
diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 941e74437..f568ea2b3 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -43,6 +43,7 @@ use proxmox_section_config::SectionConfigData;
 use crate::backup_info::{
     BackupDir, BackupGroup, BackupInfo, OLD_LOCKING, PROTECTED_MARKER_FILENAME,
 };
+use crate::chunk_map::ChunkMap;
 use crate::chunk_store::{ChunkStore, is_bad_chunk_suffix};
 use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
 use crate::fixed_index::{FixedIndexReader, FixedIndexWriter};
@@ -2117,6 +2118,7 @@ impl DataStore {
         index: Box<dyn IndexFile>,
         file_name: &Path, // only used for error reporting
         chunk_lru_cache: &mut Option<LruCache<[u8; 32], ()>>,
+        mut chunk_map_info: Option<ChunkMapInfo>,
         status: &mut GarbageCollectionStatus,
         worker: &dyn WorkerTaskContext,
         s3_client: Option<Arc<S3Client>>,
@@ -2127,7 +2129,13 @@ impl DataStore {
         for pos in 0..index.index_count() {
             worker.check_abort()?;
             worker.fail_on_shutdown()?;
-            let digest = index.index_digest(pos).unwrap();
+            let chunk_info = index.chunk_info(pos).unwrap();
+            let digest = &chunk_info.digest;
+
+            if let Some(map) = chunk_map_info.as_mut() {
+                map.digests
+                    .insert(digest, map.namespace, map.snapshot, chunk_info.size());
+            }
 
             // Avoid multiple expensive atime updates by utimensat
             if let Some(chunk_lru_cache) = chunk_lru_cache {
@@ -2171,12 +2179,13 @@ impl DataStore {
         Ok(())
     }
 
-    fn mark_used_chunks(
+    fn mark_used_chunks<'a>(
         &self,
         status: &mut GarbageCollectionStatus,
         worker: &dyn WorkerTaskContext,
         cache_capacity: usize,
         s3_client: Option<Arc<S3Client>>,
+        mut chunk_map: Option<&mut ChunkMap<'a>>,
     ) -> Result<(), Error> {
         // Iterate twice over the datastore to fetch index files, even if this comes with an
         // additional runtime cost:
@@ -2206,7 +2215,7 @@ impl DataStore {
             .context("creating namespace iterator failed")?
         {
             let namespace = namespace.context("iterating namespaces failed")?;
-            for group in arc_self.iter_backup_groups(namespace)? {
+            for group in arc_self.iter_backup_groups(namespace.clone())? {
                 let group = group.context("iterating backup groups failed")?;
 
                 // Avoid race between listing/marking of snapshots by GC and pruning the last
@@ -2264,10 +2273,21 @@ impl DataStore {
                                 }
                             };
 
+                            let digest_map = if let Some(digests) = chunk_map.as_mut() {
+                                Some(ChunkMapInfo {
+                                    digests,
+                                    namespace: &namespace,
+                                    snapshot: snapshot.backup_dir.dir(),
+                                })
+                            } else {
+                                None
+                            };
+
                             self.index_mark_used_chunks(
                                 index,
                                 &path,
                                 &mut chunk_lru_cache,
+                                digest_map,
                                 status,
                                 worker,
                                 s3_client.as_ref().cloned(),
@@ -2311,6 +2331,7 @@ impl DataStore {
                 index,
                 &path,
                 &mut chunk_lru_cache,
+                None,
                 status,
                 worker,
                 s3_client.as_ref().cloned(),
@@ -2333,6 +2354,7 @@ impl DataStore {
                 index,
                 path,
                 &mut chunk_lru_cache,
+                None,
                 status,
                 worker,
                 s3_client.as_ref().cloned(),
@@ -2461,6 +2483,13 @@ impl DataStore {
             1024 * 1024
         };
 
+        let mut chunk_map = if gc_store_config.extended_usage_statistics {
+            info!("Enabled extended usage statistics gathering.");
+            Some(ChunkMap::with_datastore(self))
+        } else {
+            None
+        };
+
         info!("Start GC phase1 (mark used chunks)");
 
         self.mark_used_chunks(
@@ -2468,6 +2497,7 @@ impl DataStore {
             worker,
             gc_cache_capacity,
             s3_client.as_ref().cloned(),
+            chunk_map.as_mut(),
         )
         .context("marking used chunks failed")?;
 
@@ -2493,6 +2523,10 @@ impl DataStore {
                             None => continue,
                         };
 
+                    if let Some(map) = chunk_map.as_mut() {
+                        map.set_raw_chunk_size(&digest, content.size);
+                    }
+
                     let timeout = std::time::Duration::from_secs(0);
                     let _chunk_guard = match self.inner.chunk_store.lock_chunk(&digest, timeout) {
                         Ok(guard) => guard,
@@ -2589,6 +2623,7 @@ impl DataStore {
                 &mut tmp_gc_status,
                 worker,
                 self.cache(),
+                None,
             )?;
         } else {
             self.inner.chunk_store.sweep_unused_chunks(
@@ -2597,6 +2632,7 @@ impl DataStore {
                 &mut gc_status,
                 worker,
                 None,
+                chunk_map.as_mut(),
             )?;
         }
 
@@ -2610,6 +2646,12 @@ impl DataStore {
                 );
             }
         }
+
+        if let Some(map) = chunk_map.take() {
+            map.accumulate_and_persist()
+                .context("failed to accumulate and persists usage statistics")?;
+        }
+
         info!(
             "Removed garbage: {}",
             HumanByte::from(gc_status.removed_bytes),
@@ -3577,3 +3619,9 @@ impl S3DeleteList {
         Ok(())
     }
 }
+
+struct ChunkMapInfo<'a, 'b> {
+    digests: &'a mut ChunkMap<'b>,
+    namespace: &'a pbs_api_types::BackupNamespace,
+    snapshot: &'a pbs_api_types::BackupDir,
+}
-- 
2.47.3