[PATCH v1 02/11] datastore: expose request counters for s3 backed datastores

[Christian Ebner <c.ebner@proxmox.com>]

Allows to introspect the current state of the request counters related
to a datastore. With the intention to show the request counter
statistics in the ui and allow to use them for soft limits and warnings
via the notification system in the future.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-datastore/src/datastore.rs | 48 ++++++++++++++++++++++++++++++----
 src/api2/admin/datastore.rs    |  4 +++
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 957e900d6..bbf4d30f4 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -3,12 +3,14 @@ use std::io::{self, Write};
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::io::AsRawFd;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::Ordering;
 use std::sync::{Arc, LazyLock, Mutex};
 use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, format_err, Context, Error};
 use http_body_util::BodyExt;
 use hyper::body::Bytes;
+use hyper::Method;
 use nix::unistd::{unlinkat, UnlinkatFlags};
 use pbs_tools::lru_cache::LruCache;
 use tokio::io::AsyncWriteExt;
@@ -17,7 +19,7 @@ use tracing::{info, warn};
 use proxmox_human_byte::HumanByte;
 use proxmox_s3_client::{
     S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions,
-    S3RequestCounterOptions,
+    S3RequestCounterOptions, SharedRequestCounters,
 };
 use proxmox_schema::ApiType;
 
@@ -32,7 +34,7 @@ use pbs_api_types::{
     ArchiveType, Authid, BackupGroupDeleteStats, BackupNamespace, BackupType, ChunkOrder,
     DataStoreConfig, DatastoreBackendConfig, DatastoreBackendType, DatastoreFSyncLevel,
     DatastoreTuning, GarbageCollectionCacheStats, GarbageCollectionStatus, MaintenanceMode,
-    MaintenanceType, Operation, UPID,
+    MaintenanceType, Operation, S3Statistics, UPID,
 };
 use pbs_config::s3::S3_CFG_TYPE_ID;
 use pbs_config::{BackupLockGuard, ConfigVersionCache};
@@ -177,6 +179,7 @@ pub struct DataStoreImpl {
     /// datastore.cfg cache generation number at lookup time, used to
     /// invalidate this cached `DataStoreImpl`
     config_generation: Option<usize>,
+    request_counters: Option<Arc<SharedRequestCounters>>,
 }
 
 impl DataStoreImpl {
@@ -194,6 +197,7 @@ impl DataStoreImpl {
             lru_store_caching: None,
             thread_settings: Default::default(),
             config_generation: None,
+            request_counters: None,
         })
     }
 }
@@ -451,6 +455,22 @@ impl DataStore {
         Ok(backend_type)
     }
 
+    /// Get the s3 statistics for this datastore
+    pub fn s3_statistics(&self) -> Option<S3Statistics> {
+        self.inner
+            .request_counters
+            .as_ref()
+            .map(|counters| S3Statistics {
+                get: counters.load(Method::GET, Ordering::SeqCst),
+                put: counters.load(Method::PUT, Ordering::SeqCst),
+                post: counters.load(Method::POST, Ordering::SeqCst),
+                delete: counters.load(Method::DELETE, Ordering::SeqCst),
+                head: counters.load(Method::HEAD, Ordering::SeqCst),
+                uploaded: counters.get_upload_traffic(Ordering::SeqCst),
+                downloaded: counters.get_download_traffic(Ordering::SeqCst),
+            })
+    }
+
     pub fn cache(&self) -> Option<&LocalDatastoreLruCache> {
         self.inner.lru_store_caching.as_ref()
     }
@@ -654,7 +674,8 @@ impl DataStore {
                 .parse_property_string(config.backend.as_deref().unwrap_or(""))?,
         )?;
 
-        let lru_store_caching = if DatastoreBackendType::S3 == backend_config.ty.unwrap_or_default()
+        let (lru_store_caching, request_counters) = if DatastoreBackendType::S3
+            == backend_config.ty.unwrap_or_default()
         {
             let mut cache_capacity = 0;
             if let Ok(fs_info) = proxmox_sys::fs::fs_info(&chunk_store.base_path()) {
@@ -676,9 +697,25 @@ impl DataStore {
             );
 
             let cache = LocalDatastoreLruCache::new(cache_capacity, chunk_store.clone());
-            Some(cache)
+
+            let path = format!(
+                "{}/{}-{}-{}.shmem",
+                S3_CLIENT_REQUEST_COUNTER_BASE_PATH,
+                backend_config
+                    .client
+                    .as_ref()
+                    .ok_or(format_err!("missing s3 endpoint id"))?,
+                backend_config
+                    .bucket
+                    .as_ref()
+                    .ok_or(format_err!("missing s3 bucket"))?,
+                config.name,
+            );
+            let request_counters =
+                SharedRequestCounters::open_shared_memory_mapped(path, pbs_config::backup_user()?)?;
+            (Some(cache), Some(Arc::new(request_counters)))
         } else {
-            None
+            (None, None)
         };
 
         let thread_settings = DatastoreThreadSettings::new(
@@ -697,6 +734,7 @@ impl DataStore {
             lru_store_caching,
             thread_settings,
             config_generation: generation,
+            request_counters,
         })
     }
 
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 88ad5d53b..d71112475 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -622,6 +622,8 @@ pub async fn status(
         (None, None)
     };
 
+    let s3_statistics = datastore.s3_statistics();
+
     Ok(if store_stats {
         let storage = crate::tools::fs::fs_info(datastore.base_path()).await?;
         DataStoreStatus {
@@ -630,6 +632,7 @@ pub async fn status(
             avail: storage.available,
             gc_status,
             counts,
+            s3_statistics,
         }
     } else {
         DataStoreStatus {
@@ -638,6 +641,7 @@ pub async fn status(
             avail: 0,
             gc_status,
             counts,
+            s3_statistics,
         }
     })
 }
-- 
2.47.3