[PATCH proxmox-backup v7 15/34] datastore: collect request statistics for s3 backed datastores

[Christian Ebner <c.ebner@proxmox.com>]

Define and provide the s3 request counter options to the s3 client
invocations so all requests and traffic is being tracked. When no
datastore is involved, account the statistics for the bucket instead.

Bucket or even endpoint wide statistics might then be gathered in the
future by parsing the corresponding memory mapped files and collecting
their contents.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
Reviewed-by: Hannes Laimer <h.laimer@proxmox.com>
Tested-by: Hannes Laimer <h.laimer@proxmox.com>
---
 pbs-datastore/src/datastore.rs | 15 +++++++++++++++
 pbs-datastore/src/lib.rs       |  2 +-
 src/api2/admin/s3.rs           | 17 +++++++++++++++--
 src/api2/config/s3.rs          |  9 ++++++++-
 4 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index ef378c696..c772e33c6 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -17,6 +17,7 @@ use tracing::{info, warn};
 use proxmox_human_byte::HumanByte;
 use proxmox_s3_client::{
     S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions,
+    S3RequestCounterConfig,
 };
 use proxmox_schema::ApiType;
 
@@ -75,6 +76,8 @@ pub const GROUP_NOTES_FILE_NAME: &str = "notes";
 pub const GROUP_OWNER_FILE_NAME: &str = "owner";
 /// Filename for in-use marker stored on S3 object store backend
 pub const S3_DATASTORE_IN_USE_MARKER: &str = ".in-use";
+/// Base directory for storing shared memory mapped s3 request counters
+pub const S3_CLIENT_REQUEST_COUNTER_BASE_PATH: &str = "/var/lib/proxmox-backup/s3-statistics";
 const S3_CLIENT_RATE_LIMITER_BASE_PATH: &str = pbs_buildcfg::rundir!("/s3/shmem/tbf");
 const NAMESPACE_MARKER_FILENAME: &str = ".namespace";
 // s3 put request times out after upload_size / 1 Kib/s, so about 2.3 hours for 8 MiB
@@ -426,6 +429,11 @@ impl DataStore {
                     user: pbs_config::backup_user()?,
                     base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
                 };
+                let request_counter_config = S3RequestCounterConfig {
+                    id: format!("{s3_client_id}-{bucket}-{}", self.name()),
+                    user: pbs_config::backup_user()?,
+                    base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+                };
 
                 let options = S3ClientOptions::from_config(
                     config.config,
@@ -434,6 +442,7 @@ impl DataStore {
                     self.name().to_owned(),
                     Some(rate_limiter_options),
                     None, //FIXME read from node.cfg
+                    Some(request_counter_config),
                 );
                 let s3_client = S3Client::new(options)?;
                 DatastoreBackend::S3(Arc::new(s3_client))
@@ -2652,6 +2661,11 @@ impl DataStore {
             user: pbs_config::backup_user()?,
             base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
         };
+        let request_counter_config = S3RequestCounterConfig {
+            id: format!("{s3_client_id}-{bucket}-{}", datastore_config.name),
+            user: pbs_config::backup_user()?,
+            base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+        };
 
         let options = S3ClientOptions::from_config(
             client_config.config,
@@ -2660,6 +2674,7 @@ impl DataStore {
             datastore_config.name.to_owned(),
             Some(rate_limiter_options),
             None, // FIXME read from node.cfg
+            Some(request_counter_config),
         );
         let s3_client = S3Client::new(options)
             .context("failed to create s3 client")
diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
index 1f7c54ae8..afe340a65 100644
--- a/pbs-datastore/src/lib.rs
+++ b/pbs-datastore/src/lib.rs
@@ -217,7 +217,7 @@ pub use store_progress::StoreProgress;
 mod datastore;
 pub use datastore::{
     check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore,
-    DatastoreBackend, S3_DATASTORE_IN_USE_MARKER,
+    DatastoreBackend, S3_CLIENT_REQUEST_COUNTER_BASE_PATH, S3_DATASTORE_IN_USE_MARKER,
 };
 
 mod hierarchy;
diff --git a/src/api2/admin/s3.rs b/src/api2/admin/s3.rs
index d3b3b11e6..1e098b4a1 100644
--- a/src/api2/admin/s3.rs
+++ b/src/api2/admin/s3.rs
@@ -6,8 +6,8 @@ use serde_json::Value;
 use proxmox_http::Body;
 use proxmox_router::{list_subdirs_api_method, Permission, Router, RpcEnvironment, SubdirMap};
 use proxmox_s3_client::{
-    S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3_BUCKET_NAME_SCHEMA,
-    S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT,
+    S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3RequestCounterConfig,
+    S3_BUCKET_NAME_SCHEMA, S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT,
 };
 use proxmox_schema::*;
 use proxmox_sortable_macro::sortable;
@@ -15,6 +15,7 @@ use proxmox_sortable_macro::sortable;
 use pbs_api_types::PRIV_SYS_MODIFY;
 
 use pbs_config::s3::S3_CFG_TYPE_ID;
+use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH;
 
 #[api(
     input: {
@@ -48,6 +49,17 @@ pub async fn check(
         .lookup(S3_CFG_TYPE_ID, &s3_client_id)
         .context("config lookup failed")?;
 
+    let request_counter_id = if let Some(store) = &store_prefix {
+        format!("{s3_client_id}-{bucket}-{store}")
+    } else {
+        format!("{s3_client_id}-{bucket}")
+    };
+    let request_counter_config = S3RequestCounterConfig {
+        id: request_counter_id,
+        user: pbs_config::backup_user()?,
+        base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+    };
+
     let store_prefix = store_prefix.unwrap_or_default();
     let options = S3ClientOptions::from_config(
         config.config,
@@ -56,6 +68,7 @@ pub async fn check(
         store_prefix,
         None,
         None, // FIXME read from node.cfg once regular datastore operations do as well
+        Some(request_counter_config),
     );
 
     let test_object_key =
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
index 9f1b4b5f7..29ac1038a 100644
--- a/src/api2/config/s3.rs
+++ b/src/api2/config/s3.rs
@@ -6,7 +6,7 @@ use serde_json::Value;
 use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
 use proxmox_s3_client::{
     S3BucketListItem, S3Client, S3ClientConf, S3ClientConfig, S3ClientConfigUpdater,
-    S3ClientConfigWithoutSecret, S3ClientOptions, S3_CLIENT_ID_SCHEMA,
+    S3ClientConfigWithoutSecret, S3ClientOptions, S3RequestCounterConfig, S3_CLIENT_ID_SCHEMA,
 };
 use proxmox_schema::{api, param_bail, ApiType};
 
@@ -16,6 +16,7 @@ use pbs_api_types::{
 };
 use pbs_config::s3::{self, S3_CFG_TYPE_ID};
 use pbs_config::CachedUserInfo;
+use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH;
 
 #[api(
     input: {
@@ -350,6 +351,11 @@ pub async fn list_buckets(
         .context("config lookup failed")?;
 
     let empty_prefix = String::new();
+    let request_counter_config = S3RequestCounterConfig {
+        id,
+        user: pbs_config::backup_user()?,
+        base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+    };
     let options = S3ClientOptions::from_config(
         config.config,
         config.secret_key,
@@ -357,6 +363,7 @@ pub async fn list_buckets(
         empty_prefix,
         None,
         None, // FIXME read from node.cfg once regular datastore operations do as well
+        Some(request_counter_config),
     );
     let client = S3Client::new(options).context("client creation failed")?;
     let list_buckets_response = client
-- 
2.47.3