[PATCH proxmox-backup v4 02/22] datastore: collect request statistics for s3 backed datastores

[Christian Ebner <c.ebner@proxmox.com>]

Define and provide the s3 request counter options to the s3 client
invocations so all requests and traffic is being tracked. When no
datastore is involved, account the statistics for the bucket instead.

Bucket or even endpoint wide statistics might then be gathered in the
future by parsing the corresponding memory mapped files and collecting
their contents.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-datastore/src/datastore.rs | 15 +++++++++++++++
 pbs-datastore/src/lib.rs       |  2 +-
 src/api2/admin/s3.rs           | 17 +++++++++++++++--
 src/api2/config/s3.rs          | 18 +++++++++++++++---
 4 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 7ad3d917d..957e900d6 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -17,6 +17,7 @@ use tracing::{info, warn};
 use proxmox_human_byte::HumanByte;
 use proxmox_s3_client::{
     S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions,
+    S3RequestCounterOptions,
 };
 use proxmox_schema::ApiType;
 
@@ -75,6 +76,8 @@ pub const GROUP_NOTES_FILE_NAME: &str = "notes";
 pub const GROUP_OWNER_FILE_NAME: &str = "owner";
 /// Filename for in-use marker stored on S3 object store backend
 pub const S3_DATASTORE_IN_USE_MARKER: &str = ".in-use";
+/// Base directory for storing shared memory mapped s3 request counters
+pub const S3_CLIENT_REQUEST_COUNTER_BASE_PATH: &str = "/var/lib/proxmox-backup/s3-statistics";
 const S3_CLIENT_RATE_LIMITER_BASE_PATH: &str = pbs_buildcfg::rundir!("/s3/shmem/tbf");
 const NAMESPACE_MARKER_FILENAME: &str = ".namespace";
 // s3 put request times out after upload_size / 1 Kib/s, so about 2.3 hours for 8 MiB
@@ -426,6 +429,11 @@ impl DataStore {
                     user: pbs_config::backup_user()?,
                     base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
                 };
+                let request_counter_options = S3RequestCounterOptions {
+                    id: format!("{s3_client_id}-{bucket}-{}", self.name()),
+                    user: pbs_config::backup_user()?,
+                    base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+                };
 
                 let options = S3ClientOptions::from_config(
                     config.config,
@@ -433,6 +441,7 @@ impl DataStore {
                     Some(bucket),
                     self.name().to_owned(),
                     Some(rate_limiter_options),
+                    Some(request_counter_options),
                 );
                 let s3_client = S3Client::new(options)?;
                 DatastoreBackend::S3(Arc::new(s3_client))
@@ -2659,6 +2668,11 @@ impl DataStore {
             user: pbs_config::backup_user()?,
             base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(),
         };
+        let request_counter_options = S3RequestCounterOptions {
+            id: format!("{s3_client_id}-{bucket}-{}", datastore_config.name),
+            user: pbs_config::backup_user()?,
+            base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+        };
 
         let options = S3ClientOptions::from_config(
             client_config.config,
@@ -2666,6 +2680,7 @@ impl DataStore {
             Some(bucket),
             datastore_config.name.to_owned(),
             Some(rate_limiter_options),
+            Some(request_counter_options),
         );
         let s3_client = S3Client::new(options)
             .context("failed to create s3 client")
diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs
index 1f7c54ae8..afe340a65 100644
--- a/pbs-datastore/src/lib.rs
+++ b/pbs-datastore/src/lib.rs
@@ -217,7 +217,7 @@ pub use store_progress::StoreProgress;
 mod datastore;
 pub use datastore::{
     check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore,
-    DatastoreBackend, S3_DATASTORE_IN_USE_MARKER,
+    DatastoreBackend, S3_CLIENT_REQUEST_COUNTER_BASE_PATH, S3_DATASTORE_IN_USE_MARKER,
 };
 
 mod hierarchy;
diff --git a/src/api2/admin/s3.rs b/src/api2/admin/s3.rs
index 73388281b..d20cae483 100644
--- a/src/api2/admin/s3.rs
+++ b/src/api2/admin/s3.rs
@@ -6,8 +6,8 @@ use serde_json::Value;
 use proxmox_http::Body;
 use proxmox_router::{list_subdirs_api_method, Permission, Router, RpcEnvironment, SubdirMap};
 use proxmox_s3_client::{
-    S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3_BUCKET_NAME_SCHEMA,
-    S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT,
+    S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3RequestCounterOptions,
+    S3_BUCKET_NAME_SCHEMA, S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT,
 };
 use proxmox_schema::*;
 use proxmox_sortable_macro::sortable;
@@ -15,6 +15,7 @@ use proxmox_sortable_macro::sortable;
 use pbs_api_types::PRIV_SYS_MODIFY;
 
 use pbs_config::s3::S3_CFG_TYPE_ID;
+use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH;
 
 #[api(
     input: {
@@ -48,6 +49,17 @@ pub async fn check(
         .lookup(S3_CFG_TYPE_ID, &s3_client_id)
         .context("config lookup failed")?;
 
+    let request_counter_id = if let Some(store) = &store_prefix {
+        format!("{s3_client_id}-{bucket}-{store}")
+    } else {
+        format!("{s3_client_id}-{bucket}")
+    };
+    let request_counter_options = S3RequestCounterOptions {
+        id: request_counter_id,
+        user: pbs_config::backup_user()?,
+        base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+    };
+
     let store_prefix = store_prefix.unwrap_or_default();
     let options = S3ClientOptions::from_config(
         config.config,
@@ -55,6 +67,7 @@ pub async fn check(
         Some(bucket),
         store_prefix,
         None,
+        Some(request_counter_options),
     );
 
     let test_object_key =
diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs
index 27b3c4cc2..046e247e4 100644
--- a/src/api2/config/s3.rs
+++ b/src/api2/config/s3.rs
@@ -6,7 +6,7 @@ use serde_json::Value;
 use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
 use proxmox_s3_client::{
     S3BucketListItem, S3Client, S3ClientConf, S3ClientConfig, S3ClientConfigUpdater,
-    S3ClientConfigWithoutSecret, S3ClientOptions, S3_CLIENT_ID_SCHEMA,
+    S3ClientConfigWithoutSecret, S3ClientOptions, S3RequestCounterOptions, S3_CLIENT_ID_SCHEMA,
 };
 use proxmox_schema::{api, param_bail, ApiType};
 
@@ -16,6 +16,7 @@ use pbs_api_types::{
 };
 use pbs_config::s3::{self, S3_CFG_TYPE_ID};
 use pbs_config::CachedUserInfo;
+use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH;
 
 #[api(
     input: {
@@ -350,8 +351,19 @@ pub async fn list_buckets(
         .context("config lookup failed")?;
 
     let empty_prefix = String::new();
-    let options =
-        S3ClientOptions::from_config(config.config, config.secret_key, None, empty_prefix, None);
+    let request_counter_options = S3RequestCounterOptions {
+        id,
+        user: pbs_config::backup_user()?,
+        base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(),
+    };
+    let options = S3ClientOptions::from_config(
+        config.config,
+        config.secret_key,
+        None,
+        empty_prefix,
+        None,
+        Some(request_counter_options),
+    );
     let client = S3Client::new(options).context("client creation failed")?;
     let list_buckets_response = client
         .list_buckets()
-- 
2.47.3