[pbs-devel] [PATCH proxmox v2 1/1] s3-client: add request statistics gathering capabilities

[Christian Ebner <c.ebner@proxmox.com>]

Track the per-method request counts and the duration for the last
successful request send by a client instance.

With the intend to utilize the request statistics in PBS, starting
with showing request statistics for garbage collection as a first
step.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 1:
- not present in previous version

 proxmox-s3-client/src/client.rs | 52 ++++++++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs
index 83176b39..2183fd18 100644
--- a/proxmox-s3-client/src/client.rs
+++ b/proxmox-s3-client/src/client.rs
@@ -1,5 +1,6 @@
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 
@@ -135,12 +136,26 @@ impl S3ClientOptions {
     }
 }
 
+#[derive(Default)]
+/// Request statistics for s3 client instance
+struct RequestStats {
+    /// Number of requests performed by method
+    delete: AtomicU64,
+    get: AtomicU64,
+    head: AtomicU64,
+    post: AtomicU64,
+    put: AtomicU64,
+    /// Duration of the last successfully completed request
+    last_request_duration: Mutex<Option<Duration>>,
+}
+
 /// S3 client for object stores compatible with the AWS S3 API
 pub struct S3Client {
     client: Client<HttpsConnector, Body>,
     options: S3ClientOptions,
     authority: Authority,
     put_rate_limiter: Option<Arc<Mutex<RateLimiter>>>,
+    request_stats: Arc<RequestStats>,
 }
 
 impl S3Client {
@@ -241,6 +256,7 @@ impl S3Client {
             options,
             authority,
             put_rate_limiter,
+            request_stats: Arc::new(RequestStats::default()),
         })
     }
 
@@ -385,6 +401,16 @@ impl S3Client {
                 tokio::time::sleep(backoff_secs).await;
             }
 
+            match parts.method {
+                Method::DELETE => self.request_stats.delete.fetch_add(1, Ordering::SeqCst),
+                Method::GET => self.request_stats.get.fetch_add(1, Ordering::SeqCst),
+                Method::HEAD => self.request_stats.head.fetch_add(1, Ordering::SeqCst),
+                Method::POST => self.request_stats.post.fetch_add(1, Ordering::SeqCst),
+                Method::PUT => self.request_stats.put.fetch_add(1, Ordering::SeqCst),
+                _ => 0,
+            };
+
+            let now = Instant::now();
             let response = if let Some(deadline) = deadline {
                 tokio::time::timeout_at(deadline, self.client.request(request)).await
             } else {
@@ -392,7 +418,10 @@ impl S3Client {
             };
 
             match response {
-                Ok(Ok(response)) => return Ok(response),
+                Ok(Ok(response)) => {
+                    *self.request_stats.last_request_duration.lock().unwrap() = Some(now.elapsed());
+                    return Ok(response);
+                }
                 Ok(Err(err)) => {
                     if retry >= MAX_S3_HTTP_REQUEST_RETRY - 1 {
                         return Err(err.into());
@@ -409,6 +438,27 @@ impl S3Client {
         bail!("failed to send request exceeding retries");
     }
 
+    /// Duration of the last successfully completed request.
+    pub fn last_request_duration(&self) -> Option<Duration> {
+        self.request_stats
+            .last_request_duration
+            .lock()
+            .unwrap()
+            .clone()
+    }
+
+    /// Request counts by method for the client since instantiation.
+    pub fn request_count(&self, method: Method) -> u64 {
+        match method {
+            Method::DELETE => self.request_stats.delete.load(Ordering::SeqCst),
+            Method::GET => self.request_stats.get.load(Ordering::SeqCst),
+            Method::HEAD => self.request_stats.head.load(Ordering::SeqCst),
+            Method::POST => self.request_stats.post.load(Ordering::SeqCst),
+            Method::PUT => self.request_stats.put.load(Ordering::SeqCst),
+            _ => 0,
+        }
+    }
+
     /// Check if bucket exists and got permissions to access it.
     /// See reference docs: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html
     pub async fn head_bucket(&self) -> Result<(), Error> {
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel