From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 1005A1FF136 for ; Mon, 09 Feb 2026 10:15:21 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 72374BC; Mon, 9 Feb 2026 10:15:58 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH v1 01/11] datastore: collect request statistics for s3 backed datastores Date: Mon, 9 Feb 2026 10:15:23 +0100 Message-ID: <20260209091533.156902-8-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260209091533.156902-1-c.ebner@proxmox.com> References: <20260209091533.156902-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1770628461233 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.048 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: PBIPAZVEVSCL57OFCHAWAHYWK7LXRPPK X-Message-ID-Hash: PBIPAZVEVSCL57OFCHAWAHYWK7LXRPPK X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Define and provide the s3 request counter options to the s3 client invocations so all requests and traffic is being tracked. When no datastore is involved, account the statistics for the bucket instead. Bucket or even endpoint wide statistics might then be gathered in the future by parsing the corresponding memory mapped files and collecting their contents. Signed-off-by: Christian Ebner --- pbs-datastore/src/datastore.rs | 15 +++++++++++++++ pbs-datastore/src/lib.rs | 2 +- src/api2/admin/s3.rs | 17 +++++++++++++++-- src/api2/config/s3.rs | 18 +++++++++++++++--- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index 7ad3d917d..957e900d6 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -17,6 +17,7 @@ use tracing::{info, warn}; use proxmox_human_byte::HumanByte; use proxmox_s3_client::{ S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions, + S3RequestCounterOptions, }; use proxmox_schema::ApiType; @@ -75,6 +76,8 @@ pub const GROUP_NOTES_FILE_NAME: &str = "notes"; pub const GROUP_OWNER_FILE_NAME: &str = "owner"; /// Filename for in-use marker stored on S3 object store backend pub const S3_DATASTORE_IN_USE_MARKER: &str = ".in-use"; +/// Base directory for storing shared memory mapped s3 request counters +pub const S3_CLIENT_REQUEST_COUNTER_BASE_PATH: &str = "/var/lib/proxmox-backup/s3-statistics"; const S3_CLIENT_RATE_LIMITER_BASE_PATH: &str = pbs_buildcfg::rundir!("/s3/shmem/tbf"); const NAMESPACE_MARKER_FILENAME: &str = ".namespace"; // s3 put request times out after upload_size / 1 Kib/s, so about 2.3 hours for 8 MiB @@ -426,6 +429,11 @@ impl DataStore { user: pbs_config::backup_user()?, base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(), }; + let request_counter_options = S3RequestCounterOptions { + id: format!("{s3_client_id}-{bucket}-{}", self.name()), + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; let options = S3ClientOptions::from_config( config.config, @@ -433,6 +441,7 @@ impl DataStore { Some(bucket), self.name().to_owned(), Some(rate_limiter_options), + Some(request_counter_options), ); let s3_client = S3Client::new(options)?; DatastoreBackend::S3(Arc::new(s3_client)) @@ -2659,6 +2668,11 @@ impl DataStore { user: pbs_config::backup_user()?, base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(), }; + let request_counter_options = S3RequestCounterOptions { + id: format!("{s3_client_id}-{bucket}-{}", datastore_config.name), + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; let options = S3ClientOptions::from_config( client_config.config, @@ -2666,6 +2680,7 @@ impl DataStore { Some(bucket), datastore_config.name.to_owned(), Some(rate_limiter_options), + Some(request_counter_options), ); let s3_client = S3Client::new(options) .context("failed to create s3 client") diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 1f7c54ae8..afe340a65 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -217,7 +217,7 @@ pub use store_progress::StoreProgress; mod datastore; pub use datastore::{ check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore, - DatastoreBackend, S3_DATASTORE_IN_USE_MARKER, + DatastoreBackend, S3_CLIENT_REQUEST_COUNTER_BASE_PATH, S3_DATASTORE_IN_USE_MARKER, }; mod hierarchy; diff --git a/src/api2/admin/s3.rs b/src/api2/admin/s3.rs index 73388281b..d20cae483 100644 --- a/src/api2/admin/s3.rs +++ b/src/api2/admin/s3.rs @@ -6,8 +6,8 @@ use serde_json::Value; use proxmox_http::Body; use proxmox_router::{list_subdirs_api_method, Permission, Router, RpcEnvironment, SubdirMap}; use proxmox_s3_client::{ - S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3_BUCKET_NAME_SCHEMA, - S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT, + S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3RequestCounterOptions, + S3_BUCKET_NAME_SCHEMA, S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT, }; use proxmox_schema::*; use proxmox_sortable_macro::sortable; @@ -15,6 +15,7 @@ use proxmox_sortable_macro::sortable; use pbs_api_types::PRIV_SYS_MODIFY; use pbs_config::s3::S3_CFG_TYPE_ID; +use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH; #[api( input: { @@ -48,6 +49,17 @@ pub async fn check( .lookup(S3_CFG_TYPE_ID, &s3_client_id) .context("config lookup failed")?; + let request_counter_id = if let Some(store) = &store_prefix { + format!("{s3_client_id}-{bucket}-{store}") + } else { + format!("{s3_client_id}-{bucket}") + }; + let request_counter_options = S3RequestCounterOptions { + id: request_counter_id, + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; + let store_prefix = store_prefix.unwrap_or_default(); let options = S3ClientOptions::from_config( config.config, @@ -55,6 +67,7 @@ pub async fn check( Some(bucket), store_prefix, None, + Some(request_counter_options), ); let test_object_key = diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs index 27b3c4cc2..046e247e4 100644 --- a/src/api2/config/s3.rs +++ b/src/api2/config/s3.rs @@ -6,7 +6,7 @@ use serde_json::Value; use proxmox_router::{http_bail, Permission, Router, RpcEnvironment}; use proxmox_s3_client::{ S3BucketListItem, S3Client, S3ClientConf, S3ClientConfig, S3ClientConfigUpdater, - S3ClientConfigWithoutSecret, S3ClientOptions, S3_CLIENT_ID_SCHEMA, + S3ClientConfigWithoutSecret, S3ClientOptions, S3RequestCounterOptions, S3_CLIENT_ID_SCHEMA, }; use proxmox_schema::{api, param_bail, ApiType}; @@ -16,6 +16,7 @@ use pbs_api_types::{ }; use pbs_config::s3::{self, S3_CFG_TYPE_ID}; use pbs_config::CachedUserInfo; +use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH; #[api( input: { @@ -350,8 +351,19 @@ pub async fn list_buckets( .context("config lookup failed")?; let empty_prefix = String::new(); - let options = - S3ClientOptions::from_config(config.config, config.secret_key, None, empty_prefix, None); + let request_counter_options = S3RequestCounterOptions { + id, + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; + let options = S3ClientOptions::from_config( + config.config, + config.secret_key, + None, + empty_prefix, + None, + Some(request_counter_options), + ); let client = S3Client::new(options).context("client creation failed")?; let list_buckets_response = client .list_buckets() -- 2.47.3