From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id ACD6C1FF13C for ; Thu, 19 Mar 2026 10:41:59 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 5514116731; Thu, 19 Mar 2026 10:42:01 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v6 04/22] datastore: collect request statistics for s3 backed datastores Date: Thu, 19 Mar 2026 10:40:42 +0100 Message-ID: <20260319094100.240765-17-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260319094100.240765-1-c.ebner@proxmox.com> References: <20260319094100.240765-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1773913232867 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.090 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_2 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_4 0.1 random spam to be learned in bayes SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: C2SMVJRVTWSKWBYF4XVKEQWEY5RW762K X-Message-ID-Hash: C2SMVJRVTWSKWBYF4XVKEQWEY5RW762K X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Define and provide the s3 request counter options to the s3 client invocations so all requests and traffic is being tracked. When no datastore is involved, account the statistics for the bucket instead. Bucket or even endpoint wide statistics might then be gathered in the future by parsing the corresponding memory mapped files and collecting their contents. Signed-off-by: Christian Ebner --- pbs-datastore/src/datastore.rs | 15 +++++++++++++++ pbs-datastore/src/lib.rs | 2 +- src/api2/admin/s3.rs | 17 +++++++++++++++-- src/api2/config/s3.rs | 9 ++++++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index ef378c696..f88f323df 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -17,6 +17,7 @@ use tracing::{info, warn}; use proxmox_human_byte::HumanByte; use proxmox_s3_client::{ S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3PathPrefix, S3RateLimiterOptions, + S3RequestCounterOptions, }; use proxmox_schema::ApiType; @@ -75,6 +76,8 @@ pub const GROUP_NOTES_FILE_NAME: &str = "notes"; pub const GROUP_OWNER_FILE_NAME: &str = "owner"; /// Filename for in-use marker stored on S3 object store backend pub const S3_DATASTORE_IN_USE_MARKER: &str = ".in-use"; +/// Base directory for storing shared memory mapped s3 request counters +pub const S3_CLIENT_REQUEST_COUNTER_BASE_PATH: &str = "/var/lib/proxmox-backup/s3-statistics"; const S3_CLIENT_RATE_LIMITER_BASE_PATH: &str = pbs_buildcfg::rundir!("/s3/shmem/tbf"); const NAMESPACE_MARKER_FILENAME: &str = ".namespace"; // s3 put request times out after upload_size / 1 Kib/s, so about 2.3 hours for 8 MiB @@ -426,6 +429,11 @@ impl DataStore { user: pbs_config::backup_user()?, base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(), }; + let request_counter_options = S3RequestCounterOptions { + id: format!("{s3_client_id}-{bucket}-{}", self.name()), + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; let options = S3ClientOptions::from_config( config.config, @@ -434,6 +442,7 @@ impl DataStore { self.name().to_owned(), Some(rate_limiter_options), None, //FIXME read from node.cfg + Some(request_counter_options), ); let s3_client = S3Client::new(options)?; DatastoreBackend::S3(Arc::new(s3_client)) @@ -2652,6 +2661,11 @@ impl DataStore { user: pbs_config::backup_user()?, base_path: S3_CLIENT_RATE_LIMITER_BASE_PATH.into(), }; + let request_counter_options = S3RequestCounterOptions { + id: format!("{s3_client_id}-{bucket}-{}", datastore_config.name), + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; let options = S3ClientOptions::from_config( client_config.config, @@ -2660,6 +2674,7 @@ impl DataStore { datastore_config.name.to_owned(), Some(rate_limiter_options), None, // FIXME read from node.cfg + Some(request_counter_options), ); let s3_client = S3Client::new(options) .context("failed to create s3 client") diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 1f7c54ae8..afe340a65 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -217,7 +217,7 @@ pub use store_progress::StoreProgress; mod datastore; pub use datastore::{ check_backup_owner, ensure_datastore_is_mounted, get_datastore_mount_status, DataStore, - DatastoreBackend, S3_DATASTORE_IN_USE_MARKER, + DatastoreBackend, S3_CLIENT_REQUEST_COUNTER_BASE_PATH, S3_DATASTORE_IN_USE_MARKER, }; mod hierarchy; diff --git a/src/api2/admin/s3.rs b/src/api2/admin/s3.rs index d3b3b11e6..156a8b660 100644 --- a/src/api2/admin/s3.rs +++ b/src/api2/admin/s3.rs @@ -6,8 +6,8 @@ use serde_json::Value; use proxmox_http::Body; use proxmox_router::{list_subdirs_api_method, Permission, Router, RpcEnvironment, SubdirMap}; use proxmox_s3_client::{ - S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3_BUCKET_NAME_SCHEMA, - S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT, + S3Client, S3ClientConf, S3ClientOptions, S3ObjectKey, S3RequestCounterOptions, + S3_BUCKET_NAME_SCHEMA, S3_CLIENT_ID_SCHEMA, S3_HTTP_REQUEST_TIMEOUT, }; use proxmox_schema::*; use proxmox_sortable_macro::sortable; @@ -15,6 +15,7 @@ use proxmox_sortable_macro::sortable; use pbs_api_types::PRIV_SYS_MODIFY; use pbs_config::s3::S3_CFG_TYPE_ID; +use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH; #[api( input: { @@ -48,6 +49,17 @@ pub async fn check( .lookup(S3_CFG_TYPE_ID, &s3_client_id) .context("config lookup failed")?; + let request_counter_id = if let Some(store) = &store_prefix { + format!("{s3_client_id}-{bucket}-{store}") + } else { + format!("{s3_client_id}-{bucket}") + }; + let request_counter_options = S3RequestCounterOptions { + id: request_counter_id, + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; + let store_prefix = store_prefix.unwrap_or_default(); let options = S3ClientOptions::from_config( config.config, @@ -56,6 +68,7 @@ pub async fn check( store_prefix, None, None, // FIXME read from node.cfg once regular datastore operations do as well + Some(request_counter_options), ); let test_object_key = diff --git a/src/api2/config/s3.rs b/src/api2/config/s3.rs index 9f1b4b5f7..d57236f0c 100644 --- a/src/api2/config/s3.rs +++ b/src/api2/config/s3.rs @@ -6,7 +6,7 @@ use serde_json::Value; use proxmox_router::{http_bail, Permission, Router, RpcEnvironment}; use proxmox_s3_client::{ S3BucketListItem, S3Client, S3ClientConf, S3ClientConfig, S3ClientConfigUpdater, - S3ClientConfigWithoutSecret, S3ClientOptions, S3_CLIENT_ID_SCHEMA, + S3ClientConfigWithoutSecret, S3ClientOptions, S3RequestCounterOptions, S3_CLIENT_ID_SCHEMA, }; use proxmox_schema::{api, param_bail, ApiType}; @@ -16,6 +16,7 @@ use pbs_api_types::{ }; use pbs_config::s3::{self, S3_CFG_TYPE_ID}; use pbs_config::CachedUserInfo; +use pbs_datastore::S3_CLIENT_REQUEST_COUNTER_BASE_PATH; #[api( input: { @@ -350,6 +351,11 @@ pub async fn list_buckets( .context("config lookup failed")?; let empty_prefix = String::new(); + let request_counter_options = S3RequestCounterOptions { + id, + user: pbs_config::backup_user()?, + base_path: S3_CLIENT_REQUEST_COUNTER_BASE_PATH.into(), + }; let options = S3ClientOptions::from_config( config.config, config.secret_key, @@ -357,6 +363,7 @@ pub async fn list_buckets( empty_prefix, None, None, // FIXME read from node.cfg once regular datastore operations do as well + Some(request_counter_options), ); let client = S3Client::new(options).context("client creation failed")?; let list_buckets_response = client -- 2.47.3