From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id ABE261FF0B6 for ; Thu, 12 Nov 2026 19:02:22 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id E539D21694; Thu, 12 Nov 2026 19:02:11 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox v3 03/11] s3-client: detect throttling request from the response body on error Date: Thu, 12 Nov 2026 19:01:11 +0100 Message-ID: <20261112180119.660923-4-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112180119.660923-1-c.ebner@proxmox.com> References: <20261112180119.660923-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794506501426 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.409 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: CEL2QP3OLPRCKBDE6HTTSH6LMNGKMYUD X-Message-ID-Hash: CEL2QP3OLPRCKBDE6HTTSH6LMNGKMYUD X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: AWS S3 signals throttling with a service unavailable response and the SlowDown error code, which is now classified as transient errors since the previous change introducing this distinction and therefore retried after the short backoff of a transient error. Follow the AWS SDKs implementation guide on this and try to identify retryable errors by the error code of the response body if present instead, the body will be discarded on retry anyways. Using the longer throttling backoff delay for retries introduced by the previous change for the known throttling codes. Other S3 compatible providers do use similar codes to signal rate limiting [0]. [0] https://forum.proxmox.com/threads/184371/post-858173 [1] https://forum.proxmox.com/threads/176950/ Link: https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html Signed-off-by: Christian Ebner --- proxmox-s3-client/src/client.rs | 71 ++++++++++++++++++++++++ proxmox-s3-client/src/response_reader.rs | 36 ++++++++++++ 2 files changed, 107 insertions(+) diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index 677557f9..c43d0973 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, LazyLock, Mutex, RwLock}; use std::time::{Duration, Instant}; use anyhow::{Context, Error, bail, format_err}; +use http_body_util::BodyExt; use hyper::body::{Bytes, Incoming}; use hyper::http::method::Method; use hyper::http::uri::{Authority, Parts, PathAndQuery, Scheme}; @@ -52,6 +53,23 @@ const MAX_S3_HTTP_REQUEST_RETRY: usize = 3; const S3_RETRY_BASE_DELAY_TRANSIENT: Duration = Duration::from_millis(50); const S3_RETRY_BASE_DELAY_THROTTLING: Duration = Duration::from_secs(1); const S3_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(20); +// Response body error codes interpreted as retry with throttled delay instead of transient delay. +const S3_THROTTLING_ERROR_CODES: &[&str] = &[ + "Throttling", + "ThrottlingException", + "ThrottledException", + "RequestThrottledException", + "TooManyRequestsException", + "ProvisionedThroughputExceededException", + "LimitExceededException", + "PriorRequestNotComplete", + "RequestThrottled", + "RequestLimitExceeded", + "SlowDown", + "BandwidthLimitExceeded", +]; +// Maximum time to wait when reading an error response body to check for throttling codes. +const S3_ERROR_BODY_READ_TIMEOUT: Duration = Duration::from_secs(5); static SHARED_COUNTER_FLUSHER: LazyLock> = LazyLock::new(|| RwLock::new(MmapFlusher::new())); @@ -540,6 +558,28 @@ impl S3Client { return Ok(response?); }; + // The response is discarded on retry, so use it to check the error code to detect + // throttling signaled with a server error status. + let retry_reason = match response { + Ok(response) if retry_reason == RetryReason::Transient => { + let response_body = response.into_body().collect(); + let read_deadline = tokio::time::Instant::now() + S3_ERROR_BODY_READ_TIMEOUT; + let body_deadline = match deadline { + Some(deadline) => read_deadline.min(deadline), + None => read_deadline, + }; + + if let Ok(Ok(body)) = + tokio::time::timeout_at(body_deadline, response_body).await + { + RetryReason::from_error_response_body(&body.to_bytes()) + } else { + RetryReason::Transient + } + } + _ => retry_reason, + }; + tokio::time::sleep(retry_backoff(retry_reason, retry)).await; } @@ -1037,6 +1077,16 @@ enum RetryReason { } impl RetryReason { + fn from_error_response_body(body: &[u8]) -> Self { + if let Some(error_code) = crate::response_reader::parse_error_code(body) + && S3_THROTTLING_ERROR_CODES.contains(&error_code.as_str()) + { + return Self::Throttling; + } + + Self::Transient + } + fn base_delay(self) -> Duration { match self { Self::Transient => S3_RETRY_BASE_DELAY_TRANSIENT, @@ -1083,6 +1133,27 @@ mod tests { ); } + #[test] + fn retry_reason_from_error_code() { + let error_body = |code: &str| format!("{code}"); + assert_eq!( + RetryReason::from_error_response_body(error_body("SlowDown").as_bytes()), + RetryReason::Throttling + ); + assert_eq!( + RetryReason::from_error_response_body(error_body("InternalError").as_bytes()), + RetryReason::Transient + ); + assert_eq!( + RetryReason::from_error_response_body(b""), + RetryReason::Transient + ); + assert_eq!( + RetryReason::from_error_response_body(b"Service Unavailable"), + RetryReason::Transient + ); + } + #[test] fn retry_backoff_is_capped() { assert_eq!( diff --git a/proxmox-s3-client/src/response_reader.rs b/proxmox-s3-client/src/response_reader.rs index 1a3a9add..f941aaa2 100644 --- a/proxmox-s3-client/src/response_reader.rs +++ b/proxmox-s3-client/src/response_reader.rs @@ -605,6 +605,42 @@ impl ResponseReader { } } +#[derive(Deserialize)] +#[serde(rename_all = "PascalCase")] +/// Common error response body of S3 api calls, only the error code is of interest. +/// https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#RESTErrorResponses +struct ErrorResponseBody { + code: Option, +} + +/// Parse the error code from an S3 api error response body, if any. +pub(crate) fn parse_error_code(body: &[u8]) -> Option { + let body = std::str::from_utf8(body).ok()?; + let error: ErrorResponseBody = serde_xml_rs::from_str(body).ok()?; + error.code +} + +#[test] +fn parse_error_code_test() { + let response_body = r#" + + SlowDown + Please reduce your request rate. + 4442587FB7D0A2F9 + "#; + assert_eq!( + parse_error_code(response_body.as_bytes()), + Some("SlowDown".to_string()) + ); + assert_eq!( + parse_error_code(b"no code"), + None + ); + assert_eq!(parse_error_code(b""), None); + assert_eq!(parse_error_code(b"Bad Gateway"), None); + assert_eq!(parse_error_code(&[0xff, 0xfe]), None); +} + #[test] fn parse_list_objects_v2_response_test() { let response_body = r#" -- 2.47.3