From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 3114F1FF0B6 for ; Thu, 12 Nov 2026 19:01:58 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id CE19821665; Thu, 12 Nov 2026 19:01:56 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox v3 05/11] s3-client: honor response header directed retry delays Date: Thu, 12 Nov 2026 19:01:13 +0100 Message-ID: <20261112180119.660923-6-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112180119.660923-1-c.ebner@proxmox.com> References: <20261112180119.660923-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794506501966 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.406 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: GEPBL26MNN3Q4V6R2OXXBGW4Q2K2DG3R X-Message-ID-Hash: GEPBL26MNN3Q4V6R2OXXBGW4Q2K2DG3R X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: S3 API servers can tell clients when they expect to have capacity available again via the x-amz-retry-after header, given in milliseconds. Retrying earlier is likely to fail again and only adds load, so try to honor these delays on retry. Use the server's requested delay instead of the computed backoff, but never wait less than the backoff and at most 5 seconds longer, so a misbehaving server cannot stall requests for a long time. No jitter is applied, the server is expected to randomize the delay itself. This follows the AWS SDKs implementation guide. Link: https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html Signed-off-by: Christian Ebner --- proxmox-s3-client/src/client.rs | 63 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index bb9132b8..b094a58d 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -53,6 +53,8 @@ const MAX_S3_HTTP_REQUEST_RETRY: usize = 3; const S3_RETRY_BASE_DELAY_TRANSIENT: Duration = Duration::from_millis(50); const S3_RETRY_BASE_DELAY_THROTTLING: Duration = Duration::from_secs(1); const S3_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(20); +// Maximum time a header directed retry delay may exceed the computed backoff. +const S3_RETRY_AFTER_MAX_EXCESS: Duration = Duration::from_secs(5); // Response body error codes interpreted as retry with throttled delay instead of transient delay. const S3_THROTTLING_ERROR_CODES: &[&str] = &[ "Throttling", @@ -558,8 +560,13 @@ impl S3Client { return Ok(response?); }; - // The response is discarded on retry, so use it to check the error code to detect - // throttling signaled with a server error status. + let retry_after = response + .as_ref() + .ok() + .and_then(|response| parse_retry_after(response.headers())); + + // The response is discarded for the retry anyway, so check its error code to detect + // throttling signaled with a server error status, for example S3's 503 SlowDown. let retry_reason = match response { Ok(response) if retry_reason == RetryReason::Transient => { let response_body = response.into_body().collect(); @@ -580,7 +587,8 @@ impl S3Client { _ => retry_reason, }; - tokio::time::sleep(retry_backoff_with_jitter(retry_reason, retry)).await; + let backoff = retry_backoff(retry_reason, retry); + tokio::time::sleep(server_directed_delay(retry_after, backoff)).await; } bail!("failed to send request exceeding retries"); @@ -1131,6 +1139,22 @@ fn with_jitter(backoff: Duration) -> Duration { } } +/// Parse the server directed retry delay in milliseconds from the response headers, if any. +fn parse_retry_after(headers: &header::HeaderMap) -> Option { + let value = headers.get("x-amz-retry-after")?.to_str().ok()?; + value.trim().parse().ok().map(Duration::from_millis) +} + +/// Use the response header directed retry delay if given, but never wait less than the computed +/// backoff and at most a few seconds longer. The server is expected to randomize the delay, so +/// no further jitter is applied. +fn server_directed_delay(retry_after: Option, backoff: Duration) -> Duration { + match retry_after { + Some(retry_after) => retry_after.clamp(backoff, backoff + S3_RETRY_AFTER_MAX_EXCESS), + None => with_jitter(backoff), + } +} + #[cfg(test)] mod tests { use super::*; @@ -1176,6 +1200,39 @@ mod tests { ); } + #[test] + fn retry_after_header() { + let mut headers = header::HeaderMap::new(); + assert_eq!(parse_retry_after(&headers), None); + headers.insert("x-amz-retry-after", HeaderValue::from_static("1500")); + assert_eq!( + parse_retry_after(&headers), + Some(Duration::from_millis(1500)) + ); + headers.insert("x-amz-retry-after", HeaderValue::from_static("soon")); + assert_eq!(parse_retry_after(&headers), None); + headers.insert("x-amz-retry-after", HeaderValue::from_static("-1")); + assert_eq!(parse_retry_after(&headers), None); + } + + #[test] + fn server_directed_delay_is_clamped() { + let backoff = Duration::from_secs(1); + assert!(server_directed_delay(None, backoff) <= backoff); + assert_eq!( + server_directed_delay(Some(Duration::from_secs(3)), backoff), + Duration::from_secs(3) + ); + assert_eq!( + server_directed_delay(Some(Duration::from_millis(10)), backoff), + backoff + ); + assert_eq!( + server_directed_delay(Some(Duration::from_secs(60)), backoff), + Duration::from_secs(6) + ); + } + #[test] fn with_jitter_within_backoff() { let backoff = Duration::from_secs(1); -- 2.47.3