From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id B11151FF0B7 for ; Fri, 13 Nov 2026 15:22:03 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id C28A7216A3; Fri, 13 Nov 2026 15:21:56 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox v4 02/11] s3-client: distinguish transient errors and throttled requests Date: Fri, 13 Nov 2026 15:21:39 +0100 Message-ID: <20261113142148.569471-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113142148.569471-1-c.ebner@proxmox.com> References: <20261113142148.569471-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794579712030 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.397 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: ITGH7LEFUOFVXPUNECZCN624RNFTF3XJ X-Message-ID-Hash: ITGH7LEFUOFVXPUNECZCN624RNFTF3XJ X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Currently failed requests are retried after a fixed 1 and 3 seconds, regardless of the cause. This is longer than needed for connection or server errors, only intended as graceful recovery period for heavily loaded S3 APIs in the previous implementation. Therefore, distinguish these 2 modes by following the backoff logic use by the AWS SDK's standard retry mode instead: wait base * 2^retry, capped at 20 seconds, with a base delay of 100 ms for transient errors and 2 second for throttling. These values are chosen to be double the ones provided by the guide, in order to give on average the same throttling backoff delay as previously used by the client, once uniformly distributed jitter is introduced as well in subsequent changes. Cover only the too many requests http status code for now, additional status codes will use this based on the error code in the response body parsing as extended in subsequent changes. Link: https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html Signed-off-by: Christian Ebner --- proxmox-s3-client/src/client.rs | 106 +++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 14 deletions(-) diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index 3f2a7107..28646aee 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -47,7 +47,11 @@ const MAX_S3_UPLOAD_RETRY: usize = 3; // Assumed minimum upload rate of 1 KiB/s for dynamic put object request timeout calculation. const S3_MIN_ASSUMED_UPLOAD_RATE: u64 = 1024; const MAX_S3_HTTP_REQUEST_RETRY: usize = 3; -const S3_HTTP_REQUEST_RETRY_BACKOFF_DEFAULT: Duration = Duration::from_secs(1); +// Retry backoff as used by the AWS SDKs standard retry mode, see +// https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html +const S3_RETRY_BASE_DELAY_TRANSIENT: Duration = Duration::from_millis(100); +const S3_RETRY_BASE_DELAY_THROTTLING: Duration = Duration::from_secs(2); +const S3_RETRY_MAX_BACKOFF: Duration = Duration::from_secs(20); static SHARED_COUNTER_FLUSHER: LazyLock> = LazyLock::new(|| RwLock::new(MmapFlusher::new())); @@ -506,19 +510,20 @@ impl S3Client { self.client.request(request).await }; - let do_retry = match &response { - Ok(response) => matches!( - response.status(), + let retry_reason = match &response { + Ok(response) => match response.status() { StatusCode::INTERNAL_SERVER_ERROR - | StatusCode::BAD_GATEWAY - | StatusCode::SERVICE_UNAVAILABLE - | StatusCode::GATEWAY_TIMEOUT - | StatusCode::TOO_MANY_REQUESTS - ), - Err(_) => true, + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => Some(RetryReason::Transient), + StatusCode::TOO_MANY_REQUESTS => Some(RetryReason::Throttling), + _ => None, + }, + Err(_) => Some(RetryReason::Transient), }; - if !do_retry || retry >= MAX_S3_HTTP_REQUEST_RETRY - 1 { + let Some(retry_reason) = retry_reason.filter(|_| retry < MAX_S3_HTTP_REQUEST_RETRY - 1) + else { if let Some(counters) = self.request_counters.as_ref() && response.is_ok() { @@ -533,10 +538,9 @@ impl S3Client { let _ = SHARED_COUNTER_FLUSHER.read().unwrap().request_flush(); } return Ok(response?); - } + }; - let backoff_secs = S3_HTTP_REQUEST_RETRY_BACKOFF_DEFAULT * 3_u32.pow(retry as u32); - tokio::time::sleep(backoff_secs).await; + tokio::time::sleep(retry_backoff(retry_reason, retry)).await; } bail!("failed to send request exceeding retries"); @@ -1026,10 +1030,84 @@ impl S3Client { } } +#[derive(Debug, PartialEq)] +enum RetryReason { + Transient, + Throttling, +} + +impl RetryReason { + fn base_delay(self) -> Duration { + match self { + Self::Transient => S3_RETRY_BASE_DELAY_TRANSIENT, + Self::Throttling => S3_RETRY_BASE_DELAY_THROTTLING, + } + } +} + +/// Helper to get exponential backoff duration based on retry reason and count. +/// +/// Calculates how long to wait before the next retry attempt for a request. +/// The duration is limited by an upper bond of [`S3_RETRY_MAX_BACKOFF`]. +fn retry_backoff(reason: RetryReason, retry: usize) -> Duration { + u32::try_from(retry) + .ok() + .and_then(|retry| 2_u32.checked_pow(retry)) + .and_then(|factor| reason.base_delay().checked_mul(factor)) + .map_or(S3_RETRY_MAX_BACKOFF, |backoff| { + backoff.min(S3_RETRY_MAX_BACKOFF) + }) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn retry_backoff_by_reason() { + assert_eq!( + retry_backoff(RetryReason::Transient, 0), + Duration::from_millis(100) + ); + assert_eq!( + retry_backoff(RetryReason::Transient, 1), + Duration::from_millis(200) + ); + assert_eq!( + retry_backoff(RetryReason::Throttling, 0), + Duration::from_secs(2) + ); + assert_eq!( + retry_backoff(RetryReason::Throttling, 1), + Duration::from_secs(4) + ); + } + + #[test] + fn retry_backoff_is_capped() { + assert_eq!( + retry_backoff(RetryReason::Throttling, 2), + Duration::from_secs(8) + ); + assert_eq!( + retry_backoff(RetryReason::Transient, 4), + Duration::from_millis(1_600) + ); + assert_eq!( + retry_backoff(RetryReason::Transient, 8), + S3_RETRY_MAX_BACKOFF + ); + // must not overflow + assert_eq!( + retry_backoff(RetryReason::Throttling, 64), + S3_RETRY_MAX_BACKOFF + ); + assert_eq!( + retry_backoff(RetryReason::Transient, usize::MAX), + S3_RETRY_MAX_BACKOFF + ); + } + struct TestDirectory(PathBuf); impl Drop for TestDirectory { -- 2.47.3