From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 051901FF0BC for ; Mon, 16 Nov 2026 10:55:08 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 2B1B92173A; Mon, 16 Nov 2026 10:54:52 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox v5 08/12] s3-client: add shared rate limiter based retry quota Date: Mon, 16 Nov 2026 10:54:30 +0100 Message-ID: <20261116095434.110541-9-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116095434.110541-1-c.ebner@proxmox.com> References: <20261116095434.110541-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794822887195 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.382 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: O37FGFASPPIJI4ZUD3HMMXVDYQCKSRGQ X-Message-ID-Hash: O37FGFASPPIJI4ZUD3HMMXVDYQCKSRGQ X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Delays for requests are currently only applied if either a shared rate limiter is configured or if the request is being retried. New requests are never delayed for latter case. If the server requested a slow down, it makes senses to rate limit also new requests. Therefore, implement a shared token bucket retry quota and reuse the SharedRateLimiter to account for all requests. Use the values indicated by the AWS SDK for token cost and refill behaviour of the retry quota. In particular: - 500 bucket size - 50 tokens/s refill rate - cost 5 for throttling retries and cost 10 for transient retries When a request has not yet reached its retry limit and the retry quota is exhausted, add the rate limiter's delay to the retry backoff instead of failing hard, avoiding premature failure. Link: https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html Signed-off-by: Christian Ebner --- proxmox-s3-client/src/client.rs | 58 +++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/proxmox-s3-client/src/client.rs b/proxmox-s3-client/src/client.rs index aae3961a..3c58946a 100644 --- a/proxmox-s3-client/src/client.rs +++ b/proxmox-s3-client/src/client.rs @@ -72,6 +72,12 @@ const S3_THROTTLING_ERROR_CODES: &[&str] = &[ ]; // Maximum time to wait when reading an error response body to check for throttling codes. const S3_ERROR_BODY_READ_TIMEOUT: Duration = Duration::from_secs(5); +// Token bucket retry quota parameters based on the AWS SDKs standard retry mode, see +// https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html +const S3_RETRY_QUOTA_BUCKET_SIZE: u64 = 500; +const S3_RETRY_QUOTA_RATE: u64 = 50; +const S3_RETRY_QUOTA_COST: u64 = 5; +const S3_RETRY_QUOTA_TIMEOUT_COST: u64 = 10; static SHARED_COUNTER_FLUSHER: LazyLock> = LazyLock::new(|| RwLock::new(MmapFlusher::new())); @@ -201,6 +207,7 @@ pub struct S3Client { authority: Authority, active_request_rate_limiter: Option>, passive_request_rate_limiter: Option>, + retry_rate_limiter: Option>, // TODO: Drop for PBS 5. put_rate_limiter: Option>>, request_counters: Option>, @@ -267,7 +274,11 @@ impl S3Client { S3_TCP_KEEPIDLE_TIME, ); - let (mut passive_request_rate_limiter, mut active_request_rate_limiter) = (None, None); + let ( + mut passive_request_rate_limiter, + mut active_request_rate_limiter, + mut retry_rate_limiter, + ) = (None, None, None); if let Some(limiter_config) = &options.rate_limiter_config { if let Some(limit) = limiter_config.rate_in { let limiter = SharedRateLimiter::mmap_shmem( @@ -312,6 +323,15 @@ impl S3Client { )?; passive_request_rate_limiter = Some(Arc::new(limiter)); } + + let limiter = SharedRateLimiter::mmap_shmem( + &format!("{}.retry-quota", limiter_config.options.id), + S3_RETRY_QUOTA_RATE, + S3_RETRY_QUOTA_BUCKET_SIZE, + limiter_config.options.user.clone(), + limiter_config.options.base_path.clone(), + )?; + retry_rate_limiter = Some(Arc::new(limiter)); } if let Some(proxy_config) = &options.proxy_config { @@ -371,6 +391,7 @@ impl S3Client { authority, active_request_rate_limiter, passive_request_rate_limiter, + retry_rate_limiter, put_rate_limiter, request_counters, }) @@ -495,6 +516,11 @@ impl S3Client { let deadline = timeout.map(|timeout| tokio::time::Instant::now() + timeout); + if let Some(limiter) = &self.retry_rate_limiter { + let sleep = limiter.register_traffic(Instant::now(), 0); + tokio::time::sleep(sleep).await; + } + for retry in 0..MAX_S3_HTTP_REQUEST_RETRY { let mut request = Request::from_parts(parts.clone(), Body::from(body_bytes.clone())); if let Some(limiter) = &self.active_request_rate_limiter { @@ -588,7 +614,11 @@ impl S3Client { }; let backoff = retry_backoff(retry_reason, retry); - tokio::time::sleep(server_directed_delay(retry_after, backoff)).await; + let mut delay = server_directed_delay(retry_after, backoff); + if let Some(limiter) = &self.retry_rate_limiter { + delay += limiter.register_traffic(Instant::now(), retry_reason.quota_cost()); + } + tokio::time::sleep(delay).await; } bail!("failed to send request exceeding retries"); @@ -688,8 +718,12 @@ impl S3Client { if retry >= MAX_S3_HTTP_REQUEST_RETRY - 1 { return Err(err); } - tokio::time::sleep(retry_backoff_with_jitter(RetryReason::Transient, retry)) - .await; + let mut delay = retry_backoff_with_jitter(RetryReason::Transient, retry); + if let Some(limiter) = &self.retry_rate_limiter { + delay += limiter + .register_traffic(Instant::now(), RetryReason::Transient.quota_cost()); + } + tokio::time::sleep(delay).await; retry += 1; } } @@ -1063,7 +1097,12 @@ impl S3Client { } } } - tokio::time::sleep(retry_backoff_with_jitter(RetryReason::Transient, retry)).await; + let mut delay = retry_backoff_with_jitter(RetryReason::Transient, retry); + if let Some(limiter) = &self.retry_rate_limiter { + delay += + limiter.register_traffic(Instant::now(), RetryReason::Transient.quota_cost()); + } + tokio::time::sleep(delay).await; } Ok(false) } @@ -1115,7 +1154,7 @@ impl S3Client { } } -#[derive(Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq)] enum RetryReason { Transient, Throttling, @@ -1138,6 +1177,13 @@ impl RetryReason { Self::Throttling => S3_RETRY_BASE_DELAY_THROTTLING, } } + + fn quota_cost(self) -> u64 { + match self { + Self::Transient => S3_RETRY_QUOTA_TIMEOUT_COST, + Self::Throttling => S3_RETRY_QUOTA_COST, + } + } } /// Helper to get exponential backoff duration based on retry reason and count with -- 2.47.3