From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 5BD2A1FF09F for ; Thu, 03 Sep 2026 14:16:21 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 2E241215BC; Thu, 03 Sep 2026 14:16:21 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup 2/2] verify: s3: retry on transient response body collection errors Date: Thu, 3 Sep 2026 14:15:57 +0200 Message-ID: <20260903121557.420018-3-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260903121557.420018-1-c.ebner@proxmox.com> References: <20260903121557.420018-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1788437774448 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.677 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: YIZNX7BKHNWMCQUAE7ZWWLK76RDD4RSR X-Message-ID-Hash: YIZNX7BKHNWMCQUAE7ZWWLK76RDD4RSR X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: If collecting the response body for the S3 object fetching requests fail, retry with exponential backoff time and only consider this as transient fetch error on retry exhaustion. Sending the requests is already retried by the s3-client, so not to be repeated. This avoids that an otherwise long running verification which already downloaded a lot of chunks needs to be redone due to transient connection issues. Signed-off-by: Christian Ebner --- src/backup/verify.rs | 94 ++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/src/backup/verify.rs b/src/backup/verify.rs index e85e120e5..b558bcaf1 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -2,7 +2,7 @@ use pbs_config::BackupLockGuard; use std::collections::HashSet; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::time::{Duration, Instant}; use anyhow::{Error, format_err}; use http_body_util::BodyExt; @@ -305,48 +305,64 @@ impl VerifyWorker { }, DatastoreBackend::S3(s3_client) => { let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?; - match proxmox_async::runtime::block_on(s3_client.get_object(object_key)) { - Ok(Some(response)) => { - match proxmox_async::runtime::block_on(response.content.collect()) { - Ok(raw_chunk) => { - match DataBlob::from_raw(raw_chunk.to_bytes().to_vec()) { - Ok(chunk) => { - let size = info.size(); - verify_state - .read_bytes - .fetch_add(chunk.raw_size(), Ordering::SeqCst); - decoder_pool.send((chunk, info.digest, size))?; - verify_state - .decoded_bytes - .fetch_add(size, Ordering::SeqCst); - } - Err(err) => Self::add_corrupt_chunk( - verify_state, - info.digest, - &format!( - "can't verify chunk with digest {} - {err}", - hex::encode(info.digest) + + for retry in 0..3 { + let verify_state = Arc::clone(&verify_state); + match proxmox_async::runtime::block_on(s3_client.get_object(object_key.clone())) + { + Ok(Some(response)) => { + match proxmox_async::runtime::block_on(response.content.collect()) { + Ok(raw_chunk) => { + match DataBlob::from_raw(raw_chunk.to_bytes().to_vec()) { + Ok(chunk) => { + let size = info.size(); + verify_state + .read_bytes + .fetch_add(chunk.raw_size(), Ordering::SeqCst); + decoder_pool.send((chunk, info.digest, size))?; + verify_state + .decoded_bytes + .fetch_add(size, Ordering::SeqCst); + } + Err(err) => Self::add_corrupt_chunk( + verify_state, + info.digest, + &format!( + "can't verify chunk with digest {} - {err}", + hex::encode(info.digest) + ), ), - ), + } + break; + } + Err(err) => { + if retry < 3 { + warn!("retry reading chunk, load failed - {err}"); + let backoff = Duration::from_secs(3_u64.pow(retry)); + std::thread::sleep(backoff); + } else { + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); + error!("can't verify chunk, load failed - {err}"); + } } - } - Err(err) => { - verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); - error!("can't verify chunk, load failed - {err}"); } } - } - Ok(None) => Self::add_corrupt_chunk( - verify_state, - info.digest, - &format!( - "can't verify missing chunk with digest {}", - hex::encode(info.digest) - ), - ), - Err(err) => { - verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); - error!("can't verify chunk, load failed - {err}"); + Ok(None) => { + Self::add_corrupt_chunk( + verify_state, + info.digest, + &format!( + "can't verify missing chunk with digest {}", + hex::encode(info.digest) + ), + ); + break; + } + Err(err) => { + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); + error!("can't verify chunk, load failed - {err}"); + break; + } } } } -- 2.47.3