From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 7AF781FF0B6 for ; Thu, 12 Nov 2026 19:02:31 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id E9355216D5; Thu, 12 Nov 2026 19:02:17 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v3 11/11] verify: s3: stop early if the endpoint appears unreachable Date: Thu, 12 Nov 2026 19:01:19 +0100 Message-ID: <20261112180119.660923-12-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112180119.660923-1-c.ebner@proxmox.com> References: <20261112180119.660923-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794506503638 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.403 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: WS6Y6LE67M3JS32Y5UJZRZSFUFH6BX5K X-Message-ID-Hash: WS6Y6LE67M3JS32Y5UJZRZSFUFH6BX5K X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: If the S3 endpoint cannot be reached, every remaining chunk fetch of a snapshot fails only after the request retries and their backoff. This causes useless requests and can block the verification for a very long time, while the outcome is already known, as the snapshot cannot be marked as verified anyway. Stopping on the first transient error is not an option though: a snapshot without verify state can still be used as reference by the next backup, so corrupt chunks which would be detected later in the same run must not be missed because of a single failed request. Therefore, only stop verifying the chunks and the remaining archives of a snapshot once a few chunk fetches failed in a row, with any successful fetch resetting the count. Errors detected up to that point are still accounted for, so known bad snapshots are marked as failed. The count is reset for each snapshot, so verification continues normally once the endpoint becomes reachable again. Signed-off-by: Christian Ebner --- src/backup/verify.rs | 64 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 624c9a9aa..c29e60678 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -22,6 +22,10 @@ use pbs_datastore::{DataBlob, DataStore, DatastoreBackend, StoreProgress}; use crate::backup::hierarchy::ListAccessibleBackupGroups; +/// Number of chunk fetches from the S3 backend failing in a row, after which the endpoint is +/// considered unreachable and verification of the current snapshot is stopped. +const S3_MAX_CONSECUTIVE_FETCH_ERRORS: usize = 3; + /// A VerifyWorker encapsulates a task worker, datastore and information about which chunks have /// already been verified or detected as corrupt. pub struct VerifyWorker { @@ -32,6 +36,7 @@ pub struct VerifyWorker { backend: DatastoreBackend, read_threads: usize, verify_threads: usize, + consecutive_fetch_errors: Arc, } struct IndexVerifyState { @@ -43,6 +48,7 @@ struct IndexVerifyState { corrupt_chunks: Arc>>, verified_chunks: Arc>>, start_time: Instant, + consecutive_fetch_errors: Arc, } impl IndexVerifyState { @@ -50,6 +56,7 @@ impl IndexVerifyState { datastore: &Arc, corrupt_chunks: &Arc>>, verified_chunks: &Arc>>, + consecutive_fetch_errors: &Arc, ) -> Self { Self { read_bytes: AtomicU64::new(0), @@ -60,6 +67,7 @@ impl IndexVerifyState { corrupt_chunks: Arc::clone(corrupt_chunks), verified_chunks: Arc::clone(verified_chunks), start_time: Instant::now(), + consecutive_fetch_errors: Arc::clone(consecutive_fetch_errors), } } } @@ -104,9 +112,17 @@ impl VerifyWorker { backend, read_threads, verify_threads, + consecutive_fetch_errors: Arc::new(AtomicUsize::new(0)), }) } + /// Check if fetching chunks from the S3 backend failed too often in a row. A single transient + /// error must not stop the verification, as that could hide corrupt chunks which would then + /// be reused by subsequent backups. + fn s3_endpoint_unreachable(&self) -> bool { + self.consecutive_fetch_errors.load(Ordering::SeqCst) >= S3_MAX_CONSECUTIVE_FETCH_ERRORS + } + fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<(), VerificationError> { let blob = backup_dir .load_blob(info.filename.as_ref()) @@ -151,6 +167,7 @@ impl VerifyWorker { &self.datastore, &self.corrupt_chunks, &self.verified_chunks, + &self.consecutive_fetch_errors, )); let decoder_pool = ParallelHandler::new("verify chunk decoder", self.verify_threads, { @@ -248,12 +265,20 @@ impl VerifyWorker { }); let mut chunk_send_error: Option = None; + let mut exhausted_error: Option = None; for (pos, _) in chunk_list { self.worker.check_abort().map_err(VerificationError::Task)?; self.worker .fail_on_shutdown() .map_err(VerificationError::Task)?; + if self.s3_endpoint_unreachable() { + // avoid further requests and long stalls, the archive cannot be verified anyway + warn!("stop verifying chunks, too many consecutive fetch errors"); + exhausted_error = Some(format_err!("S3 endpoint appears to be unreachable")); + break; + } + let info = index.chunk_info(pos).unwrap(); // we must always recheck this here, the parallel worker below alter it! @@ -271,6 +296,11 @@ impl VerifyWorker { let reader_result = reader_pool.complete(); let decoder_result = decoder_pool.complete(); + if exhausted_error.is_none() && self.s3_endpoint_unreachable() { + warn!("stop verifying chunks, too many consecutive fetch errors"); + exhausted_error = Some(format_err!("S3 endpoint appears to be unreachable")); + } + let elapsed = verify_state.start_time.elapsed().as_secs_f64(); let read_bytes = verify_state.read_bytes.load(Ordering::SeqCst); @@ -301,6 +331,12 @@ impl VerifyWorker { )); } + if let Some(err) = exhausted_error { + return Err(VerificationError::Transient( + err.context("incomplete verification"), + )); + } + if let Some(err) = chunk_send_error { return Err(VerificationError::Transient( err.context("incomplete verification because of chunk send error"), @@ -351,8 +387,21 @@ impl VerifyWorker { } }, DatastoreBackend::S3(s3_client) => { + if verify_state.consecutive_fetch_errors.load(Ordering::SeqCst) + >= S3_MAX_CONSECUTIVE_FETCH_ERRORS + { + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); + return Ok(()); + } let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?; - match proxmox_async::runtime::block_on(s3_client.get_object_content(object_key)) { + let result = + proxmox_async::runtime::block_on(s3_client.get_object_content(object_key)); + if result.is_ok() { + verify_state + .consecutive_fetch_errors + .store(0, Ordering::SeqCst); + } + match result { Ok(Some(raw_chunk)) => match DataBlob::from_raw(raw_chunk.to_vec()) { Ok(chunk) => { let size = info.size(); @@ -381,6 +430,9 @@ impl VerifyWorker { ), Err(err) => { verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); + verify_state + .consecutive_fetch_errors + .fetch_add(1, Ordering::SeqCst); error!("can't verify chunk, load failed - {err}"); } } @@ -532,6 +584,9 @@ impl VerifyWorker { info!("verify {datastore_name}:{backup_dir_name}"); + // probe the endpoint again for each snapshot, it might have become reachable again + self.consecutive_fetch_errors.store(0, Ordering::SeqCst); + let mut error_count = 0; let mut verify_result = Some(VerifyState::Ok); @@ -570,6 +625,13 @@ impl VerifyWorker { "verify {datastore_name}:{backup_dir_name}/{file_name} failed with transient error: {err}", file_name = info.filename, ); + + if self.s3_endpoint_unreachable() { + warn!( + "skip verifying remaining archives of {datastore_name}:{backup_dir_name}" + ); + break; + } } } } -- 2.47.3