From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id F04551FF0B6 for ; Thu, 12 Nov 2026 11:06:38 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3827C21625; Thu, 12 Nov 2026 11:06:38 +0100 (CET) Message-ID: Date: Thu, 12 Nov 2026 11:06:33 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird Subject: Re: [PATCH proxmox-backup v2 3/5] fix #7606: verify: s3: handle transient chunk load errors To: =?UTF-8?Q?Fabian_Gr=C3=BCnbichler?= , pbs-devel@lists.proxmox.com References: <20261110153731.449671-1-c.ebner@proxmox.com> <20261110153731.449671-4-c.ebner@proxmox.com> <1794475595.m77v4wiujn.astroid@yuna.none> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: <1794475595.m77v4wiujn.astroid@yuna.none> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794477993674 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.414 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: OT4YY5YPOUSOO4XQOPIPIT36FMVCJVCD X-Message-ID-Hash: OT4YY5YPOUSOO4XQOPIPIT36FMVCJVCD X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On 11/12/26 10:39 AM, Fabian Grünbichler wrote: > On November 10, 2026 4:37 pm, Christian Ebner wrote: >> Fetching an object for verification might fail when sending the >> request to the S3 API or while receiving the response. Both are >> currently handled as hard errors, leading to the manifest state being >> set to verify failed. This however does not reflect the actual state, >> as a subsequent verification can work just fine. >> >> Therefore, distinguish and propagate these error kinds by enum >> variants, adapting return types and call sites accordingly. >> >> Handle the variants respectively when setting/updating the verify >> state for the manifest: if only transient errors, but no permanent >> errors occur, strip any verify okay state since then it is not >> possible to determine if the snapshot is still fine. That is best >> represented by not having a verify state, as consequence sync jobs >> with verified-only filtering will then exclude such snapshots as >> well. >> >> Independent of the error variant, always keep a previously set verify >> error state in order to avoid allowing backups to reuse this snapshot >> as reference. This covers both the new state as well as the manifest >> state prior to verification. >> >> Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=7606 >> Signed-off-by: Christian Ebner >> --- >> src/backup/verify.rs | 190 ++++++++++++++++++++++++++++++++----------- >> 1 file changed, 143 insertions(+), 47 deletions(-) >> >> diff --git a/src/backup/verify.rs b/src/backup/verify.rs >> index 4d497c219..670a899ca 100644 >> --- a/src/backup/verify.rs >> +++ b/src/backup/verify.rs >> @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; >> use std::sync::{Arc, Mutex}; >> use std::time::Instant; >> >> -use anyhow::{Error, bail}; >> +use anyhow::{Error, format_err}; >> use http_body_util::BodyExt; >> use tracing::{error, info, warn}; >> >> @@ -39,6 +39,7 @@ struct IndexVerifyState { >> read_bytes: AtomicU64, >> decoded_bytes: AtomicU64, >> errors: AtomicUsize, >> + fetch_errors: AtomicUsize, >> datastore: Arc, >> corrupt_chunks: Arc>>, >> verified_chunks: Arc>>, >> @@ -55,6 +56,7 @@ impl IndexVerifyState { >> read_bytes: AtomicU64::new(0), >> decoded_bytes: AtomicU64::new(0), >> errors: AtomicUsize::new(0), >> + fetch_errors: AtomicUsize::new(0), >> datastore: Arc::clone(datastore), >> corrupt_chunks: Arc::clone(corrupt_chunks), >> verified_chunks: Arc::clone(verified_chunks), >> @@ -63,6 +65,11 @@ impl IndexVerifyState { >> } >> } >> >> +enum VerificationError { >> + Transient(Error), >> + Permanent(Error), >> +} >> + >> impl VerifyWorker { >> /// Creates a new VerifyWorker for a given task worker and datastore. >> pub fn new( >> @@ -95,27 +102,38 @@ impl VerifyWorker { >> }) >> } >> >> - fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { >> - let blob = backup_dir.load_blob(info.filename.as_ref())?; >> + fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<(), VerificationError> { >> + let blob = backup_dir >> + .load_blob(info.filename.as_ref()) >> + .map_err(VerificationError::Permanent)?; >> >> let raw_size = blob.raw_size(); >> if raw_size != info.size { >> - bail!("wrong size ({} != {})", info.size, raw_size); >> + return Err(VerificationError::Permanent(format_err!( >> + "wrong size ({} != {})", >> + info.size, >> + raw_size >> + ))); >> } >> >> let csum = openssl::sha::sha256(blob.raw_data()); >> if csum != info.csum { >> - bail!("wrong index checksum"); >> + return Err(VerificationError::Permanent(format_err!( >> + "wrong index checksum" >> + ))); >> } >> >> - match blob.crypt_mode()? { >> + match blob.crypt_mode().map_err(VerificationError::Permanent)? { >> CryptMode::Encrypt => Ok(()), >> CryptMode::None => { >> // digest already verified above >> - blob.decode(None, None)?; >> + blob.decode(None, None) >> + .map_err(VerificationError::Permanent)?; >> Ok(()) >> } >> - CryptMode::SignOnly => bail!("Invalid CryptMode for blob"), >> + CryptMode::SignOnly => Err(VerificationError::Permanent(format_err!( >> + "Invalid CryptMode for blob" >> + ))), >> } >> } >> >> @@ -123,7 +141,7 @@ impl VerifyWorker { >> &self, >> index: Box, >> crypt_mode: CryptMode, >> - ) -> Result<(), Error> { >> + ) -> Result<(), VerificationError> { >> let verify_state = Arc::new(IndexVerifyState::new( >> &self.datastore, >> &self.corrupt_chunks, >> @@ -190,7 +208,8 @@ impl VerifyWorker { >> >> let chunk_list = self >> .datastore >> - .get_chunks_in_order(&*index, skip_chunk, check_abort)?; >> + .get_chunks_in_order(&*index, skip_chunk, check_abort) >> + .map_err(VerificationError::Transient)?; > > this is a bit dangerous IMHO, it relies on get_chunks_in_order to only > return an error on worker aborts (which is the case at the moment). > would need at least a comment there that documents this behaviour and > verify's reliance on it? > > also see next comment.. > >> >> let reader_pool = ParallelHandler::new("read chunks", self.read_threads, { >> let decoder_pool = decoder_pool.channel(); >> @@ -207,8 +226,12 @@ impl VerifyWorker { >> } >> }); >> for (pos, _) in chunk_list { >> - self.worker.check_abort()?; >> - self.worker.fail_on_shutdown()?; >> + self.worker >> + .check_abort() >> + .map_err(VerificationError::Transient)?; >> + self.worker >> + .fail_on_shutdown() >> + .map_err(VerificationError::Transient)?; > > should these be a third category that leaves the verification state > untouched? if the verification aborts cause of these, it's not related > to the verification itself at all? Yeah, makes sense to have VerificationError::Task to correctly distinguish these as well. Will adapt for v3. > >> >> let info = index.chunk_info(pos).unwrap(); >> >> @@ -217,11 +240,17 @@ impl VerifyWorker { >> continue; // already verified or marked corrupt >> } >> >> - reader_pool.send(info)?; >> + reader_pool >> + .send(info) >> + .map_err(|err| VerificationError::Transient(err.into()))?; > > this needs to check for permanent errors first, else it could hide a > permanent error that has occured already? or it could be treated like > worker aborts, I haven't checked the full implications here.. good catch! the send error should never hide permanent errors > >> } >> >> - reader_pool.complete()?; >> - decoder_pool.complete()?; >> + reader_pool >> + .complete() >> + .map_err(|err| VerificationError::Transient(err.into()))?; >> + decoder_pool >> + .complete() >> + .map_err(|err| VerificationError::Transient(err.into()))?; > > same for these two > >> >> let elapsed = verify_state.start_time.elapsed().as_secs_f64(); >> >> @@ -235,13 +264,22 @@ impl VerifyWorker { >> let decode_speed = decoded_bytes_mib / elapsed; >> >> let error_count = verify_state.errors.load(Ordering::SeqCst); >> + let fetch_error_count = verify_state.fetch_errors.load(Ordering::SeqCst); >> >> info!( >> - " verified {read_bytes_mib:.2}/{decoded_bytes_mib:.2} MiB in {elapsed:.2} seconds, speed {read_speed:.2}/{decode_speed:.2} MiB/s ({error_count} errors)" >> + " verified {read_bytes_mib:.2}/{decoded_bytes_mib:.2} MiB in {elapsed:.2} seconds, speed {read_speed:.2}/{decode_speed:.2} MiB/s ({error_count} errors, {fetch_error_count} fetch errors)" >> ); >> >> - if verify_state.errors.load(Ordering::SeqCst) > 0 { >> - bail!("chunks could not be verified"); >> + if error_count > 0 { >> + return Err(VerificationError::Permanent(format_err!( >> + "chunks could not be verified" >> + ))); >> + } > > like is done here? > >> + >> + if fetch_error_count > 0 { >> + return Err(VerificationError::Transient(format_err!( >> + "chunks could not be fetched" >> + ))); >> } >> >> Ok(()) >> @@ -297,7 +335,7 @@ impl VerifyWorker { >> } >> } >> Err(err) => { >> - verify_state.errors.fetch_add(1, Ordering::SeqCst); >> + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> error!("can't verify chunk, load failed - {err}"); >> } >> } >> @@ -311,7 +349,7 @@ impl VerifyWorker { >> ), >> ), >> Err(err) => { >> - verify_state.errors.fetch_add(1, Ordering::SeqCst); >> + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> error!("can't verify chunk, load failed - {err}"); >> } >> } > >> [..]