From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 8EEFA1FF0B6 for ; Thu, 12 Nov 2026 10:39:17 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3B4522162B; Thu, 12 Nov 2026 10:39:17 +0100 (CET) Date: Thu, 12 Nov 2026 10:39:08 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= Subject: Re: [PATCH proxmox-backup v2 3/5] fix #7606: verify: s3: handle transient chunk load errors To: Christian Ebner , pbs-devel@lists.proxmox.com References: <20261110153731.449671-1-c.ebner@proxmox.com> <20261110153731.449671-4-c.ebner@proxmox.com> In-Reply-To: <20261110153731.449671-4-c.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.17.0 (https://github.com/astroidmail/astroid) Message-Id: <1794475595.m77v4wiujn.astroid@yuna.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794476351896 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.538 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: ILL32NYZJ4SSPTHZ2IYEAOOJ5D2QBDUT X-Message-ID-Hash: ILL32NYZJ4SSPTHZ2IYEAOOJ5D2QBDUT X-MailFrom: f.gruenbichler@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On November 10, 2026 4:37 pm, Christian Ebner wrote: > Fetching an object for verification might fail when sending the > request to the S3 API or while receiving the response. Both are > currently handled as hard errors, leading to the manifest state being > set to verify failed. This however does not reflect the actual state, > as a subsequent verification can work just fine. >=20 > Therefore, distinguish and propagate these error kinds by enum > variants, adapting return types and call sites accordingly. >=20 > Handle the variants respectively when setting/updating the verify > state for the manifest: if only transient errors, but no permanent > errors occur, strip any verify okay state since then it is not > possible to determine if the snapshot is still fine. That is best > represented by not having a verify state, as consequence sync jobs > with verified-only filtering will then exclude such snapshots as > well. >=20 > Independent of the error variant, always keep a previously set verify > error state in order to avoid allowing backups to reuse this snapshot > as reference. This covers both the new state as well as the manifest > state prior to verification. >=20 > Fixes: https://bugzilla.proxmox.com/show_bug.cgi?id=3D7606 > Signed-off-by: Christian Ebner > --- > src/backup/verify.rs | 190 ++++++++++++++++++++++++++++++++----------- > 1 file changed, 143 insertions(+), 47 deletions(-) >=20 > diff --git a/src/backup/verify.rs b/src/backup/verify.rs > index 4d497c219..670a899ca 100644 > --- a/src/backup/verify.rs > +++ b/src/backup/verify.rs > @@ -4,7 +4,7 @@ use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}= ; > use std::sync::{Arc, Mutex}; > use std::time::Instant; > =20 > -use anyhow::{Error, bail}; > +use anyhow::{Error, format_err}; > use http_body_util::BodyExt; > use tracing::{error, info, warn}; > =20 > @@ -39,6 +39,7 @@ struct IndexVerifyState { > read_bytes: AtomicU64, > decoded_bytes: AtomicU64, > errors: AtomicUsize, > + fetch_errors: AtomicUsize, > datastore: Arc, > corrupt_chunks: Arc>>, > verified_chunks: Arc>>, > @@ -55,6 +56,7 @@ impl IndexVerifyState { > read_bytes: AtomicU64::new(0), > decoded_bytes: AtomicU64::new(0), > errors: AtomicUsize::new(0), > + fetch_errors: AtomicUsize::new(0), > datastore: Arc::clone(datastore), > corrupt_chunks: Arc::clone(corrupt_chunks), > verified_chunks: Arc::clone(verified_chunks), > @@ -63,6 +65,11 @@ impl IndexVerifyState { > } > } > =20 > +enum VerificationError { > + Transient(Error), > + Permanent(Error), > +} > + > impl VerifyWorker { > /// Creates a new VerifyWorker for a given task worker and datastore= . > pub fn new( > @@ -95,27 +102,38 @@ impl VerifyWorker { > }) > } > =20 > - fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<()= , Error> { > - let blob =3D backup_dir.load_blob(info.filename.as_ref())?; > + fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<()= , VerificationError> { > + let blob =3D backup_dir > + .load_blob(info.filename.as_ref()) > + .map_err(VerificationError::Permanent)?; > =20 > let raw_size =3D blob.raw_size(); > if raw_size !=3D info.size { > - bail!("wrong size ({} !=3D {})", info.size, raw_size); > + return Err(VerificationError::Permanent(format_err!( > + "wrong size ({} !=3D {})", > + info.size, > + raw_size > + ))); > } > =20 > let csum =3D openssl::sha::sha256(blob.raw_data()); > if csum !=3D info.csum { > - bail!("wrong index checksum"); > + return Err(VerificationError::Permanent(format_err!( > + "wrong index checksum" > + ))); > } > =20 > - match blob.crypt_mode()? { > + match blob.crypt_mode().map_err(VerificationError::Permanent)? { > CryptMode::Encrypt =3D> Ok(()), > CryptMode::None =3D> { > // digest already verified above > - blob.decode(None, None)?; > + blob.decode(None, None) > + .map_err(VerificationError::Permanent)?; > Ok(()) > } > - CryptMode::SignOnly =3D> bail!("Invalid CryptMode for blob")= , > + CryptMode::SignOnly =3D> Err(VerificationError::Permanent(fo= rmat_err!( > + "Invalid CryptMode for blob" > + ))), > } > } > =20 > @@ -123,7 +141,7 @@ impl VerifyWorker { > &self, > index: Box, > crypt_mode: CryptMode, > - ) -> Result<(), Error> { > + ) -> Result<(), VerificationError> { > let verify_state =3D Arc::new(IndexVerifyState::new( > &self.datastore, > &self.corrupt_chunks, > @@ -190,7 +208,8 @@ impl VerifyWorker { > =20 > let chunk_list =3D self > .datastore > - .get_chunks_in_order(&*index, skip_chunk, check_abort)?; > + .get_chunks_in_order(&*index, skip_chunk, check_abort) > + .map_err(VerificationError::Transient)?; this is a bit dangerous IMHO, it relies on get_chunks_in_order to only return an error on worker aborts (which is the case at the moment). would need at least a comment there that documents this behaviour and verify's reliance on it? also see next comment.. > =20 > let reader_pool =3D ParallelHandler::new("read chunks", self.rea= d_threads, { > let decoder_pool =3D decoder_pool.channel(); > @@ -207,8 +226,12 @@ impl VerifyWorker { > } > }); > for (pos, _) in chunk_list { > - self.worker.check_abort()?; > - self.worker.fail_on_shutdown()?; > + self.worker > + .check_abort() > + .map_err(VerificationError::Transient)?; > + self.worker > + .fail_on_shutdown() > + .map_err(VerificationError::Transient)?; should these be a third category that leaves the verification state untouched? if the verification aborts cause of these, it's not related to the verification itself at all? > =20 > let info =3D index.chunk_info(pos).unwrap(); > =20 > @@ -217,11 +240,17 @@ impl VerifyWorker { > continue; // already verified or marked corrupt > } > =20 > - reader_pool.send(info)?; > + reader_pool > + .send(info) > + .map_err(|err| VerificationError::Transient(err.into()))= ?; this needs to check for permanent errors first, else it could hide a permanent error that has occured already? or it could be treated like worker aborts, I haven't checked the full implications here.. > } > =20 > - reader_pool.complete()?; > - decoder_pool.complete()?; > + reader_pool > + .complete() > + .map_err(|err| VerificationError::Transient(err.into()))?; > + decoder_pool > + .complete() > + .map_err(|err| VerificationError::Transient(err.into()))?; same for these two > =20 > let elapsed =3D verify_state.start_time.elapsed().as_secs_f64(); > =20 > @@ -235,13 +264,22 @@ impl VerifyWorker { > let decode_speed =3D decoded_bytes_mib / elapsed; > =20 > let error_count =3D verify_state.errors.load(Ordering::SeqCst); > + let fetch_error_count =3D verify_state.fetch_errors.load(Orderin= g::SeqCst); > =20 > info!( > - " verified {read_bytes_mib:.2}/{decoded_bytes_mib:.2} MiB i= n {elapsed:.2} seconds, speed {read_speed:.2}/{decode_speed:.2} MiB/s ({err= or_count} errors)" > + " verified {read_bytes_mib:.2}/{decoded_bytes_mib:.2} MiB i= n {elapsed:.2} seconds, speed {read_speed:.2}/{decode_speed:.2} MiB/s ({err= or_count} errors, {fetch_error_count} fetch errors)" > ); > =20 > - if verify_state.errors.load(Ordering::SeqCst) > 0 { > - bail!("chunks could not be verified"); > + if error_count > 0 { > + return Err(VerificationError::Permanent(format_err!( > + "chunks could not be verified" > + ))); > + } like is done here? > + > + if fetch_error_count > 0 { > + return Err(VerificationError::Transient(format_err!( > + "chunks could not be fetched" > + ))); > } > =20 > Ok(()) > @@ -297,7 +335,7 @@ impl VerifyWorker { > } > } > Err(err) =3D> { > - verify_state.errors.fetch_add(1, Orderin= g::SeqCst); > + verify_state.fetch_errors.fetch_add(1, O= rdering::SeqCst); > error!("can't verify chunk, load failed = - {err}"); > } > } > @@ -311,7 +349,7 @@ impl VerifyWorker { > ), > ), > Err(err) =3D> { > - verify_state.errors.fetch_add(1, Ordering::SeqCs= t); > + verify_state.fetch_errors.fetch_add(1, Ordering:= :SeqCst); > error!("can't verify chunk, load failed - {err}"= ); > } > } > [..]