From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 280C01FF0B6 for ; Thu, 12 Nov 2026 10:49:01 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id EC3BF2161C; Thu, 12 Nov 2026 10:49:00 +0100 (CET) Date: Thu, 12 Nov 2026 10:48:54 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= Subject: Re: [PATCH proxmox-backup v2 4/5] verify: s3: retry on transient response body collection errors To: Christian Ebner , pbs-devel@lists.proxmox.com References: <20261110153731.449671-1-c.ebner@proxmox.com> <20261110153731.449671-5-c.ebner@proxmox.com> In-Reply-To: <20261110153731.449671-5-c.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.17.0 (https://github.com/astroidmail/astroid) Message-Id: <1794476595.f1ixh0zxb1.astroid@yuna.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794476937364 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.527 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: EPC72NX2ZBVMPTGQDKJHVL5TE4JV7GQM X-Message-ID-Hash: EPC72NX2ZBVMPTGQDKJHVL5TE4JV7GQM X-MailFrom: f.gruenbichler@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On November 10, 2026 4:37 pm, Christian Ebner wrote: > If collecting the response body for the S3 object fetching requests > fail, retry with exponential backoff time and only consider this > as transient fetch error on retry exhaustion. Sending the requests > is already retried by the s3-client, so not to be repeated. >=20 > This avoids that an otherwise long running verification which already > downloaded a lot of chunks needs to be redone due to transient > connection issues. not super happy about backoff handling in two places now.. could we move this into proxmox_s3 by adding a `get_object_content` helper there? except for the "does S3 work" checks, we always immediately collect the object response anyway on the PBS side, and I guess if we want to retry collecting when verifying, we also want when restoring/.. ? >=20 > Signed-off-by: Christian Ebner > --- > src/backup/verify.rs | 102 ++++++++++++++++++++++++++----------------- > 1 file changed, 63 insertions(+), 39 deletions(-) >=20 > diff --git a/src/backup/verify.rs b/src/backup/verify.rs > index 670a899ca..edfb52a5d 100644 > --- a/src/backup/verify.rs > +++ b/src/backup/verify.rs > @@ -2,7 +2,7 @@ use pbs_config::BackupLockGuard; > use std::collections::HashSet; > use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; > use std::sync::{Arc, Mutex}; > -use std::time::Instant; > +use std::time::{Duration, Instant}; > =20 > use anyhow::{Error, format_err}; > use http_body_util::BodyExt; > @@ -309,48 +309,72 @@ impl VerifyWorker { > }, > DatastoreBackend::S3(s3_client) =3D> { > let object_key =3D pbs_datastore::s3::object_key_from_di= gest(&info.digest)?; > - match proxmox_async::runtime::block_on(s3_client.get_obj= ect(object_key)) { > - Ok(Some(response)) =3D> { > - match proxmox_async::runtime::block_on(response.= content.collect()) { > - Ok(raw_chunk) =3D> { > - match DataBlob::from_raw(raw_chunk.to_by= tes().to_vec()) { > - Ok(chunk) =3D> { > - let size =3D info.size(); > - verify_state > - .read_bytes > - .fetch_add(chunk.raw_size(),= Ordering::SeqCst); > - decoder_pool.send((chunk, info.d= igest, size))?; > - verify_state > - .decoded_bytes > - .fetch_add(size, Ordering::S= eqCst); > - } > - Err(err) =3D> Self::add_corrupt_chun= k( > - verify_state, > - info.digest, > - &format!( > - "can't verify chunk with dig= est {} - {err}", > - hex::encode(info.digest) > + > + for retry in 0..3 { > + match proxmox_async::runtime::block_on(s3_client.get= _object(object_key.clone())) > + { > + Ok(Some(response)) =3D> { > + match proxmox_async::runtime::block_on(respo= nse.content.collect()) { > + Ok(raw_chunk) =3D> { > + match DataBlob::from_raw(raw_chunk.t= o_bytes().to_vec()) { > + Ok(chunk) =3D> { > + let size =3D info.size(); > + verify_state > + .read_bytes > + .fetch_add(chunk.raw_siz= e(), Ordering::SeqCst); > + decoder_pool.send((chunk, in= fo.digest, size))?; > + verify_state > + .decoded_bytes > + .fetch_add(size, Orderin= g::SeqCst); > + } > + Err(err) =3D> Self::add_corrupt_= chunk( > + verify_state, > + info.digest, > + &format!( > + "can't verify chunk with= digest {} - {err}", > + hex::encode(info.digest) > + ), > ), > - ), > + } > + break; > + } > + Err(err) =3D> { > + if retry < 2 { > + warn!( > + "retry reading chunk with di= gest {}, load failed - {err}", > + hex::encode(info.digest), > + ); > + let backoff =3D Duration::from_s= ecs(3_u64.pow(retry)); > + std::thread::sleep(backoff); > + } else { > + verify_state.fetch_errors.fetch_= add(1, Ordering::SeqCst); > + error!( > + "can't verify chunk with dig= est {}, load failed - {err}", > + hex::encode(info.digest), > + ); > + } > } > - } > - Err(err) =3D> { > - verify_state.fetch_errors.fetch_add(1, O= rdering::SeqCst); > - error!("can't verify chunk, load failed = - {err}"); > } > } > - } > - Ok(None) =3D> Self::add_corrupt_chunk( > - verify_state, > - info.digest, > - &format!( > - "can't verify missing chunk with digest {}", > - hex::encode(info.digest) > - ), > - ), > - Err(err) =3D> { > - verify_state.fetch_errors.fetch_add(1, Ordering:= :SeqCst); > - error!("can't verify chunk, load failed - {err}"= ); > + Ok(None) =3D> { > + Self::add_corrupt_chunk( > + verify_state, > + info.digest, > + &format!( > + "can't verify missing chunk with dig= est {}", > + hex::encode(info.digest) > + ), > + ); > + break; > + } > + Err(err) =3D> { > + verify_state.fetch_errors.fetch_add(1, Order= ing::SeqCst); > + error!( > + "can't verify chunk with digest {}, load= failed - {err}", > + hex::encode(info.digest), > + ); > + break; > + } > } > } > } > --=20 > 2.47.3 >=20 >=20 >=20 >=20 >=20 >=20