From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id A4ED21FF0B6 for ; Thu, 12 Nov 2026 11:09:15 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 70DBF21613; Thu, 12 Nov 2026 11:09:15 +0100 (CET) Message-ID: Date: Thu, 12 Nov 2026 11:09:10 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird Subject: Re: [PATCH proxmox-backup v2 4/5] verify: s3: retry on transient response body collection errors To: =?UTF-8?Q?Fabian_Gr=C3=BCnbichler?= , pbs-devel@lists.proxmox.com References: <20261110153731.449671-1-c.ebner@proxmox.com> <20261110153731.449671-5-c.ebner@proxmox.com> <1794476595.f1ixh0zxb1.astroid@yuna.none> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: <1794476595.f1ixh0zxb1.astroid@yuna.none> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794478150772 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.413 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: QBVHWYT3CE57ZIYQ55TP77FU7ZPMF7PS X-Message-ID-Hash: QBVHWYT3CE57ZIYQ55TP77FU7ZPMF7PS X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On 11/12/26 10:48 AM, Fabian Grünbichler wrote: > On November 10, 2026 4:37 pm, Christian Ebner wrote: >> If collecting the response body for the S3 object fetching requests >> fail, retry with exponential backoff time and only consider this >> as transient fetch error on retry exhaustion. Sending the requests >> is already retried by the s3-client, so not to be repeated. >> >> This avoids that an otherwise long running verification which already >> downloaded a lot of chunks needs to be redone due to transient >> connection issues. > > not super happy about backoff handling in two places now.. could we move > this into proxmox_s3 by adding a `get_object_content` helper there? > > except for the "does S3 work" checks, we always immediately collect the > object response anyway on the PBS side, and I guess if we want to retry > collecting when verifying, we also want when restoring/.. ? Yeah, agreed: best to move and integrate into proxmox-s3-client crate and adapt all object fetching related call sites to use that as well. >> >> Signed-off-by: Christian Ebner >> --- >> src/backup/verify.rs | 102 ++++++++++++++++++++++++++----------------- >> 1 file changed, 63 insertions(+), 39 deletions(-) >> >> diff --git a/src/backup/verify.rs b/src/backup/verify.rs >> index 670a899ca..edfb52a5d 100644 >> --- a/src/backup/verify.rs >> +++ b/src/backup/verify.rs >> @@ -2,7 +2,7 @@ use pbs_config::BackupLockGuard; >> use std::collections::HashSet; >> use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; >> use std::sync::{Arc, Mutex}; >> -use std::time::Instant; >> +use std::time::{Duration, Instant}; >> >> use anyhow::{Error, format_err}; >> use http_body_util::BodyExt; >> @@ -309,48 +309,72 @@ impl VerifyWorker { >> }, >> DatastoreBackend::S3(s3_client) => { >> let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?; >> - match proxmox_async::runtime::block_on(s3_client.get_object(object_key)) { >> - Ok(Some(response)) => { >> - match proxmox_async::runtime::block_on(response.content.collect()) { >> - Ok(raw_chunk) => { >> - match DataBlob::from_raw(raw_chunk.to_bytes().to_vec()) { >> - Ok(chunk) => { >> - let size = info.size(); >> - verify_state >> - .read_bytes >> - .fetch_add(chunk.raw_size(), Ordering::SeqCst); >> - decoder_pool.send((chunk, info.digest, size))?; >> - verify_state >> - .decoded_bytes >> - .fetch_add(size, Ordering::SeqCst); >> - } >> - Err(err) => Self::add_corrupt_chunk( >> - verify_state, >> - info.digest, >> - &format!( >> - "can't verify chunk with digest {} - {err}", >> - hex::encode(info.digest) >> + >> + for retry in 0..3 { >> + match proxmox_async::runtime::block_on(s3_client.get_object(object_key.clone())) >> + { >> + Ok(Some(response)) => { >> + match proxmox_async::runtime::block_on(response.content.collect()) { >> + Ok(raw_chunk) => { >> + match DataBlob::from_raw(raw_chunk.to_bytes().to_vec()) { >> + Ok(chunk) => { >> + let size = info.size(); >> + verify_state >> + .read_bytes >> + .fetch_add(chunk.raw_size(), Ordering::SeqCst); >> + decoder_pool.send((chunk, info.digest, size))?; >> + verify_state >> + .decoded_bytes >> + .fetch_add(size, Ordering::SeqCst); >> + } >> + Err(err) => Self::add_corrupt_chunk( >> + verify_state, >> + info.digest, >> + &format!( >> + "can't verify chunk with digest {} - {err}", >> + hex::encode(info.digest) >> + ), >> ), >> - ), >> + } >> + break; >> + } >> + Err(err) => { >> + if retry < 2 { >> + warn!( >> + "retry reading chunk with digest {}, load failed - {err}", >> + hex::encode(info.digest), >> + ); >> + let backoff = Duration::from_secs(3_u64.pow(retry)); >> + std::thread::sleep(backoff); >> + } else { >> + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> + error!( >> + "can't verify chunk with digest {}, load failed - {err}", >> + hex::encode(info.digest), >> + ); >> + } >> } >> - } >> - Err(err) => { >> - verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> - error!("can't verify chunk, load failed - {err}"); >> } >> } >> - } >> - Ok(None) => Self::add_corrupt_chunk( >> - verify_state, >> - info.digest, >> - &format!( >> - "can't verify missing chunk with digest {}", >> - hex::encode(info.digest) >> - ), >> - ), >> - Err(err) => { >> - verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> - error!("can't verify chunk, load failed - {err}"); >> + Ok(None) => { >> + Self::add_corrupt_chunk( >> + verify_state, >> + info.digest, >> + &format!( >> + "can't verify missing chunk with digest {}", >> + hex::encode(info.digest) >> + ), >> + ); >> + break; >> + } >> + Err(err) => { >> + verify_state.fetch_errors.fetch_add(1, Ordering::SeqCst); >> + error!( >> + "can't verify chunk with digest {}, load failed - {err}", >> + hex::encode(info.digest), >> + ); >> + break; >> + } >> } >> } >> } >> -- >> 2.47.3 >> >> >> >> >> >>