From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id DFD2C1FF0B6 for ; Thu, 12 Nov 2026 19:02:32 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id C59E621643; Thu, 12 Nov 2026 19:02:18 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v3 08/11] s3: retry fetching object contents on body collection errors Date: Thu, 12 Nov 2026 19:01:16 +0100 Message-ID: <20261112180119.660923-9-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112180119.660923-1-c.ebner@proxmox.com> References: <20261112180119.660923-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794506502832 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.404 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: F2TND3A7OX2JIFVU3N4XIFOR2COWUGDE X-Message-ID-Hash: F2TND3A7OX2JIFVU3N4XIFOR2COWUGDE X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Requests to the S3 API are retried by the client on connection errors and some error responses, but the connection can still break while receiving the response body. Since the body is always collected right after the request, such errors currently fail chunk reads for restore and verification, cache refills, and the datastore refresh from the bucket, even though repeating the request would most likely succeed. Use the s3-client helper which implements the retry logic on fetch errors. The in-use marker check now fails if the marker content cannot be received, instead of falling back to an empty content, but both cases refuse to reuse the bucket anyway. As a further positive side-effect the pbs-datastore create now no longer depends on http-body-util and in particular the error handling logic for chunk fetching during verification gets simpler. Signed-off-by: Christian Ebner --- pbs-datastore/Cargo.toml | 1 - pbs-datastore/src/datastore.rs | 12 ++--- pbs-datastore/src/local_chunk_reader.rs | 4 +- .../src/local_datastore_lru_cache.rs | 6 +-- src/api2/config/datastore.rs | 13 ++--- src/api2/reader/mod.rs | 4 +- src/backup/verify.rs | 49 +++++++------------ 7 files changed, 30 insertions(+), 59 deletions(-) diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml index b51438dc6..b60a827f3 100644 --- a/pbs-datastore/Cargo.toml +++ b/pbs-datastore/Cargo.toml @@ -13,7 +13,6 @@ crc32fast.workspace = true endian_trait.workspace = true futures.workspace = true hex = { workspace = true, features = [ "serde" ] } -http-body-util.workspace = true hyper.workspace = true libc.workspace = true log.workspace = true diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index e6766a3b9..d62b21f9e 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -8,7 +8,6 @@ use std::sync::{Arc, LazyLock, Mutex}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Error, bail, format_err}; -use http_body_util::BodyExt; use hyper::Method; use hyper::body::Bytes; use nix::unistd::{UnlinkatFlags, unlinkat}; @@ -3262,18 +3261,13 @@ impl DataStore { .await .with_context(|| format!("failed to create target file {file_path:?}"))?; - if let Some(response) = s3_client - .get_object(object.key) + if let Some(data) = s3_client + .get_object_content(object.key) .await .with_context(|| format!("failed to fetch object {object_path:?}"))? { - let data = response - .content - .collect() - .await - .context("failed to collect object contents")?; target_file - .write_all(&data.to_bytes()) + .write_all(&data) .await .context("failed to write to target file")?; file_create_options diff --git a/pbs-datastore/src/local_chunk_reader.rs b/pbs-datastore/src/local_chunk_reader.rs index faae43b71..431d6f89b 100644 --- a/pbs-datastore/src/local_chunk_reader.rs +++ b/pbs-datastore/src/local_chunk_reader.rs @@ -3,7 +3,6 @@ use std::pin::Pin; use std::sync::Arc; use anyhow::{Error, bail, format_err}; -use http_body_util::BodyExt; use pbs_api_types::CryptMode; use pbs_tools::crypt_config::CryptConfig; @@ -55,8 +54,7 @@ impl LocalChunkReader { async fn fetch(s3_client: Arc, digest: &[u8; 32]) -> Result { let object_key = crate::s3::object_key_from_digest(digest)?; - if let Some(response) = s3_client.get_object(object_key).await? { - let bytes = response.content.collect().await?.to_bytes(); + if let Some(bytes) = s3_client.get_object_content(object_key).await? { DataBlob::from_raw(bytes.to_vec()) } else { bail!("no object with digest {}", hex::encode(digest)); diff --git a/pbs-datastore/src/local_datastore_lru_cache.rs b/pbs-datastore/src/local_datastore_lru_cache.rs index 1bde8ee5a..12e6e3683 100644 --- a/pbs-datastore/src/local_datastore_lru_cache.rs +++ b/pbs-datastore/src/local_datastore_lru_cache.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use anyhow::{Error, bail}; -use http_body_util::BodyExt; use pbs_tools::async_lru_cache::AsyncLruCache; use proxmox_s3_client::S3Client; @@ -124,12 +123,11 @@ impl LocalDatastoreLruCache { digest: &[u8; 32], ) -> Result { let object_key = crate::s3::object_key_from_digest(digest)?; - match client.get_object(object_key).await? { + match client.get_object_content(object_key).await? { None => { bail!("could not fetch object with key {}", hex::encode(digest)) } - Some(response) => { - let bytes = response.content.collect().await?.to_bytes(); + Some(bytes) => { let chunk = DataBlob::from_raw(bytes.to_vec())?; self.insert(digest, &chunk)?; Ok(chunk) diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs index 350e48acc..19bdd73bc 100644 --- a/src/api2/config/datastore.rs +++ b/src/api2/config/datastore.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use ::serde::{Deserialize, Serialize}; use anyhow::{Context, Error, bail, format_err}; -use http_body_util::BodyExt; use serde_json::Value; use tracing::{info, warn}; @@ -128,14 +127,12 @@ pub(crate) fn do_create_datastore( if !overwrite_in_use { let object_key = S3ObjectKey::try_from(S3_DATASTORE_IN_USE_MARKER) .context("failed to generate s3 object key")?; - if let Some(response) = - proxmox_async::runtime::block_on(s3_client.get_object(object_key.clone())) - .context("failed to get in-use marker from bucket")? + if let Some(content) = proxmox_async::runtime::block_on( + s3_client.get_object_content(object_key.clone()), + ) + .context("failed to get in-use marker from bucket")? { - let content = proxmox_async::runtime::block_on(response.content.collect()) - .unwrap_or_default(); - let content = - String::from_utf8(content.to_bytes().to_vec()).unwrap_or_default(); + let content = String::from_utf8(content.to_vec()).unwrap_or_default(); let in_use: InUseContent = serde_json::from_str(&content).unwrap_or_default(); if let Some(hostname) = in_use.hostname { diff --git a/src/api2/reader/mod.rs b/src/api2/reader/mod.rs index 29c00e74c..cbda79579 100644 --- a/src/api2/reader/mod.rs +++ b/src/api2/reader/mod.rs @@ -3,7 +3,6 @@ use anyhow::{Context, Error, bail, format_err}; use futures::*; use hex::FromHex; -use http_body_util::BodyExt; use hyper::body::Incoming; use hyper::header::{self, CONNECTION, HeaderValue, UPGRADE}; use hyper::http::request::Parts; @@ -351,8 +350,7 @@ fn download_chunk( async fn fetch_from_object_store(s3_client: &S3Client, digest: &[u8; 32]) -> Result { let object_key = pbs_datastore::s3::object_key_from_digest(digest)?; - if let Some(response) = s3_client.get_object(object_key).await? { - let data = response.content.collect().await?.to_bytes(); + if let Some(data) = s3_client.get_object_content(object_key).await? { return Ok(Body::from(data)); } bail!("cannot find chunk with digest {}", hex::encode(digest)); diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 18fb1ca4c..01aafb4f2 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -5,7 +5,6 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use anyhow::{Error, bail}; -use http_body_util::BodyExt; use tracing::{error, info, warn}; use proxmox_parallel_handler::{ParallelHandler, SendHandle}; @@ -271,37 +270,25 @@ impl VerifyWorker { }, DatastoreBackend::S3(s3_client) => { let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?; - match proxmox_async::runtime::block_on(s3_client.get_object(object_key)) { - Ok(Some(response)) => { - match proxmox_async::runtime::block_on(response.content.collect()) { - Ok(raw_chunk) => { - match DataBlob::from_raw(raw_chunk.to_bytes().to_vec()) { - Ok(chunk) => { - let size = info.size(); - verify_state - .read_bytes - .fetch_add(chunk.raw_size(), Ordering::SeqCst); - decoder_pool.send((chunk, info.digest, size))?; - verify_state - .decoded_bytes - .fetch_add(size, Ordering::SeqCst); - } - Err(err) => Self::add_corrupt_chunk( - verify_state, - info.digest, - &format!( - "can't verify chunk with digest {} - {err}", - hex::encode(info.digest) - ), - ), - } - } - Err(err) => { - verify_state.errors.fetch_add(1, Ordering::SeqCst); - error!("can't verify chunk, load failed - {err}"); - } + match proxmox_async::runtime::block_on(s3_client.get_object_content(object_key)) { + Ok(Some(raw_chunk)) => match DataBlob::from_raw(raw_chunk.to_vec()) { + Ok(chunk) => { + let size = info.size(); + verify_state + .read_bytes + .fetch_add(chunk.raw_size(), Ordering::SeqCst); + decoder_pool.send((chunk, info.digest, size))?; + verify_state.decoded_bytes.fetch_add(size, Ordering::SeqCst); } - } + Err(err) => Self::add_corrupt_chunk( + verify_state, + info.digest, + &format!( + "can't verify chunk with digest {} - {err}", + hex::encode(info.digest) + ), + ), + }, Ok(None) => Self::add_corrupt_chunk( verify_state, info.digest, -- 2.47.3