From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id EEC4C1FF16B for ; Tue, 15 Jul 2025 15:02:59 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 640C63CF4C; Tue, 15 Jul 2025 15:03:58 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 15 Jul 2025 14:53:04 +0200 Message-ID: <20250715125332.954494-27-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250715125332.954494-1-c.ebner@proxmox.com> References: <20250715125332.954494-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.046 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_MSPIKE_H2 0.001 Average reputation (+2) SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v8 17/45] verify: implement chunk verification for stores with s3 backend X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" For datastores backed by an S3 compatible object store, rather than reading the chunks to be verified from the local filesystem, fetch them via the s3 client from the configured bucket. Signed-off-by: Christian Ebner --- changes since version 7: - no changes src/backup/verify.rs | 89 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/src/backup/verify.rs b/src/backup/verify.rs index dea10f618..3a4a1d0d5 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use anyhow::{bail, Error}; +use http_body_util::BodyExt; use tracing::{error, info, warn}; use proxmox_worker_task::WorkerTaskContext; @@ -89,6 +90,38 @@ impl VerifyWorker { } } + if let Ok(DatastoreBackend::S3(s3_client)) = datastore.backend() { + let suffix = format!(".{}.bad", counter); + let target_key = + match pbs_datastore::s3::object_key_from_digest_with_suffix(digest, &suffix) { + Ok(target_key) => target_key, + Err(err) => { + info!("could not generate target key for corrupted chunk {path:?} - {err}"); + return; + } + }; + let object_key = match pbs_datastore::s3::object_key_from_digest(digest) { + Ok(object_key) => object_key, + Err(err) => { + info!("could not generate object key for corrupted chunk {path:?} - {err}"); + return; + } + }; + if proxmox_async::runtime::block_on( + s3_client.copy_object(object_key.clone(), target_key), + ) + .is_ok() + { + if proxmox_async::runtime::block_on(s3_client.delete_object(object_key)).is_err() { + info!("failed to delete corrupt chunk on s3 backend: {digest_str}"); + } + } else { + info!("failed to copy corrupt chunk on s3 backend: {digest_str}"); + } + } else { + info!("failed to get s3 backend while trying to rename bad chunk: {digest_str}"); + } + match std::fs::rename(&path, &new_path) { Ok(_) => { info!("corrupted chunk renamed to {:?}", &new_path); @@ -189,18 +222,50 @@ impl VerifyWorker { continue; // already verified or marked corrupt } - match self.datastore.load_chunk(&info.digest) { - Err(err) => { - self.corrupt_chunks.lock().unwrap().insert(info.digest); - error!("can't verify chunk, load failed - {err}"); - errors.fetch_add(1, Ordering::SeqCst); - Self::rename_corrupted_chunk(self.datastore.clone(), &info.digest); - } - Ok(chunk) => { - let size = info.size(); - read_bytes += chunk.raw_size(); - decoder_pool.send((chunk, info.digest, size))?; - decoded_bytes += size; + match &self.backend { + DatastoreBackend::Filesystem => match self.datastore.load_chunk(&info.digest) { + Err(err) => { + self.corrupt_chunks.lock().unwrap().insert(info.digest); + error!("can't verify chunk, load failed - {err}"); + errors.fetch_add(1, Ordering::SeqCst); + Self::rename_corrupted_chunk(self.datastore.clone(), &info.digest); + } + Ok(chunk) => { + let size = info.size(); + read_bytes += chunk.raw_size(); + decoder_pool.send((chunk, info.digest, size))?; + decoded_bytes += size; + } + }, + DatastoreBackend::S3(s3_client) => { + let object_key = pbs_datastore::s3::object_key_from_digest(&info.digest)?; + match proxmox_async::runtime::block_on(s3_client.get_object(object_key)) { + Ok(Some(response)) => { + let bytes = + proxmox_async::runtime::block_on(response.content.collect())? + .to_bytes(); + let chunk = DataBlob::from_raw(bytes.to_vec())?; + let size = info.size(); + read_bytes += chunk.raw_size(); + decoder_pool.send((chunk, info.digest, size))?; + decoded_bytes += size; + } + Ok(None) => { + self.corrupt_chunks.lock().unwrap().insert(info.digest); + error!( + "can't verify missing chunk with digest {}", + hex::encode(info.digest) + ); + errors.fetch_add(1, Ordering::SeqCst); + Self::rename_corrupted_chunk(self.datastore.clone(), &info.digest); + } + Err(err) => { + self.corrupt_chunks.lock().unwrap().insert(info.digest); + error!("can't verify chunk, load failed - {err}"); + errors.fetch_add(1, Ordering::SeqCst); + Self::rename_corrupted_chunk(self.datastore.clone(), &info.digest); + } + } } } } -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel