From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 8376D1FF16F for ; Tue, 22 Jul 2025 12:11:10 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CABD135D5B; Tue, 22 Jul 2025 12:12:15 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 22 Jul 2025 12:10:36 +0200 Message-ID: <20250722101106.526438-21-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.2 In-Reply-To: <20250722101106.526438-1-c.ebner@proxmox.com> References: <20250722101106.526438-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1753179082454 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.045 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v11 16/46] verify worker: add datastore backed to verify worker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" In order to fetch chunks from an S3 compatible object store, instantiate and store the s3 client in the verify worker by storing the datastore's backend. This allows to reuse the same instance for the whole verification task. Signed-off-by: Christian Ebner Reviewed-by: Lukas Wagner Reviewed-by: Hannes Laimer --- changes since version 10: - no changes src/api2/admin/datastore.rs | 2 +- src/api2/backup/environment.rs | 2 +- src/backup/verify.rs | 14 ++++++++++---- src/server/verify_job.rs | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index d742633cf..3e1610936 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -883,7 +883,7 @@ pub fn verify( auth_id.to_string(), to_stdout, move |worker| { - let verify_worker = VerifyWorker::new(worker.clone(), datastore); + let verify_worker = VerifyWorker::new(worker.clone(), datastore)?; let failed_dirs = if let Some(backup_dir) = backup_dir { let mut res = Vec::new(); if !verify_worker.verify_backup_dir( diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 4a70d7c13..6a265bcc5 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -710,7 +710,7 @@ impl BackupEnvironment { move |worker| { worker.log_message("Automatically verifying newly added snapshot"); - let verify_worker = VerifyWorker::new(worker.clone(), datastore); + let verify_worker = VerifyWorker::new(worker.clone(), datastore)?; if !verify_worker.verify_backup_dir_with_lock( &backup_dir, worker.upid().clone(), diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 9344033d8..dea10f618 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -17,7 +17,7 @@ use pbs_api_types::{ use pbs_datastore::backup_info::{BackupDir, BackupGroup, BackupInfo}; use pbs_datastore::index::IndexFile; use pbs_datastore::manifest::{BackupManifest, FileInfo}; -use pbs_datastore::{DataBlob, DataStore, StoreProgress}; +use pbs_datastore::{DataBlob, DataStore, DatastoreBackend, StoreProgress}; use crate::tools::parallel_handler::ParallelHandler; @@ -30,19 +30,25 @@ pub struct VerifyWorker { datastore: Arc, verified_chunks: Arc>>, corrupt_chunks: Arc>>, + backend: DatastoreBackend, } impl VerifyWorker { /// Creates a new VerifyWorker for a given task worker and datastore. - pub fn new(worker: Arc, datastore: Arc) -> Self { - Self { + pub fn new( + worker: Arc, + datastore: Arc, + ) -> Result { + let backend = datastore.backend()?; + Ok(Self { worker, datastore, // start with 16k chunks == up to 64G data verified_chunks: Arc::new(Mutex::new(HashSet::with_capacity(16 * 1024))), // start with 64 chunks since we assume there are few corrupt ones corrupt_chunks: Arc::new(Mutex::new(HashSet::with_capacity(64))), - } + backend, + }) } fn verify_blob(backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs index 95a7b2a9b..c8792174b 100644 --- a/src/server/verify_job.rs +++ b/src/server/verify_job.rs @@ -41,7 +41,7 @@ pub fn do_verification_job( None => Default::default(), }; - let verify_worker = VerifyWorker::new(worker.clone(), datastore); + let verify_worker = VerifyWorker::new(worker.clone(), datastore)?; let result = verify_worker.verify_all_backups( worker.upid(), ns, -- 2.47.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel