From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 9F379672B1 for ; Thu, 30 Jul 2020 09:09:07 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9D8BC13492 for ; Thu, 30 Jul 2020 09:09:07 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 438981346D for ; Thu, 30 Jul 2020 09:09:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 02646433F4 for ; Thu, 30 Jul 2020 09:09:06 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Thu, 30 Jul 2020 09:09:05 +0200 Message-Id: <20200730070905.10735-3-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200730070905.10735-1-d.csapak@proxmox.com> References: <20200730070905.10735-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.045 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pbs-devel] [PATCH proxmox-backup 3/3] verify: keep track and log which dirs failed the verification X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 30 Jul 2020 07:09:07 -0000 so that we can print a list at the end of the worker which backups are corrupt. this is useful if there are many snapshots and some in between had an error. Before this patch, the task log simply says to 'look in the logs' but if the log is very long it makes it hard to see what exactly failed. Signed-off-by: Dominik Csapak --- src/api2/admin/datastore.rs | 14 +++++++++++--- src/backup/verify.rs | 31 ++++++++++++++----------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 5e6d5720..29e74bd6 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -474,16 +474,24 @@ pub fn verify( let upid_str = WorkerTask::new_thread( "verify", Some(worker_id.clone()), &username, to_stdout, move |worker| { - let success = if let Some(backup_dir) = backup_dir { + let failed_dirs = if let Some(backup_dir) = backup_dir { let mut verified_chunks = HashSet::with_capacity(1024*16); let mut corrupt_chunks = HashSet::with_capacity(64); - verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? + let mut res = Vec::new(); + if !verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? { + res.push(backup_dir.to_string()); + } + res } else if let Some(backup_group) = backup_group { verify_backup_group(&datastore, &backup_group, &worker)? } else { verify_all_backups(&datastore, &worker)? }; - if !success { + if failed_dirs.len() > 0 { + worker.log("Failed to verify following snapshots:"); + for dir in failed_dirs { + worker.log(format!("\t{}", dir)); + } bail!("verfication failed - please check the log for details"); } Ok(()) diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 58b91bc9..cba1297f 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -198,34 +198,32 @@ pub fn verify_backup_dir( /// Errors are logged to the worker log. /// /// Returns -/// - Ok(true) if verify is successful -/// - Ok(false) if there were verification errors +/// - Ok(failed_dirs) where failed_dirs had verification errors /// - Err(_) if task was aborted -pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result { +pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result, Error> { + let mut errors = Vec::new(); let mut list = match group.list_backups(&datastore.base_path()) { Ok(list) => list, Err(err) => { worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err)); - return Ok(false); + return Ok(errors); } }; worker.log(format!("verify group {}:{}", datastore.name(), group)); - let mut error_count = 0; - let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB) let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones BackupInfo::sort_list(&mut list, false); // newest first for info in list { if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{ - error_count += 1; + errors.push(info.backup_dir.to_string()); } } - Ok(error_count == 0) + Ok(errors) } /// Verify all backups inside a datastore @@ -233,27 +231,26 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: & /// Errors are logged to the worker log. /// /// Returns -/// - Ok(true) if verify is successful -/// - Ok(false) if there were verification errors +/// - Ok(failed_dirs) where failed_dirs had verification errors /// - Err(_) if task was aborted -pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result { +pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result, Error> { + + let mut errors = Vec::new(); let list = match BackupGroup::list_groups(&datastore.base_path()) { Ok(list) => list, Err(err) => { worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err)); - return Ok(false); + return Ok(errors); } }; worker.log(format!("verify datastore {}", datastore.name())); - let mut error_count = 0; for group in list { - if !verify_backup_group(datastore, &group, worker)? { - error_count += 1; - } + let mut group_errors = verify_backup_group(datastore, &group, worker)?; + errors.append(&mut group_errors); } - Ok(error_count == 0) + Ok(errors) } -- 2.20.1