From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <dcsapak@zita.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 9F379672B1
for <pbs-devel@lists.proxmox.com>; Thu, 30 Jul 2020 09:09:07 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 9D8BC13492
for <pbs-devel@lists.proxmox.com>; Thu, 30 Jul 2020 09:09:07 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[212.186.127.180])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id 438981346D
for <pbs-devel@lists.proxmox.com>; Thu, 30 Jul 2020 09:09:06 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 02646433F4
for <pbs-devel@lists.proxmox.com>; Thu, 30 Jul 2020 09:09:06 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Thu, 30 Jul 2020 09:09:05 +0200
Message-Id: <20200730070905.10735-3-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20200730070905.10735-1-d.csapak@proxmox.com>
References: <20200730070905.10735-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL -0.045 Adjusted score from AWL reputation of From: address
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery
methods
NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records
RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/,
medium trust
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_NONE 0.001 SPF: sender does not publish an SPF Record
Subject: [pbs-devel] [PATCH proxmox-backup 3/3] verify: keep track and log
which dirs failed the verification
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Thu, 30 Jul 2020 07:09:07 -0000
so that we can print a list at the end of the worker which backups
are corrupt.
this is useful if there are many snapshots and some in between had an
error. Before this patch, the task log simply says to 'look in the logs'
but if the log is very long it makes it hard to see what exactly failed.
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/api2/admin/datastore.rs | 14 +++++++++++---
src/backup/verify.rs | 31 ++++++++++++++-----------------
2 files changed, 25 insertions(+), 20 deletions(-)
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 5e6d5720..29e74bd6 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -474,16 +474,24 @@ pub fn verify(
let upid_str = WorkerTask::new_thread(
"verify", Some(worker_id.clone()), &username, to_stdout, move |worker|
{
- let success = if let Some(backup_dir) = backup_dir {
+ let failed_dirs = if let Some(backup_dir) = backup_dir {
let mut verified_chunks = HashSet::with_capacity(1024*16);
let mut corrupt_chunks = HashSet::with_capacity(64);
- verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)?
+ let mut res = Vec::new();
+ if !verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? {
+ res.push(backup_dir.to_string());
+ }
+ res
} else if let Some(backup_group) = backup_group {
verify_backup_group(&datastore, &backup_group, &worker)?
} else {
verify_all_backups(&datastore, &worker)?
};
- if !success {
+ if failed_dirs.len() > 0 {
+ worker.log("Failed to verify following snapshots:");
+ for dir in failed_dirs {
+ worker.log(format!("\t{}", dir));
+ }
bail!("verfication failed - please check the log for details");
}
Ok(())
diff --git a/src/backup/verify.rs b/src/backup/verify.rs
index 58b91bc9..cba1297f 100644
--- a/src/backup/verify.rs
+++ b/src/backup/verify.rs
@@ -198,34 +198,32 @@ pub fn verify_backup_dir(
/// Errors are logged to the worker log.
///
/// Returns
-/// - Ok(true) if verify is successful
-/// - Ok(false) if there were verification errors
+/// - Ok(failed_dirs) where failed_dirs had verification errors
/// - Err(_) if task was aborted
-pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<bool, Error> {
+pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+ let mut errors = Vec::new();
let mut list = match group.list_backups(&datastore.base_path()) {
Ok(list) => list,
Err(err) => {
worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
- return Ok(false);
+ return Ok(errors);
}
};
worker.log(format!("verify group {}:{}", datastore.name(), group));
- let mut error_count = 0;
-
let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
BackupInfo::sort_list(&mut list, false); // newest first
for info in list {
if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
- error_count += 1;
+ errors.push(info.backup_dir.to_string());
}
}
- Ok(error_count == 0)
+ Ok(errors)
}
/// Verify all backups inside a datastore
@@ -233,27 +231,26 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &
/// Errors are logged to the worker log.
///
/// Returns
-/// - Ok(true) if verify is successful
-/// - Ok(false) if there were verification errors
+/// - Ok(failed_dirs) where failed_dirs had verification errors
/// - Err(_) if task was aborted
-pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<bool, Error> {
+pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
+
+ let mut errors = Vec::new();
let list = match BackupGroup::list_groups(&datastore.base_path()) {
Ok(list) => list,
Err(err) => {
worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
- return Ok(false);
+ return Ok(errors);
}
};
worker.log(format!("verify datastore {}", datastore.name()));
- let mut error_count = 0;
for group in list {
- if !verify_backup_group(datastore, &group, worker)? {
- error_count += 1;
- }
+ let mut group_errors = verify_backup_group(datastore, &group, worker)?;
+ errors.append(&mut group_errors);
}
- Ok(error_count == 0)
+ Ok(errors)
}
--
2.20.1