From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 95A98672B0 for ; Thu, 30 Jul 2020 09:09:07 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 93A0413489 for ; Thu, 30 Jul 2020 09:09:07 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 2D82913465 for ; Thu, 30 Jul 2020 09:09:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id EFB3D433F0 for ; Thu, 30 Jul 2020 09:09:05 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Thu, 30 Jul 2020 09:09:04 +0200 Message-Id: <20200730070905.10735-2-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200730070905.10735-1-d.csapak@proxmox.com> References: <20200730070905.10735-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.045 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pbs-devel] [PATCH proxmox-backup 2/3] verify: keep also track of corrupt chunks X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 30 Jul 2020 07:09:07 -0000 so that we do not have to verify a corrupt one multiple times Signed-off-by: Dominik Csapak --- src/api2/admin/datastore.rs | 3 ++- src/backup/verify.rs | 46 +++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index c720231c..5e6d5720 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -476,7 +476,8 @@ pub fn verify( { let success = if let Some(backup_dir) = backup_dir { let mut verified_chunks = HashSet::with_capacity(1024*16); - verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &worker)? + let mut corrupt_chunks = HashSet::with_capacity(64); + verify_backup_dir(&datastore, &backup_dir, &mut verified_chunks, &mut corrupt_chunks, &worker)? } else if let Some(backup_group) = backup_group { verify_backup_group(&datastore, &backup_group, &worker)? } else { diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 9e309b12..58b91bc9 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -39,6 +39,7 @@ fn verify_index_chunks( datastore: &DataStore, index: Box, verified_chunks: &mut HashSet<[u8;32]>, + corrupt_chunks: &mut HashSet<[u8; 32]>, worker: &WorkerTask, ) -> Result<(), Error> { @@ -51,11 +52,18 @@ fn verify_index_chunks( let size = info.range.end - info.range.start; if !verified_chunks.contains(&info.digest) { - if let Err(err) = datastore.verify_stored_chunk(&info.digest, size) { - worker.log(format!("{}", err)); - errors += 1; + if !corrupt_chunks.contains(&info.digest) { + if let Err(err) = datastore.verify_stored_chunk(&info.digest, size) { + corrupt_chunks.insert(info.digest); + worker.log(format!("{}", err)); + errors += 1; + } else { + verified_chunks.insert(info.digest); + } } else { - verified_chunks.insert(info.digest); + let digest_str = proxmox::tools::digest_to_hex(&info.digest); + worker.log(format!("chunk {} was marked as corrupt", digest_str)); + errors += 1; } } } @@ -72,6 +80,7 @@ fn verify_fixed_index( backup_dir: &BackupDir, info: &FileInfo, verified_chunks: &mut HashSet<[u8;32]>, + corrupt_chunks: &mut HashSet<[u8;32]>, worker: &WorkerTask, ) -> Result<(), Error> { @@ -89,7 +98,7 @@ fn verify_fixed_index( bail!("wrong index checksum"); } - verify_index_chunks(datastore, Box::new(index), verified_chunks, worker) + verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker) } fn verify_dynamic_index( @@ -97,6 +106,7 @@ fn verify_dynamic_index( backup_dir: &BackupDir, info: &FileInfo, verified_chunks: &mut HashSet<[u8;32]>, + corrupt_chunks: &mut HashSet<[u8;32]>, worker: &WorkerTask, ) -> Result<(), Error> { @@ -114,7 +124,7 @@ fn verify_dynamic_index( bail!("wrong index checksum"); } - verify_index_chunks(datastore, Box::new(index), verified_chunks, worker) + verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker) } /// Verify a single backup snapshot @@ -130,6 +140,7 @@ pub fn verify_backup_dir( datastore: &DataStore, backup_dir: &BackupDir, verified_chunks: &mut HashSet<[u8;32]>, + corrupt_chunks: &mut HashSet<[u8;32]>, worker: &WorkerTask ) -> Result { @@ -149,8 +160,24 @@ pub fn verify_backup_dir( let result = proxmox::try_block!({ worker.log(format!(" check {}", info.filename)); match archive_type(&info.filename)? { - ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info, verified_chunks, worker), - ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info, verified_chunks, worker), + ArchiveType::FixedIndex => + verify_fixed_index( + &datastore, + &backup_dir, + info, + verified_chunks, + corrupt_chunks, + worker + ), + ArchiveType::DynamicIndex => + verify_dynamic_index( + &datastore, + &backup_dir, + info, + verified_chunks, + corrupt_chunks, + worker + ), ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info), } }); @@ -189,10 +216,11 @@ pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: & let mut error_count = 0; let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB) + let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones BackupInfo::sort_list(&mut list, false); // newest first for info in list { - if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, worker)? { + if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{ error_count += 1; } } -- 2.20.1