From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id C76D960251 for ; Tue, 8 Sep 2020 12:27:52 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B787B86AA for ; Tue, 8 Sep 2020 12:27:52 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id B9242868F for ; Tue, 8 Sep 2020 12:27:51 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 75DAE44A97 for ; Tue, 8 Sep 2020 12:27:51 +0200 (CEST) Date: Tue, 8 Sep 2020 12:27:06 +0200 (CEST) From: Dietmar Maurer To: Proxmox Backup Server development discussion , Stefan Reiter Message-ID: <146472204.951.1599560827524@webmail.proxmox.com> In-Reply-To: <20200907153036.9324-3-s.reiter@proxmox.com> References: <20200907153036.9324-1-s.reiter@proxmox.com> <20200907153036.9324-3-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit X-Priority: 3 Importance: Normal X-Mailer: Open-Xchange Mailer v7.10.3-Rev21 X-Originating-Client: open-xchange-appsuite X-SPAM-LEVEL: Spam detection results: 0 AWL 0.087 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [verify.rs, proxmox.com] Subject: Re: [pbs-devel] [PATCH v2 proxmox-backup 2/5] verify: rename corrupted chunks with .bad extension X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 08 Sep 2020 10:27:52 -0000 possible improvement to avoid duplicate format!(): diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 697ed23..a96c572 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -48,10 +48,9 @@ fn rename_corrupted_chunk( let mut counter = 0; let mut new_path = path.clone(); - new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); - while new_path.exists() && counter < 9 { - counter += 1; + loop { new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); + if new_path.exists() && counter < 9 { counter += 1; } else { break; } } match std::fs::rename(&path, &new_path) { > On 09/07/2020 5:30 PM Stefan Reiter wrote: > > > This ensures that following backups will always upload the chunk, > thereby replacing it with a correct version again. > > Format for renaming is ..bad where is used if > a chunk is found to be bad again before a GC cleans it up. > > Care has been taken to deliberately only rename a chunk in conditions > where it is guaranteed to be an error in the chunk itself. Otherwise a > broken index file could lead to an unwanted mass-rename of chunks. > > Signed-off-by: Stefan Reiter > --- > src/backup/verify.rs | 32 +++++++++++++++++++++++++++++++- > 1 file changed, 31 insertions(+), 1 deletion(-) > > diff --git a/src/backup/verify.rs b/src/backup/verify.rs > index 1c848d57..697ed236 100644 > --- a/src/backup/verify.rs > +++ b/src/backup/verify.rs > @@ -39,6 +39,34 @@ fn verify_blob(datastore: Arc, backup_dir: &BackupDir, info: &FileInf > } > } > > +fn rename_corrupted_chunk( > + datastore: Arc, > + digest: &[u8;32], > + worker: Arc, > +) { > + let (path, digest_str) = datastore.chunk_path(digest); > + > + let mut counter = 0; > + let mut new_path = path.clone(); > + new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); > + while new_path.exists() && counter < 9 { > + counter += 1; > + new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); > + } > + > + match std::fs::rename(&path, &new_path) { > + Ok(_) => { > + worker.log(format!("corrupted chunk renamed to {:?}", &new_path)); > + }, > + Err(err) => { > + match err.kind() { > + std::io::ErrorKind::NotFound => { /* ignored */ }, > + _ => worker.log(format!("could not rename corrupted chunk {:?} - {}", &path, err)) > + } > + } > + }; > +} > + > // We use a separate thread to read/load chunks, so that we can do > // load and verify in parallel to increase performance. > fn chunk_reader_thread( > @@ -73,6 +101,7 @@ fn chunk_reader_thread( > corrupt_chunks.lock().unwrap().insert(info.digest); > worker.log(format!("can't verify chunk, load failed - {}", err)); > errors.fetch_add(1, Ordering::SeqCst); > + rename_corrupted_chunk(datastore.clone(), &info.digest, worker.clone()); > continue; > } > Ok(chunk) => { > @@ -101,7 +130,7 @@ fn verify_index_chunks( > let start_time = Instant::now(); > > let chunk_channel = chunk_reader_thread( > - datastore, > + datastore.clone(), > index, > verified_chunks.clone(), > corrupt_chunks.clone(), > @@ -148,6 +177,7 @@ fn verify_index_chunks( > corrupt_chunks.lock().unwrap().insert(digest); > worker.log(format!("{}", err)); > errors.fetch_add(1, Ordering::SeqCst); > + rename_corrupted_chunk(datastore.clone(), &digest, worker.clone()); > } else { > verified_chunks.lock().unwrap().insert(digest); > } > -- > 2.20.1 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel