From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 882A061B7C for ; Mon, 7 Sep 2020 17:31:17 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4F110D667 for ; Mon, 7 Sep 2020 17:30:47 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id D12F1D5EF for ; Mon, 7 Sep 2020 17:30:44 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9D6F944A9B for ; Mon, 7 Sep 2020 17:30:44 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Mon, 7 Sep 2020 17:30:33 +0200 Message-Id: <20200907153036.9324-3-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200907153036.9324-1-s.reiter@proxmox.com> References: <20200907153036.9324-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.054 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v2 proxmox-backup 2/5] verify: rename corrupted chunks with .bad extension X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 07 Sep 2020 15:31:17 -0000 This ensures that following backups will always upload the chunk, thereby replacing it with a correct version again. Format for renaming is ..bad where is used if a chunk is found to be bad again before a GC cleans it up. Care has been taken to deliberately only rename a chunk in conditions where it is guaranteed to be an error in the chunk itself. Otherwise a broken index file could lead to an unwanted mass-rename of chunks. Signed-off-by: Stefan Reiter --- src/backup/verify.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/backup/verify.rs b/src/backup/verify.rs index 1c848d57..697ed236 100644 --- a/src/backup/verify.rs +++ b/src/backup/verify.rs @@ -39,6 +39,34 @@ fn verify_blob(datastore: Arc, backup_dir: &BackupDir, info: &FileInf } } +fn rename_corrupted_chunk( + datastore: Arc, + digest: &[u8;32], + worker: Arc, +) { + let (path, digest_str) = datastore.chunk_path(digest); + + let mut counter = 0; + let mut new_path = path.clone(); + new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); + while new_path.exists() && counter < 9 { + counter += 1; + new_path.set_file_name(format!("{}.{}.bad", digest_str, counter)); + } + + match std::fs::rename(&path, &new_path) { + Ok(_) => { + worker.log(format!("corrupted chunk renamed to {:?}", &new_path)); + }, + Err(err) => { + match err.kind() { + std::io::ErrorKind::NotFound => { /* ignored */ }, + _ => worker.log(format!("could not rename corrupted chunk {:?} - {}", &path, err)) + } + } + }; +} + // We use a separate thread to read/load chunks, so that we can do // load and verify in parallel to increase performance. fn chunk_reader_thread( @@ -73,6 +101,7 @@ fn chunk_reader_thread( corrupt_chunks.lock().unwrap().insert(info.digest); worker.log(format!("can't verify chunk, load failed - {}", err)); errors.fetch_add(1, Ordering::SeqCst); + rename_corrupted_chunk(datastore.clone(), &info.digest, worker.clone()); continue; } Ok(chunk) => { @@ -101,7 +130,7 @@ fn verify_index_chunks( let start_time = Instant::now(); let chunk_channel = chunk_reader_thread( - datastore, + datastore.clone(), index, verified_chunks.clone(), corrupt_chunks.clone(), @@ -148,6 +177,7 @@ fn verify_index_chunks( corrupt_chunks.lock().unwrap().insert(digest); worker.log(format!("{}", err)); errors.fetch_add(1, Ordering::SeqCst); + rename_corrupted_chunk(datastore.clone(), &digest, worker.clone()); } else { verified_chunks.lock().unwrap().insert(digest); } -- 2.20.1