From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 1245060D38 for ; Thu, 3 Sep 2020 16:17:44 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0FF0B1A11F for ; Thu, 3 Sep 2020 16:17:14 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 59E721A0FA for ; Thu, 3 Sep 2020 16:17:12 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 00AA7449FD for ; Thu, 3 Sep 2020 16:17:12 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Thu, 3 Sep 2020 16:17:04 +0200 Message-Id: <20200903141705.6344-4-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200903141705.6344-1-s.reiter@proxmox.com> References: <20200903141705.6344-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.046 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs, mod.rs] Subject: [pbs-devel] [PATCH proxmox-backup 3/4] gc: remove .bad files on garbage collect X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Sep 2020 14:17:44 -0000 The iterator of get_chunk_iterator is extended with a third parameter indicating whether the current file is a chunk (false) or a .bad file (true). Count their sizes to the total of removed bytes, since it also frees disk space. Signed-off-by: Stefan Reiter --- src/api2/types/mod.rs | 3 +++ src/backup/chunk_store.rs | 43 ++++++++++++++++++++++++++++----------- src/backup/datastore.rs | 5 ++++- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs index 6854fdf0..e29a7e37 100644 --- a/src/api2/types/mod.rs +++ b/src/api2/types/mod.rs @@ -559,6 +559,8 @@ pub struct GarbageCollectionStatus { pub pending_bytes: u64, /// Number of pending chunks (pending removal - kept for safety). pub pending_chunks: usize, + /// Number of chunks marked as .bad by verify that have been removed by GC. + pub removed_bad: usize, } impl Default for GarbageCollectionStatus { @@ -573,6 +575,7 @@ impl Default for GarbageCollectionStatus { removed_chunks: 0, pending_bytes: 0, pending_chunks: 0, + removed_bad: 0, } } } diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs index e1da5a8a..5c2fb29d 100644 --- a/src/backup/chunk_store.rs +++ b/src/backup/chunk_store.rs @@ -187,7 +187,7 @@ impl ChunkStore { pub fn get_chunk_iterator( &self, ) -> Result< - impl Iterator, usize)> + std::iter::FusedIterator, + impl Iterator, usize, bool)> + std::iter::FusedIterator, Error > { use nix::dir::Dir; @@ -218,20 +218,26 @@ impl ChunkStore { match inner.next() { Some(Ok(entry)) => { // skip files if they're not a hash - let bytes = entry.file_name().to_bytes(); - if bytes.len() != 64 { - continue; + let hash = { + let bytes = entry.file_name().to_bytes(); + bytes.len() == 64 && bytes.iter().all(u8::is_ascii_hexdigit) + }; + + if hash { + return Some((Ok(entry), percentage, false)); + } else if let Ok(name) = entry.file_name().to_str() { + if name.ends_with(".bad") { + return Some((Ok(entry), percentage, true)); + } } - if !bytes.iter().all(u8::is_ascii_hexdigit) { - continue; - } - return Some((Ok(entry), percentage)); + + continue; } Some(Err(err)) => { // stop after first error done = true; // and pass the error through: - return Some((Err(err), percentage)); + return Some((Err(err), percentage, false)); } None => (), // open next directory } @@ -261,7 +267,7 @@ impl ChunkStore { // other errors are fatal, so end our iteration done = true; // and pass the error through: - return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage)); + return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage, false)); } } } @@ -292,7 +298,7 @@ impl ChunkStore { let mut last_percentage = 0; let mut chunk_count = 0; - for (entry, percentage) in self.get_chunk_iterator()? { + for (entry, percentage, bad) in self.get_chunk_iterator()? { if last_percentage != percentage { last_percentage = percentage; worker.log(format!("percentage done: phase2 {}% (processed {} chunks)", percentage, chunk_count)); @@ -321,7 +327,20 @@ impl ChunkStore { let lock = self.mutex.lock(); if let Ok(stat) = fstatat(dirfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { - if stat.st_atime < min_atime { + if bad { + let res = unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) }; + if res != 0 { + let err = nix::Error::last(); + worker.warn(format!( + "unlink .bad file {:?} failed on store '{}' - {}", + filename, + self.name, + err, + )); + } + status.removed_bad += 1; + status.removed_bytes += stat.st_size as u64; + } else if stat.st_atime < min_atime { //let age = now - stat.st_atime; //println!("UNLINK {} {:?}", age/(3600*24), filename); let res = unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) }; diff --git a/src/backup/datastore.rs b/src/backup/datastore.rs index 42866e38..ebe47487 100644 --- a/src/backup/datastore.rs +++ b/src/backup/datastore.rs @@ -85,7 +85,7 @@ impl DataStore { pub fn get_chunk_iterator( &self, ) -> Result< - impl Iterator, usize)>, + impl Iterator, usize, bool)>, Error > { self.chunk_store.get_chunk_iterator() @@ -495,6 +495,9 @@ impl DataStore { if gc_status.pending_bytes > 0 { worker.log(&format!("Pending removals: {} (in {} chunks)", HumanByte::from(gc_status.pending_bytes), gc_status.pending_chunks)); } + if gc_status.removed_bad > 0 { + worker.log(&format!("Removed bad files: {}", gc_status.removed_bad)); + } worker.log(&format!("Original data usage: {}", HumanByte::from(gc_status.index_data_bytes))); -- 2.20.1