From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id F36FE1FF179 for ; Wed, 10 Dec 2025 15:21:26 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A17AC1DBD3; Wed, 10 Dec 2025 15:22:05 +0100 (CET) MIME-Version: 1.0 In-Reply-To: <20251126133419.570874-5-c.ebner@proxmox.com> References: <20251126133419.570874-1-c.ebner@proxmox.com> <20251126133419.570874-5-c.ebner@proxmox.com> From: Fabian =?utf-8?q?Gr=C3=BCnbichler?= To: Christian Ebner , pbs-devel@lists.proxmox.com Date: Wed, 10 Dec 2025 15:21:58 +0100 Message-ID: <176537651811.371273.12546243384711800815@yuna.proxmox.com> User-Agent: alot/0.0.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1765376514789 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.047 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup 4/4] chunk store: return chunk extension and check for used marker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Quoting Christian Ebner (2025-11-26 14:34:19) > Clearly distinguish the cases for `bad` and `using` extensions for > items returned by the chunk iterator. Directory entries with > filenames which matched the expected length but not the extension > are now skipped over. I think we could go a step further and also move the parsing/rendering to ChunkExt everywhere we currently do it? which I guess is mostly verification/GC? not sure whether ChunkExt or a new Chunk struct that also contains the digest would be a better fit though.. > > Signed-off-by: Christian Ebner > --- > pbs-datastore/src/chunk_store.rs | 48 ++++++++++++++++++++++++-------- > 1 file changed, 37 insertions(+), 11 deletions(-) > > diff --git a/pbs-datastore/src/chunk_store.rs b/pbs-datastore/src/chunk_store.rs > index 7980938ad..ad517391d 100644 > --- a/pbs-datastore/src/chunk_store.rs > +++ b/pbs-datastore/src/chunk_store.rs > @@ -280,7 +280,11 @@ impl ChunkStore { > &self, > ) -> Result< > impl std::iter::FusedIterator< > - Item = (Result, usize, bool), > + Item = ( > + Result, > + usize, > + ChunkExt, > + ), > >, > Error, > > { > @@ -318,14 +322,20 @@ impl ChunkStore { > > if bytes.len() == 64 && bytes.iter().take(64).all(u8::is_ascii_hexdigit) > { > - return Some((Ok(entry), percentage, false)); > + return Some((Ok(entry), percentage, ChunkExt::None)); > } > > if bytes.len() == 64 + ".0.bad".len() > && bytes.iter().take(64).all(u8::is_ascii_hexdigit) > { > - let bad = bytes.ends_with(b".bad"); > - return Some((Ok(entry), percentage, bad)); > + let chunk_ext = if bytes.ends_with(b".bad") { > + ChunkExt::Bad > + } else if bytes.ends_with(USING_MARKER_FILENAME_EXT.as_bytes()) { > + ChunkExt::UsedMarker > + } else { > + continue; > + }; > + return Some((Ok(entry), percentage, chunk_ext)); > } > > continue; > @@ -334,7 +344,7 @@ impl ChunkStore { > // stop after first error > done = true; > // and pass the error through: > - return Some((Err(err), percentage, false)); > + return Some((Err(err), percentage, ChunkExt::None)); > } > None => (), // open next directory > } > @@ -367,7 +377,7 @@ impl ChunkStore { > return Some(( > Err(format_err!("unable to read subdir '{subdir}' - {err}")), > percentage, > - false, > + ChunkExt::None, > )); > } > } > @@ -402,7 +412,8 @@ impl ChunkStore { > let mut last_percentage = 0; > let mut chunk_count = 0; > > - for (entry, percentage, bad) in self.get_chunk_store_iterator()? { > + for (entry, percentage, chunk_ext) in self.get_chunk_store_iterator()? { > + let bad = chunk_ext.is_bad(); > if last_percentage != percentage { > last_percentage = percentage; > info!("processed {percentage}% ({chunk_count} chunks)"); > @@ -433,10 +444,7 @@ impl ChunkStore { > drop(lock); > continue; > } > - if filename > - .to_bytes() > - .ends_with(USING_MARKER_FILENAME_EXT.as_bytes()) > - { > + if chunk_ext.is_used_marker() { > unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir).map_err(|err| { > format_err!("unlinking chunk using marker {filename:?} failed - {err}") > })?; > @@ -908,6 +916,24 @@ impl ChunkStore { > } > } > > +#[derive(PartialEq)] > +/// Chunk iterator directory entry filename extension > +enum ChunkExt { > + None, > + Bad, > + UsedMarker, > +} > + > +impl ChunkExt { > + fn is_bad(&self) -> bool { > + *self == Self::Bad > + } > + > + fn is_used_marker(&self) -> bool { > + *self == Self::UsedMarker > + } > +} > + > #[test] > fn test_chunk_store1() { > let mut path = std::fs::canonicalize(".").unwrap(); // we need absolute path > -- > 2.47.3 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel > > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel