From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id B5E3B1FF15C for ; Fri, 14 Nov 2025 14:19:10 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E90A81367B; Fri, 14 Nov 2025 14:20:05 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Fri, 14 Nov 2025 14:19:00 +0100 Message-ID: <20251114131901.441650-21-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20251114131901.441650-1-c.ebner@proxmox.com> References: <20251114131901.441650-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1763126341761 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.048 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v6 20/21] GC: fix: don't drop bad extension for S3 object to chunk path helper X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" The current implementation does return a path and digest also for S3 object keys which are bad chunks, does however drop the extension. Since this will cause issues for phase 2 of garbage collection on S3 backends, include the extension and return a flag signaling if this is a bad chunk or not. Signed-off-by: Christian Ebner --- pbs-datastore/src/datastore.rs | 37 +++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index c0dc0f75d..b91e90638 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -1660,10 +1660,11 @@ impl DataStore { let mut delete_list = Vec::with_capacity(1000); loop { for content in list_bucket_result.contents { - let (chunk_path, digest) = match self.chunk_path_from_object_key(&content.key) { - Some(path) => path, - None => continue, - }; + let (chunk_path, digest, bad) = + match self.chunk_path_from_object_key(&content.key) { + Some(path) => path, + None => continue, + }; let timeout = std::time::Duration::from_secs(0); let _chunk_guard = match self.inner.chunk_store.lock_chunk(&digest, timeout) { @@ -1692,11 +1693,6 @@ impl DataStore { }; let atime = atime.duration_since(SystemTime::UNIX_EPOCH)?.as_secs() as i64; - let bad = chunk_path - .as_path() - .extension() - .is_some_and(|ext| ext == "bad"); - unsafe { self.inner.chunk_store.cond_sweep_chunk( atime, @@ -1852,14 +1848,23 @@ impl DataStore { } // Check and generate a chunk path from given object key - fn chunk_path_from_object_key(&self, object_key: &S3ObjectKey) -> Option<(PathBuf, [u8; 32])> { + fn chunk_path_from_object_key( + &self, + object_key: &S3ObjectKey, + ) -> Option<(PathBuf, [u8; 32], bool)> { // Check object is actually a chunk + let path = Path::new::(object_key); // file_name() should always be Some, as objects will have a filename - let digest = Path::new::(object_key).file_name()?; + let digest = path.file_name()?; let bytes = digest.as_bytes(); - if bytes.len() != 64 && bytes.len() != 64 + ".0.bad".len() { + let bad_ext_len = ".0.bad".len(); + let bad_chunk = if bytes.len() == 64 + bad_ext_len { + true + } else if bytes.len() == 64 { + false + } else { return None; - } + }; if !bytes.iter().take(64).all(u8::is_ascii_hexdigit) { return None; } @@ -1871,13 +1876,17 @@ impl DataStore { chunk_path.push(".chunks"); chunk_path.push(hexdigit_prefix); chunk_path.push(digest); + if bad_chunk { + let extension = unsafe { digest_str.get_unchecked(64..64 + bad_ext_len) }; + chunk_path.push(extension); + } let mut digest_bytes = [0u8; 32]; let digest = digest.as_bytes(); // safe to unwrap as already checked above hex::decode_to_slice(&digest[..64], &mut digest_bytes).unwrap(); - Some((chunk_path, digest_bytes)) + Some((chunk_path, digest_bytes, bad_chunk)) } pub fn try_shared_chunk_store_lock(&self) -> Result { -- 2.47.3 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel