From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 9E1B51FF137 for ; Tue, 14 Apr 2026 15:07:50 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 7B0F61936C; Tue, 14 Apr 2026 15:08:39 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup v3 25/30] sync: pull: extend encountered chunk by optional decrypted digest Date: Tue, 14 Apr 2026 14:59:18 +0200 Message-ID: <20260414125923.892345-26-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260414125923.892345-1-c.ebner@proxmox.com> References: <20260414125923.892345-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1776171502610 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.069 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [pull.rs] Message-ID-Hash: O5WBO3BTRHU6ZK2KWW3NJERHGBLYRUDA X-Message-ID-Hash: O5WBO3BTRHU6ZK2KWW3NJERHGBLYRUDA X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: For index files being decrypted during the pull, it is not enough to keep track of the processes source chunks, but the decrypted digest has to be known as well in order to rewrite the index file. Extend the encountered chunks such that this can be tracked as well. To not introduce clippy warnings and to keep the code readable, introduce the EncounteredChunksInfo struct as internal type for the hash map values. Signed-off-by: Christian Ebner --- changes since version 2: - no changes src/server/pull.rs | 53 +++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/server/pull.rs b/src/server/pull.rs index 9bac50d4f..61880183a 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -178,7 +178,7 @@ async fn pull_index_chunks( .filter(|info| { let guard = encountered_chunks.lock().unwrap(); match guard.check_reusable(&info.digest) { - Some(touched) => !touched, // reusable and already touched, can always skip + Some((touched, _decrypted_chunk)) => !touched, // reusable and already touched, can always skip None => true, } }), @@ -214,7 +214,7 @@ async fn pull_index_chunks( { // limit guard scope let mut guard = encountered_chunks.lock().unwrap(); - if let Some(touched) = guard.check_reusable(&info.digest) { + if let Some((touched, _decrypted_digest)) = guard.check_reusable(&info.digest) { if touched { return Ok::<_, Error>(()); } @@ -222,14 +222,14 @@ async fn pull_index_chunks( target.cond_touch_chunk(&info.digest, false) })?; if chunk_exists { - guard.mark_touched(&info.digest); + guard.mark_touched(&info.digest, None); //info!("chunk {} exists {}", pos, hex::encode(digest)); return Ok::<_, Error>(()); } } // mark before actually downloading the chunk, so this happens only once - guard.mark_reusable(&info.digest); - guard.mark_touched(&info.digest); + guard.mark_reusable(&info.digest, None); + guard.mark_touched(&info.digest, None); } //info!("sync {} chunk {}", pos, hex::encode(digest)); @@ -824,7 +824,7 @@ async fn pull_group( for pos in 0..index.index_count() { let chunk_info = index.chunk_info(pos).unwrap(); - reusable_chunks.mark_reusable(&chunk_info.digest); + reusable_chunks.mark_reusable(&chunk_info.digest, None); } } } @@ -1254,12 +1254,17 @@ async fn pull_ns( Ok((progress, sync_stats, errors)) } +struct EncounteredChunkInfo { + reusable: bool, + touched: bool, + decrypted_digest: Option<[u8; 32]>, +} + /// Store the state of encountered chunks, tracking if they can be reused for the /// index file currently being pulled and if the chunk has already been touched /// during this sync. struct EncounteredChunks { - // key: digest, value: (reusable, touched) - chunk_set: HashMap<[u8; 32], (bool, bool)>, + chunk_set: HashMap<[u8; 32], EncounteredChunkInfo>, } impl EncounteredChunks { @@ -1272,12 +1277,12 @@ impl EncounteredChunks { /// Check if the current state allows to reuse this chunk and if so, /// if the chunk has already been touched. - fn check_reusable(&self, digest: &[u8; 32]) -> Option { - if let Some((reusable, touched)) = self.chunk_set.get(digest) { - if !reusable { + fn check_reusable(&self, digest: &[u8; 32]) -> Option<(bool, Option<&[u8; 32]>)> { + if let Some(chunk_info) = self.chunk_set.get(digest) { + if !chunk_info.reusable { None } else { - Some(*touched) + Some((chunk_info.touched, chunk_info.decrypted_digest.as_ref())) } } else { None @@ -1285,28 +1290,36 @@ impl EncounteredChunks { } /// Mark chunk as reusable, inserting it as un-touched if not present - fn mark_reusable(&mut self, digest: &[u8; 32]) { + fn mark_reusable(&mut self, digest: &[u8; 32], decrypted_digest: Option<[u8; 32]>) { match self.chunk_set.entry(*digest) { Entry::Occupied(mut occupied) => { - let (reusable, _touched) = occupied.get_mut(); - *reusable = true; + let chunk_info = occupied.get_mut(); + chunk_info.reusable = true; } Entry::Vacant(vacant) => { - vacant.insert((true, false)); + vacant.insert(EncounteredChunkInfo { + reusable: true, + touched: false, + decrypted_digest, + }); } } } /// Mark chunk as touched during this sync, inserting it as not reusable /// but touched if not present. - fn mark_touched(&mut self, digest: &[u8; 32]) { + fn mark_touched(&mut self, digest: &[u8; 32], decrypted_digest: Option<[u8; 32]>) { match self.chunk_set.entry(*digest) { Entry::Occupied(mut occupied) => { - let (_reusable, touched) = occupied.get_mut(); - *touched = true; + let chunk_info = occupied.get_mut(); + chunk_info.touched = true; } Entry::Vacant(vacant) => { - vacant.insert((false, true)); + vacant.insert(EncounteredChunkInfo { + reusable: false, + touched: true, + decrypted_digest, + }); } } } -- 2.47.3