From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 998021FF139 for ; Tue, 10 Feb 2026 16:07:26 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4ECFA70B4; Tue, 10 Feb 2026 16:08:11 +0100 (CET) From: Robert Obkircher To: pbs-devel@lists.proxmox.com Subject: [PATCH v6 proxmox-backup 18/18] datastore: support incremental fidx uploads with different size Date: Tue, 10 Feb 2026 16:06:34 +0100 Message-ID: <20260210150642.469670-19-r.obkircher@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260210150642.469670-1-r.obkircher@proxmox.com> References: <20260210150642.469670-1-r.obkircher@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1770735972341 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.055 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: OJVVRAIDWXZXQO4TZCVYKJ2W7WCSESWN X-Message-ID-Hash: OJVVRAIDWXZXQO4TZCVYKJ2W7WCSESWN X-MailFrom: r.obkircher@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Copy as much as possible instead of requiring the index lengths to match exactly. For resizable writers the capacity is increased beforehand, but size and index length are only updated when chunks are added or with the final size on close. The partial chunk in the end is not tracked. It is the clients responsibility to overwrite it if necessary. Signed-off-by: Robert Obkircher --- pbs-datastore/src/fixed_index.rs | 66 +++++++++++++++++++++++++++----- src/api2/backup/environment.rs | 4 ++ 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs index ccbae72b..33a3734b 100644 --- a/pbs-datastore/src/fixed_index.rs +++ b/pbs-datastore/src/fixed_index.rs @@ -520,7 +520,10 @@ impl FixedIndexWriter { self.index_length ); } + self.add_digest_unchecked(index, digest) + } + fn add_digest_unchecked(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> { let Some(ptr) = &self.memory else { bail!("cannot write to closed index file."); }; @@ -553,23 +556,23 @@ impl FixedIndexWriter { self.add_digest(idx, digest) } + /// Copy the chunk hashes from a Reader to the start of this Writer. + /// + /// If this writer is resizable the capacity may increase, + /// but the size and length stay the same. pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> { - if self.growable_size { - bail!("reusing the index is only supported with known input size"); - } - if self.chunk_size != reader.chunk_size as u64 { bail!("can't reuse file with different chunk size"); } - if self.index_length != reader.index_count() { - bail!("clone_data_from failed - index sizes not equal"); + let count = reader.index_count(); + if self.growable_size && self.index_capacity < count { + self.set_index_capacity_or_unmap(count)?; } - for i in 0..self.index_length { - self.add_digest(i, reader.index_digest(i).unwrap())?; + for i in 0..count.min(self.index_capacity) { + self.add_digest_unchecked(i, reader.index_digest(i).unwrap())?; } - Ok(()) } } @@ -682,6 +685,51 @@ mod tests { dir.delete().unwrap(); } + #[test] + fn test_clone_data_from() { + let dir = TempDir::new().unwrap(); + let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 3) * CS as u64; + let mut expected = test_data(size); + + let reused = dir.path().join("reused"); + let mut w = FixedIndexWriter::create(&reused, Some(size), CS).unwrap(); + for c in expected.iter() { + c.add_to(&mut w); + } + w.close().unwrap(); + drop(w); + + let reused = FixedIndexReader::open(&reused).unwrap(); + + let truncated = dir.path().join("truncated"); + let size = size - CS as u64; + expected.pop(); + let mut w = FixedIndexWriter::create(&truncated, Some(size), CS).unwrap(); + w.clone_data_from(&reused).unwrap(); + w.close().unwrap(); + drop(w); + check_with_reader(&truncated, size, &expected); + compare_to_known_size_writer(&truncated, size, &expected); + + let modified = dir.path().join("modified"); + let mut w = FixedIndexWriter::create(&modified, None, CS).unwrap(); + w.clone_data_from(&reused).unwrap(); + { + let i = expected.len() / 2; + expected[i].digest[1] += 1; + let chunk = &expected[i]; + let chunk_pos = chunk.end - chunk.size as u64; + w.add_chunk(chunk_pos, chunk.size, &chunk.digest).unwrap(); + } + w.grow_to_size(size).unwrap(); + w.close().unwrap(); + drop(w); + check_with_reader(&modified, size, &expected); + compare_to_known_size_writer(&modified, size, &expected); + + dir.delete().unwrap(); + } + struct TestChunk { digest: [u8; 32], index: usize, diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs index 9645f6de..7063a706 100644 --- a/src/api2/backup/environment.rs +++ b/src/api2/backup/environment.rs @@ -609,6 +609,10 @@ impl BackupEnvironment { ); } + if data.incremental && data.size.is_none() { + data.index.grow_to_size(size)?; + } + if !data.incremental { let expected_count = data.index.index_length(); -- 2.47.3