From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id A7DEA1FF136 for ; Mon, 26 Jan 2026 12:38:58 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D9F17BD34; Mon, 26 Jan 2026 12:39:20 +0100 (CET) Message-ID: <767de04e-3f23-4cf3-8ad8-45d4c2d4013f@proxmox.com> Date: Mon, 26 Jan 2026 12:39:16 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird To: Proxmox Backup Server development discussion , Robert Obkircher References: <20260123154147.222215-1-r.obkircher@proxmox.com> <20260123154147.222215-10-r.obkircher@proxmox.com> Content-Language: en-US, de-DE From: Christian Ebner In-Reply-To: <20260123154147.222215-10-r.obkircher@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1769427493449 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.047 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH v4 proxmox-backup 09/11] datastore: use u64 instead of usize for fidx writer content size X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Not sure about these changes, maybe other devs have a stronger opinion on this one. If we do want to adapt this, then IMHO this should however be done throughout the whole codebase, for the dynamic index as well. On 1/23/26 4:43 PM, Robert Obkircher wrote: > This is closer to what the file format supports. > > Signed-off-by: Robert Obkircher > --- > pbs-datastore/src/datastore.rs | 6 +-- > pbs-datastore/src/fixed_index.rs | 69 ++++++++++++++++---------------- > src/api2/backup/environment.rs | 6 +-- > src/api2/backup/mod.rs | 2 +- > 4 files changed, 42 insertions(+), 41 deletions(-) > > diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs > index 56dfce6e..8770d942 100644 > --- a/pbs-datastore/src/datastore.rs > +++ b/pbs-datastore/src/datastore.rs > @@ -695,11 +695,11 @@ impl DataStore { > pub fn create_fixed_writer>( > &self, > filename: P, > - size: Option, > - chunk_size: usize, > + size: Option, > + chunk_size: u32, question: is this intentionally set to u32 instead of u64 like for other chunk sizes in this patch? Should be consistent ... > ) -> Result { > let full_path = self.inner.chunk_store.relative_path(filename.as_ref()); > - FixedIndexWriter::create(full_path, size, chunk_size) > + FixedIndexWriter::create(full_path, size, chunk_size.into()) > } > > pub fn open_fixed_reader>( > diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs > index 056ae07b..c2888372 100644 > --- a/pbs-datastore/src/fixed_index.rs > +++ b/pbs-datastore/src/fixed_index.rs > @@ -214,8 +214,8 @@ pub struct FixedIndexWriter { > file: File, > filename: PathBuf, > tmp_filename: PathBuf, > - chunk_size: usize, > - size: usize, > + chunk_size: u64, > + size: u64, > index_length: usize, > index_capacity: usize, > index: *mut u8, > @@ -248,8 +248,8 @@ impl FixedIndexWriter { > // Requires obtaining a shared chunk store lock beforehand > pub fn create( > full_path: impl Into, > - known_size: Option, > - chunk_size: usize, > + known_size: Option, > + chunk_size: u64, > ) -> Result { > let full_path = full_path.into(); > let mut tmp_path = full_path.clone(); > @@ -287,10 +287,13 @@ impl FixedIndexWriter { > > file.write_all(&buffer)?; > > - let (index_length, index_capacity) = known_size > - .map(|s| s.div_ceil(chunk_size)) > - .map(|len| (len, len)) > - .unwrap_or((0, Self::INITIAL_CAPACITY)); > + let (index_length, index_capacity) = match known_size { > + Some(s) => { > + let len = s.div_ceil(chunk_size).try_into()?; > + (len, len) > + } > + None => (0, Self::INITIAL_CAPACITY), > + }; > > let index_size = index_capacity * 32; > nix::unistd::ftruncate(&file, (header_size + index_size) as i64)?; > @@ -376,13 +379,13 @@ impl FixedIndexWriter { > /// The size also becomes fixed as soon as it is no longer divisible > /// by the block size, to ensure that only the last block can be > /// smaller. > - fn grow_to_size(&mut self, requested_size: usize) -> Result<(), Error> { > + fn grow_to_size(&mut self, requested_size: u64) -> Result<(), Error> { > if self.size < requested_size { > if !self.growable_size { > bail!("refusing to resize from {} to {requested_size}", self.size); > } > - let new_len = requested_size.div_ceil(self.chunk_size); > - if new_len * self.chunk_size != requested_size { > + let new_len = requested_size.div_ceil(self.chunk_size).try_into()?; > + if new_len as u64 * self.chunk_size != requested_size { > // not a full chunk, so this must be the last one > self.growable_size = false; > self.set_index_capacity_or_unmap(new_len)?; > @@ -463,12 +466,10 @@ impl FixedIndexWriter { > Ok(index_csum) > } > > - fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result { > - if offset < chunk_len { > + fn check_chunk_alignment(&self, offset: u64, chunk_len: u64) -> Result { > + let Some(pos) = offset.checked_sub(chunk_len) else { > bail!("got chunk with small offset ({} < {}", offset, chunk_len); > - } > - > - let pos = offset - chunk_len; > + }; > > if offset > self.size { > bail!("chunk data exceeds size ({} >= {})", offset, self.size); > @@ -490,7 +491,7 @@ impl FixedIndexWriter { > bail!("got unaligned chunk (pos = {})", pos); > } > > - Ok(pos / self.chunk_size) > + Ok((pos / self.chunk_size) as usize) > } > > fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> { > @@ -524,12 +525,12 @@ impl FixedIndexWriter { > /// If this writer has been created without a fixed size, the > /// index capacity and content size are increased automatically > /// until an incomplete chunk is encountered. > - pub fn add_chunk(&mut self, start: u64, size: u32, digest: &[u8; 32]) -> Result<(), Error> { > - let Some(end) = start.checked_add(size.into()) else { > + pub fn add_chunk(&mut self, start: u64, size: u64, digest: &[u8; 32]) -> Result<(), Error> { > + let Some(end) = start.checked_add(size) else { > bail!("add_chunk: start and size are too large: {start}+{size}"); > }; > - self.grow_to_size(end as usize)?; > - let idx = self.check_chunk_alignment(end as usize, size as usize)?; > + self.grow_to_size(end)?; > + let idx = self.check_chunk_alignment(end, size)?; > self.add_digest(idx, digest) > } > > @@ -538,7 +539,7 @@ impl FixedIndexWriter { > bail!("reusing the index is only supported with known input size"); > } > > - if self.chunk_size != reader.chunk_size { > + if Ok(self.chunk_size) != reader.chunk_size.try_into() { > bail!("can't reuse file with different chunk size"); > } > > @@ -560,7 +561,7 @@ mod tests { > use std::env; > use std::fs; > > - const CS: usize = 4096; > + const CS: u64 = 4096; > > #[test] > fn test_empty() { > @@ -606,7 +607,7 @@ mod tests { > > let initial = FixedIndexWriter::INITIAL_CAPACITY; > let steps = [1, 2, initial, initial + 1, 5 * initial, 10 * initial + 1]; > - let expected = test_data(steps.last().unwrap() * CS); > + let expected = test_data(*steps.last().unwrap() as u64 * CS); > > let mut begin = 0; > for chunk_count in steps { > @@ -623,7 +624,7 @@ mod tests { > w.close().unwrap(); > drop(w); > > - let size = expected.len() * CS; > + let size = expected.len() as u64 * CS; > check_with_reader(&path, size, &expected); > compare_to_known_size_writer(&path, size, &expected); > } > @@ -634,7 +635,7 @@ mod tests { > let path = dir.join("test_grow_to_misaligned_size"); > let mut w = FixedIndexWriter::create(&path, None, CS).unwrap(); > > - let size = (FixedIndexWriter::INITIAL_CAPACITY + 42) * CS - 1; // last is not full > + let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 42) * CS - 1; // last is not full > let expected = test_data(size); > > w.grow_to_size(size).unwrap(); > @@ -677,8 +678,8 @@ mod tests { > struct TestChunk { > digest: [u8; 32], > index: usize, > - size: usize, > - end: usize, > + size: u64, > + end: u64, > } > > impl TestChunk { > @@ -691,7 +692,7 @@ mod tests { > } > } > > - fn test_data(size: usize) -> Vec { > + fn test_data(size: u64) -> Vec { > (0..size.div_ceil(CS)) > .map(|index| { > let mut digest = [0u8; 32]; > @@ -706,24 +707,24 @@ mod tests { > }; > TestChunk { > digest, > - index, > + index: index as usize, > size, > - end: index * CS + size, > + end: index as u64 * CS + size, > } > }) > .collect() > } > > - fn check_with_reader(path: &Path, size: usize, chunks: &[TestChunk]) { > + fn check_with_reader(path: &Path, size: u64, chunks: &[TestChunk]) { > let reader = FixedIndexReader::open(path).unwrap(); > - assert_eq!(size as u64, reader.index_bytes()); > + assert_eq!(size, reader.index_bytes()); > assert_eq!(chunks.len(), reader.index_count()); > for c in chunks { > assert_eq!(&c.digest, reader.index_digest(c.index).unwrap()); > } > } > > - fn compare_to_known_size_writer(file: &Path, size: usize, chunks: &[TestChunk]) { > + fn compare_to_known_size_writer(file: &Path, size: u64, chunks: &[TestChunk]) { > let mut path = file.to_path_buf(); > path.set_extension("reference"); > let mut w = FixedIndexWriter::create(&path, Some(size), CS).unwrap(); > diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs > index 04c5bf84..7d49d47c 100644 > --- a/src/api2/backup/environment.rs > +++ b/src/api2/backup/environment.rs > @@ -67,7 +67,7 @@ struct DynamicWriterState { > struct FixedWriterState { > name: String, > index: FixedIndexWriter, > - size: Option, > + size: Option, > chunk_size: u32, > chunk_count: u64, > small_chunk_count: usize, // allow 0..1 small chunks (last chunk may be smaller) > @@ -349,7 +349,7 @@ impl BackupEnvironment { > &self, > index: FixedIndexWriter, > name: String, > - size: Option, > + size: Option, > chunk_size: u32, > incremental: bool, > ) -> Result { > @@ -442,7 +442,7 @@ impl BackupEnvironment { > ); > } > > - data.index.add_chunk(offset, size, digest)?; > + data.index.add_chunk(offset, size.into(), digest)?; > > data.chunk_count += 1; > > diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs > index c2822c18..54445efa 100644 > --- a/src/api2/backup/mod.rs > +++ b/src/api2/backup/mod.rs > @@ -480,7 +480,7 @@ fn create_fixed_index( > let env: &BackupEnvironment = rpcenv.as_ref(); > > let name = required_string_param(¶m, "archive-name")?.to_owned(); > - let size = param["size"].as_u64().map(usize::try_from).transpose()?; > + let size = param["size"].as_u64(); > let reuse_csum = param["reuse-csum"].as_str(); > > let archive_name = name.clone(); _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel