public inbox for pbs-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Robert Obkircher <r.obkircher@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [PATCH v5 proxmox-backup 16/16] datastore: FixedIndexWriter: switch internal chunk_size to u32
Date: Fri, 30 Jan 2026 17:45:40 +0100	[thread overview]
Message-ID: <20260130164552.281581-17-r.obkircher@proxmox.com> (raw)
In-Reply-To: <20260130164552.281581-1-r.obkircher@proxmox.com>

I don't really like this change becasue it introduces quite a few
conversions.

Signed-off-by: Robert Obkircher <r.obkircher@proxmox.com>
---
 pbs-datastore/src/datastore.rs   |  2 +-
 pbs-datastore/src/fixed_index.rs | 43 ++++++++++++++++----------------
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 98eab18d..631bdc30 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -699,7 +699,7 @@ impl DataStore {
         chunk_size: u32,
     ) -> Result<FixedIndexWriter, Error> {
         let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
-        FixedIndexWriter::create(full_path, size, chunk_size.into())
+        FixedIndexWriter::create(full_path, size, chunk_size)
     }
 
     pub fn open_fixed_reader<P: AsRef<Path>>(
diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs
index 3fa146f8..ba6874fe 100644
--- a/pbs-datastore/src/fixed_index.rs
+++ b/pbs-datastore/src/fixed_index.rs
@@ -225,7 +225,7 @@ pub struct FixedIndexWriter {
     file: File,
     filename: PathBuf,
     tmp_filename: PathBuf,
-    chunk_size: u64,
+    chunk_size: u32,
     size: u64,
     index_length: usize,
     index_capacity: usize,
@@ -263,7 +263,7 @@ impl FixedIndexWriter {
     pub fn create(
         full_path: impl Into<PathBuf>,
         known_size: Option<u64>,
-        chunk_size: u64,
+        chunk_size: u32,
     ) -> Result<Self, Error> {
         let full_path = full_path.into();
         let mut tmp_path = full_path.clone();
@@ -290,7 +290,7 @@ impl FixedIndexWriter {
 
         let (index_length, index_capacity) = match known_size {
             Some(s) => {
-                let len = s.div_ceil(chunk_size).try_into()?;
+                let len = s.div_ceil(chunk_size.into()).try_into()?;
                 (len, len)
             }
             None => (0, Self::INITIAL_CAPACITY),
@@ -313,7 +313,7 @@ impl FixedIndexWriter {
         let header = unsafe { memory.header().as_mut() };
         header.magic = file_formats::FIXED_SIZED_CHUNK_INDEX_1_0;
         header.ctime = i64::to_le(ctime);
-        header.chunk_size = u64::to_le(chunk_size);
+        header.chunk_size = u64::to_le(chunk_size.into());
         header.uuid = *uuid.as_bytes();
 
         Ok(Self {
@@ -402,8 +402,9 @@ impl FixedIndexWriter {
             if !self.growable_size {
                 bail!("refusing to resize from {} to {requested_size}", self.size);
             }
-            let new_len = requested_size.div_ceil(self.chunk_size).try_into()?;
-            if new_len as u64 * self.chunk_size != requested_size {
+            let cs = u64::from(self.chunk_size);
+            let new_len = requested_size.div_ceil(cs).try_into()?;
+            if new_len as u64 * cs != requested_size {
                 // not a full chunk, so this must be the last one
                 self.growable_size = false;
                 self.set_index_capacity_or_unmap(new_len)?;
@@ -472,8 +473,8 @@ impl FixedIndexWriter {
         Ok(index_csum)
     }
 
-    fn check_chunk_alignment(&self, offset: u64, chunk_len: u64) -> Result<usize, Error> {
-        let Some(pos) = offset.checked_sub(chunk_len) else {
+    fn check_chunk_alignment(&self, offset: u64, chunk_len: u32) -> Result<usize, Error> {
+        let Some(pos) = offset.checked_sub(chunk_len.into()) else {
             bail!("got chunk with small offset ({} < {}", offset, chunk_len);
         };
 
@@ -493,11 +494,11 @@ impl FixedIndexWriter {
             );
         }
 
-        if pos & (self.chunk_size - 1) != 0 {
+        if pos & (u64::from(self.chunk_size) - 1) != 0 {
             bail!("got unaligned chunk (pos = {})", pos);
         }
 
-        Ok((pos / self.chunk_size) as usize)
+        Ok((pos / u64::from(self.chunk_size)) as usize)
     }
 
     fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
@@ -536,7 +537,7 @@ impl FixedIndexWriter {
             bail!("add_chunk: start and size are too large: {start}+{size}");
         };
         self.grow_to_size(end)?;
-        let idx = self.check_chunk_alignment(end, size.into())?;
+        let idx = self.check_chunk_alignment(end, size)?;
         self.add_digest(idx, digest)
     }
 
@@ -568,7 +569,7 @@ mod tests {
     use super::*;
     use crate::temp_test_dir::TempTestDir;
 
-    const CS: u64 = 4096;
+    const CS: u32 = 4096;
 
     #[test]
     fn test_empty() {
@@ -594,7 +595,7 @@ mod tests {
         let path = dir.join("test_single_partial_chunk");
         let mut w = FixedIndexWriter::create(&path, None, CS).unwrap();
 
-        let size = CS - 1;
+        let size = CS as u64 - 1;
         let expected = test_data(size);
         w.grow_to_size(size).unwrap();
         expected[0].add_to(&mut w);
@@ -614,7 +615,7 @@ mod tests {
 
         let initial = FixedIndexWriter::INITIAL_CAPACITY;
         let steps = [1, 2, initial, initial + 1, 5 * initial, 10 * initial + 1];
-        let expected = test_data(*steps.last().unwrap() as u64 * CS);
+        let expected = test_data(*steps.last().unwrap() as u64 * CS as u64);
 
         let mut begin = 0;
         for chunk_count in steps {
@@ -631,7 +632,7 @@ mod tests {
         w.close().unwrap();
         drop(w);
 
-        let size = expected.len() as u64 * CS;
+        let size = expected.len() as u64 * CS as u64;
         check_with_reader(&path, size, &expected);
         compare_to_known_size_writer(&path, size, &expected);
     }
@@ -642,7 +643,7 @@ mod tests {
         let path = dir.join("test_grow_to_misaligned_size");
         let mut w = FixedIndexWriter::create(&path, None, CS).unwrap();
 
-        let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 42) * CS - 1; // last is not full
+        let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 42) * CS as u64 - 1; // last is not full
         let expected = test_data(size);
 
         w.grow_to_size(size).unwrap();
@@ -664,7 +665,7 @@ mod tests {
     struct TestChunk {
         digest: [u8; 32],
         index: usize,
-        size: u64,
+        size: u32,
         end: u64,
     }
 
@@ -679,23 +680,23 @@ mod tests {
     }
 
     fn test_data(size: u64) -> Vec<TestChunk> {
-        (0..size.div_ceil(CS))
+        (0..size.div_ceil(CS.into()))
             .map(|index| {
                 let mut digest = [0u8; 32];
                 let i = &(index as u64).to_le_bytes();
                 for c in digest.chunks_mut(i.len()) {
                     c.copy_from_slice(i);
                 }
-                let size = if ((index + 1) * CS) <= size {
+                let size = if ((index + 1) * CS as u64) <= size {
                     CS
                 } else {
-                    size % CS
+                    (size % CS as u64) as u32
                 };
                 TestChunk {
                     digest,
                     index: index as usize,
                     size,
-                    end: index as u64 * CS + size,
+                    end: index as u64 * CS as u64 + size as u64,
                 }
             })
             .collect()
-- 
2.47.3





      parent reply	other threads:[~2026-01-30 16:47 UTC|newest]

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-01-30 16:45 [PATCH v5 proxmox-backup 00/16] fix: #3847 pipe from STDIN to proxmox-backup-client Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 01/16] datastore: remove Arc<ChunkStore> from FixedIndexWriter Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 02/16] datastore: remove Arc<ChunkStore> from DynamicIndexWriter Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 03/16] datastore: add TempTestDir that is automatically deleted on drop Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 04/16] datastore: use temporary directory for chunk store test Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 05/16] datastore: support writing fidx files of unknown size Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 06/16] datastore: test FixedIndexWriter Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 07/16] api: backup: make fixed index file size optional Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 08/16] api: verify fixed index writer size on close Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 09/16] fix #3847: client: support fifo pipe inputs for images Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 10/16] client: treat minus sign as stdin Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 11/16] datastore: combine public FixedIndexWriter methods into add_chunk Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 12/16] datastore: use u64 instead of usize for fidx writer content size Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 13/16] datastore: compute fidx file size with overflow checks Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 14/16] datastore: support writing fidx files on systems with larger page size Robert Obkircher
2026-01-30 16:45 ` [PATCH v5 proxmox-backup 15/16] datastore: FixedIndexWriter: switch public chunk_size to u32 Robert Obkircher
2026-01-30 16:45 ` Robert Obkircher [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20260130164552.281581-17-r.obkircher@proxmox.com \
    --to=r.obkircher@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal