[PATCH v5 proxmox-backup 12/16] datastore: use u64 instead of usize for fidx writer content size

[Robert Obkircher <r.obkircher@proxmox.com>]

The file format is independent of the systems pointer size, so it
makes sense to use a fixed size type.

The chunk size is also a u64 to avoid casts and because that is what
is stored in the header anyway. Most other places limit it to u32, but
widening shouldn't be an issue.

Signed-off-by: Robert Obkircher <r.obkircher@proxmox.com>
---
 pbs-datastore/src/datastore.rs   |  4 +-
 pbs-datastore/src/fixed_index.rs | 71 ++++++++++++++++----------------
 src/api2/backup/environment.rs   |  8 ++--
 src/api2/backup/mod.rs           |  8 ++--
 4 files changed, 47 insertions(+), 44 deletions(-)

diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
index 694a5dd0..69bde917 100644
--- a/pbs-datastore/src/datastore.rs
+++ b/pbs-datastore/src/datastore.rs
@@ -695,8 +695,8 @@ impl DataStore {
     pub fn create_fixed_writer<P: AsRef<Path>>(
         &self,
         filename: P,
-        size: Option<usize>,
-        chunk_size: usize,
+        size: Option<u64>,
+        chunk_size: u64,
     ) -> Result<FixedIndexWriter, Error> {
         let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
         FixedIndexWriter::create(full_path, size, chunk_size)
diff --git a/pbs-datastore/src/fixed_index.rs b/pbs-datastore/src/fixed_index.rs
index f50f4517..753dd5a5 100644
--- a/pbs-datastore/src/fixed_index.rs
+++ b/pbs-datastore/src/fixed_index.rs
@@ -214,8 +214,8 @@ pub struct FixedIndexWriter {
     file: File,
     filename: PathBuf,
     tmp_filename: PathBuf,
-    chunk_size: usize,
-    size: usize,
+    chunk_size: u64,
+    size: u64,
     index_length: usize,
     index_capacity: usize,
     index: *mut u8,
@@ -252,8 +252,8 @@ impl FixedIndexWriter {
     // Requires obtaining a shared chunk store lock beforehand
     pub fn create(
         full_path: impl Into<PathBuf>,
-        known_size: Option<usize>,
-        chunk_size: usize,
+        known_size: Option<u64>,
+        chunk_size: u64,
     ) -> Result<Self, Error> {
         let full_path = full_path.into();
         let mut tmp_path = full_path.clone();
@@ -291,10 +291,13 @@ impl FixedIndexWriter {
 
         file.write_all(&buffer)?;
 
-        let (index_length, index_capacity) = known_size
-            .map(|s| s.div_ceil(chunk_size))
-            .map(|len| (len, len))
-            .unwrap_or((0, Self::INITIAL_CAPACITY));
+        let (index_length, index_capacity) = match known_size {
+            Some(s) => {
+                let len = s.div_ceil(chunk_size).try_into()?;
+                (len, len)
+            }
+            None => (0, Self::INITIAL_CAPACITY),
+        };
 
         let index_size = index_capacity * 32;
         nix::unistd::ftruncate(&file, (header_size + index_size) as i64)?;
@@ -380,13 +383,13 @@ impl FixedIndexWriter {
     /// The size also becomes fixed as soon as it is no longer divisible
     /// by the block size, to ensure that only the last block can be
     /// smaller.
-    fn grow_to_size(&mut self, requested_size: usize) -> Result<(), Error> {
+    fn grow_to_size(&mut self, requested_size: u64) -> Result<(), Error> {
         if self.size < requested_size {
             if !self.growable_size {
                 bail!("refusing to resize from {} to {requested_size}", self.size);
             }
-            let new_len = requested_size.div_ceil(self.chunk_size);
-            if new_len * self.chunk_size != requested_size {
+            let new_len = requested_size.div_ceil(self.chunk_size).try_into()?;
+            if new_len as u64 * self.chunk_size != requested_size {
                 // not a full chunk, so this must be the last one
                 self.growable_size = false;
                 self.set_index_capacity_or_unmap(new_len)?;
@@ -410,7 +413,7 @@ impl FixedIndexWriter {
 
     /// The current total size of the referenced content.
     pub fn size(&self) -> u64 {
-        self.size as u64
+        self.size
     }
 
     fn unmap(&mut self) -> Result<(), Error> {
@@ -467,12 +470,10 @@ impl FixedIndexWriter {
         Ok(index_csum)
     }
 
-    fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
-        if offset < chunk_len {
+    fn check_chunk_alignment(&self, offset: u64, chunk_len: u64) -> Result<usize, Error> {
+        let Some(pos) = offset.checked_sub(chunk_len) else {
             bail!("got chunk with small offset ({} < {}", offset, chunk_len);
-        }
-
-        let pos = offset - chunk_len;
+        };
 
         if offset > self.size {
             bail!("chunk data exceeds size ({} >= {})", offset, self.size);
@@ -494,7 +495,7 @@ impl FixedIndexWriter {
             bail!("got unaligned chunk (pos = {})", pos);
         }
 
-        Ok(pos / self.chunk_size)
+        Ok((pos / self.chunk_size) as usize)
     }
 
     fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
@@ -528,12 +529,12 @@ impl FixedIndexWriter {
     /// If this writer has been created without a fixed size, the
     /// index capacity and content size are increased automatically
     /// until an incomplete chunk is encountered.
-    pub fn add_chunk(&mut self, start: u64, size: u32, digest: &[u8; 32]) -> Result<(), Error> {
-        let Some(end) = start.checked_add(size.into()) else {
+    pub fn add_chunk(&mut self, start: u64, size: u64, digest: &[u8; 32]) -> Result<(), Error> {
+        let Some(end) = start.checked_add(size) else {
             bail!("add_chunk: start and size are too large: {start}+{size}");
         };
-        self.grow_to_size(end as usize)?;
-        let idx = self.check_chunk_alignment(end as usize, size as usize)?;
+        self.grow_to_size(end)?;
+        let idx = self.check_chunk_alignment(end, size)?;
         self.add_digest(idx, digest)
     }
 
@@ -542,7 +543,7 @@ impl FixedIndexWriter {
             bail!("reusing the index is only supported with known input size");
         }
 
-        if self.chunk_size != reader.chunk_size {
+        if Ok(self.chunk_size) != reader.chunk_size.try_into() {
             bail!("can't reuse file with different chunk size");
         }
 
@@ -565,7 +566,7 @@ mod tests {
     use super::*;
     use crate::temp_test_dir::TempTestDir;
 
-    const CS: usize = 4096;
+    const CS: u64 = 4096;
 
     #[test]
     fn test_empty() {
@@ -611,7 +612,7 @@ mod tests {
 
         let initial = FixedIndexWriter::INITIAL_CAPACITY;
         let steps = [1, 2, initial, initial + 1, 5 * initial, 10 * initial + 1];
-        let expected = test_data(steps.last().unwrap() * CS);
+        let expected = test_data(*steps.last().unwrap() as u64 * CS);
 
         let mut begin = 0;
         for chunk_count in steps {
@@ -628,7 +629,7 @@ mod tests {
         w.close().unwrap();
         drop(w);
 
-        let size = expected.len() * CS;
+        let size = expected.len() as u64 * CS;
         check_with_reader(&path, size, &expected);
         compare_to_known_size_writer(&path, size, &expected);
     }
@@ -639,7 +640,7 @@ mod tests {
         let path = dir.join("test_grow_to_misaligned_size");
         let mut w = FixedIndexWriter::create(&path, None, CS).unwrap();
 
-        let size = (FixedIndexWriter::INITIAL_CAPACITY + 42) * CS - 1; // last is not full
+        let size = (FixedIndexWriter::INITIAL_CAPACITY as u64 + 42) * CS - 1; // last is not full
         let expected = test_data(size);
 
         w.grow_to_size(size).unwrap();
@@ -661,8 +662,8 @@ mod tests {
     struct TestChunk {
         digest: [u8; 32],
         index: usize,
-        size: usize,
-        end: usize,
+        size: u64,
+        end: u64,
     }
 
     impl TestChunk {
@@ -675,7 +676,7 @@ mod tests {
         }
     }
 
-    fn test_data(size: usize) -> Vec<TestChunk> {
+    fn test_data(size: u64) -> Vec<TestChunk> {
         (0..size.div_ceil(CS))
             .map(|index| {
                 let mut digest = [0u8; 32];
@@ -690,24 +691,24 @@ mod tests {
                 };
                 TestChunk {
                     digest,
-                    index,
+                    index: index as usize,
                     size,
-                    end: index * CS + size,
+                    end: index as u64 * CS + size,
                 }
             })
             .collect()
     }
 
-    fn check_with_reader(path: &Path, size: usize, chunks: &[TestChunk]) {
+    fn check_with_reader(path: &Path, size: u64, chunks: &[TestChunk]) {
         let reader = FixedIndexReader::open(path).unwrap();
-        assert_eq!(size as u64, reader.index_bytes());
+        assert_eq!(size, reader.index_bytes());
         assert_eq!(chunks.len(), reader.index_count());
         for c in chunks {
             assert_eq!(&c.digest, reader.index_digest(c.index).unwrap());
         }
     }
 
-    fn compare_to_known_size_writer(file: &Path, size: usize, chunks: &[TestChunk]) {
+    fn compare_to_known_size_writer(file: &Path, size: u64, chunks: &[TestChunk]) {
         let mut path = file.to_path_buf();
         path.set_extension("reference");
         let mut w = FixedIndexWriter::create(&path, Some(size), CS).unwrap();
diff --git a/src/api2/backup/environment.rs b/src/api2/backup/environment.rs
index 04c5bf84..9773a87a 100644
--- a/src/api2/backup/environment.rs
+++ b/src/api2/backup/environment.rs
@@ -67,7 +67,7 @@ struct DynamicWriterState {
 struct FixedWriterState {
     name: String,
     index: FixedIndexWriter,
-    size: Option<usize>,
+    size: Option<u64>,
     chunk_size: u32,
     chunk_count: u64,
     small_chunk_count: usize, // allow 0..1 small chunks (last chunk may be smaller)
@@ -349,7 +349,7 @@ impl BackupEnvironment {
         &self,
         index: FixedIndexWriter,
         name: String,
-        size: Option<usize>,
+        size: Option<u64>,
         chunk_size: u32,
         incremental: bool,
     ) -> Result<usize, Error> {
@@ -442,7 +442,7 @@ impl BackupEnvironment {
             );
         }
 
-        data.index.add_chunk(offset, size, digest)?;
+        data.index.add_chunk(offset, size.into(), digest)?;
 
         data.chunk_count += 1;
 
@@ -622,7 +622,7 @@ impl BackupEnvironment {
             }
 
             if let Some(known_size) = data.size {
-                if size != known_size as u64 {
+                if size != known_size {
                     bail!(
                         "fixed writer '{}' close failed - unexpected file size ({} != {})",
                         data.name,
diff --git a/src/api2/backup/mod.rs b/src/api2/backup/mod.rs
index c2822c18..55207e6c 100644
--- a/src/api2/backup/mod.rs
+++ b/src/api2/backup/mod.rs
@@ -480,7 +480,7 @@ fn create_fixed_index(
     let env: &BackupEnvironment = rpcenv.as_ref();
 
     let name = required_string_param(&param, "archive-name")?.to_owned();
-    let size = param["size"].as_u64().map(usize::try_from).transpose()?;
+    let size = param["size"].as_u64();
     let reuse_csum = param["reuse-csum"].as_str();
 
     let archive_name = name.clone();
@@ -528,13 +528,15 @@ fn create_fixed_index(
         reader = Some(index);
     }
 
-    let mut writer = env.datastore.create_fixed_writer(&path, size, chunk_size)?;
+    let mut writer = env
+        .datastore
+        .create_fixed_writer(&path, size, chunk_size.into())?;
 
     if let Some(reader) = reader {
         writer.clone_data_from(&reader)?;
     }
 
-    let wid = env.register_fixed_writer(writer, name, size, chunk_size as u32, incremental)?;
+    let wid = env.register_fixed_writer(writer, name, size, chunk_size, incremental)?;
 
     env.log(format!("created new fixed index {wid} ({path:?})"));
 
-- 
2.47.3