public inbox for pbs-devel@lists.proxmox.com
 help / color / mirror / Atom feed
From: Christian Ebner <c.ebner@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH v6 pxar 05/14] encoder: move to stack based state tracking
Date: Tue, 14 May 2024 12:33:21 +0200	[thread overview]
Message-ID: <20240514103421.289431-6-c.ebner@proxmox.com> (raw)
In-Reply-To: <20240514103421.289431-1-c.ebner@proxmox.com>

In preparation for the proxmox-backup-client look-ahead caching,
where a passing around of different encoder instances with internal
references is not feasible.

Instead of creating a new encoder instance for each directory level
and keeping references to the parent state, use an internal stack.
Adds additional helper functions to solve borrow issues, when both
the state and writers have to be accessed by a mutable reference.

This is a breaking change in the pxar library API.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 examples/pxarcmd.rs  |   7 +-
 src/encoder/aio.rs   |  26 ++--
 src/encoder/mod.rs   | 323 +++++++++++++++++++++++++------------------
 src/encoder/sync.rs  |  16 ++-
 tests/simple/fs.rs   |   6 +-
 tests/simple/main.rs |   3 +
 6 files changed, 226 insertions(+), 155 deletions(-)

diff --git a/examples/pxarcmd.rs b/examples/pxarcmd.rs
index e0c779d..0294eba 100644
--- a/examples/pxarcmd.rs
+++ b/examples/pxarcmd.rs
@@ -106,6 +106,7 @@ fn cmd_create(mut args: std::env::ArgsOs) -> Result<(), Error> {
     let mut encoder = Encoder::create(file, &meta)?;
     add_directory(&mut encoder, dir, &dir_path, &mut HashMap::new())?;
     encoder.finish()?;
+    encoder.close()?;
 
     Ok(())
 }
@@ -138,14 +139,14 @@ fn add_directory<'a, T: SeqWrite + 'a>(
 
         let meta = Metadata::from(&file_meta);
         if file_type.is_dir() {
-            let mut dir = encoder.create_directory(file_name, &meta)?;
+            encoder.create_directory(file_name, &meta)?;
             add_directory(
-                &mut dir,
+                encoder,
                 std::fs::read_dir(file_path)?,
                 root_path,
                 &mut *hardlinks,
             )?;
-            dir.finish()?;
+            encoder.finish()?;
         } else if file_type.is_symlink() {
             todo!("symlink handling");
         } else if file_type.is_file() {
diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
index 31a1a2f..635e550 100644
--- a/src/encoder/aio.rs
+++ b/src/encoder/aio.rs
@@ -109,20 +109,23 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
         &mut self,
         file_name: P,
         metadata: &Metadata,
-    ) -> io::Result<Encoder<'_, T>> {
-        Ok(Encoder {
-            inner: self
-                .inner
-                .create_directory(file_name.as_ref(), metadata)
-                .await?,
-        })
+    ) -> io::Result<()> {
+        self.inner
+            .create_directory(file_name.as_ref(), metadata)
+            .await
     }
 
-    /// Finish this directory. This is mandatory, otherwise the `Drop` handler will `panic!`.
-    pub async fn finish(self) -> io::Result<()> {
+    /// Finish this directory. This is mandatory, encodes the end for the current directory.
+    pub async fn finish(&mut self) -> io::Result<()> {
         self.inner.finish().await
     }
 
+    /// Close the encoder instance. This is mandatory, encodes the end for the optional payload
+    /// output stream, if some is given
+    pub async fn close(self) -> io::Result<()> {
+        self.inner.close().await
+    }
+
     /// Add a symbolic link to the archive.
     pub async fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
         &mut self,
@@ -307,11 +310,12 @@ mod test {
                     .await
                     .unwrap();
             {
-                let mut dir = encoder
+                encoder
                     .create_directory("baba", &Metadata::dir_builder(0o700).build())
                     .await
                     .unwrap();
-                dir.create_file(&Metadata::file_builder(0o755).build(), "abab", 1024)
+                encoder
+                    .create_file(&Metadata::file_builder(0o755).build(), "abab", 1024)
                     .await
                     .unwrap();
             }
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index 99c3758..369a937 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -224,9 +224,22 @@ struct EncoderState {
 
     /// Track the bytes written to the payload writer
     payload_write_position: u64,
+
+    /// Mark the encoder state as correctly finished, ready to be dropped
+    finished: bool,
 }
 
 impl EncoderState {
+    #[inline]
+    fn position(&self) -> u64 {
+        self.write_position
+    }
+
+    #[inline]
+    fn payload_position(&self) -> u64 {
+        self.payload_write_position
+    }
+
     fn merge_error(&mut self, error: Option<EncodeError>) {
         // one error is enough:
         if self.encode_error.is_none() {
@@ -237,6 +250,23 @@ impl EncoderState {
     fn add_error(&mut self, error: EncodeError) {
         self.merge_error(Some(error));
     }
+
+    fn finish(&mut self) -> Option<EncodeError> {
+        self.finished = true;
+        self.encode_error.take()
+    }
+}
+
+impl Drop for EncoderState {
+    fn drop(&mut self) {
+        if !self.finished {
+            eprintln!("unfinished encoder state dropped");
+        }
+
+        if self.encode_error.is_some() {
+            eprintln!("finished encoder state with errors");
+        }
+    }
 }
 
 pub(crate) enum EncoderOutput<'a, T> {
@@ -244,16 +274,6 @@ pub(crate) enum EncoderOutput<'a, T> {
     Borrowed(&'a mut T),
 }
 
-impl<'a, T> EncoderOutput<'a, T> {
-    #[inline]
-    fn to_borrowed_mut<'s>(&'s mut self) -> EncoderOutput<'s, T>
-    where
-        'a: 's,
-    {
-        EncoderOutput::Borrowed(self.as_mut())
-    }
-}
-
 impl<'a, T> std::convert::AsMut<T> for EncoderOutput<'a, T> {
     fn as_mut(&mut self) -> &mut T {
         match self {
@@ -282,8 +302,8 @@ impl<'a, T> std::convert::From<&'a mut T> for EncoderOutput<'a, T> {
 pub(crate) struct EncoderImpl<'a, T: SeqWrite + 'a> {
     output: EncoderOutput<'a, T>,
     payload_output: EncoderOutput<'a, Option<T>>,
-    state: EncoderState,
-    parent: Option<&'a mut EncoderState>,
+    /// EncoderState stack storing the state for each directory level
+    state: Vec<EncoderState>,
     finished: bool,
 
     /// Since only the "current" entry can be actively writing files, we share the file copy
@@ -293,15 +313,12 @@ pub(crate) struct EncoderImpl<'a, T: SeqWrite + 'a> {
 
 impl<'a, T: SeqWrite + 'a> Drop for EncoderImpl<'a, T> {
     fn drop(&mut self) {
-        if let Some(ref mut parent) = self.parent {
-            // propagate errors:
-            parent.merge_error(self.state.encode_error);
-            if !self.finished {
-                parent.add_error(EncodeError::IncompleteDirectory);
-            }
-        } else if !self.finished {
-            // FIXME: how do we deal with this?
-            // eprintln!("Encoder dropped without finishing!");
+        if !self.finished {
+            eprintln!("unclosed encoder dropped");
+        }
+
+        if !self.state.is_empty() {
+            eprintln!("closed encoder dropped with state");
         }
     }
 }
@@ -318,8 +335,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         let mut this = Self {
             output,
             payload_output: EncoderOutput::Owned(None),
-            state: EncoderState::default(),
-            parent: None,
+            state: vec![EncoderState::default()],
             finished: false,
             file_copy_buffer: Arc::new(Mutex::new(unsafe {
                 crate::util::vec_new_uninitialized(1024 * 1024)
@@ -327,7 +343,8 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         };
 
         this.encode_metadata(metadata).await?;
-        this.state.files_offset = this.position();
+        let state = this.state_mut()?;
+        state.files_offset = state.position();
 
         if let Some(payload_output) = payload_output {
             this.payload_output = EncoderOutput::Owned(Some(payload_output));
@@ -337,13 +354,50 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     }
 
     fn check(&self) -> io::Result<()> {
-        match self.state.encode_error {
+        if self.finished {
+            io_bail!("unexpected encoder finished state");
+        }
+        let state = self.state()?;
+        match state.encode_error {
             Some(EncodeError::IncompleteFile) => io_bail!("incomplete file"),
             Some(EncodeError::IncompleteDirectory) => io_bail!("directory not finalized"),
             None => Ok(()),
         }
     }
 
+    fn state(&self) -> io::Result<&EncoderState> {
+        self.state
+            .last()
+            .ok_or_else(|| io_format_err!("encoder state stack underflow"))
+    }
+
+    fn state_mut(&mut self) -> io::Result<&mut EncoderState> {
+        self.state
+            .last_mut()
+            .ok_or_else(|| io_format_err!("encoder state stack underflow"))
+    }
+
+    fn output_state(&mut self) -> io::Result<(&mut T, &mut EncoderState)> {
+        Ok((
+            self.output.as_mut(),
+            self.state
+                .last_mut()
+                .ok_or_else(|| io_format_err!("encoder state stack underflow"))?,
+        ))
+    }
+
+    fn output_payload_output_state(
+        &mut self,
+    ) -> io::Result<(&mut T, Option<&mut T>, &mut EncoderState)> {
+        Ok((
+            self.output.as_mut(),
+            self.payload_output.as_mut().as_mut(),
+            self.state
+                .last_mut()
+                .ok_or_else(|| io_format_err!("encoder state stack underflow"))?,
+        ))
+    }
+
     pub async fn create_file<'b>(
         &'b mut self,
         metadata: &Metadata,
@@ -368,22 +422,17 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     {
         self.check()?;
 
-        let file_offset = self.position();
+        let file_offset = self.state()?.position();
         self.start_file_do(Some(metadata), file_name).await?;
 
-        if let Some(payload_output) = self.payload_output.as_mut() {
+        if let (output, Some(payload_output), state) = self.output_payload_output_state()? {
             // payload references must point to the position prior to the payload header,
             // separating payload entries in the payload stream
-            let payload_position = self.state.payload_write_position;
+            let payload_position = state.payload_position();
 
             let header = format::Header::with_content_size(format::PXAR_PAYLOAD, file_size);
             header.check_header_size()?;
-            seq_write_struct(
-                payload_output,
-                header,
-                &mut self.state.payload_write_position,
-            )
-            .await?;
+            seq_write_struct(payload_output, header, &mut state.payload_write_position).await?;
 
             let payload_ref = PayloadRef {
                 offset: payload_position,
@@ -391,32 +440,34 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
             };
 
             seq_write_pxar_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_PAYLOAD_REF,
                 &payload_ref.data(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         } else {
             let header = format::Header::with_content_size(format::PXAR_PAYLOAD, file_size);
             header.check_header_size()?;
-            seq_write_struct(self.output.as_mut(), header, &mut self.state.write_position).await?;
+            let (output, state) = self.output_state()?;
+            seq_write_struct(output, header, &mut state.write_position).await?;
         }
 
-        let payload_data_offset = self.position();
+        let (output, payload_output, state) = self.output_payload_output_state()?;
+        let payload_data_offset = state.position();
 
         let meta_size = payload_data_offset - file_offset;
 
         Ok(FileImpl {
-            output: self.output.as_mut(),
-            payload_output: self.payload_output.as_mut().as_mut(),
+            output,
+            payload_output,
             goodbye_item: GoodbyeItem {
                 hash: format::hash_filename(file_name),
                 offset: file_offset,
                 size: file_size + meta_size,
             },
             remaining_size: file_size,
-            parent: &mut self.state,
+            parent: state,
         })
     }
 
@@ -497,7 +548,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         target: &Path,
         target_offset: LinkOffset,
     ) -> io::Result<()> {
-        let current_offset = self.position();
+        let current_offset = self.state()?.position();
         if current_offset <= target_offset.0 {
             io_bail!("invalid hardlink offset, can only point to prior files");
         }
@@ -571,24 +622,20 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     ) -> io::Result<LinkOffset> {
         self.check()?;
 
-        let file_offset = self.position();
+        let file_offset = self.state()?.position();
 
         let file_name = file_name.as_os_str().as_bytes();
 
         self.start_file_do(metadata, file_name).await?;
+
+        let (output, state) = self.output_state()?;
         if let Some((htype, entry_data)) = entry_htype_data {
-            seq_write_pxar_entry(
-                self.output.as_mut(),
-                htype,
-                entry_data,
-                &mut self.state.write_position,
-            )
-            .await?;
+            seq_write_pxar_entry(output, htype, entry_data, &mut state.write_position).await?;
         }
 
-        let end_offset = self.position();
+        let end_offset = state.position();
 
-        self.state.items.push(GoodbyeItem {
+        state.items.push(GoodbyeItem {
             hash: format::hash_filename(file_name),
             offset: file_offset,
             size: end_offset - file_offset,
@@ -597,21 +644,11 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         Ok(LinkOffset(file_offset))
     }
 
-    #[inline]
-    fn position(&mut self) -> u64 {
-        self.state.write_position
-    }
-
-    #[inline]
-    fn payload_position(&mut self) -> u64 {
-        self.state.payload_write_position
-    }
-
     pub async fn create_directory(
         &mut self,
         file_name: &Path,
         metadata: &Metadata,
-    ) -> io::Result<EncoderImpl<'_, T>> {
+    ) -> io::Result<()> {
         self.check()?;
 
         if !metadata.is_dir() {
@@ -621,37 +658,32 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         let file_name = file_name.as_os_str().as_bytes();
         let file_hash = format::hash_filename(file_name);
 
-        let file_offset = self.position();
+        let file_offset = self.state()?.position();
         self.encode_filename(file_name).await?;
 
-        let entry_offset = self.position();
+        let entry_offset = self.state()?.position();
         self.encode_metadata(metadata).await?;
 
-        let files_offset = self.position();
+        let state = self.state_mut()?;
+        let files_offset = state.position();
 
         // the child will write to OUR state now:
-        let write_position = self.position();
-        let payload_write_position = self.payload_position();
-
-        let file_copy_buffer = Arc::clone(&self.file_copy_buffer);
-
-        Ok(EncoderImpl {
-            // always forward as Borrowed(), to avoid stacking references on nested calls
-            output: self.output.to_borrowed_mut(),
-            payload_output: self.payload_output.to_borrowed_mut(),
-            state: EncoderState {
-                entry_offset,
-                files_offset,
-                file_offset: Some(file_offset),
-                file_hash,
-                write_position,
-                payload_write_position,
-                ..Default::default()
-            },
-            parent: Some(&mut self.state),
+        let write_position = state.position();
+        let payload_write_position = state.payload_position();
+
+        self.state.push(EncoderState {
+            items: Vec::new(),
+            encode_error: None,
+            entry_offset,
+            files_offset,
+            file_offset: Some(file_offset),
+            file_hash,
+            write_position,
+            payload_write_position,
             finished: false,
-            file_copy_buffer,
-        })
+        });
+
+        Ok(())
     }
 
     async fn start_file_do(
@@ -667,11 +699,12 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     }
 
     async fn encode_metadata(&mut self, metadata: &Metadata) -> io::Result<()> {
+        let (output, state) = self.output_state()?;
         seq_write_pxar_struct_entry(
-            self.output.as_mut(),
+            output,
             format::PXAR_ENTRY,
             metadata.stat.clone(),
-            &mut self.state.write_position,
+            &mut state.write_position,
         )
         .await?;
 
@@ -693,72 +726,74 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     }
 
     async fn write_xattr(&mut self, xattr: &format::XAttr) -> io::Result<()> {
+        let (output, state) = self.output_state()?;
         seq_write_pxar_entry(
-            self.output.as_mut(),
+            output,
             format::PXAR_XATTR,
             &xattr.data,
-            &mut self.state.write_position,
+            &mut state.write_position,
         )
         .await
     }
 
     async fn write_acls(&mut self, acl: &crate::Acl) -> io::Result<()> {
+        let (output, state) = self.output_state()?;
         for acl in &acl.users {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_USER,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
 
         for acl in &acl.groups {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_GROUP,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
 
         if let Some(acl) = &acl.group_obj {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_GROUP_OBJ,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
 
         if let Some(acl) = &acl.default {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_DEFAULT,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
 
         for acl in &acl.default_users {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_DEFAULT_USER,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
 
         for acl in &acl.default_groups {
             seq_write_pxar_struct_entry(
-                self.output.as_mut(),
+                output,
                 format::PXAR_ACL_DEFAULT_GROUP,
                 acl.clone(),
-                &mut self.state.write_position,
+                &mut state.write_position,
             )
             .await?;
         }
@@ -767,11 +802,12 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
     }
 
     async fn write_file_capabilities(&mut self, fcaps: &format::FCaps) -> io::Result<()> {
+        let (output, state) = self.output_state()?;
         seq_write_pxar_entry(
-            self.output.as_mut(),
+            output,
             format::PXAR_FCAPS,
             &fcaps.data,
-            &mut self.state.write_position,
+            &mut state.write_position,
         )
         .await
     }
@@ -780,35 +816,32 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         &mut self,
         quota_project_id: &format::QuotaProjectId,
     ) -> io::Result<()> {
+        let (output, state) = self.output_state()?;
         seq_write_pxar_struct_entry(
-            self.output.as_mut(),
+            output,
             format::PXAR_QUOTA_PROJID,
             *quota_project_id,
-            &mut self.state.write_position,
+            &mut state.write_position,
         )
         .await
     }
 
     async fn encode_filename(&mut self, file_name: &[u8]) -> io::Result<()> {
         crate::util::validate_filename(file_name)?;
+        let (output, state) = self.output_state()?;
         seq_write_pxar_entry_zero(
-            self.output.as_mut(),
+            output,
             format::PXAR_FILENAME,
             file_name,
-            &mut self.state.write_position,
+            &mut state.write_position,
         )
         .await
     }
 
-    pub async fn finish(mut self) -> io::Result<()> {
-        let tail_bytes = self.finish_goodbye_table().await?;
-        seq_write_pxar_entry(
-            self.output.as_mut(),
-            format::PXAR_GOODBYE,
-            &tail_bytes,
-            &mut self.state.write_position,
-        )
-        .await?;
+    pub async fn close(mut self) -> io::Result<()> {
+        if !self.state.is_empty() {
+            io_bail!("unexpected state on encoder close");
+        }
 
         if let EncoderOutput::Owned(Some(output)) = &mut self.payload_output {
             flush(output).await?;
@@ -818,34 +851,60 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
             flush(output).await?;
         }
 
-        // done up here because of the self-borrow and to propagate
-        let end_offset = self.position();
-        let payload_end_offset = self.payload_position();
+        self.finished = true;
+
+        Ok(())
+    }
 
-        if let Some(parent) = &mut self.parent {
+    pub async fn finish(&mut self) -> io::Result<()> {
+        let tail_bytes = self.finish_goodbye_table().await?;
+        let mut state = self
+            .state
+            .pop()
+            .ok_or_else(|| io_format_err!("encoder state stack underflow"))?;
+        seq_write_pxar_entry(
+            self.output.as_mut(),
+            format::PXAR_GOODBYE,
+            &tail_bytes,
+            &mut state.write_position,
+        )
+        .await?;
+
+        let end_offset = state.position();
+        let payload_end_offset = state.payload_position();
+
+        let encode_error = state.finish();
+        if let Some(parent) = self.state.last_mut() {
             parent.write_position = end_offset;
             parent.payload_write_position = payload_end_offset;
 
-            let file_offset = self
-                .state
+            let file_offset = state
                 .file_offset
                 .expect("internal error: parent set but no file_offset?");
 
             parent.items.push(GoodbyeItem {
-                hash: self.state.file_hash,
+                hash: state.file_hash,
                 offset: file_offset,
                 size: end_offset - file_offset,
             });
+            // propagate errors
+            parent.merge_error(encode_error);
+            Ok(())
+        } else {
+            match encode_error {
+                Some(EncodeError::IncompleteFile) => io_bail!("incomplete file"),
+                Some(EncodeError::IncompleteDirectory) => io_bail!("directory not finalized"),
+                None => Ok(()),
+            }
         }
-        self.finished = true;
-        Ok(())
     }
 
     async fn finish_goodbye_table(&mut self) -> io::Result<Vec<u8>> {
-        let goodbye_offset = self.position();
+        let state = self.state_mut()?;
+        let goodbye_offset = state.position();
 
         // "take" out the tail (to not leave an array of endian-swapped structs in `self`)
-        let mut tail = take(&mut self.state.items);
+        let mut tail = take(&mut state.items);
         let tail_size = (tail.len() + 1) * size_of::<GoodbyeItem>();
         let goodbye_size = tail_size as u64 + size_of::<format::Header>() as u64;
 
@@ -870,7 +929,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         bst.push(
             GoodbyeItem {
                 hash: format::PXAR_GOODBYE_TAIL_MARKER,
-                offset: goodbye_offset - self.state.entry_offset,
+                offset: goodbye_offset - state.entry_offset,
                 size: goodbye_size,
             }
             .to_le(),
@@ -900,8 +959,8 @@ pub(crate) struct FileImpl<'a, S: SeqWrite> {
     /// exactly zero.
     remaining_size: u64,
 
-    /// The directory containing this file. This is where we propagate the `IncompleteFile` error
-    /// to, and where we insert our `GoodbyeItem`.
+    /// The directory stack with the last item being the directory containing this file. This is
+    /// where we propagate the `IncompleteFile` error to, and where we insert our `GoodbyeItem`.
     parent: &'a mut EncoderState,
 }
 
diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
index 96d056d..d0d62ba 100644
--- a/src/encoder/sync.rs
+++ b/src/encoder/sync.rs
@@ -106,17 +106,21 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
         &mut self,
         file_name: P,
         metadata: &Metadata,
-    ) -> io::Result<Encoder<'_, T>> {
-        Ok(Encoder {
-            inner: poll_result_once(self.inner.create_directory(file_name.as_ref(), metadata))?,
-        })
+    ) -> io::Result<()> {
+        poll_result_once(self.inner.create_directory(file_name.as_ref(), metadata))
     }
 
-    /// Finish this directory. This is mandatory, otherwise the `Drop` handler will `panic!`.
-    pub fn finish(self) -> io::Result<()> {
+    /// Finish this directory. This is mandatory, encodes the end for the current directory.
+    pub fn finish(&mut self) -> io::Result<()> {
         poll_result_once(self.inner.finish())
     }
 
+    /// Close the encoder instance. This is mandatory, encodes the end for the optional payload
+    /// output stream, if some is given
+    pub fn close(self) -> io::Result<()> {
+        poll_result_once(self.inner.close())
+    }
+
     /// Add a symbolic link to the archive.
     pub fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
         &mut self,
diff --git a/tests/simple/fs.rs b/tests/simple/fs.rs
index 9a89c4d..4284805 100644
--- a/tests/simple/fs.rs
+++ b/tests/simple/fs.rs
@@ -144,12 +144,12 @@ impl Entry {
 
             EntryKind::Directory(entries) => {
                 self.no_hardlink()?;
-                let mut dir = encoder.create_directory(&self.name, &self.metadata)?;
+                encoder.create_directory(&self.name, &self.metadata)?;
                 let path = path.join(&self.name);
                 for entry in entries {
-                    entry.encode_into(&mut dir, hardlinks, &path)?;
+                    entry.encode_into(encoder, hardlinks, &path)?;
                 }
-                dir.finish()?;
+                encoder.finish()?;
             }
 
             EntryKind::Symlink(path) => {
diff --git a/tests/simple/main.rs b/tests/simple/main.rs
index d661c7d..e55457f 100644
--- a/tests/simple/main.rs
+++ b/tests/simple/main.rs
@@ -51,6 +51,9 @@ fn test1() {
     encoder
         .finish()
         .expect("failed to finish encoding the pxar archive");
+    encoder
+        .close()
+        .expect("failed to close the encoder instance");
 
     assert!(!file.is_empty(), "encoder did not write any data");
 
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel


  parent reply	other threads:[~2024-05-14 10:34 UTC|newest]

Thread overview: 98+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2024-05-14 10:33 [pbs-devel] [PATCH v5 pxar proxmox-backup 00/62] fix #3174: improve file-level backup Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 01/14] format/examples: add header type `PXAR_PAYLOAD_REF` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 02/14] decoder: add method to read payload references Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 03/14] decoder: factor out skip part from skip_entry Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 04/14] encoder: add optional output writer for file payloads Christian Ebner
2024-05-21 10:06   ` Dominik Csapak
2024-05-21 10:21     ` Christian Ebner
2024-05-14 10:33 ` Christian Ebner [this message]
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 06/14] decoder/accessor: add optional payload input stream Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 07/14] decoder: set payload input range when decoding via accessor Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 08/14] encoder: add payload reference capability Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 09/14] encoder: add payload position capability Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 10/14] encoder: add payload advance capability Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 11/14] encoder/format: finish payload stream with marker Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 12/14] format: add payload stream start marker Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 13/14] format/encoder/decoder: new pxar entry type `Version` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 pxar 14/14] format/encoder/decoder: new pxar entry type `Prelude` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 15/65] client: pxar: switch to stack based encoder state Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 16/65] client: backup: factor out extension from backup target Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 17/65] client: pxar: combine writers into struct Christian Ebner
2024-05-21 10:29   ` Dominik Csapak
2024-05-21 13:30     ` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 18/65] client: pxar: add optional pxar payload writer instance Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 19/65] client: pxar: optionally split metadata and payload streams Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 20/65] client: helper: add helpers for creating reader instances Christian Ebner
2024-05-21 12:26   ` Dominik Csapak
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 21/65] client: helper: add method for split archive name mapping Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 22/65] client: restore: read payload from dedicated index Christian Ebner
2024-05-21 12:44   ` Dominik Csapak
2024-05-24  6:45     ` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 23/65] tools: cover extension for split pxar archives Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 24/65] restore: " Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 25/65] client: mount: make split pxar archives mountable Christian Ebner
2024-05-21 12:54   ` Dominik Csapak
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 26/65] api: datastore: refactor getting local chunk reader Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 27/65] api: datastore: attach optional payload " Christian Ebner
2024-05-21 13:12   ` Dominik Csapak
2024-05-24  6:48     ` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 28/65] catalog: shell: make split pxar archives accessible Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 29/65] www: cover metadata extension for pxar archives Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 30/65] file restore: factor out getting pxar reader Christian Ebner
2024-05-21 13:19   ` Dominik Csapak
2024-05-21 14:07     ` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 31/65] file restore: cover split metadata and payload archives Christian Ebner
2024-05-21 13:25   ` Dominik Csapak
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 32/65] file restore: show more error context when extraction fails Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 33/65] pxar: add optional payload input for achive restore Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 34/65] pxar: add more context to extraction error Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 35/65] client: pxar: include payload offset in entry listing Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 36/65] pxar: show padding in debug output on archive list Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 37/65] datastore: dynamic index: add method to get digest Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 38/65] client: pxar: helper for lookup of reusable dynamic entries Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 39/65] upload stream: implement reused chunk injector Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 40/65] client: chunk stream: add struct to hold injection state Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 41/65] chunker: add method to reset chunker state Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 42/65] client: streams: add channels for dynamic entry injection Christian Ebner
2024-05-22  9:56   ` Dominik Csapak
2024-05-24  6:57     ` Christian Ebner
2024-05-14 10:33 ` [pbs-devel] [PATCH v6 proxmox-backup 43/65] specs: add backup detection mode specification Christian Ebner
2024-05-22 13:07   ` Dominik Csapak
2024-05-24  6:59     ` Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 44/65] client: implement prepare reference method Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 45/65] client: pxar: add method for metadata comparison Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 46/65] pxar: caching: add look-ahead cache types Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 47/65] fix #3174: client: pxar: enable caching and meta comparison Christian Ebner
2024-05-22 14:45   ` Dominik Csapak
2024-05-24  8:50     ` Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 48/65] client: backup writer: add injected chunk count to stats Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 49/65] pxar: create: keep track of reused chunks and files Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 50/65] pxar: create: show chunk injection stats debug output Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 51/65] client: pxar: add helper to handle optional preludes Christian Ebner
2024-05-23  8:47   ` Dominik Csapak
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 52/65] client: pxar: opt encode cli exclude patterns as Prelude Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 53/65] docs: file formats: describe split pxar archive file layout Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 54/65] docs: add section describing change detection mode Christian Ebner
2024-05-23  9:28   ` Dominik Csapak
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 55/65] test-suite: add detection mode change benchmark Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 56/65] test-suite: add bin to deb, add shell completions Christian Ebner
2024-05-23  9:32   ` Dominik Csapak
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 57/65] datastore: chunker: add Chunker trait Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 58/65] datastore: chunker: implement chunker for payload stream Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 59/65] client: chunk stream: switch payload stream chunker Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 60/65] client: pxar: allow to restore prelude to optional path Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 61/65] client: pxar: add archive creation with reference test Christian Ebner
2024-05-23 10:04   ` Dominik Csapak
2024-05-23 10:17     ` Christian Ebner
2024-05-23 10:17       ` Dominik Csapak
2024-05-27 11:05         ` Christian Ebner
2024-05-27 11:17           ` Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 62/65] client: tools: add helper to raise nofile rlimit Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 63/65] client: pxar: set cache limit based on " Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 64/65] chunker: tests: add regression tests for payload chunker Christian Ebner
2024-05-21 11:23   ` Dominik Csapak
2024-05-21 11:27     ` Christian Ebner
2024-05-14 10:34 ` [pbs-devel] [PATCH v6 proxmox-backup 65/65] chunk stream: " Christian Ebner
2024-05-21 11:21   ` Dominik Csapak
2024-05-14 10:45 ` [pbs-devel] [PATCH v5 pxar proxmox-backup 00/62] fix #3174: improve file-level backup Christian Ebner
2024-05-27 14:35 ` Christian Ebner

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20240514103421.289431-6-c.ebner@proxmox.com \
    --to=c.ebner@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal