Re: [pbs-devel] [RFC v2 pxar 07/36] encoder: add payload reference capability

["Fabian Grünbichler" <f.gruenbichler@proxmox.com>]

On March 5, 2024 10:26 am, Christian Ebner wrote:
> Allows to encode regular files with a payload reference within a
> separate payload archive rather than encoding the payload within the
> regular archive.
> 
> Following the PXAR_PAYLOAD_REF marked header, the payload offset and
> size are encoded.
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
> changes since version 1:
> - get rid of unused sub method impl for PayloadOffset
> 
>  src/encoder/aio.rs  | 15 ++++++++++++++-
>  src/encoder/mod.rs  | 43 +++++++++++++++++++++++++++++++++++++++++++
>  src/encoder/sync.rs | 18 +++++++++++++++++-
>  3 files changed, 74 insertions(+), 2 deletions(-)
> 
> diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
> index 60b11cd..87d75ef 100644
> --- a/src/encoder/aio.rs
> +++ b/src/encoder/aio.rs
> @@ -5,7 +5,7 @@ use std::path::Path;
>  use std::pin::Pin;
>  use std::task::{Context, Poll};
>  
> -use crate::encoder::{self, LinkOffset, SeqWrite};
> +use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
>  use crate::format;
>  use crate::Metadata;
>  
> @@ -98,6 +98,19 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
>      //         content.as_async_reader(),
>      //     ).await
>      // }
> +    //
> +    /// Encode a payload reference, returning the payload offset within the payload stream
> +    pub async fn add_payload_ref(
> +        &mut self,
> +        metadata: &Metadata,
> +        file_name: &Path,
> +        file_size: u64,
> +        payload_offset: PayloadOffset,
> +    ) -> io::Result<()> {
> +        self.inner
> +            .add_payload_ref(metadata, file_name.as_ref(), file_size, payload_offset)
> +            .await
> +    }
>  
>      /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
>      /// `finish()` method, otherwise the entire archive will be in an error state.
> diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
> index 962087a..1baa221 100644
> --- a/src/encoder/mod.rs
> +++ b/src/encoder/mod.rs
> @@ -38,6 +38,24 @@ impl LinkOffset {
>      }
>  }
>  
> +/// File reference used to create payload references.
> +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd)]
> +pub struct PayloadOffset(u64);
> +
> +impl PayloadOffset {
> +    /// Get the raw byte offset of this link.
> +    #[inline]
> +    pub fn raw(self) -> u64 {
> +        self.0
> +    }
> +
> +    /// Return a new PayloadOffset, positively shifted by offset
> +    #[inline]
> +    pub fn add(&self, offset: u64) -> Self {
> +        Self(self.0 + offset)
> +    }
> +}
> +
>  /// Sequential write interface used by the encoder's state machine.
>  ///
>  /// This is our internal writer trait which is available for `std::io::Write` types in the
> @@ -472,6 +490,31 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
>          Ok(offset)
>      }
>  
> +    /// Encode a payload reference, returning the payload offset within the payload stream

this doesn't return the offset though?

> +    pub async fn add_payload_ref(
> +        &mut self,
> +        metadata: &Metadata,
> +        file_name: &Path,
> +        file_size: u64,
> +        payload_offset: PayloadOffset,
> +    ) -> io::Result<()> {
> +        if self.payload_output.as_mut().is_none() {
> +            io_bail!("unable to add payload reference");
> +        }
> +
> +        let mut payload_ref = payload_offset.raw().to_le_bytes().to_vec();
> +        payload_ref.append(&mut file_size.to_le_bytes().to_vec());

another copy of manually encoding a PayloadRef (see patch #2)

> +        let _this_offset: LinkOffset = self
> +            .add_file_entry(
> +                Some(metadata),
> +                file_name,
> +                Some((format::PXAR_PAYLOAD_REF, &payload_ref)),
> +            )
> +            .await?;
> +
> +        Ok(())
> +    }
> +
>      /// Return a file offset usable with `add_hardlink`.
>      pub async fn add_symlink(
>          &mut self,
> diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
> index a7c350e..90af8f9 100644
> --- a/src/encoder/sync.rs
> +++ b/src/encoder/sync.rs
> @@ -6,7 +6,7 @@ use std::pin::Pin;
>  use std::task::{Context, Poll};
>  
>  use crate::decoder::sync::StandardReader;
> -use crate::encoder::{self, LinkOffset, SeqWrite};
> +use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
>  use crate::format;
>  use crate::util::poll_result_once;
>  use crate::Metadata;
> @@ -100,6 +100,22 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
>          ))
>      }
>  
> +    /// Encode a payload reference, returning the payload offset within the payload stream
> +    pub async fn add_payload_ref(
> +        &mut self,
> +        metadata: &Metadata,
> +        file_name: &Path,
> +        file_size: u64,
> +        payload_offset: PayloadOffset,
> +    ) -> io::Result<()> {
> +        poll_result_once(self.inner.add_payload_ref(
> +            metadata,
> +            file_name.as_ref(),
> +            file_size,
> +            payload_offset,
> +        ))
> +    }
> +
>      /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
>      /// `finish()` method, otherwise the entire archive will be in an error state.
>      pub fn create_directory<P: AsRef<Path>>(
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 
> 
>