[pbs-devel] [PATCH v3 pxar 08/58] encoder: add payload reference capability

[Christian Ebner <c.ebner@proxmox.com>]

Allows to encode regular files with a payload reference within a
separate payload archive rather than encoding the payload within the
regular archive.

Following the PXAR_PAYLOAD_REF marked header, the payload offset and
size are encoded.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- Add additional consistency check for current payload position
- Use PayloadRefs `data` for encoding
- Fixed incorrect comments

 src/encoder/aio.rs  | 18 ++++++++++++++-
 src/encoder/mod.rs  | 54 +++++++++++++++++++++++++++++++++++++++++++++
 src/encoder/sync.rs | 21 +++++++++++++++++-
 3 files changed, 91 insertions(+), 2 deletions(-)

diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
index 635e550..17ade7d 100644
--- a/src/encoder/aio.rs
+++ b/src/encoder/aio.rs
@@ -5,7 +5,7 @@ use std::path::Path;
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-use crate::encoder::{self, LinkOffset, SeqWrite};
+use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
 use crate::format;
 use crate::Metadata;
 
@@ -103,6 +103,22 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
     //     ).await
     // }
 
+    /// Encode a payload reference pointing to given offset in the separate payload output
+    ///
+    /// Returns with error if the encoder instance has no separate payload output or encoding
+    /// failed.
+    pub async fn add_payload_ref(
+        &mut self,
+        metadata: &Metadata,
+        file_name: &Path,
+        file_size: u64,
+        payload_offset: PayloadOffset,
+    ) -> io::Result<()> {
+        self.inner
+            .add_payload_ref(metadata, file_name.as_ref(), file_size, payload_offset)
+            .await
+    }
+
     /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
     /// `finish()` method, otherwise the entire archive will be in an error state.
     pub async fn create_directory<P: AsRef<Path>>(
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index 31bb0fa..fcd636d 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -38,6 +38,24 @@ impl LinkOffset {
     }
 }
 
+/// File reference used to create payload references.
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Ord, PartialOrd)]
+pub struct PayloadOffset(u64);
+
+impl PayloadOffset {
+    /// Get the raw byte offset of this link.
+    #[inline]
+    pub fn raw(self) -> u64 {
+        self.0
+    }
+
+    /// Return a new PayloadOffset, positively shifted by offset
+    #[inline]
+    pub fn add(&self, offset: u64) -> Self {
+        Self(self.0 + offset)
+    }
+}
+
 /// Sequential write interface used by the encoder's state machine.
 ///
 /// This is our internal writer trait which is available for `std::io::Write` types in the
@@ -485,6 +503,42 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         Ok(offset)
     }
 
+    /// Encode a payload reference pointing to given offset in the separate payload output
+    ///
+    /// Returns with error if the encoder instance has no separate payload output or encoding
+    /// failed.
+    pub async fn add_payload_ref(
+        &mut self,
+        metadata: &Metadata,
+        file_name: &Path,
+        file_size: u64,
+        payload_offset: PayloadOffset,
+    ) -> io::Result<()> {
+        if self.payload_output.as_mut().is_none() {
+            io_bail!("unable to add payload reference");
+        }
+
+        let offset = payload_offset.raw();
+        let payload_position = self.state()?.payload_position();
+        if offset < payload_position {
+            io_bail!("offset smaller than current position: {offset} < {payload_position}");
+        }
+
+        let payload_ref = PayloadRef {
+            offset,
+            size: file_size,
+        };
+        let _this_offset: LinkOffset = self
+            .add_file_entry(
+                Some(metadata),
+                file_name,
+                Some((format::PXAR_PAYLOAD_REF, &payload_ref.data())),
+            )
+            .await?;
+
+        Ok(())
+    }
+
     /// Return a file offset usable with `add_hardlink`.
     pub async fn add_symlink(
         &mut self,
diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
index d0d62ba..48942b9 100644
--- a/src/encoder/sync.rs
+++ b/src/encoder/sync.rs
@@ -6,7 +6,7 @@ use std::pin::Pin;
 use std::task::{Context, Poll};
 
 use crate::decoder::sync::StandardReader;
-use crate::encoder::{self, LinkOffset, SeqWrite};
+use crate::encoder::{self, LinkOffset, PayloadOffset, SeqWrite};
 use crate::format;
 use crate::util::poll_result_once;
 use crate::Metadata;
@@ -100,6 +100,25 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
         ))
     }
 
+    /// Encode a payload reference pointing to given offset in the separate payload output
+    ///
+    /// Returns with error if the encoder instance has no separate payload output or encoding
+    /// failed.
+    pub async fn add_payload_ref(
+        &mut self,
+        metadata: &Metadata,
+        file_name: &Path,
+        file_size: u64,
+        payload_offset: PayloadOffset,
+    ) -> io::Result<()> {
+        poll_result_once(self.inner.add_payload_ref(
+            metadata,
+            file_name.as_ref(),
+            file_size,
+            payload_offset,
+        ))
+    }
+
     /// Create a new subdirectory. Note that the subdirectory has to be finished by calling the
     /// `finish()` method, otherwise the entire archive will be in an error state.
     pub fn create_directory<P: AsRef<Path>>(
-- 
2.39.2