[pbs-devel] [RFC pxar 6/20] fix #3174: enc/dec: impl PXAR_APPENDIX entrytype

[Christian Ebner <c.ebner@proxmox.com>]

Add an additional entry type for marking the start of a pxar archive
appendix section. The appendix is a concatenation of possibly
uncorrelated chunks, therefore not following the pxar archive format
anymore. The appendix is only used to access the file metadata and
payloads when a PXAR_APPENDIX_REF entry is encountered in the archive
before this point.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 examples/mk-format-hashes.rs |  1 +
 src/decoder/mod.rs           |  9 +++++++++
 src/encoder/aio.rs           |  7 +++++++
 src/encoder/mod.rs           | 20 ++++++++++++++++++++
 src/encoder/sync.rs          |  7 +++++++
 src/format/mod.rs            |  7 +++++++
 src/lib.rs                   |  4 ++++
 7 files changed, 55 insertions(+)

diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs
index 8b4f5de..f068edd 100644
--- a/examples/mk-format-hashes.rs
+++ b/examples/mk-format-hashes.rs
@@ -12,6 +12,7 @@ const CONSTANTS: &[(&str, &str, &str)] = &[
         "__PROXMOX_FORMAT_ENTRY__",
     ),
     ("", "PXAR_FILENAME", "__PROXMOX_FORMAT_FILENAME__"),
+    ("", "PXAR_APPENDIX", "__PROXMOX_FORMAT_APPENDIX__"),
     ("", "PXAR_SYMLINK", "__PROXMOX_FORMAT_SYMLINK__"),
     ("", "PXAR_DEVICE", "__PROXMOX_FORMAT_DEVICE__"),
     ("", "PXAR_XATTR", "__PROXMOX_FORMAT_XATTR__"),
diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs
index 143a01d..e54ab41 100644
--- a/src/decoder/mod.rs
+++ b/src/decoder/mod.rs
@@ -291,6 +291,7 @@ impl<I: SeqRead> DecoderImpl<I> {
                         continue;
                     }
                 }
+                format::PXAR_APPENDIX => return Ok(Some(self.entry.take())),
                 _ => io_bail!(
                     "expected filename or directory-goodbye pxar entry, got: {}",
                     self.current_header,
@@ -536,6 +537,14 @@ impl<I: SeqRead> DecoderImpl<I> {
                 };
                 return Ok(ItemResult::Entry);
             }
+            format::PXAR_APPENDIX => {
+                let bytes = self.read_entry_as_bytes().await?;
+                let total = u64::from_le_bytes(bytes[0..8].try_into().unwrap());
+                self.entry.kind = EntryKind::Appendix {
+                    total,
+                };
+                return Ok(ItemResult::Entry);
+            }
             format::PXAR_PAYLOAD => {
                 let offset = seq_read_position(&mut self.input).await.transpose()?;
                 self.entry.kind = EntryKind::File {
diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs
index 1d8e635..3b6c900 100644
--- a/src/encoder/aio.rs
+++ b/src/encoder/aio.rs
@@ -130,6 +130,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
             .await
     }
 
+    /// Add the appendix start entry marker
+    ///
+    /// Returns the LinkOffset pointing after the entry, the appendix start offset
+    pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<LinkOffset> {
+        self.inner.add_appendix(full_size).await
+    }
+
     /// Add a symbolic link to the archive.
     pub async fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
         &mut self,
diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs
index ddb0125..a513ce6 100644
--- a/src/encoder/mod.rs
+++ b/src/encoder/mod.rs
@@ -473,6 +473,26 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> {
         Ok(())
     }
 
+    /// Add the appendix start entry marker
+    ///
+    /// Returns the LinkOffset pointing after the entry, the appendix start offset
+    pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<LinkOffset> {
+        self.check()?;
+
+        let data = &full_size.to_le_bytes().to_vec();
+        seq_write_pxar_entry(
+            self.output.as_mut(),
+            format::PXAR_APPENDIX,
+            &data,
+            &mut self.state.write_position,
+        )
+        .await?;
+
+        let offset = self.position();
+
+        Ok(LinkOffset(offset))
+    }
+
     /// Return a file offset usable with `add_hardlink`.
     pub async fn add_symlink(
         &mut self,
diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs
index 6cac7eb..372ca12 100644
--- a/src/encoder/sync.rs
+++ b/src/encoder/sync.rs
@@ -126,6 +126,13 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> {
         ))
     }
 
+    /// Add the appendix start entry marker
+    ///
+    /// Returns the LinkOffset pointing after the entry, the appendix start offset
+    pub async fn add_appendix(&mut self, full_size: u64) -> io::Result<LinkOffset> {
+        poll_result_once(self.inner.add_appendix(full_size))
+    }
+
     /// Add a symbolic link to the archive.
     pub fn add_symlink<PF: AsRef<Path>, PT: AsRef<Path>>(
         &mut self,
diff --git a/src/format/mod.rs b/src/format/mod.rs
index 5eb7562..8254df9 100644
--- a/src/format/mod.rs
+++ b/src/format/mod.rs
@@ -35,6 +35,12 @@
 //!   * `<archive>`         -- serialization of the second directory entry
 //!   * ...
 //!   * `GOODBYE`           -- lookup table at the end of a list of directory entries
+//!
+//! For backups referencing previous backups to skip file payloads, the archive is followed by a
+//! appendix maker after which the concatinated pxar archive fragments containing the file payloads
+//! are appended. They are NOT guaranteed to follow the full pxar structure and should only be
+//! used to extract the file payloads by given offset.
+//!   * `APPENDIX`          -- pxar archive fragments containing file payloads
 
 use std::cmp::Ordering;
 use std::ffi::{CStr, OsStr};
@@ -85,6 +91,7 @@ pub const PXAR_ENTRY: u64 = 0xd5956474e588acef;
 /// Previous version of the entry struct
 pub const PXAR_ENTRY_V1: u64 = 0x11da850a1c1cceff;
 pub const PXAR_FILENAME: u64 = 0x16701121063917b3;
+pub const PXAR_APPENDIX: u64 = 0x9ff6c9507864b38d;
 pub const PXAR_SYMLINK: u64 = 0x27f971e7dbf5dc5f;
 pub const PXAR_DEVICE: u64 = 0x9fc9e906586d5ce9;
 pub const PXAR_XATTR: u64 = 0x0dab0229b57dcd03;
diff --git a/src/lib.rs b/src/lib.rs
index 6a30fc3..086c7f2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -416,6 +416,10 @@ pub enum EntryKind {
         file_size: u64,
     },
 
+    Appendix {
+        total: u64,
+    },
+
     /// Directory entry. When iterating through an archive, the contents follow next.
     Directory,
 
-- 
2.39.2