[pbs-devel] [PATCH pxar 1/2] decoder: move payload header check for split input

[Christian Ebner <c.ebner@proxmox.com>]

The payload entries in the payload output for split pxar archives are
separated by payload headers, which allow to perform consistency
checks for the payload references encoded in the metadata archive.

Currently, this consistency check is performed right after reading the
entry in the metadata archive, which however has the downside that the
payload has to be fetched and decoded just for this consistency check.
This greatly impacts performance when accessing a metadata archive
with attached payload input reader, e.g. in the fuse implementation to
mount pxar archives, being especially severe when accessed over the
network in combination with a remote chunk reader as the Proxmox
Backup Server does.

Therefore, move this check to the contents reader instantiation
instead and add an additional flag to the decoder's `InPayload` state.

Getting the decoder now needs to be async and the method must return
an error when the check fails.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/decoder/aio.rs  |  4 +--
 src/decoder/mod.rs  | 60 ++++++++++++++++++++++++++-------------------
 src/decoder/sync.rs |  5 ++--
 3 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/src/decoder/aio.rs b/src/decoder/aio.rs
index 3f9881d..19e7023 100644
--- a/src/decoder/aio.rs
+++ b/src/decoder/aio.rs
@@ -60,8 +60,8 @@ impl<T: SeqRead> Decoder<T> {
     }
 
     /// Get a reader for the contents of the current entry, if the entry has contents.
-    pub fn contents(&mut self) -> Option<Contents<T>> {
-        self.inner.content_reader()
+    pub async fn contents(&mut self) -> io::Result<Option<Contents<T>>> {
+        self.inner.content_reader().await
     }
 
     /// Get the size of the current contents, if the entry has contents.
diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs
index 46a21b8..8f92964 100644
--- a/src/decoder/mod.rs
+++ b/src/decoder/mod.rs
@@ -182,6 +182,7 @@ enum State {
     InPayload {
         offset: u64,
         size: u64,
+        header_checked: bool,
     },
 
     /// file entries with no data (fifo, socket)
@@ -296,8 +297,12 @@ impl<I: SeqRead> DecoderImpl<I> {
                     // hierarchy and parse the next PXAR_FILENAME or the PXAR_GOODBYE:
                     self.read_next_item().await?;
                 }
-                State::InPayload { offset, .. } => {
+                State::InPayload { offset, header_checked, .. } => {
                     if self.input.payload().is_some() {
+                        if !header_checked {
+                            // header is only checked if payload has been accessed
+                            self.payload_consumed += size_of::<Header>() as u64;
+                        }
                         // Update consumed payload as given by the offset referenced by the content reader
                         self.payload_consumed += offset;
                     } else {
@@ -370,19 +375,39 @@ impl<I: SeqRead> DecoderImpl<I> {
         }
     }
 
-    pub fn content_reader(&mut self) -> Option<Contents<I>> {
-        if let State::InPayload { offset, size } = &mut self.state {
-            if self.input.payload().is_some() {
-                Some(Contents::new(
+    pub async fn content_reader(&mut self) -> Result<Option<Contents<I>>, io::Error> {
+        if let State::InPayload { offset, size, header_checked } = &mut self.state {
+            if let Some(payload_input) = self.input.payload_mut() {
+                if !*header_checked {
+                    let header: Header = seq_read_entry(payload_input).await?;
+                    if header.htype != format::PXAR_PAYLOAD {
+                        io_bail!(
+                            "unexpected header in payload input: expected {} , got {header}",
+                            format::PXAR_PAYLOAD,
+                        );
+                    }
+                    self.payload_consumed += size_of::<Header>() as u64;
+
+                    if header.content_size() != *size {
+                        io_bail!(
+                            "encountered payload size mismatch: got {size}, expected {}",
+                            header.content_size(),
+                        );
+                    }
+
+                    *header_checked = true;
+                }
+
+                Ok(Some(Contents::new(
                     self.input.payload_mut().unwrap(),
                     offset,
                     *size,
-                ))
+                )))
             } else {
-                Some(Contents::new(self.input.archive_mut(), offset, *size))
+                Ok(Some(Contents::new(self.input.archive_mut(), offset, *size)))
             }
         } else {
-            None
+            Ok(None)
         }
     }
 
@@ -621,6 +646,7 @@ impl<I: SeqRead> DecoderImpl<I> {
                 };
                 self.state = State::InPayload {
                     offset: 0,
+                    header_checked: false,
                     size: self.current_header.content_size(),
                 };
                 return Ok(ItemResult::Entry);
@@ -652,23 +678,6 @@ impl<I: SeqRead> DecoderImpl<I> {
                         let end = start + payload_ref.size + size_of::<Header>() as u64;
                         payload_input.update_range(start..end);
                     }
-
-                    let header: Header = seq_read_entry(payload_input).await?;
-                    if header.htype != format::PXAR_PAYLOAD {
-                        io_bail!(
-                            "unexpected header in payload input: expected {} , got {header}",
-                            format::PXAR_PAYLOAD,
-                        );
-                    }
-                    self.payload_consumed += size_of::<Header>() as u64;
-
-                    if header.content_size() != payload_ref.size {
-                        io_bail!(
-                            "encountered payload size mismatch: got {}, expected {}",
-                            payload_ref.size,
-                            header.content_size(),
-                        );
-                    }
                 }
 
                 self.entry.kind = EntryKind::File {
@@ -678,6 +687,7 @@ impl<I: SeqRead> DecoderImpl<I> {
                 };
                 self.state = State::InPayload {
                     offset: 0,
+                    header_checked: false,
                     size: payload_ref.size,
                 };
                 return Ok(ItemResult::Entry);
diff --git a/src/decoder/sync.rs b/src/decoder/sync.rs
index 8779f87..1116fe8 100644
--- a/src/decoder/sync.rs
+++ b/src/decoder/sync.rs
@@ -77,8 +77,9 @@ impl<T: SeqRead> Decoder<T> {
     }
 
     /// Get a reader for the contents of the current entry, if the entry has contents.
-    pub fn contents(&mut self) -> Option<Contents<T>> {
-        self.inner.content_reader().map(|inner| Contents { inner })
+    pub fn contents(&mut self) -> io::Result<Option<Contents<T>>> {
+        let content_reader = poll_result_once(self.inner.content_reader())?;
+        Ok(content_reader.map(|inner| Contents { inner }))
     }
 
     /// Get the size of the current contents, if the entry has contents.
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel