[pbs-devel] [PATCH proxmox-backup 1/2] pbs-client: extract: rewrite create_zip with sequential decoder

[pbs-devel] [PATCH proxmox-backup 1/2] pbs-client: extract: rewrite create_zip with sequential decoder

[Dominik Csapak]

instead of an async recursive function. Not only is it less code,
recursive futures are not really nice and it should be faster too.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 pbs-client/src/pxar/extract.rs | 154 ++++++++++++++++-----------------
 1 file changed, 74 insertions(+), 80 deletions(-)

diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index 7898e255..90ce88bd 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -7,11 +7,9 @@ use std::io;
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
 use std::path::{Path, PathBuf};
-use std::pin::Pin;
 use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, format_err, Error};
-use futures::future::Future;
 use nix::dir::Dir;
 use nix::fcntl::OFlag;
 use nix::sys::stat::Mode;
@@ -699,7 +697,7 @@ where
 
 pub async fn create_zip<T, W, P>(
     output: W,
-    decoder: Accessor<T>,
+    accessor: Accessor<T>,
     path: P,
     verbose: bool,
 ) -> Result<(), Error>
@@ -708,7 +706,7 @@ where
     W: tokio::io::AsyncWrite + Unpin + Send + 'static,
     P: AsRef<Path>,
 {
-    let root = decoder.open_root().await?;
+    let root = accessor.open_root().await?;
     let file = root
         .lookup(&path)
         .await?
@@ -720,88 +718,84 @@ where
         components.as_path().to_owned()
     };
 
-    let mut zipencoder = ZipEncoder::new(output);
-    let mut decoder = decoder;
-    recurse_files_zip(&mut zipencoder, &mut decoder, &prefix, file, verbose)
-        .await
-        .map_err(|err| {
-            eprintln!("error during creating of zip: {}", err);
-            err
-        })?;
+    let mut zip = ZipEncoder::new(output);
 
-    zipencoder.finish().await.map_err(|err| {
-        eprintln!("error during finishing of zip: {}", err);
-        err
-    })
-}
+    if let Ok(dir) = file.enter_directory().await {
+        let entry = dir.lookup_self().await?;
+        let path = entry.path().strip_prefix(&prefix)?;
+        if path != Path::new("/") {
+            let metadata = entry.metadata();
+            let entry = ZipEntry::new(
+                path,
+                metadata.stat.mtime.secs,
+                metadata.stat.mode as u16,
+                false,
+            );
+            zip.add_entry::<FileContents<T>>(entry, None).await?;
+        }
 
-fn recurse_files_zip<'a, T, W>(
-    zip: &'a mut ZipEncoder<W>,
-    decoder: &'a mut Accessor<T>,
-    prefix: &'a Path,
-    file: FileEntry<T>,
-    verbose: bool,
-) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>>
-where
-    T: Clone + pxar::accessor::ReadAt + Unpin + Send + Sync + 'static,
-    W: tokio::io::AsyncWrite + Unpin + Send + 'static,
-{
-    Box::pin(async move {
-        let metadata = file.entry().metadata();
-        let path = file.entry().path().strip_prefix(&prefix)?;
-
-        match file.kind() {
-            EntryKind::File { .. } => {
-                if verbose {
-                    eprintln!("adding '{}' to zip", path.display());
-                }
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    true,
-                );
-                zip.add_entry(entry, Some(file.contents().await?))
-                    .await
-                    .map_err(|err| format_err!("could not send file entry: {}", err))?;
-            }
-            EntryKind::Hardlink(_) => {
-                let realfile = decoder.follow_hardlink(&file).await?;
-                if verbose {
-                    eprintln!("adding '{}' to zip", path.display());
+        let mut decoder = dir.decode_full().await?;
+        decoder.enable_goodbye_entries(false);
+        while let Some(entry) = decoder.next().await {
+            let entry = entry?;
+            let metadata = entry.metadata();
+            let path = entry.path().strip_prefix(&prefix)?;
+
+            match entry.kind() {
+                EntryKind::File { .. } => {
+                    if verbose {
+                        eprintln!("adding '{}' to zip", path.display());
+                    }
+                    let entry = ZipEntry::new(
+                        path,
+                        metadata.stat.mtime.secs,
+                        metadata.stat.mode as u16,
+                        true,
+                    );
+                    zip.add_entry(entry, decoder.contents())
+                        .await
+                        .map_err(|err| format_err!("could not send file entry: {}", err))?;
                 }
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    true,
-                );
-                zip.add_entry(entry, Some(realfile.contents().await?))
-                    .await
-                    .map_err(|err| format_err!("could not send file entry: {}", err))?;
-            }
-            EntryKind::Directory => {
-                let dir = file.enter_directory().await?;
-                let mut readdir = dir.read_dir();
-                if verbose {
-                    eprintln!("adding '{}' to zip", path.display());
+                EntryKind::Hardlink(_) => {
+                    let entry = root
+                        .lookup(&path)
+                        .await?
+                        .ok_or(format_err!("error looking up '{:?}'", path))?;
+                    let realfile = accessor.follow_hardlink(&entry).await?;
+                    let metadata = realfile.entry().metadata();
+                    if verbose {
+                        eprintln!("adding '{}' to zip", path.display());
+                    }
+                    let entry = ZipEntry::new(
+                        path,
+                        metadata.stat.mtime.secs,
+                        metadata.stat.mode as u16,
+                        true,
+                    );
+                    zip.add_entry(entry, decoder.contents())
+                        .await
+                        .map_err(|err| format_err!("could not send file entry: {}", err))?;
                 }
-                let entry = ZipEntry::new(
-                    path,
-                    metadata.stat.mtime.secs,
-                    metadata.stat.mode as u16,
-                    false,
-                );
-                zip.add_entry::<FileContents<T>>(entry, None).await?;
-                while let Some(entry) = readdir.next().await {
-                    let entry = entry?.decode_entry().await?;
-                    recurse_files_zip(zip, decoder, prefix, entry, verbose).await?;
+                EntryKind::Directory => {
+                    if verbose {
+                        eprintln!("adding '{}' to zip", path.display());
+                    }
+                    let entry = ZipEntry::new(
+                        path,
+                        metadata.stat.mtime.secs,
+                        metadata.stat.mode as u16,
+                        false,
+                    );
+                    zip.add_entry::<FileContents<T>>(entry, None).await?;
                 }
-            }
-            _ => {} // ignore all else
-        };
+                _ => {} // ignore all else
+            };
+        }
+    }
 
-        Ok(())
+    zip.finish().await.map_err(|err| {
+        eprintln!("error during finishing of zip: {}", err);
+        err
     })
 }
 
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 2/2] pbs-client: extract: add top-level dir in tar.zst

[Dominik Csapak]

when download a folder, include that folder as first entry (except '/')

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 pbs-client/src/pxar/extract.rs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index 90ce88bd..f18fc7d4 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -557,6 +557,22 @@ where
     let mut hardlinks: HashMap<PathBuf, PathBuf> = HashMap::new();
 
     if let Ok(dir) = file.enter_directory().await {
+        let entry = dir.lookup_self().await?;
+        let path = entry.path().strip_prefix(prefix)?;
+
+        if path != Path::new("/") {
+            let metadata = entry.metadata();
+            let mut header = tar::Header::new_gnu();
+            header.set_entry_type(tar::EntryType::Directory);
+            add_metadata_to_header(&mut header, metadata);
+            header.set_size(0);
+            header.set_cksum();
+            tarencoder
+                .add_entry(&mut header, path, tokio::io::empty())
+                .await
+                .map_err(|err| format_err!("could not send dir entry: {}", err))?;
+        }
+
         let mut decoder = dir.decode_full().await?;
         decoder.enable_goodbye_entries(false);
         while let Some(entry) = decoder.next().await {
-- 
2.30.2

[pbs-devel] applied: [PATCH proxmox-backup 1/2] pbs-client: extract: rewrite create_zip with sequential decoder

[Wolfgang Bumiller]

applied both patches