[pbs-devel] [PATCH proxmox{, -backup} v5 0/5] fix #4995: include symlinks in zip file restore

[pbs-devel] [PATCH proxmox{, -backup} v5 0/5] fix #4995: include symlinks in zip file restore

[Filip Schauer]

Include symlinks when restoring files from a backup as a zip file.

Resulting ZIP files were successfully tested on Linux with: zipinfo,
unzip, unar

When extracting with Windows Explorer, symlinks appear as regular files
containing the path to the destination. When extracting the ZIP files
with 7-Zip on Windows, symlinks are correctly extracted, provided that
the user has permission to create symlinks.

proxmox:

Filip Schauer (3):
  compression: zip: add a FileType enum
  compression: zip: add support for symlinks
  compression: add tests for the ZipEncoder

 proxmox-compression/src/zip.rs   |  50 +++++++++----
 proxmox-compression/tests/zip.rs | 118 +++++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+), 15 deletions(-)
 create mode 100644 proxmox-compression/tests/zip.rs


proxmox-backup:

Filip Schauer (2):
  pxar: Adopt FileType enum when adding a zip entry
  fix #4995: pxar: Include symlinks in zip file creation

 pbs-client/src/pxar/extract.rs | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)


Summary over all repositories:
  3 files changed, 169 insertions(+), 20 deletions(-)

-- 
Generated by git-murpp 0.6.0


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox v5 1/5] compression: zip: add a FileType enum

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-compression/src/zip.rs | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/proxmox-compression/src/zip.rs b/proxmox-compression/src/zip.rs
index 11b29de4..e67613e4 100644
--- a/proxmox-compression/src/zip.rs
+++ b/proxmox-compression/src/zip.rs
@@ -17,6 +17,7 @@ use std::time::SystemTime;
 use anyhow::{format_err, Error, Result};
 use endian_trait::Endian;
 use futures::ready;
+use libc::{S_IFDIR, S_IFREG};
 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
 
 use crc32fast::Hasher;
@@ -71,6 +72,12 @@ fn epoch_to_dos(epoch: i64) -> (u16, u16) {
     (date, time)
 }
 
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub enum FileType {
+    Directory,
+    Regular,
+}
+
 #[derive(Endian)]
 #[repr(C, packed)]
 struct Zip64Field {
@@ -202,16 +209,15 @@ pub struct ZipEntry {
     uncompressed_size: u64,
     compressed_size: u64,
     offset: u64,
-    is_file: bool,
     is_utf8_filename: bool,
 }
 
 impl ZipEntry {
     /// Creates a new ZipEntry
     ///
-    /// if is_file is false the path will contain an trailing separator,
+    /// if file_type is Directory, the path will contain a trailing separator,
     /// so that the zip file understands that it is a directory
-    pub fn new<P: AsRef<Path>>(path: P, mtime: i64, mode: u16, is_file: bool) -> Self {
+    pub fn new<P: AsRef<Path>>(path: P, mtime: i64, mut mode: u16, file_type: FileType) -> Self {
         let mut relpath = PathBuf::new();
 
         for comp in path.as_ref().components() {
@@ -220,13 +226,18 @@ impl ZipEntry {
             }
         }
 
-        if !is_file {
+        if file_type == FileType::Directory {
             relpath.push(""); // adds trailing slash
         }
 
         let filename: OsString = relpath.into();
         let is_utf8_filename = filename.to_str().is_some();
 
+        mode |= match file_type {
+            FileType::Regular => S_IFREG,
+            FileType::Directory => S_IFDIR,
+        } as u16;
+
         Self {
             filename,
             crc32: 0,
@@ -235,7 +246,6 @@ impl ZipEntry {
             uncompressed_size: 0,
             compressed_size: 0,
             offset: 0,
-            is_file,
             is_utf8_filename,
         }
     }
@@ -342,6 +352,8 @@ impl ZipEntry {
             )
         };
 
+        let is_directory = (self.mode & S_IFDIR as u16) != 0;
+
         write_struct(
             &mut buf,
             CentralDirectoryFileHeader {
@@ -360,7 +372,7 @@ impl ZipEntry {
                 comment_len: 0,
                 start_disk: 0,
                 internal_flags: 0,
-                external_flags: ((self.mode as u32) << 16) | ((!self.is_file as u32) << 4),
+                external_flags: ((self.mode as u32) << 16) | ((is_directory as u32) << 4),
                 offset,
             },
         )
@@ -437,7 +449,7 @@ where
 /// use anyhow::{Error, Result};
 /// use tokio::fs::File;
 ///
-/// use proxmox_compression::zip::{ZipEncoder, ZipEntry};
+/// use proxmox_compression::zip::{FileType, ZipEncoder, ZipEntry};
 ///
 /// //#[tokio::main]
 /// async fn main_() -> Result<(), Error> {
@@ -449,7 +461,7 @@ where
 ///         "foo.txt",
 ///         0,
 ///         0o100755,
-///         true,
+///         FileType::Regular,
 ///     ), Some(source)).await?;
 ///
 ///     zip.finish().await?;
@@ -658,10 +670,10 @@ where
 
             if entry.file_type().is_file() {
                 let file = tokio::fs::File::open(entry.path()).await?;
-                let ze = ZipEntry::new(entry_path_no_base, mtime, mode, true);
+                let ze = ZipEntry::new(entry_path_no_base, mtime, mode, FileType::Regular);
                 Ok(Some((ze, Some(file))))
             } else if entry.file_type().is_dir() {
-                let ze = ZipEntry::new(entry_path_no_base, mtime, mode, false);
+                let ze = ZipEntry::new(entry_path_no_base, mtime, mode, FileType::Directory);
                 let content: Option<tokio::fs::File> = None;
                 Ok(Some((ze, content)))
             } else {
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox v5 2/5] compression: zip: add support for symlinks

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-compression/src/zip.rs | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/proxmox-compression/src/zip.rs b/proxmox-compression/src/zip.rs
index e67613e4..d7bdd296 100644
--- a/proxmox-compression/src/zip.rs
+++ b/proxmox-compression/src/zip.rs
@@ -17,7 +17,7 @@ use std::time::SystemTime;
 use anyhow::{format_err, Error, Result};
 use endian_trait::Endian;
 use futures::ready;
-use libc::{S_IFDIR, S_IFREG};
+use libc::{S_IFDIR, S_IFLNK, S_IFREG};
 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
 
 use crc32fast::Hasher;
@@ -76,6 +76,7 @@ fn epoch_to_dos(epoch: i64) -> (u16, u16) {
 pub enum FileType {
     Directory,
     Regular,
+    Symlink,
 }
 
 #[derive(Endian)]
@@ -236,6 +237,7 @@ impl ZipEntry {
         mode |= match file_type {
             FileType::Regular => S_IFREG,
             FileType::Directory => S_IFDIR,
+            FileType::Symlink => S_IFLNK,
         } as u16;
 
         Self {
@@ -656,7 +658,8 @@ where
         let encoder = &mut encoder;
 
         let entry = async move {
-            let entry_path_no_base = entry.path().strip_prefix(base_path)?;
+            let entrypath = entry.path();
+            let entry_path_no_base = entrypath.strip_prefix(base_path)?;
             let metadata = entry.metadata()?;
             let mtime = match metadata
                 .modified()
@@ -669,13 +672,18 @@ where
             let mode = metadata.mode() as u16;
 
             if entry.file_type().is_file() {
-                let file = tokio::fs::File::open(entry.path()).await?;
+                let file = Box::new(tokio::fs::File::open(entrypath).await?);
                 let ze = ZipEntry::new(entry_path_no_base, mtime, mode, FileType::Regular);
-                Ok(Some((ze, Some(file))))
+                Ok(Some((ze, Some::<Box<dyn AsyncRead + Send + Unpin>>(file))))
             } else if entry.file_type().is_dir() {
                 let ze = ZipEntry::new(entry_path_no_base, mtime, mode, FileType::Directory);
-                let content: Option<tokio::fs::File> = None;
-                Ok(Some((ze, content)))
+                Ok(Some((ze, None)))
+            } else if entry.file_type().is_symlink() {
+                let target: Box<dyn AsyncRead + Send + Unpin> = Box::new(io::Cursor::new(
+                    entrypath.read_link()?.into_os_string().into_encoded_bytes(),
+                ));
+                let ze = ZipEntry::new(entry_path_no_base, mtime, mode, FileType::Symlink);
+                Ok(Some((ze, Some(target))))
             } else {
                 // ignore other file types
                 Ok::<_, Error>(None)
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox v5 3/5] compression: add tests for the ZipEncoder

[Filip Schauer]

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 proxmox-compression/tests/zip.rs | 118 +++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 proxmox-compression/tests/zip.rs

diff --git a/proxmox-compression/tests/zip.rs b/proxmox-compression/tests/zip.rs
new file mode 100644
index 00000000..db7aa171
--- /dev/null
+++ b/proxmox-compression/tests/zip.rs
@@ -0,0 +1,118 @@
+use std::io::Cursor;
+
+use anyhow::{ensure, Result};
+use flate2::{Decompress, FlushDecompress};
+use tokio::test;
+
+use proxmox_compression::zip::{FileType, ZipEncoder, ZipEntry};
+
+fn check_zip_with_one_file(
+    zip_file: &[u8],
+    expected_file_name: &str,
+    expected_file_attributes: u16,
+    expected_content: Option<&[u8]>,
+) -> Result<()> {
+    ensure!(zip_file.starts_with(b"PK\x03\x04"));
+
+    let general_purpose_flags = &zip_file[6..8];
+    let size_compressed = &zip_file[18..22];
+    let size_uncompressed = &zip_file[22..26];
+    let file_name_len = (zip_file[26] as usize) | ((zip_file[27] as usize) << 8);
+    let extra_len = zip_file[28] as usize | ((zip_file[29] as usize) << 8);
+    let file_name = &zip_file[30..30 + file_name_len];
+    let mut offset = 30 + file_name_len;
+
+    ensure!(file_name == expected_file_name.as_bytes());
+
+    offset += extra_len;
+
+    if let Some(expected_content) = expected_content {
+        let mut decompress = Decompress::new(false);
+        let mut decompressed = Vec::with_capacity(expected_content.len());
+        decompress.decompress_vec(
+            &zip_file[offset..],
+            &mut decompressed,
+            FlushDecompress::Finish,
+        )?;
+
+        ensure!(decompressed == expected_content);
+
+        offset += decompress.total_in() as usize;
+    }
+
+    // Optional data descriptor
+    if &zip_file[offset..offset + 4] == b"PK\x07\x08" {
+        offset += 4;
+
+        if (general_purpose_flags[0] & 8) != 0 {
+            offset += 12;
+
+            if size_compressed == b"\xff\xff\xff\xff" && size_uncompressed == b"\xff\xff\xff\xff" {
+                offset += 8;
+            }
+        }
+    }
+
+    ensure!(
+        &zip_file[offset..offset + 4] == b"PK\x01\x02",
+        "Expecting a central directory file header"
+    );
+
+    let external_file_attributes = &zip_file[offset + 38..offset + 42];
+    let file_attributes = u16::from_le_bytes(external_file_attributes[2..4].try_into()?);
+
+    ensure!(file_attributes == expected_file_attributes);
+
+    Ok(())
+}
+
+#[test]
+async fn test_zip_file() -> Result<()> {
+    let mut zip_file = Vec::new();
+    let mut zip_encoder = ZipEncoder::new(&mut zip_file);
+    zip_encoder
+        .add_entry(
+            ZipEntry::new("foo", 0, 0o755, FileType::Regular),
+            Some(Cursor::new(b"bar")),
+        )
+        .await?;
+    zip_encoder.finish().await?;
+
+    check_zip_with_one_file(&zip_file, "foo", 0o100755, Some(b"bar"))?;
+
+    Ok(())
+}
+
+#[test]
+async fn test_zip_symlink() -> Result<()> {
+    let mut zip_file = Vec::new();
+    let mut zip_encoder = ZipEncoder::new(&mut zip_file);
+    zip_encoder
+        .add_entry(
+            ZipEntry::new("link", 0, 0o755, FileType::Symlink),
+            Some(Cursor::new(b"/dev/null")),
+        )
+        .await?;
+    zip_encoder.finish().await?;
+
+    check_zip_with_one_file(&zip_file, "link", 0o120755, Some(b"/dev/null"))?;
+
+    Ok(())
+}
+
+#[test]
+async fn test_zip_directory() -> Result<()> {
+    let mut zip_file = Vec::new();
+    let mut zip_encoder = ZipEncoder::new(&mut zip_file);
+    zip_encoder
+        .add_entry::<&[u8]>(
+            ZipEntry::new("directory", 0, 0o755, FileType::Directory),
+            None,
+        )
+        .await?;
+    zip_encoder.finish().await?;
+
+    check_zip_with_one_file(&zip_file, "directory/", 0o40755, None)?;
+
+    Ok(())
+}
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup v5 4/5] pxar: Adopt FileType enum when adding a zip entry

[Filip Schauer]

Use a FileType enum instead of a boolean to specify whether a ZipEntry
is a directory or a regular file.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 pbs-client/src/pxar/extract.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index c4d0085c..f8775aaf 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -26,7 +26,7 @@ use proxmox_log::{debug, error, info};
 use proxmox_sys::c_result;
 use proxmox_sys::fs::{create_path, CreateOptions};
 
-use proxmox_compression::zip::{ZipEncoder, ZipEntry};
+use proxmox_compression::zip::{FileType, ZipEncoder, ZipEntry};
 
 use crate::pxar::dir_stack::PxarDirStack;
 use crate::pxar::metadata;
@@ -1034,7 +1034,7 @@ where
                 path,
                 metadata.stat.mtime.secs,
                 metadata.stat.mode as u16,
-                false,
+                FileType::Directory,
             );
             zip.add_entry::<FileContents<T>>(entry, None).await?;
         }
@@ -1053,7 +1053,7 @@ where
                         path,
                         metadata.stat.mtime.secs,
                         metadata.stat.mode as u16,
-                        true,
+                        FileType::Regular,
                     );
                     let contents = decoder.contents().await?;
                     zip.add_entry(entry, contents)
@@ -1072,7 +1072,7 @@ where
                         path,
                         metadata.stat.mtime.secs,
                         metadata.stat.mode as u16,
-                        true,
+                        FileType::Regular,
                     );
                     let contents = decoder.contents().await?;
                     zip.add_entry(entry, contents)
@@ -1085,7 +1085,7 @@ where
                         path,
                         metadata.stat.mtime.secs,
                         metadata.stat.mode as u16,
-                        false,
+                        FileType::Directory,
                     );
                     zip.add_entry::<FileContents<T>>(entry, None).await?;
                 }
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup v5 5/5] fix #4995: pxar: Include symlinks in zip file creation

[Filip Schauer]

Include symlinks when restoring a backup as a zip file.

Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
---
 pbs-client/src/pxar/extract.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index f8775aaf..371f0235 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -1089,6 +1089,17 @@ where
                     );
                     zip.add_entry::<FileContents<T>>(entry, None).await?;
                 }
+                EntryKind::Symlink(target) => {
+                    debug!("adding '{}' to zip", path.display());
+                    let entry = ZipEntry::new(
+                        path,
+                        metadata.stat.mtime.secs,
+                        metadata.stat.mode as u16,
+                        FileType::Symlink,
+                    );
+                    let target = io::Cursor::<&[u8]>::new(target.as_ref());
+                    zip.add_entry(entry, Some(target)).await?;
+                }
                 _ => {} // ignore all else
             };
         }
-- 
2.47.3



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel