[pbs-devel] [PATCH proxmox-backup v2 0/3] pbs-client: feature #3923

[pbs-devel] [PATCH proxmox-backup v2 0/3] pbs-client: feature #3923

[Markus Frank]

These patches are for ignore-acl, ignore-xattr, ignore-ownership,
ignore-permission & overwrite-existing-files parameter in
proxmox-backup-client.

Markus Frank (3):
  pbs-client: added overwrite-existing-files to PxarExtractOptions.
  pbs-client: added options to skip xattr/acl/ownership/permissions
  proxmox-backup-client: added ignore-acl/xattr/ownership/permission &
    overwrite parameters

 pbs-client/src/catalog_shell.rs   |  4 +--
 pbs-client/src/pxar/extract.rs    | 28 ++++++++++++++++--
 pbs-client/src/pxar/flags.rs      |  6 ++++
 pbs-client/src/pxar/metadata.rs   | 40 ++++++++++++++++---------
 proxmox-backup-client/src/main.rs | 49 ++++++++++++++++++++++++++++++-
 pxar-bin/src/main.rs              |  7 +++++
 6 files changed, 114 insertions(+), 20 deletions(-)

-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup v2 1/3] pbs-client: added overwrite-existing-files to PxarExtractOptions.

[Markus Frank]

If overwrite-existing-files is true, O_TRUNC is set (to clean the
leftovers) instead of O_EXCL and therefore overwrites the files and
does not error out.

Signed-off-by: Markus Frank <m.frank@proxmox.com>
---
v2: add O_TRUNC to oflags if overwrite_existing_files is true.

 pbs-client/src/catalog_shell.rs |  4 ++--
 pbs-client/src/pxar/extract.rs  | 28 +++++++++++++++++++++++++---
 pxar-bin/src/main.rs            |  7 +++++++
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs
index b11901ed..25e27b37 100644
--- a/pbs-client/src/catalog_shell.rs
+++ b/pbs-client/src/catalog_shell.rs
@@ -984,7 +984,7 @@ impl Shell {
             .clone();
 
         let extractor =
-            crate::pxar::extract::Extractor::new(rootdir, root_meta, true, Flags::DEFAULT);
+            crate::pxar::extract::Extractor::new(rootdir, root_meta, true, false, Flags::DEFAULT);
 
         let mut extractor = ExtractorState::new(
             &mut self.catalog,
@@ -1172,7 +1172,7 @@ impl<'a> ExtractorState<'a> {
                 let file_name = CString::new(entry.file_name().as_bytes())?;
                 let mut contents = entry.contents().await?;
                 self.extractor
-                    .async_extract_file(&file_name, entry.metadata(), *size, &mut contents)
+                    .async_extract_file(&file_name, entry.metadata(), *size, &mut contents, false)
                     .await
             }
             _ => {
diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index 161d2cef..36600789 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -34,6 +34,7 @@ pub struct PxarExtractOptions<'a> {
     pub match_list: &'a [MatchEntry],
     pub extract_match_default: bool,
     pub allow_existing_dirs: bool,
+    pub overwrite_existing_files: bool,
     pub on_error: Option<ErrorHandler>,
 }
 
@@ -80,6 +81,7 @@ where
         dir,
         root.metadata().clone(),
         options.allow_existing_dirs,
+        options.overwrite_existing_files,
         feature_flags,
     );
 
@@ -198,6 +200,7 @@ where
                 &mut decoder.contents().ok_or_else(|| {
                     format_err!("found regular file entry without contents in archive")
                 })?,
+                extractor.overwrite_existing_files,
             ),
             (false, _) => Ok(()), // skip this
         }
@@ -215,6 +218,7 @@ where
 pub struct Extractor {
     feature_flags: Flags,
     allow_existing_dirs: bool,
+    overwrite_existing_files: bool,
     dir_stack: PxarDirStack,
 
     /// For better error output we need to track the current path in the Extractor state.
@@ -231,11 +235,13 @@ impl Extractor {
         root_dir: Dir,
         metadata: Metadata,
         allow_existing_dirs: bool,
+        overwrite_existing_files: bool,
         feature_flags: Flags,
     ) -> Self {
         Self {
             dir_stack: PxarDirStack::new(root_dir, metadata),
             allow_existing_dirs,
+            overwrite_existing_files,
             feature_flags,
             current_path: Arc::new(Mutex::new(OsString::new())),
             on_error: Box::new(Err),
@@ -392,14 +398,21 @@ impl Extractor {
         metadata: &Metadata,
         size: u64,
         contents: &mut dyn io::Read,
+        overwrite_existing_files: bool,
     ) -> Result<(), Error> {
         let parent = self.parent_fd()?;
+        let mut oflags = OFlag::O_CREAT | OFlag::O_WRONLY | OFlag::O_CLOEXEC;
+        if overwrite_existing_files {
+            oflags = oflags | OFlag::O_TRUNC;
+        } else {
+            oflags = oflags | OFlag::O_EXCL;
+        }
         let mut file = unsafe {
             std::fs::File::from_raw_fd(
                 nix::fcntl::openat(
                     parent,
                     file_name,
-                    OFlag::O_CREAT | OFlag::O_EXCL | OFlag::O_WRONLY | OFlag::O_CLOEXEC,
+                    oflags,
                     Mode::from_bits(0o600).unwrap(),
                 )
                 .map_err(|err| format_err!("failed to create file {:?}: {}", file_name, err))?,
@@ -448,14 +461,21 @@ impl Extractor {
         metadata: &Metadata,
         size: u64,
         contents: &mut T,
+        overwrite_existing_files: bool,
     ) -> Result<(), Error> {
         let parent = self.parent_fd()?;
+        let mut oflags = OFlag::O_CREAT | OFlag::O_WRONLY | OFlag::O_CLOEXEC;
+        if overwrite_existing_files {
+            oflags = oflags | OFlag::O_TRUNC;
+        } else {
+            oflags = oflags | OFlag::O_EXCL;
+        }
         let mut file = tokio::fs::File::from_std(unsafe {
             std::fs::File::from_raw_fd(
                 nix::fcntl::openat(
                     parent,
                     file_name,
-                    OFlag::O_CREAT | OFlag::O_EXCL | OFlag::O_WRONLY | OFlag::O_CLOEXEC,
+                    oflags,
                     Mode::from_bits(0o600).unwrap(),
                 )
                 .map_err(|err| format_err!("failed to create file {:?}: {}", file_name, err))?,
@@ -818,7 +838,7 @@ where
         )
     })?;
 
-    Ok(Extractor::new(dir, metadata, false, Flags::DEFAULT))
+    Ok(Extractor::new(dir, metadata, false, false, Flags::DEFAULT))
 }
 
 pub async fn extract_sub_dir<T, DEST, PATH>(
@@ -951,6 +971,7 @@ where
                     &mut file.contents().await.map_err(|_| {
                         format_err!("found regular file entry without contents in archive")
                     })?,
+                    extractor.overwrite_existing_files,
                 )
                 .await?
         }
@@ -998,6 +1019,7 @@ where
                             &mut decoder.contents().ok_or_else(|| {
                                 format_err!("found regular file entry without contents in archive")
                             })?,
+                            extractor.overwrite_existing_files,
                         )
                         .await?
                 }
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index 3714eb03..1dd34deb 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -75,6 +75,11 @@ fn extract_archive_from_reader<R: std::io::Read>(
                 optional: true,
                 default: false,
             },
+            "overwrite_existing_files": {
+                description: "overwrite already existing files",
+                optional: true,
+                default: false,
+            },
             "files-from": {
                 description: "File containing match pattern for files to restore.",
                 optional: true,
@@ -112,6 +117,7 @@ fn extract_archive(
     no_fcaps: bool,
     no_acls: bool,
     allow_existing_dirs: bool,
+    overwrite_existing_files: bool,
     files_from: Option<String>,
     no_device_nodes: bool,
     no_fifos: bool,
@@ -179,6 +185,7 @@ fn extract_archive(
     let options = PxarExtractOptions {
         match_list: &match_list,
         allow_existing_dirs,
+        overwrite_existing_files,
         extract_match_default,
         on_error,
     };
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup v2 2/3] pbs-client: added options to skip xattr/acl/ownership/permissions

[Markus Frank]

Also added WITH_OWNER and WITH_PERMISSION to Default-Flags,
because otherwise it would be needed to activly set these flags and most
filesystems that support XATTR and ACL also support
POSIX-Permissions & Ownership.

Signed-off-by: Markus Frank <m.frank@proxmox.com>
---
v2: 
* created new WITH_OWNER Flag and use WITH_PERMISSIONS for skipping chmod
* removed redundant if 

 pbs-client/src/pxar/flags.rs    |  6 +++++
 pbs-client/src/pxar/metadata.rs | 40 +++++++++++++++++++++------------
 2 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/pbs-client/src/pxar/flags.rs b/pbs-client/src/pxar/flags.rs
index d46c8af3..b3280de7 100644
--- a/pbs-client/src/pxar/flags.rs
+++ b/pbs-client/src/pxar/flags.rs
@@ -71,6 +71,9 @@ bitflags! {
         /// Preserve XFS/ext4/ZFS project quota ID
         const WITH_QUOTA_PROJID                = 0x0001_0000_0000;
 
+        /// UNIX OWNERSHIP
+        const WITH_OWNER                       = 0x0002_0000_0000;
+
         /// Support ".pxarexclude" files
         const EXCLUDE_FILE                     = 0x1000_0000_0000_0000;
         /// Exclude submounts
@@ -105,6 +108,7 @@ bitflags! {
             Flags::WITH_2SEC_TIME.bits() |
             Flags::WITH_READ_ONLY.bits() |
             Flags::WITH_PERMISSIONS.bits() |
+            Flags::WITH_OWNER.bits() |
             Flags::WITH_SYMLINKS.bits() |
             Flags::WITH_DEVICE_NODES.bits() |
             Flags::WITH_FIFOS.bits() |
@@ -135,6 +139,8 @@ bitflags! {
             Flags::WITH_FLAG_PROJINHERIT.bits() |
             Flags::WITH_SUBVOLUME.bits() |
             Flags::WITH_SUBVOLUME_RO.bits() |
+            Flags::WITH_PERMISSIONS.bits() |
+            Flags::WITH_OWNER.bits() |
             Flags::WITH_XATTRS.bits() |
             Flags::WITH_ACL.bits() |
             Flags::WITH_SELINUX.bits() |
diff --git a/pbs-client/src/pxar/metadata.rs b/pbs-client/src/pxar/metadata.rs
index 22bc5f9d..be1911a7 100644
--- a/pbs-client/src/pxar/metadata.rs
+++ b/pbs-client/src/pxar/metadata.rs
@@ -100,19 +100,7 @@ pub fn apply(
     on_error: &mut (dyn FnMut(Error) -> Result<(), Error> + Send),
 ) -> Result<(), Error> {
     let c_proc_path = CString::new(format!("/proc/self/fd/{}", fd)).unwrap();
-
-    unsafe {
-        // UID and GID first, as this fails if we lose access anyway.
-        c_result!(libc::chown(
-            c_proc_path.as_ptr(),
-            metadata.stat.uid,
-            metadata.stat.gid
-        ))
-        .map(drop)
-        .or_else(allow_notsupp)
-        .map_err(|err| format_err!("failed to set ownership: {}", err))
-        .or_else(&mut *on_error)?;
-    }
+    apply_ownership(flags, c_proc_path.as_ptr(), metadata, &mut *on_error)?;
 
     let mut skip_xattrs = false;
     apply_xattrs(flags, c_proc_path.as_ptr(), metadata, &mut skip_xattrs)
@@ -125,7 +113,7 @@ pub fn apply(
 
     // Finally mode and time. We may lose access with mode, but the changing the mode also
     // affects times.
-    if !metadata.is_symlink() {
+    if !metadata.is_symlink() && flags.contains(Flags::WITH_PERMISSIONS) {
         c_result!(unsafe {
             libc::chmod(c_proc_path.as_ptr(), perms_from_metadata(metadata)?.bits())
         })
@@ -162,6 +150,30 @@ pub fn apply(
     Ok(())
 }
 
+pub fn apply_ownership(
+    flags: Flags,
+    c_proc_path: *const libc::c_char,
+    metadata: &Metadata,
+    on_error: &mut (dyn FnMut(Error) -> Result<(), Error> + Send),
+) -> Result<(), Error> {
+    if !flags.contains(Flags::WITH_OWNER) {
+        return Ok(());
+    }
+    unsafe {
+        // UID and GID first, as this fails if we lose access anyway.
+        c_result!(libc::chown(
+            c_proc_path,
+            metadata.stat.uid,
+            metadata.stat.gid
+        ))
+        .map(drop)
+        .or_else(allow_notsupp)
+        .map_err(|err| format_err!("failed to set ownership: {}", err))
+        .or_else(&mut *on_error)?;
+    }
+    Ok(())
+}
+
 fn add_fcaps(
     flags: Flags,
     c_proc_path: *const libc::c_char,
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup v2 3/3] proxmox-backup-client: added ignore-acl/xattr/ownership/permission & overwrite parameters

[Markus Frank]

If ignore-acl/ignore-xattr/ignore-ownership/ignore-permission is set,
the corresponding flag gets removed.

overwrite-existing-files is saved as an PxarExtractOption like
allow-existing-dirs.

Signed-off-by: Markus Frank <m.frank@proxmox.com>
---
v2: added ignore-permission

 proxmox-backup-client/src/main.rs | 49 ++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index 4bb9aa5e..dc038864 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -1201,6 +1201,31 @@ We do not extract '.pxar' archives when writing to standard output.
                 type: CryptMode,
                 optional: true,
             },
+            "ignore-acl": {
+                type: Boolean,
+                description: "ignore acl settings",
+                optional: true,
+            },
+            "ignore-xattr": {
+                type: Boolean,
+                description: "ignore xattr settings",
+                optional: true,
+            },
+            "ignore-ownership": {
+                type: Boolean,
+                description: "ignore owner settings (no chown)",
+                optional: true,
+            },
+            "ignore-permission": {
+                type: Boolean,
+                description: "ignore permission settings (no chmod)",
+                optional: true,
+            },
+            "overwrite-existing-files": {
+                type: Boolean,
+                description: "overwrite already existing files",
+                optional: true,
+            },
         }
     }
 )]
@@ -1210,6 +1235,12 @@ async fn restore(param: Value) -> Result<Value, Error> {
 
     let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
 
+    let ignore_acls = param["ignore-acl"].as_bool().unwrap_or(false);
+    let ignore_xattr = param["ignore-xattr"].as_bool().unwrap_or(false);
+    let ignore_ownership = param["ignore-ownership"].as_bool().unwrap_or(false);
+    let ignore_permission = param["ignore-permission"].as_bool().unwrap_or(false);
+    let overwrite_existing_files = param["overwrite-existing-files"].as_bool().unwrap_or(false);
+
     let archive_name = json::required_string_param(&param, "archive-name")?;
 
     let rate = match param["rate"].as_str() {
@@ -1331,14 +1362,30 @@ async fn restore(param: Value) -> Result<Value, Error> {
             match_list: &[],
             extract_match_default: true,
             allow_existing_dirs,
+            overwrite_existing_files,
             on_error: None,
         };
 
+        let mut feature_flags = pbs_client::pxar::Flags::DEFAULT;
+
+        if ignore_acls {
+            feature_flags.remove(pbs_client::pxar::Flags::WITH_ACL);
+        }
+        if ignore_xattr {
+            feature_flags.remove(pbs_client::pxar::Flags::WITH_XATTRS);
+        }
+        if ignore_ownership {
+            feature_flags.remove(pbs_client::pxar::Flags::WITH_OWNER);
+        }
+        if ignore_permission {
+            feature_flags.remove(pbs_client::pxar::Flags::WITH_PERMISSIONS);
+        }
+
         if let Some(target) = target {
             pbs_client::pxar::extract_archive(
                 pxar::decoder::Decoder::from_std(reader)?,
                 Path::new(target),
-                pbs_client::pxar::Flags::DEFAULT,
+                feature_flags,
                 |path| {
                     log::debug!("{:?}", path);
                 },
-- 
2.30.2

Re: [pbs-devel] [PATCH proxmox-backup v2 3/3] proxmox-backup-client: added ignore-acl/xattr/ownership/permission & overwrite parameters

[Wolfgang Bumiller]

Sorry for missing this in the v1 revie, but I think it's better to have
the parameters as actual function paramters, and the api description
contain the `default` values.

Also, I think most of these should use plural form ;-)

On Thu, Aug 18, 2022 at 01:06:54PM +0200, Markus Frank wrote:
> If ignore-acl/ignore-xattr/ignore-ownership/ignore-permission is set,
> the corresponding flag gets removed.
> 
> overwrite-existing-files is saved as an PxarExtractOption like
> allow-existing-dirs.
> 
> Signed-off-by: Markus Frank <m.frank@proxmox.com>
> ---
> v2: added ignore-permission
> 
>  proxmox-backup-client/src/main.rs | 49 ++++++++++++++++++++++++++++++-
>  1 file changed, 48 insertions(+), 1 deletion(-)
> 
> diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
> index 4bb9aa5e..dc038864 100644
> --- a/proxmox-backup-client/src/main.rs
> +++ b/proxmox-backup-client/src/main.rs
> @@ -1201,6 +1201,31 @@ We do not extract '.pxar' archives when writing to standard output.
>                  type: CryptMode,
>                  optional: true,
>              },
> +            "ignore-acl": {

acl -> acls

> +                type: Boolean,
> +                description: "ignore acl settings",
> +                optional: true,

Add `default: false,`

> +            },
> +            "ignore-xattr": {

xattr -> xattrs

> +                type: Boolean,
> +                description: "ignore xattr settings",
> +                optional: true,

Add `default: false,`

> +            },
> +            "ignore-ownership": {
> +                type: Boolean,
> +                description: "ignore owner settings (no chown)",
> +                optional: true,

Add `default: false,`

> +            },
> +            "ignore-permission": {

permission -> permissions

> +                type: Boolean,
> +                description: "ignore permission settings (no chmod)",
> +                optional: true,

Add `default: false,`

> +            },
> +            "overwrite-existing-files": {

I think 'overwrite' could be enough actually.
(eg. tar has `--overwrite`)

> +                type: Boolean,
> +                description: "overwrite already existing files",
> +                optional: true,
> +            },
>          }
>      }
>  )]
> @@ -1210,6 +1235,12 @@ async fn restore(param: Value) -> Result<Value, Error> {

^ You can just place `bool` parameters in the `fn()` signature and
they'll be filled with the `default` values automatically by the api
macro (just use underscores in place of the minuses).
The `param: Value` parameter will then only contain the remaining
arguments.

(Most of the required parameters could actually be changed this way, but
not necessarily in the same patch series...)

>  
>      let allow_existing_dirs = param["allow-existing-dirs"].as_bool().unwrap_or(false);
>  
> +    let ignore_acls = param["ignore-acl"].as_bool().unwrap_or(false);
> +    let ignore_xattr = param["ignore-xattr"].as_bool().unwrap_or(false);
> +    let ignore_ownership = param["ignore-ownership"].as_bool().unwrap_or(false);
> +    let ignore_permission = param["ignore-permission"].as_bool().unwrap_or(false);
> +    let overwrite_existing_files = param["overwrite-existing-files"].as_bool().unwrap_or(false);

^ then this entire block can be dropped