[pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag

* [pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag
@ 2024-02-13 14:04 Gabriel Goller
  2024-02-14  8:10 ` Dietmar Maurer
  2024-02-14  8:36 ` Dietmar Maurer
  0 siblings, 2 replies; 4+ messages in thread
From: Gabriel Goller @ 2024-02-13 14:04 UTC (permalink / raw)
  To: pbs-devel

Some filesystems (f.e. zfs) support xattrs bigger than 64kB, sadly we
can't get them because the kernel vfs limits us. The syscalls listxattr
and getxattr will return a E2BIG error in this case.
Added a flag --ignore-e2big-metadata to the client, this will ignore the
metadata (but still backup the file) if this error occurs. Also print
the errors using the pretty-print formatter ("{:#}"), so that we get
the full anyhow context.

Signed-off-by: Gabriel Goller <g.goller@proxmox.com>
---

cont'd from: https://lists.proxmox.com/pipermail/pbs-devel/2024-February/007780.html 

 pbs-client/src/pxar/create.rs                 | 30 +++++++++++++++++--
 pbs-client/src/pxar_backup_stream.rs          |  2 +-
 proxmox-backup-client/src/main.rs             |  8 +++++
 .../src/proxmox_restore_daemon/api.rs         |  3 +-
 pxar-bin/src/main.rs                          |  1 +
 5 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index e7053d9e..04d92714 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -7,7 +7,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
-use anyhow::{bail, Context, Error};
+use anyhow::{bail, Context, Error, anyhow};
 use futures::future::BoxFuture;
 use futures::FutureExt;
 use nix::dir::Dir;
@@ -41,6 +41,8 @@ pub struct PxarCreateOptions {
     pub entries_max: usize,
     /// Skip lost+found directory
     pub skip_lost_and_found: bool,
+    /// Skip metadata of files that return E2BIG error
+    pub skip_e2big_metadata: bool,
 }
 
 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
@@ -103,7 +105,7 @@ impl std::error::Error for ArchiveError {}
 
 impl fmt::Display for ArchiveError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "error at {:?}: {}", self.path, self.error)
+        write!(f, "error at {:?}: {:#}", self.path, self.error)
     }
 }
 
@@ -128,6 +130,7 @@ struct Archiver {
     device_set: Option<HashSet<u64>>,
     hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
     file_copy_buffer: Vec<u8>,
+    skip_e2big_metadata: bool,
 }
 
 type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
@@ -158,6 +161,7 @@ where
         feature_flags & fs_feature_flags,
         fs_magic,
         &mut fs_feature_flags,
+        options.skip_e2big_metadata
     )
     .context("failed to get metadata for source directory")?;
 
@@ -192,6 +196,7 @@ where
         device_set,
         hardlinks: HashMap::new(),
         file_copy_buffer: vec::undefined(4 * 1024 * 1024),
+        skip_e2big_metadata: options.skip_e2big_metadata,
     };
 
     archiver
@@ -540,6 +545,7 @@ impl Archiver {
             self.flags(),
             self.fs_magic,
             &mut self.fs_feature_flags,
+            self.skip_e2big_metadata
         )?;
 
         let match_path = PathBuf::from("/").join(self.path.clone());
@@ -765,6 +771,7 @@ fn get_metadata(
     flags: Flags,
     fs_magic: i64,
     fs_feature_flags: &mut Flags,
+    skip_e2big_metadata: bool,
 ) -> Result<Metadata, Error> {
     // required for some of these
     let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
@@ -780,7 +787,7 @@ fn get_metadata(
         ..Default::default()
     };
 
-    get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags)?;
+    get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags, skip_e2big_metadata)?;
     get_chattr(&mut meta, fd)?;
     get_fat_attr(&mut meta, fd, fs_magic)?;
     get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
@@ -818,6 +825,7 @@ fn get_xattr_fcaps_acl(
     proc_path: &Path,
     flags: Flags,
     fs_feature_flags: &mut Flags,
+    skip_e2big_metadata: bool,
 ) -> Result<(), Error> {
     if !flags.contains(Flags::WITH_XATTRS) {
         return Ok(());
@@ -829,6 +837,14 @@ fn get_xattr_fcaps_acl(
             fs_feature_flags.remove(Flags::WITH_XATTRS);
             return Ok(());
         }
+        Err(Errno::E2BIG) => {
+            match skip_e2big_metadata {
+                true => return Ok(()),
+                false => {
+                    return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));
+                }
+            };
+        }
         Err(Errno::EBADF) => return Ok(()), // symlinks
         Err(err) => return Err(err).context("failed to read xattrs"),
     };
@@ -855,6 +871,14 @@ fn get_xattr_fcaps_acl(
             Err(Errno::ENODATA) => (), // it got removed while we were iterating...
             Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
             Err(Errno::EBADF) => (),   // symlinks, shouldn't be able to reach this either
+            Err(Errno::E2BIG) => {
+                match skip_e2big_metadata {
+                    true => return Ok(()),
+                    false => {
+                        return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));
+                    }
+                };
+            }
             Err(err) => {
                 return Err(err).context(format!("error reading extended attribute {attr:?}"))
             }
diff --git a/pbs-client/src/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs
index 22a6ffdc..61aee7c2 100644
--- a/pbs-client/src/pxar_backup_stream.rs
+++ b/pbs-client/src/pxar_backup_stream.rs
@@ -68,7 +68,7 @@ impl PxarBackupStream {
             .await
             {
                 let mut error = error2.lock().unwrap();
-                *error = Some(err.to_string());
+                *error = Some(format!("{:#}", err));
             }
         };
 
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index e5caf87d..e4a551d5 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -665,6 +665,12 @@ fn spawn_catalog_upload(
                optional: true,
                default: false,
            },
+           "skip-e2big-metadata": {
+               type: Boolean,
+               description: "Ignore the E2BIG error when retrieving metadata. This includes the file, but discards the metadata",
+               optional: true,
+               default: false,
+           },
        }
    }
 )]
@@ -674,6 +680,7 @@ async fn create_backup(
     all_file_systems: bool,
     skip_lost_and_found: bool,
     dry_run: bool,
+    skip_e2big_metadata: bool,
     _info: &ApiMethod,
     _rpcenv: &mut dyn RpcEnvironment,
 ) -> Result<Value, Error> {
@@ -993,6 +1000,7 @@ async fn create_backup(
                     patterns: pattern_list.clone(),
                     entries_max: entries_max as usize,
                     skip_lost_and_found,
+                    skip_e2big_metadata
                 };
 
                 let upload_options = UploadOptions {
diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
index c4e97d33..33070e0b 100644
--- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
+++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
@@ -352,6 +352,7 @@ fn extract(
                         device_set: None,
                         patterns,
                         skip_lost_and_found: false,
+                        skip_e2big_metadata: false,
                     };
 
                     let pxar_writer = TokioWriter::new(writer);
@@ -360,7 +361,7 @@ fn extract(
                 }
                 .await;
                 if let Err(err) = result {
-                    error!("pxar streaming task failed - {}", err);
+                    error!("pxar streaming task failed - {:#}", err);
                 }
             });
         } else if format == "tar" {
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index bc044035..e5ea2a08 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -335,6 +335,7 @@ async fn create_archive(
         device_set,
         patterns,
         skip_lost_and_found: false,
+        skip_e2big_metadata: false,
     };
 
     let source = PathBuf::from(source);
-- 
2.43.0





^ permalink raw reply	[flat|nested] 4+ messages in thread