Re: [pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag

[Dietmar Maurer <dietmar@proxmox.com>]

comments inline

> 
> Signed-off-by: Gabriel Goller <g.goller@proxmox.com>
> ---
> 
> cont'd from: https://lists.proxmox.com/pipermail/pbs-devel/2024-February/007780.html 
> 
>  pbs-client/src/pxar/create.rs                 | 30 +++++++++++++++++--
>  pbs-client/src/pxar_backup_stream.rs          |  2 +-
>  proxmox-backup-client/src/main.rs             |  8 +++++
>  .../src/proxmox_restore_daemon/api.rs         |  3 +-
>  pxar-bin/src/main.rs                          |  1 +
>  5 files changed, 39 insertions(+), 5 deletions(-)
> 
> diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
> index e7053d9e..04d92714 100644
> --- a/pbs-client/src/pxar/create.rs
> +++ b/pbs-client/src/pxar/create.rs
> @@ -7,7 +7,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
>  use std::path::{Path, PathBuf};
>  use std::sync::{Arc, Mutex};
>  
> -use anyhow::{bail, Context, Error};
> +use anyhow::{bail, Context, Error, anyhow};

not sorted, and we do use "format_err" instead of "anyhow!".

>  use futures::future::BoxFuture;
>  use futures::FutureExt;
>  use nix::dir::Dir;
> @@ -41,6 +41,8 @@ pub struct PxarCreateOptions {
>      pub entries_max: usize,
>      /// Skip lost+found directory
>      pub skip_lost_and_found: bool,
> +    /// Skip metadata of files that return E2BIG error
> +    pub skip_e2big_metadata: bool,
>  }
>  
>  fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
> @@ -103,7 +105,7 @@ impl std::error::Error for ArchiveError {}
>  
>  impl fmt::Display for ArchiveError {
>      fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
> -        write!(f, "error at {:?}: {}", self.path, self.error)
> +        write!(f, "error at {:?}: {:#}", self.path, self.error)

please do this in a separate patch

>      }
>  }
>  
> @@ -128,6 +130,7 @@ struct Archiver {
>      device_set: Option<HashSet<u64>>,
>      hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
>      file_copy_buffer: Vec<u8>,
> +    skip_e2big_metadata: bool,
>  }
>  
>  type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
> @@ -158,6 +161,7 @@ where
>          feature_flags & fs_feature_flags,
>          fs_magic,
>          &mut fs_feature_flags,
> +        options.skip_e2big_metadata
>      )
>      .context("failed to get metadata for source directory")?;
>  
> @@ -192,6 +196,7 @@ where
>          device_set,
>          hardlinks: HashMap::new(),
>          file_copy_buffer: vec::undefined(4 * 1024 * 1024),
> +        skip_e2big_metadata: options.skip_e2big_metadata,
>      };
>  
>      archiver
> @@ -540,6 +545,7 @@ impl Archiver {
>              self.flags(),
>              self.fs_magic,
>              &mut self.fs_feature_flags,
> +            self.skip_e2big_metadata
>          )?;
>  
>          let match_path = PathBuf::from("/").join(self.path.clone());
> @@ -765,6 +771,7 @@ fn get_metadata(
>      flags: Flags,
>      fs_magic: i64,
>      fs_feature_flags: &mut Flags,
> +    skip_e2big_metadata: bool,
>  ) -> Result<Metadata, Error> {
>      // required for some of these
>      let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
> @@ -780,7 +787,7 @@ fn get_metadata(
>          ..Default::default()
>      };
>  
> -    get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags)?;
> +    get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags, skip_e2big_metadata)?;
>      get_chattr(&mut meta, fd)?;
>      get_fat_attr(&mut meta, fd, fs_magic)?;
>      get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
> @@ -818,6 +825,7 @@ fn get_xattr_fcaps_acl(
>      proc_path: &Path,
>      flags: Flags,
>      fs_feature_flags: &mut Flags,
> +    skip_e2big_metadata: bool,
>  ) -> Result<(), Error> {
>      if !flags.contains(Flags::WITH_XATTRS) {
>          return Ok(());
> @@ -829,6 +837,14 @@ fn get_xattr_fcaps_acl(
>              fs_feature_flags.remove(Flags::WITH_XATTRS);
>              return Ok(());
>          }
> +        Err(Errno::E2BIG) => {
> +            match skip_e2big_metadata {
> +                true => return Ok(()),
> +                false => {
> +                    return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));

please use format_err!() instead.

> +                }
> +            };
> +        }
>          Err(Errno::EBADF) => return Ok(()), // symlinks
>          Err(err) => return Err(err).context("failed to read xattrs"),
>      };
> @@ -855,6 +871,14 @@ fn get_xattr_fcaps_acl(
>              Err(Errno::ENODATA) => (), // it got removed while we were iterating...
>              Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
>              Err(Errno::EBADF) => (),   // symlinks, shouldn't be able to reach this either
> +            Err(Errno::E2BIG) => {
> +                match skip_e2big_metadata {
> +                    true => return Ok(()),
> +                    false => {
> +                        return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));


see above

> +                    }
> +                };
> +            }
>              Err(err) => {
>                  return Err(err).context(format!("error reading extended attribute {attr:?}"))
>              }
> diff --git a/pbs-client/src/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs
> index 22a6ffdc..61aee7c2 100644
> --- a/pbs-client/src/pxar_backup_stream.rs
> +++ b/pbs-client/src/pxar_backup_stream.rs
> @@ -68,7 +68,7 @@ impl PxarBackupStream {
>              .await
>              {
>                  let mut error = error2.lock().unwrap();
> -                *error = Some(err.to_string());
> +                *error = Some(format!("{:#}", err));

please do this in a separate patch

>              }
>          };
>  
> diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
> index e5caf87d..e4a551d5 100644
> --- a/proxmox-backup-client/src/main.rs
> +++ b/proxmox-backup-client/src/main.rs
> @@ -665,6 +665,12 @@ fn spawn_catalog_upload(
>                 optional: true,
>                 default: false,
>             },
> +           "skip-e2big-metadata": {
> +               type: Boolean,
> +               description: "Ignore the E2BIG error when retrieving metadata. This includes the file, but discards the metadata",
> +               optional: true,
> +               default: false,
> +           },
>         }
>     }
>  )]
> @@ -674,6 +680,7 @@ async fn create_backup(
>      all_file_systems: bool,
>      skip_lost_and_found: bool,
>      dry_run: bool,
> +    skip_e2big_metadata: bool,
>      _info: &ApiMethod,
>      _rpcenv: &mut dyn RpcEnvironment,
>  ) -> Result<Value, Error> {
> @@ -993,6 +1000,7 @@ async fn create_backup(
>                      patterns: pattern_list.clone(),
>                      entries_max: entries_max as usize,
>                      skip_lost_and_found,
> +                    skip_e2big_metadata
>                  };
>  
>                  let upload_options = UploadOptions {
> diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
> index c4e97d33..33070e0b 100644
> --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
> +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
> @@ -352,6 +352,7 @@ fn extract(
>                          device_set: None,
>                          patterns,
>                          skip_lost_and_found: false,
> +                        skip_e2big_metadata: false,
>                      };
>  
>                      let pxar_writer = TokioWriter::new(writer);
> @@ -360,7 +361,7 @@ fn extract(
>                  }
>                  .await;
>                  if let Err(err) = result {
> -                    error!("pxar streaming task failed - {}", err);
> +                    error!("pxar streaming task failed - {:#}", err);

please do this in a separate patch

>                  }
>              });
>          } else if format == "tar" {
> diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
> index bc044035..e5ea2a08 100644
> --- a/pxar-bin/src/main.rs
> +++ b/pxar-bin/src/main.rs
> @@ -335,6 +335,7 @@ async fn create_archive(
>          device_set,
>          patterns,
>          skip_lost_and_found: false,
> +        skip_e2big_metadata: false,
>      };
>  
>      let source = PathBuf::from(source);
> -- 
> 2.43.0
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel