From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id B3BF5910AE for ; Tue, 13 Feb 2024 15:04:38 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 954DF36DE8 for ; Tue, 13 Feb 2024 15:04:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 13 Feb 2024 15:04:37 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E749547CA5 for ; Tue, 13 Feb 2024 15:04:36 +0100 (CET) From: Gabriel Goller To: pbs-devel@lists.proxmox.com Date: Tue, 13 Feb 2024 15:04:32 +0100 Message-ID: <20240213140433.164037-1-g.goller@proxmox.com> X-Mailer: git-send-email 2.43.0 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.106 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 13 Feb 2024 14:04:38 -0000 Some filesystems (f.e. zfs) support xattrs bigger than 64kB, sadly we can't get them because the kernel vfs limits us. The syscalls listxattr and getxattr will return a E2BIG error in this case. Added a flag --ignore-e2big-metadata to the client, this will ignore the metadata (but still backup the file) if this error occurs. Also print the errors using the pretty-print formatter ("{:#}"), so that we get the full anyhow context. Signed-off-by: Gabriel Goller --- cont'd from: https://lists.proxmox.com/pipermail/pbs-devel/2024-February/007780.html pbs-client/src/pxar/create.rs | 30 +++++++++++++++++-- pbs-client/src/pxar_backup_stream.rs | 2 +- proxmox-backup-client/src/main.rs | 8 +++++ .../src/proxmox_restore_daemon/api.rs | 3 +- pxar-bin/src/main.rs | 1 + 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index e7053d9e..04d92714 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -7,7 +7,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; -use anyhow::{bail, Context, Error}; +use anyhow::{bail, Context, Error, anyhow}; use futures::future::BoxFuture; use futures::FutureExt; use nix::dir::Dir; @@ -41,6 +41,8 @@ pub struct PxarCreateOptions { pub entries_max: usize, /// Skip lost+found directory pub skip_lost_and_found: bool, + /// Skip metadata of files that return E2BIG error + pub skip_e2big_metadata: bool, } fn detect_fs_type(fd: RawFd) -> Result { @@ -103,7 +105,7 @@ impl std::error::Error for ArchiveError {} impl fmt::Display for ArchiveError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "error at {:?}: {}", self.path, self.error) + write!(f, "error at {:?}: {:#}", self.path, self.error) } } @@ -128,6 +130,7 @@ struct Archiver { device_set: Option>, hardlinks: HashMap, file_copy_buffer: Vec, + skip_e2big_metadata: bool, } type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>; @@ -158,6 +161,7 @@ where feature_flags & fs_feature_flags, fs_magic, &mut fs_feature_flags, + options.skip_e2big_metadata ) .context("failed to get metadata for source directory")?; @@ -192,6 +196,7 @@ where device_set, hardlinks: HashMap::new(), file_copy_buffer: vec::undefined(4 * 1024 * 1024), + skip_e2big_metadata: options.skip_e2big_metadata, }; archiver @@ -540,6 +545,7 @@ impl Archiver { self.flags(), self.fs_magic, &mut self.fs_feature_flags, + self.skip_e2big_metadata )?; let match_path = PathBuf::from("/").join(self.path.clone()); @@ -765,6 +771,7 @@ fn get_metadata( flags: Flags, fs_magic: i64, fs_feature_flags: &mut Flags, + skip_e2big_metadata: bool, ) -> Result { // required for some of these let proc_path = Path::new("/proc/self/fd/").join(fd.to_string()); @@ -780,7 +787,7 @@ fn get_metadata( ..Default::default() }; - get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags)?; + get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags, skip_e2big_metadata)?; get_chattr(&mut meta, fd)?; get_fat_attr(&mut meta, fd, fs_magic)?; get_quota_project_id(&mut meta, fd, flags, fs_magic)?; @@ -818,6 +825,7 @@ fn get_xattr_fcaps_acl( proc_path: &Path, flags: Flags, fs_feature_flags: &mut Flags, + skip_e2big_metadata: bool, ) -> Result<(), Error> { if !flags.contains(Flags::WITH_XATTRS) { return Ok(()); @@ -829,6 +837,14 @@ fn get_xattr_fcaps_acl( fs_feature_flags.remove(Flags::WITH_XATTRS); return Ok(()); } + Err(Errno::E2BIG) => { + match skip_e2big_metadata { + true => return Ok(()), + false => { + return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string()))); + } + }; + } Err(Errno::EBADF) => return Ok(()), // symlinks Err(err) => return Err(err).context("failed to read xattrs"), }; @@ -855,6 +871,14 @@ fn get_xattr_fcaps_acl( Err(Errno::ENODATA) => (), // it got removed while we were iterating... Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this Err(Errno::EBADF) => (), // symlinks, shouldn't be able to reach this either + Err(Errno::E2BIG) => { + match skip_e2big_metadata { + true => return Ok(()), + false => { + return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string()))); + } + }; + } Err(err) => { return Err(err).context(format!("error reading extended attribute {attr:?}")) } diff --git a/pbs-client/src/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs index 22a6ffdc..61aee7c2 100644 --- a/pbs-client/src/pxar_backup_stream.rs +++ b/pbs-client/src/pxar_backup_stream.rs @@ -68,7 +68,7 @@ impl PxarBackupStream { .await { let mut error = error2.lock().unwrap(); - *error = Some(err.to_string()); + *error = Some(format!("{:#}", err)); } }; diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index e5caf87d..e4a551d5 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -665,6 +665,12 @@ fn spawn_catalog_upload( optional: true, default: false, }, + "skip-e2big-metadata": { + type: Boolean, + description: "Ignore the E2BIG error when retrieving metadata. This includes the file, but discards the metadata", + optional: true, + default: false, + }, } } )] @@ -674,6 +680,7 @@ async fn create_backup( all_file_systems: bool, skip_lost_and_found: bool, dry_run: bool, + skip_e2big_metadata: bool, _info: &ApiMethod, _rpcenv: &mut dyn RpcEnvironment, ) -> Result { @@ -993,6 +1000,7 @@ async fn create_backup( patterns: pattern_list.clone(), entries_max: entries_max as usize, skip_lost_and_found, + skip_e2big_metadata }; let upload_options = UploadOptions { diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs index c4e97d33..33070e0b 100644 --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs @@ -352,6 +352,7 @@ fn extract( device_set: None, patterns, skip_lost_and_found: false, + skip_e2big_metadata: false, }; let pxar_writer = TokioWriter::new(writer); @@ -360,7 +361,7 @@ fn extract( } .await; if let Err(err) = result { - error!("pxar streaming task failed - {}", err); + error!("pxar streaming task failed - {:#}", err); } }); } else if format == "tar" { diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs index bc044035..e5ea2a08 100644 --- a/pxar-bin/src/main.rs +++ b/pxar-bin/src/main.rs @@ -335,6 +335,7 @@ async fn create_archive( device_set, patterns, skip_lost_and_found: false, + skip_e2big_metadata: false, }; let source = PathBuf::from(source); -- 2.43.0