From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5B6B99131E for ; Wed, 14 Feb 2024 09:11:01 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 3BC8A3F5B7 for ; Wed, 14 Feb 2024 09:10:31 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 14 Feb 2024 09:10:30 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 3249A480EE for ; Wed, 14 Feb 2024 09:10:30 +0100 (CET) Date: Wed, 14 Feb 2024 09:10:28 +0100 (CET) From: Dietmar Maurer To: Proxmox Backup Server development discussion , Gabriel Goller Message-ID: <1865314480.1832.1707898228662@webmail.proxmox.com> In-Reply-To: <20240213140433.164037-1-g.goller@proxmox.com> References: <20240213140433.164037-1-g.goller@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit X-Priority: 3 Importance: Normal X-Mailer: Open-Xchange Mailer v7.10.6-Rev59 X-Originating-Client: open-xchange-appsuite X-SPAM-LEVEL: Spam detection results: 0 AWL 0.358 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [main.rs, proxmox.com, api.rs, create.rs] Subject: Re: [pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 14 Feb 2024 08:11:01 -0000 comments inline > > Signed-off-by: Gabriel Goller > --- > > cont'd from: https://lists.proxmox.com/pipermail/pbs-devel/2024-February/007780.html > > pbs-client/src/pxar/create.rs | 30 +++++++++++++++++-- > pbs-client/src/pxar_backup_stream.rs | 2 +- > proxmox-backup-client/src/main.rs | 8 +++++ > .../src/proxmox_restore_daemon/api.rs | 3 +- > pxar-bin/src/main.rs | 1 + > 5 files changed, 39 insertions(+), 5 deletions(-) > > diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs > index e7053d9e..04d92714 100644 > --- a/pbs-client/src/pxar/create.rs > +++ b/pbs-client/src/pxar/create.rs > @@ -7,7 +7,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; > use std::path::{Path, PathBuf}; > use std::sync::{Arc, Mutex}; > > -use anyhow::{bail, Context, Error}; > +use anyhow::{bail, Context, Error, anyhow}; not sorted, and we do use "format_err" instead of "anyhow!". > use futures::future::BoxFuture; > use futures::FutureExt; > use nix::dir::Dir; > @@ -41,6 +41,8 @@ pub struct PxarCreateOptions { > pub entries_max: usize, > /// Skip lost+found directory > pub skip_lost_and_found: bool, > + /// Skip metadata of files that return E2BIG error > + pub skip_e2big_metadata: bool, > } > > fn detect_fs_type(fd: RawFd) -> Result { > @@ -103,7 +105,7 @@ impl std::error::Error for ArchiveError {} > > impl fmt::Display for ArchiveError { > fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { > - write!(f, "error at {:?}: {}", self.path, self.error) > + write!(f, "error at {:?}: {:#}", self.path, self.error) please do this in a separate patch > } > } > > @@ -128,6 +130,7 @@ struct Archiver { > device_set: Option>, > hardlinks: HashMap, > file_copy_buffer: Vec, > + skip_e2big_metadata: bool, > } > > type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>; > @@ -158,6 +161,7 @@ where > feature_flags & fs_feature_flags, > fs_magic, > &mut fs_feature_flags, > + options.skip_e2big_metadata > ) > .context("failed to get metadata for source directory")?; > > @@ -192,6 +196,7 @@ where > device_set, > hardlinks: HashMap::new(), > file_copy_buffer: vec::undefined(4 * 1024 * 1024), > + skip_e2big_metadata: options.skip_e2big_metadata, > }; > > archiver > @@ -540,6 +545,7 @@ impl Archiver { > self.flags(), > self.fs_magic, > &mut self.fs_feature_flags, > + self.skip_e2big_metadata > )?; > > let match_path = PathBuf::from("/").join(self.path.clone()); > @@ -765,6 +771,7 @@ fn get_metadata( > flags: Flags, > fs_magic: i64, > fs_feature_flags: &mut Flags, > + skip_e2big_metadata: bool, > ) -> Result { > // required for some of these > let proc_path = Path::new("/proc/self/fd/").join(fd.to_string()); > @@ -780,7 +787,7 @@ fn get_metadata( > ..Default::default() > }; > > - get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags)?; > + get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags, skip_e2big_metadata)?; > get_chattr(&mut meta, fd)?; > get_fat_attr(&mut meta, fd, fs_magic)?; > get_quota_project_id(&mut meta, fd, flags, fs_magic)?; > @@ -818,6 +825,7 @@ fn get_xattr_fcaps_acl( > proc_path: &Path, > flags: Flags, > fs_feature_flags: &mut Flags, > + skip_e2big_metadata: bool, > ) -> Result<(), Error> { > if !flags.contains(Flags::WITH_XATTRS) { > return Ok(()); > @@ -829,6 +837,14 @@ fn get_xattr_fcaps_acl( > fs_feature_flags.remove(Flags::WITH_XATTRS); > return Ok(()); > } > + Err(Errno::E2BIG) => { > + match skip_e2big_metadata { > + true => return Ok(()), > + false => { > + return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string()))); please use format_err!() instead. > + } > + }; > + } > Err(Errno::EBADF) => return Ok(()), // symlinks > Err(err) => return Err(err).context("failed to read xattrs"), > }; > @@ -855,6 +871,14 @@ fn get_xattr_fcaps_acl( > Err(Errno::ENODATA) => (), // it got removed while we were iterating... > Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this > Err(Errno::EBADF) => (), // symlinks, shouldn't be able to reach this either > + Err(Errno::E2BIG) => { > + match skip_e2big_metadata { > + true => return Ok(()), > + false => { > + return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string()))); see above > + } > + }; > + } > Err(err) => { > return Err(err).context(format!("error reading extended attribute {attr:?}")) > } > diff --git a/pbs-client/src/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs > index 22a6ffdc..61aee7c2 100644 > --- a/pbs-client/src/pxar_backup_stream.rs > +++ b/pbs-client/src/pxar_backup_stream.rs > @@ -68,7 +68,7 @@ impl PxarBackupStream { > .await > { > let mut error = error2.lock().unwrap(); > - *error = Some(err.to_string()); > + *error = Some(format!("{:#}", err)); please do this in a separate patch > } > }; > > diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs > index e5caf87d..e4a551d5 100644 > --- a/proxmox-backup-client/src/main.rs > +++ b/proxmox-backup-client/src/main.rs > @@ -665,6 +665,12 @@ fn spawn_catalog_upload( > optional: true, > default: false, > }, > + "skip-e2big-metadata": { > + type: Boolean, > + description: "Ignore the E2BIG error when retrieving metadata. This includes the file, but discards the metadata", > + optional: true, > + default: false, > + }, > } > } > )] > @@ -674,6 +680,7 @@ async fn create_backup( > all_file_systems: bool, > skip_lost_and_found: bool, > dry_run: bool, > + skip_e2big_metadata: bool, > _info: &ApiMethod, > _rpcenv: &mut dyn RpcEnvironment, > ) -> Result { > @@ -993,6 +1000,7 @@ async fn create_backup( > patterns: pattern_list.clone(), > entries_max: entries_max as usize, > skip_lost_and_found, > + skip_e2big_metadata > }; > > let upload_options = UploadOptions { > diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > index c4e97d33..33070e0b 100644 > --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs > @@ -352,6 +352,7 @@ fn extract( > device_set: None, > patterns, > skip_lost_and_found: false, > + skip_e2big_metadata: false, > }; > > let pxar_writer = TokioWriter::new(writer); > @@ -360,7 +361,7 @@ fn extract( > } > .await; > if let Err(err) = result { > - error!("pxar streaming task failed - {}", err); > + error!("pxar streaming task failed - {:#}", err); please do this in a separate patch > } > }); > } else if format == "tar" { > diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs > index bc044035..e5ea2a08 100644 > --- a/pxar-bin/src/main.rs > +++ b/pxar-bin/src/main.rs > @@ -335,6 +335,7 @@ async fn create_archive( > device_set, > patterns, > skip_lost_and_found: false, > + skip_e2big_metadata: false, > }; > > let source = PathBuf::from(source); > -- > 2.43.0 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel