From: Gabriel Goller <g.goller@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup] fix #4975: client: ignore E2BIG error flag
Date: Tue, 13 Feb 2024 15:04:32 +0100 [thread overview]
Message-ID: <20240213140433.164037-1-g.goller@proxmox.com> (raw)
Some filesystems (f.e. zfs) support xattrs bigger than 64kB, sadly we
can't get them because the kernel vfs limits us. The syscalls listxattr
and getxattr will return a E2BIG error in this case.
Added a flag --ignore-e2big-metadata to the client, this will ignore the
metadata (but still backup the file) if this error occurs. Also print
the errors using the pretty-print formatter ("{:#}"), so that we get
the full anyhow context.
Signed-off-by: Gabriel Goller <g.goller@proxmox.com>
---
cont'd from: https://lists.proxmox.com/pipermail/pbs-devel/2024-February/007780.html
pbs-client/src/pxar/create.rs | 30 +++++++++++++++++--
pbs-client/src/pxar_backup_stream.rs | 2 +-
proxmox-backup-client/src/main.rs | 8 +++++
.../src/proxmox_restore_daemon/api.rs | 3 +-
pxar-bin/src/main.rs | 1 +
5 files changed, 39 insertions(+), 5 deletions(-)
diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index e7053d9e..04d92714 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -7,7 +7,7 @@ use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
-use anyhow::{bail, Context, Error};
+use anyhow::{bail, Context, Error, anyhow};
use futures::future::BoxFuture;
use futures::FutureExt;
use nix::dir::Dir;
@@ -41,6 +41,8 @@ pub struct PxarCreateOptions {
pub entries_max: usize,
/// Skip lost+found directory
pub skip_lost_and_found: bool,
+ /// Skip metadata of files that return E2BIG error
+ pub skip_e2big_metadata: bool,
}
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
@@ -103,7 +105,7 @@ impl std::error::Error for ArchiveError {}
impl fmt::Display for ArchiveError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
- write!(f, "error at {:?}: {}", self.path, self.error)
+ write!(f, "error at {:?}: {:#}", self.path, self.error)
}
}
@@ -128,6 +130,7 @@ struct Archiver {
device_set: Option<HashSet<u64>>,
hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
file_copy_buffer: Vec<u8>,
+ skip_e2big_metadata: bool,
}
type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
@@ -158,6 +161,7 @@ where
feature_flags & fs_feature_flags,
fs_magic,
&mut fs_feature_flags,
+ options.skip_e2big_metadata
)
.context("failed to get metadata for source directory")?;
@@ -192,6 +196,7 @@ where
device_set,
hardlinks: HashMap::new(),
file_copy_buffer: vec::undefined(4 * 1024 * 1024),
+ skip_e2big_metadata: options.skip_e2big_metadata,
};
archiver
@@ -540,6 +545,7 @@ impl Archiver {
self.flags(),
self.fs_magic,
&mut self.fs_feature_flags,
+ self.skip_e2big_metadata
)?;
let match_path = PathBuf::from("/").join(self.path.clone());
@@ -765,6 +771,7 @@ fn get_metadata(
flags: Flags,
fs_magic: i64,
fs_feature_flags: &mut Flags,
+ skip_e2big_metadata: bool,
) -> Result<Metadata, Error> {
// required for some of these
let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
@@ -780,7 +787,7 @@ fn get_metadata(
..Default::default()
};
- get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags)?;
+ get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags, fs_feature_flags, skip_e2big_metadata)?;
get_chattr(&mut meta, fd)?;
get_fat_attr(&mut meta, fd, fs_magic)?;
get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
@@ -818,6 +825,7 @@ fn get_xattr_fcaps_acl(
proc_path: &Path,
flags: Flags,
fs_feature_flags: &mut Flags,
+ skip_e2big_metadata: bool,
) -> Result<(), Error> {
if !flags.contains(Flags::WITH_XATTRS) {
return Ok(());
@@ -829,6 +837,14 @@ fn get_xattr_fcaps_acl(
fs_feature_flags.remove(Flags::WITH_XATTRS);
return Ok(());
}
+ Err(Errno::E2BIG) => {
+ match skip_e2big_metadata {
+ true => return Ok(()),
+ false => {
+ return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));
+ }
+ };
+ }
Err(Errno::EBADF) => return Ok(()), // symlinks
Err(err) => return Err(err).context("failed to read xattrs"),
};
@@ -855,6 +871,14 @@ fn get_xattr_fcaps_acl(
Err(Errno::ENODATA) => (), // it got removed while we were iterating...
Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
Err(Errno::EBADF) => (), // symlinks, shouldn't be able to reach this either
+ Err(Errno::E2BIG) => {
+ match skip_e2big_metadata {
+ true => return Ok(()),
+ false => {
+ return Err(anyhow!(format!("{} (try --skip-e2big-metadata)", Errno::E2BIG.to_string())));
+ }
+ };
+ }
Err(err) => {
return Err(err).context(format!("error reading extended attribute {attr:?}"))
}
diff --git a/pbs-client/src/pxar_backup_stream.rs b/pbs-client/src/pxar_backup_stream.rs
index 22a6ffdc..61aee7c2 100644
--- a/pbs-client/src/pxar_backup_stream.rs
+++ b/pbs-client/src/pxar_backup_stream.rs
@@ -68,7 +68,7 @@ impl PxarBackupStream {
.await
{
let mut error = error2.lock().unwrap();
- *error = Some(err.to_string());
+ *error = Some(format!("{:#}", err));
}
};
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index e5caf87d..e4a551d5 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -665,6 +665,12 @@ fn spawn_catalog_upload(
optional: true,
default: false,
},
+ "skip-e2big-metadata": {
+ type: Boolean,
+ description: "Ignore the E2BIG error when retrieving metadata. This includes the file, but discards the metadata",
+ optional: true,
+ default: false,
+ },
}
}
)]
@@ -674,6 +680,7 @@ async fn create_backup(
all_file_systems: bool,
skip_lost_and_found: bool,
dry_run: bool,
+ skip_e2big_metadata: bool,
_info: &ApiMethod,
_rpcenv: &mut dyn RpcEnvironment,
) -> Result<Value, Error> {
@@ -993,6 +1000,7 @@ async fn create_backup(
patterns: pattern_list.clone(),
entries_max: entries_max as usize,
skip_lost_and_found,
+ skip_e2big_metadata
};
let upload_options = UploadOptions {
diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
index c4e97d33..33070e0b 100644
--- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
+++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs
@@ -352,6 +352,7 @@ fn extract(
device_set: None,
patterns,
skip_lost_and_found: false,
+ skip_e2big_metadata: false,
};
let pxar_writer = TokioWriter::new(writer);
@@ -360,7 +361,7 @@ fn extract(
}
.await;
if let Err(err) = result {
- error!("pxar streaming task failed - {}", err);
+ error!("pxar streaming task failed - {:#}", err);
}
});
} else if format == "tar" {
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index bc044035..e5ea2a08 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -335,6 +335,7 @@ async fn create_archive(
device_set,
patterns,
skip_lost_and_found: false,
+ skip_e2big_metadata: false,
};
let source = PathBuf::from(source);
--
2.43.0
next reply other threads:[~2024-02-13 14:04 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-02-13 14:04 Gabriel Goller [this message]
2024-02-14 8:10 ` Dietmar Maurer
2024-02-14 8:36 ` Dietmar Maurer
2024-02-14 9:52 ` Gabriel Goller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240213140433.164037-1-g.goller@proxmox.com \
--to=g.goller@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal