[pbs-devel] [PATCH proxmox/proxmox-backup v2] fix #2915: stat when necessary

[pbs-devel] [PATCH proxmox/proxmox-backup v2] fix #2915: stat when necessary

[Dominik Csapak]

this series implements a fallback to stat'ing files when the
readdir entry does not contain the filetype (since only some
filesystems support that)

changes from v1:
* omit the 'FileStat' parameter for scandir callback (we don't really
  need it there)
* add two new helpers to fstatat the file to get the type, and to
  extract the type from a FileStat struct
* use the FileStat directly in sweep_unused_chunk

proxmox:

Dominik Csapak (1):
  partially fix #2915: proxmox-sys: scandir: stat if file_type is
    unknown

 proxmox-sys/src/fs/read_dir.rs | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

proxmox-backup:

Dominik Csapak (1):
  partially fix #2915: 'stat' in case ReadDirEntry does not contain type

 pbs-datastore/src/chunk_store.rs | 23 ++++++----------
 pbs-datastore/src/hierarchy.rs   | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 14 deletions(-)

-- 
2.30.2

[pbs-devel] [PATCH proxmox v2 1/1] partially fix #2915: proxmox-sys: scandir: stat if file_type is unknown

[Dominik Csapak]

when using readdir/getdents the file type might be 'DT_UNKNOWN'
(depending on the filesystem). Do a fstatat in that case to
get the file type. Since maybe the callback wants to do
a stat anyway, pass it there (if done)

adds two new helpers:
'file_type_from_file_stat': uses a FileStat struct to get the file type
'get_file_type': calls fstatat to determine the file_type

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 proxmox-sys/src/fs/read_dir.rs | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/proxmox-sys/src/fs/read_dir.rs b/proxmox-sys/src/fs/read_dir.rs
index f687748..8c0c5a0 100644
--- a/proxmox-sys/src/fs/read_dir.rs
+++ b/proxmox-sys/src/fs/read_dir.rs
@@ -2,7 +2,7 @@ use std::borrow::{Borrow, BorrowMut};
 use std::ops::{Deref, DerefMut};
 use std::os::unix::io::{AsRawFd, RawFd};
 
-use anyhow::{bail, format_err, Error};
+use anyhow::{format_err, Error};
 use nix::dir;
 use nix::dir::Dir;
 use nix::fcntl::OFlag;
@@ -148,7 +148,7 @@ where
         let entry = entry?;
         let file_type = match entry.file_type() {
             Some(file_type) => file_type,
-            None => bail!("unable to detect file type"),
+            None => get_file_type(entry.parent_fd(), entry.file_name())?,
         };
 
         callback(
@@ -321,3 +321,33 @@ fn do_lock_dir_noblock(
 
     Ok(handle)
 }
+
+/// extracts [`nix::dir::Type`] from a [`nix::sys::stat::FileStat`] if possible
+pub fn file_type_from_file_stat(stat: &nix::sys::stat::FileStat) -> Option<nix::dir::Type> {
+    use nix::dir::Type;
+
+    // mask out all unnecessary bits
+    Some(match stat.st_mode & libc::S_IFMT {
+        libc::S_IFSOCK => Type::Socket,
+        libc::S_IFLNK => Type::Symlink,
+        libc::S_IFREG => Type::File,
+        libc::S_IFBLK => Type::BlockDevice,
+        libc::S_IFDIR => Type::Directory,
+        libc::S_IFCHR => Type::CharacterDevice,
+        libc::S_IFIFO => Type::Fifo,
+        _ => return None,
+    })
+}
+
+/// Returns the file type of the `path` in the `parent_fd`
+///
+/// calls [`nix::sys::stat::fstatat`] to determine it
+pub fn get_file_type<P: ?Sized + nix::NixPath>(
+    parent_fd: RawFd,
+    path: &P,
+) -> Result<nix::dir::Type, Error> {
+    let stat = nix::sys::stat::fstatat(parent_fd, path, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW)?;
+    let file_type =
+        file_type_from_file_stat(&stat).ok_or_else(|| format_err!("unable to detect file type"))?;
+    Ok(file_type)
+}
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup v2 1/1] partially fix #2915: 'stat' in case ReadDirEntry does not contain type

[Dominik Csapak]

readdir/getdents may return 'DT_UNKNOWN' for the file type
(which corresponds to 'None' in nix::dir::Entry), so stat the file and
check the type

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 pbs-datastore/src/chunk_store.rs | 23 ++++++----------
 pbs-datastore/src/hierarchy.rs   | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/pbs-datastore/src/chunk_store.rs b/pbs-datastore/src/chunk_store.rs
index c9714c1e..3bebc645 100644
--- a/pbs-datastore/src/chunk_store.rs
+++ b/pbs-datastore/src/chunk_store.rs
@@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex};
 use anyhow::{bail, format_err, Error};
 
 use pbs_api_types::GarbageCollectionStatus;
-use proxmox_sys::fs::{create_dir, create_path, CreateOptions};
+use proxmox_sys::fs::{create_dir, create_path, file_type_from_file_stat, CreateOptions};
 use proxmox_sys::process_locker::{
     ProcessLockExclusiveGuard, ProcessLockSharedGuard, ProcessLocker,
 };
@@ -375,24 +375,19 @@ impl ChunkStore {
                 ),
             };
 
-            let file_type = match entry.file_type() {
-                Some(file_type) => file_type,
-                None => bail!(
-                    "unsupported file system type on chunk store '{}'",
-                    self.name
-                ),
-            };
-            if file_type != nix::dir::Type::File {
-                continue;
-            }
-
-            chunk_count += 1;
-
             let filename = entry.file_name();
 
             let lock = self.mutex.lock();
 
             if let Ok(stat) = fstatat(dirfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
+                let file_type = file_type_from_file_stat(&stat);
+                if file_type != Some(nix::dir::Type::File) {
+                    drop(lock);
+                    continue;
+                }
+
+                chunk_count += 1;
+
                 if stat.st_atime < min_atime {
                     //let age = now - stat.st_atime;
                     //println!("UNLINK {}  {:?}", age/(3600*24), filename);
diff --git a/pbs-datastore/src/hierarchy.rs b/pbs-datastore/src/hierarchy.rs
index d5007b07..8609364b 100644
--- a/pbs-datastore/src/hierarchy.rs
+++ b/pbs-datastore/src/hierarchy.rs
@@ -5,6 +5,7 @@ use std::sync::Arc;
 use anyhow::{bail, format_err, Error};
 
 use pbs_api_types::{BackupNamespace, BackupType, BACKUP_DATE_REGEX, BACKUP_ID_REGEX};
+use proxmox_sys::fs::get_file_type;
 
 use crate::backup_info::{BackupDir, BackupGroup};
 use crate::DataStore;
@@ -36,6 +37,17 @@ impl Iterator for ListSnapshots {
                 Ok(ref entry) => {
                     match entry.file_type() {
                         Some(nix::dir::Type::Directory) => entry, // OK
+                        None => match get_file_type(entry.parent_fd(), entry.file_name()) {
+                            Ok(nix::dir::Type::Directory) => entry,
+                            Ok(_) => continue,
+                            Err(err) => {
+                                log::info!(
+                                    "error listing snapshots for {}: {err}",
+                                    self.group.group()
+                                );
+                                continue;
+                            }
+                        },
                         _ => continue,
                     }
                 }
@@ -93,6 +105,17 @@ impl Iterator for ListGroups {
                     Ok(ref entry) => {
                         match entry.file_type() {
                             Some(nix::dir::Type::Directory) => entry, // OK
+                            None => match get_file_type(entry.parent_fd(), entry.file_name()) {
+                                Ok(nix::dir::Type::Directory) => entry,
+                                Ok(_) => continue,
+                                Err(err) => {
+                                    log::info!(
+                                        "error listing groups for {}: {err}",
+                                        self.store.name()
+                                    );
+                                    continue;
+                                }
+                            },
                             _ => continue,
                         }
                     }
@@ -114,6 +137,17 @@ impl Iterator for ListGroups {
                     Ok(ref entry) => {
                         match entry.file_type() {
                             Some(nix::dir::Type::Directory) => entry, // OK
+                            None => match get_file_type(entry.parent_fd(), entry.file_name()) {
+                                Ok(nix::dir::Type::Directory) => entry,
+                                Ok(_) => continue,
+                                Err(err) => {
+                                    log::info!(
+                                        "error listing groups for {}: {err}",
+                                        self.store.name()
+                                    );
+                                    continue;
+                                }
+                            },
                             _ => continue,
                         }
                     }
@@ -176,6 +210,19 @@ impl Iterator for ListNamespaces {
                     Ok(ref entry) => {
                         match entry.file_type() {
                             Some(nix::dir::Type::Directory) => entry, // OK
+                            None => match get_file_type(entry.parent_fd(), entry.file_name()) {
+                                Ok(nix::dir::Type::Directory) => entry,
+                                Ok(_) => continue,
+                                Err(err) => {
+                                    let mut base_path = self.base_path.to_owned();
+                                    if !self.ns.is_root() {
+                                        base_path.push(self.ns.path());
+                                    }
+                                    base_path.push("ns");
+                                    log::info!("error listing dirs in {:?}: {err}", base_path);
+                                    continue;
+                                }
+                            },
                             _ => continue,
                         }
                     }
-- 
2.30.2

[pbs-devel] applied: [PATCH proxmox v2 1/1] partially fix #2915: proxmox-sys: scandir: stat if file_type is unknown

[Wolfgang Bumiller]

applied

On Tue, Jun 28, 2022 at 03:15:13PM +0200, Dominik Csapak wrote:
> when using readdir/getdents the file type might be 'DT_UNKNOWN'
> (depending on the filesystem). Do a fstatat in that case to
> get the file type. Since maybe the callback wants to do
> a stat anyway, pass it there (if done)
> 
> adds two new helpers:
> 'file_type_from_file_stat': uses a FileStat struct to get the file type
> 'get_file_type': calls fstatat to determine the file_type
> 
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
>  proxmox-sys/src/fs/read_dir.rs | 34 ++++++++++++++++++++++++++++++++--
>  1 file changed, 32 insertions(+), 2 deletions(-)
> 
> diff --git a/proxmox-sys/src/fs/read_dir.rs b/proxmox-sys/src/fs/read_dir.rs
> index f687748..8c0c5a0 100644
> --- a/proxmox-sys/src/fs/read_dir.rs
> +++ b/proxmox-sys/src/fs/read_dir.rs
> @@ -2,7 +2,7 @@ use std::borrow::{Borrow, BorrowMut};
>  use std::ops::{Deref, DerefMut};
>  use std::os::unix::io::{AsRawFd, RawFd};
>  
> -use anyhow::{bail, format_err, Error};
> +use anyhow::{format_err, Error};
>  use nix::dir;
>  use nix::dir::Dir;
>  use nix::fcntl::OFlag;
> @@ -148,7 +148,7 @@ where
>          let entry = entry?;
>          let file_type = match entry.file_type() {
>              Some(file_type) => file_type,
> -            None => bail!("unable to detect file type"),
> +            None => get_file_type(entry.parent_fd(), entry.file_name())?,
>          };
>  
>          callback(
> @@ -321,3 +321,33 @@ fn do_lock_dir_noblock(
>  
>      Ok(handle)
>  }
> +
> +/// extracts [`nix::dir::Type`] from a [`nix::sys::stat::FileStat`] if possible
> +pub fn file_type_from_file_stat(stat: &nix::sys::stat::FileStat) -> Option<nix::dir::Type> {
> +    use nix::dir::Type;
> +
> +    // mask out all unnecessary bits
> +    Some(match stat.st_mode & libc::S_IFMT {
> +        libc::S_IFSOCK => Type::Socket,
> +        libc::S_IFLNK => Type::Symlink,
> +        libc::S_IFREG => Type::File,
> +        libc::S_IFBLK => Type::BlockDevice,
> +        libc::S_IFDIR => Type::Directory,
> +        libc::S_IFCHR => Type::CharacterDevice,
> +        libc::S_IFIFO => Type::Fifo,
> +        _ => return None,
> +    })
> +}
> +
> +/// Returns the file type of the `path` in the `parent_fd`
> +///
> +/// calls [`nix::sys::stat::fstatat`] to determine it
> +pub fn get_file_type<P: ?Sized + nix::NixPath>(
> +    parent_fd: RawFd,
> +    path: &P,
> +) -> Result<nix::dir::Type, Error> {
> +    let stat = nix::sys::stat::fstatat(parent_fd, path, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW)?;
> +    let file_type =
> +        file_type_from_file_stat(&stat).ok_or_else(|| format_err!("unable to detect file type"))?;
> +    Ok(file_type)
> +}
> -- 
> 2.30.2

[pbs-devel] applied: [PATCH proxmox-backup v2 1/1] partially fix #2915: 'stat' in case ReadDirEntry does not contain type

[Wolfgang Bumiller]

applied, thanks