[pbs-devel] [PATCH/RFC proxmox-backup 1/2] disks: refactor partition type handling

[pbs-devel] [PATCH/RFC proxmox-backup 1/2] disks: refactor partition type handling

[Fabian Ebner]

in preparation to also get the file system type from lsblk.

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
 src/tools/disks.rs     | 35 ++++++++++++++++++++++-------------
 src/tools/disks/lvm.rs | 18 ++++++++++--------
 src/tools/disks/zfs.rs | 17 ++++++++---------
 3 files changed, 40 insertions(+), 30 deletions(-)

diff --git a/src/tools/disks.rs b/src/tools/disks.rs
index 8573695d..d374df0e 100644
--- a/src/tools/disks.rs
+++ b/src/tools/disks.rs
@@ -46,6 +46,14 @@ pub struct DiskManage {
     mounted_devices: OnceCell<HashSet<dev_t>>,
 }
 
+/// Information for a device as returned by lsblk.
+pub struct LsblkInfo {
+    /// Path to the device.
+    path: String,
+    /// Partition type GUID.
+    partition_type: Option<String>,
+}
+
 impl DiskManage {
     /// Create a new disk management context.
     pub fn new() -> Arc<Self> {
@@ -556,30 +564,31 @@ pub struct BlockDevStat {
 }
 
 /// Use lsblk to read partition type uuids.
-pub fn get_partition_type_info() -> Result<HashMap<String, Vec<String>>, Error> {
+pub fn get_lsblk_info() -> Result<Vec<LsblkInfo>, Error> {
 
     let mut command = std::process::Command::new("lsblk");
     command.args(&["--json", "-o", "path,parttype"]);
 
     let output = crate::tools::run_command(command, None)?;
 
-    let mut res: HashMap<String, Vec<String>> = HashMap::new();
-
     let output: serde_json::Value = output.parse()?;
+
+    let mut res = vec![];
+
     if let Some(list) = output["blockdevices"].as_array() {
         for info in list {
             let path = match info["path"].as_str() {
-                Some(p) => p,
-                None => continue,
-            };
-            let partition_type = match info["parttype"].as_str() {
-                Some(t) => t.to_owned(),
+                Some(p) => p.to_string(),
                 None => continue,
             };
-            let devices = res.entry(partition_type).or_insert(Vec::new());
-            devices.push(path.to_string());
+            let partition_type = info["parttype"].as_str().map(|t| t.to_string());
+            res.push(LsblkInfo {
+                path,
+                partition_type,
+            });
         }
     }
+
     Ok(res)
 }
 
@@ -736,14 +745,14 @@ pub fn get_disks(
 
     let disk_manager = DiskManage::new();
 
-    let partition_type_map = get_partition_type_info()?;
+    let lsblk_info = get_lsblk_info()?;
 
-    let zfs_devices = zfs_devices(&partition_type_map, None).or_else(|err| -> Result<HashSet<u64>, Error> {
+    let zfs_devices = zfs_devices(&lsblk_info, None).or_else(|err| -> Result<HashSet<u64>, Error> {
         eprintln!("error getting zfs devices: {}", err);
         Ok(HashSet::new())
     })?;
 
-    let lvm_devices = get_lvm_devices(&partition_type_map)?;
+    let lvm_devices = get_lvm_devices(&lsblk_info)?;
 
     // fixme: ceph journals/volumes
 
diff --git a/src/tools/disks/lvm.rs b/src/tools/disks/lvm.rs
index 1e8f825d..1a81cea0 100644
--- a/src/tools/disks/lvm.rs
+++ b/src/tools/disks/lvm.rs
@@ -1,10 +1,12 @@
-use std::collections::{HashSet, HashMap};
+use std::collections::HashSet;
 use std::os::unix::fs::MetadataExt;
 
 use anyhow::{Error};
 use serde_json::Value;
 use lazy_static::lazy_static;
 
+use super::LsblkInfo;
+
 lazy_static!{
     static ref LVM_UUIDS: HashSet<&'static str> = {
         let mut set = HashSet::new();
@@ -17,7 +19,7 @@ lazy_static!{
 ///
 /// The set is indexed by using the unix raw device number (dev_t is u64)
 pub fn get_lvm_devices(
-    partition_type_map: &HashMap<String, Vec<String>>,
+    lsblk_info: &[LsblkInfo],
 ) -> Result<HashSet<u64>, Error> {
 
     const PVS_BIN_PATH: &str = "pvs";
@@ -29,12 +31,12 @@ pub fn get_lvm_devices(
 
     let mut device_set: HashSet<u64> = HashSet::new();
 
-    for device_list in partition_type_map.iter()
-        .filter_map(|(uuid, list)| if LVM_UUIDS.contains(uuid.as_str()) { Some(list) } else { None })
-    {
-        for device in device_list {
-            let meta = std::fs::metadata(device)?;
-            device_set.insert(meta.rdev());
+    for info in lsblk_info.iter() {
+        if let Some(partition_type) = &info.partition_type {
+            if LVM_UUIDS.contains(partition_type.as_str()) {
+                let meta = std::fs::metadata(&info.path)?;
+                device_set.insert(meta.rdev());
+            }
         }
     }
 
diff --git a/src/tools/disks/zfs.rs b/src/tools/disks/zfs.rs
index e0084939..55e0aa30 100644
--- a/src/tools/disks/zfs.rs
+++ b/src/tools/disks/zfs.rs
@@ -1,5 +1,5 @@
 use std::path::PathBuf;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashSet;
 use std::os::unix::fs::MetadataExt;
 
 use anyhow::{bail, Error};
@@ -67,12 +67,11 @@ pub fn zfs_pool_stats(pool: &OsStr) -> Result<Option<BlockDevStat>, Error> {
     Ok(Some(stat))
 }
 
-
 /// Get set of devices used by zfs (or a specific zfs pool)
 ///
 /// The set is indexed by using the unix raw device number (dev_t is u64)
 pub fn zfs_devices(
-    partition_type_map: &HashMap<String, Vec<String>>,
+    lsblk_info: &[LsblkInfo],
     pool: Option<String>,
 ) -> Result<HashSet<u64>, Error> {
 
@@ -86,12 +85,12 @@ pub fn zfs_devices(
         }
     }
 
-    for device_list in partition_type_map.iter()
-        .filter_map(|(uuid, list)| if ZFS_UUIDS.contains(uuid.as_str()) { Some(list) } else { None })
-    {
-        for device in device_list {
-            let meta = std::fs::metadata(device)?;
-            device_set.insert(meta.rdev());
+    for info in lsblk_info.iter() {
+        if let Some(partition_type) = &info.partition_type {
+            if ZFS_UUIDS.contains(partition_type.as_str()) {
+                let meta = std::fs::metadata(&info.path)?;
+                device_set.insert(meta.rdev());
+            }
         }
     }
 
-- 
2.20.1

[pbs-devel] [PATCH/RFC proxmox-backup 2/2] disks: also check for file systems with lsblk

[Fabian Ebner]

so that whole disks with a file system that are not mounted are still detected
as in-use.

Reported-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---

Ideally, we'd also show the type in the result, but the api macro complained
when I tried using
    FileSystem(String)
    Mounted(String) // showing the fs type here would also be nice
in the enum. Is there a good way to do this? An alternative would be replacing
the DiskUsageType with a String in the DiskUsage struct, but that's not very
nice...

 src/tools/disks.rs | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/src/tools/disks.rs b/src/tools/disks.rs
index d374df0e..5eed723b 100644
--- a/src/tools/disks.rs
+++ b/src/tools/disks.rs
@@ -52,6 +52,8 @@ pub struct LsblkInfo {
     path: String,
     /// Partition type GUID.
     partition_type: Option<String>,
+    /// File system label.
+    file_system_type: Option<String>,
 }
 
 impl DiskManage {
@@ -563,11 +565,11 @@ pub struct BlockDevStat {
     pub io_ticks: u64, // milliseconds
 }
 
-/// Use lsblk to read partition type uuids.
+/// Use lsblk to read partition type uuids and file system types.
 pub fn get_lsblk_info() -> Result<Vec<LsblkInfo>, Error> {
 
     let mut command = std::process::Command::new("lsblk");
-    command.args(&["--json", "-o", "path,parttype"]);
+    command.args(&["--json", "-o", "path,parttype,fstype"]);
 
     let output = crate::tools::run_command(command, None)?;
 
@@ -582,9 +584,11 @@ pub fn get_lsblk_info() -> Result<Vec<LsblkInfo>, Error> {
                 None => continue,
             };
             let partition_type = info["parttype"].as_str().map(|t| t.to_string());
+            let file_system_type = info["fstype"].as_str().map(|t| t.to_string());
             res.push(LsblkInfo {
                 path,
                 partition_type,
+                file_system_type,
             });
         }
     }
@@ -592,6 +596,25 @@ pub fn get_lsblk_info() -> Result<Vec<LsblkInfo>, Error> {
     Ok(res)
 }
 
+/// Get set of devices with a file system label.
+///
+/// The set is indexed by using the unix raw device number (dev_t is u64)
+fn get_file_system_devices(
+    lsblk_info: &[LsblkInfo],
+) -> Result<HashSet<u64>, Error> {
+
+    let mut device_set: HashSet<u64> = HashSet::new();
+
+    for info in lsblk_info.iter() {
+        if info.file_system_type.is_some() {
+            let meta = std::fs::metadata(&info.path)?;
+            device_set.insert(meta.rdev());
+        }
+    }
+
+    Ok(device_set)
+}
+
 #[api()]
 #[derive(Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all="lowercase")]
@@ -608,6 +631,8 @@ pub enum DiskUsageType {
     DeviceMapper,
     /// Disk has partitions
     Partitions,
+    /// Disk contains a file system label
+    FileSystem,
 }
 
 #[api(
@@ -754,6 +779,8 @@ pub fn get_disks(
 
     let lvm_devices = get_lvm_devices(&lsblk_info)?;
 
+    let file_system_devices = get_file_system_devices(&lsblk_info)?;
+
     // fixme: ceph journals/volumes
 
     let mut result = HashMap::new();
@@ -829,6 +856,10 @@ pub fn get_disks(
             };
         }
 
+        if usage == DiskUsageType::Unused && file_system_devices.contains(&devnum) {
+            usage = DiskUsageType::FileSystem;
+        }
+
         if usage == DiskUsageType::Unused && disk.has_holders()? {
             usage = DiskUsageType::DeviceMapper;
         }
-- 
2.20.1

Re: [pbs-devel] [PATCH/RFC proxmox-backup 1/2] disks: refactor partition type handling

[Wolfgang Bumiller]

Looks fine, though I wonder if we should shorten this?

On Tue, Apr 20, 2021 at 11:53:44AM +0200, Fabian Ebner wrote:
> in preparation to also get the file system type from lsblk.
> 
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>  src/tools/disks.rs     | 35 ++++++++++++++++++++++-------------
>  src/tools/disks/lvm.rs | 18 ++++++++++--------
>  src/tools/disks/zfs.rs | 17 ++++++++---------
>  3 files changed, 40 insertions(+), 30 deletions(-)
> 
> diff --git a/src/tools/disks.rs b/src/tools/disks.rs
> index 8573695d..d374df0e 100644
> --- a/src/tools/disks.rs
> +++ b/src/tools/disks.rs
> @@ -46,6 +46,14 @@ pub struct DiskManage {
>      mounted_devices: OnceCell<HashSet<dev_t>>,
>  }
>  
> +/// Information for a device as returned by lsblk.

#[derive(Deserialize)]

> +pub struct LsblkInfo {
> +    /// Path to the device.
> +    path: String,
> +    /// Partition type GUID.

#[serde(rename = "parttype")]

> +    partition_type: Option<String>,
> +}
> +
>  impl DiskManage {
>      /// Create a new disk management context.
>      pub fn new() -> Arc<Self> {
> @@ -556,30 +564,31 @@ pub struct BlockDevStat {
>  }
>  
>  /// Use lsblk to read partition type uuids.
> -pub fn get_partition_type_info() -> Result<HashMap<String, Vec<String>>, Error> {
> +pub fn get_lsblk_info() -> Result<Vec<LsblkInfo>, Error> {
>  
>      let mut command = std::process::Command::new("lsblk");
>      command.args(&["--json", "-o", "path,parttype"]);
>  
>      let output = crate::tools::run_command(command, None)?;
>  
> -    let mut res: HashMap<String, Vec<String>> = HashMap::new();
> -
>      let output: serde_json::Value = output.parse()?;

        let mut output: serde_json::Value = output.parse()?;
        Ok(serde_json::from_value(output["blockdevices"].take())?)
    }

And pretty much skip the rest of the function ;-)

> +
> +    let mut res = vec![];
> +
>      if let Some(list) = output["blockdevices"].as_array() {
>          for info in list {
>              let path = match info["path"].as_str() {
> -                Some(p) => p,
> -                None => continue,
> -            };
> -            let partition_type = match info["parttype"].as_str() {
> -                Some(t) => t.to_owned(),
> +                Some(p) => p.to_string(),
>                  None => continue,
>              };
> -            let devices = res.entry(partition_type).or_insert(Vec::new());
> -            devices.push(path.to_string());
> +            let partition_type = info["parttype"].as_str().map(|t| t.to_string());
> +            res.push(LsblkInfo {
> +                path,
> +                partition_type,
> +            });
>          }
>      }
> +
>      Ok(res)
>  }
>  
> @@ -736,14 +745,14 @@ pub fn get_disks(
>  
>      let disk_manager = DiskManage::new();
>  
> -    let partition_type_map = get_partition_type_info()?;
> +    let lsblk_info = get_lsblk_info()?;
>  
> -    let zfs_devices = zfs_devices(&partition_type_map, None).or_else(|err| -> Result<HashSet<u64>, Error> {
> +    let zfs_devices = zfs_devices(&lsblk_info, None).or_else(|err| -> Result<HashSet<u64>, Error> {
>          eprintln!("error getting zfs devices: {}", err);
>          Ok(HashSet::new())
>      })?;
>  
> -    let lvm_devices = get_lvm_devices(&partition_type_map)?;
> +    let lvm_devices = get_lvm_devices(&lsblk_info)?;
>  
>      // fixme: ceph journals/volumes
>  
> diff --git a/src/tools/disks/lvm.rs b/src/tools/disks/lvm.rs
> index 1e8f825d..1a81cea0 100644
> --- a/src/tools/disks/lvm.rs
> +++ b/src/tools/disks/lvm.rs
> @@ -1,10 +1,12 @@
> -use std::collections::{HashSet, HashMap};
> +use std::collections::HashSet;
>  use std::os::unix::fs::MetadataExt;
>  
>  use anyhow::{Error};
>  use serde_json::Value;
>  use lazy_static::lazy_static;
>  
> +use super::LsblkInfo;
> +
>  lazy_static!{
>      static ref LVM_UUIDS: HashSet<&'static str> = {
>          let mut set = HashSet::new();
> @@ -17,7 +19,7 @@ lazy_static!{
>  ///
>  /// The set is indexed by using the unix raw device number (dev_t is u64)
>  pub fn get_lvm_devices(
> -    partition_type_map: &HashMap<String, Vec<String>>,
> +    lsblk_info: &[LsblkInfo],
>  ) -> Result<HashSet<u64>, Error> {
>  
>      const PVS_BIN_PATH: &str = "pvs";
> @@ -29,12 +31,12 @@ pub fn get_lvm_devices(
>  
>      let mut device_set: HashSet<u64> = HashSet::new();
>  
> -    for device_list in partition_type_map.iter()
> -        .filter_map(|(uuid, list)| if LVM_UUIDS.contains(uuid.as_str()) { Some(list) } else { None })
> -    {
> -        for device in device_list {
> -            let meta = std::fs::metadata(device)?;
> -            device_set.insert(meta.rdev());
> +    for info in lsblk_info.iter() {
> +        if let Some(partition_type) = &info.partition_type {
> +            if LVM_UUIDS.contains(partition_type.as_str()) {
> +                let meta = std::fs::metadata(&info.path)?;
> +                device_set.insert(meta.rdev());
> +            }
>          }
>      }
>  
> diff --git a/src/tools/disks/zfs.rs b/src/tools/disks/zfs.rs
> index e0084939..55e0aa30 100644
> --- a/src/tools/disks/zfs.rs
> +++ b/src/tools/disks/zfs.rs
> @@ -1,5 +1,5 @@
>  use std::path::PathBuf;
> -use std::collections::{HashMap, HashSet};
> +use std::collections::HashSet;
>  use std::os::unix::fs::MetadataExt;
>  
>  use anyhow::{bail, Error};
> @@ -67,12 +67,11 @@ pub fn zfs_pool_stats(pool: &OsStr) -> Result<Option<BlockDevStat>, Error> {
>      Ok(Some(stat))
>  }
>  
> -
>  /// Get set of devices used by zfs (or a specific zfs pool)
>  ///
>  /// The set is indexed by using the unix raw device number (dev_t is u64)
>  pub fn zfs_devices(
> -    partition_type_map: &HashMap<String, Vec<String>>,
> +    lsblk_info: &[LsblkInfo],
>      pool: Option<String>,
>  ) -> Result<HashSet<u64>, Error> {
>  
> @@ -86,12 +85,12 @@ pub fn zfs_devices(
>          }
>      }
>  
> -    for device_list in partition_type_map.iter()
> -        .filter_map(|(uuid, list)| if ZFS_UUIDS.contains(uuid.as_str()) { Some(list) } else { None })
> -    {
> -        for device in device_list {
> -            let meta = std::fs::metadata(device)?;
> -            device_set.insert(meta.rdev());
> +    for info in lsblk_info.iter() {
> +        if let Some(partition_type) = &info.partition_type {
> +            if ZFS_UUIDS.contains(partition_type.as_str()) {
> +                let meta = std::fs::metadata(&info.path)?;
> +                device_set.insert(meta.rdev());
> +            }
>          }
>      }
>  
> -- 
> 2.20.1