all lists on lists.proxmox.com
 help / color / mirror / Atom feed
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup v2] fix #3743: extract zfs dataset io stats from /proc/spl/kstat/zfs/POOL/objset-*
Date: Mon, 10 Jan 2022 12:08:14 +0100	[thread overview]
Message-ID: <20220110110814.2784724-1-d.csapak@proxmox.com> (raw)

Recently, ZFS removed the pool global io stats from
/proc/spl/kstat/zfs/POOL/io with no replacement.

To gather stats about the datastores, access now the objset specific
entries there. To be able to make that efficient, cache a map of
dataset <-> obset ids, so that we do not have to parse all files each time.

We update the cache each time we try to get the info for a dataset
where we do not have a mapping.

We cannot update it on datastore add/remove since that happens in the
proxmox-backup daemon, while we need the info here in proxmox-backup-proxy.

Sadly with this we lose the io wait metric, but it seems that this is no
longer tracked in zfs at all, so nothing we can do for that.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes from v1:
* rebased on master
* made zfs_dataset_stats 'pub'

 src/bin/proxmox-backup-proxy.rs |  21 +++---
 src/tools/disks/zfs.rs          | 123 +++++++++++++++++++++++++++++---
 2 files changed, 125 insertions(+), 19 deletions(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 287cff0a..903dd9bc 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -62,8 +62,7 @@ use proxmox_backup::tools::{
     PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
     disks::{
         DiskManage,
-        zfs_pool_stats,
-        get_pool_from_dataset,
+        zfs_dataset_stats,
     },
 };
 
@@ -1072,16 +1071,16 @@ fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &st
         Ok(None) => {},
         Ok(Some((fs_type, device, source))) => {
             let mut device_stat = None;
-            match fs_type.as_str() {
-                "zfs" => {
-                    if let Some(source) = source {
-                        let pool = get_pool_from_dataset(&source).unwrap_or(&source);
-                        match zfs_pool_stats(pool) {
-                            Ok(stat) => device_stat = stat,
-                            Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err),
-                        }
+            match (fs_type.as_str(), source) {
+                ("zfs", Some(source)) => match source.into_string() {
+                    Ok(dataset) => match zfs_dataset_stats(&dataset) {
+                        Ok(stat) => device_stat = Some(stat),
+                        Err(err) => eprintln!("zfs_dataset_stats({:?}) failed - {}", dataset, err),
+                    },
+                    Err(source) => {
+                        eprintln!("zfs_pool_stats({:?}) failed - invalid characters", source)
                     }
-                }
+                },
                 _ => {
                     if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) {
                         match disk.read_stat() {
diff --git a/src/tools/disks/zfs.rs b/src/tools/disks/zfs.rs
index 3da4b603..25a3a709 100644
--- a/src/tools/disks/zfs.rs
+++ b/src/tools/disks/zfs.rs
@@ -1,10 +1,13 @@
 use std::path::PathBuf;
 use std::collections::HashSet;
 use std::os::unix::fs::MetadataExt;
+use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, Error};
 use lazy_static::lazy_static;
 
+use proxmox_schema::const_regex;
+
 use super::*;
 
 lazy_static!{
@@ -16,15 +19,12 @@ lazy_static!{
     };
 }
 
-/// returns pool from dataset path of the form 'rpool/ROOT/pbs-1'
-pub fn get_pool_from_dataset(dataset: &OsStr) -> Option<&OsStr> {
-    if let Some(dataset) = dataset.to_str() {
-        if let Some(idx) = dataset.find('/') {
-            return Some(dataset[0..idx].as_ref());
-        }
+fn get_pool_from_dataset(dataset: &str) -> &str {
+    if let Some(idx) = dataset.find('/') {
+        &dataset[0..idx].as_ref()
+    } else {
+        dataset.as_ref()
     }
-
-    None
 }
 
 /// Returns kernel IO-stats for zfs pools
@@ -97,3 +97,110 @@ pub fn zfs_devices(
     Ok(device_set)
 }
 
+const ZFS_KSTAT_BASE_PATH: &str = "/proc/spl/kstat/zfs";
+const_regex! {
+    OBJSET_REGEX = r"^objset-0x[a-fA-F0-9]+$";
+}
+
+lazy_static::lazy_static! {
+    pub static ref ZFS_DATASET_OBJSET_MAP: Arc<Mutex<HashMap<String, (String, String)>>> =
+        Arc::new(Mutex::new(HashMap::new()));
+}
+
+// parses /proc/spl/kstat/zfs/POOL/objset-ID files
+// they have the following format:
+//
+// 0 0 0x00 0 0000 00000000000 000000000000000000
+// name                            type data
+// dataset_name                    7    pool/dataset
+// writes                          4    0
+// nwritten                        4    0
+// reads                           4    0
+// nread                           4    0
+// nunlinks                        4    0
+// nunlinked                       4    0
+//
+// we are only interested in the dataset_name, writes, nwrites, reads and nread
+fn parse_objset_stat(pool: &str, objset_id: &str) -> Result<(String, BlockDevStat), Error> {
+    let path = PathBuf::from(format!("{}/{}/{}", ZFS_KSTAT_BASE_PATH, pool, objset_id));
+
+    let text = match proxmox_sys::fs::file_read_optional_string(&path)? {
+        Some(text) => text,
+        None => bail!("could not parse 'objset-{}' stat file", objset_id),
+    };
+
+    let mut dataset_name = String::new();
+    let mut stat = BlockDevStat {
+        read_sectors: 0,
+        write_sectors: 0,
+        read_ios: 0,
+        write_ios: 0,
+        io_ticks: 0,
+    };
+
+    for (i, line) in text.lines().enumerate() {
+        if i < 2 {
+            continue;
+        }
+
+        let mut parts = line.split_ascii_whitespace();
+        let name = parts.next();
+        parts.next(); // discard type
+        let value = parts.next().ok_or_else(|| format_err!("no value found"))?;
+        match name {
+            Some("dataset_name") => dataset_name = value.to_string(),
+            Some("writes") => stat.write_ios = u64::from_str_radix(value, 10).unwrap_or(0),
+            Some("nwritten") => {
+                stat.write_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512
+            }
+            Some("reads") => stat.read_ios = u64::from_str_radix(value, 10).unwrap_or(0),
+            Some("nread") => stat.read_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512,
+            _ => {}
+        }
+    }
+
+    Ok((dataset_name, stat))
+}
+
+
+fn get_mapping(dataset: &str) -> Option<(String, String)> {
+    ZFS_DATASET_OBJSET_MAP
+        .lock()
+        .unwrap()
+        .get(dataset)
+        .map(|c| c.to_owned())
+}
+
+/// Updates the dataset <-> objset_map
+pub(crate) fn update_zfs_objset_map(pool: &str) -> Result<(), Error> {
+    let mut map = ZFS_DATASET_OBJSET_MAP.lock().unwrap();
+    map.clear();
+    let path = PathBuf::from(format!("{}/{}", ZFS_KSTAT_BASE_PATH, pool));
+
+    proxmox_sys::fs::scandir(
+        libc::AT_FDCWD,
+        &path,
+        &OBJSET_REGEX,
+        |_l2_fd, filename, _type| {
+            let (name, _) = parse_objset_stat(pool, filename)?;
+            map.insert(name, (pool.to_string(), filename.to_string()));
+            Ok(())
+        },
+    )?;
+
+    Ok(())
+}
+
+/// Gets io stats for the dataset from /proc/spl/kstat/zfs/POOL/objset-ID
+pub fn zfs_dataset_stats(dataset: &str) -> Result<BlockDevStat, Error> {
+    let mut mapping = get_mapping(dataset);
+    if mapping.is_none() {
+        let pool = get_pool_from_dataset(dataset);
+        update_zfs_objset_map(pool)?;
+        mapping = get_mapping(dataset);
+    }
+    let (pool, objset_id) =
+        mapping.ok_or_else(|| format_err!("could not find objset id for dataset"))?;
+    let (_, stat) = parse_objset_stat(&pool, &objset_id)?;
+    Ok(stat)
+}
-- 
2.30.2





             reply	other threads:[~2022-01-10 11:08 UTC|newest]

Thread overview: 2+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-01-10 11:08 Dominik Csapak [this message]
2022-01-11  8:04 ` [pbs-devel] applied: " Dietmar Maurer

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20220110110814.2784724-1-d.csapak@proxmox.com \
    --to=d.csapak@proxmox.com \
    --cc=pbs-devel@lists.proxmox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal