From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by lists.proxmox.com (Postfix) with ESMTPS id 906226012B
 for <pbs-devel@lists.proxmox.com>; Mon, 10 Jan 2022 12:08:47 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
 by firstgate.proxmox.com (Proxmox) with ESMTP id 7C6E760E
 for <pbs-devel@lists.proxmox.com>; Mon, 10 Jan 2022 12:08:17 +0100 (CET)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
 [94.136.29.106])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256)
 (No client certificate requested)
 by firstgate.proxmox.com (Proxmox) with ESMTPS id E6357602
 for <pbs-devel@lists.proxmox.com>; Mon, 10 Jan 2022 12:08:15 +0100 (CET)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
 by proxmox-new.maurer-it.com (Proxmox) with ESMTP id B229145062
 for <pbs-devel@lists.proxmox.com>; Mon, 10 Jan 2022 12:08:15 +0100 (CET)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Mon, 10 Jan 2022 12:08:14 +0100
Message-Id: <20220110110814.2784724-1-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results:  0
 AWL 0.165 Adjusted score from AWL reputation of From: address
 BAYES_00                 -1.9 Bayes spam probability is 0 to 1%
 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
 RCVD_IN_DNSWL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to DNSWL was
 blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block
 for more information.
 SPF_HELO_NONE           0.001 SPF: HELO does not publish an SPF Record
 SPF_PASS               -0.001 SPF: sender matches SPF record
 URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
 http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
 information. [zfs.rs, proxmox-backup-proxy.rs]
Subject: [pbs-devel] [PATCH proxmox-backup v2] fix #3743: extract zfs
 dataset io stats from /proc/spl/kstat/zfs/POOL/objset-*
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
 <pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Mon, 10 Jan 2022 11:08:47 -0000

Recently, ZFS removed the pool global io stats from
/proc/spl/kstat/zfs/POOL/io with no replacement.

To gather stats about the datastores, access now the objset specific
entries there. To be able to make that efficient, cache a map of
dataset <-> obset ids, so that we do not have to parse all files each time.

We update the cache each time we try to get the info for a dataset
where we do not have a mapping.

We cannot update it on datastore add/remove since that happens in the
proxmox-backup daemon, while we need the info here in proxmox-backup-proxy.

Sadly with this we lose the io wait metric, but it seems that this is no
longer tracked in zfs at all, so nothing we can do for that.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes from v1:
* rebased on master
* made zfs_dataset_stats 'pub'

 src/bin/proxmox-backup-proxy.rs |  21 +++---
 src/tools/disks/zfs.rs          | 123 +++++++++++++++++++++++++++++---
 2 files changed, 125 insertions(+), 19 deletions(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 287cff0a..903dd9bc 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -62,8 +62,7 @@ use proxmox_backup::tools::{
     PROXMOX_BACKUP_TCP_KEEPALIVE_TIME,
     disks::{
         DiskManage,
-        zfs_pool_stats,
-        get_pool_from_dataset,
+        zfs_dataset_stats,
     },
 };
 
@@ -1072,16 +1071,16 @@ fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &st
         Ok(None) => {},
         Ok(Some((fs_type, device, source))) => {
             let mut device_stat = None;
-            match fs_type.as_str() {
-                "zfs" => {
-                    if let Some(source) = source {
-                        let pool = get_pool_from_dataset(&source).unwrap_or(&source);
-                        match zfs_pool_stats(pool) {
-                            Ok(stat) => device_stat = stat,
-                            Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err),
-                        }
+            match (fs_type.as_str(), source) {
+                ("zfs", Some(source)) => match source.into_string() {
+                    Ok(dataset) => match zfs_dataset_stats(&dataset) {
+                        Ok(stat) => device_stat = Some(stat),
+                        Err(err) => eprintln!("zfs_dataset_stats({:?}) failed - {}", dataset, err),
+                    },
+                    Err(source) => {
+                        eprintln!("zfs_pool_stats({:?}) failed - invalid characters", source)
                     }
-                }
+                },
                 _ => {
                     if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) {
                         match disk.read_stat() {
diff --git a/src/tools/disks/zfs.rs b/src/tools/disks/zfs.rs
index 3da4b603..25a3a709 100644
--- a/src/tools/disks/zfs.rs
+++ b/src/tools/disks/zfs.rs
@@ -1,10 +1,13 @@
 use std::path::PathBuf;
 use std::collections::HashSet;
 use std::os::unix::fs::MetadataExt;
+use std::sync::{Arc, Mutex};
 
 use anyhow::{bail, Error};
 use lazy_static::lazy_static;
 
+use proxmox_schema::const_regex;
+
 use super::*;
 
 lazy_static!{
@@ -16,15 +19,12 @@ lazy_static!{
     };
 }
 
-/// returns pool from dataset path of the form 'rpool/ROOT/pbs-1'
-pub fn get_pool_from_dataset(dataset: &OsStr) -> Option<&OsStr> {
-    if let Some(dataset) = dataset.to_str() {
-        if let Some(idx) = dataset.find('/') {
-            return Some(dataset[0..idx].as_ref());
-        }
+fn get_pool_from_dataset(dataset: &str) -> &str {
+    if let Some(idx) = dataset.find('/') {
+        &dataset[0..idx].as_ref()
+    } else {
+        dataset.as_ref()
     }
-
-    None
 }
 
 /// Returns kernel IO-stats for zfs pools
@@ -97,3 +97,110 @@ pub fn zfs_devices(
     Ok(device_set)
 }
 
+const ZFS_KSTAT_BASE_PATH: &str = "/proc/spl/kstat/zfs";
+const_regex! {
+    OBJSET_REGEX = r"^objset-0x[a-fA-F0-9]+$";
+}
+
+lazy_static::lazy_static! {
+    pub static ref ZFS_DATASET_OBJSET_MAP: Arc<Mutex<HashMap<String, (String, String)>>> =
+        Arc::new(Mutex::new(HashMap::new()));
+}
+
+// parses /proc/spl/kstat/zfs/POOL/objset-ID files
+// they have the following format:
+//
+// 0 0 0x00 0 0000 00000000000 000000000000000000
+// name                            type data
+// dataset_name                    7    pool/dataset
+// writes                          4    0
+// nwritten                        4    0
+// reads                           4    0
+// nread                           4    0
+// nunlinks                        4    0
+// nunlinked                       4    0
+//
+// we are only interested in the dataset_name, writes, nwrites, reads and nread
+fn parse_objset_stat(pool: &str, objset_id: &str) -> Result<(String, BlockDevStat), Error> {
+    let path = PathBuf::from(format!("{}/{}/{}", ZFS_KSTAT_BASE_PATH, pool, objset_id));
+
+    let text = match proxmox_sys::fs::file_read_optional_string(&path)? {
+        Some(text) => text,
+        None => bail!("could not parse 'objset-{}' stat file", objset_id),
+    };
+
+    let mut dataset_name = String::new();
+    let mut stat = BlockDevStat {
+        read_sectors: 0,
+        write_sectors: 0,
+        read_ios: 0,
+        write_ios: 0,
+        io_ticks: 0,
+    };
+
+    for (i, line) in text.lines().enumerate() {
+        if i < 2 {
+            continue;
+        }
+
+        let mut parts = line.split_ascii_whitespace();
+        let name = parts.next();
+        parts.next(); // discard type
+        let value = parts.next().ok_or_else(|| format_err!("no value found"))?;
+        match name {
+            Some("dataset_name") => dataset_name = value.to_string(),
+            Some("writes") => stat.write_ios = u64::from_str_radix(value, 10).unwrap_or(0),
+            Some("nwritten") => {
+                stat.write_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512
+            }
+            Some("reads") => stat.read_ios = u64::from_str_radix(value, 10).unwrap_or(0),
+            Some("nread") => stat.read_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512,
+            _ => {}
+        }
+    }
+
+    Ok((dataset_name, stat))
+}
+
+
+fn get_mapping(dataset: &str) -> Option<(String, String)> {
+    ZFS_DATASET_OBJSET_MAP
+        .lock()
+        .unwrap()
+        .get(dataset)
+        .map(|c| c.to_owned())
+}
+
+/// Updates the dataset <-> objset_map
+pub(crate) fn update_zfs_objset_map(pool: &str) -> Result<(), Error> {
+    let mut map = ZFS_DATASET_OBJSET_MAP.lock().unwrap();
+    map.clear();
+    let path = PathBuf::from(format!("{}/{}", ZFS_KSTAT_BASE_PATH, pool));
+
+    proxmox_sys::fs::scandir(
+        libc::AT_FDCWD,
+        &path,
+        &OBJSET_REGEX,
+        |_l2_fd, filename, _type| {
+            let (name, _) = parse_objset_stat(pool, filename)?;
+            map.insert(name, (pool.to_string(), filename.to_string()));
+            Ok(())
+        },
+    )?;
+
+    Ok(())
+}
+
+/// Gets io stats for the dataset from /proc/spl/kstat/zfs/POOL/objset-ID
+pub fn zfs_dataset_stats(dataset: &str) -> Result<BlockDevStat, Error> {
+    let mut mapping = get_mapping(dataset);
+    if mapping.is_none() {
+        let pool = get_pool_from_dataset(dataset);
+        update_zfs_objset_map(pool)?;
+        mapping = get_mapping(dataset);
+    }
+    let (pool, objset_id) =
+        mapping.ok_or_else(|| format_err!("could not find objset id for dataset"))?;
+    let (_, stat) = parse_objset_stat(&pool, &objset_id)?;
+    Ok(stat)
+}
-- 
2.30.2