From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DCEA284C94 for ; Wed, 15 Dec 2021 16:16:45 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D507814FE0 for ; Wed, 15 Dec 2021 16:16:45 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 0CBFF14FD7 for ; Wed, 15 Dec 2021 16:16:45 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id CC55B451FD for ; Wed, 15 Dec 2021 16:16:44 +0100 (CET) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Wed, 15 Dec 2021 16:16:43 +0100 Message-Id: <20211215151643.1626566-1-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.173 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup] fix #3743: extract zfs dataset io stats from /proc/spl/kstat/zfs/POOL/objset-* X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Dec 2021 15:16:45 -0000 Recently, ZFS removed the pool global io stats from /proc/spl/kstat/zfs/POOL/io with no replacement. To gather stats about the datastores, access now the objset specific entries there. To be able to make that efficient, cache a map of dataset <-> obset ids, so that we do not have to parse all files each time. We update the cache each time we try to get the info for a dataset where we do not have a mapping. We cannot update it on datastore add/remove since that happens in the proxmox-backup daemon, while we need the info here in proxmox-backup-proxy. Sadly with this we lose the io wait metric, but it seems that this is no longer tracked in zfs at all, so nothing we can do for that. Signed-off-by: Dominik Csapak --- the iowait graph is still shown by default, did not have the time to come up with a mechanism to hide that for zfs backed datastores yet, but would be independet of this patch anyway src/bin/proxmox-backup-proxy.rs | 21 +++--- src/tools/disks/zfs.rs | 123 +++++++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 19 deletions(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 07a53687..b6be080c 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -62,8 +62,7 @@ use proxmox_backup::tools::{ PROXMOX_BACKUP_TCP_KEEPALIVE_TIME, disks::{ DiskManage, - zfs_pool_stats, - get_pool_from_dataset, + zfs_dataset_stats, }, }; @@ -1071,16 +1070,16 @@ fn gather_disk_stats(disk_manager: Arc, path: &Path, rrd_prefix: &st Ok(None) => {}, Ok(Some((fs_type, device, source))) => { let mut device_stat = None; - match fs_type.as_str() { - "zfs" => { - if let Some(source) = source { - let pool = get_pool_from_dataset(&source).unwrap_or(&source); - match zfs_pool_stats(pool) { - Ok(stat) => device_stat = stat, - Err(err) => eprintln!("zfs_pool_stats({:?}) failed - {}", pool, err), - } + match (fs_type.as_str(), source) { + ("zfs", Some(source)) => match source.into_string() { + Ok(dataset) => match zfs_dataset_stats(&dataset) { + Ok(stat) => device_stat = Some(stat), + Err(err) => eprintln!("zfs_dataset_stats({:?}) failed - {}", dataset, err), + }, + Err(source) => { + eprintln!("zfs_pool_stats({:?}) failed - invalid characters", source) } - } + }, _ => { if let Ok(disk) = disk_manager.clone().disk_by_dev_num(device.into_dev_t()) { match disk.read_stat() { diff --git a/src/tools/disks/zfs.rs b/src/tools/disks/zfs.rs index 2f18919d..522a04b5 100644 --- a/src/tools/disks/zfs.rs +++ b/src/tools/disks/zfs.rs @@ -1,10 +1,13 @@ use std::path::PathBuf; use std::collections::HashSet; use std::os::unix::fs::MetadataExt; +use std::sync::{Arc, Mutex}; use anyhow::{bail, Error}; use lazy_static::lazy_static; +use proxmox_schema::const_regex; + use super::*; lazy_static!{ @@ -16,15 +19,12 @@ lazy_static!{ }; } -/// returns pool from dataset path of the form 'rpool/ROOT/pbs-1' -pub fn get_pool_from_dataset(dataset: &OsStr) -> Option<&OsStr> { - if let Some(dataset) = dataset.to_str() { - if let Some(idx) = dataset.find('/') { - return Some(&dataset[0..idx].as_ref()); - } +fn get_pool_from_dataset(dataset: &str) -> &str { + if let Some(idx) = dataset.find('/') { + &dataset[0..idx].as_ref() + } else { + dataset.as_ref() } - - None } /// Returns kernel IO-stats for zfs pools @@ -97,3 +97,110 @@ pub fn zfs_devices( Ok(device_set) } +const ZFS_KSTAT_BASE_PATH: &str = "/proc/spl/kstat/zfs"; +const_regex! { + OBJSET_REGEX = r"^objset-0x[a-fA-F0-9]+$"; +} + +lazy_static::lazy_static! { + pub static ref ZFS_DATASET_OBJSET_MAP: Arc>> = + Arc::new(Mutex::new(HashMap::new())); +} + +// parses /proc/spl/kstat/zfs/POOL/objset-ID files +// they have the following format: +// +// 0 0 0x00 0 0000 00000000000 000000000000000000 +// name type data +// dataset_name 7 pool/dataset +// writes 4 0 +// nwritten 4 0 +// reads 4 0 +// nread 4 0 +// nunlinks 4 0 +// nunlinked 4 0 +// +// we are only interested in the dataset_name, writes, nwrites, reads and nread +fn parse_objset_stat(pool: &str, objset_id: &str) -> Result<(String, BlockDevStat), Error> { + let path = PathBuf::from(format!("{}/{}/{}", ZFS_KSTAT_BASE_PATH, pool, objset_id)); + + let text = match proxmox_sys::fs::file_read_optional_string(&path)? { + Some(text) => text, + None => bail!("could not parse 'objset-{}' stat file", objset_id), + }; + + let mut dataset_name = String::new(); + let mut stat = BlockDevStat { + read_sectors: 0, + write_sectors: 0, + read_ios: 0, + write_ios: 0, + io_ticks: 0, + }; + + for (i, line) in text.lines().enumerate() { + if i < 2 { + continue; + } + + let mut parts = line.split_ascii_whitespace(); + let name = parts.next(); + parts.next(); // discard type + let value = parts.next().ok_or_else(|| format_err!("no value found"))?; + match name { + Some("dataset_name") => dataset_name = value.to_string(), + Some("writes") => stat.write_ios = u64::from_str_radix(value, 10).unwrap_or(0), + Some("nwritten") => { + stat.write_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512 + } + Some("reads") => stat.read_ios = u64::from_str_radix(value, 10).unwrap_or(0), + Some("nread") => stat.read_sectors = u64::from_str_radix(value, 10).unwrap_or(0) / 512, + _ => {} + } + } + + Ok((dataset_name, stat)) +} + + +fn get_mapping(dataset: &str) -> Option<(String, String)> { + ZFS_DATASET_OBJSET_MAP + .lock() + .unwrap() + .get(dataset) + .map(|c| c.to_owned()) +} + +/// Updates the dataset <-> objset_map +pub(crate) fn update_zfs_objset_map(pool: &str) -> Result<(), Error> { + let mut map = ZFS_DATASET_OBJSET_MAP.lock().unwrap(); + map.clear(); + let path = PathBuf::from(format!("{}/{}", ZFS_KSTAT_BASE_PATH, pool)); + + proxmox_sys::fs::scandir( + libc::AT_FDCWD, + &path, + &OBJSET_REGEX, + |_l2_fd, filename, _type| { + let (name, _) = parse_objset_stat(pool, filename)?; + map.insert(name, (pool.to_string(), filename.to_string())); + Ok(()) + }, + )?; + + Ok(()) +} + +/// Gets io stats for the dataset from /proc/spl/kstat/zfs/POOL/objset-ID +pub(crate) fn zfs_dataset_stats(dataset: &str) -> Result { + let mut mapping = get_mapping(dataset); + if mapping.is_none() { + let pool = get_pool_from_dataset(dataset); + update_zfs_objset_map(pool)?; + mapping = get_mapping(dataset); + } + let (pool, objset_id) = + mapping.ok_or_else(|| format_err!("could not find objset id for dataset"))?; + let (_, stat) = parse_objset_stat(&pool, &objset_id)?; + Ok(stat) +} -- 2.30.2