From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0A5E060693 for ; Wed, 2 Feb 2022 10:50:31 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9732B19D8C for ; Wed, 2 Feb 2022 10:50:30 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 903D019D14 for ; Wed, 2 Feb 2022 10:50:26 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A8CAB4180D for ; Wed, 2 Feb 2022 10:50:22 +0100 (CET) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Wed, 2 Feb 2022 10:50:16 +0100 Message-Id: <20220202095019.1799843-8-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220202095019.1799843-1-d.csapak@proxmox.com> References: <20220202095019.1799843-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.162 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH proxmox-backup v5 5/8] proxmox-backup-proxy: send metrics to configured metrics server X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 02 Feb 2022 09:50:31 -0000 and keep the data as similar as possible to pve (tags/fields) datastores get their own 'object' type and reside in the "blockstat" measurement Signed-off-by: Dominik Csapak --- src/bin/proxmox-backup-proxy.rs | 135 +++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 302fece9..465374b7 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -23,11 +23,13 @@ use proxmox_sys::linux::{ }; use proxmox_sys::fs::{CreateOptions, FileSystemInformation}; use proxmox_lang::try_block; +use proxmox_metrics::MetricsData; use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation}; use proxmox_http::client::{RateLimitedStream, ShareableRateLimit}; use proxmox_sys::{task_log, task_warn}; use proxmox_sys::logrotate::LogRotate; +use pbs_config::metrics::get_metric_server_connections; use pbs_datastore::DataStore; use proxmox_rest_server::{ @@ -959,16 +961,123 @@ async fn run_stat_generator() { } }; - let rrd_future = tokio::task::spawn_blocking(move || { - rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); - rrd_sync_journal(); + let rrd_future = tokio::task::spawn_blocking({ + let stats = Arc::clone(&stats); + move || { + rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); + rrd_sync_journal(); + } }); + let metrics_future = send_data_to_metric_servers(stats); + + let (rrd_res, metrics_res) = join!(rrd_future, metrics_future); + if let Err(err) = rrd_res { + log::error!("rrd update panicked: {}", err); + } + if let Err(err) = metrics_res { + log::error!("error during metrics sending: {}", err); + } tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; } +} + +async fn send_data_to_metric_servers( + stats: Arc<(HostStats, DiskStat, Vec)>, +) -> Result<(), Error> { + let (config, _digest) = pbs_config::metrics::config()?; + let (channels, names) = get_metric_server_connections(config)?; + + if channels.is_empty() { + return Ok(()); + } + + let ctime = proxmox_time::epoch_i64(); + let nodename = proxmox_sys::nodename(); + + let mut values = Vec::new(); + let mut cpuvalue = match &stats.0.proc { + Some(stat) => serde_json::to_value(stat)?, + None => json!({}), + }; + + if let Some(loadavg) = &stats.0.load { + cpuvalue["avg1"] = Value::from(loadavg.0); + cpuvalue["avg5"] = Value::from(loadavg.1); + cpuvalue["avg15"] = Value::from(loadavg.2); + } + + values.push(Arc::new(MetricsData::new( + "cpustat", + &[("object", "host"), ("host", nodename)], + ctime, + cpuvalue, + )?)); + + if let Some(stat) = &stats.0.meminfo { + values.push(Arc::new(MetricsData::new( + "memory", + &[("object", "host"), ("host", nodename)], + ctime, + stat, + )?)); + } + + if let Some(netdev) = &stats.0.net { + for item in netdev { + values.push(Arc::new(MetricsData::new( + "nics", + &[ + ("object", "host"), + ("host", nodename), + ("instance", &item.device), + ], + ctime, + item, + )?)); + } + } + + values.push(Arc::new(MetricsData::new( + "blockstat", + &[("object", "host"), ("host", nodename)], + ctime, + stats.1.to_value(), + )?)); + + for datastore in stats.2.iter() { + values.push(Arc::new(MetricsData::new( + "blockstat", + &[ + ("object", "datastore"), + ("nodename", nodename), + ("datastore", &datastore.name), + ], + ctime, + datastore.to_value(), + )?)); + } + + let results = proxmox_metrics::send_data_to_channels(&values, &channels).await; + for (res, name) in results.into_iter().zip(names.iter()) { + if let Err(err) = res { + log::error!("error sending into channel of {}: {}", name, err); + } + } + + futures::future::join_all(channels.into_iter().zip(names.into_iter()).map( + |(channel, name)| async move { + if let Err(err) = channel.join().await { + log::error!("error sending to metric server {}: {}", name, err); + } + }, + )) + .await; + + Ok(()) } struct HostStats { @@ -984,6 +1093,26 @@ struct DiskStat { dev: Option, } +impl DiskStat { + fn to_value(&self) -> Value { + let mut value = json!({}); + if let Some(usage) = &self.usage { + value["total"] = Value::from(usage.total); + value["used"] = Value::from(usage.used); + value["avail"] = Value::from(usage.available); + } + + if let Some(dev) = &self.dev { + value["read_ios"] = Value::from(dev.read_ios); + value["read_bytes"] = Value::from(dev.read_sectors * 512); + value["write_ios"] = Value::from(dev.write_ios); + value["write_bytes"] = Value::from(dev.write_sectors * 512); + value["io_ticks"] = Value::from(dev.io_ticks / 1000); + } + value + } +} + fn collect_host_stats_sync() -> HostStats { use proxmox_sys::linux::procfs::{ read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg}; -- 2.30.2