From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id AC37960C08 for ; Thu, 3 Feb 2022 10:46:13 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 976902455F for ; Thu, 3 Feb 2022 10:45:43 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id CAE4124555 for ; Thu, 3 Feb 2022 10:45:42 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9D2DB45013 for ; Thu, 3 Feb 2022 10:45:42 +0100 (CET) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Thu, 3 Feb 2022 10:45:41 +0100 Message-Id: <20220203094541.2739465-1-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.161 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [self.dev, 0.net, proxmox-backup-proxy.rs] Subject: [pbs-devel] [PATCH proxmox-backup v6] proxmox-backup-proxy: send metrics to configured metrics server X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Feb 2022 09:46:13 -0000 and keep the data as similar as possible to pve (tags/fields) datastores get their own 'object' type and reside in the "blockstat" measurement Signed-off-by: Dominik Csapak --- replacement for the patch 5/8 of my v5 of 'add metrics server capability' changes: * adapted to MetrcsData changes by Wolfgang src/bin/proxmox-backup-proxy.rs | 124 +++++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 3 deletions(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index 302fece9..687d4d72 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -23,11 +23,13 @@ use proxmox_sys::linux::{ }; use proxmox_sys::fs::{CreateOptions, FileSystemInformation}; use proxmox_lang::try_block; +use proxmox_metrics::MetricsData; use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation}; use proxmox_http::client::{RateLimitedStream, ShareableRateLimit}; use proxmox_sys::{task_log, task_warn}; use proxmox_sys::logrotate::LogRotate; +use pbs_config::metrics::get_metric_server_connections; use pbs_datastore::DataStore; use proxmox_rest_server::{ @@ -959,16 +961,112 @@ async fn run_stat_generator() { } }; - let rrd_future = tokio::task::spawn_blocking(move || { - rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); - rrd_sync_journal(); + let rrd_future = tokio::task::spawn_blocking({ + let stats = Arc::clone(&stats); + move || { + rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); + rrd_sync_journal(); + } }); + let metrics_future = send_data_to_metric_servers(stats); + + let (rrd_res, metrics_res) = join!(rrd_future, metrics_future); + if let Err(err) = rrd_res { + log::error!("rrd update panicked: {}", err); + } + if let Err(err) = metrics_res { + log::error!("error during metrics sending: {}", err); + } tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; } +} + +async fn send_data_to_metric_servers( + stats: Arc<(HostStats, DiskStat, Vec)>, +) -> Result<(), Error> { + let (config, _digest) = pbs_config::metrics::config()?; + let (channels, names) = get_metric_server_connections(config)?; + + if channels.is_empty() { + return Ok(()); + } + + let ctime = proxmox_time::epoch_i64(); + let nodename = proxmox_sys::nodename(); + + let mut values = Vec::new(); + let mut cpuvalue = match &stats.0.proc { + Some(stat) => serde_json::to_value(stat)?, + None => json!({}), + }; + + if let Some(loadavg) = &stats.0.load { + cpuvalue["avg1"] = Value::from(loadavg.0); + cpuvalue["avg5"] = Value::from(loadavg.1); + cpuvalue["avg15"] = Value::from(loadavg.2); + } + + values.push(Arc::new( + MetricsData::new("cpustat", ctime, cpuvalue)? + .tag("object", "host") + .tag("host", nodename), + )); + + if let Some(stat) = &stats.0.meminfo { + values.push(Arc::new( + MetricsData::new("memory", ctime, stat)? + .tag("object", "host") + .tag("host", nodename) + )); + } + + if let Some(netdev) = &stats.0.net { + for item in netdev { + values.push(Arc::new( + MetricsData::new("nics", ctime, item)? + .tag("object", "host") + .tag("host", nodename) + .tag("instance", item.device.clone()) + )); + } + } + + values.push(Arc::new( + MetricsData::new("blockstat", ctime, stats.1.to_value())? + .tag("object", "host") + .tag("host", nodename) + )); + + for datastore in stats.2.iter() { + values.push(Arc::new( + MetricsData::new("blockstat", ctime, datastore.to_value())? + .tag("object", "host") + .tag("host", nodename) + .tag("datastore", datastore.name.clone()) + )); + } + + let results = proxmox_metrics::send_data_to_channels(&values, &channels).await; + for (res, name) in results.into_iter().zip(names.iter()) { + if let Err(err) = res { + log::error!("error sending into channel of {}: {}", name, err); + } + } + + futures::future::join_all(channels.into_iter().zip(names.into_iter()).map( + |(channel, name)| async move { + if let Err(err) = channel.join().await { + log::error!("error sending to metric server {}: {}", name, err); + } + }, + )) + .await; + + Ok(()) } struct HostStats { @@ -984,6 +1082,26 @@ struct DiskStat { dev: Option, } +impl DiskStat { + fn to_value(&self) -> Value { + let mut value = json!({}); + if let Some(usage) = &self.usage { + value["total"] = Value::from(usage.total); + value["used"] = Value::from(usage.used); + value["avail"] = Value::from(usage.available); + } + + if let Some(dev) = &self.dev { + value["read_ios"] = Value::from(dev.read_ios); + value["read_bytes"] = Value::from(dev.read_sectors * 512); + value["write_ios"] = Value::from(dev.write_ios); + value["write_bytes"] = Value::from(dev.write_sectors * 512); + value["io_ticks"] = Value::from(dev.io_ticks / 1000); + } + value + } +} + fn collect_host_stats_sync() -> HostStats { use proxmox_sys::linux::procfs::{ read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg}; -- 2.30.2