From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id F2BEA84B1B for ; Wed, 15 Dec 2021 10:20:35 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A05D0FBBA for ; Wed, 15 Dec 2021 10:20:05 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id AD264FABC for ; Wed, 15 Dec 2021 10:20:00 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id F3103451E9 for ; Wed, 15 Dec 2021 10:19:54 +0100 (CET) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Wed, 15 Dec 2021 10:19:51 +0100 Message-Id: <20211215091952.1490583-9-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211215091952.1490583-1-d.csapak@proxmox.com> References: <20211215091952.1490583-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.174 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v2 5/6] proxmox-backup-proxy: send metrics to configured metrics server X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Dec 2021 09:20:36 -0000 and keep the data as similar as possible to pve (tags/fields) datastores get their own 'object' type and reside in the "blockstat" measurement Signed-off-by: Dominik Csapak --- src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++- 1 file changed, 136 insertions(+), 3 deletions(-) diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs index f1a3b7d3..aa95a4cf 100644 --- a/src/bin/proxmox-backup-proxy.rs +++ b/src/bin/proxmox-backup-proxy.rs @@ -23,11 +23,13 @@ use proxmox_sys::linux::{ }; use proxmox_sys::fs::{CreateOptions, DiskUsage}; use proxmox_lang::try_block; +use proxmox_metrics::MetricsData; use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation}; use proxmox_http::client::{RateLimitedStream, ShareableRateLimit}; use proxmox_sys::{task_log, task_warn}; use proxmox_sys::logrotate::LogRotate; +use pbs_config::metrics::get_metric_server_connections; use pbs_datastore::DataStore; use proxmox_rest_server::{ @@ -948,16 +950,127 @@ async fn run_stat_generator() { } }; - let rrd_future = tokio::task::spawn_blocking(move || { - rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); - rrd_sync_journal(); + let rrd_future = tokio::task::spawn_blocking({ + let stats = Arc::clone(&stats); + move || { + rrd_update_host_stats_sync(&stats.0, &stats.1, &stats.2); + rrd_sync_journal(); + } }); + let metrics_future = send_data_to_metric_servers(stats); + + let (rrd_res, metrics_res) = join!(rrd_future, metrics_future); + if let Err(err) = rrd_res { + log::error!("rrd update panicked: {}", err); + } + if let Err(err) = metrics_res { + log::error!("error during metrics sending: {}", err); + } tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; } +} + +async fn send_data_to_metric_servers( + stats: Arc<(HostStats, DiskStat, Vec)>, +) -> Result<(), Error> { + let (config, _digest) = pbs_config::metrics::config()?; + let (futures, channels, names) = get_metric_server_connections(config)?; + + if futures.is_empty() { + return Ok(()); + } + + let sending_handle = tokio::spawn(async move { + future::join_all(futures).await + }); + + let ctime = proxmox_time::epoch_i64(); + let nodename = proxmox_sys::nodename(); + + let mut values = Vec::new(); + + let mut cpuvalue = match &stats.0.proc { + Some(stat) => serde_json::to_value(stat)?, + None => json!({}), + }; + + if let Some(loadavg) = &stats.0.load { + cpuvalue["avg1"] = Value::from(loadavg.0); + cpuvalue["avg5"] = Value::from(loadavg.1); + cpuvalue["avg15"] = Value::from(loadavg.2); + } + + values.push(Arc::new(MetricsData::new( + "cpustat", + &[("object", "host"), ("host", nodename)], + ctime, + cpuvalue, + )?)); + + if let Some(stat) = &stats.0.meminfo { + values.push(Arc::new(MetricsData::new( + "memory", + &[("object", "host"), ("host", nodename)], + ctime, + stat, + )?)); + } + if let Some(netdev) = &stats.0.net { + for item in netdev { + values.push(Arc::new(MetricsData::new( + "nics", + &[ + ("object", "host"), + ("host", nodename), + ("instance", &item.device), + ], + ctime, + item, + )?)); + } + } + + values.push(Arc::new(MetricsData::new( + "blockstat", + &[("object", "host"), ("host", nodename)], + ctime, + stats.1.to_value(), + )?)); + + for datastore in stats.2.iter() { + values.push(Arc::new(MetricsData::new( + "blockstat", + &[ + ("object", "datastore"), + ("nodename", nodename), + ("datastore", &datastore.name), + ], + ctime, + datastore.to_value(), + )?)); + } + + let results = proxmox_metrics::send_data_to_channels(&values, &channels).await; + for (i, res) in results.into_iter().enumerate() { + if let Err(err) = res { + log::error!("error sending into channel of {}: {}", names[i], err); + } + } + + drop(channels); + + let results = sending_handle.await?; + for (i, res) in results.into_iter().enumerate() { + if let Err(err) = res { + log::error!("error sending to metric server {}: {}", names[i], err); + } + } + + Ok(()) } struct HostStats { @@ -973,6 +1086,26 @@ struct DiskStat { dev: Option, } +impl DiskStat { + fn to_value(&self) -> Value { + let mut value = json!({}); + if let Some(usage) = &self.usage { + value["total"] = Value::from(usage.total); + value["used"] = Value::from(usage.used); + value["avail"] = Value::from(usage.available); + } + + if let Some(dev) = &self.dev { + value["read_ios"] = Value::from(dev.read_ios); + value["read_bytes"] = Value::from(dev.read_sectors * 512); + value["write_ios"] = Value::from(dev.write_ios); + value["write_bytes"] = Value::from(dev.write_sectors * 512); + value["io_ticks"] = Value::from(dev.io_ticks / 1000); + } + value + } +} + fn collect_host_stats_sync() -> HostStats { use proxmox_sys::linux::procfs::{ read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg}; -- 2.30.2