From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy: send metrics to configured metrics server
Date: Tue, 14 Dec 2021 13:24:11 +0100 [thread overview]
Message-ID: <20211214122412.4077902-9-d.csapak@proxmox.com> (raw)
In-Reply-To: <20211214122412.4077902-1-d.csapak@proxmox.com>
and keep the data as similar as possible to pve (tags/fields)
datastores get their own 'object' type and reside in the "blockstat"
measurement
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++-
1 file changed, 138 insertions(+), 1 deletion(-)
diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 2700fabf..fbb782dd 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -23,11 +23,13 @@ use proxmox_sys::linux::{
};
use proxmox_sys::fs::{CreateOptions, DiskUsage};
use proxmox_lang::try_block;
+use proxmox_metrics::MetricsData;
use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation};
use proxmox_http::client::{RateLimitedStream, ShareableRateLimit};
use proxmox_sys::{task_log, task_warn};
use proxmox_sys::logrotate::LogRotate;
+use pbs_config::metrics::get_metric_server_connections;
use pbs_datastore::DataStore;
use proxmox_rest_server::{
@@ -948,16 +950,131 @@ async fn run_stat_generator() {
}
};
+ let hoststats2 = hoststats.clone();
+ let hostdisk2 = hostdisk.clone();
+ let datastores2 = datastores.clone();
let rrd_future = tokio::task::spawn_blocking(move || {
- rrd_update_host_stats_sync(&hoststats, &hostdisk, &datastores);
+ rrd_update_host_stats_sync(&hoststats2, &hostdisk2, &datastores2);
rrd_sync_journal();
});
+ let metrics_future = send_data_to_metric_servers(hoststats, hostdisk, datastores);
+
+ let (rrd_res, metrics_res) = join!(rrd_future, metrics_future);
+ if let Err(err) = rrd_res {
+ log::error!("rrd update panicked: {}", err);
+ }
+ if let Err(err) = metrics_res {
+ log::error!("error during metrics sending: {}", err);
+ }
tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
}
+}
+
+async fn send_data_to_metric_servers(
+ host: Arc<HostStats>,
+ hostdisk: Arc<DiskStat>,
+ datastores: Arc<Vec<DiskStat>>,
+) -> Result<(), Error> {
+ let (config, _digest) = pbs_config::metrics::config()?;
+ let (futures, channels, names) = get_metric_server_connections(config)?;
+
+ if futures.is_empty() {
+ return Ok(());
+ }
+
+ let names2 = names.clone();
+ let sending_handle = tokio::spawn(async move {
+ for (i, res) in future::join_all(futures).await.into_iter().enumerate() {
+ if let Err(err) = res {
+ eprintln!("ERROR '{}': {}", names2[i], err);
+ }
+ }
+ });
+
+ let ctime = proxmox_time::epoch_i64();
+ let nodename = proxmox_sys::nodename();
+
+ let mut values = Vec::new();
+
+ let mut cpuvalue = json!({});
+ if let Some(stat) = &host.proc {
+ for (key, value) in serde_json::to_value(stat)?.as_object().unwrap().iter() {
+ cpuvalue[key.clone()] = value.clone();
+ }
+ }
+
+ if let Some(loadavg) = &host.load {
+ cpuvalue["avg1"] = Value::from(loadavg.0);
+ cpuvalue["avg5"] = Value::from(loadavg.1);
+ cpuvalue["avg15"] = Value::from(loadavg.2);
+ }
+ values.push(Arc::new(MetricsData::new(
+ "cpustat",
+ &[("object", "host"), ("host", nodename)],
+ ctime,
+ cpuvalue,
+ )?));
+
+ if let Some(stat) = &host.meminfo {
+ values.push(Arc::new(MetricsData::new(
+ "memory",
+ &[("object", "host"), ("host", nodename)],
+ ctime,
+ stat,
+ )?));
+ }
+
+ if let Some(netdev) = &host.net {
+ for item in netdev {
+ values.push(Arc::new(MetricsData::new(
+ "nics",
+ &[
+ ("object", "host"),
+ ("host", nodename),
+ ("instance", &item.device),
+ ],
+ ctime,
+ item,
+ )?));
+ }
+ }
+
+ values.push(Arc::new(MetricsData::new(
+ "blockstat",
+ &[("object", "host"), ("host", nodename)],
+ ctime,
+ hostdisk.to_value(),
+ )?));
+
+ for datastore in datastores.iter() {
+ values.push(Arc::new(MetricsData::new(
+ "blockstat",
+ &[
+ ("object", "datastore"),
+ ("nodename", nodename),
+ ("datastore", &datastore.name),
+ ],
+ ctime,
+ datastore.to_value(),
+ )?));
+ }
+
+ let results = proxmox_metrics::send_data_to_channels(&values, &channels).await;
+ for (i, res) in results.into_iter().enumerate() {
+ if let Err(err) = res {
+ log::error!("error sending to {}: {}", names[i], err);
+ }
+ }
+
+ drop(channels);
+
+ sending_handle.await?;
+
+ Ok(())
}
struct HostStats {
@@ -973,6 +1090,26 @@ struct DiskStat {
dev: Option<BlockDevStat>,
}
+impl DiskStat {
+ fn to_value(&self) -> Value {
+ let mut value = json!({});
+ if let Some(usage) = &self.usage {
+ value["total"] = Value::from(usage.total);
+ value["used"] = Value::from(usage.used);
+ value["avail"] = Value::from(usage.available);
+ }
+
+ if let Some(dev) = &self.dev {
+ value["read_ios"] = Value::from(dev.read_ios);
+ value["read_bytes"] = Value::from(dev.read_sectors * 512);
+ value["write_ios"] = Value::from(dev.write_ios);
+ value["write_bytes"] = Value::from(dev.write_sectors * 512);
+ value["io_ticks"] = Value::from(dev.io_ticks / 1000);
+ }
+ value
+ }
+}
+
fn collect_host_stats_sync() -> HostStats {
use proxmox_sys::linux::procfs::{
read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg};
--
2.30.2
next prev parent reply other threads:[~2021-12-14 12:24 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-12-14 12:24 [pbs-devel] [PATCH proxmox/proxmox-backup] add metrics server capability Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox 1/3] proxmox-sys: make some structs serializable Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox 2/3] proxmox-sys: add DiskUsage struct and helper Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox 3/3] proxmox-metrics: implement metrics server client code Dominik Csapak
2021-12-14 13:51 ` Wolfgang Bumiller
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox-backup 1/6] use 'disk_usage' from proxmox-sys Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox-backup 2/6] pbs-api-types: add metrics api types Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox-backup 3/6] pbs-config: add metrics config class Dominik Csapak
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox-backup 4/6] backup-proxy: decouple stats gathering from rrd update Dominik Csapak
2021-12-14 12:24 ` Dominik Csapak [this message]
2021-12-15 7:51 ` [pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy: send metrics to configured metrics server Wolfgang Bumiller
2021-12-14 12:24 ` [pbs-devel] [PATCH proxmox-backup 6/6] api: add metricserver endpoints Dominik Csapak
2021-12-15 7:39 ` Wolfgang Bumiller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20211214122412.4077902-9-d.csapak@proxmox.com \
--to=d.csapak@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox