[pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy: send metrics to configured metrics server

[Dominik Csapak <d.csapak@proxmox.com>]

and keep the data as similar as possible to pve (tags/fields)

datastores get their own 'object' type and reside in the "blockstat"
measurement

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++-
 1 file changed, 138 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 2700fabf..fbb782dd 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -23,11 +23,13 @@ use proxmox_sys::linux::{
 };
 use proxmox_sys::fs::{CreateOptions, DiskUsage};
 use proxmox_lang::try_block;
+use proxmox_metrics::MetricsData;
 use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation};
 use proxmox_http::client::{RateLimitedStream, ShareableRateLimit};
 use proxmox_sys::{task_log, task_warn};
 use proxmox_sys::logrotate::LogRotate;
 
+use pbs_config::metrics::get_metric_server_connections;
 use pbs_datastore::DataStore;
 
 use proxmox_rest_server::{
@@ -948,16 +950,131 @@ async fn run_stat_generator() {
             }
         };
 
+        let hoststats2 = hoststats.clone();
+        let hostdisk2 = hostdisk.clone();
+        let datastores2 = datastores.clone();
         let rrd_future = tokio::task::spawn_blocking(move || {
-            rrd_update_host_stats_sync(&hoststats, &hostdisk, &datastores);
+            rrd_update_host_stats_sync(&hoststats2, &hostdisk2, &datastores2);
             rrd_sync_journal();
         });
 
+        let metrics_future = send_data_to_metric_servers(hoststats, hostdisk, datastores);
+
+        let (rrd_res, metrics_res) = join!(rrd_future, metrics_future);
+        if let Err(err) = rrd_res {
+            log::error!("rrd update panicked: {}", err);
+        }
+        if let Err(err) = metrics_res {
+            log::error!("error during metrics sending: {}", err);
+        }
 
         tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
 
      }
+}
+
+async fn send_data_to_metric_servers(
+    host: Arc<HostStats>,
+    hostdisk: Arc<DiskStat>,
+    datastores: Arc<Vec<DiskStat>>,
+) -> Result<(), Error> {
+    let (config, _digest) = pbs_config::metrics::config()?;
+    let (futures, channels, names) = get_metric_server_connections(config)?;
+
+    if futures.is_empty() {
+        return Ok(());
+    }
+
+    let names2 = names.clone();
+    let sending_handle = tokio::spawn(async move {
+        for (i, res) in future::join_all(futures).await.into_iter().enumerate() {
+            if let Err(err) = res {
+                eprintln!("ERROR '{}': {}", names2[i], err);
+            }
+        }
+    });
+
+    let ctime = proxmox_time::epoch_i64();
+    let nodename = proxmox_sys::nodename();
+
+    let mut values = Vec::new();
+
+    let mut cpuvalue = json!({});
+    if let Some(stat) = &host.proc {
+        for (key, value) in serde_json::to_value(stat)?.as_object().unwrap().iter() {
+            cpuvalue[key.clone()] = value.clone();
+        }
+    }
+
+    if let Some(loadavg) = &host.load {
+        cpuvalue["avg1"] = Value::from(loadavg.0);
+        cpuvalue["avg5"] = Value::from(loadavg.1);
+        cpuvalue["avg15"] = Value::from(loadavg.2);
+    }
 
+    values.push(Arc::new(MetricsData::new(
+        "cpustat",
+        &[("object", "host"), ("host", nodename)],
+        ctime,
+        cpuvalue,
+    )?));
+
+    if let Some(stat) = &host.meminfo {
+        values.push(Arc::new(MetricsData::new(
+            "memory",
+            &[("object", "host"), ("host", nodename)],
+            ctime,
+            stat,
+        )?));
+    }
+
+    if let Some(netdev) = &host.net {
+        for item in netdev {
+            values.push(Arc::new(MetricsData::new(
+                "nics",
+                &[
+                    ("object", "host"),
+                    ("host", nodename),
+                    ("instance", &item.device),
+                ],
+                ctime,
+                item,
+            )?));
+        }
+    }
+
+    values.push(Arc::new(MetricsData::new(
+        "blockstat",
+        &[("object", "host"), ("host", nodename)],
+        ctime,
+        hostdisk.to_value(),
+    )?));
+
+    for datastore in datastores.iter() {
+        values.push(Arc::new(MetricsData::new(
+            "blockstat",
+            &[
+                ("object", "datastore"),
+                ("nodename", nodename),
+                ("datastore", &datastore.name),
+            ],
+            ctime,
+            datastore.to_value(),
+        )?));
+    }
+
+    let results = proxmox_metrics::send_data_to_channels(&values, &channels).await;
+    for (i, res) in results.into_iter().enumerate() {
+        if let Err(err) = res {
+            log::error!("error sending to {}: {}", names[i], err);
+        }
+    }
+
+    drop(channels);
+
+    sending_handle.await?;
+
+    Ok(())
 }
 
 struct HostStats {
@@ -973,6 +1090,26 @@ struct DiskStat {
     dev: Option<BlockDevStat>,
 }
 
+impl DiskStat {
+    fn to_value(&self) -> Value {
+        let mut value = json!({});
+        if let Some(usage) = &self.usage {
+            value["total"] = Value::from(usage.total);
+            value["used"] = Value::from(usage.used);
+            value["avail"] = Value::from(usage.available);
+        }
+
+        if let Some(dev) = &self.dev {
+            value["read_ios"] = Value::from(dev.read_ios);
+            value["read_bytes"] = Value::from(dev.read_sectors * 512);
+            value["write_ios"] = Value::from(dev.write_ios);
+            value["write_bytes"] = Value::from(dev.write_sectors * 512);
+            value["io_ticks"] = Value::from(dev.io_ticks / 1000);
+        }
+        value
+    }
+}
+
 fn collect_host_stats_sync() -> HostStats {
     use proxmox_sys::linux::procfs::{
         read_meminfo, read_proc_stat, read_proc_net_dev, read_loadavg};
-- 
2.30.2