Re: [pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy: send metrics to configured metrics server

[Wolfgang Bumiller <w.bumiller@proxmox.com>]

On Tue, Dec 14, 2021 at 01:24:11PM +0100, Dominik Csapak wrote:
> and keep the data as similar as possible to pve (tags/fields)
> 
> datastores get their own 'object' type and reside in the "blockstat"
> measurement
> 
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
>  src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++-
>  1 file changed, 138 insertions(+), 1 deletion(-)
> 
> diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
> index 2700fabf..fbb782dd 100644
> --- a/src/bin/proxmox-backup-proxy.rs
> +++ b/src/bin/proxmox-backup-proxy.rs
> @@ -23,11 +23,13 @@ use proxmox_sys::linux::{
>  };
>  use proxmox_sys::fs::{CreateOptions, DiskUsage};
>  use proxmox_lang::try_block;
> +use proxmox_metrics::MetricsData;
>  use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation};
>  use proxmox_http::client::{RateLimitedStream, ShareableRateLimit};
>  use proxmox_sys::{task_log, task_warn};
>  use proxmox_sys::logrotate::LogRotate;
>  
> +use pbs_config::metrics::get_metric_server_connections;
>  use pbs_datastore::DataStore;
>  
>  use proxmox_rest_server::{
> @@ -948,16 +950,131 @@ async fn run_stat_generator() {
>              }
>          };
>  
> +        let hoststats2 = hoststats.clone();
> +        let hostdisk2 = hostdisk.clone();
> +        let datastores2 = datastores.clone();

Please use Arc::clone, also, I'm not sure it's worth having them all as
separate Arcs, maybe just a 3-tuple in 1 Arc?

>          let rrd_future = tokio::task::spawn_blocking(move || {
> -            rrd_update_host_stats_sync(&hoststats, &hostdisk, &datastores);
> +            rrd_update_host_stats_sync(&hoststats2, &hostdisk2, &datastores2);
>              rrd_sync_journal();
>          });
>  
> +        let metrics_future = send_data_to_metric_servers(hoststats, hostdisk, datastores);
> +
> +        let (rrd_res, metrics_res) = join!(rrd_future, metrics_future);
> +        if let Err(err) = rrd_res {
> +            log::error!("rrd update panicked: {}", err);
> +        }
> +        if let Err(err) = metrics_res {
> +            log::error!("error during metrics sending: {}", err);
> +        }
>  
>          tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
>  
>       }
> +}
> +
> +async fn send_data_to_metric_servers(
> +    host: Arc<HostStats>,
> +    hostdisk: Arc<DiskStat>,
> +    datastores: Arc<Vec<DiskStat>>,
> +) -> Result<(), Error> {
> +    let (config, _digest) = pbs_config::metrics::config()?;
> +    let (futures, channels, names) = get_metric_server_connections(config)?;
> +
> +    if futures.is_empty() {
> +        return Ok(());
> +    }
> +
> +    let names2 = names.clone();
> +    let sending_handle = tokio::spawn(async move {
> +        for (i, res) in future::join_all(futures).await.into_iter().enumerate() {
> +            if let Err(err) = res {
> +                eprintln!("ERROR '{}': {}", names2[i], err);
> +            }
> +        }
> +    });
> +
> +    let ctime = proxmox_time::epoch_i64();
> +    let nodename = proxmox_sys::nodename();
> +
> +    let mut values = Vec::new();
> +
> +    let mut cpuvalue = json!({});
> +    if let Some(stat) = &host.proc {
> +        for (key, value) in serde_json::to_value(stat)?.as_object().unwrap().iter() {
> +            cpuvalue[key.clone()] = value.clone();
> +        }
I may be missing something but can I not replace the entire loop with
just:
    cpuvalue = to_value(stat);
?

> +    }

in fact:

    let mut cpuvalue = match &host.proc {
        Some(stat) => serde_json.to_value(stat),
        None => json!({}),
    };

> +
> +    if let Some(loadavg) = &host.load {
> +        cpuvalue["avg1"] = Value::from(loadavg.0);
> +        cpuvalue["avg5"] = Value::from(loadavg.1);
> +        cpuvalue["avg15"] = Value::from(loadavg.2);
> +    }

> @@ -973,6 +1090,26 @@ struct DiskStat {
>      dev: Option<BlockDevStat>,
>  }
>  
> +impl DiskStat {
> +    fn to_value(&self) -> Value {
> +        let mut value = json!({});
> +        if let Some(usage) = &self.usage {
> +            value["total"] = Value::from(usage.total);
> +            value["used"] = Value::from(usage.used);
> +            value["avail"] = Value::from(usage.available);
> +        }
> +
> +        if let Some(dev) = &self.dev {
> +            value["read_ios"] = Value::from(dev.read_ios);
> +            value["read_bytes"] = Value::from(dev.read_sectors * 512);

And bye-bye goes the hope for a generic
'serialize-by-merging-into-existing-object' helper :-(