From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 60D0984A68 for ; Wed, 15 Dec 2021 08:51:49 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4F187F069 for ; Wed, 15 Dec 2021 08:51:19 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id A499AF05E for ; Wed, 15 Dec 2021 08:51:18 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7596F4519F for ; Wed, 15 Dec 2021 08:51:18 +0100 (CET) Date: Wed, 15 Dec 2021 08:51:17 +0100 From: Wolfgang Bumiller To: Dominik Csapak Cc: pbs-devel@lists.proxmox.com Message-ID: <20211215075117.2eiykfvsa7amfxhq@olga.proxmox.com> References: <20211214122412.4077902-1-d.csapak@proxmox.com> <20211214122412.4077902-9-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20211214122412.4077902-9-d.csapak@proxmox.com> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.409 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy: send metrics to configured metrics server X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Dec 2021 07:51:49 -0000 On Tue, Dec 14, 2021 at 01:24:11PM +0100, Dominik Csapak wrote: > and keep the data as similar as possible to pve (tags/fields) > > datastores get their own 'object' type and reside in the "blockstat" > measurement > > Signed-off-by: Dominik Csapak > --- > src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++- > 1 file changed, 138 insertions(+), 1 deletion(-) > > diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs > index 2700fabf..fbb782dd 100644 > --- a/src/bin/proxmox-backup-proxy.rs > +++ b/src/bin/proxmox-backup-proxy.rs > @@ -23,11 +23,13 @@ use proxmox_sys::linux::{ > }; > use proxmox_sys::fs::{CreateOptions, DiskUsage}; > use proxmox_lang::try_block; > +use proxmox_metrics::MetricsData; > use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation}; > use proxmox_http::client::{RateLimitedStream, ShareableRateLimit}; > use proxmox_sys::{task_log, task_warn}; > use proxmox_sys::logrotate::LogRotate; > > +use pbs_config::metrics::get_metric_server_connections; > use pbs_datastore::DataStore; > > use proxmox_rest_server::{ > @@ -948,16 +950,131 @@ async fn run_stat_generator() { > } > }; > > + let hoststats2 = hoststats.clone(); > + let hostdisk2 = hostdisk.clone(); > + let datastores2 = datastores.clone(); Please use Arc::clone, also, I'm not sure it's worth having them all as separate Arcs, maybe just a 3-tuple in 1 Arc? > let rrd_future = tokio::task::spawn_blocking(move || { > - rrd_update_host_stats_sync(&hoststats, &hostdisk, &datastores); > + rrd_update_host_stats_sync(&hoststats2, &hostdisk2, &datastores2); > rrd_sync_journal(); > }); > > + let metrics_future = send_data_to_metric_servers(hoststats, hostdisk, datastores); > + > + let (rrd_res, metrics_res) = join!(rrd_future, metrics_future); > + if let Err(err) = rrd_res { > + log::error!("rrd update panicked: {}", err); > + } > + if let Err(err) = metrics_res { > + log::error!("error during metrics sending: {}", err); > + } > > tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await; > > } > +} > + > +async fn send_data_to_metric_servers( > + host: Arc, > + hostdisk: Arc, > + datastores: Arc>, > +) -> Result<(), Error> { > + let (config, _digest) = pbs_config::metrics::config()?; > + let (futures, channels, names) = get_metric_server_connections(config)?; > + > + if futures.is_empty() { > + return Ok(()); > + } > + > + let names2 = names.clone(); > + let sending_handle = tokio::spawn(async move { > + for (i, res) in future::join_all(futures).await.into_iter().enumerate() { > + if let Err(err) = res { > + eprintln!("ERROR '{}': {}", names2[i], err); > + } > + } > + }); > + > + let ctime = proxmox_time::epoch_i64(); > + let nodename = proxmox_sys::nodename(); > + > + let mut values = Vec::new(); > + > + let mut cpuvalue = json!({}); > + if let Some(stat) = &host.proc { > + for (key, value) in serde_json::to_value(stat)?.as_object().unwrap().iter() { > + cpuvalue[key.clone()] = value.clone(); > + } I may be missing something but can I not replace the entire loop with just: cpuvalue = to_value(stat); ? > + } in fact: let mut cpuvalue = match &host.proc { Some(stat) => serde_json.to_value(stat), None => json!({}), }; > + > + if let Some(loadavg) = &host.load { > + cpuvalue["avg1"] = Value::from(loadavg.0); > + cpuvalue["avg5"] = Value::from(loadavg.1); > + cpuvalue["avg15"] = Value::from(loadavg.2); > + } > @@ -973,6 +1090,26 @@ struct DiskStat { > dev: Option, > } > > +impl DiskStat { > + fn to_value(&self) -> Value { > + let mut value = json!({}); > + if let Some(usage) = &self.usage { > + value["total"] = Value::from(usage.total); > + value["used"] = Value::from(usage.used); > + value["avail"] = Value::from(usage.available); > + } > + > + if let Some(dev) = &self.dev { > + value["read_ios"] = Value::from(dev.read_ios); > + value["read_bytes"] = Value::from(dev.read_sectors * 512); And bye-bye goes the hope for a generic 'serialize-by-merging-into-existing-object' helper :-(