From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <w.bumiller@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 60D0984A68
for <pbs-devel@lists.proxmox.com>; Wed, 15 Dec 2021 08:51:49 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 4F187F069
for <pbs-devel@lists.proxmox.com>; Wed, 15 Dec 2021 08:51:19 +0100 (CET)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[94.136.29.106])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id A499AF05E
for <pbs-devel@lists.proxmox.com>; Wed, 15 Dec 2021 08:51:18 +0100 (CET)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7596F4519F
for <pbs-devel@lists.proxmox.com>; Wed, 15 Dec 2021 08:51:18 +0100 (CET)
Date: Wed, 15 Dec 2021 08:51:17 +0100
From: Wolfgang Bumiller <w.bumiller@proxmox.com>
To: Dominik Csapak <d.csapak@proxmox.com>
Cc: pbs-devel@lists.proxmox.com
Message-ID: <20211215075117.2eiykfvsa7amfxhq@olga.proxmox.com>
References: <20211214122412.4077902-1-d.csapak@proxmox.com>
<20211214122412.4077902-9-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
In-Reply-To: <20211214122412.4077902-9-d.csapak@proxmox.com>
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.409 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: Re: [pbs-devel] [PATCH proxmox-backup 5/6] proxmox-backup-proxy:
send metrics to configured metrics server
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Wed, 15 Dec 2021 07:51:49 -0000
On Tue, Dec 14, 2021 at 01:24:11PM +0100, Dominik Csapak wrote:
> and keep the data as similar as possible to pve (tags/fields)
>
> datastores get their own 'object' type and reside in the "blockstat"
> measurement
>
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
> src/bin/proxmox-backup-proxy.rs | 139 +++++++++++++++++++++++++++++++-
> 1 file changed, 138 insertions(+), 1 deletion(-)
>
> diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
> index 2700fabf..fbb782dd 100644
> --- a/src/bin/proxmox-backup-proxy.rs
> +++ b/src/bin/proxmox-backup-proxy.rs
> @@ -23,11 +23,13 @@ use proxmox_sys::linux::{
> };
> use proxmox_sys::fs::{CreateOptions, DiskUsage};
> use proxmox_lang::try_block;
> +use proxmox_metrics::MetricsData;
> use proxmox_router::{RpcEnvironment, RpcEnvironmentType, UserInformation};
> use proxmox_http::client::{RateLimitedStream, ShareableRateLimit};
> use proxmox_sys::{task_log, task_warn};
> use proxmox_sys::logrotate::LogRotate;
>
> +use pbs_config::metrics::get_metric_server_connections;
> use pbs_datastore::DataStore;
>
> use proxmox_rest_server::{
> @@ -948,16 +950,131 @@ async fn run_stat_generator() {
> }
> };
>
> + let hoststats2 = hoststats.clone();
> + let hostdisk2 = hostdisk.clone();
> + let datastores2 = datastores.clone();
Please use Arc::clone, also, I'm not sure it's worth having them all as
separate Arcs, maybe just a 3-tuple in 1 Arc?
> let rrd_future = tokio::task::spawn_blocking(move || {
> - rrd_update_host_stats_sync(&hoststats, &hostdisk, &datastores);
> + rrd_update_host_stats_sync(&hoststats2, &hostdisk2, &datastores2);
> rrd_sync_journal();
> });
>
> + let metrics_future = send_data_to_metric_servers(hoststats, hostdisk, datastores);
> +
> + let (rrd_res, metrics_res) = join!(rrd_future, metrics_future);
> + if let Err(err) = rrd_res {
> + log::error!("rrd update panicked: {}", err);
> + }
> + if let Err(err) = metrics_res {
> + log::error!("error during metrics sending: {}", err);
> + }
>
> tokio::time::sleep_until(tokio::time::Instant::from_std(delay_target)).await;
>
> }
> +}
> +
> +async fn send_data_to_metric_servers(
> + host: Arc<HostStats>,
> + hostdisk: Arc<DiskStat>,
> + datastores: Arc<Vec<DiskStat>>,
> +) -> Result<(), Error> {
> + let (config, _digest) = pbs_config::metrics::config()?;
> + let (futures, channels, names) = get_metric_server_connections(config)?;
> +
> + if futures.is_empty() {
> + return Ok(());
> + }
> +
> + let names2 = names.clone();
> + let sending_handle = tokio::spawn(async move {
> + for (i, res) in future::join_all(futures).await.into_iter().enumerate() {
> + if let Err(err) = res {
> + eprintln!("ERROR '{}': {}", names2[i], err);
> + }
> + }
> + });
> +
> + let ctime = proxmox_time::epoch_i64();
> + let nodename = proxmox_sys::nodename();
> +
> + let mut values = Vec::new();
> +
> + let mut cpuvalue = json!({});
> + if let Some(stat) = &host.proc {
> + for (key, value) in serde_json::to_value(stat)?.as_object().unwrap().iter() {
> + cpuvalue[key.clone()] = value.clone();
> + }
I may be missing something but can I not replace the entire loop with
just:
cpuvalue = to_value(stat);
?
> + }
in fact:
let mut cpuvalue = match &host.proc {
Some(stat) => serde_json.to_value(stat),
None => json!({}),
};
> +
> + if let Some(loadavg) = &host.load {
> + cpuvalue["avg1"] = Value::from(loadavg.0);
> + cpuvalue["avg5"] = Value::from(loadavg.1);
> + cpuvalue["avg15"] = Value::from(loadavg.2);
> + }
> @@ -973,6 +1090,26 @@ struct DiskStat {
> dev: Option<BlockDevStat>,
> }
>
> +impl DiskStat {
> + fn to_value(&self) -> Value {
> + let mut value = json!({});
> + if let Some(usage) = &self.usage {
> + value["total"] = Value::from(usage.total);
> + value["used"] = Value::from(usage.used);
> + value["avail"] = Value::from(usage.available);
> + }
> +
> + if let Some(dev) = &self.dev {
> + value["read_ios"] = Value::from(dev.read_ios);
> + value["read_bytes"] = Value::from(dev.read_sectors * 512);
And bye-bye goes the hope for a generic
'serialize-by-merging-into-existing-object' helper :-(