From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 4A9BE1FF16F for ; Tue, 30 Sep 2025 16:52:05 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id CAFC6B222; Tue, 30 Sep 2025 16:52:10 +0200 (CEST) Message-ID: Date: Tue, 30 Sep 2025 16:51:37 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird Beta To: Proxmox Datacenter Manager development discussion , Lukas Wagner References: <20250929094705.106650-1-l.wagner@proxmox.com> Content-Language: en-US From: Dominik Csapak In-Reply-To: <20250929094705.106650-1-l.wagner@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1759243876569 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.025 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pdm-devel] [PATCH proxmox-datacenter-manager] api: pbs: request latest metrics when using hourly RRD timeframe X-BeenThere: pdm-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Datacenter Manager development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Datacenter Manager development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pdm-devel-bounces@lists.proxmox.com Sender: "pdm-devel" code makes sense, works as advertised. consider this Reviewed-by: Dominik Csapak Tested-by: Dominik Csapak On 9/29/25 11:47 AM, Lukas Wagner wrote: > Same as for PVE, when requesting metric data in the hourly timeframe, we > trigger an out-of-order metric collection for this single remote, > waiting for its completion up to a short timeout of five seconds. If > collection does not finish in this time, we simply return what is > currently in the database, which might have a short gap (up to 10 > minutes, since this is the regular metric collection interval). > > Signed-off-by: Lukas Wagner > --- > server/src/api/pbs/rrddata.rs | 10 ++++------ > server/src/api/pve/rrddata.rs | 35 ++++------------------------------- > server/src/api/rrd_common.rs | 33 +++++++++++++++++++++++++++++++-- > 3 files changed, 39 insertions(+), 39 deletions(-) > > diff --git a/server/src/api/pbs/rrddata.rs b/server/src/api/pbs/rrddata.rs > index c6649a2d..aa980d45 100644 > --- a/server/src/api/pbs/rrddata.rs > +++ b/server/src/api/pbs/rrddata.rs > @@ -102,15 +102,14 @@ impl DataPoint for PbsDatastoreDataPoint { > }, > )] > /// Read PBS node stats > -fn get_pbs_node_rrd_data( > +async fn get_pbs_node_rrd_data( > remote: String, > timeframe: RrdTimeframe, > cf: RrdMode, > _param: Value, > ) -> Result, Error> { > let base = format!("pbs/{remote}/host"); > - > - rrd_common::create_datapoints_from_rrd(&base, timeframe, cf) > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > #[api( > @@ -128,7 +127,7 @@ fn get_pbs_node_rrd_data( > }, > )] > /// Read PBS datastore stats > -fn get_pbs_datastore_rrd_data( > +async fn get_pbs_datastore_rrd_data( > remote: String, > datastore: String, > timeframe: RrdTimeframe, > @@ -136,8 +135,7 @@ fn get_pbs_datastore_rrd_data( > _param: Value, > ) -> Result, Error> { > let base = format!("pbs/{remote}/datastore/{datastore}"); > - > - rrd_common::create_datapoints_from_rrd(&base, timeframe, cf) > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > pub const PBS_NODE_RRD_ROUTER: Router = Router::new().get(&API_METHOD_GET_PBS_NODE_RRD_DATA); > diff --git a/server/src/api/pve/rrddata.rs b/server/src/api/pve/rrddata.rs > index e08d4b43..9fed9671 100644 > --- a/server/src/api/pve/rrddata.rs > +++ b/server/src/api/pve/rrddata.rs > @@ -1,5 +1,3 @@ > -use std::time::Duration; > - > use anyhow::Error; > use serde_json::Value; > > @@ -12,7 +10,6 @@ use pdm_api_types::rrddata::{LxcDataPoint, NodeDataPoint, PveStorageDataPoint, Q > use pdm_api_types::{NODE_SCHEMA, PRIV_RESOURCE_AUDIT, PVE_STORAGE_ID_SCHEMA, VMID_SCHEMA}; > > use crate::api::rrd_common::{self, DataPoint}; > -use crate::metric_collection; > > impl DataPoint for NodeDataPoint { > fn new(time: u64) -> Self { > @@ -193,7 +190,7 @@ async fn get_qemu_rrd_data( > _param: Value, > ) -> Result, Error> { > let base = format!("pve/{remote}/qemu/{vmid}"); > - get_rrd_datapoints(remote, base, timeframe, cf).await > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > #[api( > @@ -222,7 +219,7 @@ async fn get_lxc_rrd_data( > _param: Value, > ) -> Result, Error> { > let base = format!("pve/{remote}/lxc/{vmid}"); > - get_rrd_datapoints(remote, base, timeframe, cf).await > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > #[api( > @@ -251,7 +248,7 @@ async fn get_node_rrd_data( > _param: Value, > ) -> Result, Error> { > let base = format!("pve/{remote}/node/{node}"); > - get_rrd_datapoints(remote, base, timeframe, cf).await > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > #[api( > @@ -282,31 +279,7 @@ async fn get_storage_rrd_data( > _param: Value, > ) -> Result, Error> { > let base = format!("pve/{remote}/storage/{node}/{storage}"); > - get_rrd_datapoints(remote, base, timeframe, cf).await > -} > - > -async fn get_rrd_datapoints( > - remote: String, > - basepath: String, > - timeframe: RrdTimeframe, > - mode: RrdMode, > -) -> Result, Error> { > - const WAIT_FOR_NEWEST_METRIC_TIMEOUT: Duration = Duration::from_secs(5); > - > - if timeframe == RrdTimeframe::Hour { > - // Let's wait for a limited time for the most recent metrics. If the connection to the remote > - // is super slow or if the metric collection tasks currently busy with collecting other > - // metrics, we just return the data we already have, not the newest one. > - let _ = tokio::time::timeout(WAIT_FOR_NEWEST_METRIC_TIMEOUT, async { > - metric_collection::trigger_metric_collection(Some(remote), true).await > - }) > - .await; > - } > - > - tokio::task::spawn_blocking(move || { > - rrd_common::create_datapoints_from_rrd(&basepath, timeframe, mode) > - }) > - .await? > + rrd_common::get_rrd_datapoints(remote, base, timeframe, cf).await > } > > pub const QEMU_RRD_ROUTER: Router = Router::new().get(&API_METHOD_GET_QEMU_RRD_DATA); > diff --git a/server/src/api/rrd_common.rs b/server/src/api/rrd_common.rs > index 28868bc1..b5d1a786 100644 > --- a/server/src/api/rrd_common.rs > +++ b/server/src/api/rrd_common.rs > @@ -1,9 +1,10 @@ > -use std::collections::BTreeMap; > +use std::{collections::BTreeMap, time::Duration}; > > use anyhow::{bail, Error}; > + > use proxmox_rrd_api_types::{RrdMode, RrdTimeframe}; > > -use crate::metric_collection::rrd_cache; > +use crate::metric_collection::{self, rrd_cache}; > > /// Trait common to all RRD-stored metric objects (nodes, datastores, qemu, lxc, etc.) > pub trait DataPoint { > @@ -53,3 +54,31 @@ pub fn create_datapoints_from_rrd( > > Ok(timemap.into_values().collect()) > } > + > +/// Get RRD datapoints for a given remote/RRD path. > +/// > +/// If `timeframe` is set to [`RrdTimeframe::Hour`], then this function will trigger > +/// metric collection for this remote and wait for its completion, up to a timeout of five > +/// seconds. If the timeout is exceeded, we simply go ahead and return what is in the database at > +/// the moment, which might have a gap for the last couple minutes. > +pub async fn get_rrd_datapoints( > + remote: String, > + basepath: String, > + timeframe: RrdTimeframe, > + mode: RrdMode, > +) -> Result, Error> { > + const WAIT_FOR_NEWEST_METRIC_TIMEOUT: Duration = Duration::from_secs(5); > + > + if timeframe == RrdTimeframe::Hour { > + // Let's wait for a limited time for the most recent metrics. If the connection to the remote > + // is super slow or if the metric collection tasks currently busy with collecting other > + // metrics, we just return the data we already have, not the newest one. > + let _ = tokio::time::timeout(WAIT_FOR_NEWEST_METRIC_TIMEOUT, async { > + metric_collection::trigger_metric_collection(Some(remote), true).await > + }) > + .await; > + } > + > + tokio::task::spawn_blocking(move || create_datapoints_from_rrd(&basepath, timeframe, mode)) > + .await? > +} _______________________________________________ pdm-devel mailing list pdm-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel