From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: <pdm-devel-bounces@lists.proxmox.com> Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id C5F451FF164 for <inbox@lore.proxmox.com>; Fri, 14 Feb 2025 14:07:49 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 3210719E4F; Fri, 14 Feb 2025 14:07:50 +0100 (CET) From: Lukas Wagner <l.wagner@proxmox.com> To: pdm-devel@lists.proxmox.com Date: Fri, 14 Feb 2025 14:06:30 +0100 Message-Id: <20250214130653.283012-6-l.wagner@proxmox.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250214130653.283012-1-l.wagner@proxmox.com> References: <20250214130653.283012-1-l.wagner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.010 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pdm-devel] [PATCH proxmox-datacenter-manager v2 05/28] metric collection: split top_entities split into separate module X-BeenThere: pdm-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Datacenter Manager development discussion <pdm-devel.lists.proxmox.com> List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pdm-devel>, <mailto:pdm-devel-request@lists.proxmox.com?subject=unsubscribe> List-Archive: <http://lists.proxmox.com/pipermail/pdm-devel/> List-Post: <mailto:pdm-devel@lists.proxmox.com> List-Help: <mailto:pdm-devel-request@lists.proxmox.com?subject=help> List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel>, <mailto:pdm-devel-request@lists.proxmox.com?subject=subscribe> Reply-To: Proxmox Datacenter Manager development discussion <pdm-devel@lists.proxmox.com> Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pdm-devel-bounces@lists.proxmox.com Sender: "pdm-devel" <pdm-devel-bounces@lists.proxmox.com> This makes the parent module a bit more easy not navigate. No functional changes intended. Signed-off-by: Lukas Wagner <l.wagner@proxmox.com> --- server/src/api/resources.rs | 3 +- server/src/metric_collection/mod.rs | 149 +------------------ server/src/metric_collection/top_entities.rs | 148 ++++++++++++++++++ 3 files changed, 152 insertions(+), 148 deletions(-) create mode 100644 server/src/metric_collection/top_entities.rs diff --git a/server/src/api/resources.rs b/server/src/api/resources.rs index 453d9e8e..afda51e3 100644 --- a/server/src/api/resources.rs +++ b/server/src/api/resources.rs @@ -26,6 +26,7 @@ use proxmox_subscription::SubscriptionStatus; use pve_api_types::{ClusterResource, ClusterResourceType}; use crate::connection; +use crate::metric_collection::top_entities; pub const ROUTER: Router = Router::new() .get(&list_subdirs_api_method!(SUBDIRS)) @@ -320,7 +321,7 @@ async fn get_top_entities(timeframe: Option<RrdTimeframe>) -> Result<TopEntities let (remotes_config, _) = pdm_config::remotes::config()?; let timeframe = timeframe.unwrap_or(RrdTimeframe::Day); - let res = crate::metric_collection::calculate_top(&remotes_config.sections, timeframe, 10); + let res = top_entities::calculate_top(&remotes_config.sections, timeframe, 10); Ok(res) } diff --git a/server/src/metric_collection/mod.rs b/server/src/metric_collection/mod.rs index b37d6782..5540f937 100644 --- a/server/src/metric_collection/mod.rs +++ b/server/src/metric_collection/mod.rs @@ -6,15 +6,13 @@ use anyhow::Error; use pbs_api_types::{MetricDataPoint, MetricDataType}; use proxmox_rrd::rrd::DataSourceType; -use pdm_api_types::{ - remotes::RemoteType, - resource::{Resource, ResourceRrdData, TopEntities, TopEntity}, -}; +use pdm_api_types::remotes::RemoteType; use pve_api_types::{ClusterMetricsData, ClusterMetricsDataType}; use crate::{connection, task_utils}; pub mod rrd_cache; +pub mod top_entities; const COLLECTION_INTERVAL: u64 = 60; @@ -150,146 +148,3 @@ fn store_metric_pbs(remote_name: &str, data_point: &MetricDataPoint) { data_source_type, ); } - -fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) { - let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) { - Ok(idx) | Err(idx) => idx, - }; - - vec.insert(index, value); - if vec.len() > limit { - for _ in 0..(vec.len() - limit) { - vec.remove(0); - } - } -} - -// for now simple sum of the values => area under the graph curve -fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 { - let mut coefficient = 0.0; - for point in values.data.iter() { - let value = point.unwrap_or_default(); - if value.is_finite() { - coefficient += value; - } - } - - coefficient -} - -// FIXME: cache the values instead of calculate freshly every time? -// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache) -pub fn calculate_top( - remotes: &HashMap<String, pdm_api_types::remotes::Remote>, - timeframe: proxmox_rrd_api_types::RrdTimeframe, - num: usize, -) -> TopEntities { - let mut guest_cpu = Vec::new(); - let mut node_cpu = Vec::new(); - let mut node_memory = Vec::new(); - - for remote_name in remotes.keys() { - if let Some(data) = - crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64) - { - for res in data.resources { - let id = res.id().to_string(); - let name = format!("pve/{remote_name}/{id}"); - match &res { - Resource::PveStorage(_) => {} - Resource::PveQemu(_) | Resource::PveLxc(_) => { - if let Some(entity) = - get_entity(timeframe, remote_name, res, name, "cpu_current") - { - let coefficient = (entity.0 * 100.0).round() as usize; - insert_sorted(&mut guest_cpu, (coefficient, entity.1), num); - } - } - Resource::PveNode(_) => { - if let Some(entity) = get_entity( - timeframe, - remote_name, - res.clone(), - name.clone(), - "cpu_current", - ) { - let coefficient = (entity.0 * 100.0).round() as usize; - insert_sorted(&mut node_cpu, (coefficient, entity.1), num); - } - // convert mem/mem_total into a single entity - if let Some(mut mem) = get_entity( - timeframe, - remote_name, - res.clone(), - name.clone(), - "mem_used", - ) { - if let Some(mem_total) = - get_entity(timeframe, remote_name, res, name, "mem_total") - { - // skip if we don't have the same amount of data for used and total - let mem_rrd = &mem.1.rrd_data.data; - let mem_total_rrd = &mem_total.1.rrd_data.data; - if mem_rrd.len() != mem_total_rrd.len() { - continue; - } - let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize; - let mut mem_usage = Vec::new(); - for i in 0..mem_rrd.len() { - let point = match (mem_rrd[i], mem_total_rrd[i]) { - (Some(mem), Some(total)) => Some(mem / total), - _ => None, - }; - mem_usage.push(point) - } - mem.1.rrd_data.data = mem_usage; - insert_sorted(&mut node_memory, (coefficient, mem.1), num); - } - } - } - Resource::PbsNode(_) => {} - Resource::PbsDatastore(_) => {} - } - } - } - } - - TopEntities { - guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(), - node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(), - node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(), - } -} - -fn get_entity( - timeframe: proxmox_rrd_api_types::RrdTimeframe, - remote_name: &String, - res: Resource, - name: String, - metric: &str, -) -> Option<(f64, TopEntity)> { - if let Ok(Some(values)) = rrd_cache::extract_data( - &name, - metric, - timeframe, - proxmox_rrd_api_types::RrdMode::Average, - ) { - let coefficient = calculate_coefficient(&values); - if coefficient > 0.0 { - return Some(( - coefficient, - TopEntity { - remote: remote_name.to_string(), - resource: res, - rrd_data: ResourceRrdData { - start: values.start, - resolution: values.resolution, - data: values.data, - }, - }, - )); - } - } - - None -} diff --git a/server/src/metric_collection/top_entities.rs b/server/src/metric_collection/top_entities.rs new file mode 100644 index 00000000..f8e053fb --- /dev/null +++ b/server/src/metric_collection/top_entities.rs @@ -0,0 +1,148 @@ +use std::collections::HashMap; + +use pdm_api_types::resource::{Resource, ResourceRrdData, TopEntities, TopEntity}; + +use super::rrd_cache; + +fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) { + let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) { + Ok(idx) | Err(idx) => idx, + }; + + vec.insert(index, value); + if vec.len() > limit { + for _ in 0..(vec.len() - limit) { + vec.remove(0); + } + } +} + +// for now simple sum of the values => area under the graph curve +fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 { + let mut coefficient = 0.0; + for point in values.data.iter() { + let value = point.unwrap_or_default(); + if value.is_finite() { + coefficient += value; + } + } + + coefficient +} + +// FIXME: cache the values instead of calculate freshly every time? +// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache) +pub fn calculate_top( + remotes: &HashMap<String, pdm_api_types::remotes::Remote>, + timeframe: proxmox_rrd_api_types::RrdTimeframe, + num: usize, +) -> TopEntities { + let mut guest_cpu = Vec::new(); + let mut node_cpu = Vec::new(); + let mut node_memory = Vec::new(); + + for remote_name in remotes.keys() { + if let Some(data) = + crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64) + { + for res in data.resources { + let id = res.id().to_string(); + let name = format!("pve/{remote_name}/{id}"); + match &res { + Resource::PveStorage(_) => {} + Resource::PveQemu(_) | Resource::PveLxc(_) => { + if let Some(entity) = + get_entity(timeframe, remote_name, res, name, "cpu_current") + { + let coefficient = (entity.0 * 100.0).round() as usize; + insert_sorted(&mut guest_cpu, (coefficient, entity.1), num); + } + } + Resource::PveNode(_) => { + if let Some(entity) = get_entity( + timeframe, + remote_name, + res.clone(), + name.clone(), + "cpu_current", + ) { + let coefficient = (entity.0 * 100.0).round() as usize; + insert_sorted(&mut node_cpu, (coefficient, entity.1), num); + } + // convert mem/mem_total into a single entity + if let Some(mut mem) = get_entity( + timeframe, + remote_name, + res.clone(), + name.clone(), + "mem_used", + ) { + if let Some(mem_total) = + get_entity(timeframe, remote_name, res, name, "mem_total") + { + // skip if we don't have the same amount of data for used and total + let mem_rrd = &mem.1.rrd_data.data; + let mem_total_rrd = &mem_total.1.rrd_data.data; + if mem_rrd.len() != mem_total_rrd.len() { + continue; + } + let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize; + let mut mem_usage = Vec::new(); + for i in 0..mem_rrd.len() { + let point = match (mem_rrd[i], mem_total_rrd[i]) { + (Some(mem), Some(total)) => Some(mem / total), + _ => None, + }; + mem_usage.push(point) + } + mem.1.rrd_data.data = mem_usage; + insert_sorted(&mut node_memory, (coefficient, mem.1), num); + } + } + } + Resource::PbsNode(_) => {} + Resource::PbsDatastore(_) => {} + } + } + } + } + + TopEntities { + guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(), + node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(), + node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(), + } +} + +fn get_entity( + timeframe: proxmox_rrd_api_types::RrdTimeframe, + remote_name: &String, + res: Resource, + name: String, + metric: &str, +) -> Option<(f64, TopEntity)> { + if let Ok(Some(values)) = rrd_cache::extract_data( + &name, + metric, + timeframe, + proxmox_rrd_api_types::RrdMode::Average, + ) { + let coefficient = calculate_coefficient(&values); + if coefficient > 0.0 { + return Some(( + coefficient, + TopEntity { + remote: remote_name.to_string(), + resource: res, + rrd_data: ResourceRrdData { + start: values.start, + resolution: values.resolution, + data: values.data, + }, + }, + )); + } + } + + None +} -- 2.39.5 _______________________________________________ pdm-devel mailing list pdm-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel