[pdm-devel] [PATCH proxmox-datacenter-manager v4 03/26] metric collection: split top_entities split into separate module

From: Lukas Wagner <l.wagner@proxmox.com>
To: pdm-devel@lists.proxmox.com
Subject: [pdm-devel] [PATCH proxmox-datacenter-manager v4 03/26] metric collection: split top_entities split into separate module
Date: Mon, 12 May 2025 15:37:02 +0200	[thread overview]
Message-ID: <20250512133725.262263-4-l.wagner@proxmox.com> (raw)
In-Reply-To: <20250512133725.262263-1-l.wagner@proxmox.com>

This makes the parent module a bit more easy not navigate.

No functional changes intended.

Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
Reviewed-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---
 server/src/api/resources.rs                  |   3 +-
 server/src/metric_collection/mod.rs          | 149 +------------------
 server/src/metric_collection/top_entities.rs | 148 ++++++++++++++++++
 3 files changed, 152 insertions(+), 148 deletions(-)
 create mode 100644 server/src/metric_collection/top_entities.rs

diff --git a/server/src/api/resources.rs b/server/src/api/resources.rs
index 6a8c8ef2..13a959a6 100644
--- a/server/src/api/resources.rs
+++ b/server/src/api/resources.rs
@@ -26,6 +26,7 @@ use proxmox_subscription::SubscriptionStatus;
 use pve_api_types::{ClusterResource, ClusterResourceType};
 
 use crate::connection;
+use crate::metric_collection::top_entities;
 
 pub const ROUTER: Router = Router::new()
     .get(&list_subdirs_api_method!(SUBDIRS))
@@ -320,7 +321,7 @@ async fn get_top_entities(timeframe: Option<RrdTimeframe>) -> Result<TopEntities
     let (remotes_config, _) = pdm_config::remotes::config()?;
 
     let timeframe = timeframe.unwrap_or(RrdTimeframe::Day);
-    let res = crate::metric_collection::calculate_top(&remotes_config, timeframe, 10);
+    let res = top_entities::calculate_top(&remotes_config, timeframe, 10);
     Ok(res)
 }
 
diff --git a/server/src/metric_collection/mod.rs b/server/src/metric_collection/mod.rs
index f62e2a63..588f7ad5 100644
--- a/server/src/metric_collection/mod.rs
+++ b/server/src/metric_collection/mod.rs
@@ -6,15 +6,13 @@ use anyhow::Error;
 use pbs_api_types::{MetricDataPoint, MetricDataType};
 use proxmox_rrd::rrd::DataSourceType;
 
-use pdm_api_types::{
-    remotes::RemoteType,
-    resource::{Resource, ResourceRrdData, TopEntities, TopEntity},
-};
+use pdm_api_types::remotes::RemoteType;
 use pve_api_types::{ClusterMetricsData, ClusterMetricsDataType};
 
 use crate::{connection, task_utils};
 
 pub mod rrd_cache;
+pub mod top_entities;
 
 const COLLECTION_INTERVAL: u64 = 60;
 
@@ -150,146 +148,3 @@ fn store_metric_pbs(remote_name: &str, data_point: &MetricDataPoint) {
         data_source_type,
     );
 }
-
-fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) {
-    let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) {
-        Ok(idx) | Err(idx) => idx,
-    };
-
-    vec.insert(index, value);
-    if vec.len() > limit {
-        for _ in 0..(vec.len() - limit) {
-            vec.remove(0);
-        }
-    }
-}
-
-// for now simple sum of the values => area under the graph curve
-fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 {
-    let mut coefficient = 0.0;
-    for point in values.data.iter() {
-        let value = point.unwrap_or_default();
-        if value.is_finite() {
-            coefficient += value;
-        }
-    }
-
-    coefficient
-}
-
-// FIXME: cache the values instead of calculate freshly every time?
-// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache)
-pub fn calculate_top(
-    remotes: &HashMap<String, pdm_api_types::remotes::Remote>,
-    timeframe: proxmox_rrd_api_types::RrdTimeframe,
-    num: usize,
-) -> TopEntities {
-    let mut guest_cpu = Vec::new();
-    let mut node_cpu = Vec::new();
-    let mut node_memory = Vec::new();
-
-    for remote_name in remotes.keys() {
-        if let Some(data) =
-            crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64)
-        {
-            for res in data.resources {
-                let id = res.id().to_string();
-                let name = format!("pve/{remote_name}/{id}");
-                match &res {
-                    Resource::PveStorage(_) => {}
-                    Resource::PveQemu(_) | Resource::PveLxc(_) => {
-                        if let Some(entity) =
-                            get_entity(timeframe, remote_name, res, name, "cpu_current")
-                        {
-                            let coefficient = (entity.0 * 100.0).round() as usize;
-                            insert_sorted(&mut guest_cpu, (coefficient, entity.1), num);
-                        }
-                    }
-                    Resource::PveNode(_) => {
-                        if let Some(entity) = get_entity(
-                            timeframe,
-                            remote_name,
-                            res.clone(),
-                            name.clone(),
-                            "cpu_current",
-                        ) {
-                            let coefficient = (entity.0 * 100.0).round() as usize;
-                            insert_sorted(&mut node_cpu, (coefficient, entity.1), num);
-                        }
-                        // convert mem/mem_total into a single entity
-                        if let Some(mut mem) = get_entity(
-                            timeframe,
-                            remote_name,
-                            res.clone(),
-                            name.clone(),
-                            "mem_used",
-                        ) {
-                            if let Some(mem_total) =
-                                get_entity(timeframe, remote_name, res, name, "mem_total")
-                            {
-                                // skip if we don't have the same amount of data for used and total
-                                let mem_rrd = &mem.1.rrd_data.data;
-                                let mem_total_rrd = &mem_total.1.rrd_data.data;
-                                if mem_rrd.len() != mem_total_rrd.len() {
-                                    continue;
-                                }
-                                let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize;
-                                let mut mem_usage = Vec::new();
-                                for i in 0..mem_rrd.len() {
-                                    let point = match (mem_rrd[i], mem_total_rrd[i]) {
-                                        (Some(mem), Some(total)) => Some(mem / total),
-                                        _ => None,
-                                    };
-                                    mem_usage.push(point)
-                                }
-                                mem.1.rrd_data.data = mem_usage;
-                                insert_sorted(&mut node_memory, (coefficient, mem.1), num);
-                            }
-                        }
-                    }
-                    Resource::PbsNode(_) => {}
-                    Resource::PbsDatastore(_) => {}
-                }
-            }
-        }
-    }
-
-    TopEntities {
-        guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(),
-        node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(),
-        node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(),
-    }
-}
-
-fn get_entity(
-    timeframe: proxmox_rrd_api_types::RrdTimeframe,
-    remote_name: &String,
-    res: Resource,
-    name: String,
-    metric: &str,
-) -> Option<(f64, TopEntity)> {
-    if let Ok(Some(values)) = rrd_cache::extract_data(
-        &name,
-        metric,
-        timeframe,
-        proxmox_rrd_api_types::RrdMode::Average,
-    ) {
-        let coefficient = calculate_coefficient(&values);
-        if coefficient > 0.0 {
-            return Some((
-                coefficient,
-                TopEntity {
-                    remote: remote_name.to_string(),
-                    resource: res,
-                    rrd_data: ResourceRrdData {
-                        start: values.start,
-                        resolution: values.resolution,
-                        data: values.data,
-                    },
-                },
-            ));
-        }
-    }
-
-    None
-}
diff --git a/server/src/metric_collection/top_entities.rs b/server/src/metric_collection/top_entities.rs
new file mode 100644
index 00000000..f8e053fb
--- /dev/null
+++ b/server/src/metric_collection/top_entities.rs
@@ -0,0 +1,148 @@
+use std::collections::HashMap;
+
+use pdm_api_types::resource::{Resource, ResourceRrdData, TopEntities, TopEntity};
+
+use super::rrd_cache;
+
+fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) {
+    let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) {
+        Ok(idx) | Err(idx) => idx,
+    };
+
+    vec.insert(index, value);
+    if vec.len() > limit {
+        for _ in 0..(vec.len() - limit) {
+            vec.remove(0);
+        }
+    }
+}
+
+// for now simple sum of the values => area under the graph curve
+fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 {
+    let mut coefficient = 0.0;
+    for point in values.data.iter() {
+        let value = point.unwrap_or_default();
+        if value.is_finite() {
+            coefficient += value;
+        }
+    }
+
+    coefficient
+}
+
+// FIXME: cache the values instead of calculate freshly every time?
+// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache)
+pub fn calculate_top(
+    remotes: &HashMap<String, pdm_api_types::remotes::Remote>,
+    timeframe: proxmox_rrd_api_types::RrdTimeframe,
+    num: usize,
+) -> TopEntities {
+    let mut guest_cpu = Vec::new();
+    let mut node_cpu = Vec::new();
+    let mut node_memory = Vec::new();
+
+    for remote_name in remotes.keys() {
+        if let Some(data) =
+            crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64)
+        {
+            for res in data.resources {
+                let id = res.id().to_string();
+                let name = format!("pve/{remote_name}/{id}");
+                match &res {
+                    Resource::PveStorage(_) => {}
+                    Resource::PveQemu(_) | Resource::PveLxc(_) => {
+                        if let Some(entity) =
+                            get_entity(timeframe, remote_name, res, name, "cpu_current")
+                        {
+                            let coefficient = (entity.0 * 100.0).round() as usize;
+                            insert_sorted(&mut guest_cpu, (coefficient, entity.1), num);
+                        }
+                    }
+                    Resource::PveNode(_) => {
+                        if let Some(entity) = get_entity(
+                            timeframe,
+                            remote_name,
+                            res.clone(),
+                            name.clone(),
+                            "cpu_current",
+                        ) {
+                            let coefficient = (entity.0 * 100.0).round() as usize;
+                            insert_sorted(&mut node_cpu, (coefficient, entity.1), num);
+                        }
+                        // convert mem/mem_total into a single entity
+                        if let Some(mut mem) = get_entity(
+                            timeframe,
+                            remote_name,
+                            res.clone(),
+                            name.clone(),
+                            "mem_used",
+                        ) {
+                            if let Some(mem_total) =
+                                get_entity(timeframe, remote_name, res, name, "mem_total")
+                            {
+                                // skip if we don't have the same amount of data for used and total
+                                let mem_rrd = &mem.1.rrd_data.data;
+                                let mem_total_rrd = &mem_total.1.rrd_data.data;
+                                if mem_rrd.len() != mem_total_rrd.len() {
+                                    continue;
+                                }
+                                let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize;
+                                let mut mem_usage = Vec::new();
+                                for i in 0..mem_rrd.len() {
+                                    let point = match (mem_rrd[i], mem_total_rrd[i]) {
+                                        (Some(mem), Some(total)) => Some(mem / total),
+                                        _ => None,
+                                    };
+                                    mem_usage.push(point)
+                                }
+                                mem.1.rrd_data.data = mem_usage;
+                                insert_sorted(&mut node_memory, (coefficient, mem.1), num);
+                            }
+                        }
+                    }
+                    Resource::PbsNode(_) => {}
+                    Resource::PbsDatastore(_) => {}
+                }
+            }
+        }
+    }
+
+    TopEntities {
+        guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(),
+        node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(),
+        node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(),
+    }
+}
+
+fn get_entity(
+    timeframe: proxmox_rrd_api_types::RrdTimeframe,
+    remote_name: &String,
+    res: Resource,
+    name: String,
+    metric: &str,
+) -> Option<(f64, TopEntity)> {
+    if let Ok(Some(values)) = rrd_cache::extract_data(
+        &name,
+        metric,
+        timeframe,
+        proxmox_rrd_api_types::RrdMode::Average,
+    ) {
+        let coefficient = calculate_coefficient(&values);
+        if coefficient > 0.0 {
+            return Some((
+                coefficient,
+                TopEntity {
+                    remote: remote_name.to_string(),
+                    resource: res,
+                    rrd_data: ResourceRrdData {
+                        start: values.start,
+                        resolution: values.resolution,
+                        data: values.data,
+                    },
+                },
+            ));
+        }
+    }
+
+    None
+}
-- 
2.39.5



_______________________________________________
pdm-devel mailing list
pdm-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel

