From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pdm-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9])
by lore.proxmox.com (Postfix) with ESMTPS id C5F451FF164
for <inbox@lore.proxmox.com>; Fri, 14 Feb 2025 14:07:49 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 3210719E4F;
Fri, 14 Feb 2025 14:07:50 +0100 (CET)
From: Lukas Wagner <l.wagner@proxmox.com>
To: pdm-devel@lists.proxmox.com
Date: Fri, 14 Feb 2025 14:06:30 +0100
Message-Id: <20250214130653.283012-6-l.wagner@proxmox.com>
X-Mailer: git-send-email 2.39.5
In-Reply-To: <20250214130653.283012-1-l.wagner@proxmox.com>
References: <20250214130653.283012-1-l.wagner@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.010 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pdm-devel] [PATCH proxmox-datacenter-manager v2 05/28] metric
collection: split top_entities split into separate module
X-BeenThere: pdm-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Datacenter Manager development discussion
<pdm-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pdm-devel>,
<mailto:pdm-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pdm-devel/>
List-Post: <mailto:pdm-devel@lists.proxmox.com>
List-Help: <mailto:pdm-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel>,
<mailto:pdm-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Datacenter Manager development discussion
<pdm-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pdm-devel-bounces@lists.proxmox.com
Sender: "pdm-devel" <pdm-devel-bounces@lists.proxmox.com>
This makes the parent module a bit more easy not navigate.
No functional changes intended.
Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
---
server/src/api/resources.rs | 3 +-
server/src/metric_collection/mod.rs | 149 +------------------
server/src/metric_collection/top_entities.rs | 148 ++++++++++++++++++
3 files changed, 152 insertions(+), 148 deletions(-)
create mode 100644 server/src/metric_collection/top_entities.rs
diff --git a/server/src/api/resources.rs b/server/src/api/resources.rs
index 453d9e8e..afda51e3 100644
--- a/server/src/api/resources.rs
+++ b/server/src/api/resources.rs
@@ -26,6 +26,7 @@ use proxmox_subscription::SubscriptionStatus;
use pve_api_types::{ClusterResource, ClusterResourceType};
use crate::connection;
+use crate::metric_collection::top_entities;
pub const ROUTER: Router = Router::new()
.get(&list_subdirs_api_method!(SUBDIRS))
@@ -320,7 +321,7 @@ async fn get_top_entities(timeframe: Option<RrdTimeframe>) -> Result<TopEntities
let (remotes_config, _) = pdm_config::remotes::config()?;
let timeframe = timeframe.unwrap_or(RrdTimeframe::Day);
- let res = crate::metric_collection::calculate_top(&remotes_config.sections, timeframe, 10);
+ let res = top_entities::calculate_top(&remotes_config.sections, timeframe, 10);
Ok(res)
}
diff --git a/server/src/metric_collection/mod.rs b/server/src/metric_collection/mod.rs
index b37d6782..5540f937 100644
--- a/server/src/metric_collection/mod.rs
+++ b/server/src/metric_collection/mod.rs
@@ -6,15 +6,13 @@ use anyhow::Error;
use pbs_api_types::{MetricDataPoint, MetricDataType};
use proxmox_rrd::rrd::DataSourceType;
-use pdm_api_types::{
- remotes::RemoteType,
- resource::{Resource, ResourceRrdData, TopEntities, TopEntity},
-};
+use pdm_api_types::remotes::RemoteType;
use pve_api_types::{ClusterMetricsData, ClusterMetricsDataType};
use crate::{connection, task_utils};
pub mod rrd_cache;
+pub mod top_entities;
const COLLECTION_INTERVAL: u64 = 60;
@@ -150,146 +148,3 @@ fn store_metric_pbs(remote_name: &str, data_point: &MetricDataPoint) {
data_source_type,
);
}
-
-fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) {
- let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) {
- Ok(idx) | Err(idx) => idx,
- };
-
- vec.insert(index, value);
- if vec.len() > limit {
- for _ in 0..(vec.len() - limit) {
- vec.remove(0);
- }
- }
-}
-
-// for now simple sum of the values => area under the graph curve
-fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 {
- let mut coefficient = 0.0;
- for point in values.data.iter() {
- let value = point.unwrap_or_default();
- if value.is_finite() {
- coefficient += value;
- }
- }
-
- coefficient
-}
-
-// FIXME: cache the values instead of calculate freshly every time?
-// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache)
-pub fn calculate_top(
- remotes: &HashMap<String, pdm_api_types::remotes::Remote>,
- timeframe: proxmox_rrd_api_types::RrdTimeframe,
- num: usize,
-) -> TopEntities {
- let mut guest_cpu = Vec::new();
- let mut node_cpu = Vec::new();
- let mut node_memory = Vec::new();
-
- for remote_name in remotes.keys() {
- if let Some(data) =
- crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64)
- {
- for res in data.resources {
- let id = res.id().to_string();
- let name = format!("pve/{remote_name}/{id}");
- match &res {
- Resource::PveStorage(_) => {}
- Resource::PveQemu(_) | Resource::PveLxc(_) => {
- if let Some(entity) =
- get_entity(timeframe, remote_name, res, name, "cpu_current")
- {
- let coefficient = (entity.0 * 100.0).round() as usize;
- insert_sorted(&mut guest_cpu, (coefficient, entity.1), num);
- }
- }
- Resource::PveNode(_) => {
- if let Some(entity) = get_entity(
- timeframe,
- remote_name,
- res.clone(),
- name.clone(),
- "cpu_current",
- ) {
- let coefficient = (entity.0 * 100.0).round() as usize;
- insert_sorted(&mut node_cpu, (coefficient, entity.1), num);
- }
- // convert mem/mem_total into a single entity
- if let Some(mut mem) = get_entity(
- timeframe,
- remote_name,
- res.clone(),
- name.clone(),
- "mem_used",
- ) {
- if let Some(mem_total) =
- get_entity(timeframe, remote_name, res, name, "mem_total")
- {
- // skip if we don't have the same amount of data for used and total
- let mem_rrd = &mem.1.rrd_data.data;
- let mem_total_rrd = &mem_total.1.rrd_data.data;
- if mem_rrd.len() != mem_total_rrd.len() {
- continue;
- }
- let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize;
- let mut mem_usage = Vec::new();
- for i in 0..mem_rrd.len() {
- let point = match (mem_rrd[i], mem_total_rrd[i]) {
- (Some(mem), Some(total)) => Some(mem / total),
- _ => None,
- };
- mem_usage.push(point)
- }
- mem.1.rrd_data.data = mem_usage;
- insert_sorted(&mut node_memory, (coefficient, mem.1), num);
- }
- }
- }
- Resource::PbsNode(_) => {}
- Resource::PbsDatastore(_) => {}
- }
- }
- }
- }
-
- TopEntities {
- guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(),
- node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(),
- node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(),
- }
-}
-
-fn get_entity(
- timeframe: proxmox_rrd_api_types::RrdTimeframe,
- remote_name: &String,
- res: Resource,
- name: String,
- metric: &str,
-) -> Option<(f64, TopEntity)> {
- if let Ok(Some(values)) = rrd_cache::extract_data(
- &name,
- metric,
- timeframe,
- proxmox_rrd_api_types::RrdMode::Average,
- ) {
- let coefficient = calculate_coefficient(&values);
- if coefficient > 0.0 {
- return Some((
- coefficient,
- TopEntity {
- remote: remote_name.to_string(),
- resource: res,
- rrd_data: ResourceRrdData {
- start: values.start,
- resolution: values.resolution,
- data: values.data,
- },
- },
- ));
- }
- }
-
- None
-}
diff --git a/server/src/metric_collection/top_entities.rs b/server/src/metric_collection/top_entities.rs
new file mode 100644
index 00000000..f8e053fb
--- /dev/null
+++ b/server/src/metric_collection/top_entities.rs
@@ -0,0 +1,148 @@
+use std::collections::HashMap;
+
+use pdm_api_types::resource::{Resource, ResourceRrdData, TopEntities, TopEntity};
+
+use super::rrd_cache;
+
+fn insert_sorted<T>(vec: &mut Vec<(usize, T)>, value: (usize, T), limit: usize) {
+ let index = match vec.binary_search_by_key(&value.0, |(idx, _)| *idx) {
+ Ok(idx) | Err(idx) => idx,
+ };
+
+ vec.insert(index, value);
+ if vec.len() > limit {
+ for _ in 0..(vec.len() - limit) {
+ vec.remove(0);
+ }
+ }
+}
+
+// for now simple sum of the values => area under the graph curve
+fn calculate_coefficient(values: &proxmox_rrd::Entry) -> f64 {
+ let mut coefficient = 0.0;
+ for point in values.data.iter() {
+ let value = point.unwrap_or_default();
+ if value.is_finite() {
+ coefficient += value;
+ }
+ }
+
+ coefficient
+}
+
+// FIXME: cache the values instead of calculate freshly every time?
+// FIXME: find better way to enumerate nodes/guests/etc.(instead of relying on the cache)
+pub fn calculate_top(
+ remotes: &HashMap<String, pdm_api_types::remotes::Remote>,
+ timeframe: proxmox_rrd_api_types::RrdTimeframe,
+ num: usize,
+) -> TopEntities {
+ let mut guest_cpu = Vec::new();
+ let mut node_cpu = Vec::new();
+ let mut node_memory = Vec::new();
+
+ for remote_name in remotes.keys() {
+ if let Some(data) =
+ crate::api::resources::get_cached_resources(remote_name, i64::MAX as u64)
+ {
+ for res in data.resources {
+ let id = res.id().to_string();
+ let name = format!("pve/{remote_name}/{id}");
+ match &res {
+ Resource::PveStorage(_) => {}
+ Resource::PveQemu(_) | Resource::PveLxc(_) => {
+ if let Some(entity) =
+ get_entity(timeframe, remote_name, res, name, "cpu_current")
+ {
+ let coefficient = (entity.0 * 100.0).round() as usize;
+ insert_sorted(&mut guest_cpu, (coefficient, entity.1), num);
+ }
+ }
+ Resource::PveNode(_) => {
+ if let Some(entity) = get_entity(
+ timeframe,
+ remote_name,
+ res.clone(),
+ name.clone(),
+ "cpu_current",
+ ) {
+ let coefficient = (entity.0 * 100.0).round() as usize;
+ insert_sorted(&mut node_cpu, (coefficient, entity.1), num);
+ }
+ // convert mem/mem_total into a single entity
+ if let Some(mut mem) = get_entity(
+ timeframe,
+ remote_name,
+ res.clone(),
+ name.clone(),
+ "mem_used",
+ ) {
+ if let Some(mem_total) =
+ get_entity(timeframe, remote_name, res, name, "mem_total")
+ {
+ // skip if we don't have the same amount of data for used and total
+ let mem_rrd = &mem.1.rrd_data.data;
+ let mem_total_rrd = &mem_total.1.rrd_data.data;
+ if mem_rrd.len() != mem_total_rrd.len() {
+ continue;
+ }
+ let coefficient = (100.0 * mem.0 / mem_total.0).round() as usize;
+ let mut mem_usage = Vec::new();
+ for i in 0..mem_rrd.len() {
+ let point = match (mem_rrd[i], mem_total_rrd[i]) {
+ (Some(mem), Some(total)) => Some(mem / total),
+ _ => None,
+ };
+ mem_usage.push(point)
+ }
+ mem.1.rrd_data.data = mem_usage;
+ insert_sorted(&mut node_memory, (coefficient, mem.1), num);
+ }
+ }
+ }
+ Resource::PbsNode(_) => {}
+ Resource::PbsDatastore(_) => {}
+ }
+ }
+ }
+ }
+
+ TopEntities {
+ guest_cpu: guest_cpu.into_iter().map(|(_, entity)| entity).collect(),
+ node_cpu: node_cpu.into_iter().map(|(_, entity)| entity).collect(),
+ node_memory: node_memory.into_iter().map(|(_, entity)| entity).collect(),
+ }
+}
+
+fn get_entity(
+ timeframe: proxmox_rrd_api_types::RrdTimeframe,
+ remote_name: &String,
+ res: Resource,
+ name: String,
+ metric: &str,
+) -> Option<(f64, TopEntity)> {
+ if let Ok(Some(values)) = rrd_cache::extract_data(
+ &name,
+ metric,
+ timeframe,
+ proxmox_rrd_api_types::RrdMode::Average,
+ ) {
+ let coefficient = calculate_coefficient(&values);
+ if coefficient > 0.0 {
+ return Some((
+ coefficient,
+ TopEntity {
+ remote: remote_name.to_string(),
+ resource: res,
+ rrd_data: ResourceRrdData {
+ start: values.start,
+ resolution: values.resolution,
+ data: values.data,
+ },
+ },
+ ));
+ }
+ }
+
+ None
+}
--
2.39.5
_______________________________________________
pdm-devel mailing list
pdm-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel