From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 32CED1FF0B6 for ; Thu, 12 Nov 2026 10:46:54 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id F0BDB21731; Thu, 12 Nov 2026 10:45:33 +0100 (CET) From: Dominik Rusovac To: pve-devel@lists.proxmox.com Subject: [PATCH proxmox v2 06/19] fix #7993: resource-scheduling: scheduler: add weights Date: Thu, 12 Nov 2026 10:44:24 +0100 Message-ID: <20261112094437.1951271-7-d.rusovac@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261112094437.1951271-1-d.rusovac@proxmox.com> References: <20261112094437.1951271-1-d.rusovac@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794476685547 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.033 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 3GA5HRSNJBPGTPNO4DT44QTWUSTCKXTJ X-Message-ID-Hash: 3GA5HRSNJBPGTPNO4DT44QTWUSTCKXTJ X-MailFrom: d.rusovac@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: CRS did not rebalance a cluster that was clearly imbalanced in CPU load or memory usage alone, because the combined node load looked balanced enough [0] add weights to the scheduler, so that callers can prioritize balancing one resource over the other load imbalance computations and the bruteforce method use the weighted node load for the TOPSIS methods, the weights replace the hardcoded criteria weights from commit 2b325aa5 ("add pve_static module"). those favor memory usage 5 times over CPU load. Scaling them with the weights instead would make the weights relative to that built-in emphasis, so 1:1 would effectively mean 1:5, which is undesirable. only the factor of 2 between the criteria for the highest and the average usage of each resource is kept, so custom weights of 1:5 reproduce the previous criteria weights of 1:5 (average) and 2:10 (highest) the same weights are used for the TOPSIS criteria and the node load, so the imbalance reported for the migrations ranked by TOPSIS is weighted like the criteria that ranked them the weights default to 1:1. This changes the default ranking of the TOPSIS methods (expected result of one affected test was adjusted), so callers have to pass hardcoded legacy topsis weights to keep their previous rankings. with these, the node load is weighted 1:5 as well and the previous rankings are preserved, which is also being tested [0] https://bugzilla.proxmox.com/show_bug.cgi?id=7993 Signed-off-by: Dominik Rusovac --- Notes: new with changes regarding patch [0] in v1: * add weights as field to Scheduler struct * weigh the TOPSIS criteria directly with the weights, keeping the factor of 2 between the highest and the average usage criteria of each resource - weights of 1:5 reproduce the previous criteria weights 1, 2, 5, 10 - add LEGACY_TOPSIS_WEIGHTS for callers keeping the previous rankings * fall back to previous TOPSIS ranking of migration candidates * no longer expose score and keep ScoredMigration as is * adjust one scheduler test's expected result * test that legacy topsis weights preserve previous rankings [0] https://lore.proxmox.com/all/20261021085932.1196302-4-d.rusovac@proxmox.com/ proxmox-resource-scheduling/src/pve_static.rs | 4 +- proxmox-resource-scheduling/src/scheduler.rs | 105 ++++++++++++++---- .../tests/scheduler.rs | 64 ++++++++++- 3 files changed, 149 insertions(+), 24 deletions(-) diff --git a/proxmox-resource-scheduling/src/pve_static.rs b/proxmox-resource-scheduling/src/pve_static.rs index 40b76cbc..25439a0e 100644 --- a/proxmox-resource-scheduling/src/pve_static.rs +++ b/proxmox-resource-scheduling/src/pve_static.rs @@ -1,7 +1,7 @@ use anyhow::Error; use serde::{Deserialize, Serialize}; -use crate::scheduler::{NodeUsage, Scheduler}; +use crate::scheduler::{LEGACY_TOPSIS_WEIGHTS, NodeUsage, Scheduler}; use crate::{node::NodeStats, resource::ResourceStats}; #[derive(Clone, Serialize, Deserialize)] @@ -94,7 +94,7 @@ pub fn score_nodes_to_start_service>( .map(|node| node.as_ref().clone().into()) .collect::>(); - let scheduler = Scheduler::from_nodes(nodes); + let scheduler = Scheduler::from_nodes(nodes).with_weights(LEGACY_TOPSIS_WEIGHTS); scheduler.score_nodes_to_start_resource(*service) } diff --git a/proxmox-resource-scheduling/src/scheduler.rs b/proxmox-resource-scheduling/src/scheduler.rs index a9390357..75929a14 100644 --- a/proxmox-resource-scheduling/src/scheduler.rs +++ b/proxmox-resource-scheduling/src/scheduler.rs @@ -1,6 +1,10 @@ use anyhow::Error; -use crate::{node::NodeStats, resource::ResourceStats, topsis}; +use crate::{ + node::{NodeStats, NodeStatsWeights}, + resource::ResourceStats, + topsis, +}; use serde::{Deserialize, Serialize}; use std::{ @@ -69,26 +73,69 @@ fn node_stats_after_migration(candidate: &MigrationCandidate, node: &NodeUsage) new_stats } -criteria_struct! { - /// A given alternative. - struct PveTopsisAlternative { - #[criterion("average CPU", -1.0)] - average_cpu: f64, - #[criterion("highest CPU", -2.0)] - highest_cpu: f64, - #[criterion("average memory", -5.0)] - average_memory: f64, - #[criterion("highest memory", -10.0)] - highest_memory: f64, +/// A given alternative. +struct PveTopsisAlternative { + average_cpu: f64, + highest_cpu: f64, + average_memory: f64, + highest_memory: f64, +} + +const N_CRITERIA: usize = 4; + +impl From for [f64; N_CRITERIA] { + fn from(alternative: PveTopsisAlternative) -> Self { + [ + alternative.average_cpu, + alternative.highest_cpu, + alternative.average_memory, + alternative.highest_memory, + ] } +} - const N_CRITERIA; - static PVE_HA_TOPSIS_CRITERIA; +/// Factor to compute the weight of the criterion for the highest usage of a resource relative to +/// the criterion for the average usage of that resource. This keeps the ratio of the criteria +/// weights introduced in commit 2b325aa5. +/// +/// For more details see [`topsis_criteria`]. +const TOPSIS_HIGHEST_USAGE_FACTOR: f64 = 2.0; + +/// Weights that reproduce the TOPSIS criteria weights introduced in commit 2b325aa5 with +/// [`topsis_criteria`]. +pub const LEGACY_TOPSIS_WEIGHTS: NodeStatsWeights = NodeStatsWeights { + cpu_weight: 1.0, + mem_weight: 5.0, +}; + +/// Returns the TOPSIS criteria for a [`PveTopsisAlternative`], weighted by `weights`. +/// +/// The criteria for the average usage of each resource get the respective weight in `weights`, +/// the criteria for the highest usage get [`TOPSIS_HIGHEST_USAGE_FACTOR`] times that weight. +fn topsis_criteria(weights: &NodeStatsWeights) -> Result, Error> { + let NodeStatsWeights { + cpu_weight, + mem_weight, + } = *weights; + + topsis::Criteria::new([ + topsis::Criterion::new("average CPU".to_owned(), -cpu_weight), + topsis::Criterion::new( + "highest CPU".to_owned(), + -TOPSIS_HIGHEST_USAGE_FACTOR * cpu_weight, + ), + topsis::Criterion::new("average memory".to_owned(), -mem_weight), + topsis::Criterion::new( + "highest memory".to_owned(), + -TOPSIS_HIGHEST_USAGE_FACTOR * mem_weight, + ), + ]) } #[derive(Clone, Debug)] pub struct Scheduler { nodes: Vec, + weights: NodeStatsWeights, } /// A possible migration. @@ -176,9 +223,28 @@ impl Scheduler { { Self { nodes: nodes.into_iter().map(|node| node.into()).collect(), + weights: NodeStatsWeights::EQUAL, } } + /// Sets the `weights` controlling how much balancing the CPU load and the memory usage of the + /// nodes is prioritized relative to each other. + /// + /// For [`Self::node_imbalance`], [`Self::node_imbalance_with_migration_candidate`], the + /// bruteforce method [`Self::score_best_balancing_migration_candidates`] and the imbalance + /// reported by [`Self::score_best_balancing_migration_candidates_topsis`], the `weights` + /// determine the [`NodeStats::weighted_load`] of each node. + /// + /// For the TOPSIS-based methods, [`Self::score_best_balancing_migration_candidates_topsis`] and + /// [`Self::score_nodes_to_start_resource`], the `weights` also weigh the average CPU and + /// average memory criteria, see [`topsis_criteria`]. Use [`LEGACY_TOPSIS_WEIGHTS`] to keep the + /// criteria weights introduced in commit 2b325aa5. + pub fn with_weights(mut self, weights: NodeStatsWeights) -> Self { + self.weights = weights; + + self + } + /// Maps the current node usages to a [`PveTopsisAlternative`]. /// /// The [`PveTopsisAlternative`] is derived by calculating a modified version of the root mean @@ -221,7 +287,7 @@ impl Scheduler { /// /// See [`calculate_node_imbalance`] for more information. pub fn node_imbalance(&self) -> f64 { - calculate_node_imbalance(&self.nodes, |node| node.stats.load()) + calculate_node_imbalance(&self.nodes, |node| node.stats.weighted_load(&self.weights)) } /// Returns the load imbalance among the nodes as if a specific resource was moved. @@ -229,7 +295,7 @@ impl Scheduler { /// See [`calculate_node_imbalance`] for more information. pub fn node_imbalance_with_migration_candidate(&self, candidate: &MigrationCandidate) -> f64 { calculate_node_imbalance(&self.nodes, |node| { - node_stats_after_migration(candidate, node).load() + node_stats_after_migration(candidate, node).weighted_load(&self.weights) }) } @@ -297,8 +363,9 @@ impl Scheduler { }) .collect::>(); + let criteria = topsis_criteria(&self.weights)?; let best_alternatives = - topsis::rank_alternatives(&topsis::Matrix::new(matrix)?, &PVE_HA_TOPSIS_CRITERIA)?; + topsis::rank_alternatives(&topsis::Matrix::new(matrix)?, &criteria)?; Ok(best_alternatives .into_iter() @@ -343,8 +410,8 @@ impl Scheduler { }) .collect::>(); - let scores = - topsis::score_alternatives(&topsis::Matrix::new(matrix)?, &PVE_HA_TOPSIS_CRITERIA)?; + let criteria = topsis_criteria(&self.weights)?; + let scores = topsis::score_alternatives(&topsis::Matrix::new(matrix)?, &criteria)?; Ok(scores .into_iter() diff --git a/proxmox-resource-scheduling/tests/scheduler.rs b/proxmox-resource-scheduling/tests/scheduler.rs index 61fdd840..603a9418 100644 --- a/proxmox-resource-scheduling/tests/scheduler.rs +++ b/proxmox-resource-scheduling/tests/scheduler.rs @@ -2,7 +2,7 @@ use anyhow::Error; use proxmox_resource_scheduling::{ node::NodeStats, resource::ResourceStats, - scheduler::{Migration, MigrationCandidate, NodeUsage, Scheduler}, + scheduler::{LEGACY_TOPSIS_WEIGHTS, Migration, MigrationCandidate, NodeUsage, Scheduler}, }; fn new_empty_cluster_scheduler() -> Scheduler { @@ -317,7 +317,7 @@ fn test_score_homogeneous_nodes_to_start_resource() -> Result<(), Error> { assert_eq!( rank_nodes_to_start_resource(&scheduler, combined_resource_stats)?, - vec!["node2", "node3", "node1"] + vec!["node1", "node3", "node2"] ); Ok(()) @@ -325,7 +325,7 @@ fn test_score_homogeneous_nodes_to_start_resource() -> Result<(), Error> { #[test] fn test_score_heterogeneous_nodes_to_start_resource() -> Result<(), Error> { - let scheduler = new_heterogeneous_cluster_scheduler(); + let scheduler = new_heterogeneous_cluster_scheduler().with_weights(LEGACY_TOPSIS_WEIGHTS); let heavy_memory_resource_stats = ResourceStats { cpu: 0.0, @@ -377,3 +377,61 @@ fn test_score_heterogeneous_nodes_to_start_resource() -> Result<(), Error> { Ok(()) } + +#[test] +fn test_legacy_topsis_weights_keep_unweighted_rankings() -> Result<(), Error> { + let resource_stats = + [(1.0, 12 << 30), (12.0, 0), (0.0, 0), (12.0, 12 << 30)].map(|(maxcpu, maxmem)| { + ResourceStats { + cpu: 0.0, + maxcpu, + mem: 0, + maxmem, + } + }); + + // rankings with the TOPSIS criteria weights from before the weights became adjustable + let expected_homogeneous_rankings = [ + ["node2", "node3", "node1"], + ["node1", "node3", "node2"], + ["node1", "node3", "node2"], + ["node2", "node3", "node1"], + ]; + let expected_heterogeneous_rankings = [ + ["node2", "node1", "node3"], + ["node3", "node2", "node1"], + ["node1", "node3", "node2"], + ["node2", "node1", "node3"], + ]; + + let scenarios: [(fn() -> Scheduler, _); 2] = [ + ( + new_homogeneous_cluster_scheduler, + expected_homogeneous_rankings, + ), + ( + new_heterogeneous_cluster_scheduler, + expected_heterogeneous_rankings, + ), + ]; + + for (new_scheduler, expected_rankings) in scenarios { + let legacy = new_scheduler().with_weights(LEGACY_TOPSIS_WEIGHTS); + + for (resource_stats, expected_ranking) in resource_stats.iter().zip(expected_rankings) { + assert_eq!( + rank_nodes_to_start_resource(&legacy, *resource_stats)?, + expected_ranking + ); + } + + let (candidates, migration1, migration2) = new_simple_migration_candidates(); + + assert_eq!( + rank_best_balancing_migration_candidates_topsis(&legacy, &candidates, 2)?, + vec![migration1, migration2] + ); + } + + Ok(()) +} -- 2.47.3