From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 3C7C61FF0DE for ; Thu, 19 Nov 2026 13:40:07 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 97EA0216CA; Thu, 19 Nov 2026 13:40:05 +0100 (CET) Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=UTF-8 Date: Thu, 19 Nov 2026 13:40:00 +0100 Message-Id: Subject: Re: [PATCH proxmox v2 06/19] fix #7993: resource-scheduling: scheduler: add weights From: "Daniel Kral" To: "Dominik Rusovac" , X-Mailer: aerc 0.22.0-45-gc30c872fde7b-dirty References: <20261112094437.1951271-1-d.rusovac@proxmox.com> <20261112094437.1951271-7-d.rusovac@proxmox.com> In-Reply-To: <20261112094437.1951271-7-d.rusovac@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795092000457 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.438 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 2AWHSKUY6ST6776353Y2PYBUTRGXQEL7 X-Message-ID-Hash: 2AWHSKUY6ST6776353Y2PYBUTRGXQEL7 X-MailFrom: d.kral@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On Thu Nov 12, 2026 at 10:44 AM CET, Dominik Rusovac wrote: > CRS did not rebalance a cluster that was clearly imbalanced in CPU load > or memory usage alone, because the combined node load looked balanced > enough [0] > > add weights to the scheduler, so that callers can prioritize balancing > one resource over the other > > load imbalance computations and the bruteforce method use the weighted > node load > > for the TOPSIS methods, the weights replace the hardcoded criteria > weights from commit 2b325aa5 ("add pve_static module"). those favor > memory usage 5 times over CPU load. Scaling them with the weights > instead would make the weights relative to that built-in emphasis, so > 1:1 would effectively mean 1:5, which is undesirable. only the factor of > 2 between the criteria for the highest and the average usage of each > resource is kept, so custom weights of 1:5 reproduce the previous > criteria weights of 1:5 (average) and 2:10 (highest) > > the same weights are used for the TOPSIS criteria and the node load, so > the imbalance reported for the migrations ranked by TOPSIS is weighted > like the criteria that ranked them > > the weights default to 1:1. This changes the default ranking of the > TOPSIS methods (expected result of one affected test was adjusted), so > callers have to pass hardcoded legacy topsis weights to keep their > previous rankings. with these, the node load is weighted 1:5 as well and > the previous rankings are preserved, which is also being tested > > [0] https://bugzilla.proxmox.com/show_bug.cgi?id=3D7993 > > Signed-off-by: Dominik Rusovac > --- > > Notes: > new with changes regarding patch [0] in v1: > * add weights as field to Scheduler struct > * weigh the TOPSIS criteria directly with the weights, keeping the fa= ctor > of 2 between the highest and the average usage criteria of each res= ource > - weights of 1:5 reproduce the previous criteria weights 1, 2, 5, 1= 0 > - add LEGACY_TOPSIS_WEIGHTS for callers keeping the previous rankin= gs > * fall back to previous TOPSIS ranking of migration candidates > * no longer expose score and keep ScoredMigration as is > * adjust one scheduler test's expected result > * test that legacy topsis weights preserve previous rankings > =20 > [0] https://lore.proxmox.com/all/20261021085932.1196302-4-d.rusovac@p= roxmox.com/ > > proxmox-resource-scheduling/src/pve_static.rs | 4 +- > proxmox-resource-scheduling/src/scheduler.rs | 105 ++++++++++++++---- > .../tests/scheduler.rs | 64 ++++++++++- > 3 files changed, 149 insertions(+), 24 deletions(-) > > diff --git a/proxmox-resource-scheduling/src/pve_static.rs b/proxmox-reso= urce-scheduling/src/pve_static.rs > index 40b76cbc..25439a0e 100644 > --- a/proxmox-resource-scheduling/src/pve_static.rs > +++ b/proxmox-resource-scheduling/src/pve_static.rs > @@ -1,7 +1,7 @@ > use anyhow::Error; > use serde::{Deserialize, Serialize}; > =20 > -use crate::scheduler::{NodeUsage, Scheduler}; > +use crate::scheduler::{LEGACY_TOPSIS_WEIGHTS, NodeUsage, Scheduler}; > use crate::{node::NodeStats, resource::ResourceStats}; > =20 > #[derive(Clone, Serialize, Deserialize)] > @@ -94,7 +94,7 @@ pub fn score_nodes_to_start_service>( > .map(|node| node.as_ref().clone().into()) > .collect::>(); > =20 > - let scheduler =3D Scheduler::from_nodes(nodes); > + let scheduler =3D Scheduler::from_nodes(nodes).with_weights(LEGACY_T= OPSIS_WEIGHTS); > =20 > scheduler.score_nodes_to_start_resource(*service) > } > diff --git a/proxmox-resource-scheduling/src/scheduler.rs b/proxmox-resou= rce-scheduling/src/scheduler.rs > index a9390357..75929a14 100644 > --- a/proxmox-resource-scheduling/src/scheduler.rs > +++ b/proxmox-resource-scheduling/src/scheduler.rs > @@ -1,6 +1,10 @@ > use anyhow::Error; > =20 > -use crate::{node::NodeStats, resource::ResourceStats, topsis}; > +use crate::{ > + node::{NodeStats, NodeStatsWeights}, > + resource::ResourceStats, > + topsis, > +}; > =20 > use serde::{Deserialize, Serialize}; > use std::{ > @@ -69,26 +73,69 @@ fn node_stats_after_migration(candidate: &MigrationCa= ndidate, node: &NodeUsage) > new_stats > } > =20 > -criteria_struct! { > - /// A given alternative. > - struct PveTopsisAlternative { > - #[criterion("average CPU", -1.0)] > - average_cpu: f64, > - #[criterion("highest CPU", -2.0)] > - highest_cpu: f64, > - #[criterion("average memory", -5.0)] > - average_memory: f64, > - #[criterion("highest memory", -10.0)] > - highest_memory: f64, > +/// A given alternative. > +struct PveTopsisAlternative { > + average_cpu: f64, > + highest_cpu: f64, > + average_memory: f64, > + highest_memory: f64, > +} > + > +const N_CRITERIA: usize =3D 4; > + > +impl From for [f64; N_CRITERIA] { > + fn from(alternative: PveTopsisAlternative) -> Self { > + [ > + alternative.average_cpu, > + alternative.highest_cpu, > + alternative.average_memory, > + alternative.highest_memory, > + ] > } > +} > =20 > - const N_CRITERIA; > - static PVE_HA_TOPSIS_CRITERIA; > +/// Factor to compute the weight of the criterion for the highest usage = of a resource relative to > +/// the criterion for the average usage of that resource. This keeps the= ratio of the criteria > +/// weights introduced in commit 2b325aa5. nit: include the commit title here, i.e. 2b325aa5 ("add pve_static module") > +/// > +/// For more details see [`topsis_criteria`]. > +const TOPSIS_HIGHEST_USAGE_FACTOR: f64 =3D 2.0; > + > +/// Weights that reproduce the TOPSIS criteria weights introduced in com= mit 2b325aa5 with here as well > +/// [`topsis_criteria`]. > +pub const LEGACY_TOPSIS_WEIGHTS: NodeStatsWeights =3D NodeStatsWeights { > + cpu_weight: 1.0, > + mem_weight: 5.0, > +}; > + > +/// Returns the TOPSIS criteria for a [`PveTopsisAlternative`], weighted= by `weights`. > +/// > +/// The criteria for the average usage of each resource get the respecti= ve weight in `weights`, > +/// the criteria for the highest usage get [`TOPSIS_HIGHEST_USAGE_FACTOR= `] times that weight. > +fn topsis_criteria(weights: &NodeStatsWeights) -> Result, Error> { nit: topsis_criteria_from_weights() or make this a method of Scheduler, which is its only user > + let NodeStatsWeights { > + cpu_weight, > + mem_weight, > + } =3D *weights; > + > + topsis::Criteria::new([ > + topsis::Criterion::new("average CPU".to_owned(), -cpu_weight), > + topsis::Criterion::new( > + "highest CPU".to_owned(), > + -TOPSIS_HIGHEST_USAGE_FACTOR * cpu_weight, > + ), > + topsis::Criterion::new("average memory".to_owned(), -mem_weight)= , > + topsis::Criterion::new( > + "highest memory".to_owned(), > + -TOPSIS_HIGHEST_USAGE_FACTOR * mem_weight, > + ), > + ]) > } > =20 > #[derive(Clone, Debug)] > pub struct Scheduler { > nodes: Vec, > + weights: NodeStatsWeights, > } > =20 > /// A possible migration. > @@ -176,9 +223,28 @@ impl Scheduler { > { > Self { > nodes: nodes.into_iter().map(|node| node.into()).collect(), > + weights: NodeStatsWeights::EQUAL, > } > } > =20 > + /// Sets the `weights` controlling how much balancing the CPU load a= nd the memory usage of the > + /// nodes is prioritized relative to each other. > + /// > + /// For [`Self::node_imbalance`], [`Self::node_imbalance_with_migrat= ion_candidate`], the > + /// bruteforce method [`Self::score_best_balancing_migration_candida= tes`] and the imbalance > + /// reported by [`Self::score_best_balancing_migration_candidates_to= psis`], the `weights` > + /// determine the [`NodeStats::weighted_load`] of each node. > + /// > + /// For the TOPSIS-based methods, [`Self::score_best_balancing_migra= tion_candidates_topsis`] and > + /// [`Self::score_nodes_to_start_resource`], the `weights` also weig= h the average CPU and > + /// average memory criteria, see [`topsis_criteria`]. Use [`LEGACY_T= OPSIS_WEIGHTS`] to keep the > + /// criteria weights introduced in commit 2b325aa5. > + pub fn with_weights(mut self, weights: NodeStatsWeights) -> Self { > + self.weights =3D weights; > + > + self > + } > + > /// Maps the current node usages to a [`PveTopsisAlternative`]. > /// > /// The [`PveTopsisAlternative`] is derived by calculating a modifie= d version of the root mean > @@ -221,7 +287,7 @@ impl Scheduler { > /// > /// See [`calculate_node_imbalance`] for more information. > pub fn node_imbalance(&self) -> f64 { > - calculate_node_imbalance(&self.nodes, |node| node.stats.load()) > + calculate_node_imbalance(&self.nodes, |node| node.stats.weighted= _load(&self.weights)) > } > =20 > /// Returns the load imbalance among the nodes as if a specific reso= urce was moved. > @@ -229,7 +295,7 @@ impl Scheduler { > /// See [`calculate_node_imbalance`] for more information. > pub fn node_imbalance_with_migration_candidate(&self, candidate: &Mi= grationCandidate) -> f64 { > calculate_node_imbalance(&self.nodes, |node| { > - node_stats_after_migration(candidate, node).load() > + node_stats_after_migration(candidate, node).weighted_load(&s= elf.weights) > }) > } > =20 > @@ -297,8 +363,9 @@ impl Scheduler { > }) > .collect::>(); > =20 > + let criteria =3D topsis_criteria(&self.weights)?; > let best_alternatives =3D > - topsis::rank_alternatives(&topsis::Matrix::new(matrix)?, &PV= E_HA_TOPSIS_CRITERIA)?; > + topsis::rank_alternatives(&topsis::Matrix::new(matrix)?, &cr= iteria)?; > =20 > Ok(best_alternatives > .into_iter() > @@ -343,8 +410,8 @@ impl Scheduler { > }) > .collect::>(); > =20 > - let scores =3D > - topsis::score_alternatives(&topsis::Matrix::new(matrix)?, &P= VE_HA_TOPSIS_CRITERIA)?; > + let criteria =3D topsis_criteria(&self.weights)?; > + let scores =3D topsis::score_alternatives(&topsis::Matrix::new(m= atrix)?, &criteria)?; > =20 > Ok(scores > .into_iter() > diff --git a/proxmox-resource-scheduling/tests/scheduler.rs b/proxmox-res= ource-scheduling/tests/scheduler.rs > index 61fdd840..603a9418 100644 > --- a/proxmox-resource-scheduling/tests/scheduler.rs > +++ b/proxmox-resource-scheduling/tests/scheduler.rs > @@ -2,7 +2,7 @@ use anyhow::Error; > use proxmox_resource_scheduling::{ > node::NodeStats, > resource::ResourceStats, > - scheduler::{Migration, MigrationCandidate, NodeUsage, Scheduler}, > + scheduler::{LEGACY_TOPSIS_WEIGHTS, Migration, MigrationCandidate, No= deUsage, Scheduler}, > }; > =20 > fn new_empty_cluster_scheduler() -> Scheduler { > @@ -317,7 +317,7 @@ fn test_score_homogeneous_nodes_to_start_resource() -= > Result<(), Error> { is the scheduler here intentionally not augmented with .with_weights(LEGACY_TOPSIS_WEIGHTS) ? > =20 > assert_eq!( > rank_nodes_to_start_resource(&scheduler, combined_resource_stats= )?, > - vec!["node2", "node3", "node1"] > + vec!["node1", "node3", "node2"] > ); > =20 > Ok(()) > @@ -325,7 +325,7 @@ fn test_score_homogeneous_nodes_to_start_resource() -= > Result<(), Error> { > =20 > #[test] > fn test_score_heterogeneous_nodes_to_start_resource() -> Result<(), Erro= r> { > - let scheduler =3D new_heterogeneous_cluster_scheduler(); > + let scheduler =3D new_heterogeneous_cluster_scheduler().with_weights= (LEGACY_TOPSIS_WEIGHTS); > =20 > let heavy_memory_resource_stats =3D ResourceStats { > cpu: 0.0, > @@ -377,3 +377,61 @@ fn test_score_heterogeneous_nodes_to_start_resource(= ) -> Result<(), Error> { > =20 > Ok(()) > } > + > +#[test] > +fn test_legacy_topsis_weights_keep_unweighted_rankings() -> Result<(), E= rror> { > + let resource_stats =3D > + [(1.0, 12 << 30), (12.0, 0), (0.0, 0), (12.0, 12 << 30)].map(|(m= axcpu, maxmem)| { > + ResourceStats { > + cpu: 0.0, > + maxcpu, > + mem: 0, > + maxmem, > + } > + }); > + > + // rankings with the TOPSIS criteria weights from before the weights= became adjustable > + let expected_homogeneous_rankings =3D [ > + ["node2", "node3", "node1"], > + ["node1", "node3", "node2"], > + ["node1", "node3", "node2"], > + ["node2", "node3", "node1"], > + ]; > + let expected_heterogeneous_rankings =3D [ > + ["node2", "node1", "node3"], > + ["node3", "node2", "node1"], > + ["node1", "node3", "node2"], > + ["node2", "node1", "node3"], > + ]; > + > + let scenarios: [(fn() -> Scheduler, _); 2] =3D [ > + ( > + new_homogeneous_cluster_scheduler, > + expected_homogeneous_rankings, > + ), > + ( > + new_heterogeneous_cluster_scheduler, > + expected_heterogeneous_rankings, > + ), > + ]; > + > + for (new_scheduler, expected_rankings) in scenarios { > + let legacy =3D new_scheduler().with_weights(LEGACY_TOPSIS_WEIGHT= S); > + > + for (resource_stats, expected_ranking) in resource_stats.iter().= zip(expected_rankings) { > + assert_eq!( > + rank_nodes_to_start_resource(&legacy, *resource_stats)?, > + expected_ranking > + ); > + } > + > + let (candidates, migration1, migration2) =3D new_simple_migratio= n_candidates(); > + > + assert_eq!( > + rank_best_balancing_migration_candidates_topsis(&legacy, &ca= ndidates, 2)?, > + vec![migration1, migration2] > + ); > + } > + > + Ok(()) > +}