[PATCH pve-ha-manager 10/15] fix #7993: manager,usage: implement weighted auto rebalancing

[Dominik Rusovac <d.rusovac@proxmox.com>]

Include ha-auto-rebalance-cpu-weight and ha-auto-rebalance-mem-weight in
load imbalance calculation and decision making.

The weights only play into decisions in the context of automatic
rebalancing. The bruteforce method optimizes the weighted load
imbalance. For the TOPSIS method ha-auto-rebalance-cpu-weight and
ha-auto-rebalance-mem-weight are assigned to the respective average cpu
and average memory criteria. The weights of the highest cpu and highest
memory criteria of TOPSIS are not independently configurable. Nodes to
start service are scored with the same fixed (TOPSIS) weights as before.

The weights also shape how sensitively the load balancer reacts to CPU
load changes relative to memory usage changes. This is to address: "In
our case CRS has not moved any VM around because the imbalance
percentage is low (7%). Eventhough the cluster is very imbalance [sic]
CPU wise, or memory wise alone." [0]

Skip load balancing with warning if both weights are 0.

[0] https://bugzilla.proxmox.com/show_bug.cgi?id=7993

Signed-off-by: Dominik Rusovac <d.rusovac@proxmox.com>
---
 src/PVE/HA/Manager.pm                         | 47 ++++++++++++++++---
 src/PVE/HA/Usage.pm                           | 21 +++++----
 src/PVE/HA/Usage/Basic.pm                     | 14 ++++--
 src/PVE/HA/Usage/Dynamic.pm                   | 27 +++++++----
 src/PVE/HA/Usage/Static.pm                    | 27 +++++++----
 .../log.expect                                | 13 +----
 6 files changed, 102 insertions(+), 47 deletions(-)

diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index 7a5a685..c6818e2 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -119,6 +119,12 @@ sub update_crs_scheduler_mode {
     $self->{crs}->{auto_rebalance}->{margin} =
         ($crs_cfg->{'ha-auto-rebalance-margin'} // 10) / 100.0;
 
+    my $default_weight = 1;
+    $self->{crs}->{auto_rebalance}->{cpu_weight} = $crs_cfg->{'ha-auto-rebalance-cpu-weight'} // $default_weight;
+    # by default TOPSIS weights mem higher than cpu
+    $default_weight = 5 if ($self->{crs}->{auto_rebalance}->{method} eq 'topsis');
+    $self->{crs}->{auto_rebalance}->{mem_weight} = $crs_cfg->{'ha-auto-rebalance-mem-weight'} // $default_weight;
+
     my $old_mode = $self->{crs}->{scheduler};
     my $new_mode = $crs_cfg->{ha} || 'basic';
 
@@ -236,8 +242,20 @@ sub load_balance {
     return if $crs->{scheduler} ne 'static' && $crs->{scheduler} ne 'dynamic';
     return if $self->any_resource_motion_queued_or_running();
 
-    my ($threshold, $method, $hold_duration, $margin) =
-        $auto_rebalance_opts->@{qw(threshold method hold_duration margin)};
+    my (
+        $threshold, $method, $hold_duration, $margin, $cpu_weight, $mem_weight,
+        ) =
+        $auto_rebalance_opts->@{qw(threshold method hold_duration margin cpu_weight mem_weight)};
+
+    if ($cpu_weight + $mem_weight eq 0) {
+        $haenv->log(
+            'warning',
+            "auto-rebalance weights are both set to 0, skipping load balancing -"
+                . " set at least one to a positive value to avoid constant load"
+                . " imbalance of 0",
+        );
+        return;
+    }
 
     # with threshold and margin both at 0% any non-zero imbalance triggers a non-improving
     # migration, churning the cluster forever - skip and warn instead of silently looping.
@@ -250,7 +268,7 @@ sub load_balance {
         return;
     }
 
-    my $imbalance = $online_node_usage->calculate_node_imbalance();
+    my $imbalance = $online_node_usage->calculate_node_imbalance($cpu_weight, $mem_weight);
 
     # do not load balance unless imbalance threshold has been exceeded
     # consecutively for $hold_duration calls to load_balance();
@@ -268,15 +286,24 @@ sub load_balance {
 
     my $result;
     if ($method eq 'bruteforce') {
-        $result = $online_node_usage->select_best_balancing_migration($candidates);
+        $result = $online_node_usage->select_best_balancing_migration(
+            $candidates,
+            $cpu_weight,
+            $mem_weight,
+        );
     } elsif ($method eq 'topsis') {
-        $result = $online_node_usage->select_best_balancing_migration_topsis($candidates);
+        $result = $online_node_usage->select_best_balancing_migration_topsis(
+            $candidates,
+            $cpu_weight,
+            $mem_weight,
+        );
     }
 
     # happens if $candidates is empty or $method isn't handled above
     return if !$result;
 
-    my ($migration, $target_imbalance) = $result->@{qw(migration imbalance)};
+    my ($migration, $target_imbalance, $score) = $result->@{qw(migration imbalance score)};
+    $target_imbalance = $score if $method eq 'bruteforce';
 
     my $relative_change = ($imbalance - $target_imbalance) / $imbalance;
     return if $relative_change < $margin;
@@ -311,7 +338,13 @@ sub flush_master_status {
     $ms->{node_status} = $ns->{status};
     $ms->{service_status} = $ss;
     $ms->{timestamp} = $haenv->get_time();
-    $ms->{imbalance} = $self->{online_node_usage}->calculate_node_imbalance();
+
+    my ($cpu_weight, $mem_weight) = (
+        $self->{crs}->{auto_rebalance}->{cpu_weight},
+        $self->{crs}->{auto_rebalance}->{mem_weight},
+    );
+    $ms->{imbalance} =
+        $self->{online_node_usage}->calculate_node_imbalance($cpu_weight, $mem_weight);
 
     $haenv->write_manager_status($ms);
 }
diff --git a/src/PVE/HA/Usage.pm b/src/PVE/HA/Usage.pm
index 659ab30..fbd8800 100644
--- a/src/PVE/HA/Usage.pm
+++ b/src/PVE/HA/Usage.pm
@@ -61,42 +61,47 @@ sub remove_service_usage {
 }
 
 sub calculate_node_imbalance {
-    my ($self) = @_;
+    my ($self, $cpu_weight, $mem_weight) = @_;
 
     die "implement in subclass";
 }
 
 sub score_best_balancing_migrations {
-    my ($self, $migration_candidates, $limit) = @_;
+    my ($self, $migration_candidates, $limit, $cpu_weight, $mem_weight) = @_;
 
     die "implement in subclass";
 }
 
 sub select_best_balancing_migration {
-    my ($self, $migration_candidates) = @_;
+    my ($self, $migration_candidates, $cpu_weight, $mem_weight) = @_;
 
-    my $migrations = $self->score_best_balancing_migrations($migration_candidates, 1);
+    my $migrations =
+        $self->score_best_balancing_migrations($migration_candidates, 1, $cpu_weight, $mem_weight);
 
     return $migrations->[0];
 }
 
 sub score_best_balancing_migrations_topsis {
-    my ($self, $migration_candidates, $limit) = @_;
+    my (
+        $self, $migration_candidates, $limit, $cpu_weight, $mem_weight,
+    ) = @_;
 
     die "implement in subclass";
 }
 
 sub select_best_balancing_migration_topsis {
-    my ($self, $migration_candidates) = @_;
+    my ($self, $migration_candidates, $cpu_weight, $mem_weight) = @_;
 
-    my $migrations = $self->score_best_balancing_migrations_topsis($migration_candidates, 1);
+    my $migrations = $self->score_best_balancing_migrations_topsis(
+        $migration_candidates, 1, $cpu_weight, $mem_weight,
+    );
 
     return $migrations->[0];
 }
 
 # Returns a hash with $nodename => $score pairs. A lower $score is better.
 sub score_nodes_to_start_service {
-    my ($self, $sid) = @_;
+    my ($self, $sid, $cpu_weight, $mem_weight) = @_;
 
     die "implement in subclass";
 }
diff --git a/src/PVE/HA/Usage/Basic.pm b/src/PVE/HA/Usage/Basic.pm
index 4dce9e1..57eecd3 100644
--- a/src/PVE/HA/Usage/Basic.pm
+++ b/src/PVE/HA/Usage/Basic.pm
@@ -67,19 +67,27 @@ sub remove_service_usage {
 }
 
 sub calculate_node_imbalance {
-    my ($self) = @_;
+    my ($self, $cpu_weight, $mem_weight) = @_;
 
     return 0.0;
 }
 
 sub score_best_balancing_migrations {
-    my ($self, $migration_candidates, $limit) = @_;
+    my ($self, $migration_candidates, $limit, $cpu_weight, $mem_weight) = @_;
 
     return [];
 }
 
 sub score_best_balancing_migrations_topsis {
-    my ($self, $migration_candidates, $limit) = @_;
+    my (
+        $self,
+        $migration_candidates,
+        $limit,
+        $cpu_weight,
+        $max_cpu_weight,
+        $mem_weight,
+        $max_mem_weight,
+    ) = @_;
 
     return [];
 }
diff --git a/src/PVE/HA/Usage/Dynamic.pm b/src/PVE/HA/Usage/Dynamic.pm
index 76d0fea..bd03e89 100644
--- a/src/PVE/HA/Usage/Dynamic.pm
+++ b/src/PVE/HA/Usage/Dynamic.pm
@@ -105,20 +105,22 @@ sub remove_service_usage {
 }
 
 sub calculate_node_imbalance {
-    my ($self) = @_;
+    my ($self, $cpu_weight, $mem_weight) = @_;
 
-    my $node_imbalance = eval { $self->{scheduler}->calculate_node_imbalance() };
+    my $node_imbalance =
+        eval { $self->{scheduler}->calculate_node_imbalance($cpu_weight, $mem_weight) };
     $self->{haenv}->log('warning', "unable to calculate dynamic node imbalance - $@") if $@;
 
     return $node_imbalance // 0.0;
 }
 
 sub score_best_balancing_migrations {
-    my ($self, $migration_candidates, $limit) = @_;
+    my ($self, $migration_candidates, $limit, $cpu_weight, $mem_weight) = @_;
 
     my $migrations = eval {
-        $self->{scheduler}
-            ->score_best_balancing_migration_candidates($migration_candidates, $limit);
+        $self->{scheduler}->score_best_balancing_migration_candidates(
+            $migration_candidates, $limit, $cpu_weight, $mem_weight,
+        );
     };
     $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@;
 
@@ -126,11 +128,14 @@ sub score_best_balancing_migrations {
 }
 
 sub score_best_balancing_migrations_topsis {
-    my ($self, $migration_candidates, $limit) = @_;
+    my (
+        $self, $migration_candidates, $limit, $cpu_weight, $mem_weight,
+    ) = @_;
 
     my $migrations = eval {
-        $self->{scheduler}
-            ->score_best_balancing_migration_candidates_topsis($migration_candidates, $limit);
+        $self->{scheduler}->score_best_balancing_migration_candidates_topsis(
+            $migration_candidates, $limit, $cpu_weight, $mem_weight,
+        );
     };
     $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@;
 
@@ -140,9 +145,13 @@ sub score_best_balancing_migrations_topsis {
 sub score_nodes_to_start_service {
     my ($self, $sid) = @_;
 
+    my ($cpu_weight, $mem_weight) = (1, 5);
+
     my $score_list = eval {
         my $service_usage = get_service_usage($self, $sid);
-        $self->{scheduler}->score_nodes_to_start_resource($service_usage);
+        $self->{scheduler}->score_nodes_to_start_resource(
+            $service_usage, $cpu_weight, $mem_weight,
+        );
     };
     $self->{haenv}
         ->log('err', "unable to score nodes according to dynamic usage for service '$sid' - $@")
diff --git a/src/PVE/HA/Usage/Static.pm b/src/PVE/HA/Usage/Static.pm
index e67d5f5..39308ff 100644
--- a/src/PVE/HA/Usage/Static.pm
+++ b/src/PVE/HA/Usage/Static.pm
@@ -112,20 +112,22 @@ sub remove_service_usage {
 }
 
 sub calculate_node_imbalance {
-    my ($self) = @_;
+    my ($self, $cpu_weight, $mem_weight) = @_;
 
-    my $node_imbalance = eval { $self->{scheduler}->calculate_node_imbalance() };
+    my $node_imbalance =
+        eval { $self->{scheduler}->calculate_node_imbalance($cpu_weight, $mem_weight) };
     $self->{haenv}->log('warning', "unable to calculate static node imbalance - $@") if $@;
 
     return $node_imbalance // 0.0;
 }
 
 sub score_best_balancing_migrations {
-    my ($self, $migration_candidates, $limit) = @_;
+    my ($self, $migration_candidates, $limit, $cpu_weight, $mem_weight) = @_;
 
     my $migrations = eval {
-        $self->{scheduler}
-            ->score_best_balancing_migration_candidates($migration_candidates, $limit);
+        $self->{scheduler}->score_best_balancing_migration_candidates(
+            $migration_candidates, $limit, $cpu_weight, $mem_weight,
+        );
     };
     $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@;
 
@@ -133,11 +135,14 @@ sub score_best_balancing_migrations {
 }
 
 sub score_best_balancing_migrations_topsis {
-    my ($self, $migration_candidates, $limit) = @_;
+    my (
+        $self, $migration_candidates, $limit, $cpu_weight, $mem_weight,
+    ) = @_;
 
     my $migrations = eval {
-        $self->{scheduler}
-            ->score_best_balancing_migration_candidates_topsis($migration_candidates, $limit);
+        $self->{scheduler}->score_best_balancing_migration_candidates_topsis(
+            $migration_candidates, $limit, $cpu_weight, $mem_weight,
+        );
     };
     $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@;
 
@@ -147,9 +152,13 @@ sub score_best_balancing_migrations_topsis {
 sub score_nodes_to_start_service {
     my ($self, $sid) = @_;
 
+    my ($cpu_weight, $mem_weight) = (1, 5);
+
     my $score_list = eval {
         my $service_usage = get_service_usage($self, $sid);
-        $self->{scheduler}->score_nodes_to_start_service($service_usage);
+        $self->{scheduler}->score_nodes_to_start_service(
+            $service_usage, $cpu_weight, $mem_weight,
+        );
     };
     if (my $err = $@) {
         $self->{haenv}->log(
diff --git a/src/test/test-crs-dynamic-auto-rebalance-topsis3/log.expect b/src/test/test-crs-dynamic-auto-rebalance-topsis3/log.expect
index 2f19c72..95163d4 100644
--- a/src/test/test-crs-dynamic-auto-rebalance-topsis3/log.expect
+++ b/src/test/test-crs-dynamic-auto-rebalance-topsis3/log.expect
@@ -53,7 +53,7 @@ info     25    node3/lrm: service status vm:107 started
 info    120      cmdlist: execute service vm:105 set-dynamic-stats cpu 7.8 mem 7912
 info    120      cmdlist: execute service vm:106 set-dynamic-stats cpu 5.7 mem 8192
 info    120      cmdlist: execute service vm:107 set-dynamic-stats cpu 6.0 mem 8011
-info    160    node1/crm: auto rebalance - migrate vm:105 to node2 (expected change for imbalance from 60.2% to 29.8%)
+info    160    node1/crm: auto rebalance - migrate vm:105 to node2 (expected change for imbalance from 48.2% to 24.0%)
 info    160    node1/crm: got crm command: migrate vm:105 node2
 info    160    node1/crm: migrate service 'vm:105' to node 'node2'
 info    160    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node3, target = node2)
@@ -68,7 +68,7 @@ info    220      cmdlist: execute service vm:104 set-dynamic-stats cpu 6.7 mem 8
 info    220      cmdlist: execute service vm:105 set-dynamic-stats cpu 1.8 mem 1201
 info    220      cmdlist: execute service vm:106 set-dynamic-stats cpu 2.1 mem 1211
 info    220      cmdlist: execute service vm:107 set-dynamic-stats cpu 0.9 mem 1191
-info    260    node1/crm: auto rebalance - migrate vm:103 to node3 (expected change for imbalance from 57.3% to 30.1%)
+info    260    node1/crm: auto rebalance - migrate vm:103 to node3 (expected change for imbalance from 58.4% to 27.4%)
 info    260    node1/crm: got crm command: migrate vm:103 node3
 info    260    node1/crm: migrate service 'vm:103' to node 'node3'
 info    260    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node2, target = node3)
@@ -77,13 +77,4 @@ info    263    node2/lrm: service vm:103 - end migrate to node 'node3'
 info    280    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node3)
 info    285    node3/lrm: starting service vm:103
 info    285    node3/lrm: service status vm:103 started
-info    340    node1/crm: auto rebalance - migrate vm:105 to node1 (expected change for imbalance from 30.1% to 17.0%)
-info    340    node1/crm: got crm command: migrate vm:105 node1
-info    340    node1/crm: migrate service 'vm:105' to node 'node1'
-info    340    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node2, target = node1)
-info    343    node2/lrm: service vm:105 - start migrate to node 'node1'
-info    343    node2/lrm: service vm:105 - end migrate to node 'node1'
-info    360    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node1)
-info    361    node1/lrm: starting service vm:105
-info    361    node1/lrm: service status vm:105 started
 info    820     hardware: exit simulation - done
-- 
2.47.3