From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id EF2391FF0C0 for ; Wed, 18 Nov 2026 12:09:10 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 056312198B; Wed, 18 Nov 2026 12:07:50 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH ha-manager 06/15] usage: share one scheduler base between static and dynamic Date: Wed, 18 Nov 2026 12:05:13 +0100 Message-ID: <20261118110705.595989-7-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118110705.595989-1-t.lamprecht@proxmox.com> References: <20261118110705.595989-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795000030598 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.174 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: WYETE6XUQEODJ5QPNU727ULN2NMVBPUW X-Message-ID-Hash: WYETE6XUQEODJ5QPNU727ULN2NMVBPUW X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The static and dynamic usage drivers each wrapped the resource scheduling bindings directly, in two near-identical copies. Route them through PVE::ResourceScheduling::Scheduler from the scheduling library and fold their shared logic into a common base, so static and dynamic differ only in the mode they select. The HA stack and the non-HA resource scheduling API then share one scheduler wrapper rather than copies that can drift. The wrapper forwards every call to the same bindings with the same arguments. The HA simulation tests produce byte-identical logs before and after this change, and their static and dynamic CRS cases fail when the library is missing, so they do exercise the new code path. The one intended difference is the handling of a failure to score nodes for a service: dynamic mode now also falls back to ranking the nodes by their service count, like static mode does. Before, it returned no scores, so the caller placed the service on the first node by name and logged uninitialized value warnings. Extend the fallback test to cover dynamic mode. Signed-off-by: Thomas Lamprecht --- debian/pve-ha-manager.install | 1 + src/PVE/HA/Usage/Dynamic.pm | 145 +------------------------- src/PVE/HA/Usage/Makefile | 2 +- src/PVE/HA/Usage/Scheduler.pm | 175 ++++++++++++++++++++++++++++++++ src/PVE/HA/Usage/Static.pm | 160 +---------------------------- src/test/test_usage_fallback.pl | 7 +- 6 files changed, 187 insertions(+), 303 deletions(-) create mode 100644 src/PVE/HA/Usage/Scheduler.pm diff --git a/debian/pve-ha-manager.install b/debian/pve-ha-manager.install index 301e9fc..4451dd3 100644 --- a/debian/pve-ha-manager.install +++ b/debian/pve-ha-manager.install @@ -41,6 +41,7 @@ /usr/share/perl5/PVE/HA/Tools.pm /usr/share/perl5/PVE/HA/Usage.pm /usr/share/perl5/PVE/HA/Usage/Basic.pm +/usr/share/perl5/PVE/HA/Usage/Scheduler.pm /usr/share/perl5/PVE/HA/Usage/Static.pm /usr/share/perl5/PVE/HA/Usage/Dynamic.pm /usr/share/perl5/PVE/Service/pve_ha_crm.pm diff --git a/src/PVE/HA/Usage/Dynamic.pm b/src/PVE/HA/Usage/Dynamic.pm index 76d0fea..05be1a5 100644 --- a/src/PVE/HA/Usage/Dynamic.pm +++ b/src/PVE/HA/Usage/Dynamic.pm @@ -3,153 +3,12 @@ package PVE::HA::Usage::Dynamic; use strict; use warnings; -use PVE::HA::Resources; -use PVE::RS::ResourceScheduling::Dynamic; - -use base qw(PVE::HA::Usage); +use base qw(PVE::HA::Usage::Scheduler); sub new { my ($class, $haenv, $service_stats) = @_; - my $node_stats = eval { $haenv->get_dynamic_node_stats() }; - die "did not get dynamic node usage information - $@" if $@; - - my $scheduler = eval { PVE::RS::ResourceScheduling::Dynamic->new() }; - die "unable to initialize dynamic scheduling - $@" if $@; - - return bless { - 'node-stats' => $node_stats, - 'service-stats' => $service_stats, - haenv => $haenv, - scheduler => $scheduler, - }, $class; -} - -sub add_node { - my ($self, $nodename) = @_; - - my $stats = $self->{'node-stats'}->{$nodename} - or die "did not get dynamic node usage information for '$nodename'\n"; - die "dynamic node usage information for '$nodename' missing cpu count\n" - if !defined($stats->{maxcpu}); - die "dynamic node usage information for '$nodename' missing memory\n" - if !defined($stats->{maxmem}); - - eval { $self->{scheduler}->add_node($nodename, $stats); }; - die "initializing dynamic node usage for '$nodename' failed - $@" if $@; -} - -sub remove_node { - my ($self, $nodename) = @_; - - $self->{scheduler}->remove_node($nodename); -} - -sub list_nodes { - my ($self) = @_; - - return $self->{scheduler}->list_nodes()->@*; -} - -sub contains_node { - my ($self, $nodename) = @_; - - return $self->{scheduler}->contains_node($nodename); -} - -my sub get_service_usage { - my ($self, $sid) = @_; - - my $service_stats = $self->{'service-stats'}->{$sid}->{usage} - or die "did not get dynamic service usage information for '$sid'\n"; - - return $service_stats; -} - -sub add_service { - my ($self, $sid, $current_node, $target_node, $running) = @_; - - # do not add service, which does not put any usage on the nodes - return if !defined($current_node) && !defined($target_node); - - # PVE::RS::ResourceScheduling::Dynamic::add_resource() expects $current_node - # to be set, so consider $target_node as $current_node for unset $current_node; - # - # currently, this happens for the request_start_balance service state and if - # node maintenance causes services to migrate to other nodes - if (!defined($current_node)) { - $current_node = $target_node; - undef $target_node; - } - - eval { - my $service_usage = get_service_usage($self, $sid); - - my $service = { - stats => $service_usage, - running => $running, - 'current-node' => $current_node, - 'target-node' => $target_node, - }; - - $self->{scheduler}->add_resource($sid, $service); - }; - $self->{haenv}->log('warning', "unable to add service '$sid' - $@") if $@; -} - -sub remove_service_usage { - my ($self, $sid) = @_; - - eval { $self->{scheduler}->remove_resource($sid) }; - $self->{haenv}->log('warning', "unable to remove service '$sid' usage - $@") if $@; -} - -sub calculate_node_imbalance { - my ($self) = @_; - - my $node_imbalance = eval { $self->{scheduler}->calculate_node_imbalance() }; - $self->{haenv}->log('warning', "unable to calculate dynamic node imbalance - $@") if $@; - - return $node_imbalance // 0.0; -} - -sub score_best_balancing_migrations { - my ($self, $migration_candidates, $limit) = @_; - - my $migrations = eval { - $self->{scheduler} - ->score_best_balancing_migration_candidates($migration_candidates, $limit); - }; - $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; - - return $migrations; -} - -sub score_best_balancing_migrations_topsis { - my ($self, $migration_candidates, $limit) = @_; - - my $migrations = eval { - $self->{scheduler} - ->score_best_balancing_migration_candidates_topsis($migration_candidates, $limit); - }; - $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; - - return $migrations; -} - -sub score_nodes_to_start_service { - my ($self, $sid) = @_; - - my $score_list = eval { - my $service_usage = get_service_usage($self, $sid); - $self->{scheduler}->score_nodes_to_start_resource($service_usage); - }; - $self->{haenv} - ->log('err', "unable to score nodes according to dynamic usage for service '$sid' - $@") - if $@; - - # Take minus the value, so that a lower score is better, which our caller(s) expect(s). - return { map { $_->[0] => -$_->[1] } $score_list->@* }; + return $class->SUPER::new($haenv, $service_stats, 'dynamic'); } 1; diff --git a/src/PVE/HA/Usage/Makefile b/src/PVE/HA/Usage/Makefile index 5d51a9c..619e0a3 100644 --- a/src/PVE/HA/Usage/Makefile +++ b/src/PVE/HA/Usage/Makefile @@ -1,5 +1,5 @@ SIM_SOURCES=Basic.pm -SOURCES=${SIM_SOURCES} Static.pm Dynamic.pm +SOURCES=${SIM_SOURCES} Scheduler.pm Static.pm Dynamic.pm .PHONY: install install: diff --git a/src/PVE/HA/Usage/Scheduler.pm b/src/PVE/HA/Usage/Scheduler.pm new file mode 100644 index 0000000..777ac5a --- /dev/null +++ b/src/PVE/HA/Usage/Scheduler.pm @@ -0,0 +1,175 @@ +package PVE::HA::Usage::Scheduler; + +use strict; +use warnings; + +use PVE::ResourceScheduling::Scheduler; + +use base qw(PVE::HA::Usage); + +# Shared base for the resource-scheduling-backed HA usage drivers; PVE::HA::Usage::Static and +# ::Dynamic only pick the mode. The mode selects the matching PVE::ResourceScheduling::Scheduler +# backend and the haenv node-stats getter; everything else is identical. + +sub new { + my ($class, $haenv, $service_stats, $mode) = @_; + + my $node_stats = eval { + $mode eq 'static' ? $haenv->get_static_node_stats() : $haenv->get_dynamic_node_stats(); + }; + die "did not get $mode node usage information - $@" if $@; + + my $scheduler = eval { PVE::ResourceScheduling::Scheduler->new($mode) }; + die "unable to initialize $mode scheduling - $@" if $@; + + return bless { + mode => $mode, + 'node-stats' => $node_stats, + 'service-stats' => $service_stats, + haenv => $haenv, + scheduler => $scheduler, + 'node-services' => {}, # services on each node, fallback if scoring calculation fails + }, $class; +} + +sub add_node { + my ($self, $nodename) = @_; + + $self->{'node-services'}->{$nodename} = {}; + + my $stats = $self->{'node-stats'}->{$nodename} + or die "did not get $self->{mode} node usage information for '$nodename'\n"; + die "$self->{mode} node usage information for '$nodename' missing cpu count\n" + if !defined($stats->{maxcpu}); + die "$self->{mode} node usage information for '$nodename' missing memory\n" + if !defined($stats->{maxmem}); + + eval { $self->{scheduler}->add_node($nodename, $stats); }; + die "initializing $self->{mode} node usage for '$nodename' failed - $@" if $@; +} + +sub remove_node { + my ($self, $nodename) = @_; + + delete $self->{'node-services'}->{$nodename}; + + $self->{scheduler}->remove_node($nodename); +} + +sub list_nodes { + my ($self) = @_; + + return $self->{scheduler}->list_nodes(); +} + +sub contains_node { + my ($self, $nodename) = @_; + + return $self->{scheduler}->contains_node($nodename); +} + +my sub get_service_usage { + my ($self, $sid) = @_; + + my $service_stats = $self->{'service-stats'}->{$sid}->{usage} + or die "did not get $self->{mode} service usage information for '$sid'\n"; + + return $service_stats; +} + +sub add_service { + my ($self, $sid, $current_node, $target_node, $running) = @_; + + # do not add services which do not put any usage on the nodes + return if !defined($current_node) && !defined($target_node); + + # the scheduler backend expects $current_node to be set, so consider $target_node as + # $current_node for unset $current_node; currently, this happens for the request_start_balance + # service state and if node maintenance causes services to migrate to other nodes + if (!defined($current_node)) { + $current_node = $target_node; + undef $target_node; + } + + $self->{'node-services'}->{$current_node}->{$sid} = 1; + $self->{'node-services'}->{$target_node}->{$sid} = 1 if defined($target_node); + + eval { + my $service_usage = get_service_usage($self, $sid); + + my $service = { + stats => $service_usage, + running => $running, + 'current-node' => $current_node, + 'target-node' => $target_node, + }; + + $self->{scheduler}->add_resource($sid, $service); + }; + $self->{haenv}->log('warning', "unable to add service '$sid' - $@") if $@; +} + +sub remove_service_usage { + my ($self, $sid) = @_; + + delete($self->{'node-services'}->{$_}->{$sid}) for $self->list_nodes(); + + eval { $self->{scheduler}->remove_resource($sid) }; + $self->{haenv}->log('warning', "unable to remove service '$sid' usage - $@") if $@; +} + +sub calculate_node_imbalance { + my ($self) = @_; + + my $node_imbalance = eval { $self->{scheduler}->calculate_node_imbalance() }; + $self->{haenv}->log('warning', "unable to calculate $self->{mode} node imbalance - $@") if $@; + + return $node_imbalance // 0.0; +} + +sub score_best_balancing_migrations { + my ($self, $migration_candidates, $limit) = @_; + + my $migrations = eval { + $self->{scheduler} + ->score_best_balancing_migration_candidates($migration_candidates, $limit); + }; + $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; + + return $migrations; +} + +sub score_best_balancing_migrations_topsis { + my ($self, $migration_candidates, $limit) = @_; + + my $migrations = eval { + $self->{scheduler}->score_best_balancing_migration_candidates( + $migration_candidates, $limit, 'topsis', + ); + }; + $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; + + return $migrations; +} + +sub score_nodes_to_start_service { + my ($self, $sid) = @_; + + my $score_list = eval { + my $service_usage = get_service_usage($self, $sid); + $self->{scheduler}->score_nodes_to_start($service_usage); + }; + if (my $err = $@) { + $self->{haenv}->log( + 'err', + "unable to score nodes according to $self->{mode} usage for service '$sid' - $err", + ); + my $node_services = $self->{'node-services'}; + return { map { $_ => scalar(keys $node_services->{$_}->%*) } keys $node_services->%* }; + } + + # Take minus the value, so that a lower score is better, which our caller(s) expect(s). + return { map { $_->[0] => -$_->[1] } $score_list->@* }; +} + +1; diff --git a/src/PVE/HA/Usage/Static.pm b/src/PVE/HA/Usage/Static.pm index 570a801..a7ee0a7 100644 --- a/src/PVE/HA/Usage/Static.pm +++ b/src/PVE/HA/Usage/Static.pm @@ -3,168 +3,12 @@ package PVE::HA::Usage::Static; use strict; use warnings; -use PVE::HA::Resources; -use PVE::RS::ResourceScheduling::Static; - -use base qw(PVE::HA::Usage); +use base qw(PVE::HA::Usage::Scheduler); sub new { my ($class, $haenv, $service_stats) = @_; - my $node_stats = eval { $haenv->get_static_node_stats() }; - die "did not get static node usage information - $@" if $@; - - my $scheduler = eval { PVE::RS::ResourceScheduling::Static->new(); }; - die "unable to initialize static scheduling - $@" if $@; - - return bless { - 'node-stats' => $node_stats, - 'service-stats' => $service_stats, - haenv => $haenv, - scheduler => $scheduler, - 'node-services' => {}, # Services on each node. Fallback if scoring calculation fails. - }, $class; -} - -sub add_node { - my ($self, $nodename) = @_; - - $self->{'node-services'}->{$nodename} = {}; - - my $stats = $self->{'node-stats'}->{$nodename} - or die "did not get static node usage information for '$nodename'\n"; - die "static node usage information for '$nodename' missing cpu count\n" - if !defined($stats->{maxcpu}); - die "static node usage information for '$nodename' missing memory\n" - if !defined($stats->{maxmem}); - - eval { $self->{scheduler}->add_node($nodename, int($stats->{maxcpu}), int($stats->{maxmem})); }; - die "initializing static node usage for '$nodename' failed - $@" if $@; -} - -sub remove_node { - my ($self, $nodename) = @_; - - delete $self->{'node-services'}->{$nodename}; - - $self->{scheduler}->remove_node($nodename); -} - -sub list_nodes { - my ($self) = @_; - - return $self->{scheduler}->list_nodes()->@*; -} - -sub contains_node { - my ($self, $nodename) = @_; - - return $self->{scheduler}->contains_node($nodename); -} - -my sub get_service_usage { - my ($self, $sid) = @_; - - my $service_stats = $self->{'service-stats'}->{$sid}->{usage} - or die "did not get static service usage information for '$sid'\n"; - - return $service_stats; -} - -sub add_service { - my ($self, $sid, $current_node, $target_node, $running) = @_; - - # do not add service which do not put any usage on the nodes - return if !defined($current_node) && !defined($target_node); - - # PVE::RS::ResourceScheduling::Static::add_service() expects $current_node - # to be set, so consider $target_node as $current_node for unset $current_node; - # - # currently, this happens for the request_start_balance service state and if - # node maintenance causes services to migrate to other nodes - if (!defined($current_node)) { - $current_node = $target_node; - undef $target_node; - } - - $self->{'node-services'}->{$current_node}->{$sid} = 1; - $self->{'node-services'}->{$target_node}->{$sid} = 1 if defined($target_node); - - eval { - my $service_usage = get_service_usage($self, $sid); - - my $service = { - stats => $service_usage, - running => $running, - 'current-node' => $current_node, - 'target-node' => $target_node, - }; - - $self->{scheduler}->add_service($sid, $service); - }; - $self->{haenv}->log('warning', "unable to add service '$sid' - $@") if $@; -} - -sub remove_service_usage { - my ($self, $sid) = @_; - - delete($self->{'node-services'}->{$_}->{$sid}) for $self->list_nodes(); - - eval { $self->{scheduler}->remove_service_usage($sid) }; - $self->{haenv}->log('warning', "unable to remove service '$sid' usage - $@") if $@; -} - -sub calculate_node_imbalance { - my ($self) = @_; - - my $node_imbalance = eval { $self->{scheduler}->calculate_node_imbalance() }; - $self->{haenv}->log('warning', "unable to calculate static node imbalance - $@") if $@; - - return $node_imbalance // 0.0; -} - -sub score_best_balancing_migrations { - my ($self, $migration_candidates, $limit) = @_; - - my $migrations = eval { - $self->{scheduler} - ->score_best_balancing_migration_candidates($migration_candidates, $limit); - }; - $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; - - return $migrations; -} - -sub score_best_balancing_migrations_topsis { - my ($self, $migration_candidates, $limit) = @_; - - my $migrations = eval { - $self->{scheduler} - ->score_best_balancing_migration_candidates_topsis($migration_candidates, $limit); - }; - $self->{haenv}->log('warning', "unable to score best balancing migration - $@") if $@; - - return $migrations; -} - -sub score_nodes_to_start_service { - my ($self, $sid) = @_; - - my $score_list = eval { - my $service_usage = get_service_usage($self, $sid); - $self->{scheduler}->score_nodes_to_start_service($service_usage); - }; - if (my $err = $@) { - $self->{haenv}->log( - 'err', - "unable to score nodes according to static usage for service '$sid' - $err", - ); - my $node_services = $self->{'node-services'}; - return { map { $_ => scalar(keys $node_services->{$_}->%*) } keys $node_services->%* }; - } - - # Take minus the value, so that a lower score is better, which our caller(s) expect(s). - return { map { $_->[0] => -$_->[1] } $score_list->@* }; + return $class->SUPER::new($haenv, $service_stats, 'static'); } 1; diff --git a/src/test/test_usage_fallback.pl b/src/test/test_usage_fallback.pl index b9bb6dd..e1d1d46 100755 --- a/src/test/test_usage_fallback.pl +++ b/src/test/test_usage_fallback.pl @@ -6,6 +6,7 @@ use lib qw(..); use Test::More; +use PVE::HA::Usage::Dynamic; use PVE::HA::Usage::Static; my $GiB = 1024**3; @@ -20,6 +21,10 @@ package MockEnv { return $self->{node_stats}; } + sub get_dynamic_node_stats($self) { + return $self->{node_stats}; + } + sub log($self, $level, $msg) { push $self->{logs}->@*, "$level: $msg"; } @@ -67,6 +72,6 @@ my $test_fallback = sub($class) { ); }; -$test_fallback->('PVE::HA::Usage::Static'); +$test_fallback->($_) for qw(PVE::HA::Usage::Static PVE::HA::Usage::Dynamic); done_testing(); -- 2.47.3