[RFC ha-manager 17/21] usage: add dynamic usage scheduler

[Daniel Kral <d.kral@proxmox.com>]

Signed-off-by: Daniel Kral <d.kral@proxmox.com>
---
 debian/pve-ha-manager.install |  1 +
 src/PVE/HA/Manager.pm         | 12 +++++
 src/PVE/HA/Usage/Dynamic.pm   | 99 +++++++++++++++++++++++++++++++++++
 src/PVE/HA/Usage/Makefile     |  2 +-
 4 files changed, 113 insertions(+), 1 deletion(-)
 create mode 100644 src/PVE/HA/Usage/Dynamic.pm

diff --git a/debian/pve-ha-manager.install b/debian/pve-ha-manager.install
index 38d5d60b..75220a0b 100644
--- a/debian/pve-ha-manager.install
+++ b/debian/pve-ha-manager.install
@@ -42,6 +42,7 @@
 /usr/share/perl5/PVE/HA/Usage.pm
 /usr/share/perl5/PVE/HA/Usage/Basic.pm
 /usr/share/perl5/PVE/HA/Usage/Static.pm
+/usr/share/perl5/PVE/HA/Usage/Dynamic.pm
 /usr/share/perl5/PVE/Service/pve_ha_crm.pm
 /usr/share/perl5/PVE/Service/pve_ha_lrm.pm
 /usr/share/pve-manager/templates/default/fencing-body.html.hbs
diff --git a/src/PVE/HA/Manager.pm b/src/PVE/HA/Manager.pm
index f9dec661..fc0e7fc2 100644
--- a/src/PVE/HA/Manager.pm
+++ b/src/PVE/HA/Manager.pm
@@ -14,6 +14,7 @@ use PVE::HA::Rules::NodeAffinity qw(get_node_affinity);
 use PVE::HA::Rules::ResourceAffinity
     qw(get_affinitive_resources get_resource_affinity apply_positive_resource_affinity apply_negative_resource_affinity);
 use PVE::HA::Usage::Basic;
+use PVE::HA::Usage::Dynamic;
 
 my $have_static_scheduling;
 eval {
@@ -264,6 +265,17 @@ sub recompute_online_node_usage {
                 'warning',
                 "fallback to 'basic' scheduler mode, init for 'static' failed - $@",
             ) if $@;
+        } elsif ($mode eq 'dynamic') {
+            $online_node_usage = eval {
+                $service_stats = $haenv->get_dynamic_service_stats();
+                my $scheduler = PVE::HA::Usage::Dynamic->new($haenv, $service_stats);
+                $scheduler->add_node($_) for $online_nodes->@*;
+                return $scheduler;
+            };
+            $haenv->log(
+                'warning',
+                "fallback to 'basic' scheduler mode, init for 'dynamic' failed - $@",
+            ) if $@;
         } elsif ($mode eq 'basic') {
             # handled below in the general fall-back case
         } else {
diff --git a/src/PVE/HA/Usage/Dynamic.pm b/src/PVE/HA/Usage/Dynamic.pm
new file mode 100644
index 00000000..f4049f62
--- /dev/null
+++ b/src/PVE/HA/Usage/Dynamic.pm
@@ -0,0 +1,99 @@
+package PVE::HA::Usage::Dynamic;
+
+use strict;
+use warnings;
+
+use PVE::HA::Resources;
+use PVE::RS::ResourceScheduling::Dynamic;
+
+use base qw(PVE::HA::Usage);
+
+sub new {
+    my ($class, $haenv, $service_stats) = @_;
+
+    my $node_stats = eval { $haenv->get_dynamic_node_stats() };
+    die "did not get dynamic node usage information - $@" if $@;
+
+    my $scheduler = eval { PVE::RS::ResourceScheduling::Dynamic->new() };
+    die "unable to initialize dynamic scheduling - $@" if $@;
+
+    return bless {
+        'node-stats' => $node_stats,
+        'service-stats' => $service_stats,
+        haenv => $haenv,
+        scheduler => $scheduler,
+    }, $class;
+}
+
+sub add_node {
+    my ($self, $nodename) = @_;
+
+    my $stats = $self->{'node-stats'}->{$nodename}
+        or die "did not get dynamic node usage information for '$nodename'\n";
+    die "dynamic node usage information for '$nodename' missing cpu count\n" if !$stats->{maxcpu};
+    die "dynamic node usage information for '$nodename' missing memory\n" if !$stats->{maxmem};
+
+    eval { $self->{scheduler}->add_node($nodename, $stats); };
+    die "initializing dynamic node usage for '$nodename' failed - $@" if $@;
+}
+
+sub remove_node {
+    my ($self, $nodename) = @_;
+
+    $self->{scheduler}->remove_node($nodename);
+}
+
+sub list_nodes {
+    my ($self) = @_;
+
+    return $self->{scheduler}->list_nodes()->@*;
+}
+
+sub contains_node {
+    my ($self, $nodename) = @_;
+
+    return $self->{scheduler}->contains_node($nodename);
+}
+
+my sub get_service_usage {
+    my ($self, $sid) = @_;
+
+    my $service_stats = $self->{'service-stats'}->{$sid}->{usage}
+        or die "did not get static service usage information for '$sid'\n";
+
+    return $service_stats;
+}
+
+sub add_service_usage_to_node {
+    my ($self, $nodename, $sid) = @_;
+
+    eval {
+        my $service_usage = get_service_usage($self, $sid);
+        $self->{scheduler}->add_service_usage_to_node($nodename, $sid, $service_usage);
+    };
+    $self->{haenv}->log('warning', "unable to add service '$sid' usage to node '$nodename' - $@")
+        if $@;
+}
+
+sub remove_service_usage {
+    my ($self, $sid) = @_;
+
+    eval { $self->{scheduler}->remove_service_usage($sid) };
+    $self->{haenv}->log('warning', "unable to remove service '$sid' usage - $@") if $@;
+}
+
+sub score_nodes_to_start_service {
+    my ($self, $sid) = @_;
+
+    my $score_list = eval {
+        my $service_usage = get_service_usage($self, $sid);
+        $self->{scheduler}->score_nodes_to_start_service($service_usage);
+    };
+    $self->{haenv}
+        ->log('err', "unable to score nodes according to dynamic usage for service '$sid' - $@");
+
+    # Take minus the value, so that a lower score is better, which our caller(s) expect(s).
+    return { map { $_->[0] => -$_->[1] } $score_list->@* };
+}
+
+1;
diff --git a/src/PVE/HA/Usage/Makefile b/src/PVE/HA/Usage/Makefile
index befdda60..5d51a9c1 100644
--- a/src/PVE/HA/Usage/Makefile
+++ b/src/PVE/HA/Usage/Makefile
@@ -1,5 +1,5 @@
 SIM_SOURCES=Basic.pm
-SOURCES=${SIM_SOURCES} Static.pm
+SOURCES=${SIM_SOURCES} Static.pm Dynamic.pm
 
 .PHONY: install
 install:
-- 
2.47.3