[pve-devel] [PATCH ha-manager v2 26/26] api: services: check for colocations for service motions

[Daniel Kral <d.kral@proxmox.com>]

The HA Manager already handles positive and negative colocations for
individual service migration, but the information about these is only
redirected to the HA environment's logger, i.e., for production usage
these messages are redirected to the HA Manager node's syslog.

Therefore, add checks when migrating/relocating services through their
respective API endpoints to give users information about side-effects,
i.e., positively colocated services, which are migrated together with
the service to the requested target node, and blockers, i.e., negative
colocated services, which are on the requested target node.

get_service_motion_info(...) is also callable from other packages, to
get a listing of all allowed and disallowed nodes with respect to the HA
Colocation rules, e.g., a migration precondition check.

Signed-off-by: Daniel Kral <d.kral@proxmox.com>
---
This patch is still more a draft of what I thought this should work
like, i.e., that users get notified and not only the admin through the
HA Manager node's syslog. I wrote get_service_motion_info(...) roughly
so that it can also be called by the precondition checks in qemu-server
and pve-container at a later point to easily gather allowed and
disallowed nodes.

I'd also introduce a --force flag for the ha-manager migrate/relocate
CLI endpoints so that a callee must confirm that the side-effects should
really be done.

changes since v1:
    - NEW!

 src/PVE/API2/HA/Resources.pm | 78 +++++++++++++++++++++++++++++++++---
 src/PVE/CLI/ha_manager.pm    | 38 +++++++++++++++++-
 src/PVE/HA/Config.pm         | 60 +++++++++++++++++++++++++++
 3 files changed, 168 insertions(+), 8 deletions(-)

diff --git a/src/PVE/API2/HA/Resources.pm b/src/PVE/API2/HA/Resources.pm
index f41fa2f..d217bb8 100644
--- a/src/PVE/API2/HA/Resources.pm
+++ b/src/PVE/API2/HA/Resources.pm
@@ -59,6 +59,14 @@ sub check_service_state {
     }
 }
 
+sub check_service_motion {
+    my ($sid, $req_node) = @_;
+
+    my ($allowed_nodes, $disallowed_nodes) = PVE::HA::Config::get_service_motion_info($sid);
+
+    return ($allowed_nodes->{$req_node}, $disallowed_nodes->{$req_node});
+}
+
 __PACKAGE__->register_method({
     name => 'index',
     path => '',
@@ -331,19 +339,48 @@ __PACKAGE__->register_method({
             ),
         },
     },
-    returns => { type => 'null' },
+    returns => {
+        type => 'object',
+        properties => {
+            'requested-node' => {
+                description => "Node, which was requested to be migrated to.",
+                type => 'string',
+                optional => 0,
+            },
+            'side-effects' => {
+                description => "Positively colocated HA resources, which are"
+                    . " relocated to the same requested target node.",
+                type => 'array',
+                optional => 1,
+            },
+        },
+    },
     code => sub {
         my ($param) = @_;
 
+        my $result = {};
+
         my ($sid, $type, $name) = PVE::HA::Config::parse_sid(extract_param($param, 'sid'));
+        my $req_node = extract_param($param, 'node');
 
         PVE::HA::Config::service_is_ha_managed($sid);
 
         check_service_state($sid);
 
-        PVE::HA::Config::queue_crm_commands("migrate $sid $param->{node}");
+        my ($side_effects, $blockers) = check_service_motion($sid, $req_node);
 
-        return undef;
+        PVE::HA::Config::queue_crm_commands("migrate $sid $req_node");
+        $result->{'requested-node'} = $req_node;
+
+        if (defined($blockers)) {
+            die "cannot migrate '$sid' to '$req_node' - negatively colocated service(s) "
+                . join(', ', @$blockers)
+                . " on target '$req_node'\n";
+        }
+
+        $result->{'side-effects'} = $side_effects if @$side_effects;
+
+        return $result;
     },
 });
 
@@ -373,19 +410,48 @@ __PACKAGE__->register_method({
             ),
         },
     },
-    returns => { type => 'null' },
+    returns => {
+        type => 'object',
+        properties => {
+            'requested-node' => {
+                description => "Node, which was requested to be relocated to.",
+                type => 'string',
+                optional => 0,
+            },
+            'side-effects' => {
+                description => "Positively colocated HA resources, which are"
+                    . " relocated to the same requested target node.",
+                type => 'array',
+                optional => 1,
+            },
+        },
+    },
     code => sub {
         my ($param) = @_;
 
+        my $result = {};
+
         my ($sid, $type, $name) = PVE::HA::Config::parse_sid(extract_param($param, 'sid'));
+        my $req_node = extract_param($param, 'node');
 
         PVE::HA::Config::service_is_ha_managed($sid);
 
         check_service_state($sid);
 
-        PVE::HA::Config::queue_crm_commands("relocate $sid $param->{node}");
+        my ($side_effects, $blockers) = check_service_motion($sid, $req_node);
 
-        return undef;
+        PVE::HA::Config::queue_crm_commands("relocate $sid $req_node");
+        $result->{'requested-node'} = $req_node;
+
+        if (defined($blockers)) {
+            die "cannot relocate '$sid' to '$req_node' - negatively colocated service(s) "
+                . join(', ', @$blockers)
+                . " on target '$req_node'\n";
+        }
+
+        $result->{'side-effects'} = $side_effects if @$side_effects;
+
+        return $result;
     },
 });
 
diff --git a/src/PVE/CLI/ha_manager.pm b/src/PVE/CLI/ha_manager.pm
index 564ac96..e34c8eb 100644
--- a/src/PVE/CLI/ha_manager.pm
+++ b/src/PVE/CLI/ha_manager.pm
@@ -239,8 +239,42 @@ our $cmddef = {
     relocate => { alias => 'crm-command relocate' },
 
     'crm-command' => {
-        migrate => ["PVE::API2::HA::Resources", 'migrate', ['sid', 'node']],
-        relocate => ["PVE::API2::HA::Resources", 'relocate', ['sid', 'node']],
+        migrate => [
+            "PVE::API2::HA::Resources",
+            'migrate',
+            ['sid', 'node'],
+            {},
+            sub {
+                my ($result) = @_;
+
+                if ($result->{'side-effects'}) {
+                    my $req_node = $result->{'requested-node'};
+
+                    for my $csid ($result->{'side-effects'}->@*) {
+                        print
+                            "also migrate positive colocated service '$csid' to '$req_node'\n";
+                    }
+                }
+            },
+        ],
+        relocate => [
+            "PVE::API2::HA::Resources",
+            'relocate',
+            ['sid', 'node'],
+            {},
+            sub {
+                my ($result) = @_;
+
+                if ($result->{'side-effects'}) {
+                    my $req_node = $result->{'requested-node'};
+
+                    for my $csid ($result->{'side-effects'}->@*) {
+                        print
+                            "also relocate positive colocated service '$csid' to '$req_node'\n";
+                    }
+                }
+            },
+        ],
         stop => [__PACKAGE__, 'stop', ['sid', 'timeout']],
         'node-maintenance' => {
             enable => [__PACKAGE__, 'node-maintenance-set', ['node'], { disable => 0 }],
diff --git a/src/PVE/HA/Config.pm b/src/PVE/HA/Config.pm
index de0fcec..c9172a5 100644
--- a/src/PVE/HA/Config.pm
+++ b/src/PVE/HA/Config.pm
@@ -8,6 +8,7 @@ use JSON;
 use PVE::HA::Tools;
 use PVE::HA::Groups;
 use PVE::HA::Rules;
+use PVE::HA::Rules::Colocation qw(get_colocated_services);
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
 use PVE::HA::Resources;
 
@@ -223,6 +224,24 @@ sub read_and_check_rules_config {
     return $rules;
 }
 
+sub read_and_check_full_rules_config {
+
+    my $rules = read_and_check_rules_config();
+
+    # TODO PVE 10: Remove group migration when HA groups have been fully migrated to location rules
+    if (!is_ha_location_enabled()) {
+        my $groups = read_group_config();
+        my $services = read_and_check_resources_config();
+
+        PVE::HA::Rules::Location::delete_location_rules($rules);
+        PVE::HA::Groups::migrate_groups_to_rules($rules, $groups, $services);
+    }
+
+    PVE::HA::Rules->canonicalize($rules);
+
+    return $rules;
+}
+
 sub write_rules_config {
     my ($cfg) = @_;
 
@@ -345,6 +364,47 @@ sub service_is_configured {
     return 0;
 }
 
+sub get_service_motion_info {
+    my ($sid) = @_;
+
+    my $services = read_resources_config();
+
+    my $allowed_nodes = {};
+    my $disallowed_nodes = {};
+
+    if (&$service_check_ha_state($services, $sid)) {
+        my $manager_status = read_manager_status();
+        my $ss = $manager_status->{service_status};
+        my $ns = $manager_status->{node_status};
+
+        my $rules = read_and_check_full_rules_config();
+        my ($together, $separate) = get_colocated_services($rules, $sid);
+
+        for my $node (keys %$ns) {
+            next if $ns->{$node} ne 'online';
+
+            for my $csid (sort keys %$separate) {
+                next if $ss->{$csid}->{node} && $ss->{$csid}->{node} ne $node;
+                next if $ss->{$csid}->{target} && $ss->{$csid}->{target} ne $node;
+
+                push @{ $disallowed_nodes->{$node} }, $csid;
+            }
+
+            next if $disallowed_nodes->{$node};
+
+            $allowed_nodes->{$node} = [];
+            for my $csid (sort keys %$together) {
+                next if $ss->{$csid}->{node} && $ss->{$csid}->{node} eq $node;
+                next if $ss->{$csid}->{target} && $ss->{$csid}->{target} eq $node;
+
+                push @{ $allowed_nodes->{$node} }, $csid;
+            }
+        }
+    }
+
+    return ($allowed_nodes, $disallowed_nodes);
+}
+
 # graceful, as long as locking + cfs_write works
 sub delete_service_from_config {
     my ($sid) = @_;
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel