[pve-devel] [PATCH ha-manager v3 13/13] api: resources: add check for resource affinity in resource migrations

[Daniel Kral <d.kral@proxmox.com>]

The HA Manager already handles positive and negative resource affinity
rules for individual resource migrations, but the information about
these is only redirected to the HA environment's logger, i.e., for
production usage these messages are redirected to the HA Manager node's
syslog.

Therefore, add checks when migrating/relocating resources through their
respective API endpoints to give users information about comigrated
resources, i.e., resources which are migrated together to the requested
target node because of positive resource affinity rules, and blocking
resources, i.e., resources which are blocking a resource to be migrated
to a requested target node, because of a negative resource affinity
rule.

get_resource_motion_info(...) is also callable from other packages, to
get a listing of all allowed and disallowed nodes with respect to the
Resource Affinity rules, e.g., a migration precondition check.

Signed-off-by: Daniel Kral <d.kral@proxmox.com>
---
 src/PVE/API2/HA/Resources.pm | 128 +++++++++++++++++++++++++++++++++--
 src/PVE/CLI/ha_manager.pm    |  52 +++++++++++++-
 src/PVE/HA/Config.pm         |  56 +++++++++++++++
 3 files changed, 228 insertions(+), 8 deletions(-)

diff --git a/src/PVE/API2/HA/Resources.pm b/src/PVE/API2/HA/Resources.pm
index 6ead5f0..d0b8d7e 100644
--- a/src/PVE/API2/HA/Resources.pm
+++ b/src/PVE/API2/HA/Resources.pm
@@ -319,19 +319,75 @@ __PACKAGE__->register_method({
             ),
         },
     },
-    returns => { type => 'null' },
+    returns => {
+        type => 'object',
+        properties => {
+            sid => {
+                description => "HA resource, which is requested to be migrated.",
+                type => 'string',
+                optional => 0,
+            },
+            'requested-node' => {
+                description => "Node, which was requested to be migrated to.",
+                type => 'string',
+                optional => 0,
+            },
+            'comigrated-resources' => {
+                description => "HA resources, which are migrated to the same"
+                    . " requested target node as the given HA resource, because"
+                    . " these are in positive affinity with the HA resource.",
+                type => 'array',
+                optional => 1,
+            },
+            'blocking-resources' => {
+                description => "HA resources, which are blocking the given"
+                    . " HA resource from being migrated to the requested"
+                    . " target node.",
+                type => 'array',
+                optional => 1,
+                items => {
+                    description => "A blocking HA resource",
+                    type => 'object',
+                    properties => {
+                        sid => {
+                            type => 'string',
+                            description => "The blocking HA resource id",
+                        },
+                        cause => {
+                            type => 'string',
+                            description => "The reason why the HA resource is"
+                                . " blocking the migration.",
+                            enum => ['resource-affinity'],
+                        },
+                    },
+                },
+            },
+        },
+    },
     code => sub {
         my ($param) = @_;
 
+        my $result = {};
+
         my ($sid, $type, $name) = PVE::HA::Config::parse_sid(extract_param($param, 'sid'));
+        my $req_node = extract_param($param, 'node');
 
         PVE::HA::Config::service_is_ha_managed($sid);
 
         check_service_state($sid);
 
-        PVE::HA::Config::queue_crm_commands("migrate $sid $param->{node}");
+        PVE::HA::Config::queue_crm_commands("migrate $sid $req_node");
+        $result->{sid} = $sid;
+        $result->{'requested-node'} = $req_node;
 
-        return undef;
+        my ($comigrated_resources, $blocking_resources_by_node) =
+            PVE::HA::Config::get_resource_motion_info($sid);
+        my $blocking_resources = $blocking_resources_by_node->{$req_node};
+
+        $result->{'comigrated-resources'} = $comigrated_resources if @$comigrated_resources;
+        $result->{'blocking-resources'} = $blocking_resources if $blocking_resources;
+
+        return $result;
     },
 });
 
@@ -361,19 +417,79 @@ __PACKAGE__->register_method({
             ),
         },
     },
-    returns => { type => 'null' },
+    returns => {
+        type => 'object',
+        properties => {
+            sid => {
+                description => "HA resource, which is requested to be relocated.",
+                type => 'string',
+                optional => 0,
+            },
+            'requested-node' => {
+                description => "Node, which was requested to be relocated to.",
+                type => 'string',
+                optional => 0,
+            },
+            'comigrated-resources' => {
+                description => "HA resources, which are relocated to the same"
+                    . " requested target node as the given HA resource, because"
+                    . " these are in positive affinity with the HA resource.",
+                type => 'array',
+                optional => 1,
+                items => {
+                    type => 'string',
+                    description => "A comigrated HA resource",
+                },
+            },
+            'blocking-resources' => {
+                description => "HA resources, which are blocking the given"
+                    . " HA resource from being relocated to the requested"
+                    . " target node.",
+                type => 'array',
+                optional => 1,
+                items => {
+                    description => "A blocking HA resource",
+                    type => 'object',
+                    properties => {
+                        sid => {
+                            type => 'string',
+                            description => "The blocking HA resource id",
+                        },
+                        cause => {
+                            type => 'string',
+                            description => "The reason why the HA resource is"
+                                . " blocking the relocation.",
+                            enum => ['resource-affinity'],
+                        },
+                    },
+                },
+            },
+        },
+    },
     code => sub {
         my ($param) = @_;
 
+        my $result = {};
+
         my ($sid, $type, $name) = PVE::HA::Config::parse_sid(extract_param($param, 'sid'));
+        my $req_node = extract_param($param, 'node');
 
         PVE::HA::Config::service_is_ha_managed($sid);
 
         check_service_state($sid);
 
-        PVE::HA::Config::queue_crm_commands("relocate $sid $param->{node}");
+        PVE::HA::Config::queue_crm_commands("relocate $sid $req_node");
+        $result->{sid} = $sid;
+        $result->{'requested-node'} = $req_node;
 
-        return undef;
+        my ($comigrated_resources, $blocking_resources_by_node) =
+            PVE::HA::Config::get_resource_motion_info($sid);
+        my $blocking_resources = $blocking_resources_by_node->{$req_node};
+
+        $result->{'comigrated-resources'} = $comigrated_resources if @$comigrated_resources;
+        $result->{'blocking-resources'} = $blocking_resources if @$blocking_resources;
+
+        return $result;
     },
 });
 
diff --git a/src/PVE/CLI/ha_manager.pm b/src/PVE/CLI/ha_manager.pm
index ef936cd..d1f8393 100644
--- a/src/PVE/CLI/ha_manager.pm
+++ b/src/PVE/CLI/ha_manager.pm
@@ -147,6 +147,42 @@ __PACKAGE__->register_method({
     },
 });
 
+my $print_resource_motion_output = sub {
+    my ($cmd) = @_;
+
+    return sub {
+        my ($data) = @_;
+
+        my $sid = $data->{sid};
+        my $req_node = $data->{'requested-node'};
+
+        if (my $blocking_resources = $data->{'blocking-resources'}) {
+            my $err_msg = "cannot $cmd resource '$sid' to node '$req_node':\n\n";
+
+            for my $blocking_resource (@$blocking_resources) {
+                my ($csid, $cause) = $blocking_resource->@{qw(sid cause)};
+
+                $err_msg .= "- resource '$csid' on target node '$req_node'";
+
+                if ($cause eq 'resource-affinity') {
+                    $err_msg .= " in negative affinity with resource '$sid'";
+                }
+
+                $err_msg .= "\n";
+            }
+
+            die $err_msg;
+        }
+
+        if ($data->{'comigrated-resources'}) {
+            for my $csid ($data->{'comigrated-resources'}->@*) {
+                print "also $cmd resource '$csid' in positive affinity with"
+                    . " resource '$sid' to target node '$req_node'\n";
+            }
+        }
+    };
+};
+
 our $cmddef = {
     status => [__PACKAGE__, 'status'],
     config => [
@@ -239,8 +275,20 @@ our $cmddef = {
     relocate => { alias => 'crm-command relocate' },
 
     'crm-command' => {
-        migrate => ["PVE::API2::HA::Resources", 'migrate', ['sid', 'node']],
-        relocate => ["PVE::API2::HA::Resources", 'relocate', ['sid', 'node']],
+        migrate => [
+            "PVE::API2::HA::Resources",
+            'migrate',
+            ['sid', 'node'],
+            {},
+            $print_resource_motion_output->('migrate'),
+        ],
+        relocate => [
+            "PVE::API2::HA::Resources",
+            'relocate',
+            ['sid', 'node'],
+            {},
+            $print_resource_motion_output->('relocate'),
+        ],
         stop => [__PACKAGE__, 'stop', ['sid', 'timeout']],
         'node-maintenance' => {
             enable => [__PACKAGE__, 'node-maintenance-set', ['node'], { disable => 0 }],
diff --git a/src/PVE/HA/Config.pm b/src/PVE/HA/Config.pm
index 59bafd7..003909e 100644
--- a/src/PVE/HA/Config.pm
+++ b/src/PVE/HA/Config.pm
@@ -8,6 +8,7 @@ use JSON;
 use PVE::HA::Tools;
 use PVE::HA::Groups;
 use PVE::HA::Rules;
+use PVE::HA::Rules::ResourceAffinity qw(get_affinitive_resources);
 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
 use PVE::HA::Resources;
 
@@ -223,6 +224,24 @@ sub read_and_check_rules_config {
     return $rules;
 }
 
+sub read_and_check_effective_rules_config {
+
+    my $rules = read_and_check_rules_config();
+
+    my $manager_status = read_manager_status();
+    my $nodes = [keys $manager_status->{node_status}->%*];
+
+    # TODO PVE 10: Remove group migration when HA groups have been fully migrated to location rules
+    my $groups = read_group_config();
+    my $resources = read_and_check_resources_config();
+
+    PVE::HA::Groups::migrate_groups_to_rules($rules, $groups, $resources);
+
+    PVE::HA::Rules->canonicalize($rules, $nodes);
+
+    return $rules;
+}
+
 sub write_rules_config {
     my ($cfg) = @_;
 
@@ -350,6 +369,43 @@ sub service_is_configured {
     return 0;
 }
 
+sub get_resource_motion_info {
+    my ($sid) = @_;
+
+    my $resources = read_resources_config();
+
+    my $comigrated_resources = [];
+    my $blocking_resources_by_node = {};
+
+    if (&$service_check_ha_state($resources, $sid)) {
+        my $manager_status = read_manager_status();
+        my $ss = $manager_status->{service_status};
+        my $ns = $manager_status->{node_status};
+
+        my $rules = read_and_check_effective_rules_config();
+        my ($together, $separate) = get_affinitive_resources($rules, $sid);
+
+        push @$comigrated_resources, $_ for sort keys %$together;
+
+        for my $node (keys %$ns) {
+            next if $ns->{$node} ne 'online';
+
+            for my $csid (sort keys %$separate) {
+                next if $ss->{$csid}->{node} && $ss->{$csid}->{node} ne $node;
+                next if $ss->{$csid}->{target} && $ss->{$csid}->{target} ne $node;
+
+                push $blocking_resources_by_node->{$node}->@*,
+                    {
+                        sid => $csid,
+                        cause => 'resource-affinity',
+                    };
+            }
+        }
+    }
+
+    return ($comigrated_resources, $blocking_resources_by_node);
+}
+
 # graceful, as long as locking + cfs_write works
 sub delete_service_from_config {
     my ($sid) = @_;
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel