[PATCH ha-manager v4 15/27] test: add test for migration options

[Fiona Ebner <f.ebner@proxmox.com>]

Tests that the new JSON-style migrate command works, and that
migration options are passed down the HA stack to the LRM and
resource plugin. Tests that migration options are also used for
dependent resources, but filtered by resource type.

Signed-off-by: Fiona Ebner <f.ebner@proxmox.com>
---

New in v4.

 src/PVE/HA/Sim/Hardware.pm                    |  25 ++-
 src/PVE/HA/Sim/Resources.pm                   |  13 +-
 src/PVE/HA/Sim/Resources/VirtCT.pm            |  11 ++
 src/PVE/HA/Sim/Resources/VirtVM.pm            |  11 ++
 src/test/test-migrate-options/README          |  19 ++
 src/test/test-migrate-options/cmdlist         |   7 +
 src/test/test-migrate-options/hardware_status |   5 +
 src/test/test-migrate-options/log.expect      | 163 ++++++++++++++++++
 src/test/test-migrate-options/manager_status  |   1 +
 src/test/test-migrate-options/rules_config    |   3 +
 src/test/test-migrate-options/service_config  |   8 +
 11 files changed, 262 insertions(+), 4 deletions(-)
 create mode 100644 src/test/test-migrate-options/README
 create mode 100644 src/test/test-migrate-options/cmdlist
 create mode 100644 src/test/test-migrate-options/hardware_status
 create mode 100644 src/test/test-migrate-options/log.expect
 create mode 100644 src/test/test-migrate-options/manager_status
 create mode 100644 src/test/test-migrate-options/rules_config
 create mode 100644 src/test/test-migrate-options/service_config

diff --git a/src/PVE/HA/Sim/Hardware.pm b/src/PVE/HA/Sim/Hardware.pm
index e1a9ddd..743c20e 100644
--- a/src/PVE/HA/Sim/Hardware.pm
+++ b/src/PVE/HA/Sim/Hardware.pm
@@ -795,7 +795,7 @@ sub get_cfs_state {
 #   shutdown <node>
 #   restart-lrm <node>
 #   service <sid> <started|disabled|stopped|ignored>
-#   service <sid> <migrate|relocate> <target>
+#   service <sid> <migrate|relocate> <target> [<migrate-options>]
 #   service <sid> stop <timeout>
 #   service <sid> lock/unlock [lockname]
 #   service <sid> add <node> [<request-state=started>] [<running=0>]
@@ -939,11 +939,30 @@ sub sim_hardware_cmd {
                 $self->set_service_state($sid, $action);
 
             } elsif ($action eq 'migrate' || $action eq 'relocate') {
+                my $target = $params[0];
+                my $migrate_options = $params[1];
 
                 die "sim_hardware_cmd: missing target node for '$action' command"
-                    if !$param;
+                    if !$target;
 
-                $self->queue_crm_commands_nolock("$action $sid $param");
+                my $crm_command_str;
+                if ($action eq 'migrate' && $migrate_options) {
+                    my $crm_command = {
+                        kind => $action,
+                        sid => $sid,
+                        node => $target,
+                        options => {},
+                    };
+                    for my $migrate_option (split(',', $migrate_options)) {
+                        my ($key, $val) = split('=', $migrate_option);
+                        $crm_command->{options}->{$key} = $val;
+                    }
+                    $crm_command_str = JSON::to_json($crm_command, { canonical => 1 });
+                } else {
+                    $crm_command_str = "$action $sid $target";
+                }
+
+                $self->queue_crm_commands_nolock($crm_command_str);
 
             } elsif ($action eq 'stop') {
 
diff --git a/src/PVE/HA/Sim/Resources.pm b/src/PVE/HA/Sim/Resources.pm
index 730bfaa..16963f0 100644
--- a/src/PVE/HA/Sim/Resources.pm
+++ b/src/PVE/HA/Sim/Resources.pm
@@ -103,6 +103,11 @@ sub migrate {
         return;
     }
 
+    for my $key (sort keys $migrate_options->%*) {
+        my $val = $migrate_options->{$key};
+        $haenv->log('info', "service $sid migration - got option $key=$val");
+    }
+
     # explicitly shutdown if $online isn't true (relocate)
     if (!$online && $class->check_running($haenv, $id)) {
         $haenv->log("info", "stopping service $sid (relocate)");
@@ -125,7 +130,13 @@ sub migrate {
 sub migrate_json_properties {
     my ($class) = @_;
 
-    return $PVE::HA::Resources::common_migrate_json_properties;
+    my $props = { $PVE::HA::Resources::common_migrate_json_properties->%* };
+    $props->{'common'} = {
+        type => 'string',
+        optional => 1,
+        description => 'A migration property common to resource types.',
+    };
+    return $props;
 }
 
 sub remove_locks {
diff --git a/src/PVE/HA/Sim/Resources/VirtCT.pm b/src/PVE/HA/Sim/Resources/VirtCT.pm
index ecf3af4..1e66f83 100644
--- a/src/PVE/HA/Sim/Resources/VirtCT.pm
+++ b/src/PVE/HA/Sim/Resources/VirtCT.pm
@@ -32,4 +32,15 @@ sub migrate {
     return $class->SUPER::migrate($haenv, $id, $target, $online, $migrate_options);
 }
 
+sub migrate_json_properties {
+    my ($class) = @_;
+    my $props = { $class->SUPER::migrate_json_properties()->%* };
+    $props->{'ct-foo'} = {
+        type => 'string',
+        optional => 1,
+        description => 'A migration property specific to CTs.',
+    };
+    return $props;
+}
+
 1;
diff --git a/src/PVE/HA/Sim/Resources/VirtVM.pm b/src/PVE/HA/Sim/Resources/VirtVM.pm
index 4d2852e..c3d61ab 100644
--- a/src/PVE/HA/Sim/Resources/VirtVM.pm
+++ b/src/PVE/HA/Sim/Resources/VirtVM.pm
@@ -16,4 +16,15 @@ sub exists {
     return 1;
 }
 
+sub migrate_json_properties {
+    my ($class) = @_;
+    my $props = { $class->SUPER::migrate_json_properties()->%* };
+    $props->{'vm-bar'} = {
+        type => 'string',
+        optional => 1,
+        description => 'A migration property specific to VMs.',
+    };
+    return $props;
+}
+
 1;
diff --git a/src/test/test-migrate-options/README b/src/test/test-migrate-options/README
new file mode 100644
index 0000000..9ea0b0a
--- /dev/null
+++ b/src/test/test-migrate-options/README
@@ -0,0 +1,19 @@
+Test that migrate options are correctly passed down, also when migrating
+dependent resources.
+
+The test scenario is:
+- vm:101, ct:102, vm:103, ct:104, vm:105 and ct:106 are all currently running on
+  node1.
+- vm:101 and ct:102 are migrated to node2 with the a guest-type-specific
+  migration option and a common migration option.
+- vm:103, ct:104, vm:105 and ct:106 are kept together via a positive affinity
+  rule.
+- vm:103 is migrated to node2 with a vm-specific and a common migration option.
+- ct:104 is migrated to node3 with a ct-specific and a common migration option.
+
+The expected outcome is:
+- vm:101 and ct:102 end up on node2.
+- vm:103, ct:104, vm:105 and ct:106 all end up on node3.
+- The guest-type-specific migration option is passed down for the matching guest
+  type and the common migration options is always passed down, also when
+  migrating dependent resources.
diff --git a/src/test/test-migrate-options/cmdlist b/src/test/test-migrate-options/cmdlist
new file mode 100644
index 0000000..eb36afb
--- /dev/null
+++ b/src/test/test-migrate-options/cmdlist
@@ -0,0 +1,7 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "service vm:101 migrate node2 vm-bar=one,common=first"],
+    [ "service ct:102 migrate node2 ct-foo=two,common=second"],
+    [ "service vm:103 migrate node2 vm-bar=three,common=third"],
+    [ "service ct:104 migrate node3 ct-foo=four,common=fourth"]
+]
diff --git a/src/test/test-migrate-options/hardware_status b/src/test/test-migrate-options/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-migrate-options/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-migrate-options/log.expect b/src/test/test-migrate-options/log.expect
new file mode 100644
index 0000000..a0ca883
--- /dev/null
+++ b/src/test/test-migrate-options/log.expect
@@ -0,0 +1,163 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'ct:102' on node 'node1'
+info     20    node1/crm: adding new service 'ct:104' on node 'node1'
+info     20    node1/crm: adding new service 'ct:106' on node 'node1'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:103' on node 'node1'
+info     20    node1/crm: adding new service 'vm:105' on node 'node1'
+info     20    node1/crm: service 'ct:102': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'ct:104': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'ct:106': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:105': state changed from 'request_start' to 'started'  (node = node1)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service ct:102
+info     21    node1/lrm: service status ct:102 started
+info     21    node1/lrm: starting service ct:104
+info     21    node1/lrm: service status ct:104 started
+info     21    node1/lrm: starting service ct:106
+info     21    node1/lrm: service status ct:106 started
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     21    node1/lrm: starting service vm:103
+info     21    node1/lrm: service status vm:103 started
+info     21    node1/lrm: starting service vm:105
+info     21    node1/lrm: service status vm:105 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute service vm:101 migrate node2 vm-bar=one,common=first
+info    120    node1/crm: got crm command: migrate vm:101 node2 {"common":"first","vm-bar":"one"}
+info    120    node1/crm: migrate service 'vm:101' to node 'node2'
+info    120    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (migrate-options = {"common":"first","vm-bar":"one"}, node = node1, target = node2)
+info    121    node1/lrm: service vm:101 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:101 migration - got option common=first
+info    121    node1/lrm: service vm:101 migration - got option vm-bar=one
+info    121    node1/lrm: service vm:101 - end migrate to node 'node2'
+info    123    node2/lrm: got lock 'ha_agent_node2_lock'
+info    123    node2/lrm: status change wait_for_agent_lock => active
+info    140    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node2)
+info    143    node2/lrm: starting service vm:101
+info    143    node2/lrm: service status vm:101 started
+info    220      cmdlist: execute service ct:102 migrate node2 ct-foo=two,common=second
+info    220    node1/crm: got crm command: migrate ct:102 node2 {"common":"second","ct-foo":"two"}
+info    220    node1/crm: migrate service 'ct:102' to node 'node2'
+info    220    node1/crm: service 'ct:102': state changed from 'started' to 'migrate'  (migrate-options = {"common":"second","ct-foo":"two"}, node = node1, target = node2)
+warn    221    node1/lrm: unable to live migrate running container, fallback to relocate
+info    221    node1/lrm: service ct:102 - start relocate to node 'node2'
+info    221    node1/lrm: service ct:102 migration - got option common=second
+info    221    node1/lrm: service ct:102 migration - got option ct-foo=two
+info    221    node1/lrm: stopping service ct:102 (relocate)
+info    221    node1/lrm: service status ct:102 stopped
+info    221    node1/lrm: service ct:102 - end relocate to node 'node2'
+info    240    node1/crm: service 'ct:102': state changed from 'migrate' to 'started'  (node = node2)
+info    243    node2/lrm: starting service ct:102
+info    243    node2/lrm: service status ct:102 started
+info    320      cmdlist: execute service vm:103 migrate node2 vm-bar=three,common=third
+info    320    node1/crm: got crm command: migrate vm:103 node2 {"common":"third","vm-bar":"three"}
+info    320    node1/crm: crm command 'migrate vm:103 node2 {"common":"third","vm-bar":"three"}' - migrate service 'ct:104' to node 'node2' (service 'ct:104' in positive affinity with service 'vm:103')
+info    320    node1/crm: crm command 'migrate vm:103 node2 {"common":"third","vm-bar":"three"}' - migrate service 'ct:106' to node 'node2' (service 'ct:106' in positive affinity with service 'vm:103')
+info    320    node1/crm: crm command 'migrate vm:103 node2 {"common":"third","vm-bar":"three"}' - migrate service 'vm:105' to node 'node2' (service 'vm:105' in positive affinity with service 'vm:103')
+info    320    node1/crm: migrate service 'ct:104' to node 'node2'
+info    320    node1/crm: service 'ct:104': state changed from 'started' to 'migrate'  (migrate-options = {"common":"third"}, node = node1, target = node2)
+info    320    node1/crm: migrate service 'ct:106' to node 'node2'
+info    320    node1/crm: service 'ct:106': state changed from 'started' to 'migrate'  (migrate-options = {"common":"third"}, node = node1, target = node2)
+info    320    node1/crm: migrate service 'vm:103' to node 'node2'
+info    320    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (migrate-options = {"common":"third","vm-bar":"three"}, node = node1, target = node2)
+info    320    node1/crm: migrate service 'vm:105' to node 'node2'
+info    320    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (migrate-options = {"common":"third","vm-bar":"three"}, node = node1, target = node2)
+warn    321    node1/lrm: unable to live migrate running container, fallback to relocate
+info    321    node1/lrm: service ct:104 - start relocate to node 'node2'
+info    321    node1/lrm: service ct:104 migration - got option common=third
+info    321    node1/lrm: stopping service ct:104 (relocate)
+info    321    node1/lrm: service status ct:104 stopped
+info    321    node1/lrm: service ct:104 - end relocate to node 'node2'
+warn    321    node1/lrm: unable to live migrate running container, fallback to relocate
+info    321    node1/lrm: service ct:106 - start relocate to node 'node2'
+info    321    node1/lrm: service ct:106 migration - got option common=third
+info    321    node1/lrm: stopping service ct:106 (relocate)
+info    321    node1/lrm: service status ct:106 stopped
+info    321    node1/lrm: service ct:106 - end relocate to node 'node2'
+info    321    node1/lrm: service vm:103 - start migrate to node 'node2'
+info    321    node1/lrm: service vm:103 migration - got option common=third
+info    321    node1/lrm: service vm:103 migration - got option vm-bar=three
+info    321    node1/lrm: service vm:103 - end migrate to node 'node2'
+info    321    node1/lrm: service vm:105 - start migrate to node 'node2'
+info    321    node1/lrm: service vm:105 migration - got option common=third
+info    321    node1/lrm: service vm:105 migration - got option vm-bar=three
+info    321    node1/lrm: service vm:105 - end migrate to node 'node2'
+info    340    node1/crm: service 'ct:104': state changed from 'migrate' to 'started'  (node = node2)
+info    340    node1/crm: service 'ct:106': state changed from 'migrate' to 'started'  (node = node2)
+info    340    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node2)
+info    340    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node2)
+info    343    node2/lrm: starting service ct:104
+info    343    node2/lrm: service status ct:104 started
+info    343    node2/lrm: starting service ct:106
+info    343    node2/lrm: service status ct:106 started
+info    343    node2/lrm: starting service vm:103
+info    343    node2/lrm: service status vm:103 started
+info    343    node2/lrm: starting service vm:105
+info    343    node2/lrm: service status vm:105 started
+info    420      cmdlist: execute service ct:104 migrate node3 ct-foo=four,common=fourth
+info    420    node1/crm: got crm command: migrate ct:104 node3 {"common":"fourth","ct-foo":"four"}
+info    420    node1/crm: crm command 'migrate ct:104 node3 {"common":"fourth","ct-foo":"four"}' - migrate service 'ct:106' to node 'node3' (service 'ct:106' in positive affinity with service 'ct:104')
+info    420    node1/crm: crm command 'migrate ct:104 node3 {"common":"fourth","ct-foo":"four"}' - migrate service 'vm:103' to node 'node3' (service 'vm:103' in positive affinity with service 'ct:104')
+info    420    node1/crm: crm command 'migrate ct:104 node3 {"common":"fourth","ct-foo":"four"}' - migrate service 'vm:105' to node 'node3' (service 'vm:105' in positive affinity with service 'ct:104')
+info    420    node1/crm: migrate service 'ct:104' to node 'node3'
+info    420    node1/crm: service 'ct:104': state changed from 'started' to 'migrate'  (migrate-options = {"common":"fourth","ct-foo":"four"}, node = node2, target = node3)
+info    420    node1/crm: migrate service 'ct:106' to node 'node3'
+info    420    node1/crm: service 'ct:106': state changed from 'started' to 'migrate'  (migrate-options = {"common":"fourth","ct-foo":"four"}, node = node2, target = node3)
+info    420    node1/crm: migrate service 'vm:103' to node 'node3'
+info    420    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (migrate-options = {"common":"fourth"}, node = node2, target = node3)
+info    420    node1/crm: migrate service 'vm:105' to node 'node3'
+info    420    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (migrate-options = {"common":"fourth"}, node = node2, target = node3)
+warn    423    node2/lrm: unable to live migrate running container, fallback to relocate
+info    423    node2/lrm: service ct:104 - start relocate to node 'node3'
+info    423    node2/lrm: service ct:104 migration - got option common=fourth
+info    423    node2/lrm: service ct:104 migration - got option ct-foo=four
+info    423    node2/lrm: stopping service ct:104 (relocate)
+info    423    node2/lrm: service status ct:104 stopped
+info    423    node2/lrm: service ct:104 - end relocate to node 'node3'
+warn    423    node2/lrm: unable to live migrate running container, fallback to relocate
+info    423    node2/lrm: service ct:106 - start relocate to node 'node3'
+info    423    node2/lrm: service ct:106 migration - got option common=fourth
+info    423    node2/lrm: service ct:106 migration - got option ct-foo=four
+info    423    node2/lrm: stopping service ct:106 (relocate)
+info    423    node2/lrm: service status ct:106 stopped
+info    423    node2/lrm: service ct:106 - end relocate to node 'node3'
+info    423    node2/lrm: service vm:103 - start migrate to node 'node3'
+info    423    node2/lrm: service vm:103 migration - got option common=fourth
+info    423    node2/lrm: service vm:103 - end migrate to node 'node3'
+info    423    node2/lrm: service vm:105 - start migrate to node 'node3'
+info    423    node2/lrm: service vm:105 migration - got option common=fourth
+info    423    node2/lrm: service vm:105 - end migrate to node 'node3'
+info    425    node3/lrm: got lock 'ha_agent_node3_lock'
+info    425    node3/lrm: status change wait_for_agent_lock => active
+info    440    node1/crm: service 'ct:104': state changed from 'migrate' to 'started'  (node = node3)
+info    440    node1/crm: service 'ct:106': state changed from 'migrate' to 'started'  (node = node3)
+info    440    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node3)
+info    440    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node3)
+info    445    node3/lrm: starting service ct:104
+info    445    node3/lrm: service status ct:104 started
+info    445    node3/lrm: starting service ct:106
+info    445    node3/lrm: service status ct:106 started
+info    445    node3/lrm: starting service vm:103
+info    445    node3/lrm: service status vm:103 started
+info    445    node3/lrm: starting service vm:105
+info    445    node3/lrm: service status vm:105 started
+info   1020     hardware: exit simulation - done
diff --git a/src/test/test-migrate-options/manager_status b/src/test/test-migrate-options/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-migrate-options/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-migrate-options/rules_config b/src/test/test-migrate-options/rules_config
new file mode 100644
index 0000000..d21bb86
--- /dev/null
+++ b/src/test/test-migrate-options/rules_config
@@ -0,0 +1,3 @@
+resource-affinity: must-stick-together
+	resources vm:103,ct:104,vm:105,ct:106
+	affinity positive
diff --git a/src/test/test-migrate-options/service_config b/src/test/test-migrate-options/service_config
new file mode 100644
index 0000000..44777b5
--- /dev/null
+++ b/src/test/test-migrate-options/service_config
@@ -0,0 +1,8 @@
+{
+    "vm:101": { "node": "node1", "state": "started" },
+    "ct:102": { "node": "node1", "state": "started" },
+    "vm:103": { "node": "node1", "state": "started" },
+    "ct:104": { "node": "node1", "state": "started" },
+    "vm:105": { "node": "node1", "state": "started" },
+    "ct:106": { "node": "node1", "state": "started" }
+}
-- 
2.47.3