[pve-devel] [PATCH ha-manager v2 18/26] test: ha tester: add test cases for strict positive colocation rules

[Daniel Kral <d.kral@proxmox.com>]

Add test cases for strict positive colocation rules, i.e. where services
must be kept on the same node together. These verify the behavior of the
services in strict positive colocation rules in case of a failover of
their assigned nodes in the following scenarios:

1. 2 pos. colocated services in a 3 node cluster; 1 node failing
2. 3 pos. colocated services in a 3 node cluster; 1 node failing
3. 3 pos. colocated services in a 3 node cluster; 1 node failing, but
   the recovery node cannot start one of the services
4. 3 pos. colocated services in a 3 node cluster; 1 service manually
   migrated to another node will migrate the other pos. colocated
   services to the same node as well
5. 9 pos. colocated services in a 3 node cluster; 1 service manually
   migrated to another node will migrate the other pos. colocated
   services to the same node as well

Signed-off-by: Daniel Kral <d.kral@proxmox.com>
---
changes since v1:
    - added test case 4 and 5
    - removed strict property from rules_config
    - smaller adaptions in existing READMEs as noted by @Fiona

 .../test-colocation-strict-together1/README   |  11 +
 .../test-colocation-strict-together1/cmdlist  |   4 +
 .../hardware_status                           |   5 +
 .../log.expect                                |  66 ++++
 .../manager_status                            |   1 +
 .../rules_config                              |   3 +
 .../service_config                            |   6 +
 .../test-colocation-strict-together2/README   |  10 +
 .../test-colocation-strict-together2/cmdlist  |   4 +
 .../hardware_status                           |   5 +
 .../log.expect                                |  80 +++++
 .../manager_status                            |   1 +
 .../rules_config                              |   3 +
 .../service_config                            |   8 +
 .../test-colocation-strict-together3/README   |  17 ++
 .../test-colocation-strict-together3/cmdlist  |   4 +
 .../hardware_status                           |   5 +
 .../log.expect                                |  89 ++++++
 .../manager_status                            |   1 +
 .../rules_config                              |   3 +
 .../service_config                            |   8 +
 .../test-colocation-strict-together4/README   |  11 +
 .../test-colocation-strict-together4/cmdlist  |   4 +
 .../hardware_status                           |   5 +
 .../log.expect                                |  59 ++++
 .../manager_status                            |   1 +
 .../rules_config                              |   3 +
 .../service_config                            |   5 +
 .../test-colocation-strict-together5/README   |  19 ++
 .../test-colocation-strict-together5/cmdlist  |   8 +
 .../hardware_status                           |   5 +
 .../log.expect                                | 281 ++++++++++++++++++
 .../manager_status                            |   1 +
 .../rules_config                              |  15 +
 .../service_config                            |  11 +
 35 files changed, 762 insertions(+)
 create mode 100644 src/test/test-colocation-strict-together1/README
 create mode 100644 src/test/test-colocation-strict-together1/cmdlist
 create mode 100644 src/test/test-colocation-strict-together1/hardware_status
 create mode 100644 src/test/test-colocation-strict-together1/log.expect
 create mode 100644 src/test/test-colocation-strict-together1/manager_status
 create mode 100644 src/test/test-colocation-strict-together1/rules_config
 create mode 100644 src/test/test-colocation-strict-together1/service_config
 create mode 100644 src/test/test-colocation-strict-together2/README
 create mode 100644 src/test/test-colocation-strict-together2/cmdlist
 create mode 100644 src/test/test-colocation-strict-together2/hardware_status
 create mode 100644 src/test/test-colocation-strict-together2/log.expect
 create mode 100644 src/test/test-colocation-strict-together2/manager_status
 create mode 100644 src/test/test-colocation-strict-together2/rules_config
 create mode 100644 src/test/test-colocation-strict-together2/service_config
 create mode 100644 src/test/test-colocation-strict-together3/README
 create mode 100644 src/test/test-colocation-strict-together3/cmdlist
 create mode 100644 src/test/test-colocation-strict-together3/hardware_status
 create mode 100644 src/test/test-colocation-strict-together3/log.expect
 create mode 100644 src/test/test-colocation-strict-together3/manager_status
 create mode 100644 src/test/test-colocation-strict-together3/rules_config
 create mode 100644 src/test/test-colocation-strict-together3/service_config
 create mode 100644 src/test/test-colocation-strict-together4/README
 create mode 100644 src/test/test-colocation-strict-together4/cmdlist
 create mode 100644 src/test/test-colocation-strict-together4/hardware_status
 create mode 100644 src/test/test-colocation-strict-together4/log.expect
 create mode 100644 src/test/test-colocation-strict-together4/manager_status
 create mode 100644 src/test/test-colocation-strict-together4/rules_config
 create mode 100644 src/test/test-colocation-strict-together4/service_config
 create mode 100644 src/test/test-colocation-strict-together5/README
 create mode 100644 src/test/test-colocation-strict-together5/cmdlist
 create mode 100644 src/test/test-colocation-strict-together5/hardware_status
 create mode 100644 src/test/test-colocation-strict-together5/log.expect
 create mode 100644 src/test/test-colocation-strict-together5/manager_status
 create mode 100644 src/test/test-colocation-strict-together5/rules_config
 create mode 100644 src/test/test-colocation-strict-together5/service_config

diff --git a/src/test/test-colocation-strict-together1/README b/src/test/test-colocation-strict-together1/README
new file mode 100644
index 0000000..1678cf1
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/README
@@ -0,0 +1,11 @@
+Test whether a strict positive colocation rule makes two services migrate to
+the same recovery node in case of a failover of their previously assigned node.
+
+The test scenario is:
+- vm:101 and vm:102 must be kept together
+- vm:101 and vm:102 are both currently running on node3
+- node1 and node2 have the same service count to test that the rule is applied
+  even though it would be usually balanced between both remaining nodes
+
+The expected outcome is:
+- As node3 fails, both services are migrated to node1
diff --git a/src/test/test-colocation-strict-together1/cmdlist b/src/test/test-colocation-strict-together1/cmdlist
new file mode 100644
index 0000000..c0a4daa
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-colocation-strict-together1/hardware_status b/src/test/test-colocation-strict-together1/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-colocation-strict-together1/log.expect b/src/test/test-colocation-strict-together1/log.expect
new file mode 100644
index 0000000..7d43314
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/log.expect
@@ -0,0 +1,66 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node3'
+info     20    node1/crm: adding new service 'vm:102' on node 'node3'
+info     20    node1/crm: adding new service 'vm:103' on node 'node1'
+info     20    node1/crm: adding new service 'vm:104' on node 'node2'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:104': state changed from 'request_start' to 'started'  (node = node2)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:103
+info     21    node1/lrm: service status vm:103 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:104
+info     23    node2/lrm: service status vm:104 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:101
+info     25    node3/lrm: service status vm:101 started
+info     25    node3/lrm: starting service vm:102
+info     25    node3/lrm: service status vm:102 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:101': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:101': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:101' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:101': state changed from 'recovery' to 'started'  (node = node1)
+info    240    node1/crm: recover service 'vm:102' from fenced node 'node3' to node 'node1'
+info    240    node1/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node1)
+info    241    node1/lrm: starting service vm:101
+info    241    node1/lrm: service status vm:101 started
+info    241    node1/lrm: starting service vm:102
+info    241    node1/lrm: service status vm:102 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-colocation-strict-together1/manager_status b/src/test/test-colocation-strict-together1/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-colocation-strict-together1/rules_config b/src/test/test-colocation-strict-together1/rules_config
new file mode 100644
index 0000000..1e63579
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/rules_config
@@ -0,0 +1,3 @@
+colocation: vms-must-stick-together
+	services vm:101,vm:102
+	affinity together
diff --git a/src/test/test-colocation-strict-together1/service_config b/src/test/test-colocation-strict-together1/service_config
new file mode 100644
index 0000000..9fb091d
--- /dev/null
+++ b/src/test/test-colocation-strict-together1/service_config
@@ -0,0 +1,6 @@
+{
+    "vm:101": { "node": "node3", "state": "started" },
+    "vm:102": { "node": "node3", "state": "started" },
+    "vm:103": { "node": "node1", "state": "started" },
+    "vm:104": { "node": "node2", "state": "started" }
+}
diff --git a/src/test/test-colocation-strict-together2/README b/src/test/test-colocation-strict-together2/README
new file mode 100644
index 0000000..b282e5f
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/README
@@ -0,0 +1,10 @@
+Test whether a strict positive colocation rule makes three services migrate to
+the same recovery node in case of a failover of their previously assigned node.
+
+The test scenario is:
+- vm:101, vm:102, and vm:103 must be kept together
+- vm:101, vm:102, and vm:103 are all currently running on node3
+
+The expected outcome is:
+- As node3 fails, all services are migrated to node2, as node2 is less utilized
+  than the other available node1
diff --git a/src/test/test-colocation-strict-together2/cmdlist b/src/test/test-colocation-strict-together2/cmdlist
new file mode 100644
index 0000000..c0a4daa
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-colocation-strict-together2/hardware_status b/src/test/test-colocation-strict-together2/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-colocation-strict-together2/log.expect b/src/test/test-colocation-strict-together2/log.expect
new file mode 100644
index 0000000..78f4d66
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/log.expect
@@ -0,0 +1,80 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node3'
+info     20    node1/crm: adding new service 'vm:102' on node 'node3'
+info     20    node1/crm: adding new service 'vm:103' on node 'node3'
+info     20    node1/crm: adding new service 'vm:104' on node 'node1'
+info     20    node1/crm: adding new service 'vm:105' on node 'node1'
+info     20    node1/crm: adding new service 'vm:106' on node 'node2'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:104': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:105': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:106': state changed from 'request_start' to 'started'  (node = node2)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:104
+info     21    node1/lrm: service status vm:104 started
+info     21    node1/lrm: starting service vm:105
+info     21    node1/lrm: service status vm:105 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:106
+info     23    node2/lrm: service status vm:106 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service vm:101
+info     25    node3/lrm: service status vm:101 started
+info     25    node3/lrm: starting service vm:102
+info     25    node3/lrm: service status vm:102 started
+info     25    node3/lrm: starting service vm:103
+info     25    node3/lrm: service status vm:103 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'vm:101': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'vm:101': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'vm:101' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:101': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:102' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node2)
+info    243    node2/lrm: starting service vm:101
+info    243    node2/lrm: service status vm:101 started
+info    243    node2/lrm: starting service vm:102
+info    243    node2/lrm: service status vm:102 started
+info    243    node2/lrm: starting service vm:103
+info    243    node2/lrm: service status vm:103 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-colocation-strict-together2/manager_status b/src/test/test-colocation-strict-together2/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-colocation-strict-together2/rules_config b/src/test/test-colocation-strict-together2/rules_config
new file mode 100644
index 0000000..22ffa1e
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/rules_config
@@ -0,0 +1,3 @@
+colocation: vms-must-stick-together
+	services vm:101,vm:102,vm:103
+	affinity together
diff --git a/src/test/test-colocation-strict-together2/service_config b/src/test/test-colocation-strict-together2/service_config
new file mode 100644
index 0000000..fd4a87e
--- /dev/null
+++ b/src/test/test-colocation-strict-together2/service_config
@@ -0,0 +1,8 @@
+{
+    "vm:101": { "node": "node3", "state": "started" },
+    "vm:102": { "node": "node3", "state": "started" },
+    "vm:103": { "node": "node3", "state": "started" },
+    "vm:104": { "node": "node1", "state": "started" },
+    "vm:105": { "node": "node1", "state": "started" },
+    "vm:106": { "node": "node2", "state": "started" }
+}
diff --git a/src/test/test-colocation-strict-together3/README b/src/test/test-colocation-strict-together3/README
new file mode 100644
index 0000000..35ce2e4
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/README
@@ -0,0 +1,17 @@
+Test whether a strict positive colocation rule makes three services migrate to
+the same recovery node in case of a failover of their previously assigned node.
+If one of those fail to start on the recovery node (e.g. insufficient
+resources), the failing service will be kept on the recovery node.
+
+The test scenario is:
+- vm:101, vm:102, and fa:120002 must be kept together
+- vm:101, vm:102, and fa:120002 are all currently running on node3
+- fa:120002 will fail to start on node2
+- node1 has a higher service count than node2 so that node2 is selected for
+  migration so that fa:12002 is guaranteed to fail there
+
+The expected outcome is:
+- As node3 fails, all services are migrated to node2
+- Two of those services will start successfully, but fa:120002 will stay in
+  recovery, since it cannot be started on this node, but cannot be relocated to
+  another one either due to the strict colocation rule
diff --git a/src/test/test-colocation-strict-together3/cmdlist b/src/test/test-colocation-strict-together3/cmdlist
new file mode 100644
index 0000000..c0a4daa
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "network node3 off" ]
+]
diff --git a/src/test/test-colocation-strict-together3/hardware_status b/src/test/test-colocation-strict-together3/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-colocation-strict-together3/log.expect b/src/test/test-colocation-strict-together3/log.expect
new file mode 100644
index 0000000..4a54cb3
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/log.expect
@@ -0,0 +1,89 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'fa:120002' on node 'node3'
+info     20    node1/crm: adding new service 'vm:101' on node 'node3'
+info     20    node1/crm: adding new service 'vm:102' on node 'node3'
+info     20    node1/crm: adding new service 'vm:104' on node 'node1'
+info     20    node1/crm: adding new service 'vm:105' on node 'node1'
+info     20    node1/crm: adding new service 'vm:106' on node 'node2'
+info     20    node1/crm: service 'fa:120002': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node3)
+info     20    node1/crm: service 'vm:104': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:105': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:106': state changed from 'request_start' to 'started'  (node = node2)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:104
+info     21    node1/lrm: service status vm:104 started
+info     21    node1/lrm: starting service vm:105
+info     21    node1/lrm: service status vm:105 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     23    node2/lrm: got lock 'ha_agent_node2_lock'
+info     23    node2/lrm: status change wait_for_agent_lock => active
+info     23    node2/lrm: starting service vm:106
+info     23    node2/lrm: service status vm:106 started
+info     24    node3/crm: status change wait_for_quorum => slave
+info     25    node3/lrm: got lock 'ha_agent_node3_lock'
+info     25    node3/lrm: status change wait_for_agent_lock => active
+info     25    node3/lrm: starting service fa:120002
+info     25    node3/lrm: service status fa:120002 started
+info     25    node3/lrm: starting service vm:101
+info     25    node3/lrm: service status vm:101 started
+info     25    node3/lrm: starting service vm:102
+info     25    node3/lrm: service status vm:102 started
+info    120      cmdlist: execute network node3 off
+info    120    node1/crm: node 'node3': state changed from 'online' => 'unknown'
+info    124    node3/crm: status change slave => wait_for_quorum
+info    125    node3/lrm: status change active => lost_agent_lock
+info    160    node1/crm: service 'fa:120002': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:101': state changed from 'started' to 'fence'
+info    160    node1/crm: service 'vm:102': state changed from 'started' to 'fence'
+info    160    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
+emai    160    node1/crm: FENCE: Try to fence node 'node3'
+info    166     watchdog: execute power node3 off
+info    165    node3/crm: killed by poweroff
+info    166    node3/lrm: killed by poweroff
+info    166     hardware: server 'node3' stopped by poweroff (watchdog)
+info    240    node1/crm: got lock 'ha_agent_node3_lock'
+info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
+emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
+info    240    node1/crm: service 'fa:120002': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:101': state changed from 'fence' to 'recovery'
+info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
+info    240    node1/crm: recover service 'fa:120002' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'fa:120002': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:101' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:101': state changed from 'recovery' to 'started'  (node = node2)
+info    240    node1/crm: recover service 'vm:102' from fenced node 'node3' to node 'node2'
+info    240    node1/crm: service 'vm:102': state changed from 'recovery' to 'started'  (node = node2)
+info    243    node2/lrm: starting service fa:120002
+warn    243    node2/lrm: unable to start service fa:120002
+warn    243    node2/lrm: restart policy: retry number 1 for service 'fa:120002'
+info    243    node2/lrm: starting service vm:101
+info    243    node2/lrm: service status vm:101 started
+info    243    node2/lrm: starting service vm:102
+info    243    node2/lrm: service status vm:102 started
+info    263    node2/lrm: starting service fa:120002
+warn    263    node2/lrm: unable to start service fa:120002
+err     263    node2/lrm: unable to start service fa:120002 on local node after 1 retries
+warn    280    node1/crm: starting service fa:120002 on node 'node2' failed, relocating service.
+warn    280    node1/crm: Start Error Recovery: Tried all available nodes for service 'fa:120002', retry start on current node. Tried nodes: node2
+info    283    node2/lrm: starting service fa:120002
+info    283    node2/lrm: service status fa:120002 started
+info    300    node1/crm: relocation policy successful for 'fa:120002' on node 'node2', failed nodes: node2
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-colocation-strict-together3/manager_status b/src/test/test-colocation-strict-together3/manager_status
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/manager_status
@@ -0,0 +1 @@
+{}
diff --git a/src/test/test-colocation-strict-together3/rules_config b/src/test/test-colocation-strict-together3/rules_config
new file mode 100644
index 0000000..46c00c8
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/rules_config
@@ -0,0 +1,3 @@
+colocation: vms-must-stick-together
+	services vm:101,vm:102,fa:120002
+	affinity together
diff --git a/src/test/test-colocation-strict-together3/service_config b/src/test/test-colocation-strict-together3/service_config
new file mode 100644
index 0000000..3ce5f27
--- /dev/null
+++ b/src/test/test-colocation-strict-together3/service_config
@@ -0,0 +1,8 @@
+{
+    "vm:101": { "node": "node3", "state": "started" },
+    "vm:102": { "node": "node3", "state": "started" },
+    "fa:120002": { "node": "node3", "state": "started" },
+    "vm:104": { "node": "node1", "state": "started" },
+    "vm:105": { "node": "node1", "state": "started" },
+    "vm:106": { "node": "node2", "state": "started" }
+}
diff --git a/src/test/test-colocation-strict-together4/README b/src/test/test-colocation-strict-together4/README
new file mode 100644
index 0000000..7ef7e69
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/README
@@ -0,0 +1,11 @@
+Test whether a strict positive colocation rule of three services makes the
+services stay together, if one of the services is manually migrated to another
+node, i.e., migrate to the same node.
+
+The test scenario is:
+- vm:101, vm:102, and vm:103 must be kept together
+- vm:101, vm:102, and vm:103 are all currently running on node1
+
+The expected outcome is:
+- As vm:101 is migrated to node2, vm:102 and vm:103 are migrated to node2 as
+  well as a side-effect to follow the positive colocation rule.
diff --git a/src/test/test-colocation-strict-together4/cmdlist b/src/test/test-colocation-strict-together4/cmdlist
new file mode 100644
index 0000000..2e420cc
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/cmdlist
@@ -0,0 +1,4 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "service vm:101 migrate node2" ]
+]
diff --git a/src/test/test-colocation-strict-together4/hardware_status b/src/test/test-colocation-strict-together4/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-colocation-strict-together4/log.expect b/src/test/test-colocation-strict-together4/log.expect
new file mode 100644
index 0000000..545f4eb
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/log.expect
@@ -0,0 +1,59 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node1'
+info     20    node1/crm: adding new service 'vm:103' on node 'node1'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node1)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     21    node1/lrm: starting service vm:102
+info     21    node1/lrm: service status vm:102 started
+info     21    node1/lrm: starting service vm:103
+info     21    node1/lrm: service status vm:103 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute service vm:101 migrate node2
+info    120    node1/crm: got crm command: migrate vm:101 node2
+info    120    node1/crm: crm command 'migrate vm:101 node2' - migrate positively colocated service 'vm:102' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:101 node2' - migrate positively colocated service 'vm:103' to 'node2'
+info    120    node1/crm: migrate service 'vm:101' to node 'node2'
+info    120    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:102' to node 'node2'
+info    120    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:103' to node 'node2'
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    121    node1/lrm: service vm:101 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:101 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:103 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:103 - end migrate to node 'node2'
+info    140    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node2)
+info    143    node2/lrm: got lock 'ha_agent_node2_lock'
+info    143    node2/lrm: status change wait_for_agent_lock => active
+info    143    node2/lrm: starting service vm:101
+info    143    node2/lrm: service status vm:101 started
+info    143    node2/lrm: starting service vm:102
+info    143    node2/lrm: service status vm:102 started
+info    143    node2/lrm: starting service vm:103
+info    143    node2/lrm: service status vm:103 started
+info    720     hardware: exit simulation - done
diff --git a/src/test/test-colocation-strict-together4/manager_status b/src/test/test-colocation-strict-together4/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-colocation-strict-together4/rules_config b/src/test/test-colocation-strict-together4/rules_config
new file mode 100644
index 0000000..22ffa1e
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/rules_config
@@ -0,0 +1,3 @@
+colocation: vms-must-stick-together
+	services vm:101,vm:102,vm:103
+	affinity together
diff --git a/src/test/test-colocation-strict-together4/service_config b/src/test/test-colocation-strict-together4/service_config
new file mode 100644
index 0000000..57e3579
--- /dev/null
+++ b/src/test/test-colocation-strict-together4/service_config
@@ -0,0 +1,5 @@
+{
+    "vm:101": { "node": "node1", "state": "started" },
+    "vm:102": { "node": "node1", "state": "started" },
+    "vm:103": { "node": "node1", "state": "started" }
+}
diff --git a/src/test/test-colocation-strict-together5/README b/src/test/test-colocation-strict-together5/README
new file mode 100644
index 0000000..22d5883
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/README
@@ -0,0 +1,19 @@
+Test whether multiple connected positive colocation rules makes the services
+stay together, if one of the services is manually migrated to another node,
+i.e., migrate all of them to the same node.
+
+The test scenario is:
+- vm:101, vm:102, and vm:103 must be kept together
+- vm:103, vm:104, and vm:105 must be kept together
+- vm:105, vm:106, and vm:107 must be kept together
+- vm:105, vm:108, and vm:109 must be kept together
+- So essentially, vm:101 through vm:109 must be kept together
+- vm:101 through vm:109 are all on node1
+
+The expected outcome is:
+- As vm:103 is migrated to node2, all of vm:101 through vm:109 are migrated to
+  node2 as well, as these all must be kept together
+- As vm:101 is migrated to node3, all of vm:101 through vm:109 are migrated to
+  node3 as well, as these all must be kept together
+- As vm:109 is migrated to node1, all of vm:101 through vm:109 are migrated to
+  node1 as well, as these all must be kept together
diff --git a/src/test/test-colocation-strict-together5/cmdlist b/src/test/test-colocation-strict-together5/cmdlist
new file mode 100644
index 0000000..85c33d0
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/cmdlist
@@ -0,0 +1,8 @@
+[
+    [ "power node1 on", "power node2 on", "power node3 on" ],
+    [ "service vm:103 migrate node2" ],
+    [ "delay 100" ],
+    [ "service vm:101 migrate node3" ],
+    [ "delay 100" ],
+    [ "service vm:109 migrate node1" ]
+]
diff --git a/src/test/test-colocation-strict-together5/hardware_status b/src/test/test-colocation-strict-together5/hardware_status
new file mode 100644
index 0000000..451beb1
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/hardware_status
@@ -0,0 +1,5 @@
+{
+  "node1": { "power": "off", "network": "off" },
+  "node2": { "power": "off", "network": "off" },
+  "node3": { "power": "off", "network": "off" }
+}
diff --git a/src/test/test-colocation-strict-together5/log.expect b/src/test/test-colocation-strict-together5/log.expect
new file mode 100644
index 0000000..4f5a0e6
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/log.expect
@@ -0,0 +1,281 @@
+info      0     hardware: starting simulation
+info     20      cmdlist: execute power node1 on
+info     20    node1/crm: status change startup => wait_for_quorum
+info     20    node1/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node2 on
+info     20    node2/crm: status change startup => wait_for_quorum
+info     20    node2/lrm: status change startup => wait_for_agent_lock
+info     20      cmdlist: execute power node3 on
+info     20    node3/crm: status change startup => wait_for_quorum
+info     20    node3/lrm: status change startup => wait_for_agent_lock
+info     20    node1/crm: got lock 'ha_manager_lock'
+info     20    node1/crm: status change wait_for_quorum => master
+info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
+info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
+info     20    node1/crm: adding new service 'vm:101' on node 'node1'
+info     20    node1/crm: adding new service 'vm:102' on node 'node1'
+info     20    node1/crm: adding new service 'vm:103' on node 'node1'
+info     20    node1/crm: adding new service 'vm:104' on node 'node1'
+info     20    node1/crm: adding new service 'vm:105' on node 'node1'
+info     20    node1/crm: adding new service 'vm:106' on node 'node1'
+info     20    node1/crm: adding new service 'vm:107' on node 'node1'
+info     20    node1/crm: adding new service 'vm:108' on node 'node1'
+info     20    node1/crm: adding new service 'vm:109' on node 'node1'
+info     20    node1/crm: service 'vm:101': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:102': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:104': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:105': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:106': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:107': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:108': state changed from 'request_start' to 'started'  (node = node1)
+info     20    node1/crm: service 'vm:109': state changed from 'request_start' to 'started'  (node = node1)
+info     21    node1/lrm: got lock 'ha_agent_node1_lock'
+info     21    node1/lrm: status change wait_for_agent_lock => active
+info     21    node1/lrm: starting service vm:101
+info     21    node1/lrm: service status vm:101 started
+info     21    node1/lrm: starting service vm:102
+info     21    node1/lrm: service status vm:102 started
+info     21    node1/lrm: starting service vm:103
+info     21    node1/lrm: service status vm:103 started
+info     21    node1/lrm: starting service vm:104
+info     21    node1/lrm: service status vm:104 started
+info     21    node1/lrm: starting service vm:105
+info     21    node1/lrm: service status vm:105 started
+info     21    node1/lrm: starting service vm:106
+info     21    node1/lrm: service status vm:106 started
+info     21    node1/lrm: starting service vm:107
+info     21    node1/lrm: service status vm:107 started
+info     21    node1/lrm: starting service vm:108
+info     21    node1/lrm: service status vm:108 started
+info     21    node1/lrm: starting service vm:109
+info     21    node1/lrm: service status vm:109 started
+info     22    node2/crm: status change wait_for_quorum => slave
+info     24    node3/crm: status change wait_for_quorum => slave
+info    120      cmdlist: execute service vm:103 migrate node2
+info    120    node1/crm: got crm command: migrate vm:103 node2
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:101' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:102' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:104' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:105' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:106' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:107' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:108' to 'node2'
+info    120    node1/crm: crm command 'migrate vm:103 node2' - migrate positively colocated service 'vm:109' to 'node2'
+info    120    node1/crm: migrate service 'vm:101' to node 'node2'
+info    120    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:102' to node 'node2'
+info    120    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:103' to node 'node2'
+info    120    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:104' to node 'node2'
+info    120    node1/crm: service 'vm:104': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:105' to node 'node2'
+info    120    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:106' to node 'node2'
+info    120    node1/crm: service 'vm:106': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:107' to node 'node2'
+info    120    node1/crm: service 'vm:107': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:108' to node 'node2'
+info    120    node1/crm: service 'vm:108': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    120    node1/crm: migrate service 'vm:109' to node 'node2'
+info    120    node1/crm: service 'vm:109': state changed from 'started' to 'migrate'  (node = node1, target = node2)
+info    121    node1/lrm: service vm:101 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:101 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:102 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:103 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:103 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:104 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:104 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:105 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:105 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:106 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:106 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:107 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:107 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:108 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:108 - end migrate to node 'node2'
+info    121    node1/lrm: service vm:109 - start migrate to node 'node2'
+info    121    node1/lrm: service vm:109 - end migrate to node 'node2'
+info    140    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:104': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:106': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:107': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:108': state changed from 'migrate' to 'started'  (node = node2)
+info    140    node1/crm: service 'vm:109': state changed from 'migrate' to 'started'  (node = node2)
+info    143    node2/lrm: got lock 'ha_agent_node2_lock'
+info    143    node2/lrm: status change wait_for_agent_lock => active
+info    143    node2/lrm: starting service vm:101
+info    143    node2/lrm: service status vm:101 started
+info    143    node2/lrm: starting service vm:102
+info    143    node2/lrm: service status vm:102 started
+info    143    node2/lrm: starting service vm:103
+info    143    node2/lrm: service status vm:103 started
+info    143    node2/lrm: starting service vm:104
+info    143    node2/lrm: service status vm:104 started
+info    143    node2/lrm: starting service vm:105
+info    143    node2/lrm: service status vm:105 started
+info    143    node2/lrm: starting service vm:106
+info    143    node2/lrm: service status vm:106 started
+info    143    node2/lrm: starting service vm:107
+info    143    node2/lrm: service status vm:107 started
+info    143    node2/lrm: starting service vm:108
+info    143    node2/lrm: service status vm:108 started
+info    143    node2/lrm: starting service vm:109
+info    143    node2/lrm: service status vm:109 started
+info    220      cmdlist: execute delay 100
+info    400      cmdlist: execute service vm:101 migrate node3
+info    400    node1/crm: got crm command: migrate vm:101 node3
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:102' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:103' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:104' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:105' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:106' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:107' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:108' to 'node3'
+info    400    node1/crm: crm command 'migrate vm:101 node3' - migrate positively colocated service 'vm:109' to 'node3'
+info    400    node1/crm: migrate service 'vm:101' to node 'node3'
+info    400    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:102' to node 'node3'
+info    400    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:103' to node 'node3'
+info    400    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:104' to node 'node3'
+info    400    node1/crm: service 'vm:104': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:105' to node 'node3'
+info    400    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:106' to node 'node3'
+info    400    node1/crm: service 'vm:106': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:107' to node 'node3'
+info    400    node1/crm: service 'vm:107': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:108' to node 'node3'
+info    400    node1/crm: service 'vm:108': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    400    node1/crm: migrate service 'vm:109' to node 'node3'
+info    400    node1/crm: service 'vm:109': state changed from 'started' to 'migrate'  (node = node2, target = node3)
+info    403    node2/lrm: service vm:101 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:101 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:102 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:102 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:103 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:103 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:104 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:104 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:105 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:105 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:106 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:106 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:107 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:107 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:108 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:108 - end migrate to node 'node3'
+info    403    node2/lrm: service vm:109 - start migrate to node 'node3'
+info    403    node2/lrm: service vm:109 - end migrate to node 'node3'
+info    420    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:104': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:106': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:107': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:108': state changed from 'migrate' to 'started'  (node = node3)
+info    420    node1/crm: service 'vm:109': state changed from 'migrate' to 'started'  (node = node3)
+info    425    node3/lrm: got lock 'ha_agent_node3_lock'
+info    425    node3/lrm: status change wait_for_agent_lock => active
+info    425    node3/lrm: starting service vm:101
+info    425    node3/lrm: service status vm:101 started
+info    425    node3/lrm: starting service vm:102
+info    425    node3/lrm: service status vm:102 started
+info    425    node3/lrm: starting service vm:103
+info    425    node3/lrm: service status vm:103 started
+info    425    node3/lrm: starting service vm:104
+info    425    node3/lrm: service status vm:104 started
+info    425    node3/lrm: starting service vm:105
+info    425    node3/lrm: service status vm:105 started
+info    425    node3/lrm: starting service vm:106
+info    425    node3/lrm: service status vm:106 started
+info    425    node3/lrm: starting service vm:107
+info    425    node3/lrm: service status vm:107 started
+info    425    node3/lrm: starting service vm:108
+info    425    node3/lrm: service status vm:108 started
+info    425    node3/lrm: starting service vm:109
+info    425    node3/lrm: service status vm:109 started
+info    500      cmdlist: execute delay 100
+info    680      cmdlist: execute service vm:109 migrate node1
+info    680    node1/crm: got crm command: migrate vm:109 node1
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:101' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:102' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:103' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:104' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:105' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:106' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:107' to 'node1'
+info    680    node1/crm: crm command 'migrate vm:109 node1' - migrate positively colocated service 'vm:108' to 'node1'
+info    680    node1/crm: migrate service 'vm:101' to node 'node1'
+info    680    node1/crm: service 'vm:101': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:102' to node 'node1'
+info    680    node1/crm: service 'vm:102': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:103' to node 'node1'
+info    680    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:104' to node 'node1'
+info    680    node1/crm: service 'vm:104': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:105' to node 'node1'
+info    680    node1/crm: service 'vm:105': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:106' to node 'node1'
+info    680    node1/crm: service 'vm:106': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:107' to node 'node1'
+info    680    node1/crm: service 'vm:107': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:108' to node 'node1'
+info    680    node1/crm: service 'vm:108': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    680    node1/crm: migrate service 'vm:109' to node 'node1'
+info    680    node1/crm: service 'vm:109': state changed from 'started' to 'migrate'  (node = node3, target = node1)
+info    685    node3/lrm: service vm:101 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:101 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:102 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:102 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:103 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:103 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:104 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:104 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:105 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:105 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:106 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:106 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:107 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:107 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:108 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:108 - end migrate to node 'node1'
+info    685    node3/lrm: service vm:109 - start migrate to node 'node1'
+info    685    node3/lrm: service vm:109 - end migrate to node 'node1'
+info    700    node1/crm: service 'vm:101': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:102': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:104': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:105': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:106': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:107': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:108': state changed from 'migrate' to 'started'  (node = node1)
+info    700    node1/crm: service 'vm:109': state changed from 'migrate' to 'started'  (node = node1)
+info    701    node1/lrm: starting service vm:101
+info    701    node1/lrm: service status vm:101 started
+info    701    node1/lrm: starting service vm:102
+info    701    node1/lrm: service status vm:102 started
+info    701    node1/lrm: starting service vm:103
+info    701    node1/lrm: service status vm:103 started
+info    701    node1/lrm: starting service vm:104
+info    701    node1/lrm: service status vm:104 started
+info    701    node1/lrm: starting service vm:105
+info    701    node1/lrm: service status vm:105 started
+info    701    node1/lrm: starting service vm:106
+info    701    node1/lrm: service status vm:106 started
+info    701    node1/lrm: starting service vm:107
+info    701    node1/lrm: service status vm:107 started
+info    701    node1/lrm: starting service vm:108
+info    701    node1/lrm: service status vm:108 started
+info    701    node1/lrm: starting service vm:109
+info    701    node1/lrm: service status vm:109 started
+info   1280     hardware: exit simulation - done
diff --git a/src/test/test-colocation-strict-together5/manager_status b/src/test/test-colocation-strict-together5/manager_status
new file mode 100644
index 0000000..9e26dfe
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/manager_status
@@ -0,0 +1 @@
+{}
\ No newline at end of file
diff --git a/src/test/test-colocation-strict-together5/rules_config b/src/test/test-colocation-strict-together5/rules_config
new file mode 100644
index 0000000..481bce5
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/rules_config
@@ -0,0 +1,15 @@
+colocation: vms-must-stick-together1
+	services vm:101,vm:102,vm:103
+	affinity together
+
+colocation: vms-must-stick-together2
+	services vm:103,vm:104,vm:105
+	affinity together
+
+colocation: vms-must-stick-together3
+	services vm:105,vm:106,vm:107
+	affinity together
+
+colocation: vms-must-stick-together4
+	services vm:105,vm:108,vm:109
+	affinity together
diff --git a/src/test/test-colocation-strict-together5/service_config b/src/test/test-colocation-strict-together5/service_config
new file mode 100644
index 0000000..48db7b1
--- /dev/null
+++ b/src/test/test-colocation-strict-together5/service_config
@@ -0,0 +1,11 @@
+{
+    "vm:101": { "node": "node1", "state": "started" },
+    "vm:102": { "node": "node1", "state": "started" },
+    "vm:103": { "node": "node1", "state": "started" },
+    "vm:104": { "node": "node1", "state": "started" },
+    "vm:105": { "node": "node1", "state": "started" },
+    "vm:106": { "node": "node1", "state": "started" },
+    "vm:107": { "node": "node1", "state": "started" },
+    "vm:108": { "node": "node1", "state": "started" },
+    "vm:109": { "node": "node1", "state": "started" }
+}
-- 
2.39.5



_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel