[PATCH e2e-tests 35/38] tests: migration: verify container migration between nodes

["Michael Köppl" <m.koeppl@proxmox.com>]

Migrate a stopped container with its root file system on local-lvm
to another node, then migrate it back while it runs, in restart
mode. Check that the container and its volume end up on the target
node, that nothing is left behind on the source node and that its
data is intact. Migrating a running container without restart mode
must be refused.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/pve_ct_migration.pl | 159 ++++++++++++++++++++++++++++++++++++++
 tests/testcases.ron       |  11 +++
 2 files changed, 170 insertions(+)
 create mode 100755 tests/pve_ct_migration.pl

diff --git a/tests/pve_ct_migration.pl b/tests/pve_ct_migration.pl
new file mode 100755
index 0000000..b29faf7
--- /dev/null
+++ b/tests/pve_ct_migration.pl
@@ -0,0 +1,159 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<pve_ct_migration> - Test migrating a container between cluster nodes.
+
+=head1 DESCRIPTION
+
+This testcase creates a container with its root file system on the local-lvm
+storage of the first cluster node and writes a file to it. Migrating the running
+container must be refused unless restart mode is used.
+
+It migrates the stopped container to the second node, which must move both the
+container and its volume there. It then starts the container on the second node
+and migrates it back in restart mode, after which the container must run on the
+first node again. The file must be intact after each migration.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use lib '.';
+use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instance);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_a = get_pve_instance('pve1-cluster');
+my $pve_b = get_pve_instance('pve2-cluster');
+my $node_a = $pve_a->get_nodename();
+my $node_b = $pve_b->get_nodename();
+my $client = $pve_a->client();
+
+my $ctid;
+my $marker = 'e2e-ct-migration';
+
+# The node the container lives on according to the cluster resources.
+my $resource_node = sub {
+    my $resources = $client->get('/cluster/resources', { type => 'vm' });
+    my ($entry) = grep { ($_->{vmid} // 0) == $ctid } $resources->@*;
+    return $entry ? $entry->{node} : undef;
+};
+
+my $wait_for_node = sub {
+    my ($node) = @_;
+    poll_until(
+        sub { ($resource_node->() // '') eq $node },
+        timeout => 60,
+        description => "container $ctid to show up on node $node",
+    );
+};
+
+my $volumes_on = sub {
+    my ($node) = @_;
+    my $content = $client->get("/nodes/$node/storage/local-lvm/content", { vmid => $ctid });
+    return scalar($content->@*);
+};
+
+# Verifies that both the container and its volume are on the expected node.
+my $verify_location = sub {
+    my ($instance, $other) = @_;
+    my $node = $instance->get_nodename();
+    my $other_node = $other->get_nodename();
+
+    $wait_for_node->($node);
+    is($resource_node->(), $node, "container $ctid is on node $node");
+    ok(
+        (grep { $_->{vmid} == $ctid } $client->get("/nodes/$node/lxc", {})->@*),
+        "node $node lists container $ctid",
+    );
+    ok($volumes_on->($node), "volume of container $ctid is on node $node");
+    ok(!$volumes_on->($other_node), "no volume of container $ctid is left on node $other_node");
+};
+
+Proxmox::Test::Testcase->new('pve_ct_migration')->with([{}])->step(
+    'create container and write file',
+    sub {
+        $ctid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node_a/lxc",
+            {
+                vmid => $ctid,
+                ostemplate => CONTAINER_TEMPLATE,
+                rootfs => 'local-lvm:2',
+                memory => 256,
+                unprivileged => 1,
+                features => 'nesting=1',
+                hostname => 'ct-migration',
+            },
+        );
+        $pve_a->task_ok($upid, "created container $ctid on node $node_a")
+            or die "creating container $ctid failed\n";
+
+        $pve_a->start_guest('lxc', $ctid);
+        $pve_a->ct_exec($ctid, "sh -c 'echo $marker > /root/marker'");
+    },
+    sub {
+        my $node = $resource_node->();
+        $pve_a->destroy_guest_on($node, 'lxc', $ctid) if defined($node);
+    },
+)->step(
+    'migrating the running container needs restart mode',
+    sub {
+        eval { $client->post("/nodes/$node_a/lxc/$ctid/migrate", { target => $node_b }) };
+        # The next step sends the same request for the stopped container, which only succeeds if
+        # this one failed because hte container is running.
+        ok($@, 'migrating the running container without restart mode fails');
+        is($resource_node->(), $node_a, "container $ctid stays on node $node_a");
+    },
+)->step(
+    'migrate stopped container',
+    sub {
+        $pve_a->stop_guest('lxc', $ctid);
+
+        my $upid = $client->post("/nodes/$node_a/lxc/$ctid/migrate", { target => $node_b });
+        $pve_a->task_ok($upid, "migrated container $ctid to node $node_b")
+            or die "migrating container $ctid failed\n";
+
+        $verify_location->($pve_b, $pve_a);
+
+        $pve_b->start_guest('lxc', $ctid);
+        is(
+            $pve_b->ct_exec($ctid, 'cat /root/marker'),
+            "$marker\n",
+            'file is intact after the migration',
+        );
+    },
+)->step(
+    'migrate running container in restart mode',
+    sub {
+        my $upid = $pve_b->client()->post(
+            "/nodes/$node_b/lxc/$ctid/migrate",
+            { target => $node_a, restart => 1, timeout => 60 },
+        );
+        $pve_b->task_ok($upid, "migrated container $ctid back to node $node_a in restart mode")
+            or die "migrating container $ctid back failed\n";
+
+        $verify_location->($pve_a, $pve_b);
+
+        poll_until(
+            sub {
+                $client->get("/nodes/$node_a/lxc/$ctid/status/current", {})->{status} eq
+                    'running';
+            },
+            timeout => 60,
+            description => "container $ctid to run on node $node_a",
+        );
+        pass("container $ctid runs again after the restart migration");
+        is(
+            $pve_a->ct_exec($ctid, 'cat /root/marker'),
+            "$marker\n",
+            'file is intact after the migration',
+        );
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index f664c62..9903d79 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -140,6 +140,17 @@ TestcaseConfig(
             ],
             tag: "extended",
         ),
+        Testcase(
+            name: "pve_ct_migration",
+            run: "./tests/pve_ct_migration.pl",
+            timeout: 600,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "extended",
+        ),
         Testcase(
             name: "pve_replication",
             run: "./tests/pve_replication.pl",
-- 
2.47.3