[PATCH e2e-tests 36/38] tests: migration: verify VM migration with local disks

["Michael Köppl" <m.koeppl@proxmox.com>]

Live migrate a VM with a disk on node-local ZFS storage to another node
onto local-lvm and back onto ZFS, then migrate the stopped VM to the
other node again. Check that the disk is moved to the target storage
each time, that no volume is left behind and that its content stays the
same. A live migration without allowing local disks must fail and leave
the VM running on its node.

The offline migration keeps the disk on ZFS if the disk cannot be
converted in a format both storages support. The disk content is
compared by checksum on the node.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/pve_vm_migration_local_disks.pl | 213 ++++++++++++++++++++++++++
 tests/testcases.ron                   |  11 ++
 2 files changed, 224 insertions(+)
 create mode 100755 tests/pve_vm_migration_local_disks.pl

diff --git a/tests/pve_vm_migration_local_disks.pl b/tests/pve_vm_migration_local_disks.pl
new file mode 100755
index 0000000..55d2e14
--- /dev/null
+++ b/tests/pve_vm_migration_local_disks.pl
@@ -0,0 +1,213 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<pve_vm_migration_local_disks> - Test migrating a VM with local disks between cluster nodes.
+
+=head1 DESCRIPTION
+
+This testcase creates a VM with a disk on the node-local ZFS storage of the first
+cluster node, writes random data to the disk and records its checksum. The
+migration preconditions must report the disk as local.
+
+It starts the VM, and a live migration without allowing the migration of local
+disks must fail, leaving the VM running on the first node. A live migration with
+local disks to the second node, onto its local-lvm storage, must move the VM and
+its disk, and so must a live migration back onto the ZFS storage. After stopping
+the VM, an offline migration to the second node must move it to the storage with
+the same storage ID. The disk must have the original content after each
+migration, and no volume may be left behind on the source node.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use lib '.';
+use Proxmox::Test::Helper qw(get_pve_instance);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_a = get_pve_instance('pve1-cluster');
+my $pve_b = get_pve_instance('pve2-cluster');
+my $node_a = $pve_a->get_nodename();
+my $node_b = $pve_b->get_nodename();
+my $client = $pve_a->client();
+
+my $zfs_storage = 'zfstest';
+
+my $vmid;
+my $checksum;
+
+# The node the VM lives on according to the cluster resources. undef if the node could not be
+# determined.
+my $resource_node = sub {
+    my $resources = $client->get('/cluster/resources', { type => 'vm' });
+    my ($entry) = grep { ($_->{vmid} // 0) == $vmid } $resources->@*;
+    return $entry ? $entry->{node} : undef;
+};
+
+my $disk_volid = sub {
+    my ($node) = @_;
+    my $conf = $client->get("/nodes/$node/qemu/$vmid/config", {});
+    my ($volid) = ($conf->{scsi0} // '') =~ m/^([^,]+)/;
+    return $volid;
+};
+
+my $volumes_on = sub {
+    my ($node, $storage) = @_;
+    my $content = $client->get("/nodes/$node/storage/$storage/content", { vmid => $vmid });
+    return { map { $_->{volid} => 1 } $content->@* };
+};
+
+my $disk_checksum = sub {
+    my ($instance) = @_;
+    my $volid = $disk_volid->($instance->get_nodename());
+    my ($path) = $instance->run_command("pvesm path $volid");
+    chomp($path);
+    my ($out) = $instance->run_command("sha256sum $path");
+    my ($sum) = split(/\s+/, $out // '');
+    return $sum;
+};
+
+my $migrate = sub {
+    my ($instance, $params) = @_;
+    my $node = $instance->get_nodename();
+    return $instance->client()->post("/nodes/$node/qemu/$vmid/migrate", $params);
+};
+
+my $verify_location = sub {
+    my ($instance, $storage, $source, $source_storage) = @_;
+    my $node = $instance->get_nodename();
+
+    poll_until(
+        sub { ($resource_node->() // '') eq $node },
+        timeout => 60,
+        description => "VM $vmid to show up on node $node",
+    );
+    pass("VM $vmid is on node $node");
+
+    my $volid = $disk_volid->($node);
+    like($volid // '', qr/^\Q$storage\E:/, "disk is on $storage");
+    ok($volid && $volumes_on->($node, $storage)->{$volid}, "disk volume exists on node $node");
+    is_deeply(
+        $volumes_on->($source->get_nodename(), $source_storage),
+        {},
+        'no volume is left on the source node',
+    );
+    is($disk_checksum->($instance), $checksum, 'disk has the original content');
+};
+
+Proxmox::Test::Testcase->new('pve_vm_migration_local_disks')->with([{}])->step(
+    'create VM with a local disk',
+    sub {
+        $vmid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node_a/qemu",
+            {
+                vmid => $vmid,
+                name => 'vm-migration-local-disks',
+                memory => 256,
+                scsihw => 'virtio-scsi-pci',
+                scsi0 => "$zfs_storage:1",
+            },
+        );
+        $pve_a->task_ok($upid, "created VM $vmid on node $node_a")
+            or die "creating VM $vmid failed\n";
+    },
+    sub {
+        my $node = $resource_node->();
+        $pve_a->destroy_guest_on($node, 'qemu', $vmid) if defined($node);
+    },
+)->step(
+    'write data to the disk',
+    sub {
+        my ($path) = $pve_a->run_command('pvesm path ' . $disk_volid->($node_a));
+        chomp($path);
+        $pve_a->run_command("dd if=/dev/urandom of=$path bs=1M count=8 seek=16 conv=notrunc");
+
+        $checksum = $disk_checksum->($pve_a);
+        ok($checksum, 'recorded the checksum of the disk');
+    },
+)->step(
+    'check migration preconditions',
+    sub {
+        my $precondition = $client->get("/nodes/$node_a/qemu/$vmid/migrate", {});
+        my $volid = $disk_volid->($node_a);
+        ok(
+            (grep { $_->{volid} eq $volid } ($precondition->{local_disks} // [])->@*),
+            'disk reported as local',
+        );
+    },
+)->step(
+    'live migration needs to allow local disks',
+    sub {
+        $pve_a->start_guest('qemu', $vmid);
+
+        my $upid = $migrate->($pve_a, { target => $node_b, online => 1 });
+        ok(
+            !$pve_a->task_successful($pve_a->wait_for_task($upid, 1)),
+            'live migration without local disks allowed fails',
+        );
+        is($resource_node->(), $node_a, "VM $vmid stays on node $node_a");
+        is(
+            $client->get("/nodes/$node_a/qemu/$vmid/status/current", {})->{status},
+            'running',
+            "VM $vmid keeps running",
+        );
+    },
+)->step(
+    'live migrate VM with local disk to another storage',
+    sub {
+        my $upid = $migrate->(
+            $pve_a,
+            {
+                target => $node_b,
+                online => 1,
+                'with-local-disks' => 1,
+                targetstorage => 'local-lvm',
+            },
+        );
+        $pve_a->task_ok($upid, "live migrated VM $vmid to node $node_b")
+            or die "live migrating VM $vmid failed\n";
+
+        is(
+            $client->get("/nodes/$node_b/qemu/$vmid/status/current", {})->{status},
+            'running',
+            "VM $vmid keeps running during the live migration",
+        );
+        $verify_location->($pve_b, 'local-lvm', $pve_a, $zfs_storage);
+    },
+)->step(
+    'live migrate VM with local disk back to the ZFS storage',
+    sub {
+        my $upid = $migrate->(
+            $pve_b,
+            {
+                target => $node_a,
+                online => 1,
+                'with-local-disks' => 1,
+                targetstorage => $zfs_storage,
+            },
+        );
+        $pve_b->task_ok($upid, "live migrated VM $vmid back to node $node_a")
+            or die "live migrating VM $vmid back failed\n";
+
+        $verify_location->($pve_a, $zfs_storage, $pve_b, 'local-lvm');
+    },
+)->step(
+    'migrate stopped VM with local disk',
+    sub {
+        $pve_a->stop_guest('qemu', $vmid);
+
+        my $upid = $migrate->($pve_a, { target => $node_b });
+        $pve_a->task_ok($upid, "migrated VM $vmid to node $node_b")
+            or die "migrating VM $vmid failed\n";
+
+        $verify_location->($pve_b, $zfs_storage, $pve_a, $zfs_storage);
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index 9903d79..e0d622c 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -151,6 +151,17 @@ TestcaseConfig(
             ],
             tag: "extended",
         ),
+        Testcase(
+            name: "pve_vm_migration_local_disks",
+            run: "./tests/pve_vm_migration_local_disks.pl",
+            timeout: 600,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "extended",
+        ),
         Testcase(
             name: "pve_replication",
             run: "./tests/pve_replication.pl",
-- 
2.47.3