[PATCH e2e-tests 23/38] tests: vm: verify lifecycle operations and cloud-init guest setup

["Michael Köppl" <m.koeppl@proxmox.com>]

Create a VM from the Debian cloud image with a user, an SSH key, a
static address and DNS settings for cloud-init. Check the user,
network and meta data that Proxmox VE generates from them, then boot
the VM and check that the node can reach it at the address, log in
as the user with the key, and that the guest uses the VM name as
hostname. Package upgrades on first boot are disabled, as the guest
cannot reach any mirror.

While the VM runs, change a cloud-init option and check that it is
pending until the cloud-init drive is regenerated, and that it
changes the instance ID.

Then pause and resume the VM, reset it, reboot it to apply a pending
memory change. Then hibernate and start again. The guest's boot ID shows
whether the guest kept running or booted again. Finally, shut the VM
down through ACPI and check that destroying it removes all of its
volumes, including the cloud-init drive. A second VM without an OS
ignores ACPI requests, so the shutdown succeeds when forced.

Checks inside the guest are limited to what follows directly from the
config.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/pve_vm_lifecycle.pl | 442 ++++++++++++++++++++++++++++++++++++++
 tests/testcases.ron       |   9 +
 2 files changed, 451 insertions(+)
 create mode 100755 tests/pve_vm_lifecycle.pl

diff --git a/tests/pve_vm_lifecycle.pl b/tests/pve_vm_lifecycle.pl
new file mode 100755
index 0000000..2168258
--- /dev/null
+++ b/tests/pve_vm_lifecycle.pl
@@ -0,0 +1,442 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<pve_vm_lifecycle> - Test the lifecycle operations and cloud-init guest setup of a VM.
+
+=head1 DESCRIPTION
+
+This testcase creates a VM with a disk imported from the Debian cloud image that
+the setup plan downloads onto pve1's local storage, and a cloud-init drive. It
+configures a user with an SSH key, a static address and DNS settings through
+cloud-init and verifies the cloud-init configuration that Proxmox VE generates
+from them.
+
+It then starts the VM and verifies that the node can reach it at the configured
+address and log in as the configured user with the configured key, and that the
+guest uses the VM's name as hostname. While the VM is running, it changes a
+cloud-init option, which must be reported as pending until the cloud-init drive
+is regenerated.
+
+Afterwards, it pauses and resumes the VM, resets it, reboots it to apply a
+pending memory change and hibernates it to disk and starts it again. The guest's
+boot ID shows whether the guest kept running or was restarted by each of these.
+Finally, it shuts the VM down through ACPI and destroys it, which must remove all
+of its volumes. A second VM without an OS, which ignores ACPI, must only stop
+when the shutdown is forced.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use URI::Escape qw(uri_escape);
+
+use lib '.';
+use Proxmox::Test::Helper qw(get_pve_instance);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve = get_pve_instance('pve1');
+my $node = $pve->get_nodename();
+my $client = $pve->client();
+
+my $vm_ip = '10.250.253.10';
+my $node_ip = '10.250.253.1';
+my $nameserver = '10.250.253.53';
+my $searchdomain = 'e2e.test';
+my $changed_searchdomain = 'changed.e2e.test';
+my $user = 'e2e';
+
+my $vmid = $client->get('/cluster/nextid');
+my $name = 'vm-lifecycle';
+my $no_os_vmid;
+my $key_dir;
+my $public_key;
+my $instance_id;
+
+my $vm_config = sub {
+    my ($id) = @_;
+    return $client->get("/nodes/$node/qemu/$id/config", { current => 1 });
+};
+
+my $vm_status = sub {
+    my ($id) = @_;
+    return $client->get("/nodes/$node/qemu/$id/status/current", {});
+};
+
+my $volumes = sub {
+    my $content = $client->get("/nodes/$node/storage/local-lvm/content", { vmid => $vmid });
+    return { map { $_->{volid} => 1 } $content->@* };
+};
+
+my $volume_exists = sub {
+    my ($volid) = @_;
+
+    my ($storage) = $volid =~ m/^([^:]+):/;
+    my $content = $client->get("/nodes/$node/storage/$storage/content", { vmid => $vmid });
+    return grep { $_->{volid} eq $volid } $content->@*;
+};
+
+my $pending_value = sub {
+    my ($key) = @_;
+    my ($entry) =
+        grep { $_->{key} eq $key } $client->get("/nodes/$node/qemu/$vmid/pending", {})->@*;
+    return $entry ? $entry->{pending} : undef;
+};
+
+my $cloudinit_dump = sub {
+    my ($type) = @_;
+    return $client->get("/nodes/$node/qemu/$vmid/cloudinit/dump", { type => $type });
+};
+
+my $cloudinit_option = sub {
+    my ($key) = @_;
+    my ($entry) =
+        grep { $_->{key} eq $key } $client->get("/nodes/$node/qemu/$vmid/cloudinit", {})->@*;
+    return $entry // {};
+};
+
+my $guest_exec = sub {
+    my ($cmd) = @_;
+    my ($out) =
+        $pve->run_command("ssh -i $key_dir/id_ed25519 -o BatchMode=yes -o ConnectTimeout=5"
+            . " -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
+            . " $user\@$vm_ip $cmd");
+    return $out // '';
+};
+
+my $wait_for_ssh = sub {
+    poll_until(
+        sub {
+            eval { $guest_exec->('true'); 1 }
+        },
+        timeout => 300,
+        interval => 5,
+        description => "SSH login to VM $vmid as '$user'",
+    );
+};
+
+# Changes whenever the guest boots, but not while paused or hibernated
+my $boot_id = sub {
+    my $id = $guest_exec->('cat /proc/sys/kernel/random/boot_id');
+    chomp($id);
+    return $id;
+};
+
+Proxmox::Test::Testcase->new('pve_vm_lifecycle')->with([{}])->step(
+    'create SSH key pair on the node',
+    sub {
+        ($key_dir) = $pve->run_command('mktemp -d');
+        chomp($key_dir);
+        $pve->run_command("ssh-keygen -q -t ed25519 -N '' -f $key_dir/id_ed25519");
+
+        ($public_key) = $pve->run_command("cat $key_dir/id_ed25519.pub");
+        chomp($public_key);
+    },
+    sub {
+        $pve->run_command("rm -rf $key_dir") if defined($key_dir);
+    },
+)->step(
+    'create VM from cloud image',
+    sub {
+        my $upid = $client->post(
+            "/nodes/$node/qemu",
+            {
+                vmid => $vmid,
+                name => $name,
+                ostype => 'l26',
+                memory => 1024,
+                scsihw => 'virtio-scsi-pci',
+                scsi0 =>
+                    'local-lvm:0,import-from=local:import/debian-13-genericcloud-amd64.qcow2',
+                ide2 => 'local-lvm:cloudinit',
+                boot => 'order=scsi0',
+                net0 => 'virtio,bridge=vmbr0',
+                ciuser => $user,
+                sshkeys => uri_escape($public_key),
+                ipconfig0 => "ip=$vm_ip/24,gw=$node_ip",
+                nameserver => $nameserver,
+                searchdomain => $searchdomain,
+                # The guest cannot reach any package mirror.
+                ciupgrade => 0,
+            },
+        );
+        $pve->task_ok($upid, "created VM $vmid from the cloud image")
+            or die "creating VM $vmid failed\n";
+    },
+    sub {
+        $pve->destroy_guest('qemu', $vmid);
+    },
+)->step(
+    'verify VM volumes',
+    sub {
+        my $conf = $vm_config->($vmid);
+
+        my ($disk) = ($conf->{scsi0} // '') =~ m/^(local-lvm:[^,]+)/;
+        ok(defined($disk) && $volumes->()->{$disk},
+            'disk is imported to a volume on local-lvm');
+
+        my ($cidrive) = ($conf->{ide2} // '') =~ m/^(local-lvm:[^,]+),media=cdrom\b/;
+        ok(
+            defined($cidrive) && $volumes->()->{$cidrive},
+            'cloud-init drive is a volume on local-lvm',
+        );
+    },
+)->step(
+    'verify generated cloud-init config',
+    sub {
+        my $user_data = $cloudinit_dump->('user');
+        like($user_data, qr/^hostname: \Q$name\E$/m, 'user data sets the VM name as hostname');
+        like($user_data, qr/^fqdn: \Q$name.$searchdomain\E$/m, 'user data sets the FQDN');
+        like($user_data, qr/^user: \Q$user\E$/m, 'user data sets the user');
+        like($user_data, qr/^\s*- \Q$public_key\E$/m, 'user data authorizes the SSH key');
+
+        my ($mac) = ($vm_config->($vmid)->{net0} // '') =~ m/^[^=,]+=([0-9A-Fa-f:]{17})\b/
+            or die "net0 has no MAC address\n";
+        my $network_data = $cloudinit_dump->('network');
+        like(
+            $network_data,
+            qr/\bmac_address: '\Q$mac\E'/i,
+            'network data configures the interface with the MAC address of net0',
+        );
+        like($network_data, qr/\baddress: '\Q$vm_ip\E'/, 'network data sets the address');
+        like($network_data, qr/\bnetmask: '255\.255\.255\.0'/, 'network data sets the netmask');
+        like($network_data, qr/\bgateway: '\Q$node_ip\E'/, 'network data sets the gateway');
+        like($network_data, qr/- '\Q$nameserver\E'/, 'network data sets the nameserver');
+        like($network_data, qr/- '\Q$searchdomain\E'/, 'network data sets the search domain');
+
+        ($instance_id) = $cloudinit_dump->('meta') =~ m/^instance-id: (\S+)$/m;
+        ok(defined($instance_id), 'meta data sets an instance ID');
+    },
+)->step(
+    'add node address on vmbr0',
+    sub {
+        $pve->run_command("ip addr add $node_ip/24 dev vmbr0");
+    },
+    sub {
+        $pve->run_command("ip addr del $node_ip/24 dev vmbr0 2>/dev/null; true");
+    },
+)->step(
+    'start VM',
+    sub {
+        $pve->start_guest('qemu', $vmid);
+    },
+)->step(
+    'verify guest setup',
+    sub {
+        poll_until(
+            sub { $pve->node_can_ping($vm_ip) },
+            timeout => 300,
+            interval => 5,
+            description => "VM $vmid to answer on $vm_ip",
+        );
+        pass("node can ping VM $vmid at the configured address");
+
+        $wait_for_ssh->();
+        is($guest_exec->('id -un'), "$user\n", 'node logs in as the user with the SSH key');
+        is($guest_exec->('hostname'), "$name\n", 'guest uses the VM name as hostname');
+        like(
+            $guest_exec->('ip -4 -o addr show'),
+            qr/\binet \Q$vm_ip\E\/24\b/,
+            'guest has the configured address and prefix length',
+        );
+    },
+)->step(
+    'change cloud-init option of the running VM',
+    sub {
+        my $upid = $client->post(
+            "/nodes/$node/qemu/$vmid/config",
+            { searchdomain => $changed_searchdomain },
+        );
+        $pve->task_ok($upid, 'changed the search domain') or die "changing the config failed\n";
+
+        my $option = $cloudinit_option->('searchdomain');
+        is($option->{value}, $searchdomain, 'cloud-init drive still has the old search domain');
+        is($option->{pending}, $changed_searchdomain, 'new search domain is pending');
+
+        my ($new_instance_id) = $cloudinit_dump->('meta') =~ m/^instance-id: (\S+)$/m;
+        isnt($new_instance_id, $instance_id, 'changed config gets a new instance ID');
+    },
+)->step(
+    'regenerate cloud-init drive',
+    sub {
+        $client->put("/nodes/$node/qemu/$vmid/cloudinit", {});
+
+        my $option = $cloudinit_option->('searchdomain');
+        is(
+            $option->{value},
+            $changed_searchdomain,
+            'cloud-init drive has the new search domain',
+        );
+        ok(!defined($option->{pending}), 'no change of the search domain is pending');
+    },
+)->step(
+    'pause and resume VM',
+    sub {
+        my $old_boot_id = $boot_id->();
+
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/status/suspend", {});
+        $pve->task_ok($upid, "paused VM $vmid") or die "pausing VM $vmid failed\n";
+
+        my $status = $vm_status->($vmid);
+        is($status->{status}, 'running', 'paused VM still reports status running');
+        is($status->{qmpstatus}, 'paused', 'QEMU reports the VM as paused');
+        ok(!$pve->node_can_ping($vm_ip), 'paused VM does not answer ping');
+
+        eval { $client->post("/nodes/$node/qemu/$vmid/status/reboot", {}) };
+        ok($@, 'rebooting the paused VM fails');
+        is($vm_status->($vmid)->{qmpstatus}, 'paused', 'VM is still paused');
+
+        $upid = $client->post("/nodes/$node/qemu/$vmid/status/resume", {});
+        $pve->task_ok($upid, "resumed VM $vmid") or die "resuming VM $vmid failed\n";
+
+        is($vm_status->($vmid)->{qmpstatus}, 'running', 'QEMU reports the VM as running');
+        $wait_for_ssh->();
+        is($boot_id->(), $old_boot_id, 'guest continues without booting after the resume');
+    },
+)->step(
+    'reset VM',
+    sub {
+        my $old_pid = $vm_status->($vmid)->{pid};
+        my $old_boot_id = $boot_id->();
+
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/status/reset", {});
+        $pve->task_ok($upid, "reset VM $vmid") or die "resetting VM $vmid failed\n";
+
+        my $status = $vm_status->($vmid);
+        is($status->{status}, 'running', 'VM keeps running during the reset');
+        is($status->{pid}, $old_pid, 'reset keeps the QEMU process');
+
+        $wait_for_ssh->();
+        isnt($boot_id->(), $old_boot_id, 'guest boots again after the reset');
+    },
+)->step(
+    'reboot VM to apply pending change',
+    sub {
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/config", { memory => 1536 });
+        $pve->task_ok($upid, 'changed the memory') or die "changing the memory failed\n";
+
+        is($pending_value->('memory'), 1536, 'memory change of the running VM is pending');
+
+        my $old_pid = $vm_status->($vmid)->{pid};
+        my $old_boot_id = $boot_id->();
+
+        $upid = $client->post("/nodes/$node/qemu/$vmid/status/reboot", { timeout => 120 });
+        $pve->task_ok($upid, "rebooted VM $vmid") or die "rebooting VM $vmid failed\n";
+
+        # The reboot task only shuts the VM down, a separate start brings it back up.
+        poll_until(
+            sub {
+                my $status = $vm_status->($vmid);
+                return $status->{status} eq 'running' && $status->{pid} != $old_pid;
+            },
+            timeout => 120,
+            interval => 2,
+            description => "VM $vmid to run in a new QEMU process",
+        );
+        pass('reboot starts the VM in a new QEMU process');
+
+        is($vm_config->($vmid)->{memory}, 1536, 'reboot applied the pending memory change');
+        ok(!defined($pending_value->('memory')), 'memory change is no longer pending');
+
+        $wait_for_ssh->();
+        isnt($boot_id->(), $old_boot_id, 'guest boots again after the reboot');
+
+        # The guest only sees part of the memory, but more than before the change.
+        my ($mem_kib) = $guest_exec->('cat /proc/meminfo') =~ m/^MemTotal:\s+(\d+) kB$/m;
+        cmp_ok($mem_kib // 0, '>', 1024 * 1024, 'guest has more than 1 GiB of memory');
+    },
+)->step(
+    'hibernate VM',
+    sub {
+        my $old_boot_id = $boot_id->();
+        my ($since) = $pve->run_command('date +%s');
+        chomp($since);
+
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/status/suspend", { todisk => 1 });
+        $pve->task_ok($upid, "hibernated VM $vmid") or die "hibernating VM $vmid failed\n";
+
+        is($vm_status->($vmid)->{status}, 'stopped', 'hibernated VM reports status stopped');
+
+        my $conf = $vm_config->($vmid);
+        is($conf->{lock}, 'suspended', 'hibernated VM is locked');
+        my $vmstate = $conf->{vmstate};
+        ok(defined($vmstate) && $volume_exists->($vmstate), 'RAM is saved to a volume');
+
+        eval { $client->put("/nodes/$node/qemu/$vmid/config", { name => "$name-changed" }) };
+        ok($@, 'changing the config of the hibernated VM fails');
+        is($vm_config->($vmid)->{name}, $name, 'name of the hibernated VM is unchanged');
+
+        # Nothing waits for the cleanup operation of qmeventd after hibernation. If the VM is
+        # started before the cleanup takes the config lock, operations after that would fail.
+        $pve->wait_for_qemu_cleanup($vmid, $since);
+
+        $pve->start_guest('qemu', $vmid);
+
+        $wait_for_ssh->();
+        is($boot_id->(), $old_boot_id, 'guest continues without booting after the start');
+
+        $conf = $vm_config->($vmid);
+        ok(!defined($conf->{lock}), 'VM is no longer locked');
+        ok(!defined($conf->{vmstate}), 'config no longer references the saved RAM');
+        ok(!defined($vmstate) || !$volume_exists->($vmstate), 'saved RAM is removed');
+    },
+)->step(
+    'shutdown VM',
+    sub {
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/status/shutdown", { timeout => 120 });
+        $pve->task_ok($upid, "shut down VM $vmid") or die "shutting down VM $vmid failed\n";
+
+        is($vm_status->($vmid)->{status}, 'stopped', "VM $vmid reports status 'stopped'");
+    },
+)->step(
+    'destroy VM',
+    sub {
+        my $upid = $client->delete("/nodes/$node/qemu/$vmid", {});
+        $pve->task_ok($upid, "destroyed VM $vmid") or die "destroying VM $vmid failed\n";
+
+        ok(!$pve->guest_exists('qemu', $vmid), "VM $vmid is gone");
+        is_deeply($volumes->(), {}, "no volume of VM $vmid is left on local-lvm");
+    },
+)->step(
+    'create and start VM without OS',
+    sub {
+        $no_os_vmid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node/qemu",
+            { vmid => $no_os_vmid, name => 'vm-lifecycle-no-os', memory => 128 },
+        );
+        $pve->task_ok($upid, "created VM $no_os_vmid")
+            or die "creating VM $no_os_vmid failed\n";
+
+        $pve->start_guest('qemu', $no_os_vmid);
+    },
+    sub {
+        $pve->destroy_guest('qemu', $no_os_vmid) if defined($no_os_vmid);
+    },
+)->step(
+    'shutdown VM without OS',
+    sub {
+        my $upid = $client->post(
+            "/nodes/$node/qemu/$no_os_vmid/status/shutdown", { timeout => 10 },
+        );
+        ok(
+            !$pve->task_successful($pve->wait_for_task($upid, 1)),
+            'shutdown fails as nothing handles the ACPI request',
+        );
+        is($vm_status->($no_os_vmid)->{status}, 'running', 'VM without OS keeps running');
+
+        $upid = $client->post(
+            "/nodes/$node/qemu/$no_os_vmid/status/shutdown",
+            { timeout => 10, forceStop => 1 },
+        );
+        $pve->task_ok($upid, "forced shutdown of VM $no_os_vmid")
+            or die "forcing the shutdown of VM $no_os_vmid failed\n";
+        is($vm_status->($no_os_vmid)->{status}, 'stopped', 'forced shutdown stops the VM');
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index b1a35b6..fba3e06 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -29,6 +29,15 @@ TestcaseConfig(
             ],
             tag: "basic",
         ),
+        Testcase(
+            name: "pve_vm_lifecycle",
+            run: "./tests/pve_vm_lifecycle.pl",
+            timeout: 600,
+            instances: [
+                "pve1",
+            ],
+            tag: "basic",
+        ),
         Testcase(
             name: "pve_ldap_realm",
             run: "./tests/ldap_realm.pl",
-- 
2.47.3