[PATCH e2e-tests 27/38] tests: ceph: verify guests on RBD storage

["Michael Köppl" <m.koeppl@proxmox.com>]

Containers always access their volumes through krbd, so the container
part runs once, on a storage without krbd, and checks that the volume
is mapped. Other tests already cover operations on local-lvm, so this
test checks what the RBD storage does for them. A backup in snapshot
mode takes an RBD snapshot and removes it and its mapping afterwards.
Migrating the container in restart mode leaves its volume on the storage
and moves the mapping to the target node, as the file system on it must
only ever be mounted on one node. Converting the container to a template
gives it a base image with a protected snapshot, from which the image of
a linked clone is cloned. The template cannot be destroyed while the
linked clone exists.

For VMs, the krbd option of the storage decides whether QEMU accesses
the disk through librbd or through a mapped device. With librbd, QEMU
takes snapshots and resizes the disk of a running VM itself. So with
and without krbd, check that the disk is only mapped with krbd,
snapshot a running VM and roll the snapshot back, delete it while the
VM runs, and resize the disk of the running VM. Also live-migrate the
VM to another node, which must not copy its disk and, with krbd, must
move the mapping of its image to the target node.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/ceph/test_ceph_rbd_storage.pl | 597 ++++++++++++++++++++++++++++
 tests/testcases.ron                 |  11 +
 2 files changed, 608 insertions(+)
 create mode 100755 tests/ceph/test_ceph_rbd_storage.pl

diff --git a/tests/ceph/test_ceph_rbd_storage.pl b/tests/ceph/test_ceph_rbd_storage.pl
new file mode 100755
index 0000000..9bdde6a
--- /dev/null
+++ b/tests/ceph/test_ceph_rbd_storage.pl
@@ -0,0 +1,597 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<test_ceph_rbd_storage> - Test guests on an RBD storage of a hyper-converged Ceph cluster.
+
+=head1 DESCRIPTION
+
+This testcase sets up Ceph on the cluster nodes and creates a pool. It adds RBD
+storages for the pool and verifies that they are active on every node.
+
+Containers always access their volumes through krbd, independent of the krbd
+option of the storage, so the container part runs once, on a storage without
+krbd. It verifies that the volume of a running container is mapped through krbd
+anyway. Other tests cover the generic container operations on local-lvm, so on
+that storage, it verifies what the RBD storage does for them:
+
+ - Backs the running container up in snapshot mode, which must take an RBD
+   snapshot of the volume and remove it and its mapping afterwards.
+ - Migrates the running container to another node and back in restart mode.
+   The volume must stay on the shared storage and only be mapped on the node
+   the container runs on.
+ - Converts the container to a template, which must rename its image to a base
+   image with a protected snapshot. Creates a linked clone, whose image must be
+   cloned from that snapshot, and verifies that the template cannot be
+   destroyed while the linked clone exists.
+
+For VMs, the krbd option decides whether QEMU accesses the disk through librbd
+or through a device mapped with krbd, which also changes how snapshots and
+resizing of a running VM are done. So the VM part runs once without and once
+with krbd. It writes data to the disk of a VM, starts it and verifies that the
+disk is only mapped with krbd. It then performs the following:
+
+ - Takes a snapshot of the running VM, which QEMU takes itself with librbd,
+   and verifies that rolling it back restores the data. Deletes the snapshot
+   while the VM is running again.
+ - Resizes the disk of the running VM.
+ - Live-migrates the VM to another node, which must not copy the disk since the
+   storage is shared. With krbd, the image must also move from the source
+   node's mapped devices to the target node's.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use JSON;
+
+use lib '.';
+use Proxmox::Test::Ceph qw(create_ceph_cluster create_pool check_storage_active);
+use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pve_instances);
+use Proxmox::Test::Testcase;
+use Proxmox::Test::Util qw(poll_until);
+
+my $pve_instances = get_pve_instances();
+die "not enough PVE instances" if scalar($pve_instances->@*) < 3;
+
+my $first = $pve_instances->[0];
+my $second = $pve_instances->[1];
+my $node = $first->get_nodename();
+my $target = $second->get_nodename();
+my $client = $first->client();
+my $pool = 'e2e-rbd';
+my $snapname = 'e2esnap';
+my $marker = 'e2e-rbd-container';
+
+my $wait_for_status = sub {
+    my ($instance, $type, $vmid, $status) = @_;
+    my $nodename = $instance->get_nodename();
+
+    poll_until(
+        sub {
+            $instance->client()->get("/nodes/$nodename/$type/$vmid/status/current", {})
+                ->{status} eq $status;
+        },
+        timeout => 120,
+        description => "$type $vmid on '$nodename' to report status '$status'",
+    );
+};
+
+# The RBD images mapped through krbd on $instance, by image name, and the mapped snapshots, by
+# '<image>@<snapshot>'.
+my $mapped_images = sub {
+    my ($instance) = @_;
+    my ($out) = $instance->run_command('rbd device list --format json');
+
+    my $mapped = {};
+    for my $device (decode_json($out)->@*) {
+        my $snap = $device->{snap} // '-';
+        $mapped->{ $snap eq '-' ? $device->{name} : "$device->{name}\@$snap" } = 1;
+    }
+    return $mapped;
+};
+
+# The snapshots of the RBD image $image, by name.
+my $rbd_snapshots = sub {
+    my ($image) = @_;
+    my ($out) = $first->run_command("rbd snap ls --format json $pool/$image");
+    return { map { $_->{name} => $_ } decode_json($out)->@* };
+};
+
+# The information 'rbd info' shows about the RBD image $image, or about a snapshot with
+# '$image@<snapshot>'.
+my $rbd_info = sub {
+    my ($image) = @_;
+    my ($out) = $first->run_command("rbd info --format json $pool/$image");
+    return decode_json($out);
+};
+
+# A checksum of the data of the RBD image $image.
+my $image_checksum = sub {
+    my ($image) = @_;
+    my ($out) =
+        $first->run_command("set -o pipefail; rbd export --no-progress $pool/$image - | sha256sum");
+    my ($sum) = split(/\s+/, $out // '');
+    return $sum;
+};
+
+# Overwrite part of the RBD image $image, which must not be in use, with random data.
+my $write_random_data = sub {
+    my ($image) = @_;
+
+    my ($dev) = $first->run_command("rbd device map $pool/$image");
+    chomp($dev);
+    eval {
+        $first->run_command(
+            "dd if=/dev/urandom of=$dev bs=1M count=8 seek=16 oflag=direct status=none");
+    };
+    my $err = $@;
+    $first->run_command("rbd device unmap $dev");
+    die $err if $err;
+};
+
+my $add_storage = sub {
+    my ($testcase, $content) = @_;
+
+    $client->post(
+        '/storage',
+        {
+            storage => $testcase->{storage},
+            type => 'rbd',
+            pool => $pool,
+            krbd => $testcase->{krbd},
+            content => $content,
+        },
+    );
+
+    check_storage_active($_, $testcase->{storage}) for $pve_instances->@*;
+};
+
+create_ceph_cluster($pve_instances);
+create_pool($first, $pool);
+
+Proxmox::Test::Testcase->new('ceph_rbd_containers')->with([
+    {
+        storage => 'e2e-rbd-ct',
+        krbd => 0,
+    },
+])->step(
+        'add RBD storage without krbd',
+        sub {
+            my ($testcase) = @_;
+            $add_storage->($testcase, 'rootdir');
+        },
+        sub {
+            my ($testcase) = @_;
+            $client->delete("/storage/$testcase->{storage}");
+        },
+)->step(
+    'create and start container',
+    sub {
+        my ($testcase) = @_;
+
+        $testcase->{ctid} = $client->get('/cluster/nextid');
+        $testcase->{ct_node} = $node;
+        my $upid = $client->post(
+            "/nodes/$node/lxc",
+            {
+                vmid => $testcase->{ctid},
+                ostemplate => CONTAINER_TEMPLATE,
+                rootfs => "$testcase->{storage}:2",
+                memory => 256,
+                unprivileged => 1,
+                features => 'nesting=1',
+                hostname => "e2e-ct-$testcase->{storage}",
+            },
+        );
+        $first->task_ok($upid, "created container $testcase->{ctid} on '$testcase->{storage}'")
+            or die "creating the container failed\n";
+
+        $first->start_guest('lxc', $testcase->{ctid});
+
+        my $conf = $client->get("/nodes/$node/lxc/$testcase->{ctid}/config", {});
+        my ($image) = ($conf->{rootfs} // '') =~ m/^[^:]+:([^,]+)/;
+        $testcase->{image} = $image;
+        ok(
+            $image && $mapped_images->($first)->{$image},
+            'container volume is mapped through krbd, although the storage does not enable it',
+        );
+
+        $first->ct_exec($testcase->{ctid}, "sh -c 'echo $marker > /root/marker'");
+    },
+    sub {
+        my ($testcase) = @_;
+        return if !defined($testcase->{ctid});
+        $first->destroy_guest_on($testcase->{ct_node}, 'lxc', $testcase->{ctid});
+    },
+)->step(
+    'back up running container in snapshot mode',
+    sub {
+        my ($testcase) = @_;
+        my $ctid = $testcase->{ctid};
+
+        my $upid = $client->post(
+            "/nodes/$node/vzdump",
+            { vmid => $ctid, storage => 'local', mode => 'snapshot' },
+        );
+        $first->task_ok($upid, 'backed up the running container') or die "backup failed\n";
+
+        my $content =
+            $client->get("/nodes/$node/storage/local/content", { content => 'backup' });
+        ($testcase->{backup}) = map { $_->{volid} } grep { $_->{vmid} == $ctid } $content->@*;
+        ok($testcase->{backup}, 'backup archive exists on local storage');
+
+        # The backup reads the container's data from an RBD snapshot 'vzdump' of its volume, which
+        # it maps through krbd.
+        my $log = $client->get("/nodes/$node/tasks/$upid/log", { limit => 1000 });
+        ok(
+            (grep { ($_->{t} // '') =~ m/create storage snapshot 'vzdump'/ } $log->@*),
+            'backup used a storage snapshot of the container volume',
+        );
+
+        my $image = $testcase->{image};
+        ok(
+            !$rbd_snapshots->($image)->{vzdump},
+            'backup snapshot is removed from the RBD image',
+        );
+        ok(
+            !$mapped_images->($first)->{"$image\@vzdump"},
+            'backup snapshot is no longer mapped',
+        );
+    },
+    sub {
+        my ($testcase) = @_;
+        $first->destroy_volume('local', $testcase->{backup}) if $testcase->{backup};
+    },
+)->step(
+    'migrate running container in restart mode',
+    sub {
+        my ($testcase) = @_;
+        my $ctid = $testcase->{ctid};
+        my $image = $testcase->{image};
+        my $volid = "$testcase->{storage}:$image";
+
+        # Migrate the container to another node (and back, so that the remaining steps find it on
+        # the first node again).
+        for my $instances ([$first, $second], [$second, $first]) {
+            my ($source, $dest) = $instances->@*;
+            my $from = $source->get_nodename();
+            my $to = $dest->get_nodename();
+
+            my $upid = $client->post(
+                "/nodes/$from/lxc/$ctid/migrate",
+                { target => $to, restart => 1, timeout => 60 },
+            );
+            $source->task_ok($upid, "migrated container $ctid to '$to' in restart mode")
+                or die "migrating the container failed\n";
+            $testcase->{ct_node} = $to;
+
+            $wait_for_status->($dest, 'lxc', $ctid, 'running');
+
+            my $log = $client->get("/nodes/$from/tasks/$upid/log", { limit => 1000 });
+            my $kept =
+                grep { ($_->{t} // '') =~ m/'\Q$volid\E' is on shared storage/ } $log->@*;
+            ok($kept, "migration to '$to' leaves the volume on the shared storage");
+            like(
+                $client->get("/nodes/$to/lxc/$ctid/config", {})->{rootfs} // '',
+                qr/^\Q$volid\E,/,
+                "container uses the same volume on '$to'",
+            );
+
+            # The file system on the volume must only ever be mounted on one node.
+            ok($mapped_images->($dest)->{$image}, "container volume is mapped on '$to'");
+            ok(
+                !$mapped_images->($source)->{$image},
+                "container volume is no longer mapped on '$from'",
+            );
+
+            is(
+                $dest->ct_exec($ctid, 'cat /root/marker'),
+                "$marker\n",
+                "container data is intact on '$to'",
+            );
+        }
+    },
+)->step(
+    'convert container to template',
+    sub {
+        my ($testcase) = @_;
+        my $ctid = $testcase->{ctid};
+
+        # Only a stopped container can become a template.
+        $first->stop_guest('lxc', $ctid);
+
+        # The conversion runs in a worker task, but the API call does not return its task ID.
+        $client->post("/nodes/$node/lxc/$ctid/template", {});
+
+        my $conf;
+        eval {
+            poll_until(
+                sub {
+                    $conf = $client->get("/nodes/$node/lxc/$ctid/config", {});
+                    return $conf->{template};
+                },
+                timeout => 60,
+                interval => 2,
+                description => "container $ctid to be converted to a template",
+            );
+        };
+        is($conf->{template}, 1, "container $ctid is marked as template")
+            or die "converting the container to a template failed\n";
+
+        # Linked clones of the template are cloned from the protected snapshot of the base image.
+        my $base = $testcase->{image} =~ s/^vm-/base-/r;
+        my ($volume) = ($conf->{rootfs} // '') =~ m/^[^:]+:([^,]+)/;
+        is($volume, $base, "container volume is renamed to the base image '$base'");
+        $testcase->{base} = $base;
+
+        my $info = eval { $rbd_info->("$base\@__base__") } // {};
+        is($info->{protected}, 'true', "base image has the protected snapshot '__base__'");
+    },
+)->step(
+    'create linked clone',
+    sub {
+        my ($testcase) = @_;
+
+        $testcase->{clone} = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node/lxc/$testcase->{ctid}/clone",
+            {
+                newid => $testcase->{clone},
+                hostname => "e2e-clone-$testcase->{storage}",
+                full => 0,
+            },
+        );
+        $first->task_ok($upid, "created linked clone $testcase->{clone}")
+            or die "creating the linked clone failed\n";
+
+        my $conf = $client->get("/nodes/$node/lxc/$testcase->{clone}/config", {});
+        my ($base, $image) = ($conf->{rootfs} // '') =~ m!^[^:]+:([^/,]+)/([^,]+)!;
+        is(
+            $base,
+            $testcase->{base},
+            'linked clone volume refers to the base image of the template',
+        );
+
+        my $parent = eval { $rbd_info->($image // '')->{parent} } // {};
+        is_deeply(
+            [$parent->@{qw(pool image snapshot)}],
+            [$pool, $testcase->{base}, '__base__'],
+            "RBD image of the linked clone is cloned from '$testcase->{base}\@__base__'",
+        );
+    },
+    sub {
+        my ($testcase) = @_;
+        $first->destroy_guest('lxc', $testcase->{clone}) if defined($testcase->{clone});
+    },
+)->step(
+    'verify linked clone',
+    sub {
+        my ($testcase) = @_;
+        my $clone = $testcase->{clone};
+
+        $first->start_guest('lxc', $clone);
+
+        is(
+            $first->ct_exec($clone, 'cat /root/marker'),
+            "$marker\n",
+            'linked clone has the data of the template',
+        );
+
+        $first->stop_guest('lxc', $clone);
+    },
+)->step(
+    'template is protected while a linked clone exists',
+    sub {
+        my ($testcase) = @_;
+        my $ctid = $testcase->{ctid};
+
+        my $upid = eval { $client->delete("/nodes/$node/lxc/$ctid", {}) };
+        my $status = defined($upid) ? $first->wait_for_task($upid, 1) : "API error: $@";
+        ok(!$first->task_successful($status), 'destroying the template is refused');
+        like(
+            $status,
+            qr/still in use by linked clone/,
+            'destroying the template is refused because of the linked clone',
+        );
+
+        my $conf = eval { $client->get("/nodes/$node/lxc/$ctid/config", {}) };
+        ok($conf && $conf->{template}, 'template still exists');
+    },
+)->run();
+
+Proxmox::Test::Testcase->new('ceph_rbd_vms')->with([
+    {
+        storage => 'e2e-rbd-librbd',
+        krbd => 0,
+    },
+    {
+        storage => 'e2e-rbd-krbd',
+        krbd => 1,
+    },
+])->step(
+        'add RBD storage',
+        sub {
+            my ($testcase) = @_;
+            $add_storage->($testcase, 'images');
+        },
+        sub {
+            my ($testcase) = @_;
+            $client->delete("/storage/$testcase->{storage}");
+        },
+)->step(
+    'create VM and write data to its disk',
+    sub {
+        my ($testcase) = @_;
+
+        $testcase->{vmid} = $client->get('/cluster/nextid');
+        $testcase->{vm_node} = $node;
+        my $upid = $client->post(
+            "/nodes/$node/qemu",
+            {
+                vmid => $testcase->{vmid},
+                memory => 512,
+                scsihw => 'virtio-scsi-pci',
+                scsi0 => "$testcase->{storage}:1",
+            },
+        );
+        $first->task_ok($upid, "created VM $testcase->{vmid} on '$testcase->{storage}'")
+            or die "creating the VM failed\n";
+
+        my $conf = $client->get("/nodes/$node/qemu/$testcase->{vmid}/config", {});
+        ($testcase->{disk}) = ($conf->{scsi0} // '') =~ m/^([^,]+)/;
+        ($testcase->{image}) = ($testcase->{disk} // '') =~ m/^[^:]+:(.+)$/;
+        ok($testcase->{image}, "VM disk is '$testcase->{disk}'") or die "VM has no disk\n";
+
+        $write_random_data->($testcase->{image});
+        $testcase->{checksum} = $image_checksum->($testcase->{image});
+    },
+    sub {
+        my ($testcase) = @_;
+        return if !defined($testcase->{vmid});
+        $first->destroy_guest_on($testcase->{vm_node}, 'qemu', $testcase->{vmid});
+    },
+)->step(
+    'start VM',
+    sub {
+        my ($testcase) = @_;
+
+        $first->start_guest('qemu', $testcase->{vmid});
+
+        my $mapped = $mapped_images->($first)->{ $testcase->{image} };
+        if ($testcase->{krbd}) {
+            ok($mapped, "VM disk is mapped through krbd on '$node'");
+        } else {
+            ok(!$mapped, 'VM disk is not mapped, as QEMU accesses it through librbd');
+        }
+    },
+)->step(
+    'snapshot running VM',
+    sub {
+        my ($testcase) = @_;
+        my $vmid = $testcase->{vmid};
+
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/snapshot", { snapname => $snapname });
+        $first->task_ok($upid, 'created snapshot of the running VM')
+            or die "creating the snapshot failed\n";
+
+        my $log = $client->get("/nodes/$node/tasks/$upid/log", { limit => 1000 });
+        my $by_qemu = grep { ($_->{t} // '') eq 'internal qemu snapshot' } $log->@*;
+        if ($testcase->{krbd}) {
+            ok(!$by_qemu, 'the storage takes the snapshot of the mapped disk');
+        } else {
+            ok($by_qemu, 'QEMU takes the snapshot itself through librbd');
+        }
+
+        ok(
+            $rbd_snapshots->($testcase->{image})->{$snapname},
+            "RBD image of the disk has the snapshot '$snapname'",
+        );
+    },
+)->step(
+    'roll back snapshot',
+    sub {
+        my ($testcase) = @_;
+        my $vmid = $testcase->{vmid};
+
+        $first->stop_guest('qemu', $vmid);
+
+        $write_random_data->($testcase->{image});
+        isnt(
+            $image_checksum->($testcase->{image}),
+            $testcase->{checksum},
+            'data on the disk changed after the snapshot',
+        );
+
+        my $upid = $client->post("/nodes/$node/qemu/$vmid/snapshot/$snapname/rollback", {});
+        $first->task_ok($upid, 'rolled back the snapshot') or die "rolling back failed\n";
+
+        is(
+            $image_checksum->($testcase->{image}),
+            $testcase->{checksum},
+            'rollback restored the data from the time of the snapshot',
+        );
+    },
+)->step(
+    'delete snapshot of running VM',
+    sub {
+        my ($testcase) = @_;
+        my $vmid = $testcase->{vmid};
+
+        $first->start_guest('qemu', $vmid);
+
+        my $upid = $client->delete("/nodes/$node/qemu/$vmid/snapshot/$snapname", {});
+        $first->task_ok($upid, 'deleted the snapshot of the running VM')
+            or die "deleting the snapshot failed\n";
+
+        is_deeply(
+            [keys $rbd_snapshots->($testcase->{image})->%*],
+            [],
+            'snapshot is removed from the RBD image',
+        );
+    },
+)->step(
+    'resize disk of running VM',
+    sub {
+        my ($testcase) = @_;
+        my $vmid = $testcase->{vmid};
+
+        my $upid =
+            $client->put("/nodes/$node/qemu/$vmid/resize", { disk => 'scsi0', size => '+1G' });
+        $first->task_ok($upid, 'resized the disk of the running VM by 1 GiB')
+            or die "resizing the disk failed\n";
+
+        like(
+            $client->get("/nodes/$node/qemu/$vmid/config", {})->{scsi0} // '',
+            qr/\bsize=2G\b/,
+            'config reports the new disk size',
+        );
+
+        is(
+            $rbd_info->($testcase->{image})->{size},
+            2 * 1024 * 1024 * 1024,
+            'RBD image has the new size',
+        );
+    },
+)->step(
+    'live-migrate VM',
+    sub {
+        my ($testcase) = @_;
+        my $vmid = $testcase->{vmid};
+
+        my $upid =
+            $client->post("/nodes/$node/qemu/$vmid/migrate", { target => $target, online => 1 });
+        $first->task_ok($upid, "live-migrated VM $vmid to '$target'")
+            or die "migrating the VM failed\n";
+        $testcase->{vm_node} = $target;
+
+        $wait_for_status->($second, 'qemu', $vmid, 'running');
+
+        my $conf = $second->client()->get("/nodes/$target/qemu/$vmid/config", {});
+        like(
+            $conf->{scsi0} // '',
+            qr/^\Q$testcase->{disk}\E\b/,
+            'VM still uses the same disk on the shared storage',
+        );
+
+        my $log = $second->client()->get("/nodes/$node/tasks/$upid/log", { limit => 10000 });
+        ok(
+            !(grep { ($_->{t} // '') =~ m/drive mirror|starting storage migration/ } $log->@*),
+            'migration did not copy the disk',
+        );
+
+        my $mapped_on_target = $mapped_images->($second)->{ $testcase->{image} };
+        my $mapped_on_source = $mapped_images->($first)->{ $testcase->{image} };
+        if ($testcase->{krbd}) {
+            ok($mapped_on_target, "VM disk is mapped on '$target'");
+            ok(!$mapped_on_source, "VM disk is no longer mapped on '$node'");
+        } else {
+            ok(!$mapped_on_target && !$mapped_on_source, 'VM disk is mapped on neither node');
+        }
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index 47c06b0..fec8667 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -403,6 +403,17 @@ TestcaseConfig(
             ],
             tag: "ceph",
         ),
+        Testcase(
+            name: "ceph_rbd_storage",
+            run: "./tests/ceph/test_ceph_rbd_storage.pl",
+            timeout: 2400,
+            instances: [
+                "pve1-cluster",
+                "pve2-cluster",
+                "pve3-cluster",
+            ],
+            tag: "ceph",
+        ),
     ],
     rollback_between_tests: Some("pre-test"),
 )
-- 
2.47.3