[pve-devel] [PATCH v2 storage 3/3] rbd: add integration test for namespace handling

[Aaron Lauterer <a.lauterer@proxmox.com>]

This test is intended to be run on a hyperconverged PVE cluster to test
the most common operations of VMs using a namespaced Ceph RBD pool.

Signed-off-by: Aaron Lauterer <a.lauterer@proxmox.com>
---
v1 -> v2:
reworked the test from the feedback I got [0].

* tests are now defined in deeper hashes/arrays and can contain testing
  steps as well as preparation and cleanup steps where needed.
* command calls don't use fixed paths
* command calls use arrays
* a new Ceph pool will be created and removed at the end by default. Can
  be overriden with the '--use_existing' parameter. Most likely in
  combination with the '--pool' parameter.
* debug flag has been introduced to be used if some debug printing via
  the `jp` sub is needed. Currenctly not used but might be useful when
  tinkering with the test in the future. Thx @Thomas for the hint


[0] https://lists.proxmox.com/pipermail/pve-devel/2021-April/047472.html

 test/rbd_namespace.pl | 370 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 370 insertions(+)
 create mode 100755 test/rbd_namespace.pl

diff --git a/test/rbd_namespace.pl b/test/rbd_namespace.pl
new file mode 100755
index 0000000..c6a9468
--- /dev/null
+++ b/test/rbd_namespace.pl
@@ -0,0 +1,370 @@
+#!/usr/bin/perl
+
+# This script is meant to be run manually on hyperconverged PVE server with a
+# Ceph cluster. It tests how PVE handles RBD namespaces.
+#
+# The pool (default: rbd) must already exist. The namespace and VMs will be
+# created.
+#
+# Parameters like names for the pool an namespace and the VMID can be
+# configured.  The VMIDs for the clones is $vmid -1 and $vmid -2.
+#
+# Cleanup is done after a successful run. Cleanup can also be called manually.
+#
+# Known issues:
+#
+# * Snapshot rollback can sometimes be racy with stopping the VM and Ceph
+#  recognizing that the disk image is not in use anymore.
+
+use strict;
+use warnings;
+
+use Test::More;
+use Getopt::Long;
+use JSON;
+
+use PVE::Tools qw(run_command);
+
+my $pool = "testpool";
+my $use_existing= undef;
+my $namespace = "testspace";
+my $showhelp = '';
+my $vmid = 999999;
+my $cleanup = undef;
+my $DEBUG = 0;
+
+my $helpstring = "To override default values, set them as named parameters:
+
+--pool		pool name, default: ${pool}
+--use_existing  use existing pool, default: 0, needs --pool set
+--namespace	rbd namespace, default: ${namespace}
+--vmid		VMID of the test VM, default: ${vmid}
+--cleanup	Remove the storage definitions, namespaces and VMs
+--debug		Enable debug output\n";
+
+GetOptions (
+	"pool=s" => \$pool,
+	"use_existing" => \$use_existing,
+	"namespace=s" => \$namespace,
+	"vmid=i" => \$vmid,
+	"help" => \$showhelp,
+	"cleanup" => \$cleanup,
+	"debug" => \$DEBUG,
+) or die ($helpstring);
+
+die $helpstring if $showhelp;
+
+my $storage_name = "${pool}-${namespace}";
+
+my $vmid_clone = int($vmid) - 1;
+my $vmid_linked_clone = int($vmid) - 2;
+
+sub jp {
+    return if !$DEBUG;
+    print to_json($_[0], { utf8 => 8, pretty => 1, canonical => 1 }) . "\n";
+}
+
+sub run_cmd {
+    my ($cmd, $json, $ignore_errors) = @_;
+
+    my $raw = '';
+    my $parser = sub {$raw .= shift;};
+
+    eval {
+	run_command($cmd, outfunc => $parser);
+    };
+    if (my $err = $@) {
+	die $err if !$ignore_errors;
+    }
+
+    if ($json) {
+	my $result;
+	if ($raw eq '') {
+	    $result = [];
+	} elsif ($raw =~ m/^(\[.*\])$/s) { # untaint
+	    $result = JSON::decode_json($1);
+	} else {
+	    die "got unexpected data from command: '$cmd' -> '$raw'\n";
+	}
+	return $result;
+	}
+    return $raw;
+}
+
+sub run_test_cmd {
+    my ($cmd) = @_;
+
+    my $raw = '';
+    my $out = sub {
+	my $line = shift;
+	$raw .= "${line}\n";
+    };
+
+    eval {
+	run_command($cmd, outfunc => $out);
+    };
+    if (my $err = $@) {
+	print $raw;
+	print $err;
+	return 0;
+    }
+    print $raw;
+    return 1;
+}
+
+sub prepare {
+    print "Preparing test environent\n";
+
+    my $pools = run_cmd("ceph osd pool ls --format json", 1);
+
+    my %poolnames = map {$_ => 1} @$pools;
+    die "Pool '$pool' does not exist!\n"
+	if !exists($poolnames{$pool}) && $use_existing;
+
+    run_cmd(['pveceph', 'pool', 'create', ${pool}, '--add_storages', 1])
+	if !$use_existing;
+
+    my $namespaces = run_cmd(['rbd', '-p', ${pool}, 'namespace', 'ls', '--format', 'json'], 1);
+    my $ns_found = 0;
+    for my $i (@$namespaces) {
+	#print Dumper $i;
+	$ns_found = 1 if $i->{name} eq $namespace;
+    }
+
+    if (!$ns_found) {
+	print "Create namespace '${namespace}' in pool '${pool}'\n";
+	run_cmd(['rbd', 'namespace', 'create', "${pool}/${namespace}"]);
+    }
+
+    my $storages = run_cmd(['pvesh', 'get', 'storage', '--output-format', 'json'], 1);
+    #print Dumper $storages;
+    my $rbd_found = 0;
+    my $pool_found = 0;
+
+    print "Create storage definition\n";
+    for my $stor (@$storages) {
+	$pool_found = 1 if $stor->{storage} eq $pool;
+	$rbd_found = 1 if $stor->{storage} eq $storage_name;
+
+	if ($rbd_found) {
+	    run_cmd(['pvesm', 'set', ${storage_name}, '--krbd', '0']);
+	    die "Enable the storage '$stor->{storage}'!" if $stor->{disable};
+	}
+    }
+    if (!$pool_found) {
+	die "No storage for pool '${pool}' found! Must have same name as pool!\n"
+	    if $use_existing;
+
+	run_cmd(['pvesm', 'add', 'rbd', $pool, '--pool', $pool, '--content', 'images,rootdir']);
+    }
+    # create PVE storages (librbd / krbd)
+    run_cmd(['pvesm', 'add', 'rbd', ${storage_name}, '--krbd', '0', '--pool', ${pool}, '--namespace', ${namespace}, '--content', 'images,rootdir'])
+	if !$rbd_found;
+
+
+    # create test VM
+    print "Create test VM ${vmid}\n";
+    my $vms = run_cmd(['pvesh', 'get', 'cluster/resources', '--type', 'vm', '--output-format', 'json'], 1);
+    for my $vm (@$vms) {
+	# TODO: introduce a force flag to make this behaviour configurable
+
+	if ($vm->{vmid} eq $vmid) {
+	    print "Test VM '${vmid}' already exists. It will be removed and recreated!\n";
+	    run_cmd(['qm', 'stop', ${vmid}], 0, 1);
+	    run_cmd(['qm', 'destroy', ${vmid}]);
+	}
+    }
+    run_cmd(['qm', 'create', ${vmid}, '--bios', 'ovmf', '--efidisk0', "${storage_name}:1", '--scsi0', "${storage_name}:2"]);
+}
+
+
+sub cleanup {
+    print "Cleaning up test environment!\n";
+    print "Removing VMs\n";
+    run_cmd(['qm', 'stop', ${vmid}], 0, 1);
+    run_cmd(['qm', 'stop', ${vmid_linked_clone}], 0, 1);
+    run_cmd(['qm', 'stop', ${vmid_clone}], 0, 1);
+    run_cmd(['qm', 'destroy', ${vmid_linked_clone}], 0, 1);
+    run_cmd(['qm', 'destroy', ${vmid_clone}], 0, 1);
+    run_cmd(['for', 'i', 'in', "/dev/rbd/${pool}/${namespace}/*;", 'do', '/usr/bin/rbd', 'unmap', '\$i;', 'done'], 0, 1);
+    run_cmd(['qm', 'unlock', ${vmid}], 0, 1);
+    run_cmd(['qm', 'destroy', ${vmid}], 0, 1);
+
+    print "Removing Storage definition for ${storage_name}\n";
+    run_cmd(['pvesm', 'remove', ${storage_name}], 0, 1);
+
+    print "Removing RBD namespace '${pool}/${namespace}'\n";
+    run_cmd(['rbd', 'namespace', 'remove', "${pool}/${namespace}"], 0, 1);
+
+    if (!$use_existing) {
+	print "Removing Storage definition for ${pool}\n";
+	run_cmd(['pvesm', 'remove', ${pool}], 0, 1);
+	print "Removing test pool\n";
+	run_cmd(['pveceph', 'pool', 'destroy', $pool]);
+    }
+}
+
+my $tests = [
+    # Example structure for tests
+    # {
+    #     name => "name of test section",
+    #     preparations => [
+    #         ['some', 'prep', 'command'],
+    #     ],
+    #     steps => [
+    #         ['test', 'cmd', $vmid],
+    #         ['second', 'step', $vmid],
+    #     ],
+    #     cleanup => [
+    #         ['cleanup', 'command'],
+    #     ],
+    # },
+    {
+	name => 'first VM start',
+	steps => [
+	    ['qm', 'start', $vmid],
+	],
+    },
+    {
+	name => 'snapshot/rollback',
+	steps => [
+	    ['qm', 'snapshot', $vmid, 'test'],
+	    ['qm', 'rollback', $vmid, 'test'],
+	],
+	cleanup => [
+	    ['qm', 'unlock', $vmid],
+	],
+    },
+    {
+	name => 'remove snapshot',
+	steps => [
+	    ['qm', 'delsnapshot', $vmid, 'test'],
+	],
+    },
+    {
+	name => 'moving disk between namespaces',
+	steps => [
+	    ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
+	    ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
+	],
+    },
+    {
+	name => 'switch to krbd',
+	preparations => [
+	    ['qm', 'stop', $vmid],
+	    ['pvesm', 'set', $storage_name, '--krbd', 1]
+	],
+    },
+    {
+	name => 'start VM with krbd',
+	steps => [
+	    ['qm', 'start', $vmid],
+	],
+    },
+    {
+	name => 'snapshot/rollback with krbd',
+	steps => [
+	    ['qm', 'snapshot', $vmid, 'test'],
+	    ['qm', 'rollback', $vmid, 'test'],
+	],
+	cleanup => [
+	    ['qm', 'unlock', $vmid],
+	],
+    },
+    {
+	name => 'remove snapshot with krbd',
+	steps => [
+	    ['qm', 'delsnapshot', $vmid, 'test'],
+	],
+    },
+    {
+	name => 'moving disk between namespaces with krbd',
+	steps => [
+	    ['qm', 'move_disk', $vmid, 'scsi0', $pool, '--delete', 1],
+	    ['qm', 'move_disk', $vmid, 'scsi0', $storage_name, '--delete', 1],
+	],
+    },
+    {
+	name => 'clone VM with krbd',
+	steps => [
+	    ['qm', 'clone', $vmid, $vmid_clone],
+	],
+    },
+    {
+	name => 'switch to non krbd',
+	preparations => [
+	    ['qm', 'stop', $vmid],
+	    ['qm', 'stop', $vmid_clone],
+	    ['pvesm', 'set', $storage_name, '--krbd', 0]
+	],
+    },
+    {
+	name => 'templates and linked clone',
+	steps => [
+	    ['qm', 'template', $vmid],
+	    ['qm', 'clone', $vmid, $vmid_linked_clone],
+	    ['qm', 'start', $vmid_linked_clone],
+	    ['qm', 'stop', $vmid_linked_clone],
+	],
+    },
+    {
+	name => 'start linked clone with krbd',
+	preparations => [
+	    ['pvesm', 'set', $storage_name, '--krbd', 1]
+	],
+	steps => [
+	    ['qm', 'start', $vmid_linked_clone],
+	    ['qm', 'stop', $vmid_linked_clone],
+	],
+    },
+];
+
+sub run_prep_cleanup {
+    my ($cmds) = @_;
+
+    for (@$cmds) {
+	print join(' ', @$_). "\n";
+	run_cmd($_);
+    }
+}
+
+sub run_steps {
+    my ($steps) = @_;
+
+    for (@$steps) {
+	ok(run_test_cmd($_), join(' ', @$_));
+    }
+}
+
+sub run_tests {
+    print "Running tests:\n";
+
+    my $num_tests = 0;
+    for (@$tests) {
+	$num_tests += scalar(@{$_->{steps}}) if defined $_->{steps};
+    }
+
+    print("Tests: $num_tests\n");
+    plan tests => $num_tests;
+
+    for my $test (@$tests) {
+	print "Section: $test->{name}\n";
+	run_prep_cleanup($test->{preparations}) if defined $test->{preparations};
+	run_steps($test->{steps}) if defined $test->{steps};
+	run_prep_cleanup($test->{cleanup}) if defined $test->{cleanup};
+    }
+
+    done_testing();
+
+    if (Test::More->builder->is_passing()) {
+	cleanup();
+    }
+}
+
+if ($cleanup) {
+    cleanup();
+} else {
+    prepare();
+    run_tests();
+}
+
-- 
2.20.1