[PATCH e2e-tests 32/38] tests: backup: verify VM backup and restore, locally and on PBS

["Michael Köppl" <m.koeppl@proxmox.com>]

Back up a running VM in snapshot mode to the local storage and to a
PBS datastore, and check that the archives are listed with the
expected format and with the notes from the notes template. Restore
the local backup as a new VM, which keeps the MAC address, and the
PBS backup as a new VM with a unique MAC address. Finally, restore
the PBS backup over the original VM, which needs to be forced.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---
 tests/pve_vm_backup_restore.pl | 269 +++++++++++++++++++++++++++++++++
 tests/testcases.ron            |  10 ++
 2 files changed, 279 insertions(+)
 create mode 100755 tests/pve_vm_backup_restore.pl

diff --git a/tests/pve_vm_backup_restore.pl b/tests/pve_vm_backup_restore.pl
new file mode 100755
index 0000000..3aa5add
--- /dev/null
+++ b/tests/pve_vm_backup_restore.pl
@@ -0,0 +1,269 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<pve_vm_backup_restore> - Test backing up and restoring a VM, locally and on PBS.
+
+=head1 DESCRIPTION
+
+This testcase creates a VM with a disk on a temporary directory storage, writes
+random data to the disk and keeps a copy of it. It backs up the running VM in
+snapshot mode to the local storage and to a datastore on PBS (including a notes
+template) and verifies that the archives are listed with the expected format and
+notes while the VM keeps running.
+
+It restores the local backup as a new VM, which must keep the MAC address of the
+original, and the PBS backup as a new VM with a unique MAC address. Both restored
+VMs are expected to have the original disk content.
+
+Finally, it changes the data on the disk of the stopped original VM and restores
+the PBS backup over it. Restoring over an existing VM must be refused unless it
+is forced. The forced restore must bring back the original disk content.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use lib '.';
+use Proxmox::Test::Helper qw(get_pbs_instance get_pve_instance);
+use Proxmox::Test::Testcase;
+
+my $pve = get_pve_instance('pve1');
+my $pbs = get_pbs_instance('pbs1');
+my $node = $pve->get_nodename();
+my $client = $pve->client();
+
+my $disk_storage = 'e2e-backup-disks';
+my $disk_storage_path = '/var/lib/e2e-backup-disks';
+my $datastore = 'e2e-vm';
+my $pbs_storage = 'e2e-pbs-vm';
+my $name = 'vm-backup';
+
+my $vmid;
+my $reference_dir;
+my $backups = {};
+my $restored_vmids = {};
+
+my $config = sub {
+    my ($id) = @_;
+    return $client->get("/nodes/$node/qemu/$id/config", {});
+};
+
+my $mac_of = sub {
+    my ($conf) = @_;
+    my ($mac) = ($conf->{net0} // '') =~ m/^[^=,]+=([0-9A-Fa-f:]{17})\b/;
+    return $mac;
+};
+
+my $disk_path = sub {
+    my ($id) = @_;
+    my ($volid) = ($config->($id)->{scsi0} // '') =~ m/^([^,]+)/;
+    die "VM $id has no scsi0\n" if !defined($volid);
+    my ($path) = $pve->run_command("pvesm path $volid");
+    chomp($path);
+    return $path;
+};
+
+my $has_reference_content = sub {
+    my ($id) = @_;
+    my $path = $disk_path->($id);
+    return eval {
+        $pve->run_command("qemu-img compare -f raw $reference_dir/disk.raw $path");
+        1;
+    };
+};
+
+my $restore = sub {
+    my ($id, $archive, %opts) = @_;
+    return $client->post(
+        "/nodes/$node/qemu",
+        { vmid => $id, archive => $archive, storage => $disk_storage, %opts },
+    );
+};
+
+Proxmox::Test::Testcase->new('pve_vm_backup_restore')->with([{}])->step(
+    'create directory storage for VM disks',
+    sub {
+        $pve->run_command("mkdir -p $disk_storage_path");
+        $client->post(
+            '/storage',
+            {
+                storage => $disk_storage,
+                type => 'dir',
+                path => $disk_storage_path,
+                content => 'images',
+            },
+        );
+    },
+    sub {
+        eval { $client->delete("/storage/$disk_storage") };
+        $pve->run_command("rm -rf $disk_storage_path");
+    },
+)->step(
+    'create datastore on PBS',
+    sub {
+        $pbs->create_datastore($datastore);
+    },
+    sub {
+        $pbs->remove_datastore($datastore);
+    },
+)->step(
+    'add PBS as storage on PVE',
+    sub {
+        $pve->add_pbs_storage($pbs_storage, $pbs, $datastore);
+    },
+    sub {
+        $client->delete("/storage/$pbs_storage");
+    },
+)->step(
+    'create VM',
+    sub {
+        $vmid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node/qemu",
+            {
+                vmid => $vmid,
+                name => $name,
+                memory => 256,
+                scsihw => 'virtio-scsi-pci',
+                scsi0 => "$disk_storage:1,format=raw",
+                net0 => 'virtio,bridge=vmbr0',
+            },
+        );
+        $pve->task_ok($upid, "created VM $vmid") or die "creating VM $vmid failed\n";
+    },
+    sub {
+        $pve->destroy_guest('qemu', $vmid) if defined($vmid);
+    },
+)->step(
+    'write data to the disk and keep a copy',
+    sub {
+        my $path = $disk_path->($vmid);
+        $pve->run_command("dd if=/dev/urandom of=$path bs=1M count=8 seek=16 conv=notrunc");
+
+        ($reference_dir) = $pve->run_command('mktemp -d');
+        chomp($reference_dir);
+        $pve->run_command("cp --sparse=always $path $reference_dir/disk.raw");
+    },
+    sub {
+        $pve->run_command("rm -rf $reference_dir") if defined($reference_dir);
+    },
+)->step(
+    'start VM',
+    sub {
+        $pve->start_guest('qemu', $vmid);
+    },
+)->step(
+    'back up running VM',
+    sub {
+        for my $storage ('local', $pbs_storage) {
+            my $upid = $client->post(
+                "/nodes/$node/vzdump",
+                {
+                    vmid => $vmid,
+                    storage => $storage,
+                    mode => 'snapshot',
+                    'notes-template' => 'e2e {{guestname}}',
+                },
+            );
+            $pve->task_ok($upid, "backed up VM $vmid to $storage")
+                or die "backing up VM $vmid to $storage failed\n";
+
+            my $content =
+                $client->get("/nodes/$node/storage/$storage/content", { content => 'backup' });
+            my ($backup) = grep { ($_->{vmid} // 0) == $vmid } $content->@*;
+            ok($backup, "backup is listed on $storage") or die "backup not found\n";
+            $backups->{$storage} = $backup->{volid};
+
+            like(
+                $backup->{format},
+                $storage eq 'local' ? qr/^vma\b/ : qr/^pbs-vm$/,
+                "backup on $storage has the expected format",
+            );
+            is($backup->{notes}, "e2e $name", "backup on $storage has notes from the template");
+        }
+
+        is(
+            $client->get("/nodes/$node/qemu/$vmid/status/current", {})->{status},
+            'running',
+            'VM keeps running during the backups',
+        );
+    },
+    sub {
+        $pve->destroy_volume('local', $backups->{local}) if $backups->{local};
+    },
+)->step(
+    'restore local backup as new VM',
+    sub {
+        my $id = $restored_vmids->{local} = $client->get('/cluster/nextid');
+        my $upid = $restore->($id, $backups->{local});
+        $pve->task_ok($upid, "restored local backup as VM $id")
+            or die "restoring the local backup failed\n";
+
+        my $conf = $config->($id);
+        is($conf->{name}, $name, 'restored VM has the name of the original');
+        is(
+            $mac_of->($conf),
+            $mac_of->($config->($vmid)),
+            'restored VM keeps the MAC address of the original',
+        );
+        like(
+            $conf->{scsi0} // '',
+            qr/^\Q$disk_storage\E:/,
+            "disk is restored to $disk_storage",
+        );
+        ok($has_reference_content->($id), 'restored disk has the original content');
+    },
+    sub {
+        $pve->destroy_guest('qemu', $restored_vmids->{local})
+            if defined($restored_vmids->{local});
+    },
+)->step(
+    'restore PBS backup as new VM with unique MAC address',
+    sub {
+        my $id = $restored_vmids->{pbs} = $client->get('/cluster/nextid');
+        my $upid = $restore->($id, $backups->{$pbs_storage}, unique => 1);
+        $pve->task_ok($upid, "restored PBS backup as VM $id")
+            or die "restoring the PBS backup failed\n";
+
+        my $conf = $config->($id);
+        is($conf->{name}, $name, 'restored VM has the name of the original');
+        my $mac = $mac_of->($conf);
+        ok(defined($mac), 'restored VM has a MAC address');
+        isnt($mac, $mac_of->($config->($vmid)), 'restored VM has a new MAC address');
+        ok($has_reference_content->($id), 'restored disk has the original content');
+    },
+    sub {
+        $pve->destroy_guest('qemu', $restored_vmids->{pbs})
+            if defined($restored_vmids->{pbs});
+    },
+)->step(
+    'restore PBS backup over the changed original VM',
+    sub {
+        $pve->stop_guest('qemu', $vmid);
+
+        my $path = $disk_path->($vmid);
+        $pve->run_command("dd if=/dev/urandom of=$path bs=1M count=8 seek=16 conv=notrunc");
+        ok(!$has_reference_content->($vmid), 'disk content differs from the backup');
+
+        my $mac = $mac_of->($config->($vmid));
+
+        eval { $restore->($vmid, $backups->{$pbs_storage}) };
+        ok($@, 'restoring over the existing VM without force fails');
+        ok(!$has_reference_content->($vmid), 'disk content is unchanged');
+
+        my $upid = $restore->($vmid, $backups->{$pbs_storage}, force => 1);
+        $pve->task_ok($upid, "restored PBS backup over VM $vmid")
+            or die "restoring over VM $vmid failed\n";
+
+        my $conf = $config->($vmid);
+        is($conf->{name}, $name, 'VM keeps its name');
+        is($mac_of->($conf), $mac, 'VM keeps its MAC address');
+        ok($has_reference_content->($vmid), 'disk has the content from the time of the backup');
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index ace5792..b107835 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -283,6 +283,16 @@ TestcaseConfig(
             ],
             tag: "pbs",
         ),
+        Testcase(
+            name: "pve_vm_backup_restore",
+            run: "./tests/pve_vm_backup_restore.pl",
+            timeout: 900,
+            instances: [
+                "pve1",
+                "pbs1",
+            ],
+            tag: "pbs",
+        ),
         Testcase(
             name: "sdn_create_vlan_zone",
             run: "./tests/sdn/test_vlan_zone.pl",
-- 
2.47.3