[PATCH e2e-tests 33/38] tests: pbs: verify single file restore from a container backup

["Michael Köppl" <m.koeppl@proxmox.com>]

Back up a container with a marker file to PBS, browse the backup
through the file restore API down to the file and download it.

Signed-off-by: Michael Köppl <m.koeppl@proxmox.com>
---

Notes:
    The test accepts curl's exit code 56 for the download, which curl
    with OpenSSL 3 returns because pveproxy ends the streamed response
    without a TLS close_notify. This workaround is only meant to stay
    until http-server patch [0] is applied, or the issue is resolved in
    some other way.
    
    [0] https://lore.proxmox.com/pve-devel/20261103155223.1960754-1-m.koeppl@proxmox.com/

 tests/pbs/test_pbs_file_restore.pl | 151 +++++++++++++++++++++++++++++
 tests/testcases.ron                |  10 ++
 2 files changed, 161 insertions(+)
 create mode 100755 tests/pbs/test_pbs_file_restore.pl

diff --git a/tests/pbs/test_pbs_file_restore.pl b/tests/pbs/test_pbs_file_restore.pl
new file mode 100755
index 0000000..1d9a20e
--- /dev/null
+++ b/tests/pbs/test_pbs_file_restore.pl
@@ -0,0 +1,151 @@
+#!/usr/bin/perl
+
+=head1 NAME
+
+C<test_pbs_file_restore> - Restore single files from a container backup on PBS.
+
+=head1 DESCRIPTION
+
+This testcase creates a local datastore on PBS and adds it to PVE as a storage.
+It creates a container, writes a file to it and backs it up to the PBS datastore.
+It then browses the backup through the single file restore API down to the file,
+which must be listed with its size. It then downloads the file, which must have
+the content written before the backup.
+
+=cut
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use URI::Escape qw(uri_escape);
+
+use lib '.';
+use Proxmox::Test::Helper qw(CONTAINER_TEMPLATE get_pbs_instance get_pve_instance);
+use Proxmox::Test::Testcase;
+
+my $pve = get_pve_instance('pve1');
+my $pbs = get_pbs_instance('pbs1');
+my $node = $pve->get_nodename();
+my $client = $pve->client();
+
+my $datastore = 'e2e-file-restore';
+my $storage_id = 'e2e-pbs-file-restore';
+
+my $ctid;
+my $backup_volid;
+my $marker_file = 'e2e-marker';
+my $marker = "e2e-file-restore-$$";
+my $marker_entry;
+
+# Entries below $filepath in the backup
+my $list = sub {
+    my ($filepath) = @_;
+    return $client->get(
+        "/nodes/$node/storage/$storage_id/file-restore/list",
+        { volume => $backup_volid, filepath => $filepath },
+    );
+};
+
+my $find_entry = sub {
+    my ($entries, $match) = @_;
+    my ($entry) = grep { $_->{text} =~ $match } $entries->@*;
+    return $entry;
+};
+
+Proxmox::Test::Testcase->new('pbs_file_restore')->with([{}])->step(
+    'create datastore on PBS',
+    sub {
+        $pbs->create_datastore($datastore);
+    },
+    sub {
+        $pbs->remove_datastore($datastore);
+    },
+)->step(
+    'add PBS as storage on PVE',
+    sub {
+        $pve->add_pbs_storage($storage_id, $pbs, $datastore);
+    },
+    sub {
+        $client->delete("/storage/$storage_id");
+    },
+)->step(
+    'create container and write file',
+    sub {
+        $ctid = $client->get('/cluster/nextid');
+        my $upid = $client->post(
+            "/nodes/$node/lxc",
+            {
+                vmid => $ctid,
+                ostemplate => CONTAINER_TEMPLATE,
+                rootfs => 'local-lvm:2',
+                memory => 256,
+                unprivileged => 1,
+                features => 'nesting=1',
+                hostname => 'ct-file-restore',
+            },
+        );
+        $pve->task_ok($upid, "created container $ctid")
+            or die "creating container $ctid failed\n";
+
+        $pve->start_guest('lxc', $ctid);
+        $pve->ct_exec($ctid, "sh -c 'echo $marker > /root/$marker_file'");
+    },
+    sub {
+        $pve->destroy_guest('lxc', $ctid) if defined($ctid);
+    },
+)->step(
+    'back up container to PBS',
+    sub {
+        my $upid = $client->post(
+            "/nodes/$node/vzdump",
+            { vmid => $ctid, storage => $storage_id, mode => 'snapshot' },
+        );
+        $pve->task_ok($upid, 'backed up container to PBS') or die "backup failed\n";
+
+        my $content =
+            $client->get("/nodes/$node/storage/$storage_id/content", { content => 'backup' });
+        my ($backup) = grep { ($_->{vmid} // 0) == $ctid } $content->@*;
+        $backup_volid = $backup->{volid};
+        ok($backup_volid, 'found container backup on PBS storage') or die "backup not found\n";
+    },
+)->step(
+    'browse backup down to the file',
+    sub {
+        my $archive = $find_entry->($list->('/'), qr/^root\.pxar/);
+        ok($archive, 'backup contains the root file system archive')
+            or die "archive not found\n";
+
+        my $root_dir = $find_entry->($list->($archive->{filepath}), qr/^root$/);
+        ok($root_dir, "archive contains the directory '/root'") or die "'/root' not found\n";
+
+        $marker_entry = $find_entry->($list->($root_dir->{filepath}), qr/^\Q$marker_file\E$/);
+        ok($marker_entry, 'directory contains the file') or die "file not found\n";
+        is($marker_entry->{size}, length("$marker\n"), 'file is listed with its size');
+    },
+)->step(
+    'download file',
+    sub {
+        # This doesn't work with the API client (expects JSON), to use ticket from the node for
+        # manual request.
+        my $ticket = $client->post(
+            '/access/ticket',
+            { username => 'root@pam', password => $pve->get_root_password() },
+        )->{ticket};
+
+        my $query = join(
+            '&',
+            'volume=' . uri_escape($backup_volid),
+            'filepath=' . uri_escape($marker_entry->{filepath}),
+        );
+        # FIXME: remove workaround once api server sends TLS close notify
+        my ($content) =
+            $pve->run_command("curl -sSfk -H 'Cookie: PVEAuthCookie=$ticket'"
+                . " 'https://localhost:8006/api2/json/nodes/$node/storage/$storage_id"
+                . "/file-restore/download?$query'; rc=\$?; [ \$rc -eq 0 ] || [ \$rc -eq 56 ]");
+        is($content, "$marker\n", 'downloaded file has the content from the backup');
+    },
+)->run();
+
+done_testing();
diff --git a/tests/testcases.ron b/tests/testcases.ron
index b107835..51ceb81 100644
--- a/tests/testcases.ron
+++ b/tests/testcases.ron
@@ -293,6 +293,16 @@ TestcaseConfig(
             ],
             tag: "pbs",
         ),
+        Testcase(
+            name: "pbs_file_restore",
+            run: "./tests/pbs/test_pbs_file_restore.pl",
+            timeout: 600,
+            instances: [
+                "pve1",
+                "pbs1",
+            ],
+            tag: "pbs",
+        ),
         Testcase(
             name: "sdn_create_vlan_zone",
             run: "./tests/sdn/test_vlan_zone.pl",
-- 
2.47.3