Re: [pbs-devel] [PATCH proxmox-backup] pxar: bin: rework and dynamically generate `list` test data

[Gabriel Goller <g.goller@proxmox.com>]

On 10.09.2024 12:28, Christian Ebner wrote:
>Commit f16c5de757 ("pxar: bin: test `pxar list` with payload-input")
>introduced a regression test for listing of split pxar archives. This
>test relies on a large pxar blob file, the large size (> 100M) being
>overlooked when writing the test.
>
>In order to not depend on this file any further in the future, drop
>it and rewrite the test to dynamically generate the files, needed and
>further extend the test thereby also cover the archive creation and
>extraction for split pxar archives.
>
>Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
>---
> pxar-bin/tests/pxar.rs                       | 210 ++++++++-----------
> tests/pxar/backup-client-pxar-expected.ppxar | Bin 104859264 -> 0 bytes
> 2 files changed, 92 insertions(+), 118 deletions(-)
> delete mode 100644 tests/pxar/backup-client-pxar-expected.ppxar
>
>diff --git a/pxar-bin/tests/pxar.rs b/pxar-bin/tests/pxar.rs
>index e15c2e647..a1fde4b60 100644
>--- a/pxar-bin/tests/pxar.rs
>+++ b/pxar-bin/tests/pxar.rs
>@@ -82,7 +82,10 @@ fn pxar_create_and_extract() {
> }
>
> #[test]
>-fn pxar_list_with_payload_input() {
>+fn pxar_split_archive_test() {
>+    let src_dir = "../tests/catar_data/";
>+    let dest_dir = "../tests/catar_data/";
>+
>     let target_subdir = std::env::var("DEB_HOST_RUST_TYPE").unwrap_or_default();
>
>     let exec_path = if cfg!(debug_assertions) {
>@@ -91,129 +94,100 @@ fn pxar_list_with_payload_input() {
>         format!("../target/{target_subdir}/release/pxar")
>     };
>
>-    let output = Command::new(exec_path)
>-        .args([
>-            "list",
>-            "../tests/pxar/backup-client-pxar-expected.mpxar",
>-            "--payload-input",
>-            "../tests/pxar/backup-client-pxar-expected.ppxar",
>-        ])
>+    println!("run '{exec_path} create archive.mpxar {src_dir} --payload-output archive.ppxar'");
>+
>+    Command::new(&exec_path)
>+        .arg("create")
>+        .arg("./tests/archive.mpxar")
>+        .arg(src_dir)
>+        .arg("--payload-output=./tests/archive.ppxar")
>+        .status()
>+        .unwrap_or_else(|err| panic!("Failed to invoke '{exec_path}': {err}"));
>+
>+    let output = Command::new(&exec_path)
>+        .arg("list")
>+        .arg("./tests/archive.mpxar")
>+        .arg("--payload-input=./tests/archive.ppxar")
>         .output()
>         .expect("failed to run pxar list");
>     assert!(output.status.success());
>
>     let expected = "\"/\"
>-\"/folder_0\"
>-\"/folder_0/file_0\"
>-\"/folder_0/file_1\"
>-\"/folder_0/file_2\"
>-\"/folder_0/file_3\"
>-\"/folder_0/file_4\"
>-\"/folder_0/file_5\"
>-\"/folder_0/file_6\"
>-\"/folder_0/file_7\"
>-\"/folder_0/file_8\"
>-\"/folder_0/file_9\"
>-\"/folder_1\"
>-\"/folder_1/file_0\"
>-\"/folder_1/file_1\"
>-\"/folder_1/file_2\"
>-\"/folder_1/file_3\"
>-\"/folder_1/file_4\"
>-\"/folder_1/file_5\"
>-\"/folder_1/file_6\"
>-\"/folder_1/file_7\"
>-\"/folder_1/file_8\"
>-\"/folder_1/file_9\"
>-\"/folder_2\"
>-\"/folder_2/file_0\"
>-\"/folder_2/file_1\"
>-\"/folder_2/file_2\"
>-\"/folder_2/file_3\"
>-\"/folder_2/file_4\"
>-\"/folder_2/file_5\"
>-\"/folder_2/file_6\"
>-\"/folder_2/file_7\"
>-\"/folder_2/file_8\"
>-\"/folder_2/file_9\"
>-\"/folder_3\"
>-\"/folder_3/file_0\"
>-\"/folder_3/file_1\"
>-\"/folder_3/file_2\"
>-\"/folder_3/file_3\"
>-\"/folder_3/file_4\"
>-\"/folder_3/file_5\"
>-\"/folder_3/file_6\"
>-\"/folder_3/file_7\"
>-\"/folder_3/file_8\"
>-\"/folder_3/file_9\"
>-\"/folder_4\"
>-\"/folder_4/file_0\"
>-\"/folder_4/file_1\"
>-\"/folder_4/file_2\"
>-\"/folder_4/file_3\"
>-\"/folder_4/file_4\"
>-\"/folder_4/file_5\"
>-\"/folder_4/file_6\"
>-\"/folder_4/file_7\"
>-\"/folder_4/file_8\"
>-\"/folder_4/file_9\"
>-\"/folder_5\"
>-\"/folder_5/file_0\"
>-\"/folder_5/file_1\"
>-\"/folder_5/file_2\"
>-\"/folder_5/file_3\"
>-\"/folder_5/file_4\"
>-\"/folder_5/file_5\"
>-\"/folder_5/file_6\"
>-\"/folder_5/file_7\"
>-\"/folder_5/file_8\"
>-\"/folder_5/file_9\"
>-\"/folder_6\"
>-\"/folder_6/file_0\"
>-\"/folder_6/file_1\"
>-\"/folder_6/file_2\"
>-\"/folder_6/file_3\"
>-\"/folder_6/file_4\"
>-\"/folder_6/file_5\"
>-\"/folder_6/file_6\"
>-\"/folder_6/file_7\"
>-\"/folder_6/file_8\"
>-\"/folder_6/file_9\"
>-\"/folder_7\"
>-\"/folder_7/file_0\"
>-\"/folder_7/file_1\"
>-\"/folder_7/file_2\"
>-\"/folder_7/file_3\"
>-\"/folder_7/file_4\"
>-\"/folder_7/file_5\"
>-\"/folder_7/file_6\"
>-\"/folder_7/file_7\"
>-\"/folder_7/file_8\"
>-\"/folder_7/file_9\"
>-\"/folder_8\"
>-\"/folder_8/file_0\"
>-\"/folder_8/file_1\"
>-\"/folder_8/file_2\"
>-\"/folder_8/file_3\"
>-\"/folder_8/file_4\"
>-\"/folder_8/file_5\"
>-\"/folder_8/file_6\"
>-\"/folder_8/file_7\"
>-\"/folder_8/file_8\"
>-\"/folder_8/file_9\"
>-\"/folder_9\"
>-\"/folder_9/file_0\"
>-\"/folder_9/file_1\"
>-\"/folder_9/file_2\"
>-\"/folder_9/file_3\"
>-\"/folder_9/file_4\"
>-\"/folder_9/file_5\"
>-\"/folder_9/file_6\"
>-\"/folder_9/file_7\"
>-\"/folder_9/file_8\"
>-\"/folder_9/file_9\"
>+\"/test_file\"
>+\"/test_file/file1\"
>+\"/test_files_and_subdirs\"
>+\"/test_files_and_subdirs/a-test-symlink\"
>+\"/test_files_and_subdirs/file1\"
>+\"/test_files_and_subdirs/file2\"
>+\"/test_files_and_subdirs/subdir1\"
>+\"/test_files_and_subdirs/subdir1/subfile1\"
>+\"/test_files_and_subdirs/subdir1/subfile2\"
>+\"/test_goodbye_sort_order\"
>+\"/test_goodbye_sort_order/file1\"
>+\"/test_goodbye_sort_order/file2\"
>+\"/test_goodbye_sort_order/file3\"
>+\"/test_goodbye_sort_order/file4\"
>+\"/test_symlink\"
>+\"/test_symlink/symlink1\"
>+\"/test_xattrs_src\"
>+\"/test_xattrs_src/file.txt\"
> ";
>
>     assert_eq!(expected.as_bytes(), output.stdout);
>+
>+    println!("run '{exec_path} extract archive.mpxar {dest_dir} --payload-input archive.ppxar'");
>+
>+    Command::new(&exec_path)
>+        .arg("extract")
>+        .arg("./tests/archive.mpxar")
>+        .arg("--payload-input=./tests/archive.ppxar")
>+        .arg("--target")
>+        .arg(dest_dir)
>+        .status()
>+        .unwrap_or_else(|err| panic!("Failed to invoke '{exec_path}': {err}"));
>+
>+    println!("run 'rsync --dry-run --itemize-changes --archive {src_dir} {dest_dir}' to verify'");
>+
>+    /* Use rsync with --dry-run and --itemize-changes to compare
>+    src_dir and dest_dir */
>+    let stdout = Command::new("rsync")
>+        .arg("--dry-run")
>+        .arg("--itemize-changes")
>+        .arg("--archive")
>+        .arg(src_dir)
>+        .arg(dest_dir)
>+        .stdout(Stdio::piped())
>+        .spawn()
>+        .unwrap()
>+        .stdout
>+        .unwrap();
>+
>+    let reader = BufReader::new(stdout);
>+    let line_iter = reader.lines().map(|l| l.unwrap());
>+    let mut linecount = 0;
>+    for curr in line_iter {
>+        println!("{curr}");
>+        linecount += 1;
>+    }
>+    println!("Rsync listed {linecount} differences to address");
>+
>+    // Cleanup archive
>+    Command::new("rm")
>+        .arg("./tests/archive.mpxar")
>+        .arg("./tests/archive.ppxar")
>+        .status()
>+        .unwrap_or_else(|err| panic!("Failed to invoke 'rm': {err}"));
>+
>+    // Cleanup destination dir
>+    Command::new("rm")
>+        .arg("-r")
>+        .arg(dest_dir)
>+        .status()
>+        .unwrap_or_else(|err| panic!("Failed to invoke 'rm': {err}"));
>+
>+    // If source and destination folder contain the same content,
>+    // the output of the rsync invocation should yield no lines.
>+    if linecount != 0 {
>+        panic!("pxar create and extract did not yield the same contents");
>+    }
> }

Both tests do not work for me. It seems that the test folders/files are
missing?

And also:
  - the `catar_data` folder does not exist, the tests need to create it
    (and delete afterwords)
  - the `pxar_split_archive_test` test does not correctly clean up
    afterwards and leaves the two ppxar and mpxar files in the `tests`
    folder.


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel