From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id EE805B937C for ; Wed, 13 Mar 2024 12:48:55 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id C823A3535C for ; Wed, 13 Mar 2024 12:48:25 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 13 Mar 2024 12:48:24 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 6B29243F1D for ; Wed, 13 Mar 2024 12:48:24 +0100 (CET) Date: Wed, 13 Mar 2024 12:48:16 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20240305092703.126906-1-c.ebner@proxmox.com> <20240305092703.126906-36-c.ebner@proxmox.com> In-Reply-To: <20240305092703.126906-36-c.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1710330466.1wq075nzib.astroid@yuna.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.064 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: Re: [pbs-devel] [RFC v2 proxmox-backup 35/36] test-suite: add detection mode change benchmark X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 13 Mar 2024 11:48:56 -0000 On March 5, 2024 10:27 am, Christian Ebner wrote: > Introduces the proxmox-backup-test-suite create intended for > benchmarking and high level user facing testing. >=20 > The initial code includes a benchmark intended for regression testing of > the proxmox-backup-client when using different file detection modes > during backup. >=20 > Signed-off-by: Christian Ebner > --- > changes since version 1: > - no changes >=20 > Cargo.toml | 1 + > proxmox-backup-test-suite/Cargo.toml | 18 ++ > .../src/detection_mode_bench.rs | 294 ++++++++++++++++++ > proxmox-backup-test-suite/src/main.rs | 17 + > 4 files changed, 330 insertions(+) > create mode 100644 proxmox-backup-test-suite/Cargo.toml > create mode 100644 proxmox-backup-test-suite/src/detection_mode_bench.rs > create mode 100644 proxmox-backup-test-suite/src/main.rs >=20 > diff --git a/Cargo.toml b/Cargo.toml > index 00dc4d86..76635b4e 100644 > --- a/Cargo.toml > +++ b/Cargo.toml > @@ -45,6 +45,7 @@ members =3D [ > "proxmox-restore-daemon", > =20 > "pxar-bin", > + "proxmox-backup-test-suite", > ] > =20 > [lib] > diff --git a/proxmox-backup-test-suite/Cargo.toml b/proxmox-backup-test-s= uite/Cargo.toml > new file mode 100644 > index 00000000..3f899e9b > --- /dev/null > +++ b/proxmox-backup-test-suite/Cargo.toml > @@ -0,0 +1,18 @@ > +[package] > +name =3D "proxmox-backup-test-suite" > +version =3D "0.1.0" > +authors.workspace =3D true > +edition.workspace =3D true > + > +[dependencies] > +anyhow.workspace =3D true > +futures.workspace =3D true > +serde.workspace =3D true > +serde_json.workspace =3D true > + > +pbs-client.workspace =3D true > +pbs-key-config.workspace =3D true > +pbs-tools.workspace =3D true > +proxmox-async.workspace =3D true > +proxmox-router =3D { workspace =3D true, features =3D ["cli"] } > +proxmox-schema =3D { workspace =3D true, features =3D [ "api-macro" ] } > diff --git a/proxmox-backup-test-suite/src/detection_mode_bench.rs b/prox= mox-backup-test-suite/src/detection_mode_bench.rs > new file mode 100644 > index 00000000..9a3c7680 > --- /dev/null > +++ b/proxmox-backup-test-suite/src/detection_mode_bench.rs > @@ -0,0 +1,294 @@ > +use std::path::Path; > +use std::process::Command; > +use std::{thread, time}; > + > +use anyhow::{bail, format_err, Error}; > +use serde_json::Value; > + > +use pbs_client::{ > + tools::{complete_repository, key_source::KEYFILE_SCHEMA, REPO_URL_SC= HEMA}, > + BACKUP_SOURCE_SCHEMA, > +}; > +use pbs_tools::json; > +use proxmox_router::cli::*; > +use proxmox_schema::api; > + > +const DEFAULT_NUMBER_OF_RUNS: u64 =3D 5; > +// Homepage https://cocodataset.org/ > +const COCO_DATASET_SRC_URL: &'static str =3D "http://images.cocodataset.= org/zips/unlabeled2017.zip"; > +// Homepage https://kernel.org/ > +const LINUX_GIT_REPOSITORY: &'static str =3D > + "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git"; > +const LINUX_GIT_TAG: &'static str =3D "v6.5.5"; > + > +pub(crate) fn detection_mode_bench_mgtm_cli() -> CliCommandMap { > + let run_cmd_def =3D CliCommand::new(&API_METHOD_DETECTION_MODE_BENCH= _RUN) > + .arg_param(&["backupspec"]) > + .completion_cb("repository", complete_repository) > + .completion_cb("keyfile", complete_file_name); > + > + let prepare_cmd_def =3D CliCommand::new(&API_METHOD_DETECTION_MODE_B= ENCH_PREPARE); > + CliCommandMap::new() > + .insert("prepare", prepare_cmd_def) > + .insert("run", run_cmd_def) > +} > + > +#[api( > + input: { > + properties: { > + backupspec: { > + type: Array, > + description: "List of backup source specifications ([:] ...)", > + items: { > + schema: BACKUP_SOURCE_SCHEMA, > + } > + }, > + repository: { > + schema: REPO_URL_SCHEMA, > + optional: true, > + }, > + keyfile: { > + schema: KEYFILE_SCHEMA, > + optional: true, > + }, > + "number-of-runs": { > + description: "Number of times to repeat the run", > + type: Integer, > + optional: true, > + }, > + } > + } > +)] > +/// Run benchmark to compare performance for backups using different cha= nge detection modes. > +fn detection_mode_bench_run(param: Value) -> Result<(), Error> { > + let mut pbc =3D Command::new("proxmox-backup-client"); > + pbc.arg("backup"); > + > + let backupspec_list =3D json::required_array_param(¶m, "backupsp= ec")?; > + for backupspec in backupspec_list { > + let arg =3D backupspec > + .as_str() > + .ok_or_else(|| format_err!("failed to parse backupspec"))?; > + pbc.arg(arg); > + } > + > + if let Some(repo) =3D param["repository"].as_str() { > + pbc.arg("--repository"); > + pbc.arg::<&str>(repo); > + } > + > + if let Some(keyfile) =3D param["keyfile"].as_str() { > + pbc.arg("--keyfile"); > + pbc.arg::<&str>(keyfile); > + } > + > + let number_of_runs =3D match param["number_of_runs"].as_u64() { > + Some(n) =3D> n, > + None =3D> DEFAULT_NUMBER_OF_RUNS, > + }; > + if number_of_runs < 1 { > + bail!("Number of runs must be greater than 1, aborting."); > + } > + > + // First run is an initial run to make sure all chunks are present a= lready, reduce side effects > + // by filesystem caches ecc. > + let _stats_initial =3D do_run(&mut pbc, 1)?; this run here > + > + println!("\nStarting benchmarking backups with regular detection mod= e...\n"); > + let stats_reg =3D do_run(&mut pbc, number_of_runs)?; > + > + // Make sure to have a valid reference with catalog fromat version 2 > + pbc.arg("--change-detection-mode=3Dmetadata"); > + let _stats_initial =3D do_run(&mut pbc, 1)?; and this run here make the output a bit confusing to read, maybe they could get their own intro line? > + println!("\nStarting benchmarking backups with metadata detection mo= de...\n"); > + let stats_meta =3D do_run(&mut pbc, number_of_runs)?; > + > + println!("\nCompleted benchmark with {number_of_runs} runs for each = tested mode."); > + println!("\nCompleted regular backup with:"); > + println!("Total runtime: {:.2} s", stats_reg.total); > + println!("Average: {:.2} =C2=B1 {:.2} s", stats_reg.avg, stats_reg.s= tddev); > + println!("Min: {:.2} s", stats_reg.min); > + println!("Max: {:.2} s", stats_reg.max); > + > + println!("\nCompleted metadata detection mode backup with:"); > + println!("Total runtime: {:.2} s", stats_meta.total); > + println!( > + "Average: {:.2} =C2=B1 {:.2} s", > + stats_meta.avg, stats_meta.stddev > + ); > + println!("Min: {:.2} s", stats_meta.min); > + println!("Max: {:.2} s", stats_meta.max); > + > + let diff_stddev =3D > + ((stats_meta.stddev * stats_meta.stddev) + (stats_reg.stddev * s= tats_reg.stddev)).sqrt(); > + println!("\nDifferences (metadata based - regular):"); > + println!( > + "Delta total runtime: {:.2} s ({:.2} %)", > + stats_meta.total - stats_reg.total, > + 100.0 * (stats_meta.total / stats_reg.total - 1.0), > + ); > + println!( > + "Delta average: {:.2} =C2=B1 {:.2} s ({:.2} %)", > + stats_meta.avg - stats_reg.avg, > + diff_stddev, > + 100.0 * (stats_meta.avg / stats_reg.avg - 1.0), > + ); > + println!( > + "Delta min: {:.2} s ({:.2} %)", > + stats_meta.min - stats_reg.min, > + 100.0 * (stats_meta.min / stats_reg.min - 1.0), > + ); > + println!( > + "Delta max: {:.2} s ({:.2} %)", > + stats_meta.max - stats_reg.max, > + 100.0 * (stats_meta.max / stats_reg.max - 1.0), > + ); > + > + Ok(()) > +} > + > +fn do_run(cmd: &mut Command, n_runs: u64) -> Result { > + // Avoid consecutive snapshot timestamps collision > + thread::sleep(time::Duration::from_millis(1000)); > + let mut timings =3D Vec::with_capacity(n_runs as usize); > + for iteration in 1..n_runs + 1 { > + let start =3D std::time::SystemTime::now(); > + let mut child =3D cmd.spawn()?; > + let exit_code =3D child.wait()?; > + let elapsed =3D start.elapsed()?; > + timings.push(elapsed); > + if !exit_code.success() { > + bail!("Run number {iteration} of {n_runs} failed, aborting."= ); > + } > + } > + > + Ok(statistics(timings)) > +} > + > +struct Statistics { > + total: f64, > + avg: f64, > + stddev: f64, > + min: f64, > + max: f64, > +} > + > +fn statistics(timings: Vec) -> Statistics { > + let total =3D timings > + .iter() > + .fold(0f64, |sum, time| sum + time.as_secs_f64()); > + let avg =3D total / timings.len() as f64; > + let var =3D 1f64 / (timings.len() - 1) as f64 > + * timings.iter().fold(0f64, |sq_sum, time| { > + let diff =3D time.as_secs_f64() - avg; > + sq_sum + diff * diff > + }); > + let stddev =3D var.sqrt(); > + let min =3D timings.iter().min().unwrap().as_secs_f64(); > + let max =3D timings.iter().max().unwrap().as_secs_f64(); > + > + Statistics { > + total, > + avg, > + stddev, > + min, > + max, > + } > +} > + > +#[api( > + input: { > + properties: { > + target: { > + description: "target path to prepare test data.", > + }, > + }, > + }, > +)] > +/// Prepare files required for detection mode backup benchmarks. > +fn detection_mode_bench_prepare(target: String) -> Result<(), Error> { > + let linux_repo_target =3D format!("{target}/linux"); > + let coco_dataset_target =3D format!("{target}/coco"); > + git_clone(LINUX_GIT_REPOSITORY, linux_repo_target.as_str())?; > + git_checkout(LINUX_GIT_TAG, linux_repo_target.as_str())?; > + wget_download(COCO_DATASET_SRC_URL, coco_dataset_target.as_str())?; > + > + Ok(()) > +} > + > +fn git_clone(repo: &str, target: &str) -> Result<(), Error> { > + println!("Calling git clone for '{repo}'."); > + let target_git =3D format!("{target}/.git"); > + let path =3D Path::new(&target_git); > + if let Ok(true) =3D path.try_exists() { > + println!("Target '{target}' already contains a git repository, s= kip."); > + return Ok(()); > + } > + > + let mut git =3D Command::new("git"); > + git.args(["clone", repo, target]); > + > + let mut child =3D git.spawn()?; > + let exit_code =3D child.wait()?; > + if exit_code.success() { > + println!("git clone finished with success."); > + } else { > + bail!("git clone failed for '{target}'."); > + } > + > + Ok(()) > +} > + > +fn git_checkout(checkout_target: &str, target: &str) -> Result<(), Error= > { > + println!("Calling git checkout '{checkout_target}'."); > + let mut git =3D Command::new("git"); > + git.args(["-C", target, "checkout", checkout_target]); > + > + let mut child =3D git.spawn()?; > + let exit_code =3D child.wait()?; > + if exit_code.success() { > + println!("git checkout finished with success."); > + } else { > + bail!("git checkout '{checkout_target}' failed for '{target}'.")= ; > + } > + Ok(()) > +} > + > +fn wget_download(source_url: &str, target: &str) -> Result<(), Error> { > + let path =3D Path::new(&target); > + if let Ok(true) =3D path.try_exists() { > + println!("Target '{target}' already exists, skip."); > + return Ok(()); > + } > + let zip =3D format!("{}/unlabeled2017.zip", target); > + let path =3D Path::new(&zip); > + if !path.try_exists()? { > + println!("Download archive using wget from '{source_url}' to '{t= arget}'."); > + let mut wget =3D Command::new("wget"); > + wget.args(["-P", target, source_url]); > + > + let mut child =3D wget.spawn()?; > + let exit_code =3D child.wait()?; > + if exit_code.success() { > + println!("Download finished with success."); > + } else { > + bail!("Failed to download '{source_url}' to '{target}'."); > + } > + return Ok(()); > + } else { > + println!("Target '{target}' already contains download, skip down= load."); > + } > + > + let mut unzip =3D Command::new("unzip"); > + unzip.args([&zip, "-d", target]); > + > + let mut child =3D unzip.spawn()?; > + let exit_code =3D child.wait()?; > + if exit_code.success() { > + println!("Extracting zip archive finished with success."); > + } else { > + bail!("Failed to extract zip archive '{zip}' to '{target}'."); > + } > + Ok(()) > +} > diff --git a/proxmox-backup-test-suite/src/main.rs b/proxmox-backup-test-= suite/src/main.rs > new file mode 100644 > index 00000000..0a5b436a > --- /dev/null > +++ b/proxmox-backup-test-suite/src/main.rs > @@ -0,0 +1,17 @@ > +use proxmox_router::cli::*; > + > +mod detection_mode_bench; > + > +fn main() { > + let cmd_def =3D CliCommandMap::new().insert( > + "detection-mode-bench", > + detection_mode_bench::detection_mode_bench_mgtm_cli(), > + ); > + > + let rpcenv =3D CliEnvironment::new(); > + run_cli_command( > + cmd_def, > + rpcenv, > + Some(|future| proxmox_async::runtime::main(future)), > + ); > +} > --=20 > 2.39.2 >=20 >=20 >=20 > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >=20