Re: [pbs-devel] [PATCH vma-to-pbs 4/4] improve readability of stdout

["Fabian Grünbichler" <f.gruenbichler@proxmox.com>]

On July 24, 2024 6:18 pm, Filip Schauer wrote:
> Do not spam stdout with messages for every single chunk uploaded.
> Instead, show the progress as a percentage every 1000 chunks and improve
> overall readability.

seems good enough as a stop-gap measure, but maybe we want to switch to
tracing-based logging here? having the option to turn on debug or trace
level logging and for example, printing VMA offsets and chunk info might
be nice sometimes..

> Signed-off-by: Filip Schauer <f.schauer@proxmox.com>
> ---
>  src/vma2pbs.rs | 50 +++++++++++++++++++++++++++++++-------------------
>  1 file changed, 31 insertions(+), 19 deletions(-)
> 
> diff --git a/src/vma2pbs.rs b/src/vma2pbs.rs
> index 5c0d03f..560ebd0 100644
> --- a/src/vma2pbs.rs
> +++ b/src/vma2pbs.rs
> @@ -6,6 +6,7 @@ use std::fs::File;
>  use std::io::{stdin, BufRead, BufReader, Read};
>  use std::process::{Command, Stdio};
>  use std::ptr;
> +use std::sync::{Arc, Mutex};
>  use std::time::SystemTime;
>  
>  use anyhow::{anyhow, bail, Error};
> @@ -25,7 +26,7 @@ use proxmox_time::epoch_to_rfc3339;
>  use scopeguard::defer;
>  use serde_json::Value;
>  
> -use crate::vma::VmaReader;
> +use crate::vma::{VmaReader, VMA_MAX_DEVICES};
>  
>  const VMA_CLUSTER_SIZE: usize = 65536;
>  
> @@ -77,16 +78,14 @@ fn handle_pbs_error(pbs_err: *mut c_char, function_name: &str) -> Result<(), Err
>      bail!("{function_name} failed: {pbs_err_str}");
>  }
>  
> -fn create_pbs_backup_task(pbs_args: &PbsArgs, backup_args: &VmaBackupArgs) -> Result<*mut ProxmoxBackupHandle, Error> {
> -    println!("PBS repository: {}", pbs_args.pbs_repository);
> -    if let Some(ns) = &pbs_args.namespace {
> -        println!("PBS namespace: {}", ns);
> -    }
> -    println!("PBS fingerprint: {}", pbs_args.fingerprint);
> -    println!("compress: {}", pbs_args.compress);
> -    println!("encrypt: {}", pbs_args.encrypt);
> -
> -    println!("backup time: {}", epoch_to_rfc3339(backup_args.backup_time)?);
> +fn create_pbs_backup_task(
> +    pbs_args: &PbsArgs,
> +    backup_args: &VmaBackupArgs,
> +) -> Result<*mut ProxmoxBackupHandle, Error> {
> +    println!(
> +        "\tbackup time: {}",
> +        epoch_to_rfc3339(backup_args.backup_time)?
> +    );
>  
>      let mut pbs_err: *mut c_char = ptr::null_mut();
>  
> @@ -153,7 +152,7 @@ where
>          let config_name = config.name;
>          let config_data = config.content;
>  
> -        println!("CFG: size: {} name: {}", config_data.len(), config_name);
> +        println!("\tCFG: size: {} name: {}", config_data.len(), config_name);
>  
>          let config_name_cstr = CString::new(config_name)?;
>  
> @@ -192,7 +191,7 @@ where
>          let device_size = vma_reader.get_device_size(device_id)?;
>  
>          println!(
> -            "DEV: dev_id={} size: {} devname: {}",
> +            "\tDEV: dev_id={} size: {} devname: {}",
>              device_id, device_size, device_name
>          );
>  
> @@ -235,6 +234,8 @@ where
>          non_zero_mask: u64,
>      }
>  
> +    let chunk_stats = Arc::new(Mutex::new([0u64; VMA_MAX_DEVICES]));
> +
>      let images_chunks: RefCell<HashMap<u8, HashMap<u64, ImageChunk>>> =
>          RefCell::new(HashMap::new());
>  
> @@ -279,12 +280,14 @@ where
>          };
>  
>          let pbs_upload_chunk = |pbs_chunk_buffer: Option<&[u8]>| {
> -            println!(
> -                "Uploading dev_id: {} offset: {:#0X} - {:#0X}",
> -                dev_id,
> -                pbs_chunk_offset,
> -                pbs_chunk_offset + pbs_chunk_size,
> -            );
> +            let mut chunk_stats_locked = chunk_stats.lock().unwrap();
> +            chunk_stats_locked[dev_id as usize] += 1;
> +            if (chunk_stats_locked[dev_id as usize] % 1000) == 0 {
> +                let percentage =
> +                    100 * PROXMOX_BACKUP_DEFAULT_CHUNK_SIZE * chunk_stats_locked[dev_id as usize]
> +                        / device_size;
> +                println!("\tUploading dev_id: {} ({}%)", dev_id, percentage);
> +            }
>  
>              let mut pbs_err: *mut c_char = ptr::null_mut();
>  
> @@ -468,6 +471,15 @@ fn set_notes(
>  }
>  
>  pub fn vma2pbs(args: BackupVmaToPbsArgs) -> Result<(), Error> {
> +    let pbs_args = &args.pbs_args;
> +    println!("PBS repository: {}", pbs_args.pbs_repository);
> +    if let Some(ns) = &pbs_args.namespace {
> +        println!("PBS namespace: {}", ns);
> +    }
> +    println!("PBS fingerprint: {}", pbs_args.fingerprint);
> +    println!("compress: {}", pbs_args.compress);
> +    println!("encrypt: {}", pbs_args.encrypt);
> +
>      let start_transfer_time = SystemTime::now();
>  
>      for backup_args in args.vmas {
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 
> 
> 


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel