Re: [pbs-devel] [PATCH v3 proxmox-backup 2/3] add file(.blob, .fidx, .didx) inspection to pb-debug

[Wolfgang Bumiller <w.bumiller@proxmox.com>]

On Fri, Feb 05, 2021 at 08:58:05AM +0100, Hannes Laimer wrote:
> Adds possibility to inspect .blob, .fidx and .didx files. For index
> files a list of the chunks referenced will be printed in addition to
> some other inforation. .blob files can be decoded into file or directly
> into stdout. Options:
>  - file: path to the file
>  - [opt] decode: path to a file or stdout(-), if specidied, the file will be
>    decoded into the specified location [only for blob files, no effect
>    with index files]
>  - [opt] keyfile: path to a keyfile, needed if decode is specified and the
>    data was encrypted
> 
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---
> 
>  src/bin/proxmox_backup_debug/inspect.rs | 113 +++++++++++++++++++++++-
>  1 file changed, 111 insertions(+), 2 deletions(-)
> 
> diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
> index a66818be..367d01c2 100644
> --- a/src/bin/proxmox_backup_debug/inspect.rs
> +++ b/src/bin/proxmox_backup_debug/inspect.rs
> @@ -1,8 +1,9 @@
> +use std::collections::HashSet;
>  use std::fs::File;
>  use std::io::Write;
>  use std::path::Path;
>  
> -use anyhow::{format_err, Error};
> +use anyhow::{bail, format_err, Error};
>  use proxmox::api::cli::{
>      format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
>  };
> @@ -163,8 +164,116 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
>      Ok(Value::Null)
>  }
>  
> +#[api(
> +    input: {
> +        properties: {
> +            file: {
> +                schema: PATH_SCHEMA,
> +            },
> +            "decode": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "keyfile": {
> +                schema: KEYFILE_SCHEMA,
> +                optional: true,
> +            },
> +            "output-format": {
> +                schema: OUTPUT_FORMAT,
> +                optional: true,
> +            },
> +        }
> +    }
> +)]
> +/// Inspect a file
> +fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {

^ same as in the other patch (fn parameters)

> +    let path = tools::required_string_param(&param, "file")?;
> +    let output_format = get_output_format(&param);
> +
> +    let val;
> +    if path.ends_with(".blob") {
> +        let decode_output_param = param["decode"].as_str();
> +        let key_file_param = param["keyfile"].as_str();
> +
> +        let mut file = std::fs::File::open(&path)
> +            .map_err(|e| format_err!("could not open blob file - {}", e))?;
> +        let data_blob = DataBlob::load_from_reader(&mut file)?;

Since `load_from_reader` takes a trait object, this is mostly fine,
but I *would* prefer to limit the file's scope to this call so it gets
closed early.

    load_from_reader(&mut File::open(&path).map_err(...)?)

> +
> +        let key_file_path = key_file_param.map(|path| Path::new(path));
> +
> +        let (decode_output_path, to_stdout) = match decode_output_param {
> +            Some(path) if path.eq("-") => (None, true),
> +            Some(path) => (Some(Path::new(path)), false),
> +            None => (None, false),
> +        };

^ same as in the other patch (call the "open stdout or path" helper fn)

> +
> +        let crypt_mode = data_blob.crypt_mode()?;
> +        val = json!({
> +            "encryption": crypt_mode,
> +            "raw_size": data_blob.raw_size(),
> +        });
> +
> +        if decode_output_path.is_some() || to_stdout {
> +            decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
> +        }
> +    } else if path.ends_with(".fidx") {
> +        let index = FixedIndexReader::open(Path::new(path))
> +            .map_err(|e| format_err!("could not open fidx file - {}", e))?;
> +
> +        let mut ctime_str = index.ctime.to_string();
> +        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
> +            ctime_str = s;
> +        }
> +
> +        let mut chunk_digests = HashSet::new();
> +
> +        for pos in 0..index.index_count() {
> +            let digest = index.index_digest(pos).unwrap();
> +            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
> +        }
> +
> +        val = json!({
> +            "size": index.size,
> +            "ctime": ctime_str,
> +            "chunk-digests": chunk_digests
> +
> +        })
> +    } else if path.ends_with("didx") {

^ misses the dot in the string

> +        let index = DynamicIndexReader::open(Path::new(path))
> +            .map_err(|e| format_err!("could not open didx file - {}", e))?;
> +
> +        let mut ctime_str = index.ctime.to_string();
> +        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
> +            ctime_str = s;
> +        }
> +
> +        let mut chunk_digests = HashSet::new();
> +
> +        for pos in 0..index.index_count() {
> +            let digest = index.index_digest(pos).unwrap();
> +            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
> +        }
> +
> +        val = json!({
> +            "size": index.size,
> +            "ctime": ctime_str,
> +            "chunk-digests": chunk_digests
> +        })
> +    } else {
> +        bail!("Only .blob, .fidx and .didx files may be inspected");
> +    }
> +
> +    if !val.is_null() {
> +        format_and_print_result(&val, &output_format);
> +    }
> +
> +    Ok(Value::Null)

Note that you can just let the method return `()` and use `Ok(())`.

> +}
> +
>  pub fn inspect_commands() -> CommandLineInterface {
> -    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
> +    let cmd_def = CliCommandMap::new()
> +        .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
> +        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
>  
>      cmd_def.into()
>  }
> -- 
> 2.20.1