[pbs-devel] [PATCH proxmox-backup 1/2] add inspection of chunk files

[Hannes Laimer <h.laimer@proxmox.com>]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/bin/proxmox-backup-manager.rs         |   8 +
 src/bin/proxmox_backup_manager/inspect.rs | 201 ++++++++++++++++++++++
 src/bin/proxmox_backup_manager/mod.rs     |   2 +
 3 files changed, 211 insertions(+)
 create mode 100644 src/bin/proxmox_backup_manager/inspect.rs

diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 8ad4c7dc..b2ef08de 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -13,6 +13,13 @@ use proxmox_backup::client::*;
 
 mod proxmox_backup_manager;
 use proxmox_backup_manager::*;
+use proxmox::api::schema::{Schema, StringSchema};
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
+    "Path to encryption key. If the data was encrypted, this key will be used for decryption.")
+    .schema();
 
 async fn view_task_result(
     client: HttpClient,
@@ -386,6 +393,7 @@ fn main() {
         .insert("datastore", datastore_commands())
         .insert("disk", disk_commands())
         .insert("dns", dns_commands())
+        .insert("inspect", inspect_commands())
         .insert("network", network_commands())
         .insert("user", user_commands())
         .insert("remote", remote_commands())
diff --git a/src/bin/proxmox_backup_manager/inspect.rs b/src/bin/proxmox_backup_manager/inspect.rs
new file mode 100644
index 00000000..d9fa736e
--- /dev/null
+++ b/src/bin/proxmox_backup_manager/inspect.rs
@@ -0,0 +1,201 @@
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::Error;
+use proxmox::api::cli::{
+    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*, RpcEnvironment};
+use proxmox::sys::linux::tty;
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::api2::types::SHA256_HEX_REGEX;
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{KEYFILE_SCHEMA, PATH_SCHEMA};
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+    tty::read_password("Encryption Key Password: ")
+}
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+    output_path: Option<&Path>,
+    key_file: Option<&Path>,
+    digest: Option<&[u8; 32]>,
+    blob: &DataBlob,
+) -> Result<(), Error> {
+    let mut crypt_conf_opt = None;
+    let crypt_conf;
+
+    if blob.is_encrypted() && key_file.is_some() {
+        let (key, _created, _fingerprint) =
+            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+        crypt_conf = CryptConfig::new(key)?;
+        crypt_conf_opt = Some(&crypt_conf);
+    }
+
+    if output_path.is_some() {
+        let mut file = File::create(output_path.unwrap())?;
+        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
+    } else {
+        Ok(println!(
+            "{}",
+            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
+        ))
+    }
+}
+
+#[api(
+    input: {
+        properties: {
+            chunk: {
+                schema: PATH_SCHEMA,
+            },
+            "reference-filter": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a chunk
+fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
+    let output_format = get_output_format(&param);
+    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+
+    if !SHA256_HEX_REGEX.is_match(digest_str) {
+        println!("chunk filename is not valid");
+        return Ok(Value::Null);
+    }
+
+    let digest_raw = proxmox::tools::hex_to_digest(digest_str)?;
+
+    let reference_filter_param = param["reference-filter"].as_str();
+    let decode_output_param = param["decode"].as_str();
+    let key_file_param = param["keyfile"].as_str();
+
+    let mut search_path = None;
+    let mut decode_output_path = None;
+    let mut key_file_path = None;
+    let mut to_stdout = false;
+
+    if let Some(path) = reference_filter_param {
+        search_path = Some(Path::new(path))
+    };
+
+    if let Some(path) = decode_output_param {
+        to_stdout = path.eq("-");
+        decode_output_path = Some(Path::new(path))
+    };
+
+    if let Some(path) = key_file_param {
+        key_file_path = Some(Path::new(path))
+    };
+
+    let mut file = std::fs::File::open(&chunk_path)?;
+    let blob = DataBlob::load_from_reader(&mut file)?;
+
+    let mut referenced_by = None;
+    if let Some(search_path) = search_path {
+        let mut references = Vec::new();
+        for entry in WalkDir::new(search_path)
+            .follow_links(false)
+            .into_iter()
+            .filter_map(|e| e.ok())
+        {
+            let file_name = entry.file_name().to_string_lossy();
+            let mut in_index = HashSet::new();
+            let mut index: Option<Box<dyn IndexFile>> = None;
+
+            if file_name.ends_with(".fidx") {
+                index = match FixedIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if file_name.ends_with(".didx") {
+                index = match DynamicIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if let Some(index) = index {
+                for pos in 0..index.index_count() {
+                    if let Some(index_chunk_digest) = index.index_digest(pos) {
+                        in_index.insert(proxmox::tools::digest_to_hex(index_chunk_digest));
+                    }
+                }
+            }
+
+            if in_index.contains(digest_str) {
+                references.push(entry.path().to_string_lossy().into_owned());
+            }
+        }
+        referenced_by = Some(references);
+    }
+
+    if let Some(decode_output_path) = decode_output_path {
+        if to_stdout {
+            decode_blob(None, key_file_path, Some(&digest_raw), &blob)?;
+        } else {
+            decode_blob(
+                Some(decode_output_path),
+                key_file_path,
+                Some(&digest_raw),
+                &blob,
+            )?;
+        }
+    }
+
+    let crc_status = format!(
+        "{}({})",
+        blob.compute_crc(),
+        blob.verify_crc().map_or("NOK", |_x| "OK")
+    );
+
+    let val = match referenced_by {
+        Some(references) => json!({
+            "digest": digest_str,
+            "crc": crc_status,
+            "encryption": blob.crypt_mode()?,
+            "referenced-by": references
+        }),
+        None => json!({
+             "digest": digest_str,
+             "crc": crc_status,
+             "encryption": blob.crypt_mode()?,
+        }),
+    };
+
+    format_and_print_result(&val, &output_format);
+    Ok(Value::Null)
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new()
+        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+    cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_manager/mod.rs b/src/bin/proxmox_backup_manager/mod.rs
index 1f3ff92e..7b2854d7 100644
--- a/src/bin/proxmox_backup_manager/mod.rs
+++ b/src/bin/proxmox_backup_manager/mod.rs
@@ -6,6 +6,8 @@ mod datastore;
 pub use datastore::*;
 mod dns;
 pub use dns::*;
+mod inspect;
+pub use inspect::*;
 mod network;
 pub use network::*;
 mod remote;
-- 
2.20.1