[pbs-devel] [PATCH v3 proxmox-backup 0/3] add proxmox-backup-debug binary

[pbs-devel] [PATCH v3 proxmox-backup 0/3] add proxmox-backup-debug binary

[Hannes Laimer]

Adds a new proxmox-backup-debug binary for recovery or inspections of
index and blob files.

v3:
  - move description into commit message

v2:
  - appiled suggestions + rewrote a few parts
  - renamed 'restore' to 'debug'

Hannes Laimer (3):
  add chunk inspection to pb-debug
  add file(.blob, .fidx, .didx) inspection to pb-debug
  add index recovery to pb-debug

 Makefile                                |   3 +-
 src/bin/proxmox-backup-debug.rs         |  35 +++
 src/bin/proxmox_backup_debug/inspect.rs | 279 ++++++++++++++++++++++++
 src/bin/proxmox_backup_debug/mod.rs     |   4 +
 src/bin/proxmox_backup_debug/recover.rs | 109 +++++++++
 5 files changed, 429 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-debug.rs
 create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
 create mode 100644 src/bin/proxmox_backup_debug/mod.rs
 create mode 100644 src/bin/proxmox_backup_debug/recover.rs

-- 
2.20.1

[pbs-devel] [PATCH v3 proxmox-backup 1/3] add chunk inspection to pb-debug

[Hannes Laimer]

Adds possibility to inspect chunks and find indexes that reference the
chunk. Options:
 - chunk: path to the chunk file
 - [opt] decode: path to a file or to stdout(-), if specified, the chunk will
   be decoded into the specified location
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted
 - [opt] reference-filter: path in which indexes that reference the chunk
   should be searched, can be a group, snapshot or the whole datastore,
   if not specified no references will be searched

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---

 Makefile                                |   3 +-
 src/bin/proxmox-backup-debug.rs         |  33 +++++
 src/bin/proxmox_backup_debug/inspect.rs | 170 ++++++++++++++++++++++++
 src/bin/proxmox_backup_debug/mod.rs     |   2 +
 4 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-debug.rs
 create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
 create mode 100644 src/bin/proxmox_backup_debug/mod.rs

diff --git a/Makefile b/Makefile
index a1af9d51..3c780bcf 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,8 @@ USR_BIN := \
 
 # Binaries usable by admins
 USR_SBIN := \
-	proxmox-backup-manager
+	proxmox-backup-manager \
+	proxmox-backup-debug \
 
 # Binaries for services:
 SERVICE_BIN := \
diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
new file mode 100644
index 00000000..079b2bc8
--- /dev/null
+++ b/src/bin/proxmox-backup-debug.rs
@@ -0,0 +1,33 @@
+use anyhow::Error;
+use proxmox::api::cli::*;
+use proxmox::api::schema::{Schema, StringSchema};
+
+use proxmox::sys::linux::tty;
+
+use proxmox_backup_debug::*;
+
+mod proxmox_backup_debug;
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+    tty::read_password("Encryption Key Password: ")
+}
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
+    "Path to encryption key. If the data was encrypted, this key will be used for decryption.",
+)
+.schema();
+
+fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
+    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+
+    let rpcenv = CliEnvironment::new();
+    run_cli_command(
+        cmd_def,
+        rpcenv,
+        Some(|future| proxmox_backup::tools::runtime::main(future)),
+    );
+}
diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
new file mode 100644
index 00000000..a66818be
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -0,0 +1,170 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::{format_err, Error};
+use proxmox::api::cli::{
+    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*, RpcEnvironment};
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+    output_path: Option<&Path>,
+    key_file: Option<&Path>,
+    digest: Option<&[u8; 32]>,
+    blob: &DataBlob,
+) -> Result<(), Error> {
+    let mut crypt_conf_opt = None;
+    let crypt_conf;
+
+    if blob.is_encrypted() && key_file.is_some() {
+        let (key, _created, _fingerprint) =
+            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+        crypt_conf = CryptConfig::new(key)?;
+        crypt_conf_opt = Some(&crypt_conf);
+    }
+
+    if output_path.is_some() {
+        let mut file = File::create(output_path.unwrap())?;
+        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
+    } else {
+        Ok(println!(
+            "{}",
+            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
+        ))
+    }
+}
+
+#[api(
+    input: {
+        properties: {
+            chunk: {
+                schema: PATH_SCHEMA,
+            },
+            "reference-filter": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a chunk
+fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
+    let output_format = get_output_format(&param);
+    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+    let digest_raw = proxmox::tools::hex_to_digest(digest_str)
+        .map_err(|e| format_err!("could not parse chunk - {}", e))?;
+
+    let reference_filter_param = param["reference-filter"].as_str();
+    let decode_output_param = param["decode"].as_str();
+    let key_file_param = param["keyfile"].as_str();
+
+    let search_path = reference_filter_param.map(|path| Path::new(path));
+    let key_file_path = key_file_param.map(|path| Path::new(path));
+
+    let (decode_output_path, to_stdout) = match decode_output_param {
+        Some(path) if path.eq("-") => (None, true),
+        Some(path) => (Some(Path::new(path)), false),
+        None => (None, false),
+    };
+
+    let mut file = std::fs::File::open(&chunk_path)
+        .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+    let blob = DataBlob::load_from_reader(&mut file)?;
+
+    let mut referenced_by = None;
+    if let Some(search_path) = search_path {
+        let mut references = Vec::new();
+        for entry in WalkDir::new(search_path)
+            .follow_links(false)
+            .into_iter()
+            .filter_map(|e| e.ok())
+        {
+            let file_name = entry.file_name().to_string_lossy();
+            let mut index: Option<Box<dyn IndexFile>> = None;
+
+            if file_name.ends_with(".fidx") {
+                index = match FixedIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if file_name.ends_with(".didx") {
+                index = match DynamicIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if let Some(index) = index {
+                for pos in 0..index.index_count() {
+                    if let Some(index_chunk_digest) = index.index_digest(pos) {
+                        let str = proxmox::tools::digest_to_hex(index_chunk_digest);
+                        if str.eq(digest_str) {
+                            references.push(entry.path().to_string_lossy().into_owned());
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        referenced_by = Some(references);
+    }
+
+    if decode_output_path.is_some() || to_stdout {
+        decode_blob(decode_output_path, key_file_path, Some(&digest_raw), &blob)?;
+    }
+
+    let crc_status = format!(
+        "{}({})",
+        blob.compute_crc(),
+        blob.verify_crc().map_or("NOK", |_| "OK")
+    );
+
+    let val = match referenced_by {
+        Some(references) => json!({
+            "digest": digest_str,
+            "crc": crc_status,
+            "encryption": blob.crypt_mode()?,
+            "referenced-by": references
+        }),
+        None => json!({
+             "digest": digest_str,
+             "crc": crc_status,
+             "encryption": blob.crypt_mode()?,
+        }),
+    };
+
+    format_and_print_result(&val, &output_format);
+    Ok(Value::Null)
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+    cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
new file mode 100644
index 00000000..644583db
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -0,0 +1,2 @@
+mod inspect;
+pub use inspect::*;
-- 
2.20.1

[pbs-devel] [PATCH v3 proxmox-backup 2/3] add file(.blob, .fidx, .didx) inspection to pb-debug

[Hannes Laimer]

Adds possibility to inspect .blob, .fidx and .didx files. For index
files a list of the chunks referenced will be printed in addition to
some other inforation. .blob files can be decoded into file or directly
into stdout. Options:
 - file: path to the file
 - [opt] decode: path to a file or stdout(-), if specidied, the file will be
   decoded into the specified location [only for blob files, no effect
   with index files]
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---

 src/bin/proxmox_backup_debug/inspect.rs | 113 +++++++++++++++++++++++-
 1 file changed, 111 insertions(+), 2 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
index a66818be..367d01c2 100644
--- a/src/bin/proxmox_backup_debug/inspect.rs
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -1,8 +1,9 @@
+use std::collections::HashSet;
 use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
 use proxmox::api::cli::{
     format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
 };
@@ -163,8 +164,116 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
     Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a file
+fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let path = tools::required_string_param(&param, "file")?;
+    let output_format = get_output_format(&param);
+
+    let val;
+    if path.ends_with(".blob") {
+        let decode_output_param = param["decode"].as_str();
+        let key_file_param = param["keyfile"].as_str();
+
+        let mut file = std::fs::File::open(&path)
+            .map_err(|e| format_err!("could not open blob file - {}", e))?;
+        let data_blob = DataBlob::load_from_reader(&mut file)?;
+
+        let key_file_path = key_file_param.map(|path| Path::new(path));
+
+        let (decode_output_path, to_stdout) = match decode_output_param {
+            Some(path) if path.eq("-") => (None, true),
+            Some(path) => (Some(Path::new(path)), false),
+            None => (None, false),
+        };
+
+        let crypt_mode = data_blob.crypt_mode()?;
+        val = json!({
+            "encryption": crypt_mode,
+            "raw_size": data_blob.raw_size(),
+        });
+
+        if decode_output_path.is_some() || to_stdout {
+            decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
+        }
+    } else if path.ends_with(".fidx") {
+        let index = FixedIndexReader::open(Path::new(path))
+            .map_err(|e| format_err!("could not open fidx file - {}", e))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+
+        })
+    } else if path.ends_with("didx") {
+        let index = DynamicIndexReader::open(Path::new(path))
+            .map_err(|e| format_err!("could not open didx file - {}", e))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+        })
+    } else {
+        bail!("Only .blob, .fidx and .didx files may be inspected");
+    }
+
+    if !val.is_null() {
+        format_and_print_result(&val, &output_format);
+    }
+
+    Ok(Value::Null)
+}
+
 pub fn inspect_commands() -> CommandLineInterface {
-    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+    let cmd_def = CliCommandMap::new()
+        .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
+        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
 
     cmd_def.into()
 }
-- 
2.20.1

[pbs-devel] [PATCH v3 proxmox-backup 3/3] add index recovery to pb-debug

[Hannes Laimer]

Adds possibility to recover data from an index file. Options:
 - chunks: path to the directory where the chunks are saved
 - file: the index file that should be recovered(must be either .fidx or
   didx)
 - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
   needed
 - [opt] skip-crc: boolean, if true, read chunks wont be verified with their
   crc-sum, increases the restore speed by a lot

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---

 src/bin/proxmox-backup-debug.rs         |   6 +-
 src/bin/proxmox_backup_debug/mod.rs     |   2 +
 src/bin/proxmox_backup_debug/recover.rs | 109 ++++++++++++++++++++++++
 3 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 src/bin/proxmox_backup_debug/recover.rs

diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
index 079b2bc8..8bd01a59 100644
--- a/src/bin/proxmox-backup-debug.rs
+++ b/src/bin/proxmox-backup-debug.rs
@@ -22,7 +22,9 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
 fn main() {
     proxmox_backup::tools::setup_safe_path_env();
 
-    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+    let cmd_def = CliCommandMap::new()
+        .insert("inspect", inspect_commands())
+        .insert("recover", recover_commands());
 
     let rpcenv = CliEnvironment::new();
     run_cli_command(
@@ -30,4 +32,4 @@ fn main() {
         rpcenv,
         Some(|future| proxmox_backup::tools::runtime::main(future)),
     );
-}
+}
\ No newline at end of file
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
index 644583db..62df7754 100644
--- a/src/bin/proxmox_backup_debug/mod.rs
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -1,2 +1,4 @@
 mod inspect;
 pub use inspect::*;
+mod recover;
+pub use recover::*;
diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
new file mode 100644
index 00000000..61025adf
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/recover.rs
@@ -0,0 +1,109 @@
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox::api::api;
+use proxmox::api::cli::{CliCommand, CliCommandMap, CommandLineInterface};
+use serde_json::Value;
+
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+use proxmox::tools::digest_to_hex;
+
+use std::time::Instant;
+
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            chunks: {
+                schema: PATH_SCHEMA,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "skip-crc": {
+                type: Boolean,
+                optional: true,
+                default: false,
+                description: "Skip the crc verification, increases the restore speed immensely",
+            }
+        }
+    }
+)]
+/// Recover a index file
+fn recover_index(skip_crc: bool, param: Value) -> Result<Value, Error> {
+    let start = Instant::now();
+    let file_path = Path::new(tools::required_string_param(&param, "file")?);
+    let chunks_path = Path::new(tools::required_string_param(&param, "chunks")?);
+
+    let key_file_param = param["keyfile"].as_str();
+    let key_file_path = key_file_param.map(|path| Path::new(path));
+
+    let index: Box<dyn IndexFile> = match file_path.extension() {
+        Some(ext) if ext.eq("fidx") => Box::new(
+            FixedIndexReader::open(file_path)
+                .map_err(|e| format_err!("could not read index - {}", e))?,
+        ),
+        Some(ext) if ext.eq("didx") => Box::new(
+            DynamicIndexReader::open(file_path)
+                .map_err(|e| format_err!("could not read index - {}", e))?,
+        ),
+        _ => bail!("index file must either be a .fidx or a .didx file"),
+    };
+
+    let mut crypt_conf_opt = None;
+    let mut crypt_conf;
+
+    let output_filename = file_path.file_stem().unwrap().to_str().unwrap();
+    let output_path = Path::new(output_filename);
+    let mut output_file = File::create(output_path)
+        .map_err(|e| format_err!("could not create output file - {}", e))?;
+
+    for pos in 0..index.index_count() {
+        let chunk_digest = index.index_digest(pos).unwrap();
+        let digest_str = digest_to_hex(chunk_digest);
+        let digest_prefix = &digest_str[0..4];
+        let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
+        let mut chunk_file = std::fs::File::open(&chunk_path)
+            .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+
+        let mut data = Vec::with_capacity(1024 * 1024);
+        chunk_file.read_to_end(&mut data)?;
+        let chunk_blob = DataBlob::from_raw(data)?;
+
+        if !skip_crc {
+            chunk_blob.verify_crc()?;
+        }
+
+        if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() {
+            let (key, _created, _fingerprint) =
+                load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?;
+            crypt_conf = CryptConfig::new(key)?;
+            crypt_conf_opt = Some(&crypt_conf);
+        }
+
+        output_file.write_all(
+            chunk_blob
+                .decode(crypt_conf_opt, Some(chunk_digest))?
+                .as_slice(),
+        )?;
+    }
+    println!("{} sec.", start.elapsed().as_secs_f32());
+    Ok(Value::Null)
+}
+
+pub fn recover_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new().insert("index", CliCommand::new(&API_METHOD_RECOVER_INDEX));
+    cmd_def.into()
+}
-- 
2.20.1

Re: [pbs-devel] [PATCH v3 proxmox-backup 1/3] add chunk inspection to pb-debug

[Wolfgang Bumiller]

On Fri, Feb 05, 2021 at 08:58:04AM +0100, Hannes Laimer wrote:
> Adds possibility to inspect chunks and find indexes that reference the
> chunk. Options:
>  - chunk: path to the chunk file
>  - [opt] decode: path to a file or to stdout(-), if specified, the chunk will
>    be decoded into the specified location
>  - [opt] keyfile: path to a keyfile, needed if decode is specified and the
>    data was encrypted
>  - [opt] reference-filter: path in which indexes that reference the chunk
>    should be searched, can be a group, snapshot or the whole datastore,
>    if not specified no references will be searched
> 
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---
> 
>  Makefile                                |   3 +-
>  src/bin/proxmox-backup-debug.rs         |  33 +++++
>  src/bin/proxmox_backup_debug/inspect.rs | 170 ++++++++++++++++++++++++
>  src/bin/proxmox_backup_debug/mod.rs     |   2 +
>  4 files changed, 207 insertions(+), 1 deletion(-)
>  create mode 100644 src/bin/proxmox-backup-debug.rs
>  create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
>  create mode 100644 src/bin/proxmox_backup_debug/mod.rs
> 
> diff --git a/Makefile b/Makefile
> index a1af9d51..3c780bcf 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -13,7 +13,8 @@ USR_BIN := \
>  
>  # Binaries usable by admins
>  USR_SBIN := \
> -	proxmox-backup-manager
> +	proxmox-backup-manager \
> +	proxmox-backup-debug \
>  
>  # Binaries for services:
>  SERVICE_BIN := \
> diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
> new file mode 100644
> index 00000000..079b2bc8
> --- /dev/null
> +++ b/src/bin/proxmox-backup-debug.rs
> @@ -0,0 +1,33 @@
> +use anyhow::Error;

newline here

> +use proxmox::api::cli::*;
> +use proxmox::api::schema::{Schema, StringSchema};

no onewline here

> +use proxmox::sys::linux::tty;
> +
> +use proxmox_backup_debug::*;

no newline here, `mod` line first, and please expand the `*`.

> +mod proxmox_backup_debug;
> +
> +pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
> +    tty::read_password("Encryption Key Password: ")
> +}
> +
> +pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
> +
> +pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
> +    "Path to encryption key. If the data was encrypted, this key will be used for decryption.",
> +)
> +.schema();

^ could probably move this from the client to `src/api2/types.rs` and
reuse for both binaries

> +
> +fn main() {
> +    proxmox_backup::tools::setup_safe_path_env();
> +
> +    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
> +
> +    let rpcenv = CliEnvironment::new();
> +    run_cli_command(
> +        cmd_def,
> +        rpcenv,
> +        Some(|future| proxmox_backup::tools::runtime::main(future)),
> +    );
> +}
> diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
> new file mode 100644
> index 00000000..a66818be
> --- /dev/null
> +++ b/src/bin/proxmox_backup_debug/inspect.rs
> @@ -0,0 +1,170 @@
> +use std::fs::File;
> +use std::io::Write;
> +use std::path::Path;
> +
> +use anyhow::{format_err, Error};
> +use proxmox::api::cli::{
> +    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
> +};
> +use proxmox::api::{api, cli::*, RpcEnvironment};
> +use serde_json::{json, Value};
> +use walkdir::WalkDir;
> +
> +use proxmox_backup::backup::{
> +    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
> +};
> +use proxmox_backup::tools;
> +
> +use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
> +
> +/// Decodes a blob and writes its content either to stdout or into a file
> +fn decode_blob(
> +    output_path: Option<&Path>,
> +    key_file: Option<&Path>,
> +    digest: Option<&[u8; 32]>,
> +    blob: &DataBlob,
> +) -> Result<(), Error> {
> +    let mut crypt_conf_opt = None;
> +    let crypt_conf;
> +
> +    if blob.is_encrypted() && key_file.is_some() {
> +        let (key, _created, _fingerprint) =
> +            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
> +        crypt_conf = CryptConfig::new(key)?;
> +        crypt_conf_opt = Some(&crypt_conf);
> +    }
> +
> +    if output_path.is_some() {
> +        let mut file = File::create(output_path.unwrap())?;
> +        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
> +    } else {
> +        Ok(println!(
> +            "{}",
> +            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
> +        ))

Do we really want to use lossy utf8 here? We should at least warn when
it's invalid utf8, or maybe we just dump it to stdout as bytes.

In fact, the whole "output file or stdout" thing could probably use a
helper. If we don't have one already, maybe add to tools a
`pub fn output_or_stdout<P: AsRef<Path>>(path: Option<P>) -> File`

(or if we want to keep the 'locking' we get from the `std::io::Stdout`
type, this function would have to return a custom enum which is either a
`File` or a `Stdout` and just implements `io::Write` by forwarding)

> +    }
> +}
> +
> +#[api(
> +    input: {
> +        properties: {
> +            chunk: {
> +                schema: PATH_SCHEMA,
> +            },
> +            "reference-filter": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "decode": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "keyfile": {
> +                schema: KEYFILE_SCHEMA,
> +                optional: true,
> +            },
> +            "output-format": {
> +                schema: OUTPUT_FORMAT,
> +                optional: true,
> +            },
> +        }
> +    }
> +)]
> +/// Inspect a chunk
> +fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {

New code is much easier to read if the parameters are written out into
the `fn()`.
Also the `_rpcenv` can simply be dropped if you don't use it:

    fn inspect_chunk(
        chunk: String,
        reference_filter: Option<String>,
        decode: Option<String>,
        keyfile: Option<String>,
        param: Value, // for keep `output-format` in there for `get_output_format()`
    ) -> Result<Value, Error> {

> +    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
> +    let output_format = get_output_format(&param);
> +    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
> +    let digest_raw = proxmox::tools::hex_to_digest(digest_str)
> +        .map_err(|e| format_err!("could not parse chunk - {}", e))?;
> +
> +    let reference_filter_param = param["reference-filter"].as_str();
> +    let decode_output_param = param["decode"].as_str();
> +    let key_file_param = param["keyfile"].as_str();
> +
> +    let search_path = reference_filter_param.map(|path| Path::new(path));
> +    let key_file_path = key_file_param.map(|path| Path::new(path));

^ hint: should be possible to shorten this to `.map(Path::new)`

> +
> +    let (decode_output_path, to_stdout) = match decode_output_param {
> +        Some(path) if path.eq("-") => (None, true),
> +        Some(path) => (Some(Path::new(path)), false),
> +        None => (None, false),
> +    };

Here we could call the `ouput_or_stdout()` helper to get an
`Option<File>` and the `to_stdout` bool could be dropped and the
`.is_some()` check would be sufficient.

> +
> +    let mut file = std::fs::File::open(&chunk_path)
> +        .map_err(|e| format_err!("could not open chunk file - {}", e))?;
> +    let blob = DataBlob::load_from_reader(&mut file)?;
> +
> +    let mut referenced_by = None;
> +    if let Some(search_path) = search_path {
> +        let mut references = Vec::new();
> +        for entry in WalkDir::new(search_path)
> +            .follow_links(false)
> +            .into_iter()
> +            .filter_map(|e| e.ok())
> +        {
> +            let file_name = entry.file_name().to_string_lossy();
> +            let mut index: Option<Box<dyn IndexFile>> = None;
> +
> +            if file_name.ends_with(".fidx") {
> +                index = match FixedIndexReader::open(entry.path()) {
> +                    Ok(index) => Some(Box::new(index)),
> +                    Err(_) => None,
> +                };
> +            }
> +
> +            if file_name.ends_with(".didx") {
> +                index = match DynamicIndexReader::open(entry.path()) {
> +                    Ok(index) => Some(Box::new(index)),
> +                    Err(_) => None,
> +                };
> +            }
> +
> +            if let Some(index) = index {
> +                for pos in 0..index.index_count() {
> +                    if let Some(index_chunk_digest) = index.index_digest(pos) {
> +                        let str = proxmox::tools::digest_to_hex(index_chunk_digest);
> +                        if str.eq(digest_str) {
> +                            references.push(entry.path().to_string_lossy().into_owned());
> +                            break;
> +                        }
> +                    }
> +                }
> +            }
> +        }
> +        referenced_by = Some(references);
> +    }
> +
> +    if decode_output_path.is_some() || to_stdout {
> +        decode_blob(decode_output_path, key_file_path, Some(&digest_raw), &blob)?;
> +    }
> +
> +    let crc_status = format!(
> +        "{}({})",
> +        blob.compute_crc(),
> +        blob.verify_crc().map_or("NOK", |_| "OK")
> +    );
> +
> +    let val = match referenced_by {
> +        Some(references) => json!({
> +            "digest": digest_str,
> +            "crc": crc_status,
> +            "encryption": blob.crypt_mode()?,
> +            "referenced-by": references
> +        }),
> +        None => json!({
> +             "digest": digest_str,
> +             "crc": crc_status,
> +             "encryption": blob.crypt_mode()?,
> +        }),
> +    };
> +
> +    format_and_print_result(&val, &output_format);
> +    Ok(Value::Null)
> +}
> +
> +pub fn inspect_commands() -> CommandLineInterface {
> +    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
> +
> +    cmd_def.into()
> +}
> diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
> new file mode 100644
> index 00000000..644583db
> --- /dev/null
> +++ b/src/bin/proxmox_backup_debug/mod.rs
> @@ -0,0 +1,2 @@
> +mod inspect;
> +pub use inspect::*;
> -- 
> 2.20.1

Re: [pbs-devel] [PATCH v3 proxmox-backup 2/3] add file(.blob, .fidx, .didx) inspection to pb-debug

[Wolfgang Bumiller]

On Fri, Feb 05, 2021 at 08:58:05AM +0100, Hannes Laimer wrote:
> Adds possibility to inspect .blob, .fidx and .didx files. For index
> files a list of the chunks referenced will be printed in addition to
> some other inforation. .blob files can be decoded into file or directly
> into stdout. Options:
>  - file: path to the file
>  - [opt] decode: path to a file or stdout(-), if specidied, the file will be
>    decoded into the specified location [only for blob files, no effect
>    with index files]
>  - [opt] keyfile: path to a keyfile, needed if decode is specified and the
>    data was encrypted
> 
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---
> 
>  src/bin/proxmox_backup_debug/inspect.rs | 113 +++++++++++++++++++++++-
>  1 file changed, 111 insertions(+), 2 deletions(-)
> 
> diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
> index a66818be..367d01c2 100644
> --- a/src/bin/proxmox_backup_debug/inspect.rs
> +++ b/src/bin/proxmox_backup_debug/inspect.rs
> @@ -1,8 +1,9 @@
> +use std::collections::HashSet;
>  use std::fs::File;
>  use std::io::Write;
>  use std::path::Path;
>  
> -use anyhow::{format_err, Error};
> +use anyhow::{bail, format_err, Error};
>  use proxmox::api::cli::{
>      format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
>  };
> @@ -163,8 +164,116 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
>      Ok(Value::Null)
>  }
>  
> +#[api(
> +    input: {
> +        properties: {
> +            file: {
> +                schema: PATH_SCHEMA,
> +            },
> +            "decode": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "keyfile": {
> +                schema: KEYFILE_SCHEMA,
> +                optional: true,
> +            },
> +            "output-format": {
> +                schema: OUTPUT_FORMAT,
> +                optional: true,
> +            },
> +        }
> +    }
> +)]
> +/// Inspect a file
> +fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {

^ same as in the other patch (fn parameters)

> +    let path = tools::required_string_param(&param, "file")?;
> +    let output_format = get_output_format(&param);
> +
> +    let val;
> +    if path.ends_with(".blob") {
> +        let decode_output_param = param["decode"].as_str();
> +        let key_file_param = param["keyfile"].as_str();
> +
> +        let mut file = std::fs::File::open(&path)
> +            .map_err(|e| format_err!("could not open blob file - {}", e))?;
> +        let data_blob = DataBlob::load_from_reader(&mut file)?;

Since `load_from_reader` takes a trait object, this is mostly fine,
but I *would* prefer to limit the file's scope to this call so it gets
closed early.

    load_from_reader(&mut File::open(&path).map_err(...)?)

> +
> +        let key_file_path = key_file_param.map(|path| Path::new(path));
> +
> +        let (decode_output_path, to_stdout) = match decode_output_param {
> +            Some(path) if path.eq("-") => (None, true),
> +            Some(path) => (Some(Path::new(path)), false),
> +            None => (None, false),
> +        };

^ same as in the other patch (call the "open stdout or path" helper fn)

> +
> +        let crypt_mode = data_blob.crypt_mode()?;
> +        val = json!({
> +            "encryption": crypt_mode,
> +            "raw_size": data_blob.raw_size(),
> +        });
> +
> +        if decode_output_path.is_some() || to_stdout {
> +            decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
> +        }
> +    } else if path.ends_with(".fidx") {
> +        let index = FixedIndexReader::open(Path::new(path))
> +            .map_err(|e| format_err!("could not open fidx file - {}", e))?;
> +
> +        let mut ctime_str = index.ctime.to_string();
> +        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
> +            ctime_str = s;
> +        }
> +
> +        let mut chunk_digests = HashSet::new();
> +
> +        for pos in 0..index.index_count() {
> +            let digest = index.index_digest(pos).unwrap();
> +            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
> +        }
> +
> +        val = json!({
> +            "size": index.size,
> +            "ctime": ctime_str,
> +            "chunk-digests": chunk_digests
> +
> +        })
> +    } else if path.ends_with("didx") {

^ misses the dot in the string

> +        let index = DynamicIndexReader::open(Path::new(path))
> +            .map_err(|e| format_err!("could not open didx file - {}", e))?;
> +
> +        let mut ctime_str = index.ctime.to_string();
> +        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
> +            ctime_str = s;
> +        }
> +
> +        let mut chunk_digests = HashSet::new();
> +
> +        for pos in 0..index.index_count() {
> +            let digest = index.index_digest(pos).unwrap();
> +            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
> +        }
> +
> +        val = json!({
> +            "size": index.size,
> +            "ctime": ctime_str,
> +            "chunk-digests": chunk_digests
> +        })
> +    } else {
> +        bail!("Only .blob, .fidx and .didx files may be inspected");
> +    }
> +
> +    if !val.is_null() {
> +        format_and_print_result(&val, &output_format);
> +    }
> +
> +    Ok(Value::Null)

Note that you can just let the method return `()` and use `Ok(())`.

> +}
> +
>  pub fn inspect_commands() -> CommandLineInterface {
> -    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
> +    let cmd_def = CliCommandMap::new()
> +        .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
> +        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
>  
>      cmd_def.into()
>  }
> -- 
> 2.20.1

Re: [pbs-devel] [PATCH v3 proxmox-backup 3/3] add index recovery to pb-debug

[Wolfgang Bumiller]

On Fri, Feb 05, 2021 at 08:58:06AM +0100, Hannes Laimer wrote:
> Adds possibility to recover data from an index file. Options:
>  - chunks: path to the directory where the chunks are saved
>  - file: the index file that should be recovered(must be either .fidx or
>    didx)
>  - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
>    needed
>  - [opt] skip-crc: boolean, if true, read chunks wont be verified with their
>    crc-sum, increases the restore speed by a lot
> 
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---
> 
>  src/bin/proxmox-backup-debug.rs         |   6 +-
>  src/bin/proxmox_backup_debug/mod.rs     |   2 +
>  src/bin/proxmox_backup_debug/recover.rs | 109 ++++++++++++++++++++++++
>  3 files changed, 115 insertions(+), 2 deletions(-)
>  create mode 100644 src/bin/proxmox_backup_debug/recover.rs
> 
> diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
> index 079b2bc8..8bd01a59 100644
> --- a/src/bin/proxmox-backup-debug.rs
> +++ b/src/bin/proxmox-backup-debug.rs
> @@ -22,7 +22,9 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
>  fn main() {
>      proxmox_backup::tools::setup_safe_path_env();
>  
> -    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
> +    let cmd_def = CliCommandMap::new()
> +        .insert("inspect", inspect_commands())
> +        .insert("recover", recover_commands());
>  
>      let rpcenv = CliEnvironment::new();
>      run_cli_command(
> @@ -30,4 +32,4 @@ fn main() {
>          rpcenv,
>          Some(|future| proxmox_backup::tools::runtime::main(future)),
>      );
> -}
> +}
> \ No newline at end of file
> diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
> index 644583db..62df7754 100644
> --- a/src/bin/proxmox_backup_debug/mod.rs
> +++ b/src/bin/proxmox_backup_debug/mod.rs
> @@ -1,2 +1,4 @@
>  mod inspect;
>  pub use inspect::*;
> +mod recover;
> +pub use recover::*;
> diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
> new file mode 100644
> index 00000000..61025adf
> --- /dev/null
> +++ b/src/bin/proxmox_backup_debug/recover.rs
> @@ -0,0 +1,109 @@
> +use std::fs::File;
> +use std::io::{Read, Write};
> +use std::path::Path;
> +
> +use anyhow::{bail, format_err, Error};
> +
> +use proxmox::api::api;
> +use proxmox::api::cli::{CliCommand, CliCommandMap, CommandLineInterface};
> +use serde_json::Value;
> +
> +use proxmox_backup::backup::{
> +    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
> +};
> +use proxmox_backup::tools;
> +
> +use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
> +
> +use proxmox::tools::digest_to_hex;
> +
> +use std::time::Instant;
> +
> +#[api(
> +    input: {
> +        properties: {
> +            file: {
> +                schema: PATH_SCHEMA,
> +            },
> +            chunks: {
> +                schema: PATH_SCHEMA,
> +            },
> +            "keyfile": {
> +                schema: KEYFILE_SCHEMA,
> +                optional: true,
> +            },
> +            "skip-crc": {
> +                type: Boolean,
> +                optional: true,
> +                default: false,
> +                description: "Skip the crc verification, increases the restore speed immensely",
> +            }
> +        }
> +    }
> +)]
> +/// Recover a index file

'an'

Perhapse add some more information of what this actually does and
particularly what its limitations are.

> +fn recover_index(skip_crc: bool, param: Value) -> Result<Value, Error> {
> +    let start = Instant::now();
> +    let file_path = Path::new(tools::required_string_param(&param, "file")?);
> +    let chunks_path = Path::new(tools::required_string_param(&param, "chunks")?);
> +
> +    let key_file_param = param["keyfile"].as_str();
> +    let key_file_path = key_file_param.map(|path| Path::new(path));
> +
> +    let index: Box<dyn IndexFile> = match file_path.extension() {
> +        Some(ext) if ext.eq("fidx") => Box::new(
> +            FixedIndexReader::open(file_path)
> +                .map_err(|e| format_err!("could not read index - {}", e))?,
> +        ),
> +        Some(ext) if ext.eq("didx") => Box::new(
> +            DynamicIndexReader::open(file_path)
> +                .map_err(|e| format_err!("could not read index - {}", e))?,
> +        ),
> +        _ => bail!("index file must either be a .fidx or a .didx file"),
> +    };

Come to think of it, we do have magic bytes at the top of all of these
file types, so shouldn't a *debug* binary use *that* instead of the
file's *name*? ;-)

*And* perhaps warn if the name doesn't match its type...

So for this we could just have a helper `fn detect_file_type() -> FileType`,
which we could use in all the other functions of this binary as well.

> +
> +    let mut crypt_conf_opt = None;
> +    let mut crypt_conf;
> +
> +    let output_filename = file_path.file_stem().unwrap().to_str().unwrap();
> +    let output_path = Path::new(output_filename);
> +    let mut output_file = File::create(output_path)
> +        .map_err(|e| format_err!("could not create output file - {}", e))?;
> +

I feel like the code below should already exist somewhere in some
half-way reusable form. It might be worth trying to instantiate a
`LocalDataStore` and use its `read_chunk` method, but I'm not sure if it
helps much. Plus it wouldn't reuse the buffer...

> +    for pos in 0..index.index_count() {
> +        let chunk_digest = index.index_digest(pos).unwrap();
> +        let digest_str = digest_to_hex(chunk_digest);
> +        let digest_prefix = &digest_str[0..4];
> +        let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
> +        let mut chunk_file = std::fs::File::open(&chunk_path)
> +            .map_err(|e| format_err!("could not open chunk file - {}", e))?;
> +
> +        let mut data = Vec::with_capacity(1024 * 1024);

...but neither do you right now ;-)
Please move this above the loop and just use `.clear()` here (which does
not deallocate).

> +        chunk_file.read_to_end(&mut data)?;
> +        let chunk_blob = DataBlob::from_raw(data)?;
> +
> +        if !skip_crc {
> +            chunk_blob.verify_crc()?;
> +        }
> +
> +        if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() {
> +            let (key, _created, _fingerprint) =
> +                load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?;
> +            crypt_conf = CryptConfig::new(key)?;
> +            crypt_conf_opt = Some(&crypt_conf);
> +        }
> +
> +        output_file.write_all(
> +            chunk_blob
> +                .decode(crypt_conf_opt, Some(chunk_digest))?
> +                .as_slice(),
> +        )?;
> +    }
> +    println!("{} sec.", start.elapsed().as_secs_f32());

As a frequent user of the `time` shell command I'm wondering if this is
really something I want to always see? ;-)

> +    Ok(Value::Null)
> +}
> +
> +pub fn recover_commands() -> CommandLineInterface {
> +    let cmd_def = CliCommandMap::new().insert("index", CliCommand::new(&API_METHOD_RECOVER_INDEX));
> +    cmd_def.into()
> +}
> -- 
> 2.20.1