[pbs-devel] [PATCH v2 proxmox-backup 0/3] add proxmox-backup-debug binary

[pbs-devel] [PATCH v2 proxmox-backup 0/3] add proxmox-backup-debug binary

[Hannes Laimer]

Adds a new proxmox-backup-debug binary for recovery or inspections of
index and blob files.

v2:
  - appiled suggestions + rewrote a few parts
  - renamed 'restore' to 'debug'

Hannes Laimer (3):
  add chunk inspection to pb-debug
  add file(.blob, .fidx, .didx) inspection to pb-debug
  add index recovery to pb-debug

 Makefile                                |   3 +-
 src/bin/proxmox-backup-debug.rs         |  35 +++
 src/bin/proxmox_backup_debug/inspect.rs | 279 ++++++++++++++++++++++++
 src/bin/proxmox_backup_debug/mod.rs     |   4 +
 src/bin/proxmox_backup_debug/recover.rs | 109 +++++++++
 5 files changed, 429 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-debug.rs
 create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
 create mode 100644 src/bin/proxmox_backup_debug/mod.rs
 create mode 100644 src/bin/proxmox_backup_debug/recover.rs

-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 1/3] add chunk inspection to pb-debug

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to inspect chunks and find indexes that reference the
chunk. Options:
 - chunk: path to the chunk file
 - [opt] decode: path to a file or to stdout(-), if specified, the chunk will
   be decoded into the specified location
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted
 - [opt] reference-filter: path in which indexes that reference the chunk
   should be searched, can be a group, snapshot or the whole datastore,
   if not specified no references will be searched

 Makefile                                |   3 +-
 src/bin/proxmox-backup-debug.rs         |  33 +++++
 src/bin/proxmox_backup_debug/inspect.rs | 170 ++++++++++++++++++++++++
 src/bin/proxmox_backup_debug/mod.rs     |   2 +
 4 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-debug.rs
 create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
 create mode 100644 src/bin/proxmox_backup_debug/mod.rs

diff --git a/Makefile b/Makefile
index a1af9d51..3c780bcf 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,8 @@ USR_BIN := \
 
 # Binaries usable by admins
 USR_SBIN := \
-	proxmox-backup-manager
+	proxmox-backup-manager \
+	proxmox-backup-debug \
 
 # Binaries for services:
 SERVICE_BIN := \
diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
new file mode 100644
index 00000000..079b2bc8
--- /dev/null
+++ b/src/bin/proxmox-backup-debug.rs
@@ -0,0 +1,33 @@
+use anyhow::Error;
+use proxmox::api::cli::*;
+use proxmox::api::schema::{Schema, StringSchema};
+
+use proxmox::sys::linux::tty;
+
+use proxmox_backup_debug::*;
+
+mod proxmox_backup_debug;
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+    tty::read_password("Encryption Key Password: ")
+}
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
+    "Path to encryption key. If the data was encrypted, this key will be used for decryption.",
+)
+.schema();
+
+fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
+    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+
+    let rpcenv = CliEnvironment::new();
+    run_cli_command(
+        cmd_def,
+        rpcenv,
+        Some(|future| proxmox_backup::tools::runtime::main(future)),
+    );
+}
diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
new file mode 100644
index 00000000..a66818be
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -0,0 +1,170 @@
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::{format_err, Error};
+use proxmox::api::cli::{
+    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*, RpcEnvironment};
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+    output_path: Option<&Path>,
+    key_file: Option<&Path>,
+    digest: Option<&[u8; 32]>,
+    blob: &DataBlob,
+) -> Result<(), Error> {
+    let mut crypt_conf_opt = None;
+    let crypt_conf;
+
+    if blob.is_encrypted() && key_file.is_some() {
+        let (key, _created, _fingerprint) =
+            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+        crypt_conf = CryptConfig::new(key)?;
+        crypt_conf_opt = Some(&crypt_conf);
+    }
+
+    if output_path.is_some() {
+        let mut file = File::create(output_path.unwrap())?;
+        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
+    } else {
+        Ok(println!(
+            "{}",
+            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
+        ))
+    }
+}
+
+#[api(
+    input: {
+        properties: {
+            chunk: {
+                schema: PATH_SCHEMA,
+            },
+            "reference-filter": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a chunk
+fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
+    let output_format = get_output_format(&param);
+    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+    let digest_raw = proxmox::tools::hex_to_digest(digest_str)
+        .map_err(|e| format_err!("could not parse chunk - {}", e))?;
+
+    let reference_filter_param = param["reference-filter"].as_str();
+    let decode_output_param = param["decode"].as_str();
+    let key_file_param = param["keyfile"].as_str();
+
+    let search_path = reference_filter_param.map(|path| Path::new(path));
+    let key_file_path = key_file_param.map(|path| Path::new(path));
+
+    let (decode_output_path, to_stdout) = match decode_output_param {
+        Some(path) if path.eq("-") => (None, true),
+        Some(path) => (Some(Path::new(path)), false),
+        None => (None, false),
+    };
+
+    let mut file = std::fs::File::open(&chunk_path)
+        .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+    let blob = DataBlob::load_from_reader(&mut file)?;
+
+    let mut referenced_by = None;
+    if let Some(search_path) = search_path {
+        let mut references = Vec::new();
+        for entry in WalkDir::new(search_path)
+            .follow_links(false)
+            .into_iter()
+            .filter_map(|e| e.ok())
+        {
+            let file_name = entry.file_name().to_string_lossy();
+            let mut index: Option<Box<dyn IndexFile>> = None;
+
+            if file_name.ends_with(".fidx") {
+                index = match FixedIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if file_name.ends_with(".didx") {
+                index = match DynamicIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if let Some(index) = index {
+                for pos in 0..index.index_count() {
+                    if let Some(index_chunk_digest) = index.index_digest(pos) {
+                        let str = proxmox::tools::digest_to_hex(index_chunk_digest);
+                        if str.eq(digest_str) {
+                            references.push(entry.path().to_string_lossy().into_owned());
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        referenced_by = Some(references);
+    }
+
+    if decode_output_path.is_some() || to_stdout {
+        decode_blob(decode_output_path, key_file_path, Some(&digest_raw), &blob)?;
+    }
+
+    let crc_status = format!(
+        "{}({})",
+        blob.compute_crc(),
+        blob.verify_crc().map_or("NOK", |_| "OK")
+    );
+
+    let val = match referenced_by {
+        Some(references) => json!({
+            "digest": digest_str,
+            "crc": crc_status,
+            "encryption": blob.crypt_mode()?,
+            "referenced-by": references
+        }),
+        None => json!({
+             "digest": digest_str,
+             "crc": crc_status,
+             "encryption": blob.crypt_mode()?,
+        }),
+    };
+
+    format_and_print_result(&val, &output_format);
+    Ok(Value::Null)
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+    cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
new file mode 100644
index 00000000..644583db
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -0,0 +1,2 @@
+mod inspect;
+pub use inspect::*;
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 2/3] add file(.blob, .fidx, .didx) inspection to pb-debug

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to inspect .blob, .fidx and .didx files. For index
files a list of the chunks referenced will be printed in addition to
some other inforation. .blob files can be decoded into file or directly
into stdout. Options:
 - file: path to the file
 - [opt] decode: path to a file or stdout(-), if specidied, the file will be
   decoded into the specified location [only for blob files, no effect
   with index files]
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted

 src/bin/proxmox_backup_debug/inspect.rs | 113 +++++++++++++++++++++++-
 1 file changed, 111 insertions(+), 2 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
index a66818be..367d01c2 100644
--- a/src/bin/proxmox_backup_debug/inspect.rs
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -1,8 +1,9 @@
+use std::collections::HashSet;
 use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
 use proxmox::api::cli::{
     format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
 };
@@ -163,8 +164,116 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
     Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a file
+fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let path = tools::required_string_param(&param, "file")?;
+    let output_format = get_output_format(&param);
+
+    let val;
+    if path.ends_with(".blob") {
+        let decode_output_param = param["decode"].as_str();
+        let key_file_param = param["keyfile"].as_str();
+
+        let mut file = std::fs::File::open(&path)
+            .map_err(|e| format_err!("could not open blob file - {}", e))?;
+        let data_blob = DataBlob::load_from_reader(&mut file)?;
+
+        let key_file_path = key_file_param.map(|path| Path::new(path));
+
+        let (decode_output_path, to_stdout) = match decode_output_param {
+            Some(path) if path.eq("-") => (None, true),
+            Some(path) => (Some(Path::new(path)), false),
+            None => (None, false),
+        };
+
+        let crypt_mode = data_blob.crypt_mode()?;
+        val = json!({
+            "encryption": crypt_mode,
+            "raw_size": data_blob.raw_size(),
+        });
+
+        if decode_output_path.is_some() || to_stdout {
+            decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
+        }
+    } else if path.ends_with(".fidx") {
+        let index = FixedIndexReader::open(Path::new(path))
+            .map_err(|e| format_err!("could not open fidx file - {}", e))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+
+        })
+    } else if path.ends_with("didx") {
+        let index = DynamicIndexReader::open(Path::new(path))
+            .map_err(|e| format_err!("could not open didx file - {}", e))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+        })
+    } else {
+        bail!("Only .blob, .fidx and .didx files may be inspected");
+    }
+
+    if !val.is_null() {
+        format_and_print_result(&val, &output_format);
+    }
+
+    Ok(Value::Null)
+}
+
 pub fn inspect_commands() -> CommandLineInterface {
-    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+    let cmd_def = CliCommandMap::new()
+        .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
+        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
 
     cmd_def.into()
 }
-- 
2.20.1

[pbs-devel] [PATCH v2 proxmox-backup 3/3] add index recovery to pb-debug

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to recover data from an index file. Options:
 - chunks: path to the directory where the chunks are saved
 - file: the index file that should be recovered(must be either .fidx or
   didx)
 - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
   needed
 - [opt] skip-crc: boolean, if true, read chunks wont be verified with their
   crc-sum, increases the restore speed by a lot

 src/bin/proxmox-backup-debug.rs         |   6 +-
 src/bin/proxmox_backup_debug/mod.rs     |   2 +
 src/bin/proxmox_backup_debug/recover.rs | 109 ++++++++++++++++++++++++
 3 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 src/bin/proxmox_backup_debug/recover.rs

diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
index 079b2bc8..8bd01a59 100644
--- a/src/bin/proxmox-backup-debug.rs
+++ b/src/bin/proxmox-backup-debug.rs
@@ -22,7 +22,9 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
 fn main() {
     proxmox_backup::tools::setup_safe_path_env();
 
-    let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+    let cmd_def = CliCommandMap::new()
+        .insert("inspect", inspect_commands())
+        .insert("recover", recover_commands());
 
     let rpcenv = CliEnvironment::new();
     run_cli_command(
@@ -30,4 +32,4 @@ fn main() {
         rpcenv,
         Some(|future| proxmox_backup::tools::runtime::main(future)),
     );
-}
+}
\ No newline at end of file
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
index 644583db..62df7754 100644
--- a/src/bin/proxmox_backup_debug/mod.rs
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -1,2 +1,4 @@
 mod inspect;
 pub use inspect::*;
+mod recover;
+pub use recover::*;
diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
new file mode 100644
index 00000000..61025adf
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/recover.rs
@@ -0,0 +1,109 @@
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox::api::api;
+use proxmox::api::cli::{CliCommand, CliCommandMap, CommandLineInterface};
+use serde_json::Value;
+
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+use proxmox::tools::digest_to_hex;
+
+use std::time::Instant;
+
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            chunks: {
+                schema: PATH_SCHEMA,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "skip-crc": {
+                type: Boolean,
+                optional: true,
+                default: false,
+                description: "Skip the crc verification, increases the restore speed immensely",
+            }
+        }
+    }
+)]
+/// Recover a index file
+fn recover_index(skip_crc: bool, param: Value) -> Result<Value, Error> {
+    let start = Instant::now();
+    let file_path = Path::new(tools::required_string_param(&param, "file")?);
+    let chunks_path = Path::new(tools::required_string_param(&param, "chunks")?);
+
+    let key_file_param = param["keyfile"].as_str();
+    let key_file_path = key_file_param.map(|path| Path::new(path));
+
+    let index: Box<dyn IndexFile> = match file_path.extension() {
+        Some(ext) if ext.eq("fidx") => Box::new(
+            FixedIndexReader::open(file_path)
+                .map_err(|e| format_err!("could not read index - {}", e))?,
+        ),
+        Some(ext) if ext.eq("didx") => Box::new(
+            DynamicIndexReader::open(file_path)
+                .map_err(|e| format_err!("could not read index - {}", e))?,
+        ),
+        _ => bail!("index file must either be a .fidx or a .didx file"),
+    };
+
+    let mut crypt_conf_opt = None;
+    let mut crypt_conf;
+
+    let output_filename = file_path.file_stem().unwrap().to_str().unwrap();
+    let output_path = Path::new(output_filename);
+    let mut output_file = File::create(output_path)
+        .map_err(|e| format_err!("could not create output file - {}", e))?;
+
+    for pos in 0..index.index_count() {
+        let chunk_digest = index.index_digest(pos).unwrap();
+        let digest_str = digest_to_hex(chunk_digest);
+        let digest_prefix = &digest_str[0..4];
+        let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
+        let mut chunk_file = std::fs::File::open(&chunk_path)
+            .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+
+        let mut data = Vec::with_capacity(1024 * 1024);
+        chunk_file.read_to_end(&mut data)?;
+        let chunk_blob = DataBlob::from_raw(data)?;
+
+        if !skip_crc {
+            chunk_blob.verify_crc()?;
+        }
+
+        if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() {
+            let (key, _created, _fingerprint) =
+                load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?;
+            crypt_conf = CryptConfig::new(key)?;
+            crypt_conf_opt = Some(&crypt_conf);
+        }
+
+        output_file.write_all(
+            chunk_blob
+                .decode(crypt_conf_opt, Some(chunk_digest))?
+                .as_slice(),
+        )?;
+    }
+    println!("{} sec.", start.elapsed().as_secs_f32());
+    Ok(Value::Null)
+}
+
+pub fn recover_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new().insert("index", CliCommand::new(&API_METHOD_RECOVER_INDEX));
+    cmd_def.into()
+}
-- 
2.20.1