[pbs-devel] [PATCH proxmox-backup 0/3] add proxmox-backup-recovery binary

[pbs-devel] [PATCH proxmox-backup 0/3] add proxmox-backup-recovery binary

[Hannes Laimer]

Adds a (standalone) binary that can be used for inspection of index,
blob and chunk files. It can also be used to restore data from an index
files without relying on a running pbs instance.

Hannes Laimer (3):
  add inspection of chunks
  add inspection of .blob, .fidx and .didx files
  add restore of index files

 Makefile                                   |   3 +-
 src/bin/proxmox-backup-recovery.rs         | 137 +++++++++
 src/bin/proxmox_backup_recovery/inspect.rs | 308 +++++++++++++++++++++
 src/bin/proxmox_backup_recovery/mod.rs     |   2 +
 4 files changed, 449 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-recovery.rs
 create mode 100644 src/bin/proxmox_backup_recovery/inspect.rs
 create mode 100644 src/bin/proxmox_backup_recovery/mod.rs

-- 
2.20.1

[pbs-devel] [PATCH proxmox-backup 1/3] add inspection of chunks

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to inspect chunks and find indexes that reference the
chunk. Options:
 - chunk: path to the chunk file
 - [opt] decode: path to a file or to stdout(-), if specified, the chunk will
   be decoded into the specified location
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted
 - [opt] reference-filter: path in which indexes that reference the chunk
   should be searched, can be a group, snapshot or the whole datastore,
   if not specified no references will be searched


 Makefile                                   |   3 +-
 src/bin/proxmox-backup-recovery.rs         |  32 ++++
 src/bin/proxmox_backup_recovery/inspect.rs | 195 +++++++++++++++++++++
 src/bin/proxmox_backup_recovery/mod.rs     |   2 +
 4 files changed, 231 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox-backup-recovery.rs
 create mode 100644 src/bin/proxmox_backup_recovery/inspect.rs
 create mode 100644 src/bin/proxmox_backup_recovery/mod.rs

diff --git a/Makefile b/Makefile
index a1af9d51..a9c2a711 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,8 @@ USR_BIN := \
 
 # Binaries usable by admins
 USR_SBIN := \
-	proxmox-backup-manager
+	proxmox-backup-manager \
+	proxmox-backup-recovery \
 
 # Binaries for services:
 SERVICE_BIN := \
diff --git a/src/bin/proxmox-backup-recovery.rs b/src/bin/proxmox-backup-recovery.rs
new file mode 100644
index 00000000..ae8c18d5
--- /dev/null
+++ b/src/bin/proxmox-backup-recovery.rs
@@ -0,0 +1,32 @@
+use anyhow::Error;
+use proxmox::api::schema::{Schema, StringSchema};
+use proxmox::api::cli::*;
+
+use proxmox::sys::linux::tty;
+use proxmox_backup_recovery::*;
+mod proxmox_backup_recovery;
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+    tty::read_password("Encryption Key Password: ")
+}
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
+    "Path to encryption key. If the data was encrypted, this key will be used for decryption.",
+)
+.schema();
+
+fn main() {
+    proxmox_backup::tools::setup_safe_path_env();
+
+    let cmd_def = CliCommandMap::new()
+        .insert("inspect", inspect_commands());
+
+    let rpcenv = CliEnvironment::new();
+    run_cli_command(
+        cmd_def,
+        rpcenv,
+        Some(|future| proxmox_backup::tools::runtime::main(future)),
+    );
+}
diff --git a/src/bin/proxmox_backup_recovery/inspect.rs b/src/bin/proxmox_backup_recovery/inspect.rs
new file mode 100644
index 00000000..6bf4a0a3
--- /dev/null
+++ b/src/bin/proxmox_backup_recovery/inspect.rs
@@ -0,0 +1,195 @@
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::Error;
+use proxmox::api::cli::{
+    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*, RpcEnvironment};
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::api2::types::SHA256_HEX_REGEX;
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+    output_path: Option<&Path>,
+    key_file: Option<&Path>,
+    digest: Option<&[u8; 32]>,
+    blob: &DataBlob,
+) -> Result<(), Error> {
+    let mut crypt_conf_opt = None;
+    let crypt_conf;
+
+    if blob.is_encrypted() && key_file.is_some() {
+        let (key, _created, _fingerprint) =
+            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+        crypt_conf = CryptConfig::new(key)?;
+        crypt_conf_opt = Some(&crypt_conf);
+    }
+
+    if output_path.is_some() {
+        let mut file = File::create(output_path.unwrap())?;
+        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
+    } else {
+        Ok(println!(
+            "{}",
+            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
+        ))
+    }
+}
+
+#[api(
+    input: {
+        properties: {
+            chunk: {
+                schema: PATH_SCHEMA,
+            },
+            "reference-filter": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a chunk
+fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
+    let output_format = get_output_format(&param);
+    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+
+    if !SHA256_HEX_REGEX.is_match(digest_str) {
+        println!("chunk filename is not valid");
+        return Ok(Value::Null);
+    }
+
+    let digest_raw = proxmox::tools::hex_to_digest(digest_str)?;
+
+    let reference_filter_param = param["reference-filter"].as_str();
+    let decode_output_param = param["decode"].as_str();
+    let key_file_param = param["keyfile"].as_str();
+
+    let mut search_path = None;
+    let mut decode_output_path = None;
+    let mut key_file_path = None;
+    let mut to_stdout = false;
+
+    if let Some(path) = reference_filter_param {
+        search_path = Some(Path::new(path))
+    };
+
+    if let Some(path) = decode_output_param {
+        to_stdout = path.eq("-");
+        decode_output_path = Some(Path::new(path))
+    };
+
+    if let Some(path) = key_file_param {
+        key_file_path = Some(Path::new(path))
+    };
+
+    let mut file = std::fs::File::open(&chunk_path)?;
+    let blob = DataBlob::load_from_reader(&mut file)?;
+
+    let mut referenced_by = None;
+    if let Some(search_path) = search_path {
+        let mut references = Vec::new();
+        for entry in WalkDir::new(search_path)
+            .follow_links(false)
+            .into_iter()
+            .filter_map(|e| e.ok())
+        {
+            let file_name = entry.file_name().to_string_lossy();
+            let mut in_index = HashSet::new();
+            let mut index: Option<Box<dyn IndexFile>> = None;
+
+            if file_name.ends_with(".fidx") {
+                index = match FixedIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if file_name.ends_with(".didx") {
+                index = match DynamicIndexReader::open(entry.path()) {
+                    Ok(index) => Some(Box::new(index)),
+                    Err(_) => None,
+                };
+            }
+
+            if let Some(index) = index {
+                for pos in 0..index.index_count() {
+                    if let Some(index_chunk_digest) = index.index_digest(pos) {
+                        in_index.insert(proxmox::tools::digest_to_hex(index_chunk_digest));
+                    }
+                }
+            }
+
+            if in_index.contains(digest_str) {
+                references.push(entry.path().to_string_lossy().into_owned());
+            }
+        }
+        referenced_by = Some(references);
+    }
+
+    if let Some(decode_output_path) = decode_output_path {
+        if to_stdout {
+            decode_blob(None, key_file_path, Some(&digest_raw), &blob)?;
+        } else {
+            decode_blob(
+                Some(decode_output_path),
+                key_file_path,
+                Some(&digest_raw),
+                &blob,
+            )?;
+        }
+    }
+
+    let crc_status = format!(
+        "{}({})",
+        blob.compute_crc(),
+        blob.verify_crc().map_or("NOK", |_x| "OK")
+    );
+
+    let val = match referenced_by {
+        Some(references) => json!({
+            "digest": digest_str,
+            "crc": crc_status,
+            "encryption": blob.crypt_mode()?,
+            "referenced-by": references
+        }),
+        None => json!({
+             "digest": digest_str,
+             "crc": crc_status,
+             "encryption": blob.crypt_mode()?,
+        }),
+    };
+
+    format_and_print_result(&val, &output_format);
+    Ok(Value::Null)
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+    cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_recovery/mod.rs b/src/bin/proxmox_backup_recovery/mod.rs
new file mode 100644
index 00000000..644583db
--- /dev/null
+++ b/src/bin/proxmox_backup_recovery/mod.rs
@@ -0,0 +1,2 @@
+mod inspect;
+pub use inspect::*;
-- 
2.20.1

Re: [pbs-devel] [PATCH proxmox-backup 1/3] add inspection of chunks

[Thomas Lamprecht]

Still no patch version, that must be used! Use the `-vX` option to your git
format-patch or git send-email command, please really read, IIRC I'm linking that
the third time now!

https://pve.proxmox.com/wiki/Developer_Documentation#Versioned_Patches

On 15.12.20 09:16, Hannes Laimer wrote:
> Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
> ---

below this triple dash is for meta information, it is not the commit message
but should describe changes since the last version of the patch, it won't get
included in git.

> Adds possibility to inspect chunks and find indexes that reference the
> chunk. Options:
>  - chunk: path to the chunk file
>  - [opt] decode: path to a file or to stdout(-), if specified, the chunk will
>    be decoded into the specified location
>  - [opt] keyfile: path to a keyfile, needed if decode is specified and the
>    data was encrypted
>  - [opt] reference-filter: path in which indexes that reference the chunk
>    should be searched, can be a group, snapshot or the whole datastore,
>    if not specified no references will be searched
>  Makefile                                   |   3 +-
>  src/bin/proxmox-backup-recovery.rs         |  32 ++++

what are they now, recovery or inspect tools? Maybe a more generic name like:

# proxmox-backup-debug 
   inspect chunk ...
   inspect file ...
   recover index ...

I mean what is the plan for the whole CLI tool, what additional commands
could be added, I'd like to not just tape this together somehow, but have
somewhat of a design plan..

Documentation patches would be also good to have, e.g., what is the reference-filter,
full path to datastore, index, ...?

>  src/bin/proxmox_backup_recovery/inspect.rs | 195 +++++++++++++++++++++
>  src/bin/proxmox_backup_recovery/mod.rs     |   2 +
>  4 files changed, 231 insertions(+), 1 deletion(-)
>  create mode 100644 src/bin/proxmox-backup-recovery.rs
>  create mode 100644 src/bin/proxmox_backup_recovery/inspect.rs
>  create mode 100644 src/bin/proxmox_backup_recovery/mod.rs
> 
> diff --git a/Makefile b/Makefile
> index a1af9d51..a9c2a711 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -13,7 +13,8 @@ USR_BIN := \
>  
>  # Binaries usable by admins
>  USR_SBIN := \
> -	proxmox-backup-manager
> +	proxmox-backup-manager \
> +	proxmox-backup-recovery \
>  
>  # Binaries for services:
>  SERVICE_BIN := \
> diff --git a/src/bin/proxmox-backup-recovery.rs b/src/bin/proxmox-backup-recovery.rs
> new file mode 100644
> index 00000000..ae8c18d5
> --- /dev/null
> +++ b/src/bin/proxmox-backup-recovery.rs
> @@ -0,0 +1,32 @@
> +use anyhow::Error;
> +use proxmox::api::schema::{Schema, StringSchema};
> +use proxmox::api::cli::*;
> +
> +use proxmox::sys::linux::tty;
> +use proxmox_backup_recovery::*;
> +mod proxmox_backup_recovery;
> +
> +pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
> +    tty::read_password("Encryption Key Password: ")
> +}
> +
> +pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
> +
> +pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
> +    "Path to encryption key. If the data was encrypted, this key will be used for decryption.",
> +)
> +.schema();
> +
> +fn main() {
> +    proxmox_backup::tools::setup_safe_path_env();
> +
> +    let cmd_def = CliCommandMap::new()
> +        .insert("inspect", inspect_commands());
> +
> +    let rpcenv = CliEnvironment::new();
> +    run_cli_command(
> +        cmd_def,
> +        rpcenv,
> +        Some(|future| proxmox_backup::tools::runtime::main(future)),
> +    );
> +}
> diff --git a/src/bin/proxmox_backup_recovery/inspect.rs b/src/bin/proxmox_backup_recovery/inspect.rs
> new file mode 100644
> index 00000000..6bf4a0a3
> --- /dev/null
> +++ b/src/bin/proxmox_backup_recovery/inspect.rs
> @@ -0,0 +1,195 @@
> +use std::collections::HashSet;
> +use std::fs::File;
> +use std::io::Write;
> +use std::path::Path;
> +
> +use anyhow::Error;
> +use proxmox::api::cli::{
> +    format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
> +};
> +use proxmox::api::{api, cli::*, RpcEnvironment};
> +use serde_json::{json, Value};
> +use walkdir::WalkDir;
> +
> +use proxmox_backup::api2::types::SHA256_HEX_REGEX;
> +use proxmox_backup::backup::{
> +    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
> +};
> +use proxmox_backup::tools;
> +
> +use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
> +
> +/// Decodes a blob and writes its content either to stdout or into a file
> +fn decode_blob(
> +    output_path: Option<&Path>,
> +    key_file: Option<&Path>,
> +    digest: Option<&[u8; 32]>,
> +    blob: &DataBlob,
> +) -> Result<(), Error> {
> +    let mut crypt_conf_opt = None;
> +    let crypt_conf;
> +
> +    if blob.is_encrypted() && key_file.is_some() {
> +        let (key, _created, _fingerprint) =
> +            load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
> +        crypt_conf = CryptConfig::new(key)?;
> +        crypt_conf_opt = Some(&crypt_conf);
> +    }
> +
> +    if output_path.is_some() {
> +        let mut file = File::create(output_path.unwrap())?;
> +        Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
> +    } else {
> +        Ok(println!(
> +            "{}",
> +            String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
> +        ))
> +    }
> +}
> +
> +#[api(
> +    input: {
> +        properties: {
> +            chunk: {
> +                schema: PATH_SCHEMA,
> +            },
> +            "reference-filter": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "decode": {
> +                schema: PATH_SCHEMA,
> +                optional: true,
> +            },
> +            "keyfile": {
> +                schema: KEYFILE_SCHEMA,
> +                optional: true,
> +            },
> +            "output-format": {
> +                schema: OUTPUT_FORMAT,
> +                optional: true,
> +            },
> +        }
> +    }
> +)]
> +/// Inspect a chunk
> +fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
> +    let chunk_path = Path::new(tools::required_string_param(&param, "chunk")?);
> +    let output_format = get_output_format(&param);
> +    let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
> +
> +    if !SHA256_HEX_REGEX.is_match(digest_str) {
> +        println!("chunk filename is not valid");

at least to stderr, using eprintln! maybe also refer why is it not valid.

> +        return Ok(Value::Null);

maybe even use bail!(...), I mean, feels weird that we exit with 0 in such a case

> +    }
> +
> +    let digest_raw = proxmox::tools::hex_to_digest(digest_str)?;

or, you may even drop above REGEX check + if and just do:

let digest_raw = proxmox::tools::hex_to_digest(digest_str)
    .map_err(|e| format_err!("could not parse chunk - {}", e))?;

> +
> +    let reference_filter_param = param["reference-filter"].as_str();
> +    let decode_output_param = param["decode"].as_str();
> +    let key_file_param = param["keyfile"].as_str();
> +
> +    let mut search_path = None;
> +    let mut decode_output_path = None;
> +    let mut key_file_path = None;
> +    let mut to_stdout = false;
> +
> +    if let Some(path) = reference_filter_param {
> +        search_path = Some(Path::new(path))
> +    };

Rather avoid mutable plus three extra lines (noise) and do

let search_path = reference_filter_param.map(|path| Path::new(path)):

> +
> +    if let Some(path) = decode_output_param {
> +        to_stdout = path.eq("-");
> +        decode_output_path = Some(Path::new(path))

maybe a match could fit above better and do away with mutables

let (decode_output_path, to_stdout) = match decode_output_param {
    Some(path) if path.eq("-") => (None, true),
    Some(path) => (Some(path), false),
    None => (None, false),
};

just a line less total, but more explicit,  no hard feelings here though


> +    };
> +
> +    if let Some(path) = key_file_param {
> +        key_file_path = Some(Path::new(path))
> +    };

use map here

let key_file_path = key_file_param.map(|path| Path::new(path));

> +
> +    let mut file = std::fs::File::open(&chunk_path)?;

may map_err + format! can make this a better UX if the open fails

> +    let blob = DataBlob::load_from_reader(&mut file)?;
> +
> +    let mut referenced_by = None;
> +    if let Some(search_path) = search_path {
> +        let mut references = Vec::new();
> +        for entry in WalkDir::new(search_path)
> +            .follow_links(false)
> +            .into_iter()
> +            .filter_map(|e| e.ok())
> +        {
> +            let file_name = entry.file_name().to_string_lossy();
> +            let mut in_index = HashSet::new();

why is this defined here, but only used below, actually it should not be required anyway.


> +            let mut index: Option<Box<dyn IndexFile>> = None;
> +
> +            if file_name.ends_with(".fidx") {
> +                index = match FixedIndexReader::open(entry.path()) {
> +                    Ok(index) => Some(Box::new(index)),
> +                    Err(_) => None,
> +                };
> +            }
> +
> +            if file_name.ends_with(".didx") {
> +                index = match DynamicIndexReader::open(entry.path()) {
> +                    Ok(index) => Some(Box::new(index)),
> +                    Err(_) => None,
> +                };
> +            }
> +
> +            if let Some(index) = index {
> +                for pos in 0..index.index_count() {
> +                    if let Some(index_chunk_digest) = index.index_digest(pos) {
> +                        in_index.insert(proxmox::tools::digest_to_hex(index_chunk_digest));
> +                    }
> +                }
> +            }
> +

you could move below if hunk in the for loop's if above, avoiding the need for any hashset

> +            if in_index.contains(digest_str) {
> +                references.push(entry.path().to_string_lossy().into_owned());
> +            }
> +        }
> +        referenced_by = Some(references);
> +    }
> +
> +    if let Some(decode_output_path) = decode_output_path {
> +        if to_stdout {
> +            decode_blob(None, key_file_path, Some(&digest_raw), &blob)?;
> +        } else {
> +            decode_blob(
> +                Some(decode_output_path),
> +                key_file_path,
> +                Some(&digest_raw),
> +                &blob,
> +            )?;
> +        }
> +    }

Above 12 lines could be reduced with something like (untested):

if decode_output_path.is_some() {
    let decode_output_path =  if to_stdout { None } else { decode_output_path };

    decode_blob(decode_output_path, key_file_path, Some(&digest_raw), &blob)?;
}

> +
> +    let crc_status = format!(
> +        "{}({})",
> +        blob.compute_crc(),
> +        blob.verify_crc().map_or("NOK", |_x| "OK")

rather use just _ if unused param

> +    );
> +
> +    let val = match referenced_by {
> +        Some(references) => json!({
> +            "digest": digest_str,
> +            "crc": crc_status,
> +            "encryption": blob.crypt_mode()?,
> +            "referenced-by": references
> +        }),
> +        None => json!({
> +             "digest": digest_str,
> +             "crc": crc_status,
> +             "encryption": blob.crypt_mode()?,
> +        }),
> +    };
> +
> +    format_and_print_result(&val, &output_format);
> +    Ok(Value::Null)
> +}
> +
> +pub fn inspect_commands() -> CommandLineInterface {
> +    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
> +
> +    cmd_def.into()
> +}
> diff --git a/src/bin/proxmox_backup_recovery/mod.rs b/src/bin/proxmox_backup_recovery/mod.rs
> new file mode 100644
> index 00000000..644583db
> --- /dev/null
> +++ b/src/bin/proxmox_backup_recovery/mod.rs
> @@ -0,0 +1,2 @@
> +mod inspect;
> +pub use inspect::*;
> 

[pbs-devel] [PATCH proxmox-backup 2/3] add inspection of .blob, .fidx and .didx files

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to inspect .blob, .fidx and .didx files. For index
files a list of the chunks referenced will be printed in addition to
some other inforation. .blob files can be decoded into file or directly
into stdout. Options:
 - file: path to the file
 - [opt] decode: path to a file or stdout(-), if specidied, the file will be
   decoded into the specified location [only for blob files, no effect
   with index files]
 - [opt] keyfile: path to a keyfile, needed if decode is specified and the
   data was encrypted


 src/bin/proxmox_backup_recovery/inspect.rs | 115 ++++++++++++++++++++-
 1 file changed, 114 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox_backup_recovery/inspect.rs b/src/bin/proxmox_backup_recovery/inspect.rs
index 6bf4a0a3..0b8d8e48 100644
--- a/src/bin/proxmox_backup_recovery/inspect.rs
+++ b/src/bin/proxmox_backup_recovery/inspect.rs
@@ -188,8 +188,121 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
     Ok(Value::Null)
 }
 
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            "decode": {
+                schema: PATH_SCHEMA,
+                optional: true,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Inspect a file
+fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let path = tools::required_string_param(&param, "file")?;
+    let output_format = get_output_format(&param);
+
+    let val;
+    if path.ends_with(".blob") {
+        let decode_output_param = param["decode"].as_str();
+        let key_file_param = param["keyfile"].as_str();
+
+        let mut file = std::fs::File::open(&path)?;
+        let data_blob = DataBlob::load_from_reader(&mut file)?;
+
+        let mut decode_output_path = None;
+        let mut key_file_path = None;
+
+        if let Some(path) = decode_output_param {
+            decode_output_path = Some(Path::new(path))
+        };
+
+        if let Some(path) = key_file_param {
+            key_file_path = Some(Path::new(path))
+        };
+
+        let crypt_mode = data_blob.crypt_mode()?;
+        val = json!({
+            "encryption": crypt_mode,
+            "raw_size": data_blob.raw_size(),
+        });
+
+        if decode_output_path.is_some() {
+            if decode_output_param.unwrap().eq("-") {
+                decode_blob(None, key_file_path, None, &data_blob)?;
+            } else {
+                decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
+            }
+        }
+    } else if path.ends_with(".fidx") {
+        let index = FixedIndexReader::open(Path::new(path))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+
+        })
+    } else if path.ends_with("didx") {
+        let index = DynamicIndexReader::open(Path::new(path))?;
+
+        let mut ctime_str = index.ctime.to_string();
+        if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+            ctime_str = s;
+        }
+
+        let mut chunk_digests = HashSet::new();
+
+        for pos in 0..index.index_count() {
+            let digest = index.index_digest(pos).unwrap();
+            chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+        }
+
+        val = json!({
+            "size": index.size,
+            "ctime": ctime_str,
+            "chunk-digests": chunk_digests
+        })
+    } else {
+        val = Value::Null;
+        println!("Only .blob, .fidx and .didx files may be inspected");
+    }
+
+    if !val.is_null() {
+        format_and_print_result(&val, &output_format);
+    }
+
+    Ok(Value::Null)
+}
+
 pub fn inspect_commands() -> CommandLineInterface {
-    let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+    let cmd_def = CliCommandMap::new()
+        .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
+        .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
 
     cmd_def.into()
 }
-- 
2.20.1

[pbs-devel] [PATCH proxmox-backup 3/3] add restore of index files

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to restore data from an index file. Options:
 - chunks: path to the directory where the chunks are saved
 - file: the index file that should be restored(must be either .fidx or
   didx)
 - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
   needed
 - [opt] skip-crc: boolean, if true, read chunks wont be verified with their
   crc-sum, increases the restore speed by a lot

 src/bin/proxmox-backup-recovery.rs | 109 ++++++++++++++++++++++++++++-
 1 file changed, 107 insertions(+), 2 deletions(-)

diff --git a/src/bin/proxmox-backup-recovery.rs b/src/bin/proxmox-backup-recovery.rs
index ae8c18d5..c2b9fdfa 100644
--- a/src/bin/proxmox-backup-recovery.rs
+++ b/src/bin/proxmox-backup-recovery.rs
@@ -1,9 +1,21 @@
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+
 use anyhow::Error;
 use proxmox::api::schema::{Schema, StringSchema};
-use proxmox::api::cli::*;
+use proxmox::api::{api, cli::*};
+use proxmox::tools::digest_to_hex;
+use serde_json::Value;
 
 use proxmox::sys::linux::tty;
+use proxmox_backup::backup::{
+    load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
 use proxmox_backup_recovery::*;
+use std::time::Instant;
+
 mod proxmox_backup_recovery;
 
 pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
@@ -17,11 +29,104 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
 )
 .schema();
 
+#[api(
+    input: {
+        properties: {
+            file: {
+                schema: PATH_SCHEMA,
+            },
+            chunks: {
+                schema: PATH_SCHEMA,
+            },
+            "keyfile": {
+                schema: KEYFILE_SCHEMA,
+                optional: true,
+            },
+            "skip-crc": {
+                type: Boolean,
+                optional: true,
+                default: false,
+                description: "Skip the crc verification, increases the restore speed immensely",
+            }
+        }
+    }
+)]
+/// Restore a index file
+fn restore(skip_crc: bool, param: Value) -> Result<Value, Error> {
+    let start = Instant::now();
+    let file_path = Path::new(tools::required_string_param(&param, "file")?);
+    let chunks_path = Path::new(tools::required_string_param(&param, "chunks")?);
+
+    let key_file_param = param["keyfile"].as_str();
+    let mut key_file_path = None;
+
+    if let Some(path) = key_file_param {
+        key_file_path = Some(Path::new(path))
+    };
+
+    let file_name = file_path.file_name().unwrap().to_str().unwrap();
+    let mut index: Option<Box<dyn IndexFile>> = None;
+
+    if file_name.ends_with(".fidx") {
+        index = match FixedIndexReader::open(file_path) {
+            Ok(index) => Some(Box::new(index)),
+            Err(_) => None,
+        };
+    }
+
+    if file_name.ends_with(".didx") {
+        index = match DynamicIndexReader::open(file_path) {
+            Ok(index) => Some(Box::new(index)),
+            Err(_) => None,
+        };
+    }
+
+    let mut crypt_conf_opt = None;
+    let mut crypt_conf;
+
+    let output_filename = &file_name[0..file_name.len() - 5];
+    let output_path = Path::new(output_filename);
+    let mut output_file = File::create(output_path)?;
+    if let Some(index) = index {
+        for pos in 0..index.index_count() {
+            let chunk_digest = index.index_digest(pos).unwrap();
+            let digest_str = digest_to_hex(chunk_digest);
+            let digest_prefix = &digest_str[0..4];
+            let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
+            let mut chunk_file = std::fs::File::open(&chunk_path)?;
+
+            let mut data = Vec::with_capacity(1024 * 1024);
+            chunk_file.read_to_end(&mut data)?;
+            let chunk_blob = DataBlob::from_raw(data)?;
+
+            if !skip_crc {
+                chunk_blob.verify_crc()?;
+            }
+
+            if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() {
+                let (key, _created, _fingerprint) =
+                    load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?;
+                crypt_conf = CryptConfig::new(key)?;
+                crypt_conf_opt = Some(&crypt_conf);
+            }
+
+            output_file.write_all(
+                chunk_blob
+                    .decode(crypt_conf_opt, Some(chunk_digest))?
+                    .as_slice(),
+            )?;
+        }
+    }
+    println!("{} sec.", start.elapsed().as_secs_f32());
+    Ok(Value::Null)
+}
+
 fn main() {
     proxmox_backup::tools::setup_safe_path_env();
 
     let cmd_def = CliCommandMap::new()
-        .insert("inspect", inspect_commands());
+        .insert("inspect", inspect_commands())
+        .insert("restore", CliCommand::new(&API_METHOD_RESTORE));
 
     let rpcenv = CliEnvironment::new();
     run_cli_command(
-- 
2.20.1