* [pbs-devel] [PATCH proxmox-backup 1/2] add inspection of chunk files
2020-12-14 6:54 [pbs-devel] [PATCH proxmox-backup 0/2] add inspection of chunks, indexes and blobs Hannes Laimer
@ 2020-12-14 6:54 ` Hannes Laimer
2020-12-14 6:54 ` [pbs-devel] [PATCH proxmox-backup 2/2] add inspection of index and blob files Hannes Laimer
2020-12-14 8:44 ` [pbs-devel] [PATCH proxmox-backup 0/2] add inspection of chunks, indexes and blobs Thomas Lamprecht
2 siblings, 0 replies; 4+ messages in thread
From: Hannes Laimer @ 2020-12-14 6:54 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/bin/proxmox-backup-manager.rs | 8 +
src/bin/proxmox_backup_manager/inspect.rs | 201 ++++++++++++++++++++++
src/bin/proxmox_backup_manager/mod.rs | 2 +
3 files changed, 211 insertions(+)
create mode 100644 src/bin/proxmox_backup_manager/inspect.rs
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 8ad4c7dc..b2ef08de 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -13,6 +13,13 @@ use proxmox_backup::client::*;
mod proxmox_backup_manager;
use proxmox_backup_manager::*;
+use proxmox::api::schema::{Schema, StringSchema};
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
+ "Path to encryption key. If the data was encrypted, this key will be used for decryption.")
+ .schema();
async fn view_task_result(
client: HttpClient,
@@ -386,6 +393,7 @@ fn main() {
.insert("datastore", datastore_commands())
.insert("disk", disk_commands())
.insert("dns", dns_commands())
+ .insert("inspect", inspect_commands())
.insert("network", network_commands())
.insert("user", user_commands())
.insert("remote", remote_commands())
diff --git a/src/bin/proxmox_backup_manager/inspect.rs b/src/bin/proxmox_backup_manager/inspect.rs
new file mode 100644
index 00000000..d9fa736e
--- /dev/null
+++ b/src/bin/proxmox_backup_manager/inspect.rs
@@ -0,0 +1,201 @@
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::Error;
+use proxmox::api::cli::{
+ format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*, RpcEnvironment};
+use proxmox::sys::linux::tty;
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::api2::types::SHA256_HEX_REGEX;
+use proxmox_backup::backup::{
+ load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
+
+use crate::{KEYFILE_SCHEMA, PATH_SCHEMA};
+
+pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
+ tty::read_password("Encryption Key Password: ")
+}
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+ output_path: Option<&Path>,
+ key_file: Option<&Path>,
+ digest: Option<&[u8; 32]>,
+ blob: &DataBlob,
+) -> Result<(), Error> {
+ let mut crypt_conf_opt = None;
+ let crypt_conf;
+
+ if blob.is_encrypted() && key_file.is_some() {
+ let (key, _created, _fingerprint) =
+ load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+ crypt_conf = CryptConfig::new(key)?;
+ crypt_conf_opt = Some(&crypt_conf);
+ }
+
+ if output_path.is_some() {
+ let mut file = File::create(output_path.unwrap())?;
+ Ok(file.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?)
+ } else {
+ Ok(println!(
+ "{}",
+ String::from_utf8_lossy(blob.decode(crypt_conf_opt, digest)?.as_slice())
+ ))
+ }
+}
+
+#[api(
+ input: {
+ properties: {
+ chunk: {
+ schema: PATH_SCHEMA,
+ },
+ "reference-filter": {
+ schema: PATH_SCHEMA,
+ optional: true,
+ },
+ "decode": {
+ schema: PATH_SCHEMA,
+ optional: true,
+ },
+ "keyfile": {
+ schema: KEYFILE_SCHEMA,
+ optional: true,
+ },
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// Inspect a chunk
+fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+ let chunk_path = Path::new(tools::required_string_param(¶m, "chunk")?);
+ let output_format = get_output_format(¶m);
+ let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+
+ if !SHA256_HEX_REGEX.is_match(digest_str) {
+ println!("chunk filename is not valid");
+ return Ok(Value::Null);
+ }
+
+ let digest_raw = proxmox::tools::hex_to_digest(digest_str)?;
+
+ let reference_filter_param = param["reference-filter"].as_str();
+ let decode_output_param = param["decode"].as_str();
+ let key_file_param = param["keyfile"].as_str();
+
+ let mut search_path = None;
+ let mut decode_output_path = None;
+ let mut key_file_path = None;
+ let mut to_stdout = false;
+
+ if let Some(path) = reference_filter_param {
+ search_path = Some(Path::new(path))
+ };
+
+ if let Some(path) = decode_output_param {
+ to_stdout = path.eq("-");
+ decode_output_path = Some(Path::new(path))
+ };
+
+ if let Some(path) = key_file_param {
+ key_file_path = Some(Path::new(path))
+ };
+
+ let mut file = std::fs::File::open(&chunk_path)?;
+ let blob = DataBlob::load_from_reader(&mut file)?;
+
+ let mut referenced_by = None;
+ if let Some(search_path) = search_path {
+ let mut references = Vec::new();
+ for entry in WalkDir::new(search_path)
+ .follow_links(false)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ {
+ let file_name = entry.file_name().to_string_lossy();
+ let mut in_index = HashSet::new();
+ let mut index: Option<Box<dyn IndexFile>> = None;
+
+ if file_name.ends_with(".fidx") {
+ index = match FixedIndexReader::open(entry.path()) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ if file_name.ends_with(".didx") {
+ index = match DynamicIndexReader::open(entry.path()) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ if let Some(index) = index {
+ for pos in 0..index.index_count() {
+ if let Some(index_chunk_digest) = index.index_digest(pos) {
+ in_index.insert(proxmox::tools::digest_to_hex(index_chunk_digest));
+ }
+ }
+ }
+
+ if in_index.contains(digest_str) {
+ references.push(entry.path().to_string_lossy().into_owned());
+ }
+ }
+ referenced_by = Some(references);
+ }
+
+ if let Some(decode_output_path) = decode_output_path {
+ if to_stdout {
+ decode_blob(None, key_file_path, Some(&digest_raw), &blob)?;
+ } else {
+ decode_blob(
+ Some(decode_output_path),
+ key_file_path,
+ Some(&digest_raw),
+ &blob,
+ )?;
+ }
+ }
+
+ let crc_status = format!(
+ "{}({})",
+ blob.compute_crc(),
+ blob.verify_crc().map_or("NOK", |_x| "OK")
+ );
+
+ let val = match referenced_by {
+ Some(references) => json!({
+ "digest": digest_str,
+ "crc": crc_status,
+ "encryption": blob.crypt_mode()?,
+ "referenced-by": references
+ }),
+ None => json!({
+ "digest": digest_str,
+ "crc": crc_status,
+ "encryption": blob.crypt_mode()?,
+ }),
+ };
+
+ format_and_print_result(&val, &output_format);
+ Ok(Value::Null)
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+ let cmd_def = CliCommandMap::new()
+ .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+ cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_manager/mod.rs b/src/bin/proxmox_backup_manager/mod.rs
index 1f3ff92e..7b2854d7 100644
--- a/src/bin/proxmox_backup_manager/mod.rs
+++ b/src/bin/proxmox_backup_manager/mod.rs
@@ -6,6 +6,8 @@ mod datastore;
pub use datastore::*;
mod dns;
pub use dns::*;
+mod inspect;
+pub use inspect::*;
mod network;
pub use network::*;
mod remote;
--
2.20.1
^ permalink raw reply [flat|nested] 4+ messages in thread* [pbs-devel] [PATCH proxmox-backup 2/2] add inspection of index and blob files
2020-12-14 6:54 [pbs-devel] [PATCH proxmox-backup 0/2] add inspection of chunks, indexes and blobs Hannes Laimer
2020-12-14 6:54 ` [pbs-devel] [PATCH proxmox-backup 1/2] add inspection of chunk files Hannes Laimer
@ 2020-12-14 6:54 ` Hannes Laimer
2020-12-14 8:44 ` [pbs-devel] [PATCH proxmox-backup 0/2] add inspection of chunks, indexes and blobs Thomas Lamprecht
2 siblings, 0 replies; 4+ messages in thread
From: Hannes Laimer @ 2020-12-14 6:54 UTC (permalink / raw)
To: pbs-devel
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
src/bin/proxmox_backup_manager/inspect.rs | 110 ++++++++++++++++++++++
1 file changed, 110 insertions(+)
diff --git a/src/bin/proxmox_backup_manager/inspect.rs b/src/bin/proxmox_backup_manager/inspect.rs
index d9fa736e..b5587cf3 100644
--- a/src/bin/proxmox_backup_manager/inspect.rs
+++ b/src/bin/proxmox_backup_manager/inspect.rs
@@ -193,8 +193,118 @@ fn inspect_chunk(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value
Ok(Value::Null)
}
+#[api(
+ input: {
+ properties: {
+ file: {
+ schema: PATH_SCHEMA,
+ },
+ "decode": {
+ schema: PATH_SCHEMA,
+ optional: true,
+ },
+ "keyfile": {
+ schema: KEYFILE_SCHEMA,
+ optional: true,
+ },
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// Inspect a file
+fn inspect_file(param: Value, _rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+ let path = tools::required_string_param(¶m, "file")?;
+ let output_format = get_output_format(¶m);
+
+ let val;
+ if path.ends_with(".blob") {
+ let decode_output_param = param["decode"].as_str();
+ let key_file_param = param["keyfile"].as_str();
+
+ let mut file = std::fs::File::open(&path)?;
+ let data_blob = DataBlob::load_from_reader(&mut file)?;
+
+ let mut decode_output_path = None;
+ let mut key_file_path = None;
+
+ if let Some(path) = decode_output_param {
+ decode_output_path = Some(Path::new(path))
+ };
+
+ if let Some(path) = key_file_param {
+ key_file_path = Some(Path::new(path))
+ };
+
+ let crypt_mode = data_blob.crypt_mode()?;
+ val = json!({
+ "encryption": crypt_mode,
+ "raw_size": data_blob.raw_size(),
+ });
+
+ if decode_output_path.is_some() {
+ if decode_output_param.unwrap().eq("-") {
+ decode_blob(None, key_file_path, None, &data_blob)?;
+ } else {
+ decode_blob(decode_output_path, key_file_path, None, &data_blob)?;
+ }
+ }
+ } else if path.ends_with(".fidx") {
+ let index = FixedIndexReader::open(Path::new(path))?;
+
+ let mut ctime_str = index.ctime.to_string();
+ if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+ ctime_str = s;
+ }
+
+ let mut chunk_digests = HashSet::new();
+
+ for pos in 0..index.index_count() {
+ let digest = index.index_digest(pos).unwrap();
+ chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+ }
+
+ val = json!({
+ "size": index.size,
+ "ctime": ctime_str,
+ "chunk-digests": chunk_digests
+
+ })
+ } else if path.ends_with("didx") {
+ let index = DynamicIndexReader::open(Path::new(path))?;
+
+ let mut ctime_str = index.ctime.to_string();
+ if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.ctime) {
+ ctime_str = s;
+ }
+
+ let mut chunk_digests = HashSet::new();
+
+ for pos in 0..index.index_count() {
+ let digest = index.index_digest(pos).unwrap();
+ chunk_digests.insert(proxmox::tools::digest_to_hex(digest));
+ }
+
+ val = json!({
+ "size": index.size,
+ "ctime": ctime_str,
+ "chunk-digests": chunk_digests
+ })
+ } else {
+ val = Value::Null;
+ println!("Only .blob, .fidx and didx files may be inspected");
+ }
+
+ format_and_print_result(&val, &output_format);
+
+ Ok(Value::Null)
+}
+
pub fn inspect_commands() -> CommandLineInterface {
let cmd_def = CliCommandMap::new()
+ .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE))
.insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
cmd_def.into()
--
2.20.1
^ permalink raw reply [flat|nested] 4+ messages in thread