From: Hannes Laimer <h.laimer@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH v5 proxmox-backup 1/4] add chunk inspection to pb-debug
Date: Thu, 29 Apr 2021 13:00:13 +0200 [thread overview]
Message-ID: <20210429110016.1467670-2-h.laimer@proxmox.com> (raw)
In-Reply-To: <20210429110016.1467670-1-h.laimer@proxmox.com>
Adds possibility to inspect chunks and find indexes that reference the
chunk. Options:
- chunk: path to the chunk file
- [opt] decode: path to a file or to stdout(-), if specified, the chunk will
be decoded into the specified location
- [opt] keyfile: path to a keyfile, needed if decode is specified and the
data was encrypted
- [opt] reference-filter: path in which indexes that reference the chunk
should be searched, can be a group, snapshot or the whole datastore,
if not specified no references will be searched
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
v5:
- compair digests directly
- use KEYFILE_SCHEMA from proxmox_client_tools
- outfile_out_stdout now returns something that is writebale, instead
of writing directly
v4:
- left the two schemas here since they are quite specific to this binary
- output_or_stdout() directly outputs the data instead of returning
stdout or an open file (could not find a type that allows to properly
return either stdout or a file)
Makefile | 3 +-
src/bin/proxmox-backup-debug.rs | 23 ++++
src/bin/proxmox_backup_debug/inspect.rs | 162 ++++++++++++++++++++++++
src/bin/proxmox_backup_debug/mod.rs | 2 +
src/tools.rs | 12 +-
5 files changed, 200 insertions(+), 2 deletions(-)
create mode 100644 src/bin/proxmox-backup-debug.rs
create mode 100644 src/bin/proxmox_backup_debug/inspect.rs
create mode 100644 src/bin/proxmox_backup_debug/mod.rs
diff --git a/Makefile b/Makefile
index e5533561..f80e2dc4 100644
--- a/Makefile
+++ b/Makefile
@@ -17,7 +17,8 @@ USR_BIN := \
# Binaries usable by admins
USR_SBIN := \
- proxmox-backup-manager
+q proxmox-backup-manager \
+ proxmox-backup-debug \
# Binaries for services:
SERVICE_BIN := \
diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
new file mode 100644
index 00000000..fb548a31
--- /dev/null
+++ b/src/bin/proxmox-backup-debug.rs
@@ -0,0 +1,23 @@
+use proxmox::api::cli::*;
+use proxmox::api::schema::{Schema, StringSchema};
+
+mod proxmox_backup_debug;
+use proxmox_backup_debug::inspect_commands;
+
+pub mod proxmox_client_tools;
+use proxmox_client_tools::key_source::{get_encryption_key_password, KEYFILE_SCHEMA};
+
+pub const PATH_SCHEMA: Schema = StringSchema::new("Path to a file or a directory").schema();
+
+fn main() {
+ proxmox_backup::tools::setup_safe_path_env();
+
+ let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+
+ let rpcenv = CliEnvironment::new();
+ run_cli_command(
+ cmd_def,
+ rpcenv,
+ Some(|future| proxmox_backup::tools::runtime::main(future)),
+ );
+}
diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
new file mode 100644
index 00000000..dd6ee287
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -0,0 +1,162 @@
+use std::path::Path;
+
+use anyhow::{format_err, Error};
+use proxmox::api::cli::{
+ format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
+};
+use proxmox::api::{api, cli::*};
+use serde_json::{json, Value};
+use walkdir::WalkDir;
+
+use proxmox_backup::backup::{
+ load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+
+use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA};
+
+use proxmox_backup::tools::outfile_or_stdout;
+
+/// Decodes a blob and writes its content either to stdout or into a file
+fn decode_blob(
+ output_path: Option<&Path>,
+ key_file: Option<&Path>,
+ digest: Option<&[u8; 32]>,
+ blob: &DataBlob,
+) -> Result<(), Error> {
+ let mut crypt_conf_opt = None;
+ let crypt_conf;
+
+ if blob.is_encrypted() && key_file.is_some() {
+ let (key, _created, _fingerprint) =
+ load_and_decrypt_key(&key_file.unwrap(), &get_encryption_key_password)?;
+ crypt_conf = CryptConfig::new(key)?;
+ crypt_conf_opt = Some(&crypt_conf);
+ }
+
+ outfile_or_stdout(output_path)?.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?;
+ Ok(())
+}
+
+#[api(
+ input: {
+ properties: {
+ chunk: {
+ schema: PATH_SCHEMA,
+ },
+ "reference-filter": {
+ schema: PATH_SCHEMA,
+ optional: true,
+ },
+ "decode": {
+ schema: PATH_SCHEMA,
+ optional: true,
+ },
+ "keyfile": {
+ schema: KEYFILE_SCHEMA,
+ optional: true,
+ },
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// Inspect a chunk
+fn inspect_chunk(
+ chunk: String,
+ reference_filter: Option<String>,
+ decode: Option<String>,
+ keyfile: Option<String>,
+ param: Value,
+) -> Result<(), Error> {
+ let output_format = get_output_format(¶m);
+ let chunk_path = Path::new(&chunk);
+ let digest_str = chunk_path.file_name().unwrap().to_str().unwrap();
+ let digest_raw = proxmox::tools::hex_to_digest(digest_str)
+ .map_err(|e| format_err!("could not parse chunk - {}", e))?;
+
+ let search_path = reference_filter.as_ref().map(Path::new);
+ let key_file_path = keyfile.as_ref().map(Path::new);
+
+ let (decode_output_path, to_stdout) = (
+ decode.as_ref().map(Path::new),
+ decode.clone().map_or(false, |p| p.eq("-")),
+ );
+
+ let mut file = std::fs::File::open(&chunk_path)
+ .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+ let blob = DataBlob::load_from_reader(&mut file)?;
+
+ let mut referenced_by = None;
+ if let Some(search_path) = search_path {
+ let mut references = Vec::new();
+ for entry in WalkDir::new(search_path)
+ .follow_links(false)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ {
+ let file_name = entry.file_name().to_string_lossy();
+ let mut index: Option<Box<dyn IndexFile>> = None;
+
+ if file_name.ends_with(".fidx") {
+ index = match FixedIndexReader::open(entry.path()) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ if file_name.ends_with(".didx") {
+ index = match DynamicIndexReader::open(entry.path()) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ if let Some(index) = index {
+ for pos in 0..index.index_count() {
+ if let Some(index_chunk_digest) = index.index_digest(pos) {
+ if digest_raw.eq(index_chunk_digest) {
+ references.push(entry.path().to_string_lossy().into_owned());
+ break;
+ }
+ }
+ }
+ }
+ }
+ referenced_by = Some(references);
+ }
+
+ if decode_output_path.is_some() || to_stdout {
+ decode_blob(decode_output_path, key_file_path, Some(&digest_raw), &blob)?;
+ }
+
+ let crc_status = format!(
+ "{}({})",
+ blob.compute_crc(),
+ blob.verify_crc().map_or("NOK", |_| "OK")
+ );
+
+ let val = match referenced_by {
+ Some(references) => json!({
+ "digest": digest_str,
+ "crc": crc_status,
+ "encryption": blob.crypt_mode()?,
+ "referenced-by": references
+ }),
+ None => json!({
+ "digest": digest_str,
+ "crc": crc_status,
+ "encryption": blob.crypt_mode()?,
+ }),
+ };
+
+ format_and_print_result(&val, &output_format);
+ Ok(())
+}
+
+pub fn inspect_commands() -> CommandLineInterface {
+ let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK));
+
+ cmd_def.into()
+}
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
new file mode 100644
index 00000000..644583db
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -0,0 +1,2 @@
+mod inspect;
+pub use inspect::*;
diff --git a/src/tools.rs b/src/tools.rs
index 08af55e5..16ebf3fc 100644
--- a/src/tools.rs
+++ b/src/tools.rs
@@ -6,7 +6,7 @@ use std::borrow::Borrow;
use std::collections::HashMap;
use std::hash::BuildHasher;
use std::fs::File;
-use std::io::{self, BufRead, Read, Seek, SeekFrom};
+use std::io::{self, BufRead, Read, Seek, SeekFrom, stdout, Write};
use std::os::unix::io::RawFd;
use std::path::Path;
@@ -571,3 +571,13 @@ pub fn create_run_dir() -> Result<(), Error> {
let _: bool = proxmox::tools::fs::create_path(PROXMOX_BACKUP_RUN_DIR_M!(), None, None)?;
Ok(())
}
+
+/// Returns either a new file, if a path is given, or stdout, if no path is given.
+pub fn outfile_or_stdout<P: AsRef<Path>>(path: Option<P>) -> Result<Box<dyn Write>, Error> {
+ if let Some(path) = path {
+ let f = File::create(path)?;
+ Ok(Box::new(f) as Box<dyn Write>)
+ } else {
+ Ok(Box::new(stdout()) as Box<dyn Write>)
+ }
+}
--
2.20.1
next prev parent reply other threads:[~2021-04-29 11:00 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-29 11:00 [pbs-devel] [PATCH v5 proxmox-backup 0/4] add proxmox-backup-debug binary Hannes Laimer
2021-04-29 11:00 ` Hannes Laimer [this message]
2021-04-30 9:39 ` [pbs-devel] [PATCH v5 proxmox-backup 1/4] add chunk inspection to pb-debug Wolfgang Bumiller
2021-04-29 11:00 ` [pbs-devel] [PATCH v5 proxmox-backup 2/4] add ctime and size function to IndexFile trait Hannes Laimer
2021-04-30 9:41 ` [pbs-devel] applied: " Wolfgang Bumiller
2021-04-29 11:00 ` [pbs-devel] [PATCH v5 proxmox-backup 3/4] add file inspection to pb-debug Hannes Laimer
2021-04-30 9:44 ` Wolfgang Bumiller
2021-04-29 11:00 ` [pbs-devel] [PATCH v5 proxmox-backup 4/4] add index recovery " Hannes Laimer
2021-04-30 9:55 ` Wolfgang Bumiller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210429110016.1467670-2-h.laimer@proxmox.com \
--to=h.laimer@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox