From: Hannes Laimer <h.laimer@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH v6 proxmox-backup 3/3] add index recovery to pb-debug
Date: Fri, 30 Jul 2021 10:00:12 +0200 [thread overview]
Message-ID: <20210730080012.55834-4-h.laimer@proxmox.com> (raw)
In-Reply-To: <20210730080012.55834-1-h.laimer@proxmox.com>
Adds possibility to recover data from an index file. Options:
- chunks: path to the directory where the chunks are saved
- file: the index file that should be recovered(must be either .fidx or
didx)
- [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
needed
- [opt] skip-crc: boolean, if true, read chunks wont be verified with their
crc-sum, increases the restore speed by a lot
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
v6:
- remove SCHEMA ref.
- add "file" and "chunks" with .arg_param()
- remove not needed Result wrapping
v5:
- combine path for fixed and dynamic index fliles to avoid duplicate
code
v4:
- only the types of file that are passed by the user are check with the
magic number, when looking for index files just the filename ending
is checked -> don't have to open the file for that
- not sure if a function for the magic nr reading, seek reset makes
sense(?), it's just two lines
src/bin/proxmox-backup-debug.rs | 6 +-
src/bin/proxmox_backup_debug/mod.rs | 2 +
src/bin/proxmox_backup_debug/recover.rs | 119 ++++++++++++++++++++++++
3 files changed, 125 insertions(+), 2 deletions(-)
create mode 100644 src/bin/proxmox_backup_debug/recover.rs
diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs
index 897fa221..b3360c93 100644
--- a/src/bin/proxmox-backup-debug.rs
+++ b/src/bin/proxmox-backup-debug.rs
@@ -1,12 +1,14 @@
use proxmox::api::cli::*;
mod proxmox_backup_debug;
-use proxmox_backup_debug::inspect_commands;
+use proxmox_backup_debug::{inspect_commands, recover_commands};
fn main() {
proxmox_backup::tools::setup_safe_path_env();
- let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands());
+ let cmd_def = CliCommandMap::new()
+ .insert("inspect", inspect_commands())
+ .insert("recover", recover_commands());
let rpcenv = CliEnvironment::new();
run_cli_command(cmd_def, rpcenv, Some(|future| pbs_runtime::main(future)));
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
index 644583db..62df7754 100644
--- a/src/bin/proxmox_backup_debug/mod.rs
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -1,2 +1,4 @@
mod inspect;
pub use inspect::*;
+mod recover;
+pub use recover::*;
diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
new file mode 100644
index 00000000..706223b1
--- /dev/null
+++ b/src/bin/proxmox_backup_debug/recover.rs
@@ -0,0 +1,119 @@
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom, Write};
+use std::path::Path;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox::api::api;
+use proxmox::api::cli::{CliCommand, CliCommandMap, CommandLineInterface};
+use proxmox_backup::backup::{DYNAMIC_SIZED_CHUNK_INDEX_1_0, FIXED_SIZED_CHUNK_INDEX_1_0};
+use serde_json::Value;
+
+use proxmox_backup::backup::{
+ load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+
+use pbs_client::tools::key_source::get_encryption_key_password;
+
+use proxmox::tools::digest_to_hex;
+
+#[api(
+ input: {
+ properties: {
+ file: {
+ description: "Path to the index file, either .fidx or .didx.",
+ type: String,
+ },
+ chunks: {
+ description: "Path to the directorty that contains the chunks, usually <datastore>/.chunks.",
+ type: String,
+ },
+ "keyfile": {
+ description: "Path to a keyfile, if the data was encrypted, a keyfile is needed for decryption.",
+ type: String,
+ optional: true,
+ },
+ "skip-crc": {
+ description: "Skip the crc verification, increases the restore speed by lot.",
+ type: Boolean,
+ optional: true,
+ default: false,
+ }
+ }
+ }
+)]
+/// Restore the data from an index file, given the directory of where chunks
+/// are saved, the index file and a keyfile, if needed for decryption.
+fn recover_index(
+ file: String,
+ chunks: String,
+ keyfile: Option<String>,
+ skip_crc: bool,
+ _param: Value,
+) -> Result<(), Error> {
+ let file_path = Path::new(&file);
+ let chunks_path = Path::new(&chunks);
+
+ let key_file_path = keyfile.as_ref().map(Path::new);
+
+ let mut file = File::open(Path::new(&file))?;
+ let mut magic = [0; 8];
+ file.read_exact(&mut magic)?;
+ file.seek(SeekFrom::Start(0))?;
+ let index: Box<dyn IndexFile> = match magic {
+ FIXED_SIZED_CHUNK_INDEX_1_0 => Box::new(FixedIndexReader::new(file)?) as Box<dyn IndexFile>,
+ DYNAMIC_SIZED_CHUNK_INDEX_1_0 => {
+ Box::new(DynamicIndexReader::new(file)?) as Box<dyn IndexFile>
+ }
+ _ => bail!(format_err!(
+ "index file must either be a .fidx or a .didx file"
+ )),
+ };
+
+ let crypt_conf_opt = if let Some(key_file_path) = key_file_path {
+ let (key, _created, _fingerprint) =
+ load_and_decrypt_key(&key_file_path, &get_encryption_key_password)?;
+ Some(CryptConfig::new(key)?)
+ } else {
+ None
+ };
+
+ let output_filename = file_path.file_stem().unwrap().to_str().unwrap();
+ let output_path = Path::new(output_filename);
+ let mut output_file = File::create(output_path)
+ .map_err(|e| format_err!("could not create output file - {}", e))?;
+
+ let mut data = Vec::with_capacity(4 * 1024 * 1024);
+ for pos in 0..index.index_count() {
+ let chunk_digest = index.index_digest(pos).unwrap();
+ let digest_str = digest_to_hex(chunk_digest);
+ let digest_prefix = &digest_str[0..4];
+ let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
+ let mut chunk_file = std::fs::File::open(&chunk_path)
+ .map_err(|e| format_err!("could not open chunk file - {}", e))?;
+
+ data.clear();
+ chunk_file.read_to_end(&mut data)?;
+ let chunk_blob = DataBlob::from_raw(data.clone())?;
+
+ if !skip_crc {
+ chunk_blob.verify_crc()?;
+ }
+
+ output_file.write_all(
+ chunk_blob
+ .decode(crypt_conf_opt.as_ref(), Some(chunk_digest))?
+ .as_slice(),
+ )?;
+ }
+
+ Ok(())
+}
+
+pub fn recover_commands() -> CommandLineInterface {
+ let cmd_def = CliCommandMap::new().insert(
+ "index",
+ CliCommand::new(&API_METHOD_RECOVER_INDEX).arg_param(&["file", "chunks"]),
+ );
+ cmd_def.into()
+}
--
2.30.2
prev parent reply other threads:[~2021-07-30 8:11 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-30 8:00 [pbs-devel] [PATCH v6 proxmox-backup 0/3] add proxmox-backup-debug binary Hannes Laimer
2021-07-30 8:00 ` [pbs-devel] [PATCH v6 proxmox-backup 1/3] add chunk inspection to pb-debug Hannes Laimer
2021-07-30 8:00 ` [pbs-devel] [PATCH v6 proxmox-backup 2/3] add file " Hannes Laimer
2021-07-30 8:00 ` Hannes Laimer [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210730080012.55834-4-h.laimer@proxmox.com \
--to=h.laimer@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal