From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with UTF8SMTPS id 94FF67251C for ; Mon, 12 Apr 2021 15:40:13 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with UTF8SMTP id 7B3B21F85B for ; Mon, 12 Apr 2021 15:39:43 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with UTF8SMTPS id 87D171F84E for ; Mon, 12 Apr 2021 15:39:42 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with UTF8SMTP id 5576D4201E; Mon, 12 Apr 2021 15:39:42 +0200 (CEST) Message-ID: <117f5784-fc35-57cc-0f03-4e773bffd5f9@proxmox.com> Date: Mon, 12 Apr 2021 15:39:41 +0200 MIME-Version: 1.0 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:88.0) Gecko/20100101 Thunderbird/88.0 Content-Language: en-US To: Proxmox Backup Server development discussion , Hannes Laimer References: <20210219120941.2100665-1-h.laimer@proxmox.com> <20210219120941.2100665-4-h.laimer@proxmox.com> From: Dominik Csapak In-Reply-To: <20210219120941.2100665-4-h.laimer@proxmox.com> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 7bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.166 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment NICE_REPLY_A -0.001 Looks like a legit reply (A) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH v4 proxmox-backup 3/3] add index recovery to pb-debug X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 12 Apr 2021 13:40:13 -0000 small comments inline, rest looks ok On 2/19/21 13:09, Hannes Laimer wrote: > Adds possibility to recover data from an index file. Options: > - chunks: path to the directory where the chunks are saved > - file: the index file that should be recovered(must be either .fidx or > didx) > - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is > needed > - [opt] skip-crc: boolean, if true, read chunks wont be verified with their > crc-sum, increases the restore speed by a lot > > Signed-off-by: Hannes Laimer > --- > v4: > - there might be a better way to do the match magic block > > src/bin/proxmox-backup-debug.rs | 8 +- > src/bin/proxmox_backup_debug/mod.rs | 2 + > src/bin/proxmox_backup_debug/recover.rs | 117 ++++++++++++++++++++++++ > 3 files changed, 124 insertions(+), 3 deletions(-) > create mode 100644 src/bin/proxmox_backup_debug/recover.rs > > diff --git a/src/bin/proxmox-backup-debug.rs b/src/bin/proxmox-backup-debug.rs > index ae747c5d..00f06870 100644 > --- a/src/bin/proxmox-backup-debug.rs > +++ b/src/bin/proxmox-backup-debug.rs > @@ -5,7 +5,7 @@ use proxmox::api::schema::{Schema, StringSchema}; > use proxmox::sys::linux::tty; > > mod proxmox_backup_debug; > -use proxmox_backup_debug::inspect_commands; > +use proxmox_backup_debug::{inspect_commands, recover_commands}; > > pub fn get_encryption_key_password() -> Result, Error> { > tty::read_password("Encryption Key Password: ") > @@ -21,7 +21,9 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new( > fn main() { > proxmox_backup::tools::setup_safe_path_env(); > > - let cmd_def = CliCommandMap::new().insert("inspect", inspect_commands()); > + let cmd_def = CliCommandMap::new() > + .insert("inspect", inspect_commands()) > + .insert("recover", recover_commands()); > > let rpcenv = CliEnvironment::new(); > run_cli_command( > @@ -29,4 +31,4 @@ fn main() { > rpcenv, > Some(|future| proxmox_backup::tools::runtime::main(future)), > ); > -} > +} > \ No newline at end of file > diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs > index 644583db..62df7754 100644 > --- a/src/bin/proxmox_backup_debug/mod.rs > +++ b/src/bin/proxmox_backup_debug/mod.rs > @@ -1,2 +1,4 @@ > mod inspect; > pub use inspect::*; > +mod recover; > +pub use recover::*; > diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs > new file mode 100644 > index 00000000..451686bc > --- /dev/null > +++ b/src/bin/proxmox_backup_debug/recover.rs > @@ -0,0 +1,117 @@ > +use std::fs::File; > +use std::io::{Read, Seek, SeekFrom, Write}; > +use std::path::Path; > + > +use anyhow::{format_err, Error}; > + > +use proxmox::api::api; > +use proxmox::api::cli::{CliCommand, CliCommandMap, CommandLineInterface}; > +use proxmox_backup::backup::{DYNAMIC_SIZED_CHUNK_INDEX_1_0, FIXED_SIZED_CHUNK_INDEX_1_0}; > +use serde_json::Value; > + > +use proxmox_backup::backup::{ > + load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile, > +}; > + > +use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA}; > + > +use proxmox::tools::digest_to_hex; > + > +#[api( > + input: { > + properties: { > + file: { > + schema: PATH_SCHEMA, > + }, > + chunks: { > + schema: PATH_SCHEMA, > + }, > + "keyfile": { > + schema: KEYFILE_SCHEMA, > + optional: true, > + }, > + "skip-crc": { > + type: Boolean, > + optional: true, > + default: false, > + description: "Skip the crc verification, increases the restore speed immensely", > + } > + } > + } > +)] > +/// Restore the data from an index file, given the directory of where chunks > +/// are saved, the index file and a keyfile, if needed for decryption. > +fn recover_index( > + file: String, > + chunks: String, > + keyfile: Option, > + skip_crc: bool, > + _param: Value, > +) -> Result<(), Error> { > + let file_path = Path::new(&file); > + let chunks_path = Path::new(&chunks); > + > + let key_file_path = keyfile.as_ref().map(Path::new); > + > + let mut file = File::open(Path::new(&file))?; > + let mut magic = [0; 8]; > + file.read_exact(&mut magic)?; > + file.seek(SeekFrom::Start(0))?; > + let index: Box = match magic { > + FIXED_SIZED_CHUNK_INDEX_1_0 => { > + Ok(Box::new(FixedIndexReader::new(file)?) as Box) > + } > + DYNAMIC_SIZED_CHUNK_INDEX_1_0 => { > + Ok(Box::new(DynamicIndexReader::new(file)?) as Box) > + } > + _ => Err(format_err!( > + "index file must either be a .fidx or a .didx file" > + )), > + }?; > + > + let mut crypt_conf_opt = None; > + let mut crypt_conf; i do not get this, why not simply crypt_conf_opt = Some(CryptConfig::new(key)?); below ? > + > + let output_filename = file_path.file_stem().unwrap().to_str().unwrap(); > + let output_path = Path::new(output_filename); > + let mut output_file = File::create(output_path) > + .map_err(|e| format_err!("could not create output file - {}", e))?; > + > + let mut data = Vec::with_capacity(1024 * 1024); nit: we generally target 4 MiB chunks, so it would make sense to try to allocate as much here > + for pos in 0..index.index_count() { > + let chunk_digest = index.index_digest(pos).unwrap(); > + let digest_str = digest_to_hex(chunk_digest); > + let digest_prefix = &digest_str[0..4]; > + let chunk_path = chunks_path.join(digest_prefix).join(digest_str); > + let mut chunk_file = std::fs::File::open(&chunk_path) > + .map_err(|e| format_err!("could not open chunk file - {}", e))?; > + > + data.clear(); > + chunk_file.read_to_end(&mut data)?; > + let chunk_blob = DataBlob::from_raw(data.clone())?; > + > + if !skip_crc { > + chunk_blob.verify_crc()?; > + } > + > + if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() { > + let (key, _created, _fingerprint) = > + load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?; > + crypt_conf = CryptConfig::new(key)?; > + crypt_conf_opt = Some(&crypt_conf); > + } we probably want to load the key once at the start of the function and always give it to decode? this only tries to decrypt if the blob has the ENCRYPTED magic header anyway > + > + output_file.write_all( > + chunk_blob > + .decode(crypt_conf_opt, Some(chunk_digest))? > + .as_slice(), > + )?; > + } > + > + Ok(()) > +} > + > +pub fn recover_commands() -> CommandLineInterface { > + let cmd_def = CliCommandMap::new().insert("index", CliCommand::new(&API_METHOD_RECOVER_INDEX)); > + cmd_def.into() > +} >