From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 316ED60946 for ; Tue, 15 Dec 2020 09:17:04 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 2841AEF23 for ; Tue, 15 Dec 2020 09:16:34 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C9E6FEF10 for ; Tue, 15 Dec 2020 09:16:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8D1794516F for ; Tue, 15 Dec 2020 09:16:32 +0100 (CET) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Tue, 15 Dec 2020 09:16:26 +0100 Message-Id: <20201215081626.73888-4-h.laimer@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201215081626.73888-1-h.laimer@proxmox.com> References: <20201215081626.73888-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox-backup-recovery.rs] Subject: [pbs-devel] [PATCH proxmox-backup 3/3] add restore of index files X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 15 Dec 2020 08:17:04 -0000 Signed-off-by: Hannes Laimer --- Adds possibility to restore data from an index file. Options: - chunks: path to the directory where the chunks are saved - file: the index file that should be restored(must be either .fidx or didx) - [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is needed - [opt] skip-crc: boolean, if true, read chunks wont be verified with their crc-sum, increases the restore speed by a lot src/bin/proxmox-backup-recovery.rs | 109 ++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 2 deletions(-) diff --git a/src/bin/proxmox-backup-recovery.rs b/src/bin/proxmox-backup-recovery.rs index ae8c18d5..c2b9fdfa 100644 --- a/src/bin/proxmox-backup-recovery.rs +++ b/src/bin/proxmox-backup-recovery.rs @@ -1,9 +1,21 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; + use anyhow::Error; use proxmox::api::schema::{Schema, StringSchema}; -use proxmox::api::cli::*; +use proxmox::api::{api, cli::*}; +use proxmox::tools::digest_to_hex; +use serde_json::Value; use proxmox::sys::linux::tty; +use proxmox_backup::backup::{ + load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile, +}; +use proxmox_backup::tools; use proxmox_backup_recovery::*; +use std::time::Instant; + mod proxmox_backup_recovery; pub fn get_encryption_key_password() -> Result, Error> { @@ -17,11 +29,104 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new( ) .schema(); +#[api( + input: { + properties: { + file: { + schema: PATH_SCHEMA, + }, + chunks: { + schema: PATH_SCHEMA, + }, + "keyfile": { + schema: KEYFILE_SCHEMA, + optional: true, + }, + "skip-crc": { + type: Boolean, + optional: true, + default: false, + description: "Skip the crc verification, increases the restore speed immensely", + } + } + } +)] +/// Restore a index file +fn restore(skip_crc: bool, param: Value) -> Result { + let start = Instant::now(); + let file_path = Path::new(tools::required_string_param(¶m, "file")?); + let chunks_path = Path::new(tools::required_string_param(¶m, "chunks")?); + + let key_file_param = param["keyfile"].as_str(); + let mut key_file_path = None; + + if let Some(path) = key_file_param { + key_file_path = Some(Path::new(path)) + }; + + let file_name = file_path.file_name().unwrap().to_str().unwrap(); + let mut index: Option> = None; + + if file_name.ends_with(".fidx") { + index = match FixedIndexReader::open(file_path) { + Ok(index) => Some(Box::new(index)), + Err(_) => None, + }; + } + + if file_name.ends_with(".didx") { + index = match DynamicIndexReader::open(file_path) { + Ok(index) => Some(Box::new(index)), + Err(_) => None, + }; + } + + let mut crypt_conf_opt = None; + let mut crypt_conf; + + let output_filename = &file_name[0..file_name.len() - 5]; + let output_path = Path::new(output_filename); + let mut output_file = File::create(output_path)?; + if let Some(index) = index { + for pos in 0..index.index_count() { + let chunk_digest = index.index_digest(pos).unwrap(); + let digest_str = digest_to_hex(chunk_digest); + let digest_prefix = &digest_str[0..4]; + let chunk_path = chunks_path.join(digest_prefix).join(digest_str); + let mut chunk_file = std::fs::File::open(&chunk_path)?; + + let mut data = Vec::with_capacity(1024 * 1024); + chunk_file.read_to_end(&mut data)?; + let chunk_blob = DataBlob::from_raw(data)?; + + if !skip_crc { + chunk_blob.verify_crc()?; + } + + if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() { + let (key, _created, _fingerprint) = + load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?; + crypt_conf = CryptConfig::new(key)?; + crypt_conf_opt = Some(&crypt_conf); + } + + output_file.write_all( + chunk_blob + .decode(crypt_conf_opt, Some(chunk_digest))? + .as_slice(), + )?; + } + } + println!("{} sec.", start.elapsed().as_secs_f32()); + Ok(Value::Null) +} + fn main() { proxmox_backup::tools::setup_safe_path_env(); let cmd_def = CliCommandMap::new() - .insert("inspect", inspect_commands()); + .insert("inspect", inspect_commands()) + .insert("restore", CliCommand::new(&API_METHOD_RESTORE)); let rpcenv = CliEnvironment::new(); run_cli_command( -- 2.20.1