From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <h.laimer@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 316ED60946
for <pbs-devel@lists.proxmox.com>; Tue, 15 Dec 2020 09:17:04 +0100 (CET)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 2841AEF23
for <pbs-devel@lists.proxmox.com>; Tue, 15 Dec 2020 09:16:34 +0100 (CET)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[212.186.127.180])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id C9E6FEF10
for <pbs-devel@lists.proxmox.com>; Tue, 15 Dec 2020 09:16:32 +0100 (CET)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8D1794516F
for <pbs-devel@lists.proxmox.com>; Tue, 15 Dec 2020 09:16:32 +0100 (CET)
From: Hannes Laimer <h.laimer@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Tue, 15 Dec 2020 09:16:26 +0100
Message-Id: <20201215081626.73888-4-h.laimer@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20201215081626.73888-1-h.laimer@proxmox.com>
References: <20201215081626.73888-1-h.laimer@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/,
medium trust
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
information. [proxmox-backup-recovery.rs]
Subject: [pbs-devel] [PATCH proxmox-backup 3/3] add restore of index files
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Tue, 15 Dec 2020 08:17:04 -0000
Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
Adds possibility to restore data from an index file. Options:
- chunks: path to the directory where the chunks are saved
- file: the index file that should be restored(must be either .fidx or
didx)
- [opt] keyfile: path to a keyfile, if the data was encrypted, a keyfile is
needed
- [opt] skip-crc: boolean, if true, read chunks wont be verified with their
crc-sum, increases the restore speed by a lot
src/bin/proxmox-backup-recovery.rs | 109 ++++++++++++++++++++++++++++-
1 file changed, 107 insertions(+), 2 deletions(-)
diff --git a/src/bin/proxmox-backup-recovery.rs b/src/bin/proxmox-backup-recovery.rs
index ae8c18d5..c2b9fdfa 100644
--- a/src/bin/proxmox-backup-recovery.rs
+++ b/src/bin/proxmox-backup-recovery.rs
@@ -1,9 +1,21 @@
+use std::fs::File;
+use std::io::{Read, Write};
+use std::path::Path;
+
use anyhow::Error;
use proxmox::api::schema::{Schema, StringSchema};
-use proxmox::api::cli::*;
+use proxmox::api::{api, cli::*};
+use proxmox::tools::digest_to_hex;
+use serde_json::Value;
use proxmox::sys::linux::tty;
+use proxmox_backup::backup::{
+ load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile,
+};
+use proxmox_backup::tools;
use proxmox_backup_recovery::*;
+use std::time::Instant;
+
mod proxmox_backup_recovery;
pub fn get_encryption_key_password() -> Result<Vec<u8>, Error> {
@@ -17,11 +29,104 @@ pub const KEYFILE_SCHEMA: Schema = StringSchema::new(
)
.schema();
+#[api(
+ input: {
+ properties: {
+ file: {
+ schema: PATH_SCHEMA,
+ },
+ chunks: {
+ schema: PATH_SCHEMA,
+ },
+ "keyfile": {
+ schema: KEYFILE_SCHEMA,
+ optional: true,
+ },
+ "skip-crc": {
+ type: Boolean,
+ optional: true,
+ default: false,
+ description: "Skip the crc verification, increases the restore speed immensely",
+ }
+ }
+ }
+)]
+/// Restore a index file
+fn restore(skip_crc: bool, param: Value) -> Result<Value, Error> {
+ let start = Instant::now();
+ let file_path = Path::new(tools::required_string_param(¶m, "file")?);
+ let chunks_path = Path::new(tools::required_string_param(¶m, "chunks")?);
+
+ let key_file_param = param["keyfile"].as_str();
+ let mut key_file_path = None;
+
+ if let Some(path) = key_file_param {
+ key_file_path = Some(Path::new(path))
+ };
+
+ let file_name = file_path.file_name().unwrap().to_str().unwrap();
+ let mut index: Option<Box<dyn IndexFile>> = None;
+
+ if file_name.ends_with(".fidx") {
+ index = match FixedIndexReader::open(file_path) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ if file_name.ends_with(".didx") {
+ index = match DynamicIndexReader::open(file_path) {
+ Ok(index) => Some(Box::new(index)),
+ Err(_) => None,
+ };
+ }
+
+ let mut crypt_conf_opt = None;
+ let mut crypt_conf;
+
+ let output_filename = &file_name[0..file_name.len() - 5];
+ let output_path = Path::new(output_filename);
+ let mut output_file = File::create(output_path)?;
+ if let Some(index) = index {
+ for pos in 0..index.index_count() {
+ let chunk_digest = index.index_digest(pos).unwrap();
+ let digest_str = digest_to_hex(chunk_digest);
+ let digest_prefix = &digest_str[0..4];
+ let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
+ let mut chunk_file = std::fs::File::open(&chunk_path)?;
+
+ let mut data = Vec::with_capacity(1024 * 1024);
+ chunk_file.read_to_end(&mut data)?;
+ let chunk_blob = DataBlob::from_raw(data)?;
+
+ if !skip_crc {
+ chunk_blob.verify_crc()?;
+ }
+
+ if key_file_path.is_some() && chunk_blob.is_encrypted() && crypt_conf_opt.is_none() {
+ let (key, _created, _fingerprint) =
+ load_and_decrypt_key(&key_file_path.unwrap(), &get_encryption_key_password)?;
+ crypt_conf = CryptConfig::new(key)?;
+ crypt_conf_opt = Some(&crypt_conf);
+ }
+
+ output_file.write_all(
+ chunk_blob
+ .decode(crypt_conf_opt, Some(chunk_digest))?
+ .as_slice(),
+ )?;
+ }
+ }
+ println!("{} sec.", start.elapsed().as_secs_f32());
+ Ok(Value::Null)
+}
+
fn main() {
proxmox_backup::tools::setup_safe_path_env();
let cmd_def = CliCommandMap::new()
- .insert("inspect", inspect_commands());
+ .insert("inspect", inspect_commands())
+ .insert("restore", CliCommand::new(&API_METHOD_RESTORE));
let rpcenv = CliEnvironment::new();
run_cli_command(
--
2.20.1