From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 843507BC32; Wed, 13 Jul 2022 11:43:23 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 7FC3A2FB30; Wed, 13 Jul 2022 11:43:23 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS; Wed, 13 Jul 2022 11:43:20 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4ABF842066; Wed, 13 Jul 2022 11:43:19 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com, pve-devel@lists.proxmox.com Date: Wed, 13 Jul 2022 11:43:14 +0200 Message-Id: <20220713094317.2423116-5-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220713094317.2423116-1-d.csapak@proxmox.com> References: <20220713094317.2423116-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.098 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH proxmox-backup v2 4/4] file-restore: add 'format' and 'zstd' parameters to 'extract' command X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 13 Jul 2022 09:43:23 -0000 if the target ist stdout, we can now specify the exact format by making use of the new 'format' parameter of the restore daemons 'extract' api note that extracting a pxar from a source pxar (container/host backups) won't work currently since we would have to reencode as pxar first Signed-off-by: Dominik Csapak --- proxmox-file-restore/Cargo.toml | 1 + proxmox-file-restore/src/block_driver.rs | 12 ++- proxmox-file-restore/src/block_driver_qemu.rs | 15 +-- proxmox-file-restore/src/main.rs | 100 +++++++++++++++--- 4 files changed, 102 insertions(+), 26 deletions(-) diff --git a/proxmox-file-restore/Cargo.toml b/proxmox-file-restore/Cargo.toml index 9ffac708..c0daf530 100644 --- a/proxmox-file-restore/Cargo.toml +++ b/proxmox-file-restore/Cargo.toml @@ -14,6 +14,7 @@ log = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time" ] } +tokio-util = { version = "0.7", features = ["io"] } pxar = { version = "0.10.1", features = [ "tokio-io" ] } diff --git a/proxmox-file-restore/src/block_driver.rs b/proxmox-file-restore/src/block_driver.rs index eb6de82c..3c774e5d 100644 --- a/proxmox-file-restore/src/block_driver.rs +++ b/proxmox-file-restore/src/block_driver.rs @@ -11,7 +11,7 @@ use serde_json::{json, Value}; use proxmox_router::cli::*; use proxmox_schema::api; -use pbs_api_types::{BackupDir, BackupNamespace}; +use pbs_api_types::{file_restore::FileRestoreFormat, BackupDir, BackupNamespace}; use pbs_client::BackupRepository; use pbs_datastore::catalog::ArchiveEntry; use pbs_datastore::manifest::BackupManifest; @@ -55,7 +55,8 @@ pub trait BlockRestoreDriver { details: SnapRestoreDetails, img_file: String, path: Vec, - pxar: bool, + format: Option, + zstd: bool, ) -> Async, Error>>; /// Return status of all running/mapped images, result value is (id, extra data), where id must @@ -101,10 +102,13 @@ pub async fn data_extract( details: SnapRestoreDetails, img_file: String, path: Vec, - pxar: bool, + format: Option, + zstd: bool, ) -> Result, Error> { let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve(); - driver.data_extract(details, img_file, path, pxar).await + driver + .data_extract(details, img_file, path, format, zstd) + .await } #[api( diff --git a/proxmox-file-restore/src/block_driver_qemu.rs b/proxmox-file-restore/src/block_driver_qemu.rs index 55d15e8d..736ae2fd 100644 --- a/proxmox-file-restore/src/block_driver_qemu.rs +++ b/proxmox-file-restore/src/block_driver_qemu.rs @@ -10,7 +10,7 @@ use serde_json::json; use proxmox_sys::fs::lock_file; -use pbs_api_types::{BackupDir, BackupNamespace}; +use pbs_api_types::{file_restore::FileRestoreFormat, BackupDir, BackupNamespace}; use pbs_client::{BackupRepository, VsockClient, DEFAULT_VSOCK_PORT}; use pbs_datastore::catalog::ArchiveEntry; @@ -217,7 +217,8 @@ impl BlockRestoreDriver for QemuBlockDriver { details: SnapRestoreDetails, img_file: String, mut path: Vec, - pxar: bool, + format: Option, + zstd: bool, ) -> Async, Error>> { async move { let client = ensure_running(&details).await?; @@ -226,13 +227,13 @@ impl BlockRestoreDriver for QemuBlockDriver { } let path = base64::encode(img_file.bytes().chain(path).collect::>()); let (mut tx, rx) = tokio::io::duplex(1024 * 4096); + let mut data = json!({ "path": path, "zstd": zstd }); + if let Some(format) = format { + data["format"] = serde_json::to_value(format)?; + } tokio::spawn(async move { if let Err(err) = client - .download( - "api2/json/extract", - Some(json!({ "path": path, "pxar": pxar })), - &mut tx, - ) + .download("api2/json/extract", Some(data), &mut tx) .await { log::error!("reading file extraction stream failed - {}", err); diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs index 562c8ca7..d5deb44a 100644 --- a/proxmox-file-restore/src/main.rs +++ b/proxmox-file-restore/src/main.rs @@ -4,8 +4,11 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{bail, format_err, Error}; +use futures::StreamExt; use serde_json::{json, Value}; +use tokio::io::AsyncWriteExt; +use proxmox_compression::zstd::ZstdEncoder; use proxmox_router::cli::{ complete_file_name, default_table_format_options, format_and_print_result_full, get_output_format, init_cli_logger, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, @@ -17,8 +20,8 @@ use proxmox_sys::fs::{create_path, CreateOptions}; use pxar::accessor::aio::Accessor; use pxar::decoder::aio::Decoder; -use pbs_api_types::{BackupDir, BackupNamespace, CryptMode}; -use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq}; +use pbs_api_types::{file_restore::FileRestoreFormat, BackupDir, BackupNamespace, CryptMode}; +use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq}; use pbs_client::tools::{ complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value, key_source::{ @@ -346,9 +349,19 @@ async fn list( description: "Group/Snapshot path.", }, "path": { - description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.", + description: "Path to restore. Directories will be restored as archive files if extracted to stdout.", type: String, }, + "format": { + type: FileRestoreFormat, + optional: true, + }, + "zstd": { + type: bool, + description: "If true, output will be zstd compressed.", + optional: true, + default: false, + }, "base64": { type: Boolean, description: "If set, 'path' will be interpreted as base64 encoded.", @@ -392,6 +405,8 @@ async fn extract( path: String, base64: bool, target: Option, + format: Option, + zstd: bool, param: Value, ) -> Result<(), Error> { let repo = extract_repository_from_value(¶m)?; @@ -450,7 +465,7 @@ async fn extract( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); let decoder = Accessor::new(reader, archive_size).await?; - extract_to_target(decoder, &path, target).await?; + extract_to_target(decoder, &path, target, format, zstd).await?; } ExtractPath::VM(file, path) => { let details = SnapRestoreDetails { @@ -466,7 +481,15 @@ async fn extract( }; if let Some(mut target) = target { - let reader = data_extract(driver, details, file, path.clone(), true).await?; + let reader = data_extract( + driver, + details, + file, + path.clone(), + Some(FileRestoreFormat::Pxar), + false, + ) + .await?; let decoder = Decoder::from_tokio(reader).await?; extract_sub_dir_seq(&target, decoder).await?; @@ -477,7 +500,8 @@ async fn extract( format_err!("unable to remove temporary .pxarexclude-cli file - {}", e) })?; } else { - let mut reader = data_extract(driver, details, file, path.clone(), false).await?; + let mut reader = + data_extract(driver, details, file, path.clone(), format, zstd).await?; tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?; } } @@ -493,29 +517,75 @@ async fn extract_to_target( decoder: Accessor, path: &[u8], target: Option, + format: Option, + zstd: bool, ) -> Result<(), Error> where T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static, { let path = if path.is_empty() { b"/" } else { path }; + let path = OsStr::from_bytes(path); + + if let Some(target) = target { + extract_sub_dir(target, decoder, path).await?; + } else { + extract_archive(decoder, path, format, zstd).await?; + } + + Ok(()) +} +async fn extract_archive( + decoder: Accessor, + path: &OsStr, + format: Option, + zstd: bool, +) -> Result<(), Error> +where + T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static, +{ + let path = path.to_owned(); let root = decoder.open_root().await?; let file = root - .lookup(OsStr::from_bytes(path)) + .lookup(&path) .await? - .ok_or_else(|| format_err!("error opening '{:?}'", path))?; + .ok_or_else(|| format_err!("error opening '{:?}'", &path))?; - if let Some(target) = target { - extract_sub_dir(target, decoder, OsStr::from_bytes(path)).await?; + let (mut writer, mut reader) = tokio::io::duplex(1024 * 1024); + if file.is_regular_file() { + match format { + Some(FileRestoreFormat::Plain) | None => {} + _ => bail!("cannot extract single files as archive"), + } + tokio::spawn( + async move { tokio::io::copy(&mut file.contents().await?, &mut writer).await }, + ); } else { - match file.kind() { - pxar::EntryKind::File { .. } => { - tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout()).await?; + match format { + Some(FileRestoreFormat::Pxar) => { + bail!("pxar target not supported for pxar source"); } - _ => { - create_zip(tokio::io::stdout(), decoder, OsStr::from_bytes(path)).await?; + Some(FileRestoreFormat::Plain) => { + bail!("plain file not supported for non-regular files"); } + Some(FileRestoreFormat::Zip) | None => { + tokio::spawn(create_zip(writer, decoder, path)); + } + Some(FileRestoreFormat::Tar) => { + tokio::spawn(create_tar(writer, decoder, path)); + } + } + } + + if zstd { + let mut zstdstream = ZstdEncoder::new(tokio_util::io::ReaderStream::new(reader))?; + let mut stdout = tokio::io::stdout(); + while let Some(buf) = zstdstream.next().await { + let buf = buf?; + stdout.write_all(&buf).await?; } + } else { + tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?; } Ok(()) -- 2.30.2