From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id C961074190; Tue, 31 May 2022 13:18:30 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id EB617806C; Tue, 31 May 2022 13:17:33 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id DE9ED8041; Tue, 31 May 2022 13:17:27 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A742943983; Tue, 31 May 2022 13:17:27 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com, pbs-devel@lists.proxmox.com Date: Tue, 31 May 2022 13:17:23 +0200 Message-Id: <20220531111726.2972022-5-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220531111726.2972022-1-d.csapak@proxmox.com> References: <20220531111726.2972022-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.112 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH proxmox-backup 2/2] file-restore: add 'tar' option to 'extract' command X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 31 May 2022 11:18:30 -0000 if the target ist stdout, we can now either have a zip (default) or a tar.zst (with --tar 1) by making use of the new 'format' parameter of the restore daemons 'extract' api Signed-off-by: Dominik Csapak --- proxmox-file-restore/Cargo.toml | 1 + proxmox-file-restore/src/block_driver.rs | 6 +-- proxmox-file-restore/src/block_driver_qemu.rs | 4 +- proxmox-file-restore/src/main.rs | 51 ++++++++++++++----- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/proxmox-file-restore/Cargo.toml b/proxmox-file-restore/Cargo.toml index 81eb7299..222a244f 100644 --- a/proxmox-file-restore/Cargo.toml +++ b/proxmox-file-restore/Cargo.toml @@ -13,6 +13,7 @@ nix = "0.19.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time" ] } +tokio-util = { version = "0.6", features = ["io"] } pxar = { version = "0.10.1", features = [ "tokio-io" ] } diff --git a/proxmox-file-restore/src/block_driver.rs b/proxmox-file-restore/src/block_driver.rs index 0b5face9..ed8a19d0 100644 --- a/proxmox-file-restore/src/block_driver.rs +++ b/proxmox-file-restore/src/block_driver.rs @@ -55,7 +55,7 @@ pub trait BlockRestoreDriver { details: SnapRestoreDetails, img_file: String, path: Vec, - pxar: bool, + format: String, ) -> Async, Error>>; /// Return status of all running/mapped images, result value is (id, extra data), where id must @@ -101,10 +101,10 @@ pub async fn data_extract( details: SnapRestoreDetails, img_file: String, path: Vec, - pxar: bool, + format: String, ) -> Result, Error> { let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve(); - driver.data_extract(details, img_file, path, pxar).await + driver.data_extract(details, img_file, path, format).await } #[api( diff --git a/proxmox-file-restore/src/block_driver_qemu.rs b/proxmox-file-restore/src/block_driver_qemu.rs index 362fff0d..dca5681e 100644 --- a/proxmox-file-restore/src/block_driver_qemu.rs +++ b/proxmox-file-restore/src/block_driver_qemu.rs @@ -215,7 +215,7 @@ impl BlockRestoreDriver for QemuBlockDriver { details: SnapRestoreDetails, img_file: String, mut path: Vec, - pxar: bool, + format: String, ) -> Async, Error>> { async move { let client = ensure_running(&details).await?; @@ -228,7 +228,7 @@ impl BlockRestoreDriver for QemuBlockDriver { if let Err(err) = client .download( "api2/json/extract", - Some(json!({ "path": path, "pxar": pxar })), + Some(json!({ "path": path, "format": format })), &mut tx, ) .await diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs index 3420ea8e..693da091 100644 --- a/proxmox-file-restore/src/main.rs +++ b/proxmox-file-restore/src/main.rs @@ -4,8 +4,11 @@ use std::path::PathBuf; use std::sync::Arc; use anyhow::{bail, format_err, Error}; +use futures::StreamExt; use serde_json::{json, Value}; +use tokio::io::AsyncWriteExt; +use proxmox_compression::zstd::ZstdEncoder; use proxmox_router::cli::{ complete_file_name, default_table_format_options, format_and_print_result_full, get_output_format, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, ColumnConfig, @@ -18,7 +21,7 @@ use pxar::accessor::aio::Accessor; use pxar::decoder::aio::Decoder; use pbs_api_types::{BackupDir, BackupNamespace, CryptMode}; -use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq}; +use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq}; use pbs_client::tools::{ complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value, key_source::{ @@ -346,9 +349,15 @@ async fn list( description: "Group/Snapshot path.", }, "path": { - description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.", + description: "Path to restore. Directories will be restored as archive files if extracted to stdout.", type: String, }, + "tar": { + description: "If true, the resulting archive is a 'tar.zst' else it will be 'zip.", + optional: true, + default: false, + type: bool, + }, "base64": { type: Boolean, description: "If set, 'path' will be interpreted as base64 encoded.", @@ -393,6 +402,7 @@ async fn extract( base64: bool, target: Option, verbose: bool, + tar: bool, param: Value, ) -> Result<(), Error> { let repo = extract_repository_from_value(¶m)?; @@ -451,7 +461,7 @@ async fn extract( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); let decoder = Accessor::new(reader, archive_size).await?; - extract_to_target(decoder, &path, target, verbose).await?; + extract_to_target(decoder, &path, target, verbose, tar).await?; } ExtractPath::VM(file, path) => { let details = SnapRestoreDetails { @@ -467,7 +477,8 @@ async fn extract( }; if let Some(mut target) = target { - let reader = data_extract(driver, details, file, path.clone(), true).await?; + let reader = + data_extract(driver, details, file, path.clone(), "pxar".to_string()).await?; let decoder = Decoder::from_tokio(reader).await?; extract_sub_dir_seq(&target, decoder, verbose).await?; @@ -478,7 +489,9 @@ async fn extract( format_err!("unable to remove temporary .pxarexclude-cli file - {}", e) })?; } else { - let mut reader = data_extract(driver, details, file, path.clone(), false).await?; + let format = if tar { "tar.zst" } else { "zip" }; + let mut reader = + data_extract(driver, details, file, path.clone(), format.to_owned()).await?; tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?; } } @@ -495,6 +508,7 @@ async fn extract_to_target( path: &[u8], target: Option, verbose: bool, + tar: bool, ) -> Result<(), Error> where T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static, @@ -515,13 +529,26 @@ where tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout()).await?; } _ => { - create_zip( - tokio::io::stdout(), - decoder, - OsStr::from_bytes(path), - verbose, - ) - .await?; + if tar { + let (writer, reader) = tokio::io::duplex(1024 * 1024); + let path = OsStr::from_bytes(path).to_owned(); + tokio::spawn(async move { create_tar(writer, decoder, &path, verbose).await }); + let mut zstdstream = + ZstdEncoder::new(tokio_util::io::ReaderStream::new(reader))?; + let mut stdout = tokio::io::stdout(); + while let Some(buf) = zstdstream.next().await { + let buf = buf?; + stdout.write_all(&buf).await?; + } + } else { + create_zip( + tokio::io::stdout(), + decoder, + OsStr::from_bytes(path), + verbose, + ) + .await?; + } } } } -- 2.30.2