From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 4BCDE7A888; Tue, 5 Jul 2022 13:43:38 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 438D72E6A4; Tue, 5 Jul 2022 13:43:38 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS; Tue, 5 Jul 2022 13:43:37 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 17478406C5; Tue, 5 Jul 2022 13:43:37 +0200 (CEST) Date: Tue, 5 Jul 2022 13:43:35 +0200 From: Wolfgang Bumiller To: Dominik Csapak Cc: pve-devel@lists.proxmox.com, pbs-devel@lists.proxmox.com Message-ID: <20220705114335.ysjgcoc4vmwjaj7r@casey.proxmox.com> References: <20220531111726.2972022-1-d.csapak@proxmox.com> <20220531111726.2972022-5-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20220531111726.2972022-5-d.csapak@proxmox.com> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.295 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [main.rs] Subject: Re: [pve-devel] [pbs-devel] [PATCH proxmox-backup 2/2] file-restore: add 'tar' option to 'extract' command X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 05 Jul 2022 11:43:38 -0000 needs a rebase ;-) On Tue, May 31, 2022 at 01:17:23PM +0200, Dominik Csapak wrote: > if the target ist stdout, we can now either have a zip (default) or a > tar.zst (with --tar 1) by making use of the new 'format' parameter > of the restore daemons 'extract' api > > Signed-off-by: Dominik Csapak > --- > proxmox-file-restore/Cargo.toml | 1 + > proxmox-file-restore/src/block_driver.rs | 6 +-- > proxmox-file-restore/src/block_driver_qemu.rs | 4 +- > proxmox-file-restore/src/main.rs | 51 ++++++++++++++----- > 4 files changed, 45 insertions(+), 17 deletions(-) > > diff --git a/proxmox-file-restore/Cargo.toml b/proxmox-file-restore/Cargo.toml > index 81eb7299..222a244f 100644 > --- a/proxmox-file-restore/Cargo.toml > +++ b/proxmox-file-restore/Cargo.toml > @@ -13,6 +13,7 @@ nix = "0.19.1" > serde = { version = "1.0", features = ["derive"] } > serde_json = "1.0" > tokio = { version = "1.6", features = [ "io-std", "rt", "rt-multi-thread", "time" ] } > +tokio-util = { version = "0.6", features = ["io"] } > > pxar = { version = "0.10.1", features = [ "tokio-io" ] } > > diff --git a/proxmox-file-restore/src/block_driver.rs b/proxmox-file-restore/src/block_driver.rs > index 0b5face9..ed8a19d0 100644 > --- a/proxmox-file-restore/src/block_driver.rs > +++ b/proxmox-file-restore/src/block_driver.rs > @@ -55,7 +55,7 @@ pub trait BlockRestoreDriver { > details: SnapRestoreDetails, > img_file: String, > path: Vec, > - pxar: bool, > + format: String, Similarly, an enum (and possibly zstd bool) would make more sense here IMO. > ) -> Async, Error>>; > > /// Return status of all running/mapped images, result value is (id, extra data), where id must > @@ -101,10 +101,10 @@ pub async fn data_extract( > details: SnapRestoreDetails, > img_file: String, > path: Vec, > - pxar: bool, > + format: String, > ) -> Result, Error> { > let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve(); > - driver.data_extract(details, img_file, path, pxar).await > + driver.data_extract(details, img_file, path, format).await > } > > #[api( > diff --git a/proxmox-file-restore/src/block_driver_qemu.rs b/proxmox-file-restore/src/block_driver_qemu.rs > index 362fff0d..dca5681e 100644 > --- a/proxmox-file-restore/src/block_driver_qemu.rs > +++ b/proxmox-file-restore/src/block_driver_qemu.rs > @@ -215,7 +215,7 @@ impl BlockRestoreDriver for QemuBlockDriver { > details: SnapRestoreDetails, > img_file: String, > mut path: Vec, > - pxar: bool, > + format: String, > ) -> Async, Error>> { > async move { > let client = ensure_running(&details).await?; > @@ -228,7 +228,7 @@ impl BlockRestoreDriver for QemuBlockDriver { > if let Err(err) = client > .download( > "api2/json/extract", > - Some(json!({ "path": path, "pxar": pxar })), > + Some(json!({ "path": path, "format": format })), > &mut tx, > ) > .await > diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs > index 3420ea8e..693da091 100644 > --- a/proxmox-file-restore/src/main.rs > +++ b/proxmox-file-restore/src/main.rs > @@ -4,8 +4,11 @@ use std::path::PathBuf; > use std::sync::Arc; > > use anyhow::{bail, format_err, Error}; > +use futures::StreamExt; > use serde_json::{json, Value}; > +use tokio::io::AsyncWriteExt; > > +use proxmox_compression::zstd::ZstdEncoder; > use proxmox_router::cli::{ > complete_file_name, default_table_format_options, format_and_print_result_full, > get_output_format, run_cli_command, CliCommand, CliCommandMap, CliEnvironment, ColumnConfig, > @@ -18,7 +21,7 @@ use pxar::accessor::aio::Accessor; > use pxar::decoder::aio::Decoder; > > use pbs_api_types::{BackupDir, BackupNamespace, CryptMode}; > -use pbs_client::pxar::{create_zip, extract_sub_dir, extract_sub_dir_seq}; > +use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq}; > use pbs_client::tools::{ > complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value, > key_source::{ > @@ -346,9 +349,15 @@ async fn list( > description: "Group/Snapshot path.", > }, > "path": { > - description: "Path to restore. Directories will be restored as .zip files if extracted to stdout.", > + description: "Path to restore. Directories will be restored as archive files if extracted to stdout.", > type: String, > }, > + "tar": { Why not stick with `format` here as well? (and the remaining hunks) > + description: "If true, the resulting archive is a 'tar.zst' else it will be 'zip.", > + optional: true, > + default: false, > + type: bool, > + }, > "base64": { > type: Boolean, > description: "If set, 'path' will be interpreted as base64 encoded.", > @@ -393,6 +402,7 @@ async fn extract( > base64: bool, > target: Option, > verbose: bool, > + tar: bool, > param: Value, > ) -> Result<(), Error> { > let repo = extract_repository_from_value(¶m)?; > @@ -451,7 +461,7 @@ async fn extract( > let archive_size = reader.archive_size(); > let reader = LocalDynamicReadAt::new(reader); > let decoder = Accessor::new(reader, archive_size).await?; > - extract_to_target(decoder, &path, target, verbose).await?; > + extract_to_target(decoder, &path, target, verbose, tar).await?; > } > ExtractPath::VM(file, path) => { > let details = SnapRestoreDetails { > @@ -467,7 +477,8 @@ async fn extract( > }; > > if let Some(mut target) = target { > - let reader = data_extract(driver, details, file, path.clone(), true).await?; > + let reader = > + data_extract(driver, details, file, path.clone(), "pxar".to_string()).await?; > let decoder = Decoder::from_tokio(reader).await?; > extract_sub_dir_seq(&target, decoder, verbose).await?; > > @@ -478,7 +489,9 @@ async fn extract( > format_err!("unable to remove temporary .pxarexclude-cli file - {}", e) > })?; > } else { > - let mut reader = data_extract(driver, details, file, path.clone(), false).await?; > + let format = if tar { "tar.zst" } else { "zip" }; > + let mut reader = > + data_extract(driver, details, file, path.clone(), format.to_owned()).await?; > tokio::io::copy(&mut reader, &mut tokio::io::stdout()).await?; > } > } > @@ -495,6 +508,7 @@ async fn extract_to_target( > path: &[u8], > target: Option, > verbose: bool, > + tar: bool, > ) -> Result<(), Error> > where > T: pxar::accessor::ReadAt + Clone + Send + Sync + Unpin + 'static, > @@ -515,13 +529,26 @@ where > tokio::io::copy(&mut file.contents().await?, &mut tokio::io::stdout()).await?; > } > _ => { > - create_zip( > - tokio::io::stdout(), > - decoder, > - OsStr::from_bytes(path), > - verbose, > - ) > - .await?; > + if tar { > + let (writer, reader) = tokio::io::duplex(1024 * 1024); > + let path = OsStr::from_bytes(path).to_owned(); > + tokio::spawn(async move { create_tar(writer, decoder, &path, verbose).await }); > + let mut zstdstream = > + ZstdEncoder::new(tokio_util::io::ReaderStream::new(reader))?; > + let mut stdout = tokio::io::stdout(); > + while let Some(buf) = zstdstream.next().await { > + let buf = buf?; > + stdout.write_all(&buf).await?; > + } > + } else { > + create_zip( > + tokio::io::stdout(), > + decoder, > + OsStr::from_bytes(path), > + verbose, > + ) > + .await?; > + } > } > } > } > -- > 2.30.2