From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id AF1DB73F87; Tue, 31 May 2022 13:18:01 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A68D28067; Tue, 31 May 2022 13:17:31 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 78D6C802D; Tue, 31 May 2022 13:17:27 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4F594429B2; Tue, 31 May 2022 13:17:27 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com, pbs-devel@lists.proxmox.com Date: Tue, 31 May 2022 13:17:22 +0200 Message-Id: <20220531111726.2972022-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220531111726.2972022-1-d.csapak@proxmox.com> References: <20220531111726.2972022-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.113 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH proxmox-backup 1/2] restore-daemon: add 'format' parameter to the 'extract' handler X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 31 May 2022 11:18:01 -0000 this can be 'plain', 'pxar', 'zip' or 'tar.zst', and it returns the content in the given format (with fallback to the old behaviour if not given) Signed-off-by: Dominik Csapak --- note: needs a bumped 'proxmox-compression' in the Cargo.toml to build .../src/proxmox_restore_daemon/api.rs | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs index aeb5a71d..c4977ce6 100644 --- a/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs +++ b/proxmox-restore-daemon/src/proxmox_restore_daemon/api.rs @@ -13,7 +13,7 @@ use serde_json::Value; use tokio::sync::Semaphore; use pathpatterns::{MatchEntry, MatchPattern, MatchType, Pattern}; -use proxmox_compression::zip::zip_directory; +use proxmox_compression::{tar::tar_directory, zip::zip_directory, zstd::ZstdEncoder}; use proxmox_router::{ list_subdirs_api_method, ApiHandler, ApiMethod, ApiResponseFuture, Permission, Router, RpcEnvironment, SubdirMap, @@ -236,11 +236,24 @@ pub const API_METHOD_EXTRACT: ApiMethod = ApiMethod::new( true, &BooleanSchema::new(concat!( "if true, return a pxar archive, otherwise either the ", - "file content or the directory as a zip file" + "file content or the directory as a zip file. DEPRECATED: use 'format' instead." )) .default(true) .schema() - ) + ), + ( + "format", + true, + &StringSchema::new("The desired format of the result.") + .format(&ApiStringFormat::Enum(&[ + EnumEntry::new("plain", "Plain file (only works for single files)"), + EnumEntry::new("pxar", "PXAR archive"), + EnumEntry::new("zip", "ZIP archive"), + EnumEntry::new("tar.zst", "Zstd compressed TAR archive"), + ])) + .default("pxar") + .schema() + ), ]), ), ) @@ -271,6 +284,11 @@ fn extract( let path = Path::new(OsStr::from_bytes(&path[..])); let pxar = param["pxar"].as_bool().unwrap_or(true); + let format = match param["format"].as_str() { + Some(format) => format.to_string(), + // FIXME: old default was plain or zip, remove that with 3.0 + None => if pxar { "pxar".to_string() } else { String::new() } + }; let query_result = proxmox_async::runtime::block_in_place(move || { let mut disk_state = crate::DISK_STATE.lock().unwrap(); @@ -290,7 +308,9 @@ fn extract( let (mut writer, reader) = tokio::io::duplex(1024 * 64); - if pxar { + let mut zstd = false; + + if format == "pxar" { tokio::spawn(async move { let _inhibitor = _inhibitor; let _permit = _permit; @@ -349,12 +369,24 @@ fn extract( error!("pxar streaming task failed - {}", err); } }); + } else if format == "tar.zst" { + zstd = true; + tokio::spawn(async move { + let _inhibitor = _inhibitor; + let _permit = _permit; + if let Err(err) = tar_directory(&mut writer, &vm_path).await { + error!("file or dir streaming task failed - {}", err); + } + }); } else { + if format == "plain" && vm_path.is_dir() { + bail!("cannot stream dir with format 'plain'"); + } tokio::spawn(async move { let _inhibitor = _inhibitor; let _permit = _permit; let result = async move { - if vm_path.is_dir() { + if vm_path.is_dir() || format == "zip" { zip_directory(&mut writer, &vm_path).await?; Ok(()) } else if vm_path.is_file() { @@ -377,7 +409,12 @@ fn extract( let stream = tokio_util::io::ReaderStream::new(reader); - let body = Body::wrap_stream(stream); + let body = if zstd { + let stream = ZstdEncoder::new(stream)?; + Body::wrap_stream(stream) + } else { + Body::wrap_stream(stream) + }; Ok(Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, "application/octet-stream") -- 2.30.2