From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id BB375CA7D for ; Tue, 12 Apr 2022 13:04:21 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id B6E40DC4C for ; Tue, 12 Apr 2022 13:04:21 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C5303DC34 for ; Tue, 12 Apr 2022 13:04:19 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9DE8740C37 for ; Tue, 12 Apr 2022 13:04:19 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Tue, 12 Apr 2022 13:04:17 +0200 Message-Id: <20220412110418.3360746-6-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220412110418.3360746-1-d.csapak@proxmox.com> References: <20220412110418.3360746-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.139 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs] Subject: [pbs-devel] [PATCH proxmox-backup 2/3] api: admin/datastore: add tar support for pxar_file_download X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 12 Apr 2022 11:04:21 -0000 by using the newly added 'create_tar' and the 'ZstdEncoder' Signed-off-by: Dominik Csapak --- Cargo.toml | 1 + src/api2/admin/datastore.rs | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bd21117a..c7677c33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,6 +105,7 @@ proxmox-uuid = "1" proxmox-serde = "0.1" proxmox-shared-memory = "0.2" proxmox-sys = { version = "0.2", features = [ "sortable-macro" ] } +proxmox-compression = "0.1" proxmox-acme-rs = "0.4" diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index ef82b426..02e92640 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -12,6 +12,7 @@ use hyper::{header, Body, Response, StatusCode}; use serde_json::{json, Value}; use tokio_stream::wrappers::ReceiverStream; +use proxmox_compression::zstd::ZstdEncoder; use proxmox_sys::sortable; use proxmox_sys::fs::{ file_read_firstline, file_read_optional_string, replace_file, CreateOptions, @@ -40,7 +41,7 @@ use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_VERIFY, }; -use pbs_client::pxar::create_zip; +use pbs_client::pxar::{create_tar, create_zip}; use pbs_datastore::{ check_backup_owner, DataStore, BackupDir, BackupGroup, StoreProgress, LocalChunkReader, CATALOG_NAME, @@ -1432,6 +1433,7 @@ pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new( ("backup-id", false, &BACKUP_ID_SCHEMA), ("backup-time", false, &BACKUP_TIME_SCHEMA), ("filepath", false, &StringSchema::new("Base64 encoded path").schema()), + ("tar", true, &BooleanSchema::new("Download as .tar.zst").schema()), ]), ) ).access(None, &Permission::Privilege( @@ -1460,6 +1462,8 @@ pub fn pxar_file_download( let backup_id = required_string_param(¶m, "backup-id")?; let backup_time = required_integer_param(¶m, "backup-time")?; + let tar = param["tar"].as_bool().unwrap_or(false); + let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?; check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?; @@ -1519,15 +1523,26 @@ pub fn pxar_file_download( }), ), EntryKind::Directory => { - let (sender, receiver) = tokio::sync::mpsc::channel(100); + let (sender, receiver) = tokio::sync::mpsc::channel::>(100); let channelwriter = AsyncChannelWriter::new(sender, 1024 * 1024); - proxmox_rest_server::spawn_internal_task( - create_zip(channelwriter, decoder, path.clone(), false) - ); - Body::wrap_stream(ReceiverStream::new(receiver).map_err(move |err| { - eprintln!("error during streaming of zip '{:?}' - {}", path, err); - err - })) + if tar { + proxmox_rest_server::spawn_internal_task( + create_tar(channelwriter, decoder, path.clone(), false) + ); + let zstdstream = ZstdEncoder::new(ReceiverStream::new(receiver))?; + Body::wrap_stream(zstdstream.map_err(move |err| { + eprintln!("error during streaming of tar.zst '{:?}' - {}", path, err); + err + })) + } else { + proxmox_rest_server::spawn_internal_task( + create_zip(channelwriter, decoder, path.clone(), false) + ); + Body::wrap_stream(ReceiverStream::new(receiver).map_err(move |err| { + eprintln!("error during streaming of zip '{:?}' - {}", path, err); + err + })) + } } other => bail!("cannot download file of type {:?}", other), }; -- 2.30.2