From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by lists.proxmox.com (Postfix) with ESMTPS id BB375CA7D
 for <pbs-devel@lists.proxmox.com>; Tue, 12 Apr 2022 13:04:21 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
 by firstgate.proxmox.com (Proxmox) with ESMTP id B6E40DC4C
 for <pbs-devel@lists.proxmox.com>; Tue, 12 Apr 2022 13:04:21 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
 [94.136.29.106])
 (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
 key-exchange X25519 server-signature RSA-PSS (2048 bits))
 (No client certificate requested)
 by firstgate.proxmox.com (Proxmox) with ESMTPS id C5303DC34
 for <pbs-devel@lists.proxmox.com>; Tue, 12 Apr 2022 13:04:19 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
 by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9DE8740C37
 for <pbs-devel@lists.proxmox.com>; Tue, 12 Apr 2022 13:04:19 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Tue, 12 Apr 2022 13:04:17 +0200
Message-Id: <20220412110418.3360746-6-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.30.2
In-Reply-To: <20220412110418.3360746-1-d.csapak@proxmox.com>
References: <20220412110418.3360746-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results:  0
 AWL 0.139 Adjusted score from AWL reputation of From: address
 BAYES_00                 -1.9 Bayes spam probability is 0 to 1%
 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
 SPF_HELO_NONE           0.001 SPF: HELO does not publish an SPF Record
 SPF_PASS               -0.001 SPF: sender matches SPF record
 T_SCC_BODY_TEXT_LINE    -0.01 -
 URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
 http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
 information. [datastore.rs]
Subject: [pbs-devel] [PATCH proxmox-backup 2/3] api: admin/datastore: add
 tar support for pxar_file_download
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
 <pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>, 
 <mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Tue, 12 Apr 2022 11:04:21 -0000

by using the newly added 'create_tar' and the 'ZstdEncoder'

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 Cargo.toml                  |  1 +
 src/api2/admin/datastore.rs | 33 ++++++++++++++++++++++++---------
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index bd21117a..c7677c33 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -105,6 +105,7 @@ proxmox-uuid = "1"
 proxmox-serde = "0.1"
 proxmox-shared-memory = "0.2"
 proxmox-sys = { version = "0.2", features = [ "sortable-macro" ] }
+proxmox-compression = "0.1"
 
 
 proxmox-acme-rs = "0.4"
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index ef82b426..02e92640 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -12,6 +12,7 @@ use hyper::{header, Body, Response, StatusCode};
 use serde_json::{json, Value};
 use tokio_stream::wrappers::ReceiverStream;
 
+use proxmox_compression::zstd::ZstdEncoder;
 use proxmox_sys::sortable;
 use proxmox_sys::fs::{
     file_read_firstline, file_read_optional_string, replace_file, CreateOptions,
@@ -40,7 +41,7 @@ use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode,
     PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_VERIFY,
 
 };
-use pbs_client::pxar::create_zip;
+use pbs_client::pxar::{create_tar, create_zip};
 use pbs_datastore::{
     check_backup_owner, DataStore, BackupDir, BackupGroup, StoreProgress, LocalChunkReader,
     CATALOG_NAME,
@@ -1432,6 +1433,7 @@ pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new(
             ("backup-id", false,  &BACKUP_ID_SCHEMA),
             ("backup-time", false, &BACKUP_TIME_SCHEMA),
             ("filepath", false, &StringSchema::new("Base64 encoded path").schema()),
+            ("tar", true, &BooleanSchema::new("Download as .tar.zst").schema()),
         ]),
     )
 ).access(None, &Permission::Privilege(
@@ -1460,6 +1462,8 @@ pub fn pxar_file_download(
         let backup_id = required_string_param(&param, "backup-id")?;
         let backup_time = required_integer_param(&param, "backup-time")?;
 
+        let tar = param["tar"].as_bool().unwrap_or(false);
+
         let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
 
         check_priv_or_backup_owner(&datastore, backup_dir.group(), &auth_id, PRIV_DATASTORE_READ)?;
@@ -1519,15 +1523,26 @@ pub fn pxar_file_download(
                     }),
             ),
             EntryKind::Directory => {
-                let (sender, receiver) = tokio::sync::mpsc::channel(100);
+                let (sender, receiver) = tokio::sync::mpsc::channel::<Result<_, Error>>(100);
                 let channelwriter = AsyncChannelWriter::new(sender, 1024 * 1024);
-                proxmox_rest_server::spawn_internal_task(
-                    create_zip(channelwriter, decoder, path.clone(), false)
-                );
-                Body::wrap_stream(ReceiverStream::new(receiver).map_err(move |err| {
-                    eprintln!("error during streaming of zip '{:?}' - {}", path, err);
-                    err
-                }))
+                if tar {
+                    proxmox_rest_server::spawn_internal_task(
+                        create_tar(channelwriter, decoder, path.clone(), false)
+                    );
+                    let zstdstream = ZstdEncoder::new(ReceiverStream::new(receiver))?;
+                    Body::wrap_stream(zstdstream.map_err(move |err| {
+                        eprintln!("error during streaming of tar.zst '{:?}' - {}", path, err);
+                        err
+                    }))
+                } else {
+                    proxmox_rest_server::spawn_internal_task(
+                        create_zip(channelwriter, decoder, path.clone(), false)
+                    );
+                    Body::wrap_stream(ReceiverStream::new(receiver).map_err(move |err| {
+                        eprintln!("error during streaming of zip '{:?}' - {}", path, err);
+                        err
+                    }))
+                }
             }
             other => bail!("cannot download file of type {:?}", other),
         };
-- 
2.30.2