From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 19BFE615AA for ; Tue, 13 Oct 2020 11:50:45 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id F121BDB01 for ; Tue, 13 Oct 2020 11:50:44 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5657DDAF0 for ; Tue, 13 Oct 2020 11:50:43 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 18DC245D2B for ; Tue, 13 Oct 2020 11:50:43 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Tue, 13 Oct 2020 11:50:42 +0200 Message-Id: <20201013095042.31337-2-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201013095042.31337-1-d.csapak@proxmox.com> References: <20201013095042.31337-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.500 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [datastore.rs, atag.download] Subject: [pbs-devel] [PATCH proxmox-backup 2/2] api2/admin/datastore/pxar_file_download: download directory as zip X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 13 Oct 2020 09:50:45 -0000 by using the new ZipEncoderStream and recursively add files to it the zip only contains normal files and hardlinks (by simply copying the content), no empty directories/symlinks/etc Signed-off-by: Dominik Csapak --- src/api2/admin/datastore.rs | 132 +++++++++++++++++++++++++++++++----- www/window/FileBrowser.js | 8 +++ 2 files changed, 123 insertions(+), 17 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index c260b62d..3f22ffe7 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -2,12 +2,15 @@ use std::collections::{HashSet, HashMap}; use std::ffi::OsStr; use std::os::unix::ffi::OsStrExt; use std::sync::{Arc, Mutex}; +use std::pin::Pin; +use std::path::PathBuf; use anyhow::{bail, format_err, Error}; use futures::*; use hyper::http::request::Parts; use hyper::{header, Body, Response, StatusCode}; use serde_json::{json, Value}; +use tokio::sync::mpsc::Sender; use proxmox::api::{ api, ApiResponseFuture, ApiHandler, ApiMethod, Router, @@ -19,7 +22,7 @@ use proxmox::tools::fs::{replace_file, CreateOptions}; use proxmox::try_block; use proxmox::{http_err, identity, list_subdirs_api_method, sortable}; -use pxar::accessor::aio::Accessor; +use pxar::accessor::aio::{Accessor, FileContents, FileEntry}; use pxar::EntryKind; use crate::api2::types::*; @@ -29,7 +32,15 @@ use crate::config::datastore; use crate::config::cached_user_info::CachedUserInfo; use crate::server::WorkerTask; -use crate::tools::{self, AsyncReaderStream, WrappedReaderStream}; +use crate::tools::{ + self, + AsyncReaderStream, + WrappedReaderStream, + zip::{ + self, + ZipEncoderStream, + } +}; use crate::config::acl::{ PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, @@ -1243,6 +1254,68 @@ fn catalog( Ok(res.into()) } +fn recurse_files( + mut sender: Sender>>, + mut decoder: Accessor, + prefix: PathBuf, + file: FileEntry, +) -> Pin< + Box< + dyn Future>>, Accessor), Error>> + + Send + + 'static, + >, +> +where + T: Clone + pxar::accessor::ReadAt + Unpin + Send + Sync, +{ + Box::pin(async move { + let metadata = file.entry().metadata(); + let path = file.entry().path().strip_prefix(&prefix)?.to_path_buf(); + + match file.kind() { + EntryKind::File { .. } => { + let entry = ( + path, + metadata.stat.mtime.secs, + metadata.stat.mode as u16, + file.contents().await?, + ); + sender + .send(entry) + .await + .map_err(|err| format_err!("could not send file entry: {}", err))?; + } + EntryKind::Hardlink(_) => { + let realfile = decoder.follow_hardlink(&file).await?; + let entry = ( + path, + metadata.stat.mtime.secs, + metadata.stat.mode as u16, + realfile.contents().await?, + ); + sender + .send(entry) + .await + .map_err(|err| format_err!("could not send file entry: {}", err))?; + } + EntryKind::Directory => { + let dir = file.enter_directory().await?; + let mut readdir = dir.read_dir(); + while let Some(entry) = readdir.next().await { + let entry = entry?.decode_entry().await?; + let (sender_tmp, decoder_tmp) = recurse_files(sender, decoder, prefix.clone(), entry).await?; + sender = sender_tmp; + decoder = decoder_tmp; + } + } + _ => {} // ignore all else + }; + + Ok((sender, decoder)) + }) +} + #[sortable] pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new( &ApiHandler::AsyncHttp(&pxar_file_download), @@ -1325,22 +1398,47 @@ fn pxar_file_download( .lookup(OsStr::from_bytes(file_path)).await? .ok_or(format_err!("error opening '{:?}'", file_path))?; - let file = match file.kind() { - EntryKind::File { .. } => file, - EntryKind::Hardlink(_) => { - decoder.follow_hardlink(&file).await? - }, - // TODO symlink - other => bail!("cannot download file of type {:?}", other), - }; + let body = match file.kind() { + EntryKind::File { .. } => + Body::wrap_stream( + AsyncReaderStream::new(file.contents().await?) + .map_err(move |err| { + eprintln!("error during streaming of file '{:?}' - {}", filepath, err); + err + }) + ), + EntryKind::Hardlink(_) => + Body::wrap_stream( + AsyncReaderStream::new(decoder.follow_hardlink(&file).await?.contents().await?) + .map_err(move |err| { + eprintln!("error during streaming of hardlink '{:?}' - {}", filepath, err); + err + }) + ), + EntryKind::Directory => { + let (sender, receiver) = tokio::sync::mpsc::channel(100); + let mut prefix = PathBuf::new(); + let mut components = file.entry().path().components(); + components.next_back(); // discar last + for comp in components { + prefix.push(comp); + } - let body = Body::wrap_stream( - AsyncReaderStream::new(file.contents().await?) - .map_err(move |err| { - eprintln!("error during streaming of '{:?}' - {}", filepath, err); - err - }) - ); + crate::server::spawn_internal_task(async move { + let _ = recurse_files(sender, decoder, prefix, file).await?; + Ok::<(), Error>(()) + }); + + Body::wrap_stream( + ZipEncoderStream::new(receiver) + .map_err(move |err| { + eprintln!("error during streaming of zip '{:?}' - {}", filepath, err); + err + }) + ) + } + other => bail!("cannot download file of type {:?}", other) + }; // fixme: set other headers ? Ok(Response::builder() diff --git a/www/window/FileBrowser.js b/www/window/FileBrowser.js index 2ac50e1a..01b5d79b 100644 --- a/www/window/FileBrowser.js +++ b/www/window/FileBrowser.js @@ -87,6 +87,9 @@ Ext.define("PBS.window.FileBrowser", { }; params.filepath = data.filepath; atag.download = data.text; + if (data.type === 'd') { + atag.download += ".zip"; + } atag.href = me .buildUrl(`/api2/json/admin/datastore/${view.datastore}/pxar-file-download`, params); atag.click(); @@ -106,6 +109,11 @@ Ext.define("PBS.window.FileBrowser", { case 'f': canDownload = true; break; + case 'd': + if (data.depth > 1) { + canDownload = true; + } + break; default: break; } -- 2.20.1