From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 8801A6CD8D for ; Wed, 31 Mar 2021 11:44:51 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 7414FD20D for ; Wed, 31 Mar 2021 11:44:21 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 677B1D1E6 for ; Wed, 31 Mar 2021 11:44:19 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 300DD45983 for ; Wed, 31 Mar 2021 11:44:19 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Wed, 31 Mar 2021 11:44:16 +0200 Message-Id: <20210331094418.16609-5-d.csapak@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210331094418.16609-1-d.csapak@proxmox.com> References: <20210331094418.16609-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.176 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [rest.rs] Subject: [pbs-devel] [PATCH proxmox-backup 4/6] server/rest: add helpers for compression X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 31 Mar 2021 09:44:51 -0000 these helpers will be used in compressing api calls and static files Signed-off-by: Dominik Csapak --- src/server/rest.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/src/server/rest.rs b/src/server/rest.rs index 150125ec..bf9aab81 100644 --- a/src/server/rest.rs +++ b/src/server/rest.rs @@ -39,6 +39,7 @@ use crate::api2::types::{Authid, Userid}; use crate::auth_helpers::*; use crate::config::cached_user_info::CachedUserInfo; use crate::tools; +use crate::tools::compression::{self, CompressionMethod}; use crate::tools::ticket::Ticket; use crate::tools::FileLogger; @@ -525,6 +526,81 @@ fn extension_to_content_type(filename: &Path) -> (&'static str, bool) { ("application/octet-stream", false) } +// returns the compressed file path, and the optionial Path that needs to be written +async fn get_compressed_file_path( + source: PathBuf, + compression: &Option, +) -> Result<(PathBuf, Option), Error> { + let source_ext = source + .extension() + .and_then(|osstr| osstr.to_str()) + .unwrap_or(""); + + let target_ext = if let Some(method) = compression { + format!("{}.{}", source_ext, method.extension()) + } else { + return Ok((source, None)); + }; + + let target_filename = source + .file_name() + .and_then(|osstr| osstr.to_str()) + .ok_or_else(|| format_err!("invalid filename"))?; + + let mut target: PathBuf = PathBuf::from(format!( + "{}/{}", + crate::buildcfg::PROXMOX_BACKUP_CACHE_DIR, + target_filename + )); + target.set_extension(target_ext); + + let source_metadata = tokio::fs::metadata(&source).await?; + let target_metadata = if let Ok(metadata) = tokio::fs::metadata(&target).await { + metadata + } else { + return Ok((source, Some(target))); + }; + + if source_metadata.modified()? > target_metadata.modified()? { + Ok((source, Some(target))) + } else { + Ok((target, None)) + } +} + +// returns a handle to either the compressed file, or to the original one, +// if comrpessing fails (e.g. someone else is compressing it right now) +// in that case, the compression method is changed to None +async fn get_possibly_compressed_file( + source: PathBuf, + compression: &mut Option, +) -> Result { + let (filename, to_write) = get_compressed_file_path(source, compression) + .await + .map_err(|err| http_err!(BAD_REQUEST, "failed to find compress target: {}", err))?; + + // if compressing fails, return the regular file instead + match (&to_write, &compression) { + (Some(target), Some(method)) => { + match compression::compress_file(&filename, &target, &method).await { + Ok(file) => Ok(file), + Err(_) => { + // we are returning the regular file, reset compression + *compression = None; + File::open(filename) + .await + .map_err(|err| http_err!(BAD_REQUEST, "File open failed: {}", err)) + } + } + } + (_, _) => { + File::open(filename) + .await + .map_err(|err| http_err!(BAD_REQUEST, "File open failed: {}", err)) + } + } +} + async fn simple_static_file_download(filename: PathBuf) -> Result, Error> { let (content_type, _nocomp) = extension_to_content_type(&filename); @@ -598,6 +674,28 @@ enum AuthData { ApiToken(String), } +fn extract_compression_method(headers: &http::HeaderMap) -> Option { + let mut compression = None; + if let Some(raw_encoding) = headers.get(header::ACCEPT_ENCODING) { + if let Ok(encoding) = raw_encoding.to_str() { + for encoding in encoding.split(&[',', ' '][..]) { + if let Ok(method) = encoding.parse() { + if method != CompressionMethod::Deflate { + // fixme: implement other compressors + continue; + } + let method = Some(method); + if method > compression { + compression = method; + } + } + } + } + } + + return compression; +} + fn extract_auth_data(headers: &http::HeaderMap) -> Option { if let Some(raw_cookie) = headers.get(header::COOKIE) { if let Ok(cookie) = raw_cookie.to_str() { -- 2.20.1