From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <d.csapak@proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by lists.proxmox.com (Postfix) with ESMTPS id 8801A6CD8D
for <pbs-devel@lists.proxmox.com>; Wed, 31 Mar 2021 11:44:51 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 7414FD20D
for <pbs-devel@lists.proxmox.com>; Wed, 31 Mar 2021 11:44:21 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com
[212.186.127.180])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (2048 bits))
(No client certificate requested)
by firstgate.proxmox.com (Proxmox) with ESMTPS id 677B1D1E6
for <pbs-devel@lists.proxmox.com>; Wed, 31 Mar 2021 11:44:19 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 300DD45983
for <pbs-devel@lists.proxmox.com>; Wed, 31 Mar 2021 11:44:19 +0200 (CEST)
From: Dominik Csapak <d.csapak@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Wed, 31 Mar 2021 11:44:16 +0200
Message-Id: <20210331094418.16609-5-d.csapak@proxmox.com>
X-Mailer: git-send-email 2.20.1
In-Reply-To: <20210331094418.16609-1-d.csapak@proxmox.com>
References: <20210331094418.16609-1-d.csapak@proxmox.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.176 Adjusted score from AWL reputation of From: address
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/,
medium trust
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
information. [rest.rs]
Subject: [pbs-devel] [PATCH proxmox-backup 4/6] server/rest: add helpers for
compression
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
X-List-Received-Date: Wed, 31 Mar 2021 09:44:51 -0000
these helpers will be used in compressing api calls and static files
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
src/server/rest.rs | 98 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
diff --git a/src/server/rest.rs b/src/server/rest.rs
index 150125ec..bf9aab81 100644
--- a/src/server/rest.rs
+++ b/src/server/rest.rs
@@ -39,6 +39,7 @@ use crate::api2::types::{Authid, Userid};
use crate::auth_helpers::*;
use crate::config::cached_user_info::CachedUserInfo;
use crate::tools;
+use crate::tools::compression::{self, CompressionMethod};
use crate::tools::ticket::Ticket;
use crate::tools::FileLogger;
@@ -525,6 +526,81 @@ fn extension_to_content_type(filename: &Path) -> (&'static str, bool) {
("application/octet-stream", false)
}
+// returns the compressed file path, and the optionial Path that needs to be written
+async fn get_compressed_file_path(
+ source: PathBuf,
+ compression: &Option<CompressionMethod>,
+) -> Result<(PathBuf, Option<PathBuf>), Error> {
+ let source_ext = source
+ .extension()
+ .and_then(|osstr| osstr.to_str())
+ .unwrap_or("");
+
+ let target_ext = if let Some(method) = compression {
+ format!("{}.{}", source_ext, method.extension())
+ } else {
+ return Ok((source, None));
+ };
+
+ let target_filename = source
+ .file_name()
+ .and_then(|osstr| osstr.to_str())
+ .ok_or_else(|| format_err!("invalid filename"))?;
+
+ let mut target: PathBuf = PathBuf::from(format!(
+ "{}/{}",
+ crate::buildcfg::PROXMOX_BACKUP_CACHE_DIR,
+ target_filename
+ ));
+ target.set_extension(target_ext);
+
+ let source_metadata = tokio::fs::metadata(&source).await?;
+ let target_metadata = if let Ok(metadata) = tokio::fs::metadata(&target).await {
+ metadata
+ } else {
+ return Ok((source, Some(target)));
+ };
+
+ if source_metadata.modified()? > target_metadata.modified()? {
+ Ok((source, Some(target)))
+ } else {
+ Ok((target, None))
+ }
+}
+
+// returns a handle to either the compressed file, or to the original one,
+// if comrpessing fails (e.g. someone else is compressing it right now)
+// in that case, the compression method is changed to None
+async fn get_possibly_compressed_file(
+ source: PathBuf,
+ compression: &mut Option<CompressionMethod>,
+) -> Result<File, Error> {
+ let (filename, to_write) = get_compressed_file_path(source, compression)
+ .await
+ .map_err(|err| http_err!(BAD_REQUEST, "failed to find compress target: {}", err))?;
+
+ // if compressing fails, return the regular file instead
+ match (&to_write, &compression) {
+ (Some(target), Some(method)) => {
+ match compression::compress_file(&filename, &target, &method).await {
+ Ok(file) => Ok(file),
+ Err(_) => {
+ // we are returning the regular file, reset compression
+ *compression = None;
+ File::open(filename)
+ .await
+ .map_err(|err| http_err!(BAD_REQUEST, "File open failed: {}", err))
+ }
+ }
+ }
+ (_, _) => {
+ File::open(filename)
+ .await
+ .map_err(|err| http_err!(BAD_REQUEST, "File open failed: {}", err))
+ }
+ }
+}
+
async fn simple_static_file_download(filename: PathBuf) -> Result<Response<Body>, Error> {
let (content_type, _nocomp) = extension_to_content_type(&filename);
@@ -598,6 +674,28 @@ enum AuthData {
ApiToken(String),
}
+fn extract_compression_method(headers: &http::HeaderMap) -> Option<CompressionMethod> {
+ let mut compression = None;
+ if let Some(raw_encoding) = headers.get(header::ACCEPT_ENCODING) {
+ if let Ok(encoding) = raw_encoding.to_str() {
+ for encoding in encoding.split(&[',', ' '][..]) {
+ if let Ok(method) = encoding.parse() {
+ if method != CompressionMethod::Deflate {
+ // fixme: implement other compressors
+ continue;
+ }
+ let method = Some(method);
+ if method > compression {
+ compression = method;
+ }
+ }
+ }
+ }
+ }
+
+ return compression;
+}
+
fn extract_auth_data(headers: &http::HeaderMap) -> Option<AuthData> {
if let Some(raw_cookie) = headers.get(header::COOKIE) {
if let Ok(cookie) = raw_cookie.to_str() {
--
2.20.1