From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5FF679F065 for ; Fri, 3 Nov 2023 16:17:13 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 47E651FE5A for ; Fri, 3 Nov 2023 16:17:13 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 3 Nov 2023 16:17:08 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7DAC744350 for ; Fri, 3 Nov 2023 16:17:08 +0100 (CET) From: Maximiliano Sandoval R To: pbs-devel@lists.proxmox.com Date: Fri, 3 Nov 2023 16:17:07 +0100 Message-Id: <20231103151707.191010-2-m.sandoval@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20231103151707.191010-1-m.sandoval@proxmox.com> References: <20231103151707.191010-1-m.sandoval@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.002 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com, main.rs] Subject: [pbs-devel] [RFC proxmox-backup 2/2] client: add --compresion flag to create_backup X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 03 Nov 2023 15:17:13 -0000 This partially addresses https://bugzilla.proxmox.com/show_bug.cgi?id=4900. We take from borg the idea of passing compression levels in the form `--compression=zstd,3`. This future proof in the sense that it can accommodate future algorithms which might have more than one parameter, it also works with negative arguments. Signed-off-by: Maximiliano Sandoval R --- pbs-datastore/src/data_blob.rs | 40 +++++++++++++++++++++++++++++++ proxmox-backup-client/src/main.rs | 29 ++++++++++++++++------ 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/pbs-datastore/src/data_blob.rs b/pbs-datastore/src/data_blob.rs index 765f5a98..2ea986af 100644 --- a/pbs-datastore/src/data_blob.rs +++ b/pbs-datastore/src/data_blob.rs @@ -4,6 +4,7 @@ use anyhow::{bail, Error}; use openssl::symm::{decrypt_aead, Mode}; use proxmox_io::{ReadExt, WriteExt}; +use proxmox_schema::{Schema, StringSchema}; use pbs_api_types::CryptMode; use pbs_tools::crypt_config::CryptConfig; @@ -24,6 +25,45 @@ impl Default for Compression { } } +impl std::str::FromStr for Compression { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + if s == "none" { + return Ok(Self::None); + } + + if let Some(("zstd", level)) = s.split_once(',') { + let level = level.parse::()?; + + if zstd::compression_level_range().contains(&level) { + Ok(Self::Zstd(level)) + } else { + bail!("Invalid ZSTD compression level: {level}"); + } + } else { + bail!("Unknown compression: {s}"); + } + } +} + +impl std::fmt::Display for Compression { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Compression::None => write!(f, "none"), + Compression::Zstd(level) => write!(f, "zstd,{level}"), + } + } +} + +proxmox_serde::forward_deserialize_to_from_str!(Compression); +proxmox_serde::forward_serialize_to_display!(Compression); + +impl proxmox_schema::ApiType for Compression { + const API_SCHEMA: Schema = + StringSchema::new("Compression (e.g. 'none', 'zstd,')").schema(); +} + /// Encoded data chunk with digest and positional information pub struct ChunkInfo { pub chunk: DataBlob, diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index e5d67c7d..3aaf127e 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -526,6 +526,7 @@ struct CatalogUploadResult { fn spawn_catalog_upload( client: Arc, encrypt: bool, + compression: Compression, ) -> Result { let (catalog_tx, catalog_rx) = std::sync::mpsc::sync_channel(10); // allow to buffer 10 writes let catalog_stream = proxmox_async::blocking::StdChannelStream(catalog_rx); @@ -540,7 +541,7 @@ fn spawn_catalog_upload( let upload_options = UploadOptions { encrypt, - compression: Compression::default(), + compression, ..UploadOptions::default() }; @@ -585,6 +586,10 @@ fn spawn_catalog_upload( description: "Path to file.", } }, + "compression": { + optional: true, + type: Compression, + }, "all-file-systems": { type: Boolean, description: "Include all mounted subdirectories.", @@ -720,6 +725,16 @@ async fn create_backup( let empty = Vec::new(); let exclude_args = param["exclude"].as_array().unwrap_or(&empty); + let compression = match param["compression"].as_str() { + Some(c) => { + match Compression::from_str(c) { + Err(err) => bail!("Invalid compression '{c}': {err}"), + Ok(comp) => comp, + } + }, + None => Compression::default(), + }; + let mut pattern_list = Vec::with_capacity(exclude_args.len()); for entry in exclude_args { let entry = entry @@ -948,7 +963,7 @@ async fn create_backup( // no dry-run (BackupSpecificationType::CONFIG, false) => { let upload_options = UploadOptions { - compression: Compression::default(), + compression, encrypt: crypto.mode == CryptMode::Encrypt, ..UploadOptions::default() }; @@ -962,7 +977,7 @@ async fn create_backup( (BackupSpecificationType::LOGFILE, false) => { // fixme: remove - not needed anymore ? let upload_options = UploadOptions { - compression: Compression::default(), + compression, encrypt: crypto.mode == CryptMode::Encrypt, ..UploadOptions::default() }; @@ -977,7 +992,7 @@ async fn create_backup( // start catalog upload on first use if catalog.is_none() { let catalog_upload_res = - spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt)?; + spawn_catalog_upload(client.clone(), crypto.mode == CryptMode::Encrypt, compression)?; catalog = Some(catalog_upload_res.catalog_writer); catalog_result_rx = Some(catalog_upload_res.result); } @@ -998,7 +1013,7 @@ async fn create_backup( let upload_options = UploadOptions { previous_manifest: previous_manifest.clone(), - compression: Compression::default(), + compression, encrypt: crypto.mode == CryptMode::Encrypt, ..UploadOptions::default() }; @@ -1022,7 +1037,7 @@ async fn create_backup( let upload_options = UploadOptions { previous_manifest: previous_manifest.clone(), fixed_size: Some(size), - compression: Compression::default(), + compression, encrypt: crypto.mode == CryptMode::Encrypt, }; @@ -1077,7 +1092,7 @@ async fn create_backup( log::debug!("Upload index.json to '{}'", repo); let options = UploadOptions { - compression: Compression::default(), + compression, encrypt: false, ..UploadOptions::default() }; -- 2.39.2