From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 212796AF1D for ; Mon, 25 Jan 2021 14:43:32 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1F18AAD22 for ; Mon, 25 Jan 2021 14:43:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 2D8E8AD18 for ; Mon, 25 Jan 2021 14:43:31 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E6A55460E9 for ; Mon, 25 Jan 2021 14:43:30 +0100 (CET) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Mon, 25 Jan 2021 14:42:52 +0100 Message-Id: <20210125134302.3394328-8-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210125134302.3394328-1-f.gruenbichler@proxmox.com> References: <20210125134302.3394328-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.026 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 07/15] client: factor out UploadOptions X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 25 Jan 2021 13:43:32 -0000 to reduce function signature complexity. Signed-off-by: Fabian Grünbichler --- breaks/requires corresponding patch in proxmox-backup-qemu src/bin/proxmox-backup-client.rs | 79 ++++++++++++++++++++++---------- src/client/backup_writer.rs | 44 ++++++++++-------- 2 files changed, 80 insertions(+), 43 deletions(-) diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index dce8f0b8..af3b8464 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -280,7 +280,6 @@ pub async fn api_datastore_latest_snapshot( async fn backup_directory>( client: &BackupWriter, - previous_manifest: Option>, dir_path: P, archive_name: &str, chunk_size: Option, @@ -290,8 +289,7 @@ async fn backup_directory>( catalog: Arc>>, exclude_pattern: Vec, entries_max: usize, - compress: bool, - encrypt: bool, + upload_options: UploadOptions, ) -> Result { let pxar_stream = PxarBackupStream::open( @@ -317,8 +315,12 @@ async fn backup_directory>( } }); + if upload_options.fixed_size.is_some() { + bail!("cannot backup directory with fixed chunk size!"); + } + let stats = client - .upload_stream(previous_manifest, archive_name, stream, "dynamic", None, compress, encrypt) + .upload_stream(archive_name, stream, upload_options) .await?; Ok(stats) @@ -326,14 +328,10 @@ async fn backup_directory>( async fn backup_image>( client: &BackupWriter, - previous_manifest: Option>, image_path: P, archive_name: &str, - image_size: u64, chunk_size: Option, - compress: bool, - encrypt: bool, - _verbose: bool, + upload_options: UploadOptions, ) -> Result { let path = image_path.as_ref().to_owned(); @@ -345,8 +343,12 @@ async fn backup_image>( let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4*1024*1024)); + if upload_options.fixed_size.is_none() { + bail!("cannot backup image with dynamic chunk size!"); + } + let stats = client - .upload_stream(previous_manifest, archive_name, stream, "fixed", Some(image_size), compress, encrypt) + .upload_stream(archive_name, stream, upload_options) .await?; Ok(stats) @@ -604,9 +606,15 @@ fn spawn_catalog_upload( let (catalog_result_tx, catalog_result_rx) = tokio::sync::oneshot::channel(); + let upload_options = UploadOptions { + encrypt, + compress: true, + ..UploadOptions::default() + }; + tokio::spawn(async move { let catalog_upload_result = client - .upload_stream(None, CATALOG_NAME, catalog_chunk_stream, "dynamic", None, true, encrypt) + .upload_stream(CATALOG_NAME, catalog_chunk_stream, upload_options) .await; if let Err(ref err) = catalog_upload_result { @@ -995,16 +1003,28 @@ async fn create_backup( for (backup_type, filename, target, size) in upload_list { match backup_type { BackupSpecificationType::CONFIG => { + let upload_options = UploadOptions { + compress: true, + encrypt: crypt_mode == CryptMode::Encrypt, + ..UploadOptions::default() + }; + println!("Upload config file '{}' to '{}' as {}", filename, repo, target); let stats = client - .upload_blob_from_file(&filename, &target, true, crypt_mode == CryptMode::Encrypt) + .upload_blob_from_file(&filename, &target, upload_options) .await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; } BackupSpecificationType::LOGFILE => { // fixme: remove - not needed anymore ? + let upload_options = UploadOptions { + compress: true, + encrypt: crypt_mode == CryptMode::Encrypt, + ..UploadOptions::default() + }; + println!("Upload log file '{}' to '{}' as {}", filename, repo, target); let stats = client - .upload_blob_from_file(&filename, &target, true, crypt_mode == CryptMode::Encrypt) + .upload_blob_from_file(&filename, &target, upload_options) .await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; } @@ -1019,9 +1039,15 @@ async fn create_backup( println!("Upload directory '{}' to '{}' as {}", filename, repo, target); catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?; + let upload_options = UploadOptions { + previous_manifest: previous_manifest.clone(), + compress: true, + encrypt: crypt_mode == CryptMode::Encrypt, + ..UploadOptions::default() + }; + let stats = backup_directory( &client, - previous_manifest.clone(), &filename, &target, chunk_size_opt, @@ -1031,24 +1057,27 @@ async fn create_backup( catalog.clone(), pattern_list.clone(), entries_max as usize, - true, - crypt_mode == CryptMode::Encrypt, + upload_options, ).await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; catalog.lock().unwrap().end_directory()?; } BackupSpecificationType::IMAGE => { println!("Upload image '{}' to '{:?}' as {}", filename, repo, target); + + let upload_options = UploadOptions { + previous_manifest: previous_manifest.clone(), + fixed_size: Some(size), + compress: true, + encrypt: crypt_mode == CryptMode::Encrypt, + }; + let stats = backup_image( &client, - previous_manifest.clone(), - &filename, + &filename, &target, - size, chunk_size_opt, - true, - crypt_mode == CryptMode::Encrypt, - verbose, + upload_options, ).await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; } @@ -1074,8 +1103,9 @@ async fn create_backup( if let Some(rsa_encrypted_key) = rsa_encrypted_key { let target = ENCRYPTED_KEY_BLOB_NAME; println!("Upload RSA encoded key to '{:?}' as {}", repo, target); + let options = UploadOptions { compress: false, encrypt: false, ..UploadOptions::default() }; let stats = client - .upload_blob_from_data(rsa_encrypted_key, target, false, false) + .upload_blob_from_data(rsa_encrypted_key, target, options) .await?; manifest.add_file(target.to_string(), stats.size, stats.csum, crypt_mode)?; @@ -1087,8 +1117,9 @@ async fn create_backup( if verbose { println!("Upload index.json to '{}'", repo) }; + let options = UploadOptions { compress: true, encrypt: false, ..UploadOptions::default() }; client - .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, true, false) + .upload_blob_from_data(manifest.into_bytes(), MANIFEST_BLOB_NAME, options) .await?; client.finish().await?; diff --git a/src/client/backup_writer.rs b/src/client/backup_writer.rs index e7a6d6bd..38953a45 100644 --- a/src/client/backup_writer.rs +++ b/src/client/backup_writer.rs @@ -39,6 +39,15 @@ pub struct BackupStats { pub csum: [u8; 32], } +/// Options for uploading blobs/streams to the server +#[derive(Default, Clone)] +pub struct UploadOptions { + pub previous_manifest: Option>, + pub compress: bool, + pub encrypt: bool, + pub fixed_size: Option, +} + type UploadQueueSender = mpsc::Sender<(MergedChunkInfo, Option)>; type UploadResultReceiver = oneshot::Receiver>; @@ -168,13 +177,12 @@ impl BackupWriter { &self, data: Vec, file_name: &str, - compress: bool, - encrypt: bool, + options: UploadOptions, ) -> Result { - let blob = match (encrypt, &self.crypt_config) { - (false, _) => DataBlob::encode(&data, None, compress)?, + let blob = match (options.encrypt, &self.crypt_config) { + (false, _) => DataBlob::encode(&data, None, options.compress)?, (true, None) => bail!("requested encryption without a crypt config"), - (true, Some(crypt_config)) => DataBlob::encode(&data, Some(crypt_config), compress)?, + (true, Some(crypt_config)) => DataBlob::encode(&data, Some(crypt_config), options.compress)?, }; let raw_data = blob.into_inner(); @@ -190,8 +198,7 @@ impl BackupWriter { &self, src_path: P, file_name: &str, - compress: bool, - encrypt: bool, + options: UploadOptions, ) -> Result { let src_path = src_path.as_ref(); @@ -206,34 +213,33 @@ impl BackupWriter { .await .map_err(|err| format_err!("unable to read file {:?} - {}", src_path, err))?; - self.upload_blob_from_data(contents, file_name, compress, encrypt).await + self.upload_blob_from_data(contents, file_name, options).await } pub async fn upload_stream( &self, - previous_manifest: Option>, archive_name: &str, stream: impl Stream>, - prefix: &str, - fixed_size: Option, - compress: bool, - encrypt: bool, + options: UploadOptions, ) -> Result { let known_chunks = Arc::new(Mutex::new(HashSet::new())); let mut param = json!({ "archive-name": archive_name }); - if let Some(size) = fixed_size { + let prefix = if let Some(size) = options.fixed_size { param["size"] = size.into(); - } + "fixed" + } else { + "dynamic" + }; - if encrypt && self.crypt_config.is_none() { + if options.encrypt && self.crypt_config.is_none() { bail!("requested encryption without a crypt config"); } let index_path = format!("{}_index", prefix); let close_path = format!("{}_close", prefix); - if let Some(manifest) = previous_manifest { + if let Some(manifest) = options.previous_manifest { // try, but ignore errors match archive_type(archive_name) { Ok(ArchiveType::FixedIndex) => { @@ -255,8 +261,8 @@ impl BackupWriter { stream, &prefix, known_chunks.clone(), - if encrypt { self.crypt_config.clone() } else { None }, - compress, + if options.encrypt { self.crypt_config.clone() } else { None }, + options.compress, self.verbose, ) .await?; -- 2.20.1