From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id CAF201FF13D for ; Thu, 08 Jan 2026 12:24:44 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BC6DE20957; Thu, 8 Jan 2026 11:44:15 +0100 (CET) Message-ID: <91b0f702-94a2-49c0-8bed-727396c78005@proxmox.com> Date: Thu, 8 Jan 2026 11:44:11 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird From: Christian Ebner To: Proxmox Backup Server development discussion , Robert Obkircher References: <20251219161850.244154-1-r.obkircher@proxmox.com> <20251219161850.244154-4-r.obkircher@proxmox.com> Content-Language: en-US, de-DE In-Reply-To: <20251219161850.244154-4-r.obkircher@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1767869013855 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.048 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH v2 proxmox-backup 3/5] fix #3847: client: support fifo pipe inputs for images X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" some comments inline On 12/19/25 5:19 PM, Robert Obkircher wrote: > Accept fifo files as inputs for images and omit the size when > uploading the fixed index file. > > Signed-off-by: Robert Obkircher > --- > pbs-client/src/backup_writer.rs | 37 ++++++++++++++++++++++--------- > proxmox-backup-client/src/main.rs | 30 ++++++++++++++----------- > src/server/push.rs | 13 ++++++----- > 3 files changed, 51 insertions(+), 29 deletions(-) > > diff --git a/pbs-client/src/backup_writer.rs b/pbs-client/src/backup_writer.rs > index dbd177d8..1963b700 100644 > --- a/pbs-client/src/backup_writer.rs > +++ b/pbs-client/src/backup_writer.rs > @@ -52,7 +52,16 @@ pub struct UploadOptions { > pub previous_manifest: Option>, > pub compress: bool, > pub encrypt: bool, > - pub fixed_size: Option, > + pub chunk_size: ChunkSize, above is ill-named as this is not the chunk size, but rather the image file size. I suggest to rename this to ``` index_type: IndexType ``` or another even better fitting name and define the IndexType with tuple enum variant for the size > +} > + > +#[derive(Default, Clone)] > +pub enum ChunkSize { > + #[default] > + Dynamic, > + Fixed { > + file_size: Option, > + }, /// Index type for upload options pub enum IndexType { #[default] /// Dynamic chunking Dynamic, /// Fixed size chunking with optional image file size Fixed(Option), } > } > > struct ChunkUploadResponse { > @@ -292,11 +301,14 @@ impl BackupWriter { > options: UploadOptions, > ) -> Result { > let mut param = json!({ "archive-name": archive_name }); > - let prefix = if let Some(size) = options.fixed_size { > - param["size"] = size.into(); > - "fixed" > - } else { > - "dynamic" > + let prefix = match options.chunk_size { > + ChunkSize::Fixed { file_size } => { ... above makes this to ``` IndexType::Fixed(file_size) => { ``` > + if let Some(size) = file_size { > + param["size"] = size.into(); > + } > + "fixed" > + } > + ChunkSize::Dynamic => "dynamic", and ``` IndexType::Dynamic => "dynamic", ``` as well as for other occurences. > }; > > if options.encrypt && self.crypt_config.is_none() { > @@ -387,11 +399,14 @@ impl BackupWriter { > let known_chunks = Arc::new(Mutex::new(HashSet::new())); > > let mut param = json!({ "archive-name": archive_name }); > - let prefix = if let Some(size) = options.fixed_size { > - param["size"] = size.into(); > - "fixed" > - } else { > - "dynamic" > + let prefix = match options.chunk_size { > + ChunkSize::Fixed { file_size } => { > + if let Some(size) = file_size { > + param["size"] = size.into(); > + } > + "fixed" > + } > + ChunkSize::Dynamic => "dynamic", > }; > > if options.encrypt && self.crypt_config.is_none() { > diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs > index 999e5020..828643da 100644 > --- a/proxmox-backup-client/src/main.rs > +++ b/proxmox-backup-client/src/main.rs > @@ -46,7 +46,7 @@ use pbs_client::tools::{ > use pbs_client::{ > delete_ticket_info, parse_backup_specification, view_task_result, BackupDetectionMode, > BackupReader, BackupRepository, BackupSpecificationType, BackupStats, BackupWriter, > - BackupWriterOptions, ChunkStream, FixedChunkStream, HttpClient, InjectionData, > + BackupWriterOptions, ChunkSize, ChunkStream, FixedChunkStream, HttpClient, InjectionData, > PxarBackupStream, RemoteChunkReader, UploadOptions, BACKUP_SOURCE_SCHEMA, > }; > use pbs_datastore::catalog::{BackupCatalogWriter, CatalogReader, CatalogWriter}; > @@ -205,7 +205,7 @@ async fn backup_directory>( > pxar_create_options: pbs_client::pxar::PxarCreateOptions, > upload_options: UploadOptions, > ) -> Result<(BackupStats, Option), Error> { > - if upload_options.fixed_size.is_some() { > + if let ChunkSize::Fixed { .. } = upload_options.chunk_size { > bail!("cannot backup directory with fixed chunk size!"); > } > > @@ -295,7 +295,7 @@ async fn backup_image>( > > let stream = FixedChunkStream::new(stream, chunk_size.unwrap_or(4 * 1024 * 1024)); > > - if upload_options.fixed_size.is_none() { > + if let ChunkSize::Dynamic = upload_options.chunk_size { > bail!("cannot backup image with dynamic chunk size!"); > } > > @@ -859,15 +859,17 @@ async fn create_backup( > upload_list.push((BackupSpecificationType::PXAR, filename, target, "didx", 0)); > } > BackupSpecificationType::IMAGE => { > - if !(file_type.is_file() || file_type.is_block_device()) { > - bail!("got unexpected file type (expected file or block device)"); > - } > - > - let size = image_size(&PathBuf::from(&filename))?; > - > - if size == 0 { > - bail!("got zero-sized file '{}'", filename); > - } > + let size = if file_type.is_file() || file_type.is_block_device() { > + let size = image_size(&PathBuf::from(&filename))?; > + if size == 0 { > + bail!("got zero-sized file '{}'", filename); > + } > + size > + } else if file_type.is_fifo() { > + 0 > + } else { > + bail!("got unexpected file type (expected file, block device, or fifo"); > + }; > > upload_list.push(( > BackupSpecificationType::IMAGE, > @@ -1191,9 +1193,11 @@ async fn create_backup( > (BackupSpecificationType::IMAGE, false) => { > log_file("image", &filename, target.as_ref()); > > + // 0 means fifo pipe with unknown size > + let file_size = (size != 0).then_some(size); > let upload_options = UploadOptions { > previous_manifest: previous_manifest.clone(), > - fixed_size: Some(size), > + chunk_size: ChunkSize::Fixed { file_size }, > compress: true, > encrypt: crypto.mode == CryptMode::Encrypt, > }; > diff --git a/src/server/push.rs b/src/server/push.rs > index d7884fce..a1216ba9 100644 > --- a/src/server/push.rs > +++ b/src/server/push.rs > @@ -17,7 +17,8 @@ use pbs_api_types::{ > PRIV_REMOTE_DATASTORE_MODIFY, PRIV_REMOTE_DATASTORE_PRUNE, > }; > use pbs_client::{ > - BackupRepository, BackupWriter, BackupWriterOptions, HttpClient, MergedChunkInfo, UploadOptions, > + BackupRepository, BackupWriter, BackupWriterOptions, ChunkSize, HttpClient, MergedChunkInfo, > + UploadOptions, > }; > use pbs_config::CachedUserInfo; > use pbs_datastore::data_blob::ChunkInfo; > @@ -917,7 +918,7 @@ pub(crate) async fn push_snapshot( > index, > chunk_reader, > &backup_writer, > - None, > + ChunkSize::Dynamic, > known_chunks.clone(), > ) > .await?; > @@ -944,7 +945,9 @@ pub(crate) async fn push_snapshot( > index, > chunk_reader, > &backup_writer, > - Some(size), > + ChunkSize::Fixed { > + file_size: Some(size), > + }, > known_chunks.clone(), > ) > .await?; > @@ -1002,7 +1005,7 @@ async fn push_index( > index: impl IndexFile + Send + 'static, > chunk_reader: Arc, > backup_writer: &BackupWriter, > - size: Option, > + chunk_size: ChunkSize, > known_chunks: Arc>>, > ) -> Result { > let (upload_channel_tx, upload_channel_rx) = mpsc::channel(20); > @@ -1048,7 +1051,7 @@ async fn push_index( > let upload_options = UploadOptions { > compress: true, > encrypt: false, > - fixed_size: size, > + chunk_size, > ..UploadOptions::default() > }; > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel