From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id EB006BC08F for ; Thu, 28 Mar 2024 13:38:05 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BAA3B9F9F for ; Thu, 28 Mar 2024 13:37:47 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 28 Mar 2024 13:37:43 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4B0574297D for ; Thu, 28 Mar 2024 13:37:43 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 28 Mar 2024 13:36:32 +0100 Message-Id: <20240328123707.336951-24-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240328123707.336951-1-c.ebner@proxmox.com> References: <20240328123707.336951-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.030 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v3 proxmox-backup 23/58] client: restore: read payload from dedicated index X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Mar 2024 12:38:06 -0000 Whenever a split pxar archive is encountered, instantiate and attach the required dedicated reader instance to the decoder instance on restore. Piping the output to stdout is not possible, this would require a decoder instance which can decode the input stream, while maintaining the pxar stream format as output. Signed-off-by: Christian Ebner --- changes since version 2: - use newly introduced helpers for archive name - use helper to get pxar reader - refactoring pbs-pxar-fuse/src/lib.rs | 2 +- proxmox-backup-client/src/main.rs | 39 ++++++++++++++++++---------- proxmox-file-restore/src/main.rs | 4 +-- pxar-bin/src/main.rs | 2 +- src/bin/proxmox_backup_debug/diff.rs | 2 +- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/pbs-pxar-fuse/src/lib.rs b/pbs-pxar-fuse/src/lib.rs index bf196b6c4..dff7aac31 100644 --- a/pbs-pxar-fuse/src/lib.rs +++ b/pbs-pxar-fuse/src/lib.rs @@ -66,7 +66,7 @@ impl Session { let file = std::fs::File::open(archive_path)?; let file_size = file.metadata()?.len(); let reader: Reader = Arc::new(accessor::sync::FileReader::new(file)); - let accessor = Accessor::new(reader, file_size).await?; + let accessor = Accessor::new(reader, file_size, None).await?; Self::mount(accessor, options, verbose, mountpoint) } diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index 975f6fdf8..294b52ddb 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -1223,7 +1223,7 @@ async fn dump_image( fn parse_archive_type(name: &str) -> (String, ArchiveType) { if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") { (name.into(), archive_type(name).unwrap()) - } else if name.ends_with(".pxar") { + } else if name.ends_with(".pxar") || name.ends_with(".mpxar") || name.ends_with(".ppxar") { (format!("{}.didx", name), ArchiveType::DynamicIndex) } else if name.ends_with(".img") { (format!("{}.fidx", name), ArchiveType::FixedIndex) @@ -1457,20 +1457,15 @@ async fn restore( .map_err(|err| format_err!("unable to pipe data - {}", err))?; } } else if archive_type == ArchiveType::DynamicIndex { - let index = client - .download_dynamic_index(&manifest, &archive_name) - .await?; + let (archive_name, payload_archive_name) = helper::get_pxar_archive_names(&archive_name); - let most_used = index.find_most_used_chunks(8); - - let chunk_reader = RemoteChunkReader::new( + let mut reader = get_buffered_pxar_reader( + &archive_name, client.clone(), - crypt_config, - file_info.chunk_crypt_mode(), - most_used, - ); - - let mut reader = BufferedDynamicReader::new(index, chunk_reader); + &manifest, + crypt_config.clone(), + ) + .await?; let on_error = if ignore_extract_device_errors { let handler: PxarErrorHandler = Box::new(move |err: Error| { @@ -1525,8 +1520,21 @@ async fn restore( } if let Some(target) = target { + let decoder = if let Some(payload_archive_name) = payload_archive_name { + let payload_reader = get_buffered_pxar_reader( + &payload_archive_name, + client.clone(), + &manifest, + crypt_config.clone(), + ) + .await?; + pxar::decoder::Decoder::from_std(reader, Some(payload_reader))? + } else { + pxar::decoder::Decoder::from_std(reader, None)? + }; + pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + decoder, Path::new(target), feature_flags, |path| { @@ -1536,6 +1544,9 @@ async fn restore( ) .map_err(|err| format_err!("error extracting archive - {:#}", err))?; } else { + if archive_name.ends_with(".mpxar.didx") || archive_name.ends_with(".ppxar.didx") { + bail!("unable to pipe split archive"); + } let mut writer = std::fs::OpenOptions::new() .write(true) .open("/dev/stdout") diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs index 50875a636..dbab69942 100644 --- a/proxmox-file-restore/src/main.rs +++ b/proxmox-file-restore/src/main.rs @@ -457,7 +457,7 @@ async fn extract( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); - let decoder = Accessor::new(reader, archive_size).await?; + let decoder = Accessor::new(reader, archive_size, None).await?; extract_to_target(decoder, &path, target, format, zstd).await?; } ExtractPath::VM(file, path) => { @@ -483,7 +483,7 @@ async fn extract( false, ) .await?; - let decoder = Decoder::from_tokio(reader).await?; + let decoder = Decoder::from_tokio(reader, None).await?; extract_sub_dir_seq(&target, decoder).await?; // we extracted a .pxarexclude-cli file auto-generated by the VM when encoding the diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs index 6c13c3b17..34944cf16 100644 --- a/pxar-bin/src/main.rs +++ b/pxar-bin/src/main.rs @@ -27,7 +27,7 @@ fn extract_archive_from_reader( options: PxarExtractOptions, ) -> Result<(), Error> { pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + pxar::decoder::Decoder::from_std(reader, None)?, Path::new(target), feature_flags, |path| { diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs index 5b68941a4..140c573c1 100644 --- a/src/bin/proxmox_backup_debug/diff.rs +++ b/src/bin/proxmox_backup_debug/diff.rs @@ -277,7 +277,7 @@ async fn open_dynamic_index( let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: Arc = Arc::new(LocalDynamicReadAt::new(reader)); - let accessor = Accessor::new(reader, archive_size).await?; + let accessor = Accessor::new(reader, archive_size, None).await?; Ok((lookup_index, accessor)) } -- 2.39.2