From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 480D91FF398 for ; Mon, 27 May 2024 16:33:52 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 461EE533F; Mon, 27 May 2024 16:34:05 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 27 May 2024 16:32:32 +0200 Message-Id: <20240527143323.456002-19-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240527143323.456002-1-c.ebner@proxmox.com> References: <20240527143323.456002-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.029 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v7 proxmox-backup 18/69] client: pxar: switch to stack based encoder state X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" ... and adapt to the new reader/writer variant for encoder or decoder/accessor to attach a dedicated payload input/output for split pxar archives. In preparation for look-ahead caching, where a passing around of per-directory level encoder instances with internal references is not feasible. Previously, for each directory level a new encoder instance has been generated, restricting possible implementation errors. These encoder instances have been internally linked by references to keep track of the state changes in a parent child relationship. This is however not feasible when the encoder has to be passed by mutable reference, as required by the look-ahead cache implementation. The encoder has therefore been adapted to use a single instance implementation with an internal stack keeping track of the state. Depends on the bumped pxar library version, including the patches to attach the corresponding variant for the pxar reader/writer instantiation. Signed-off-by: Christian Ebner --- changes since version 6: - adapt to new PxarVariant pxar interface type pbs-client/src/pxar/create.rs | 8 +++++--- pbs-pxar-fuse/src/lib.rs | 2 +- proxmox-backup-client/src/catalog.rs | 3 ++- proxmox-backup-client/src/main.rs | 2 +- proxmox-backup-client/src/mount.rs | 3 ++- proxmox-file-restore/src/main.rs | 4 ++-- pxar-bin/src/main.rs | 2 +- src/api2/admin/datastore.rs | 2 +- src/api2/tape/restore.rs | 5 +++-- src/bin/proxmox_backup_debug/diff.rs | 2 +- src/tape/file_formats/snapshot_archive.rs | 7 +++++-- 11 files changed, 24 insertions(+), 16 deletions(-) diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index 60efb0ce5..1b1bac2d4 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -170,7 +170,7 @@ where set.insert(stat.st_dev); } - let mut encoder = Encoder::new(&mut writer, &metadata).await?; + let mut encoder = Encoder::new(pxar::PxarVariant::Unified(&mut writer), &metadata).await?; let mut patterns = options.patterns; @@ -203,6 +203,8 @@ where .archive_dir_contents(&mut encoder, source_dir, true) .await?; encoder.finish().await?; + encoder.close().await?; + Ok(()) } @@ -663,7 +665,7 @@ impl Archiver { ) -> Result<(), Error> { let dir_name = OsStr::from_bytes(dir_name.to_bytes()); - let mut encoder = encoder.create_directory(dir_name, metadata).await?; + encoder.create_directory(dir_name, metadata).await?; let old_fs_magic = self.fs_magic; let old_fs_feature_flags = self.fs_feature_flags; @@ -686,7 +688,7 @@ impl Archiver { log::info!("skipping mount point: {:?}", self.path); Ok(()) } else { - self.archive_dir_contents(&mut encoder, dir, false).await + self.archive_dir_contents(encoder, dir, false).await }; self.fs_magic = old_fs_magic; diff --git a/pbs-pxar-fuse/src/lib.rs b/pbs-pxar-fuse/src/lib.rs index bf196b6c4..377635b2a 100644 --- a/pbs-pxar-fuse/src/lib.rs +++ b/pbs-pxar-fuse/src/lib.rs @@ -66,7 +66,7 @@ impl Session { let file = std::fs::File::open(archive_path)?; let file_size = file.metadata()?.len(); let reader: Reader = Arc::new(accessor::sync::FileReader::new(file)); - let accessor = Accessor::new(reader, file_size).await?; + let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), file_size).await?; Self::mount(accessor, options, verbose, mountpoint) } diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs index 72b22e67f..e72b6a1e0 100644 --- a/proxmox-backup-client/src/catalog.rs +++ b/proxmox-backup-client/src/catalog.rs @@ -220,7 +220,8 @@ async fn catalog_shell(param: Value) -> Result<(), Error> { let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?; + let decoder = + pbs_pxar_fuse::Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; client.download(CATALOG_NAME, &mut tmpfile).await?; let index = DynamicIndexReader::new(tmpfile) diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index 4453c7756..ad2bc5a66 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -1458,7 +1458,7 @@ async fn restore( if let Some(target) = target { pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + pxar::decoder::Decoder::from_std(pxar::PxarVariant::Unified(reader))?, Path::new(target), feature_flags, |path| { diff --git a/proxmox-backup-client/src/mount.rs b/proxmox-backup-client/src/mount.rs index 4a2f83357..4d352b6e4 100644 --- a/proxmox-backup-client/src/mount.rs +++ b/proxmox-backup-client/src/mount.rs @@ -296,7 +296,8 @@ async fn mount_do(param: Value, pipe: Option) -> Result { let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?; + let decoder = + pbs_pxar_fuse::Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; let session = pbs_pxar_fuse::Session::mount(decoder, options, false, Path::new(target.unwrap())) diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs index 50875a636..6a6379f27 100644 --- a/proxmox-file-restore/src/main.rs +++ b/proxmox-file-restore/src/main.rs @@ -457,7 +457,7 @@ async fn extract( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); - let decoder = Accessor::new(reader, archive_size).await?; + let decoder = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; extract_to_target(decoder, &path, target, format, zstd).await?; } ExtractPath::VM(file, path) => { @@ -483,7 +483,7 @@ async fn extract( false, ) .await?; - let decoder = Decoder::from_tokio(reader).await?; + let decoder = Decoder::from_tokio(pxar::PxarVariant::Unified(reader)).await?; extract_sub_dir_seq(&target, decoder).await?; // we extracted a .pxarexclude-cli file auto-generated by the VM when encoding the diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs index 2bbe90e34..0729db1c9 100644 --- a/pxar-bin/src/main.rs +++ b/pxar-bin/src/main.rs @@ -26,7 +26,7 @@ fn extract_archive_from_reader( options: PxarExtractOptions, ) -> Result<(), Error> { pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + pxar::decoder::Decoder::from_std(pxar::PxarVariant::Unified(reader))?, Path::new(target), feature_flags, |path| { diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index ca72a2f2b..af1c12cc0 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -1813,7 +1813,7 @@ pub fn pxar_file_download( let (reader, archive_size) = get_local_pxar_reader(datastore.clone(), &manifest, &backup_dir, pxar_name)?; - let decoder = Accessor::new(reader, archive_size).await?; + let decoder = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; let root = decoder.open_root().await?; let path = OsStr::from_bytes(file_path).to_os_string(); let file = root diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index 84557bce1..9184ff934 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -1069,7 +1069,8 @@ fn restore_snapshots_to_tmpdir( "File {file_num}: snapshot archive {source_datastore}:{snapshot}", ); - let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?; + let mut decoder = + pxar::decoder::sync::Decoder::from_std(pxar::PxarVariant::Unified(reader))?; let target_datastore = match store_map.target_store(&source_datastore) { Some(datastore) => datastore, @@ -1685,7 +1686,7 @@ fn restore_snapshot_archive<'a>( reader: Box, snapshot_path: &Path, ) -> Result { - let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?; + let mut decoder = pxar::decoder::sync::Decoder::from_std(pxar::PxarVariant::Unified(reader))?; match try_restore_snapshot_archive(worker, &mut decoder, snapshot_path) { Ok(_) => Ok(true), Err(err) => { diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs index 5b68941a4..e6767c17c 100644 --- a/src/bin/proxmox_backup_debug/diff.rs +++ b/src/bin/proxmox_backup_debug/diff.rs @@ -277,7 +277,7 @@ async fn open_dynamic_index( let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: Arc = Arc::new(LocalDynamicReadAt::new(reader)); - let accessor = Accessor::new(reader, archive_size).await?; + let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), archive_size).await?; Ok((lookup_index, accessor)) } diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs index 252384b50..82f466980 100644 --- a/src/tape/file_formats/snapshot_archive.rs +++ b/src/tape/file_formats/snapshot_archive.rs @@ -58,8 +58,10 @@ pub fn tape_write_snapshot_archive<'a>( )); } - let mut encoder = - pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?; + let mut encoder = pxar::encoder::sync::Encoder::new( + pxar::PxarVariant::Unified(PxarTapeWriter::new(writer)), + &root_metadata, + )?; for filename in file_list.iter() { let mut file = snapshot_reader.open_file(filename).map_err(|err| { @@ -89,6 +91,7 @@ pub fn tape_write_snapshot_archive<'a>( } } encoder.finish()?; + encoder.close()?; Ok(()) }); -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel