From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 4FDB31FF389 for ; Tue, 7 May 2024 17:54:16 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 835781274C; Tue, 7 May 2024 17:54:13 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 7 May 2024 17:51:57 +0200 Message-Id: <20240507155244.793819-16-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240507155244.793819-1-c.ebner@proxmox.com> References: <20240507155244.793819-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.027 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v5 proxmox-backup 15/62] client: pxar: switch to stack based encoder state X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" ... and add the new optional encoder/decoder/accessor parameter to attach a dedicated payload input/output. In preparation for look-ahead caching, where a passing around of per-directory level encoder instances with internal references is not feasible. Previously, for each directory level a new encoder instance has been generated, restricting possible implementation errors. These encoder instances have been internally linked by references to keep track of the state changes in a parent child relationship. This is however not feasible when the encoder has to be passed by mutable reference, as required by the look-ahead cache implementation. The encoder has therefore been adapted to use a single instance implementation with an internal stack keeping track of the state. Depends on the bumped pxar library version, including the patches to optionally attach the dedicated payload input/output. Signed-off-by: Christian Ebner --- changes since version 4: - no changes pbs-client/src/pxar/create.rs | 8 +++++--- pbs-pxar-fuse/src/lib.rs | 2 +- proxmox-backup-client/src/catalog.rs | 2 +- proxmox-backup-client/src/main.rs | 2 +- proxmox-backup-client/src/mount.rs | 2 +- proxmox-file-restore/src/main.rs | 4 ++-- pxar-bin/src/main.rs | 2 +- src/api2/admin/datastore.rs | 2 +- src/api2/tape/restore.rs | 4 ++-- src/bin/proxmox_backup_debug/diff.rs | 2 +- src/tape/file_formats/snapshot_archive.rs | 3 ++- 11 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index 60efb0ce5..c9bf6df85 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -170,7 +170,7 @@ where set.insert(stat.st_dev); } - let mut encoder = Encoder::new(&mut writer, &metadata).await?; + let mut encoder = Encoder::new(&mut writer, &metadata, None).await?; let mut patterns = options.patterns; @@ -203,6 +203,8 @@ where .archive_dir_contents(&mut encoder, source_dir, true) .await?; encoder.finish().await?; + encoder.close().await?; + Ok(()) } @@ -663,7 +665,7 @@ impl Archiver { ) -> Result<(), Error> { let dir_name = OsStr::from_bytes(dir_name.to_bytes()); - let mut encoder = encoder.create_directory(dir_name, metadata).await?; + encoder.create_directory(dir_name, metadata).await?; let old_fs_magic = self.fs_magic; let old_fs_feature_flags = self.fs_feature_flags; @@ -686,7 +688,7 @@ impl Archiver { log::info!("skipping mount point: {:?}", self.path); Ok(()) } else { - self.archive_dir_contents(&mut encoder, dir, false).await + self.archive_dir_contents(encoder, dir, false).await }; self.fs_magic = old_fs_magic; diff --git a/pbs-pxar-fuse/src/lib.rs b/pbs-pxar-fuse/src/lib.rs index bf196b6c4..dff7aac31 100644 --- a/pbs-pxar-fuse/src/lib.rs +++ b/pbs-pxar-fuse/src/lib.rs @@ -66,7 +66,7 @@ impl Session { let file = std::fs::File::open(archive_path)?; let file_size = file.metadata()?.len(); let reader: Reader = Arc::new(accessor::sync::FileReader::new(file)); - let accessor = Accessor::new(reader, file_size).await?; + let accessor = Accessor::new(reader, file_size, None).await?; Self::mount(accessor, options, verbose, mountpoint) } diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs index 72b22e67f..db919477f 100644 --- a/proxmox-backup-client/src/catalog.rs +++ b/proxmox-backup-client/src/catalog.rs @@ -220,7 +220,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> { let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?; + let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size, None).await?; client.download(CATALOG_NAME, &mut tmpfile).await?; let index = DynamicIndexReader::new(tmpfile) diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs index 32fe914c4..287005024 100644 --- a/proxmox-backup-client/src/main.rs +++ b/proxmox-backup-client/src/main.rs @@ -1453,7 +1453,7 @@ async fn restore( if let Some(target) = target { pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + pxar::decoder::Decoder::from_std(reader, None)?, Path::new(target), feature_flags, |path| { diff --git a/proxmox-backup-client/src/mount.rs b/proxmox-backup-client/src/mount.rs index 4a2f83357..67fd23468 100644 --- a/proxmox-backup-client/src/mount.rs +++ b/proxmox-backup-client/src/mount.rs @@ -296,7 +296,7 @@ async fn mount_do(param: Value, pipe: Option) -> Result { let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: pbs_pxar_fuse::Reader = Arc::new(BufferedDynamicReadAt::new(reader)); - let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size).await?; + let decoder = pbs_pxar_fuse::Accessor::new(reader, archive_size, None).await?; let session = pbs_pxar_fuse::Session::mount(decoder, options, false, Path::new(target.unwrap())) diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs index 50875a636..dbab69942 100644 --- a/proxmox-file-restore/src/main.rs +++ b/proxmox-file-restore/src/main.rs @@ -457,7 +457,7 @@ async fn extract( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); - let decoder = Accessor::new(reader, archive_size).await?; + let decoder = Accessor::new(reader, archive_size, None).await?; extract_to_target(decoder, &path, target, format, zstd).await?; } ExtractPath::VM(file, path) => { @@ -483,7 +483,7 @@ async fn extract( false, ) .await?; - let decoder = Decoder::from_tokio(reader).await?; + let decoder = Decoder::from_tokio(reader, None).await?; extract_sub_dir_seq(&target, decoder).await?; // we extracted a .pxarexclude-cli file auto-generated by the VM when encoding the diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs index 2bbe90e34..d475da4e3 100644 --- a/pxar-bin/src/main.rs +++ b/pxar-bin/src/main.rs @@ -26,7 +26,7 @@ fn extract_archive_from_reader( options: PxarExtractOptions, ) -> Result<(), Error> { pbs_client::pxar::extract_archive( - pxar::decoder::Decoder::from_std(reader)?, + pxar::decoder::Decoder::from_std(reader, None)?, Path::new(target), feature_flags, |path| { diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 3ea174998..068b6a61e 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -1803,7 +1803,7 @@ pub fn pxar_file_download( let archive_size = reader.archive_size(); let reader = LocalDynamicReadAt::new(reader); - let decoder = Accessor::new(reader, archive_size).await?; + let decoder = Accessor::new(reader, archive_size, None).await?; let root = decoder.open_root().await?; let path = OsStr::from_bytes(file_path).to_os_string(); let file = root diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs index 84557bce1..11fb2b4cd 100644 --- a/src/api2/tape/restore.rs +++ b/src/api2/tape/restore.rs @@ -1069,7 +1069,7 @@ fn restore_snapshots_to_tmpdir( "File {file_num}: snapshot archive {source_datastore}:{snapshot}", ); - let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?; + let mut decoder = pxar::decoder::sync::Decoder::from_std(reader, None)?; let target_datastore = match store_map.target_store(&source_datastore) { Some(datastore) => datastore, @@ -1685,7 +1685,7 @@ fn restore_snapshot_archive<'a>( reader: Box, snapshot_path: &Path, ) -> Result { - let mut decoder = pxar::decoder::sync::Decoder::from_std(reader)?; + let mut decoder = pxar::decoder::sync::Decoder::from_std(reader, None)?; match try_restore_snapshot_archive(worker, &mut decoder, snapshot_path) { Ok(_) => Ok(true), Err(err) => { diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs index 5b68941a4..140c573c1 100644 --- a/src/bin/proxmox_backup_debug/diff.rs +++ b/src/bin/proxmox_backup_debug/diff.rs @@ -277,7 +277,7 @@ async fn open_dynamic_index( let reader = BufferedDynamicReader::new(index, chunk_reader); let archive_size = reader.archive_size(); let reader: Arc = Arc::new(LocalDynamicReadAt::new(reader)); - let accessor = Accessor::new(reader, archive_size).await?; + let accessor = Accessor::new(reader, archive_size, None).await?; Ok((lookup_index, accessor)) } diff --git a/src/tape/file_formats/snapshot_archive.rs b/src/tape/file_formats/snapshot_archive.rs index 252384b50..43d1cf9c3 100644 --- a/src/tape/file_formats/snapshot_archive.rs +++ b/src/tape/file_formats/snapshot_archive.rs @@ -59,7 +59,7 @@ pub fn tape_write_snapshot_archive<'a>( } let mut encoder = - pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?; + pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata, None)?; for filename in file_list.iter() { let mut file = snapshot_reader.open_file(filename).map_err(|err| { @@ -89,6 +89,7 @@ pub fn tape_write_snapshot_archive<'a>( } } encoder.finish()?; + encoder.close()?; Ok(()) }); -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel