[pbs-devel] [PATCH v3 proxmox-backup 23/58] client: restore: read payload from dedicated index

[Christian Ebner <c.ebner@proxmox.com>]

Whenever a split pxar archive is encountered, instantiate and attach
the required dedicated reader instance to the decoder instance on
restore.

Piping the output to stdout is not possible, this would require a
decoder instance which can decode the input stream, while maintaining
the pxar stream format as output.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- use newly introduced helpers for archive name
- use helper to get pxar reader
- refactoring

 pbs-pxar-fuse/src/lib.rs             |  2 +-
 proxmox-backup-client/src/main.rs    | 39 ++++++++++++++++++----------
 proxmox-file-restore/src/main.rs     |  4 +--
 pxar-bin/src/main.rs                 |  2 +-
 src/bin/proxmox_backup_debug/diff.rs |  2 +-
 5 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/pbs-pxar-fuse/src/lib.rs b/pbs-pxar-fuse/src/lib.rs
index bf196b6c4..dff7aac31 100644
--- a/pbs-pxar-fuse/src/lib.rs
+++ b/pbs-pxar-fuse/src/lib.rs
@@ -66,7 +66,7 @@ impl Session {
         let file = std::fs::File::open(archive_path)?;
         let file_size = file.metadata()?.len();
         let reader: Reader = Arc::new(accessor::sync::FileReader::new(file));
-        let accessor = Accessor::new(reader, file_size).await?;
+        let accessor = Accessor::new(reader, file_size, None).await?;
         Self::mount(accessor, options, verbose, mountpoint)
     }
 
diff --git a/proxmox-backup-client/src/main.rs b/proxmox-backup-client/src/main.rs
index 975f6fdf8..294b52ddb 100644
--- a/proxmox-backup-client/src/main.rs
+++ b/proxmox-backup-client/src/main.rs
@@ -1223,7 +1223,7 @@ async fn dump_image<W: Write>(
 fn parse_archive_type(name: &str) -> (String, ArchiveType) {
     if name.ends_with(".didx") || name.ends_with(".fidx") || name.ends_with(".blob") {
         (name.into(), archive_type(name).unwrap())
-    } else if name.ends_with(".pxar") {
+    } else if name.ends_with(".pxar") || name.ends_with(".mpxar") || name.ends_with(".ppxar") {
         (format!("{}.didx", name), ArchiveType::DynamicIndex)
     } else if name.ends_with(".img") {
         (format!("{}.fidx", name), ArchiveType::FixedIndex)
@@ -1457,20 +1457,15 @@ async fn restore(
                 .map_err(|err| format_err!("unable to pipe data - {}", err))?;
         }
     } else if archive_type == ArchiveType::DynamicIndex {
-        let index = client
-            .download_dynamic_index(&manifest, &archive_name)
-            .await?;
+        let (archive_name, payload_archive_name) = helper::get_pxar_archive_names(&archive_name);
 
-        let most_used = index.find_most_used_chunks(8);
-
-        let chunk_reader = RemoteChunkReader::new(
+        let mut reader = get_buffered_pxar_reader(
+            &archive_name,
             client.clone(),
-            crypt_config,
-            file_info.chunk_crypt_mode(),
-            most_used,
-        );
-
-        let mut reader = BufferedDynamicReader::new(index, chunk_reader);
+            &manifest,
+            crypt_config.clone(),
+        )
+        .await?;
 
         let on_error = if ignore_extract_device_errors {
             let handler: PxarErrorHandler = Box::new(move |err: Error| {
@@ -1525,8 +1520,21 @@ async fn restore(
         }
 
         if let Some(target) = target {
+            let decoder = if let Some(payload_archive_name) = payload_archive_name {
+                let payload_reader = get_buffered_pxar_reader(
+                    &payload_archive_name,
+                    client.clone(),
+                    &manifest,
+                    crypt_config.clone(),
+                )
+                .await?;
+                pxar::decoder::Decoder::from_std(reader, Some(payload_reader))?
+            } else {
+                pxar::decoder::Decoder::from_std(reader, None)?
+            };
+
             pbs_client::pxar::extract_archive(
-                pxar::decoder::Decoder::from_std(reader)?,
+                decoder,
                 Path::new(target),
                 feature_flags,
                 |path| {
@@ -1536,6 +1544,9 @@ async fn restore(
             )
             .map_err(|err| format_err!("error extracting archive - {:#}", err))?;
         } else {
+            if archive_name.ends_with(".mpxar.didx") || archive_name.ends_with(".ppxar.didx") {
+                bail!("unable to pipe split archive");
+            }
             let mut writer = std::fs::OpenOptions::new()
                 .write(true)
                 .open("/dev/stdout")
diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
index 50875a636..dbab69942 100644
--- a/proxmox-file-restore/src/main.rs
+++ b/proxmox-file-restore/src/main.rs
@@ -457,7 +457,7 @@ async fn extract(
 
             let archive_size = reader.archive_size();
             let reader = LocalDynamicReadAt::new(reader);
-            let decoder = Accessor::new(reader, archive_size).await?;
+            let decoder = Accessor::new(reader, archive_size, None).await?;
             extract_to_target(decoder, &path, target, format, zstd).await?;
         }
         ExtractPath::VM(file, path) => {
@@ -483,7 +483,7 @@ async fn extract(
                     false,
                 )
                 .await?;
-                let decoder = Decoder::from_tokio(reader).await?;
+                let decoder = Decoder::from_tokio(reader, None).await?;
                 extract_sub_dir_seq(&target, decoder).await?;
 
                 // we extracted a .pxarexclude-cli file auto-generated by the VM when encoding the
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index 6c13c3b17..34944cf16 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -27,7 +27,7 @@ fn extract_archive_from_reader<R: std::io::Read>(
     options: PxarExtractOptions,
 ) -> Result<(), Error> {
     pbs_client::pxar::extract_archive(
-        pxar::decoder::Decoder::from_std(reader)?,
+        pxar::decoder::Decoder::from_std(reader, None)?,
         Path::new(target),
         feature_flags,
         |path| {
diff --git a/src/bin/proxmox_backup_debug/diff.rs b/src/bin/proxmox_backup_debug/diff.rs
index 5b68941a4..140c573c1 100644
--- a/src/bin/proxmox_backup_debug/diff.rs
+++ b/src/bin/proxmox_backup_debug/diff.rs
@@ -277,7 +277,7 @@ async fn open_dynamic_index(
     let reader = BufferedDynamicReader::new(index, chunk_reader);
     let archive_size = reader.archive_size();
     let reader: Arc<dyn ReadAt + Send + Sync> = Arc::new(LocalDynamicReadAt::new(reader));
-    let accessor = Accessor::new(reader, archive_size).await?;
+    let accessor = Accessor::new(reader, archive_size, None).await?;
 
     Ok((lookup_index, accessor))
 }
-- 
2.39.2