[pbs-devel] [PATCH v3 proxmox-backup 00/10] fix catalog dump and shell for split pxar archives

[pbs-devel] [PATCH v3 proxmox-backup 00/10] fix catalog dump and shell for split pxar archives

[Christian Ebner]

This patch series fixes the `catalog dump` and `catalog shell`
commands for snapshots created by setting the `change-detection-mode`
to either `data` or `metadata`, therefore using split pxar archive
encoding without encoding the dedicated catalog.

If no catalog file can be found in the snaphsots manifest, the fallback
behaviour is now to check if there are metadata archives present in the
manifest, and if so use the metadata archive accessor for either
dumping the contents in a format compatible to the catalog dump or to
interactively navigate and restore contents via the catalog shell.

Changes since version 2 (thanks to Fabian for the comments and
suggestions):
- Move and factor out helpers to pxar/tools submodule instead of using
  the tools module, which is for more generally client tools.
- Improve error handling in catalog shell by avoiding unwrap of missing
  directory stack entries, bail instead.
- fix issue with factoring out of `entry_path_with_prefix` helper
- Convert to `DirEntryAttribute` by implementing the `TryFrom` trait,
  replacing previous helper.

Changes since version 1:
- Implement also the `catalog shell` command for split pxar archives
- Factor out common functionality into dedicated helpers

Christian Ebner (10):
  client: tools: make tools module public
  client: pxar: move catalog lookup helper to pxar tools
  client: tools: move pxar root entry helper to pxar submodule
  client: make helper to get remote pxar reader reusable
  client: tools: factor out entry path prefix helper
  client: tools: factor out pxar entry to dir entry mapping
  client: add helper to dump catalog from metadata archive
  client: catalog: fallback to metadata archives for catalog dump
  client: helper to mimic catalog find using metadata archive
  client: catalog shell: fallback to accessor for navigation

 pbs-client/src/catalog_shell.rs      | 291 +++++++++++++++++++++------
 pbs-client/src/pxar/extract.rs       |   2 +-
 pbs-client/src/pxar/mod.rs           |   4 +-
 pbs-client/src/pxar/tools.rs         | 256 ++++++++++++++++++++++-
 pbs-client/src/tools/mod.rs          | 120 -----------
 pbs-datastore/src/catalog.rs         |  40 ++++
 proxmox-backup-client/src/catalog.rs |  65 +++++-
 proxmox-file-restore/src/main.rs     |  38 +---
 pxar-bin/src/main.rs                 |   4 +-
 src/api2/admin/datastore.rs          |   2 +-
 src/api2/tape/restore.rs             |   2 +-
 11 files changed, 599 insertions(+), 225 deletions(-)

-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 01/10] client: tools: make tools module public

[Christian Ebner]

Change namespace visibility for tools submodule to be accessible from
other creates, to be used for common pxar related helpers.

Switch helpers declared as `pub` to `pub(crate)` in order to keep module
encapsulation, adapt namespace for functions required to be `pub`.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- not present in previous version

 pbs-client/src/catalog_shell.rs | 2 +-
 pbs-client/src/pxar/mod.rs      | 4 +---
 pbs-client/src/pxar/tools.rs    | 8 +++++---
 pxar-bin/src/main.rs            | 4 ++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs
index 349bb7cbc..f568f6676 100644
--- a/pbs-client/src/catalog_shell.rs
+++ b/pbs-client/src/catalog_shell.rs
@@ -705,7 +705,7 @@ impl Shell {
 
         let file = Self::walk_pxar_archive(&self.accessor, &mut stack).await?;
         std::io::stdout()
-            .write_all(crate::pxar::format_multi_line_entry(file.entry()).as_bytes())?;
+            .write_all(crate::pxar::tools::format_multi_line_entry(file.entry()).as_bytes())?;
         Ok(())
     }
 
diff --git a/pbs-client/src/pxar/mod.rs b/pbs-client/src/pxar/mod.rs
index 334759df6..661501782 100644
--- a/pbs-client/src/pxar/mod.rs
+++ b/pbs-client/src/pxar/mod.rs
@@ -52,7 +52,7 @@ pub(crate) mod dir_stack;
 pub(crate) mod extract;
 pub(crate) mod look_ahead_cache;
 pub(crate) mod metadata;
-pub(crate) mod tools;
+pub mod tools;
 
 mod flags;
 pub use flags::Flags;
@@ -69,5 +69,3 @@ pub use extract::{
 /// memory, so we restrict the number of allowed entries to limit
 /// maximum memory usage.
 pub const ENCODER_MAX_ENTRIES: usize = 1024 * 1024;
-
-pub use tools::{format_multi_line_entry, format_single_line_entry};
diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 27e5185a3..9d4ad6a4f 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -10,7 +10,7 @@ use nix::sys::stat::Mode;
 use pxar::{format::StatxTimestamp, mode, Entry, EntryKind, Metadata};
 
 /// Get the file permissions as `nix::Mode`
-pub fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
+pub(crate) fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
     let mode = meta.stat.get_permission_bits();
 
     u32::try_from(mode)
@@ -22,12 +22,14 @@ pub fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
 }
 
 /// Make sure path is relative and not '.' or '..'.
-pub fn assert_relative_path<S: AsRef<OsStr> + ?Sized>(path: &S) -> Result<(), Error> {
+pub(crate) fn assert_relative_path<S: AsRef<OsStr> + ?Sized>(path: &S) -> Result<(), Error> {
     assert_relative_path_do(Path::new(path))
 }
 
 /// Make sure path is a single component and not '.' or '..'.
-pub fn assert_single_path_component<S: AsRef<OsStr> + ?Sized>(path: &S) -> Result<(), Error> {
+pub(crate) fn assert_single_path_component<S: AsRef<OsStr> + ?Sized>(
+    path: &S,
+) -> Result<(), Error> {
     assert_single_path_component_do(Path::new(path))
 }
 
diff --git a/pxar-bin/src/main.rs b/pxar-bin/src/main.rs
index 6549ccf13..767a6668e 100644
--- a/pxar-bin/src/main.rs
+++ b/pxar-bin/src/main.rs
@@ -12,9 +12,9 @@ use futures::select;
 use tokio::signal::unix::{signal, SignalKind};
 
 use pathpatterns::{MatchEntry, MatchType, PatternFlag};
+use pbs_client::pxar::tools::format_single_line_entry;
 use pbs_client::pxar::{
-    format_single_line_entry, Flags, OverwriteFlags, PxarExtractOptions, PxarWriters,
-    ENCODER_MAX_ENTRIES,
+    Flags, OverwriteFlags, PxarExtractOptions, PxarWriters, ENCODER_MAX_ENTRIES,
 };
 use pxar::EntryKind;
 
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 02/10] client: pxar: move catalog lookup helper to pxar tools

[Christian Ebner]

The lookup helper used to generate catalog entries via the metadata
archive for split archive backups is pxar specific, therefore move it
to the appropriate pxar tools submodlue.
---
changes since version 2:
- not present in previous version

 pbs-client/src/pxar/tools.rs     | 86 +++++++++++++++++++++++++++++++-
 pbs-client/src/tools/mod.rs      | 84 -------------------------------
 proxmox-file-restore/src/main.rs |  8 ++-
 src/api2/admin/datastore.rs      |  2 +-
 4 files changed, 91 insertions(+), 89 deletions(-)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 9d4ad6a4f..c444a8941 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -3,11 +3,17 @@
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
 use std::path::Path;
+use std::path::PathBuf;
 
-use anyhow::{bail, Context, Error};
+use anyhow::{bail, format_err, Context, Error};
 use nix::sys::stat::Mode;
 
-use pxar::{format::StatxTimestamp, mode, Entry, EntryKind, Metadata};
+use pxar::accessor::aio::Accessor;
+use pxar::accessor::ReadAt;
+use pxar::format::{SignedDuration, StatxTimestamp};
+use pxar::{mode, Entry, EntryKind, Metadata};
+
+use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
 
 /// Get the file permissions as `nix::Mode`
 pub(crate) fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
@@ -257,3 +263,79 @@ pub fn format_multi_line_entry(entry: &Entry) -> String {
         )
     }
 }
+
+/// Look up the directory entries of the given directory `path` in a pxar archive via it's given
+/// `accessor` and return the entries formatted as [`ArchiveEntry`]'s, compatible with reading
+/// entries from the catalog.
+///
+/// If the optional `path_prefix` is given, all returned entry paths will be prefixed with it.
+pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
+    accessor: Accessor<T>,
+    path: &OsStr,
+    path_prefix: Option<&str>,
+) -> Result<Vec<ArchiveEntry>, Error> {
+    let root = accessor.open_root().await?;
+    let dir_entry = root
+        .lookup(&path)
+        .await
+        .map_err(|err| format_err!("lookup failed - {err}"))?
+        .ok_or_else(|| format_err!("lookup failed - error opening '{path:?}'"))?;
+
+    let mut entries = Vec::new();
+    if let EntryKind::Directory = dir_entry.kind() {
+        let dir_entry = dir_entry
+            .enter_directory()
+            .await
+            .map_err(|err| format_err!("failed to enter directory - {err}"))?;
+
+        let mut entries_iter = dir_entry.read_dir();
+        while let Some(entry) = entries_iter.next().await {
+            let entry = entry?.decode_entry().await?;
+
+            let entry_attr = match entry.kind() {
+                EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => continue,
+                EntryKind::Directory => DirEntryAttribute::Directory {
+                    start: entry.entry_range_info().entry_range.start,
+                },
+                EntryKind::File { size, .. } => {
+                    let mtime = match entry.metadata().mtime_as_duration() {
+                        SignedDuration::Positive(val) => i64::try_from(val.as_secs())?,
+                        SignedDuration::Negative(val) => -i64::try_from(val.as_secs())?,
+                    };
+                    DirEntryAttribute::File { size: *size, mtime }
+                }
+                EntryKind::Device(_) => match entry.metadata().file_type() {
+                    mode::IFBLK => DirEntryAttribute::BlockDevice,
+                    mode::IFCHR => DirEntryAttribute::CharDevice,
+                    _ => bail!("encountered unknown device type"),
+                },
+                EntryKind::Symlink(_) => DirEntryAttribute::Symlink,
+                EntryKind::Hardlink(_) => DirEntryAttribute::Hardlink,
+                EntryKind::Fifo => DirEntryAttribute::Fifo,
+                EntryKind::Socket => DirEntryAttribute::Socket,
+            };
+
+            let entry_path = if let Some(prefix) = path_prefix {
+                let mut entry_path = PathBuf::from(prefix);
+                match entry.path().strip_prefix("/") {
+                    Ok(path) => entry_path.push(path),
+                    Err(_) => entry_path.push(entry.path()),
+                }
+                entry_path
+            } else {
+                PathBuf::from(entry.path())
+            };
+            entries.push(ArchiveEntry::new(
+                entry_path.as_os_str().as_bytes(),
+                Some(&entry_attr),
+            ));
+        }
+    } else {
+        bail!(format!(
+            "expected directory entry, got entry kind '{:?}'",
+            dir_entry.kind()
+        ));
+    }
+
+    Ok(entries)
+}
diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs
index 772cc1263..87e74de6e 100644
--- a/pbs-client/src/tools/mod.rs
+++ b/pbs-client/src/tools/mod.rs
@@ -1,13 +1,10 @@
 //! Shared tools useful for common CLI clients.
 use std::collections::HashMap;
 use std::env::VarError::{NotPresent, NotUnicode};
-use std::ffi::OsStr;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
-use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::FromRawFd;
-use std::path::PathBuf;
 use std::process::Command;
 use std::sync::OnceLock;
 
@@ -21,12 +18,7 @@ use proxmox_schema::*;
 use proxmox_sys::fs::file_get_json;
 
 use pbs_api_types::{Authid, BackupNamespace, RateLimitConfig, UserWithTokens, BACKUP_REPO_URL};
-use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
 use pbs_datastore::BackupManifest;
-use pxar::accessor::aio::Accessor;
-use pxar::accessor::ReadAt;
-use pxar::format::SignedDuration;
-use pxar::{mode, EntryKind};
 
 use crate::{BackupRepository, HttpClient, HttpClientOptions};
 
@@ -663,82 +655,6 @@ pub fn raise_nofile_limit() -> Result<libc::rlimit64, Error> {
     Ok(old)
 }
 
-/// Look up the directory entries of the given directory `path` in a pxar archive via it's given
-/// `accessor` and return the entries formatted as [`ArchiveEntry`]'s, compatible with reading
-/// entries from the catalog.
-///
-/// If the optional `path_prefix` is given, all returned entry paths will be prefixed with it.
-pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
-    accessor: Accessor<T>,
-    path: &OsStr,
-    path_prefix: Option<&str>,
-) -> Result<Vec<ArchiveEntry>, Error> {
-    let root = accessor.open_root().await?;
-    let dir_entry = root
-        .lookup(&path)
-        .await
-        .map_err(|err| format_err!("lookup failed - {err}"))?
-        .ok_or_else(|| format_err!("lookup failed - error opening '{path:?}'"))?;
-
-    let mut entries = Vec::new();
-    if let EntryKind::Directory = dir_entry.kind() {
-        let dir_entry = dir_entry
-            .enter_directory()
-            .await
-            .map_err(|err| format_err!("failed to enter directory - {err}"))?;
-
-        let mut entries_iter = dir_entry.read_dir();
-        while let Some(entry) = entries_iter.next().await {
-            let entry = entry?.decode_entry().await?;
-
-            let entry_attr = match entry.kind() {
-                EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => continue,
-                EntryKind::Directory => DirEntryAttribute::Directory {
-                    start: entry.entry_range_info().entry_range.start,
-                },
-                EntryKind::File { size, .. } => {
-                    let mtime = match entry.metadata().mtime_as_duration() {
-                        SignedDuration::Positive(val) => i64::try_from(val.as_secs())?,
-                        SignedDuration::Negative(val) => -i64::try_from(val.as_secs())?,
-                    };
-                    DirEntryAttribute::File { size: *size, mtime }
-                }
-                EntryKind::Device(_) => match entry.metadata().file_type() {
-                    mode::IFBLK => DirEntryAttribute::BlockDevice,
-                    mode::IFCHR => DirEntryAttribute::CharDevice,
-                    _ => bail!("encountered unknown device type"),
-                },
-                EntryKind::Symlink(_) => DirEntryAttribute::Symlink,
-                EntryKind::Hardlink(_) => DirEntryAttribute::Hardlink,
-                EntryKind::Fifo => DirEntryAttribute::Fifo,
-                EntryKind::Socket => DirEntryAttribute::Socket,
-            };
-
-            let entry_path = if let Some(prefix) = path_prefix {
-                let mut entry_path = PathBuf::from(prefix);
-                match entry.path().strip_prefix("/") {
-                    Ok(path) => entry_path.push(path),
-                    Err(_) => entry_path.push(entry.path()),
-                }
-                entry_path
-            } else {
-                PathBuf::from(entry.path())
-            };
-            entries.push(ArchiveEntry::new(
-                entry_path.as_os_str().as_bytes(),
-                Some(&entry_attr),
-            ));
-        }
-    } else {
-        bail!(format!(
-            "expected directory entry, got entry kind '{:?}'",
-            dir_entry.kind()
-        ));
-    }
-
-    Ok(entries)
-}
-
 /// Creates a temporary file (with `O_TMPFILE`) in `XDG_CACHE_HOME`. If we
 /// cannot create the file there it will be created in `/tmp` instead.
 pub fn create_tmp_file() -> std::io::Result<std::fs::File> {
diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
index 69d811fc1..40b063ebc 100644
--- a/proxmox-file-restore/src/main.rs
+++ b/proxmox-file-restore/src/main.rs
@@ -186,8 +186,12 @@ async fn list_files(
                 let accessor = Accessor::new(reader, archive_size).await?;
                 let path = OsStr::from_bytes(&path);
 
-                pbs_client::tools::pxar_metadata_catalog_lookup(accessor, path, Some(&archive_name))
-                    .await
+                pbs_client::pxar::tools::pxar_metadata_catalog_lookup(
+                    accessor,
+                    path,
+                    Some(&archive_name),
+                )
+                .await
             }
         }
         ExtractPath::VM(file, path) => {
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 976617d9f..52b32dc48 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -1737,7 +1737,7 @@ pub async fn catalog(
         let accessor = Accessor::new(reader, archive_size).await?;
 
         let file_path = decode_path(&filepath)?;
-        pbs_client::tools::pxar_metadata_catalog_lookup(
+        pbs_client::pxar::tools::pxar_metadata_catalog_lookup(
             accessor,
             OsStr::from_bytes(&file_path),
             None,
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 03/10] client: tools: move pxar root entry helper to pxar submodule

[Christian Ebner]

Move the `handle_root_with_optional_format_version_prelude` helper,
purely related to handling the root entry for pxar format version 2
archives, to the more fitting pxar tools submodule.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- not present in previous version

 pbs-client/src/pxar/extract.rs |  2 +-
 pbs-client/src/pxar/tools.rs   | 36 ++++++++++++++++++++++++++++++++++
 pbs-client/src/tools/mod.rs    | 36 ----------------------------------
 src/api2/tape/restore.rs       |  2 +-
 4 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index b1245c5fc..bea37ee10 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -29,8 +29,8 @@ use proxmox_compression::zip::{ZipEncoder, ZipEntry};
 
 use crate::pxar::dir_stack::PxarDirStack;
 use crate::pxar::metadata;
+use crate::pxar::tools::handle_root_with_optional_format_version_prelude;
 use crate::pxar::Flags;
-use crate::tools::handle_root_with_optional_format_version_prelude;
 
 pub struct PxarExtractOptions<'a> {
     pub match_list: &'a [MatchEntry],
diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index c444a8941..2f517022f 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -339,3 +339,39 @@ pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
 
     Ok(entries)
 }
+
+/// Decode possible format version and prelude entries before getting the root directory
+/// entry.
+///
+/// Returns the root directory entry and, if present, the prelude entry
+pub fn handle_root_with_optional_format_version_prelude<R: pxar::decoder::SeqRead>(
+    decoder: &mut pxar::decoder::sync::Decoder<R>,
+) -> Result<(pxar::Entry, Option<pxar::Entry>), Error> {
+    let first = decoder
+        .next()
+        .ok_or_else(|| format_err!("missing root entry"))??;
+    match first.kind() {
+        pxar::EntryKind::Directory => {
+            let version = pxar::format::FormatVersion::Version1;
+            log::debug!("pxar format version '{version:?}'");
+            Ok((first, None))
+        }
+        pxar::EntryKind::Version(version) => {
+            log::debug!("pxar format version '{version:?}'");
+            let second = decoder
+                .next()
+                .ok_or_else(|| format_err!("missing root entry"))??;
+            match second.kind() {
+                pxar::EntryKind::Directory => Ok((second, None)),
+                pxar::EntryKind::Prelude(_prelude) => {
+                    let third = decoder
+                        .next()
+                        .ok_or_else(|| format_err!("missing root entry"))??;
+                    Ok((third, Some(second)))
+                }
+                _ => bail!("unexpected entry kind {:?}", second.kind()),
+            }
+        }
+        _ => bail!("unexpected entry kind {:?}", first.kind()),
+    }
+}
diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs
index 87e74de6e..28db6f348 100644
--- a/pbs-client/src/tools/mod.rs
+++ b/pbs-client/src/tools/mod.rs
@@ -596,42 +596,6 @@ pub fn has_pxar_filename_extension(name: &str, with_didx_extension: bool) -> boo
     }
 }
 
-/// Decode possible format version and prelude entries before getting the root directory
-/// entry.
-///
-/// Returns the root directory entry and, if present, the prelude entry
-pub fn handle_root_with_optional_format_version_prelude<R: pxar::decoder::SeqRead>(
-    decoder: &mut pxar::decoder::sync::Decoder<R>,
-) -> Result<(pxar::Entry, Option<pxar::Entry>), Error> {
-    let first = decoder
-        .next()
-        .ok_or_else(|| format_err!("missing root entry"))??;
-    match first.kind() {
-        pxar::EntryKind::Directory => {
-            let version = pxar::format::FormatVersion::Version1;
-            log::debug!("pxar format version '{version:?}'");
-            Ok((first, None))
-        }
-        pxar::EntryKind::Version(version) => {
-            log::debug!("pxar format version '{version:?}'");
-            let second = decoder
-                .next()
-                .ok_or_else(|| format_err!("missing root entry"))??;
-            match second.kind() {
-                pxar::EntryKind::Directory => Ok((second, None)),
-                pxar::EntryKind::Prelude(_prelude) => {
-                    let third = decoder
-                        .next()
-                        .ok_or_else(|| format_err!("missing root entry"))??;
-                    Ok((third, Some(second)))
-                }
-                _ => bail!("unexpected entry kind {:?}", second.kind()),
-            }
-        }
-        _ => bail!("unexpected entry kind {:?}", first.kind()),
-    }
-}
-
 /// Raise the soft limit for open file handles to the hard limit
 ///
 /// Returns the values set before raising the limit as libc::rlimit64
diff --git a/src/api2/tape/restore.rs b/src/api2/tape/restore.rs
index b28db6e39..f7481bacc 100644
--- a/src/api2/tape/restore.rs
+++ b/src/api2/tape/restore.rs
@@ -25,7 +25,7 @@ use pbs_api_types::{
     PRIV_DATASTORE_MODIFY, PRIV_TAPE_READ, TAPE_RESTORE_NAMESPACE_SCHEMA,
     TAPE_RESTORE_SNAPSHOT_SCHEMA, UPID_SCHEMA,
 };
-use pbs_client::tools::handle_root_with_optional_format_version_prelude;
+use pbs_client::pxar::tools::handle_root_with_optional_format_version_prelude;
 use pbs_config::CachedUserInfo;
 use pbs_datastore::dynamic_index::DynamicIndexReader;
 use pbs_datastore::fixed_index::FixedIndexReader;
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 04/10] client: make helper to get remote pxar reader reusable

[Christian Ebner]

Move the `get_remote_pxar_reader` helper function so it can be reused
also for getting the metadata archive reader instance for the catalog
dump.

No functional changes.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- move to pxar tools submodule instead of general client's tools module

 pbs-client/src/pxar/tools.rs     | 36 ++++++++++++++++++++++++++++++--
 proxmox-file-restore/src/main.rs | 30 +++-----------------------
 2 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 2f517022f..1598aeb2e 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -2,8 +2,8 @@
 
 use std::ffi::OsStr;
 use std::os::unix::ffi::OsStrExt;
-use std::path::Path;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use anyhow::{bail, format_err, Context, Error};
 use nix::sys::stat::Mode;
@@ -15,6 +15,13 @@ use pxar::{mode, Entry, EntryKind, Metadata};
 
 use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
 
+use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
+use pbs_datastore::index::IndexFile;
+use pbs_datastore::BackupManifest;
+use pbs_tools::crypt_config::CryptConfig;
+
+use crate::{BackupReader, RemoteChunkReader};
+
 /// Get the file permissions as `nix::Mode`
 pub(crate) fn perms_from_metadata(meta: &Metadata) -> Result<Mode, Error> {
     let mode = meta.stat.get_permission_bits();
@@ -375,3 +382,28 @@ pub fn handle_root_with_optional_format_version_prelude<R: pxar::decoder::SeqRea
         _ => bail!("unexpected entry kind {:?}", first.kind()),
     }
 }
+
+pub async fn get_remote_pxar_reader(
+    archive_name: &str,
+    client: Arc<BackupReader>,
+    manifest: &BackupManifest,
+    crypt_config: Option<Arc<CryptConfig>>,
+) -> Result<(LocalDynamicReadAt<RemoteChunkReader>, u64), Error> {
+    let index = client
+        .download_dynamic_index(manifest, archive_name)
+        .await?;
+    let most_used = index.find_most_used_chunks(8);
+
+    let file_info = manifest.lookup_file_info(archive_name)?;
+    let chunk_reader = RemoteChunkReader::new(
+        client.clone(),
+        crypt_config,
+        file_info.chunk_crypt_mode(),
+        most_used,
+    );
+
+    let reader = BufferedDynamicReader::new(index, chunk_reader);
+    let archive_size = reader.archive_size();
+
+    Ok((LocalDynamicReadAt::new(reader), archive_size))
+}
diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
index 40b063ebc..5a563b8e5 100644
--- a/proxmox-file-restore/src/main.rs
+++ b/proxmox-file-restore/src/main.rs
@@ -21,6 +21,7 @@ use pxar::accessor::aio::Accessor;
 use pxar::decoder::aio::Decoder;
 
 use pbs_api_types::{file_restore::FileRestoreFormat, BackupDir, BackupNamespace, CryptMode};
+use pbs_client::pxar::tools::get_remote_pxar_reader;
 use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq};
 use pbs_client::tools::{
     complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
@@ -33,9 +34,9 @@ use pbs_client::tools::{
 };
 use pbs_client::{BackupReader, BackupRepository, RemoteChunkReader};
 use pbs_datastore::catalog::{ArchiveEntry, CatalogReader, DirEntryAttribute};
-use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
+use pbs_datastore::dynamic_index::BufferedDynamicReader;
 use pbs_datastore::index::IndexFile;
-use pbs_datastore::{BackupManifest, CATALOG_NAME};
+use pbs_datastore::CATALOG_NAME;
 use pbs_key_config::decrypt_key;
 use pbs_tools::crypt_config::CryptConfig;
 
@@ -357,31 +358,6 @@ async fn list(
     Ok(())
 }
 
-async fn get_remote_pxar_reader(
-    archive_name: &str,
-    client: Arc<BackupReader>,
-    manifest: &BackupManifest,
-    crypt_config: Option<Arc<CryptConfig>>,
-) -> Result<(LocalDynamicReadAt<RemoteChunkReader>, u64), Error> {
-    let index = client
-        .download_dynamic_index(manifest, archive_name)
-        .await?;
-    let most_used = index.find_most_used_chunks(8);
-
-    let file_info = manifest.lookup_file_info(archive_name)?;
-    let chunk_reader = RemoteChunkReader::new(
-        client.clone(),
-        crypt_config,
-        file_info.chunk_crypt_mode(),
-        most_used,
-    );
-
-    let reader = BufferedDynamicReader::new(index, chunk_reader);
-    let archive_size = reader.archive_size();
-
-    Ok((LocalDynamicReadAt::new(reader), archive_size))
-}
-
 #[api(
     input: {
         properties: {
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 05/10] client: tools: factor out entry path prefix helper

[Christian Ebner]

Move the logic to generate `FileEntry` paths with a given prefix to
its own helper function for it to be reusable for the catalog shell
implementation of split pxar archives.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- only factor out code as is, thereby avoiding issue with
  relative/absolute paths depending on given prefix

 pbs-client/src/pxar/tools.rs | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 1598aeb2e..25500f748 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
 use anyhow::{bail, format_err, Context, Error};
 use nix::sys::stat::Mode;
 
-use pxar::accessor::aio::Accessor;
+use pxar::accessor::aio::{Accessor, FileEntry};
 use pxar::accessor::ReadAt;
 use pxar::format::{SignedDuration, StatxTimestamp};
 use pxar::{mode, Entry, EntryKind, Metadata};
@@ -322,16 +322,7 @@ pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
                 EntryKind::Socket => DirEntryAttribute::Socket,
             };
 
-            let entry_path = if let Some(prefix) = path_prefix {
-                let mut entry_path = PathBuf::from(prefix);
-                match entry.path().strip_prefix("/") {
-                    Ok(path) => entry_path.push(path),
-                    Err(_) => entry_path.push(entry.path()),
-                }
-                entry_path
-            } else {
-                PathBuf::from(entry.path())
-            };
+            let entry_path = entry_path_with_prefix(&entry, path_prefix);
             entries.push(ArchiveEntry::new(
                 entry_path.as_os_str().as_bytes(),
                 Some(&entry_attr),
@@ -407,3 +398,20 @@ pub async fn get_remote_pxar_reader(
 
     Ok((LocalDynamicReadAt::new(reader), archive_size))
 }
+
+/// Generate entry path for given [`FileEntry`], prefixed by given `path_prefix` component(s).
+pub(crate) fn entry_path_with_prefix<T: Clone + ReadAt>(
+    entry: &FileEntry<T>,
+    path_prefix: Option<&str>,
+) -> PathBuf {
+    if let Some(prefix) = path_prefix {
+        let mut entry_path = PathBuf::from(prefix);
+        match entry.path().strip_prefix("/") {
+            Ok(path) => entry_path.push(path),
+            Err(_) => entry_path.push(entry.path()),
+        }
+        entry_path
+    } else {
+        PathBuf::from(entry.path())
+    }
+}
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 06/10] client: tools: factor out pxar entry to dir entry mapping

[Christian Ebner]

Perform the conversion from pxar file entries to catalog entry
attributes by implementing `TryFrom<&FileEntry<T>>` for
`DirEntryAttribute` and use that.

Allows the reuse for the catalog shell, when using the split pxar
archive instead of the catalog.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- implement `TryFrom<&FileEntry<T>>` for `DirEntryAttribute` conversion
  instead of a helper.

 pbs-client/src/pxar/tools.rs | 26 ++++-------------------
 pbs-datastore/src/catalog.rs | 40 ++++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 25500f748..68207cd31 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -299,30 +299,12 @@ pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
         while let Some(entry) = entries_iter.next().await {
             let entry = entry?.decode_entry().await?;
 
-            let entry_attr = match entry.kind() {
-                EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => continue,
-                EntryKind::Directory => DirEntryAttribute::Directory {
-                    start: entry.entry_range_info().entry_range.start,
-                },
-                EntryKind::File { size, .. } => {
-                    let mtime = match entry.metadata().mtime_as_duration() {
-                        SignedDuration::Positive(val) => i64::try_from(val.as_secs())?,
-                        SignedDuration::Negative(val) => -i64::try_from(val.as_secs())?,
-                    };
-                    DirEntryAttribute::File { size: *size, mtime }
-                }
-                EntryKind::Device(_) => match entry.metadata().file_type() {
-                    mode::IFBLK => DirEntryAttribute::BlockDevice,
-                    mode::IFCHR => DirEntryAttribute::CharDevice,
-                    _ => bail!("encountered unknown device type"),
-                },
-                EntryKind::Symlink(_) => DirEntryAttribute::Symlink,
-                EntryKind::Hardlink(_) => DirEntryAttribute::Hardlink,
-                EntryKind::Fifo => DirEntryAttribute::Fifo,
-                EntryKind::Socket => DirEntryAttribute::Socket,
+            let entry_attr = match DirEntryAttribute::try_from(&entry) {
+                Ok(attr) => attr,
+                Err(_) => continue,
             };
 
-            let entry_path = entry_path_with_prefix(&entry, path_prefix);
+            let entry_path = crate::pxar::tools::entry_path_with_prefix(&entry, path_prefix);
             entries.push(ArchiveEntry::new(
                 entry_path.as_os_str().as_bytes(),
                 Some(&entry_attr),
diff --git a/pbs-datastore/src/catalog.rs b/pbs-datastore/src/catalog.rs
index eb5318370..f0589f888 100644
--- a/pbs-datastore/src/catalog.rs
+++ b/pbs-datastore/src/catalog.rs
@@ -7,6 +7,10 @@ use anyhow::{bail, format_err, Error};
 use serde::{Deserialize, Serialize};
 
 use pathpatterns::{MatchList, MatchType};
+use pxar::accessor::aio::FileEntry;
+use pxar::accessor::ReadAt;
+use pxar::format::SignedDuration;
+use pxar::{mode, EntryKind};
 
 use proxmox_io::ReadExt;
 use proxmox_schema::api;
@@ -104,6 +108,42 @@ pub enum DirEntryAttribute {
     Socket,
 }
 
+impl<T> TryFrom<&FileEntry<T>> for DirEntryAttribute
+where
+    T: Clone + ReadAt,
+{
+    type Error = Error;
+
+    fn try_from(entry: &FileEntry<T>) -> Result<Self, Self::Error> {
+        let attr = match entry.kind() {
+            EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => bail!(
+                "cannot convert pxar entry kind {:?} to catalog directory entry attribute",
+                entry.kind(),
+            ),
+            EntryKind::Directory => DirEntryAttribute::Directory {
+                start: entry.entry_range_info().entry_range.start,
+            },
+            EntryKind::File { size, .. } => {
+                let mtime = match entry.metadata().mtime_as_duration() {
+                    SignedDuration::Positive(val) => i64::try_from(val.as_secs())?,
+                    SignedDuration::Negative(val) => -i64::try_from(val.as_secs())?,
+                };
+                DirEntryAttribute::File { size: *size, mtime }
+            }
+            EntryKind::Device(_) => match entry.metadata().file_type() {
+                mode::IFBLK => DirEntryAttribute::BlockDevice,
+                mode::IFCHR => DirEntryAttribute::CharDevice,
+                _ => bail!("encountered unknown device type"),
+            },
+            EntryKind::Symlink(_) => DirEntryAttribute::Symlink,
+            EntryKind::Hardlink(_) => DirEntryAttribute::Hardlink,
+            EntryKind::Fifo => DirEntryAttribute::Fifo,
+            EntryKind::Socket => DirEntryAttribute::Socket,
+        };
+        Ok(attr)
+    }
+}
+
 impl DirEntry {
     fn new(etype: CatalogEntryType, name: Vec<u8>, start: u64, size: u64, mtime: i64) -> Self {
         match etype {
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 07/10] client: add helper to dump catalog from metadata archive

[Christian Ebner]

Implements the methods to dump the contents of a metadata pxar
archive using the same output format as used by the catalog dump.

The helper function has been split into 2 for async recursion to
work.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- place into pxar tools module instead of general client tools module

 pbs-client/src/pxar/tools.rs | 66 ++++++++++++++++++++++++++++++++++--
 1 file changed, 63 insertions(+), 3 deletions(-)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 68207cd31..c99cf46fd 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -1,19 +1,21 @@
 //! Some common methods used within the pxar code.
 
 use std::ffi::OsStr;
+use std::future::Future;
 use std::os::unix::ffi::OsStrExt;
 use std::path::{Path, PathBuf};
+use std::pin::Pin;
 use std::sync::Arc;
 
 use anyhow::{bail, format_err, Context, Error};
 use nix::sys::stat::Mode;
 
-use pxar::accessor::aio::{Accessor, FileEntry};
+use pxar::accessor::aio::{Accessor, Directory, FileEntry};
 use pxar::accessor::ReadAt;
-use pxar::format::{SignedDuration, StatxTimestamp};
+use pxar::format::StatxTimestamp;
 use pxar::{mode, Entry, EntryKind, Metadata};
 
-use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
+use pbs_datastore::catalog::{ArchiveEntry, CatalogEntryType, DirEntryAttribute};
 
 use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
 use pbs_datastore::index::IndexFile;
@@ -397,3 +399,61 @@ pub(crate) fn entry_path_with_prefix<T: Clone + ReadAt>(
         PathBuf::from(entry.path())
     }
 }
+
+/// Read a sorted list of pxar archive entries from given parent entry via the pxar accessor.
+pub(crate) async fn pxar_metadata_read_dir<T: Clone + Send + Sync + ReadAt>(
+    parent_dir: Directory<T>,
+) -> Result<Vec<FileEntry<T>>, Error> {
+    let mut entries_iter = parent_dir.read_dir();
+    let mut entries = Vec::new();
+    while let Some(entry) = entries_iter.next().await {
+        let entry = entry?.decode_entry().await?;
+        entries.push(entry);
+    }
+    entries.sort_unstable_by(|a, b| a.path().cmp(b.path()));
+    Ok(entries)
+}
+
+/// Dump pxar archive entry by using the same format used to dump entries from a catalog.
+fn pxar_metadata_catalog_dump_entry<'future, T: Clone + Send + Sync + ReadAt + 'future>(
+    entry: FileEntry<T>,
+    path_prefix: Option<&'future str>,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'future>> {
+    let entry_path = entry_path_with_prefix(&entry, path_prefix);
+
+    Box::pin(async move {
+        if let Ok(attr) = DirEntryAttribute::try_from(&entry) {
+            let etype = CatalogEntryType::from(&attr);
+            match attr {
+                DirEntryAttribute::File { size, mtime } => {
+                    let mut mtime_string = mtime.to_string();
+                    if let Ok(s) = proxmox_time::strftime_local("%FT%TZ", mtime) {
+                        mtime_string = s;
+                    }
+                    log::info!("{etype} {entry_path:?} {size} {mtime_string}");
+                }
+                DirEntryAttribute::Directory { .. } => {
+                    log::info!("{etype} {entry_path:?}");
+                    let dir = entry.enter_directory().await?;
+                    pxar_metadata_catalog_dump_dir(dir, path_prefix).await?;
+                }
+                _ => log::info!("{etype} {entry_path:?}"),
+            }
+        }
+
+        Ok(())
+    })
+}
+
+/// Recursively iterate over pxar archive entries and dump them using the same format used to dump
+/// entries from a catalog.
+pub async fn pxar_metadata_catalog_dump_dir<T: Clone + Send + Sync + ReadAt>(
+    parent_dir: Directory<T>,
+    path_prefix: Option<&str>,
+) -> Result<(), Error> {
+    let entries = pxar_metadata_read_dir(parent_dir).await?;
+    for entry in entries {
+        pxar_metadata_catalog_dump_entry(entry, path_prefix).await?;
+    }
+    Ok(())
+}
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 08/10] client: catalog: fallback to metadata archives for catalog dump

[Christian Ebner]

Commit c0302805c "client: backup: conditionally write catalog for
file level backups" drops encoding of the dedicated catalog when
archives are encoded as split metadata/data archives with the
`change-detection-mode` set to `data` or `metadata`.

Since the catalog is not present anymore, fallback to use the pxar
metadata archives in the manifest (if present) for generating the
listing of contents in a compatible manner.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- adapt to new helper module namespaces

 proxmox-backup-client/src/catalog.rs | 41 ++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs
index 0a374c011..7bb003788 100644
--- a/proxmox-backup-client/src/catalog.rs
+++ b/proxmox-backup-client/src/catalog.rs
@@ -8,11 +8,13 @@ use proxmox_router::cli::*;
 use proxmox_schema::api;
 
 use pbs_api_types::BackupNamespace;
+use pbs_client::pxar::tools::get_remote_pxar_reader;
 use pbs_client::tools::has_pxar_filename_extension;
 use pbs_client::tools::key_source::get_encryption_key_password;
 use pbs_client::{BackupReader, RemoteChunkReader};
 use pbs_tools::crypt_config::CryptConfig;
 use pbs_tools::json::required_string_param;
+use pxar::accessor::aio::Accessor;
 
 use crate::helper;
 use crate::{
@@ -88,14 +90,49 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
     let (manifest, _) = client.download_manifest().await?;
     manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
 
+    let file_info = match manifest.lookup_file_info(CATALOG_NAME) {
+        Ok(file_info) => file_info,
+        Err(err) => {
+            let mut metadata_archives = Vec::new();
+            // No catalog, fallback to metadata archives if present
+            for archive in manifest.files() {
+                if archive.filename.ends_with(".mpxar.didx") {
+                    metadata_archives.push(archive.filename.clone());
+                }
+            }
+            metadata_archives.sort_unstable_by(|a, b| a.cmp(b));
+
+            for archive in &metadata_archives {
+                let (reader, archive_size) = get_remote_pxar_reader(
+                    &archive,
+                    client.clone(),
+                    &manifest,
+                    crypt_config.clone(),
+                )
+                .await?;
+                // only care about the metadata, don't attach a payload reader
+                let reader = pxar::PxarVariant::Unified(reader);
+                let accessor = Accessor::new(reader, archive_size).await?;
+                let root_dir = accessor.open_root().await?;
+                let prefix = format!("./{archive}");
+                pbs_client::pxar::tools::pxar_metadata_catalog_dump_dir(root_dir, Some(&prefix))
+                    .await?;
+            }
+
+            if !metadata_archives.is_empty() {
+                return Ok(Value::Null);
+            }
+
+            bail!(err);
+        }
+    };
+
     let index = client
         .download_dynamic_index(&manifest, CATALOG_NAME)
         .await?;
 
     let most_used = index.find_most_used_chunks(8);
 
-    let file_info = manifest.lookup_file_info(CATALOG_NAME)?;
-
     let chunk_reader = RemoteChunkReader::new(
         client.clone(),
         crypt_config,
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 09/10] client: helper to mimic catalog find using metadata archive

[Christian Ebner]

Adds helper functions to reimplement the catalog shell functionality
for snapshots being encoded as split pxar archives.

Just as the `CatalogReader`s find method, recursively iterate entries
and call the given callback on all entries matched by the match
patterns, starting from the given parent entry.

The helper has been split into 2 functions for the async recursion to
work.
---
changes since version 2:
- no changes

 pbs-client/src/pxar/tools.rs | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index c99cf46fd..900eeab83 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -10,6 +10,7 @@ use std::sync::Arc;
 use anyhow::{bail, format_err, Context, Error};
 use nix::sys::stat::Mode;
 
+use pathpatterns::{MatchList, MatchType};
 use pxar::accessor::aio::{Accessor, Directory, FileEntry};
 use pxar::accessor::ReadAt;
 use pxar::format::StatxTimestamp;
@@ -457,3 +458,44 @@ pub async fn pxar_metadata_catalog_dump_dir<T: Clone + Send + Sync + ReadAt>(
     }
     Ok(())
 }
+
+/// Call the callback on given entry if matched by the match patterns.
+fn pxar_metadata_catalog_find_entry<'future, T: Clone + Send + Sync + ReadAt + 'future>(
+    entry: FileEntry<T>,
+    match_list: &'future (impl MatchList<'future> + Sync),
+    callback: &'future (dyn Fn(&[u8]) -> Result<(), Error> + Send + Sync),
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'future>> {
+    Box::pin(async move {
+        let file_mode = entry.metadata().file_type() as u32;
+        let entry_path = entry_path_with_prefix(&entry, Some("/"))
+            .as_os_str()
+            .to_owned();
+
+        match match_list.matches(entry_path.as_bytes(), file_mode) {
+            Ok(Some(MatchType::Exclude)) => return Ok(()),
+            Ok(Some(MatchType::Include)) => callback(entry_path.as_bytes())?,
+            _ => (),
+        }
+
+        if let EntryKind::Directory = entry.kind() {
+            let dir_entry = entry.enter_directory().await?;
+            pxar_metadata_catalog_find(dir_entry, match_list, callback).await?;
+        }
+
+        Ok(())
+    })
+}
+
+/// Recursively iterate over pxar archive entries and call the callback on entries matching the
+/// match patterns.
+pub async fn pxar_metadata_catalog_find<'future, T: Clone + Send + Sync + ReadAt + 'future>(
+    parent_dir: Directory<T>,
+    match_list: &'future (impl pathpatterns::MatchList<'future> + Sync),
+    callback: &'future (dyn Fn(&[u8]) -> Result<(), Error> + Send + Sync),
+) -> Result<(), Error> {
+    let entries = pxar_metadata_read_dir(parent_dir).await?;
+    for entry in entries {
+        pxar_metadata_catalog_find_entry(entry, match_list, callback).await?;
+    }
+    Ok(())
+}
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH v3 proxmox-backup 10/10] client: catalog shell: fallback to accessor for navigation

[Christian Ebner]

Make the catalog optional and use the pxar accessor for navigation if
the catalog is not provided.
This allows to use the metadata archive for navigraion, as for split
pxar archives no dedicated catalog is encoded.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since version 2:
- Improve error handling by avoiding unwrap, bail instead
- add helper function to get the parents pxar entry from the directory
  stack

 pbs-client/src/catalog_shell.rs      | 289 +++++++++++++++++++++------
 proxmox-backup-client/src/catalog.rs |  24 ++-
 2 files changed, 254 insertions(+), 59 deletions(-)

diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs
index f568f6676..8c8e9a654 100644
--- a/pbs-client/src/catalog_shell.rs
+++ b/pbs-client/src/catalog_shell.rs
@@ -21,7 +21,7 @@ use pxar::accessor::ReadAt;
 use pxar::{EntryKind, Metadata};
 
 use pbs_datastore::catalog::{self, DirEntryAttribute};
-use proxmox_async::runtime::block_in_place;
+use proxmox_async::runtime::{block_in_place, block_on};
 
 use crate::pxar::Flags;
 
@@ -312,8 +312,9 @@ pub struct Shell {
     /// Interactive prompt.
     prompt: String,
 
-    /// Catalog reader instance to navigate
-    catalog: CatalogReader,
+    /// Optional catalog reader instance to navigate, if not present the Accessor is used for
+    /// navigation
+    catalog: Option<CatalogReader>,
 
     /// List of selected paths for restore
     selected: HashMap<OsString, MatchEntry>,
@@ -347,7 +348,7 @@ impl PathStackEntry {
 impl Shell {
     /// Create a new shell for the given catalog and pxar archive.
     pub async fn new(
-        mut catalog: CatalogReader,
+        mut catalog: Option<CatalogReader>,
         archive_name: &str,
         archive: Accessor,
     ) -> Result<Self, Error> {
@@ -355,11 +356,31 @@ impl Shell {
         let mut rl = rustyline::Editor::<CliHelper>::new();
         rl.set_helper(Some(cli_helper));
 
-        let catalog_root = catalog.root()?;
-        let archive_root = catalog
-            .lookup(&catalog_root, archive_name.as_bytes())?
-            .ok_or_else(|| format_err!("archive not found in catalog"))?;
-        let position = vec![PathStackEntry::new(archive_root)];
+        let mut position = Vec::new();
+        if let Some(catalog) = catalog.as_mut() {
+            let catalog_root = catalog.root()?;
+            let archive_root = catalog
+                .lookup(&catalog_root, archive_name.as_bytes())?
+                .ok_or_else(|| format_err!("archive not found in catalog"))?;
+            position.push(PathStackEntry::new(archive_root));
+        } else {
+            let root = archive.open_root().await?;
+            let root_entry = root.lookup_self().await?;
+            if let EntryKind::Directory = root_entry.kind() {
+                let entry_attr = DirEntryAttribute::Directory {
+                    start: root_entry.entry_range_info().entry_range.start,
+                };
+                position.push(PathStackEntry {
+                    catalog: catalog::DirEntry {
+                        name: archive_name.into(),
+                        attr: entry_attr,
+                    },
+                    pxar: Some(root_entry),
+                });
+            } else {
+                bail!("unexpected root entry type");
+            }
+        }
 
         let mut this = Self {
             rl,
@@ -450,7 +471,7 @@ impl Shell {
 
     async fn resolve_symlink(
         stack: &mut Vec<PathStackEntry>,
-        catalog: &mut CatalogReader,
+        catalog: &mut Option<CatalogReader>,
         accessor: &Accessor,
         follow_symlinks: &mut Option<usize>,
     ) -> Result<(), Error> {
@@ -468,7 +489,7 @@ impl Shell {
             };
 
             let new_stack =
-                Self::lookup(stack, &mut *catalog, accessor, Some(path), follow_symlinks).await?;
+                Self::lookup(stack, catalog, accessor, Some(path), follow_symlinks).await?;
 
             *stack = new_stack;
 
@@ -484,7 +505,7 @@ impl Shell {
     /// out.
     async fn step(
         stack: &mut Vec<PathStackEntry>,
-        catalog: &mut CatalogReader,
+        catalog: &mut Option<CatalogReader>,
         accessor: &Accessor,
         component: std::path::Component<'_>,
         follow_symlinks: &mut Option<usize>,
@@ -503,9 +524,27 @@ impl Shell {
                 if stack.last().unwrap().catalog.is_symlink() {
                     Self::resolve_symlink(stack, catalog, accessor, follow_symlinks).await?;
                 }
-                match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? {
-                    Some(dir) => stack.push(PathStackEntry::new(dir)),
-                    None => bail!("no such file or directory: {:?}", entry),
+                if let Some(catalog) = catalog {
+                    match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? {
+                        Some(dir) => stack.push(PathStackEntry::new(dir)),
+                        None => bail!("no such file or directory: {entry:?}"),
+                    }
+                } else {
+                    let pxar_entry = parent_pxar_entry(&stack)?;
+                    let parent_dir = pxar_entry.enter_directory().await?;
+                    match parent_dir.lookup(entry).await? {
+                        Some(entry) => {
+                            let entry_attr = DirEntryAttribute::try_from(&entry)?;
+                            stack.push(PathStackEntry {
+                                catalog: catalog::DirEntry {
+                                    name: entry.entry().file_name().as_bytes().into(),
+                                    attr: entry_attr,
+                                },
+                                pxar: Some(entry),
+                            })
+                        }
+                        None => bail!("no such file or directory: {entry:?}"),
+                    }
                 }
             }
         }
@@ -515,7 +554,7 @@ impl Shell {
 
     fn step_nofollow(
         stack: &mut Vec<PathStackEntry>,
-        catalog: &mut CatalogReader,
+        catalog: &mut Option<CatalogReader>,
         component: std::path::Component<'_>,
     ) -> Result<(), Error> {
         use std::path::Component;
@@ -531,11 +570,27 @@ impl Shell {
             Component::Normal(entry) => {
                 if stack.last().unwrap().catalog.is_symlink() {
                     bail!("target is a symlink");
-                } else {
+                } else if let Some(catalog) = catalog.as_mut() {
                     match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? {
                         Some(dir) => stack.push(PathStackEntry::new(dir)),
                         None => bail!("no such file or directory: {:?}", entry),
                     }
+                } else {
+                    let pxar_entry = parent_pxar_entry(&stack)?;
+                    let parent_dir = block_on(pxar_entry.enter_directory())?;
+                    match block_on(parent_dir.lookup(entry))? {
+                        Some(entry) => {
+                            let entry_attr = DirEntryAttribute::try_from(&entry)?;
+                            stack.push(PathStackEntry {
+                                catalog: catalog::DirEntry {
+                                    name: entry.entry().file_name().as_bytes().into(),
+                                    attr: entry_attr,
+                                },
+                                pxar: Some(entry),
+                            })
+                        }
+                        None => bail!("no such file or directory: {entry:?}"),
+                    }
                 }
             }
         }
@@ -545,7 +600,7 @@ impl Shell {
     /// The pxar accessor is required to resolve symbolic links
     async fn walk_catalog(
         stack: &mut Vec<PathStackEntry>,
-        catalog: &mut CatalogReader,
+        catalog: &mut Option<CatalogReader>,
         accessor: &Accessor,
         path: &Path,
         follow_symlinks: &mut Option<usize>,
@@ -559,7 +614,7 @@ impl Shell {
     /// Non-async version cannot follow symlinks.
     fn walk_catalog_nofollow(
         stack: &mut Vec<PathStackEntry>,
-        catalog: &mut CatalogReader,
+        catalog: &mut Option<CatalogReader>,
         path: &Path,
     ) -> Result<(), Error> {
         for c in path.components() {
@@ -612,12 +667,34 @@ impl Shell {
                     tmp_stack = self.position.clone();
                 }
                 Self::walk_catalog_nofollow(&mut tmp_stack, &mut self.catalog, &path)?;
-                (&tmp_stack.last().unwrap().catalog, base, part)
+                (&tmp_stack.last().unwrap(), base, part)
             }
-            None => (&self.position.last().unwrap().catalog, "", input),
+            None => (&self.position.last().unwrap(), "", input),
         };
 
-        let entries = self.catalog.read_dir(parent)?;
+        let entries = if let Some(catalog) = self.catalog.as_mut() {
+            catalog.read_dir(&parent.catalog)?
+        } else {
+            let dir = if let Some(entry) = parent.pxar.as_ref() {
+                block_on(entry.enter_directory())?
+            } else {
+                bail!("missing pxar entry for parent");
+            };
+            let mut out = Vec::new();
+            let entries = block_on(crate::pxar::tools::pxar_metadata_read_dir(dir))?;
+            for entry in entries {
+                let mut name = base.to_string();
+                let file_name = entry.file_name().as_bytes();
+                if file_name.starts_with(part.as_bytes()) {
+                    name.push_str(std::str::from_utf8(file_name)?);
+                    if entry.is_dir() {
+                        name.push('/');
+                    }
+                    out.push(name);
+                }
+            }
+            return Ok(out);
+        };
 
         let mut out = Vec::new();
         for entry in entries {
@@ -637,7 +714,7 @@ impl Shell {
     // Break async recursion here: lookup -> walk_catalog -> step -> lookup
     fn lookup<'future, 's, 'c, 'a, 'p, 'y>(
         stack: &'s [PathStackEntry],
-        catalog: &'c mut CatalogReader,
+        catalog: &'c mut Option<CatalogReader>,
         accessor: &'a Accessor,
         path: Option<&'p Path>,
         follow_symlinks: &'y mut Option<usize>,
@@ -678,7 +755,23 @@ impl Shell {
 
         let last = stack.last().unwrap();
         if last.catalog.is_directory() {
-            let items = self.catalog.read_dir(&stack.last().unwrap().catalog)?;
+            let items = if let Some(catalog) = self.catalog.as_mut() {
+                catalog.read_dir(&stack.last().unwrap().catalog)?
+            } else {
+                let dir = if let Some(entry) = last.pxar.as_ref() {
+                    entry.enter_directory().await?
+                } else {
+                    bail!("missing pxar entry for parent");
+                };
+
+                let mut out = std::io::stdout();
+                let items = crate::pxar::tools::pxar_metadata_read_dir(dir).await?;
+                for item in items {
+                    out.write_all(&item.file_name().as_bytes())?;
+                    out.write_all(b"\n")?;
+                }
+                return Ok(());
+            };
             let mut out = std::io::stdout();
             // FIXME: columnize
             for item in items {
@@ -820,17 +913,36 @@ impl Shell {
     async fn list_matching_files(&mut self) -> Result<(), Error> {
         let matches = self.build_match_list();
 
-        self.catalog.find(
-            &self.position[0].catalog,
-            &mut Vec::new(),
-            &matches,
-            &mut |path: &[u8]| -> Result<(), Error> {
-                let mut out = std::io::stdout();
-                out.write_all(path)?;
-                out.write_all(b"\n")?;
-                Ok(())
-            },
-        )?;
+        if let Some(catalog) = self.catalog.as_mut() {
+            catalog.find(
+                &self.position[0].catalog,
+                &mut Vec::new(),
+                &matches,
+                &mut |path: &[u8]| -> Result<(), Error> {
+                    let mut out = std::io::stdout();
+                    out.write_all(path)?;
+                    out.write_all(b"\n")?;
+                    Ok(())
+                },
+            )?;
+        } else {
+            let parent_dir = if let Some(pxar_entry) = self.position[0].pxar.as_ref() {
+                pxar_entry.enter_directory().await?
+            } else {
+                bail!("missing pxar entry for archive root");
+            };
+            crate::pxar::tools::pxar_metadata_catalog_find(
+                parent_dir,
+                &matches,
+                &|path: &[u8]| -> Result<(), Error> {
+                    let mut out = std::io::stdout();
+                    out.write_all(path)?;
+                    out.write_all(b"\n")?;
+                    Ok(())
+                },
+            )
+            .await?;
+        }
 
         Ok(())
     }
@@ -841,18 +953,37 @@ impl Shell {
             MatchEntry::parse_pattern(pattern, PatternFlag::PATH_NAME, MatchType::Include)?;
 
         let mut found_some = false;
-        self.catalog.find(
-            &self.position[0].catalog,
-            &mut Vec::new(),
-            &[&pattern_entry],
-            &mut |path: &[u8]| -> Result<(), Error> {
-                found_some = true;
-                let mut out = std::io::stdout();
-                out.write_all(path)?;
-                out.write_all(b"\n")?;
-                Ok(())
-            },
-        )?;
+        if let Some(catalog) = self.catalog.as_mut() {
+            catalog.find(
+                &self.position[0].catalog,
+                &mut Vec::new(),
+                &[&pattern_entry],
+                &mut |path: &[u8]| -> Result<(), Error> {
+                    found_some = true;
+                    let mut out = std::io::stdout();
+                    out.write_all(path)?;
+                    out.write_all(b"\n")?;
+                    Ok(())
+                },
+            )?;
+        } else {
+            let parent_dir = if let Some(pxar_entry) = self.position[0].pxar.as_ref() {
+                pxar_entry.enter_directory().await?
+            } else {
+                bail!("missing pxar entry for archive root");
+            };
+            crate::pxar::tools::pxar_metadata_catalog_find(
+                parent_dir,
+                &[&pattern_entry],
+                &|path: &[u8]| -> Result<(), Error> {
+                    let mut out = std::io::stdout();
+                    out.write_all(path)?;
+                    out.write_all(b"\n")?;
+                    Ok(())
+                },
+            )
+            .await?;
+        }
 
         if found_some && select {
             self.selected.insert(pattern_os, pattern_entry);
@@ -945,6 +1076,18 @@ impl Shell {
     }
 }
 
+fn parent_pxar_entry(dir_stack: &[PathStackEntry]) -> Result<&FileEntry, Error> {
+    if let Some(parent) = dir_stack.last().as_ref() {
+        if let Some(entry) = parent.pxar.as_ref() {
+            Ok(entry)
+        } else {
+            bail!("missing pxar entry for parent");
+        }
+    } else {
+        bail!("missing parent entry on stack");
+    }
+}
+
 struct ExtractorState<'a> {
     path: Vec<u8>,
     path_len: usize,
@@ -960,22 +1103,38 @@ struct ExtractorState<'a> {
 
     extractor: crate::pxar::extract::Extractor,
 
-    catalog: &'a mut CatalogReader,
+    catalog: &'a mut Option<CatalogReader>,
     match_list: &'a [MatchEntry],
     accessor: &'a Accessor,
 }
 
 impl<'a> ExtractorState<'a> {
     pub fn new(
-        catalog: &'a mut CatalogReader,
+        catalog: &'a mut Option<CatalogReader>,
         dir_stack: Vec<PathStackEntry>,
         extractor: crate::pxar::extract::Extractor,
         match_list: &'a [MatchEntry],
         accessor: &'a Accessor,
     ) -> Result<Self, Error> {
-        let read_dir = catalog
-            .read_dir(&dir_stack.last().unwrap().catalog)?
-            .into_iter();
+        let read_dir = if let Some(catalog) = catalog.as_mut() {
+            catalog
+                .read_dir(&dir_stack.last().unwrap().catalog)?
+                .into_iter()
+        } else {
+            let pxar_entry = parent_pxar_entry(&dir_stack)?;
+            let dir = block_on(pxar_entry.enter_directory())?;
+            let entries = block_on(crate::pxar::tools::pxar_metadata_read_dir(dir))?;
+
+            let mut catalog_entries = Vec::with_capacity(entries.len());
+            for entry in entries {
+                let entry_attr = DirEntryAttribute::try_from(&entry).unwrap();
+                catalog_entries.push(catalog::DirEntry {
+                    name: entry.entry().file_name().as_bytes().into(),
+                    attr: entry_attr,
+                });
+            }
+            catalog_entries.into_iter()
+        };
         Ok(Self {
             path: Vec::new(),
             path_len: 0,
@@ -1053,11 +1212,29 @@ impl<'a> ExtractorState<'a> {
         entry: catalog::DirEntry,
         match_result: Option<MatchType>,
     ) -> Result<(), Error> {
+        let entry_iter = if let Some(catalog) = self.catalog.as_mut() {
+            catalog.read_dir(&entry)?.into_iter()
+        } else {
+            self.dir_stack.push(PathStackEntry::new(entry.clone()));
+            let dir = Shell::walk_pxar_archive(self.accessor, &mut self.dir_stack).await?;
+            self.dir_stack.pop();
+            let dir = dir.enter_directory().await?;
+            let entries = block_on(crate::pxar::tools::pxar_metadata_read_dir(dir))?;
+            entries
+                .into_iter()
+                .map(|entry| {
+                    let entry_attr = DirEntryAttribute::try_from(&entry).unwrap();
+                    catalog::DirEntry {
+                        name: entry.entry().file_name().as_bytes().into(),
+                        attr: entry_attr,
+                    }
+                })
+                .collect::<Vec<catalog::DirEntry>>()
+                .into_iter()
+        };
         // enter a new directory:
-        self.read_dir_stack.push(mem::replace(
-            &mut self.read_dir,
-            self.catalog.read_dir(&entry)?.into_iter(),
-        ));
+        self.read_dir_stack
+            .push(mem::replace(&mut self.read_dir, entry_iter));
         self.matches_stack.push(self.matches);
         self.dir_stack.push(PathStackEntry::new(entry));
         self.path_len_stack.push(self.path_len);
diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs
index 7bb003788..a55c9effe 100644
--- a/proxmox-backup-client/src/catalog.rs
+++ b/proxmox-backup-client/src/catalog.rs
@@ -230,11 +230,29 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
     )
     .await?;
 
-    let mut tmpfile = pbs_client::tools::create_tmp_file()?;
-
     let (manifest, _) = client.download_manifest().await?;
     manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
 
+    if let Err(_err) = manifest.lookup_file_info(CATALOG_NAME) {
+        // No catalog, fallback to pxar archive accessor if present
+        let accessor = helper::get_pxar_fuse_accessor(
+            &server_archive_name,
+            client.clone(),
+            &manifest,
+            crypt_config.clone(),
+        )
+        .await?;
+
+        let state = Shell::new(None, &server_archive_name, accessor).await?;
+        log::info!("Starting interactive shell");
+        state.shell().await?;
+        record_repository(&repo);
+
+        return Ok(());
+    }
+
+    let mut tmpfile = pbs_client::tools::create_tmp_file()?;
+
     let decoder = helper::get_pxar_fuse_accessor(
         &server_archive_name,
         client.clone(),
@@ -268,7 +286,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> {
 
     catalogfile.seek(SeekFrom::Start(0))?;
     let catalog_reader = CatalogReader::new(catalogfile);
-    let state = Shell::new(catalog_reader, &server_archive_name, decoder).await?;
+    let state = Shell::new(Some(catalog_reader), &server_archive_name, decoder).await?;
 
     log::info!("Starting interactive shell");
     state.shell().await?;
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel