[pbs-devel] [PATCH proxmox-backup 0/3] fix catalog dump for split pxar archives

[pbs-devel] [PATCH proxmox-backup 0/3] fix catalog dump for split pxar archives

[Christian Ebner]

This patch series fixes the catalog dump command for snapshots created
by setting the `change-detection-mode` to either `data` or `metadata`,
therefore using split pxar archive encoding without encoding the
dedicated catalog.

If no catalog file can be found in the snaphsots manifest, the fallback
behaviour is now to check if there are metadata archives present in the
manifest, and if so dump their content using the same formatting as the
regular catalog dump.

The catalog shell will be addressed in a separate patch series.

Christian Ebner (3):
  client: make helper to get remote pxar reader reusable
  client: add helper to dump catalog from metadata archive
  client: catalog: fallback to metadata archives for catalog dump

 pbs-client/src/tools/mod.rs          | 116 ++++++++++++++++++++++++++-
 proxmox-backup-client/src/catalog.rs |  39 ++++++++-
 proxmox-file-restore/src/main.rs     |  31 +------
 3 files changed, 152 insertions(+), 34 deletions(-)

-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 1/3] client: make helper to get remote pxar reader reusable

[Christian Ebner]

Move the get_remote_pxar_reader helper function so it can be reused
also for getting the metadata archive reader instance for the catalog
dump.

No functional changes.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-client/src/tools/mod.rs      | 31 ++++++++++++++++++++++++++++++-
 proxmox-file-restore/src/main.rs | 31 +++----------------------------
 2 files changed, 33 insertions(+), 29 deletions(-)

diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs
index 90fac696e..f2d2f48dc 100644
--- a/pbs-client/src/tools/mod.rs
+++ b/pbs-client/src/tools/mod.rs
@@ -8,6 +8,7 @@ use std::os::unix::ffi::OsStrExt;
 use std::os::unix::io::FromRawFd;
 use std::path::PathBuf;
 use std::process::Command;
+use std::sync::Arc;
 
 use anyhow::{bail, format_err, Context, Error};
 use serde_json::{json, Value};
@@ -20,13 +21,16 @@ use proxmox_sys::fs::file_get_json;
 
 use pbs_api_types::{Authid, BackupNamespace, RateLimitConfig, UserWithTokens, BACKUP_REPO_URL};
 use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
+use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
+use pbs_datastore::index::IndexFile;
 use pbs_datastore::BackupManifest;
+use pbs_tools::crypt_config::CryptConfig;
 use pxar::accessor::aio::Accessor;
 use pxar::accessor::ReadAt;
 use pxar::format::SignedDuration;
 use pxar::{mode, EntryKind};
 
-use crate::{BackupRepository, HttpClient, HttpClientOptions};
+use crate::{BackupReader, BackupRepository, HttpClient, HttpClientOptions, RemoteChunkReader};
 
 pub mod key_source;
 
@@ -553,6 +557,31 @@ pub fn place_xdg_file(
         .with_context(|| format!("failed to place {} in xdg home", description))
 }
 
+pub async fn get_remote_pxar_reader(
+    archive_name: &str,
+    client: Arc<BackupReader>,
+    manifest: &BackupManifest,
+    crypt_config: Option<Arc<CryptConfig>>,
+) -> Result<(LocalDynamicReadAt<RemoteChunkReader>, u64), Error> {
+    let index = client
+        .download_dynamic_index(manifest, archive_name)
+        .await?;
+    let most_used = index.find_most_used_chunks(8);
+
+    let file_info = manifest.lookup_file_info(archive_name)?;
+    let chunk_reader = RemoteChunkReader::new(
+        client.clone(),
+        crypt_config,
+        file_info.chunk_crypt_mode(),
+        most_used,
+    );
+
+    let reader = BufferedDynamicReader::new(index, chunk_reader);
+    let archive_size = reader.archive_size();
+
+    Ok((LocalDynamicReadAt::new(reader), archive_size))
+}
+
 pub fn get_pxar_archive_names(
     archive_name: &str,
     manifest: &BackupManifest,
diff --git a/proxmox-file-restore/src/main.rs b/proxmox-file-restore/src/main.rs
index 69d811fc1..db4ccae08 100644
--- a/proxmox-file-restore/src/main.rs
+++ b/proxmox-file-restore/src/main.rs
@@ -24,7 +24,7 @@ use pbs_api_types::{file_restore::FileRestoreFormat, BackupDir, BackupNamespace,
 use pbs_client::pxar::{create_tar, create_zip, extract_sub_dir, extract_sub_dir_seq};
 use pbs_client::tools::{
     complete_group_or_snapshot, complete_repository, connect, extract_repository_from_value,
-    has_pxar_filename_extension,
+    get_remote_pxar_reader, has_pxar_filename_extension,
     key_source::{
         crypto_parameters_keep_fd, format_key_source, get_encryption_key_password, KEYFD_SCHEMA,
         KEYFILE_SCHEMA,
@@ -33,9 +33,9 @@ use pbs_client::tools::{
 };
 use pbs_client::{BackupReader, BackupRepository, RemoteChunkReader};
 use pbs_datastore::catalog::{ArchiveEntry, CatalogReader, DirEntryAttribute};
-use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
+use pbs_datastore::dynamic_index::BufferedDynamicReader;
 use pbs_datastore::index::IndexFile;
-use pbs_datastore::{BackupManifest, CATALOG_NAME};
+use pbs_datastore::CATALOG_NAME;
 use pbs_key_config::decrypt_key;
 use pbs_tools::crypt_config::CryptConfig;
 
@@ -353,31 +353,6 @@ async fn list(
     Ok(())
 }
 
-async fn get_remote_pxar_reader(
-    archive_name: &str,
-    client: Arc<BackupReader>,
-    manifest: &BackupManifest,
-    crypt_config: Option<Arc<CryptConfig>>,
-) -> Result<(LocalDynamicReadAt<RemoteChunkReader>, u64), Error> {
-    let index = client
-        .download_dynamic_index(manifest, archive_name)
-        .await?;
-    let most_used = index.find_most_used_chunks(8);
-
-    let file_info = manifest.lookup_file_info(archive_name)?;
-    let chunk_reader = RemoteChunkReader::new(
-        client.clone(),
-        crypt_config,
-        file_info.chunk_crypt_mode(),
-        most_used,
-    );
-
-    let reader = BufferedDynamicReader::new(index, chunk_reader);
-    let archive_size = reader.archive_size();
-
-    Ok((LocalDynamicReadAt::new(reader), archive_size))
-}
-
 #[api(
     input: {
         properties: {
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 2/3] client: add helper to dump catalog from metadata archive

[Christian Ebner]

Implements the methods to dump the contents of a metadata pxar
archive using the same output format as used by the catalog dump.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-client/src/tools/mod.rs | 85 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 83 insertions(+), 2 deletions(-)

diff --git a/pbs-client/src/tools/mod.rs b/pbs-client/src/tools/mod.rs
index f2d2f48dc..48ef43696 100644
--- a/pbs-client/src/tools/mod.rs
+++ b/pbs-client/src/tools/mod.rs
@@ -1,12 +1,15 @@
 //! Shared tools useful for common CLI clients.
+use pxar::accessor::aio::FileEntry;
 use std::collections::HashMap;
 use std::env::VarError::{NotPresent, NotUnicode};
 use std::ffi::OsStr;
 use std::fs::File;
+use std::future::Future;
 use std::io::{BufRead, BufReader};
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::io::FromRawFd;
 use std::path::PathBuf;
+use std::pin::Pin;
 use std::process::Command;
 use std::sync::Arc;
 
@@ -20,12 +23,12 @@ use proxmox_schema::*;
 use proxmox_sys::fs::file_get_json;
 
 use pbs_api_types::{Authid, BackupNamespace, RateLimitConfig, UserWithTokens, BACKUP_REPO_URL};
-use pbs_datastore::catalog::{ArchiveEntry, DirEntryAttribute};
+use pbs_datastore::catalog::{ArchiveEntry, CatalogEntryType, DirEntryAttribute};
 use pbs_datastore::dynamic_index::{BufferedDynamicReader, LocalDynamicReadAt};
 use pbs_datastore::index::IndexFile;
 use pbs_datastore::BackupManifest;
 use pbs_tools::crypt_config::CryptConfig;
-use pxar::accessor::aio::Accessor;
+use pxar::accessor::aio::{Accessor, Directory};
 use pxar::accessor::ReadAt;
 use pxar::format::SignedDuration;
 use pxar::{mode, EntryKind};
@@ -765,3 +768,81 @@ pub async fn pxar_metadata_catalog_lookup<T: Clone + ReadAt>(
 
     Ok(entries)
 }
+
+fn pxar_metadata_catalog_dump_entry<'future, T: Clone + Send + Sync + ReadAt + 'future>(
+    entry: FileEntry<T>,
+    path_prefix: &'future str,
+) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'future>> {
+    let mut entry_path = PathBuf::from(&path_prefix);
+    match entry.path().strip_prefix("/") {
+        Ok(path) => entry_path.push(path),
+        Err(_) => entry_path.push(entry.path()),
+    }
+
+    Box::pin(async move {
+        match entry.kind() {
+            EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => {
+                return Ok(())
+            }
+            EntryKind::Directory => {
+                log::info!("{} {entry_path:?}", CatalogEntryType::Directory);
+                let dir_entry = entry.enter_directory().await?;
+                pxar_metadata_catalog_dump_dir(dir_entry, path_prefix).await?;
+            }
+            EntryKind::File { size, .. } => {
+                let mtime = match entry.metadata().mtime_as_duration() {
+                    SignedDuration::Positive(val) => i64::try_from(val.as_secs())?,
+                    SignedDuration::Negative(val) => -i64::try_from(val.as_secs())?,
+                };
+                let mut mtime_string = mtime.to_string();
+                if let Ok(s) = proxmox_time::strftime_local("%FT%TZ", mtime) {
+                    mtime_string = s;
+                }
+                log::info!(
+                    "{} {entry_path:?} {size} {mtime_string}",
+                    CatalogEntryType::File
+                );
+            }
+            EntryKind::Device(_) => match entry.metadata().file_type() {
+                mode::IFBLK => log::info!("{} {entry_path:?}", CatalogEntryType::BlockDevice),
+                mode::IFCHR => log::info!("{} {entry_path:?}", CatalogEntryType::CharDevice),
+                _ => bail!("encountered unknown device type"),
+            },
+            EntryKind::Symlink(_) => log::info!("{} {entry_path:?}", CatalogEntryType::Symlink),
+            EntryKind::Hardlink(_) => log::info!("{} {entry_path:?}", CatalogEntryType::Hardlink),
+            EntryKind::Fifo => log::info!("{} {entry_path:?}", CatalogEntryType::Fifo),
+            EntryKind::Socket => log::info!("{} {entry_path:?}", CatalogEntryType::Socket),
+        }
+
+        Ok(())
+    })
+}
+
+async fn pxar_metadata_catalog_dump_dir<T: Clone + Send + Sync + ReadAt>(
+    parent_dir: Directory<T>,
+    path_prefix: &str,
+) -> Result<(), Error> {
+    let mut entries_iter = parent_dir.read_dir();
+    let mut entries = Vec::new();
+    while let Some(entry) = entries_iter.next().await {
+        let entry = entry?.decode_entry().await?;
+        entries.push(entry);
+    }
+    entries.sort_unstable_by(|a, b| a.path().cmp(b.path()));
+
+    for entry in entries {
+        pxar_metadata_catalog_dump_entry(entry, path_prefix).await?;
+    }
+
+    Ok(())
+}
+
+pub async fn pxar_metadata_catalog_dump<T: Clone + Send + Sync + ReadAt>(
+    accessor: Accessor<T>,
+    path_prefix: &str,
+) -> Result<(), Error> {
+    let path_prefix = format!("./{path_prefix}");
+    let root = accessor.open_root().await?;
+    pxar_metadata_catalog_dump_dir(root, &path_prefix).await?;
+    Ok(())
+}
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 3/3] client: catalog: fallback to metadata archives for catalog dump

[Christian Ebner]

Commit c0302805c "client: backup: conditionally write catalog for
file level backups" drops encoding of the dedicated catalog when
archives are encoded as split metadata/data archives with the
`change-detection-mode` set to `data` or `metadata`.

Since the catalog is not present anymore, fallback to use the pxar
metadata archives in the manifest (if present) for generating the
listing of contents in a compatible manner.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 proxmox-backup-client/src/catalog.rs | 39 +++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs
index 276457c1d..983da3f06 100644
--- a/proxmox-backup-client/src/catalog.rs
+++ b/proxmox-backup-client/src/catalog.rs
@@ -9,11 +9,12 @@ use proxmox_router::cli::*;
 use proxmox_schema::api;
 
 use pbs_api_types::BackupNamespace;
-use pbs_client::tools::has_pxar_filename_extension;
 use pbs_client::tools::key_source::get_encryption_key_password;
+use pbs_client::tools::{get_remote_pxar_reader, has_pxar_filename_extension};
 use pbs_client::{BackupReader, RemoteChunkReader};
 use pbs_tools::crypt_config::CryptConfig;
 use pbs_tools::json::required_string_param;
+use pxar::accessor::aio::Accessor;
 
 use crate::helper;
 use crate::{
@@ -89,14 +90,46 @@ async fn dump_catalog(param: Value) -> Result<Value, Error> {
     let (manifest, _) = client.download_manifest().await?;
     manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?;
 
+    let file_info = match manifest.lookup_file_info(CATALOG_NAME) {
+        Ok(file_info) => file_info,
+        Err(err) => {
+            let mut metadata_archives = Vec::new();
+            // No catalog, fallback to metadata archives if present
+            for archive in manifest.files() {
+                if archive.filename.ends_with(".mpxar.didx") {
+                    metadata_archives.push(archive.filename.clone());
+                }
+            }
+            metadata_archives.sort_unstable_by(|a, b| a.cmp(b));
+
+            for archive in &metadata_archives {
+                let (reader, archive_size) = get_remote_pxar_reader(
+                    &archive,
+                    client.clone(),
+                    &manifest,
+                    crypt_config.clone(),
+                )
+                .await?;
+                // only care about the metadata, don't attach a payload reader
+                let reader = pxar::PxarVariant::Unified(reader);
+                let accessor = Accessor::new(reader, archive_size).await?;
+                pbs_client::tools::pxar_metadata_catalog_dump(accessor, &archive).await?;
+            }
+
+            if !metadata_archives.is_empty() {
+                return Ok(Value::Null);
+            }
+
+            bail!(err);
+        }
+    };
+
     let index = client
         .download_dynamic_index(&manifest, CATALOG_NAME)
         .await?;
 
     let most_used = index.find_most_used_chunks(8);
 
-    let file_info = manifest.lookup_file_info(CATALOG_NAME)?;
-
     let chunk_reader = RemoteChunkReader::new(
         client.clone(),
         crypt_config,
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

Re: [pbs-devel] [PATCH proxmox-backup 0/3] fix catalog dump for split pxar archives

[Christian Ebner]

superseded-by version 2: 
https://lists.proxmox.com/pipermail/pbs-devel/2024-July/010260.html

The new version also covers the catalog shell implementation for split 
pxar archives, not yet present in version 1.


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

Re: [pbs-devel] [PATCH proxmox-backup 0/3] fix catalog dump for split pxar archives

[Christian Ebner]

On 7/22/24 12:34, Christian Ebner wrote:
> superseded-by version 2: 
> https://lists.proxmox.com/pipermail/pbs-devel/2024-July/010260.html
> 
> The new version also covers the catalog shell implementation for split 
> pxar archives, not yet present in version 1.

wrong link, the new series is 
https://lists.proxmox.com/pipermail/pbs-devel/2024-July/010314.html



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel