[pbs-devel] [PATCH proxmox-backup 1/2] client: pxar: json encode cli exclude pattern in prelude

[pbs-devel] [PATCH proxmox-backup 1/2] client: pxar: json encode cli exclude pattern in prelude

[Christian Ebner]

The current encoding is not extensible, so encode the cli exclude
patterns as json instead. By this, the prelude is easily seralized
and deserialized, while remaining human readable.

Originally-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-client/src/pxar/create.rs | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index 12a4d92c3..dc0a31e7a 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -16,6 +16,7 @@ use nix::dir::Dir;
 use nix::errno::Errno;
 use nix::fcntl::OFlag;
 use nix::sys::stat::{FileStat, Mode};
+use serde::{Deserialize, Serialize};
 
 use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternFlag};
 use proxmox_sys::error::SysError;
@@ -154,6 +155,13 @@ struct ReuseStats {
     total_reencoded_size: u64,
 }
 
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct PbsClientPrelude {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    exclude_patterns: Option<String>,
+}
+
 struct Archiver {
     feature_flags: Flags,
     fs_feature_flags: Flags,
@@ -239,9 +247,13 @@ where
         )?);
     }
 
-    let cli_params_content = generate_pxar_excludes_cli(&patterns[..]);
-    let cli_params = if options.previous_ref.is_some() {
-        Some(cli_params_content.as_slice())
+    let prelude = if options.previous_ref.is_some() && !patterns.is_empty() {
+        let prelude = PbsClientPrelude {
+            exclude_patterns: Some(String::from_utf8(generate_pxar_excludes_cli(
+                &patterns[..],
+            ))?),
+        };
+        Some(serde_json::to_vec(&prelude)?)
     } else {
         None
     };
@@ -257,7 +269,7 @@ where
             (None, None)
         };
 
-    let mut encoder = Encoder::new(writers.archive, &metadata, cli_params).await?;
+    let mut encoder = Encoder::new(writers.archive, &metadata, prelude.as_deref()).await?;
 
     let mut archiver = Archiver {
         feature_flags,
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] [PATCH proxmox-backup 2/2] client: pxar: encode prelude based on writer variant

[Christian Ebner]

Currently, whether to encode the exlcude patterns passed via cli as
prelude or via the `.pxar-exclude-cli` is based on the presence of
a previous metadata accessor.
That leaves however to the encoding of the file entry instead of the
prelude for split archives in `data` mode and for the first snapshot
in a backup, creating undesired padding in the first payload chunk.

Therefore, use the pxar writer variant to make the decision instead.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-client/src/pxar/create.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
index dc0a31e7a..9a41bf3ee 100644
--- a/pbs-client/src/pxar/create.rs
+++ b/pbs-client/src/pxar/create.rs
@@ -183,6 +183,7 @@ struct Archiver {
     previous_payload_index: Option<DynamicIndexReader>,
     cache: PxarLookaheadCache,
     reuse_stats: ReuseStats,
+    split_archive: bool,
 }
 
 type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
@@ -247,7 +248,8 @@ where
         )?);
     }
 
-    let prelude = if options.previous_ref.is_some() && !patterns.is_empty() {
+    let split_archive = writers.archive.payload().is_some();
+    let prelude = if split_archive && !patterns.is_empty() {
         let prelude = PbsClientPrelude {
             exclude_patterns: Some(String::from_utf8(generate_pxar_excludes_cli(
                 &patterns[..],
@@ -258,7 +260,7 @@ where
         None
     };
 
-    let metadata_mode = options.previous_ref.is_some() && writers.archive.payload().is_some();
+    let metadata_mode = options.previous_ref.is_some() && split_archive;
     let (previous_payload_index, previous_metadata_accessor) =
         if let Some(refs) = options.previous_ref {
             (
@@ -291,6 +293,7 @@ where
         previous_payload_index,
         cache: PxarLookaheadCache::new(options.max_cache_size),
         reuse_stats: ReuseStats::default(),
+        split_archive,
     };
 
     archiver
@@ -388,12 +391,11 @@ impl Archiver {
             for file_entry in file_list {
                 let file_name = file_entry.name.to_bytes();
 
-                if is_root
-                    && file_name == b".pxarexclude-cli"
-                    && previous_metadata_accessor.is_none()
-                {
-                    self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)
-                        .await?;
+                if is_root && file_name == b".pxarexclude-cli" {
+                    if !self.split_archive {
+                        self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)
+                            .await?;
+                    }
                     continue;
                 }
 
@@ -1944,6 +1946,7 @@ mod tests {
                 suggested_boundaries: Some(suggested_boundaries),
                 cache: PxarLookaheadCache::new(None),
                 reuse_stats: ReuseStats::default(),
+                split_archive: true,
             };
 
             let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), metadata_size)
-- 
2.39.2



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] applied: [PATCH proxmox-backup 1/2] client: pxar: json encode cli exclude pattern in prelude

[Fabian Grünbichler]

On June 10, 2024 1:06 pm, Christian Ebner wrote:
> The current encoding is not extensible, so encode the cli exclude
> patterns as json instead. By this, the prelude is easily seralized
> and deserialized, while remaining human readable.
> 
> Originally-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
>  pbs-client/src/pxar/create.rs | 20 ++++++++++++++++----
>  1 file changed, 16 insertions(+), 4 deletions(-)
> 
> diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
> index 12a4d92c3..dc0a31e7a 100644
> --- a/pbs-client/src/pxar/create.rs
> +++ b/pbs-client/src/pxar/create.rs
> @@ -16,6 +16,7 @@ use nix::dir::Dir;
>  use nix::errno::Errno;
>  use nix::fcntl::OFlag;
>  use nix::sys::stat::{FileStat, Mode};
> +use serde::{Deserialize, Serialize};
>  
>  use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternFlag};
>  use proxmox_sys::error::SysError;
> @@ -154,6 +155,13 @@ struct ReuseStats {
>      total_reencoded_size: u64,
>  }
>  
> +#[derive(Serialize, Deserialize)]
> +#[serde(rename_all = "kebab-case")]
> +pub(crate) struct PbsClientPrelude {
> +    #[serde(skip_serializing_if = "Option::is_none")]
> +    exclude_patterns: Option<String>,
> +}
> +
>  struct Archiver {
>      feature_flags: Flags,
>      fs_feature_flags: Flags,
> @@ -239,9 +247,13 @@ where
>          )?);
>      }
>  
> -    let cli_params_content = generate_pxar_excludes_cli(&patterns[..]);
> -    let cli_params = if options.previous_ref.is_some() {
> -        Some(cli_params_content.as_slice())
> +    let prelude = if options.previous_ref.is_some() && !patterns.is_empty() {
> +        let prelude = PbsClientPrelude {
> +            exclude_patterns: Some(String::from_utf8(generate_pxar_excludes_cli(
> +                &patterns[..],
> +            ))?),
> +        };
> +        Some(serde_json::to_vec(&prelude)?)
>      } else {
>          None
>      };
> @@ -257,7 +269,7 @@ where
>              (None, None)
>          };
>  
> -    let mut encoder = Encoder::new(writers.archive, &metadata, cli_params).await?;
> +    let mut encoder = Encoder::new(writers.archive, &metadata, prelude.as_deref()).await?;
>  
>      let mut archiver = Archiver {
>          feature_flags,
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel

[pbs-devel] applied: [PATCH proxmox-backup 2/2] client: pxar: encode prelude based on writer variant

[Fabian Grünbichler]

On June 10, 2024 1:06 pm, Christian Ebner wrote:
> Currently, whether to encode the exlcude patterns passed via cli as
> prelude or via the `.pxar-exclude-cli` is based on the presence of
> a previous metadata accessor.
> That leaves however to the encoding of the file entry instead of the
> prelude for split archives in `data` mode and for the first snapshot
> in a backup, creating undesired padding in the first payload chunk.
> 
> Therefore, use the pxar writer variant to make the decision instead.
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
>  pbs-client/src/pxar/create.rs | 19 +++++++++++--------
>  1 file changed, 11 insertions(+), 8 deletions(-)
> 
> diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
> index dc0a31e7a..9a41bf3ee 100644
> --- a/pbs-client/src/pxar/create.rs
> +++ b/pbs-client/src/pxar/create.rs
> @@ -183,6 +183,7 @@ struct Archiver {
>      previous_payload_index: Option<DynamicIndexReader>,
>      cache: PxarLookaheadCache,
>      reuse_stats: ReuseStats,
> +    split_archive: bool,
>  }
>  
>  type Encoder<'a, T> = pxar::encoder::aio::Encoder<'a, T>;
> @@ -247,7 +248,8 @@ where
>          )?);
>      }
>  
> -    let prelude = if options.previous_ref.is_some() && !patterns.is_empty() {
> +    let split_archive = writers.archive.payload().is_some();
> +    let prelude = if split_archive && !patterns.is_empty() {
>          let prelude = PbsClientPrelude {
>              exclude_patterns: Some(String::from_utf8(generate_pxar_excludes_cli(
>                  &patterns[..],
> @@ -258,7 +260,7 @@ where
>          None
>      };
>  
> -    let metadata_mode = options.previous_ref.is_some() && writers.archive.payload().is_some();
> +    let metadata_mode = options.previous_ref.is_some() && split_archive;
>      let (previous_payload_index, previous_metadata_accessor) =
>          if let Some(refs) = options.previous_ref {
>              (
> @@ -291,6 +293,7 @@ where
>          previous_payload_index,
>          cache: PxarLookaheadCache::new(options.max_cache_size),
>          reuse_stats: ReuseStats::default(),
> +        split_archive,
>      };
>  
>      archiver
> @@ -388,12 +391,11 @@ impl Archiver {
>              for file_entry in file_list {
>                  let file_name = file_entry.name.to_bytes();
>  
> -                if is_root
> -                    && file_name == b".pxarexclude-cli"
> -                    && previous_metadata_accessor.is_none()
> -                {
> -                    self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)
> -                        .await?;
> +                if is_root && file_name == b".pxarexclude-cli" {
> +                    if !self.split_archive {
> +                        self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)
> +                            .await?;
> +                    }
>                      continue;
>                  }
>  
> @@ -1944,6 +1946,7 @@ mod tests {
>                  suggested_boundaries: Some(suggested_boundaries),
>                  cache: PxarLookaheadCache::new(None),
>                  reuse_stats: ReuseStats::default(),
> +                split_archive: true,
>              };
>  
>              let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), metadata_size)
> -- 
> 2.39.2
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 
> 
> 


_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel