[pve-devel] [PATCH proxmox-offline-mirror 1/4] mirror: add exclusion of packages/sections

["Fabian Grünbichler" <f.gruenbichler@proxmox.com>]

to keep the size of mirror snapshots down by excluding unnecessary files
(e.g., games data, browsers, debug packages, ..).

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---

Notes:
    requires proxmox-apt with 'section' field

    we could suggest excluding sections like 'games' in the
    wizard/docs..

 Cargo.toml                                    |  1 +
 debian/control                                |  2 +
 src/bin/proxmox-offline-mirror.rs             |  4 +-
 src/bin/proxmox_offline_mirror_cmds/config.rs |  8 +++
 src/config.rs                                 | 40 ++++++++++++-
 src/mirror.rs                                 | 59 ++++++++++++++++++-
 6 files changed, 111 insertions(+), 3 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 76791c8..b2bb188 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ anyhow = "1.0"
 base64 = "0.13"
 bzip2 = "0.4"
 flate2 = "1.0.22"
+globset = "0.4.8"
 hex = "0.4.3"
 lazy_static = "1.4"
 nix = "0.24"
diff --git a/debian/control b/debian/control
index 0741a7b..9fe6605 100644
--- a/debian/control
+++ b/debian/control
@@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 12),
  librust-base64-0.13+default-dev,
  librust-bzip2-0.4+default-dev,
  librust-flate2-1+default-dev (>= 1.0.22-~~),
+ librust-globset-0.4+default-dev (>= 0.4.8-~~),
  librust-hex-0.4+default-dev (>= 0.4.3-~~),
  librust-lazy-static-1+default-dev (>= 1.4-~~),
  librust-nix-0.24+default-dev,
@@ -57,6 +58,7 @@ Depends:
  librust-base64-0.13+default-dev,
  librust-bzip2-0.4+default-dev,
  librust-flate2-1+default-dev (>= 1.0.22-~~),
+ librust-globset-0.4+default-dev (>= 0.4.8-~~),
  librust-hex-0.4+default-dev (>= 0.4.3-~~),
  librust-lazy-static-1+default-dev (>= 1.4-~~),
  librust-nix-0.24+default-dev,
diff --git a/src/bin/proxmox-offline-mirror.rs b/src/bin/proxmox-offline-mirror.rs
index 522056b..07b6ce6 100644
--- a/src/bin/proxmox-offline-mirror.rs
+++ b/src/bin/proxmox-offline-mirror.rs
@@ -13,7 +13,7 @@ use proxmox_offline_mirror::helpers::tty::{
     read_bool_from_tty, read_selection_from_tty, read_string_from_tty,
 };
 use proxmox_offline_mirror::{
-    config::{save_config, MediaConfig, MirrorConfig},
+    config::{save_config, MediaConfig, MirrorConfig, SkipConfig},
     mirror,
     types::{ProductType, MEDIA_ID_SCHEMA, MIRROR_ID_SCHEMA},
 };
@@ -387,6 +387,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
                 base_dir: base_dir.clone(),
                 use_subscription: None,
                 ignore_errors: false,
+                skip: SkipConfig::default(), // TODO sensible default?
             });
         }
     }
@@ -401,6 +402,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
         base_dir,
         use_subscription,
         ignore_errors: false,
+        skip: SkipConfig::default(),
     };
 
     configs.push(main_config);
diff --git a/src/bin/proxmox_offline_mirror_cmds/config.rs b/src/bin/proxmox_offline_mirror_cmds/config.rs
index 5ebf6d5..3ebf4ad 100644
--- a/src/bin/proxmox_offline_mirror_cmds/config.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/config.rs
@@ -266,6 +266,14 @@ pub fn update_mirror(
         data.ignore_errors = ignore_errors
     }
 
+    if let Some(skip_packages) = update.skip.skip_packages {
+        data.skip.skip_packages = Some(skip_packages);
+    }
+
+    if let Some(skip_sections) = update.skip.skip_sections {
+        data.skip.skip_sections = Some(skip_sections);
+    }
+
     config.set_data(&id, "mirror", &data)?;
     proxmox_offline_mirror::config::save_config(&config_file, &config)?;
 
diff --git a/src/config.rs b/src/config.rs
index be8f96b..39b1193 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -14,6 +14,38 @@ use crate::types::{
     PROXMOX_SUBSCRIPTION_KEY_SCHEMA,
 };
 
+/// Skip Configuration
+#[api(
+    properties: {
+        "skip-sections": {
+            type: Array,
+            optional: true,
+            items: {
+                type: String,
+                description: "Section name",
+            },
+        },
+        "skip-packages": {
+            type: Array,
+            optional: true,
+            items: {
+                type: String,
+                description: "Package name",
+            },
+        },
+    },
+)]
+#[derive(Default, Serialize, Deserialize, Updater, Clone, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct SkipConfig {
+    /// Sections which should be skipped
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub skip_sections: Option<Vec<String>>,
+    /// Packages which should be skipped, supports globbing
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub skip_packages: Option<Vec<String>>,
+}
+
 #[api(
     properties: {
         id: {
@@ -46,6 +78,9 @@ use crate::types::{
             optional: true,
             default: false,
         },
+        "skip": {
+            type: SkipConfig,
+        },
     }
 )]
 #[derive(Clone, Debug, Serialize, Deserialize, Updater)]
@@ -73,6 +108,9 @@ pub struct MirrorConfig {
     /// Whether to downgrade download errors to warnings
     #[serde(default)]
     pub ignore_errors: bool,
+    /// Skip package files using these criteria
+    #[serde(default, flatten)]
+    pub skip: SkipConfig,
 }
 
 #[api(
@@ -191,7 +229,7 @@ fn init() -> SectionConfig {
     let mut config = SectionConfig::new(&MIRROR_ID_SCHEMA);
 
     let mirror_schema = match MirrorConfig::API_SCHEMA {
-        Schema::Object(ref obj_schema) => obj_schema,
+        Schema::AllOf(ref all_of_schema) => all_of_schema,
         _ => unreachable!(),
     };
     let mirror_plugin = SectionConfigPlugin::new(
diff --git a/src/mirror.rs b/src/mirror.rs
index dfb4cc9..22dc716 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -7,12 +7,13 @@ use std::{
 
 use anyhow::{bail, format_err, Error};
 use flate2::bufread::GzDecoder;
+use globset::{Glob, GlobSetBuilder};
 use nix::libc;
 use proxmox_http::{client::sync::Client, HttpClient, HttpOptions};
 use proxmox_sys::fs::file_get_contents;
 
 use crate::{
-    config::{MirrorConfig, SubscriptionKey},
+    config::{MirrorConfig, SkipConfig, SubscriptionKey},
     convert_repo_line,
     pool::Pool,
     types::{Diff, Snapshot, SNAPSHOT_REGEX},
@@ -47,6 +48,7 @@ struct ParsedMirrorConfig {
     pub auth: Option<String>,
     pub client: Client,
     pub ignore_errors: bool,
+    pub skip: SkipConfig,
 }
 
 impl TryInto<ParsedMirrorConfig> for MirrorConfig {
@@ -76,6 +78,7 @@ impl TryInto<ParsedMirrorConfig> for MirrorConfig {
             auth: None,
             client,
             ignore_errors: self.ignore_errors,
+            skip: self.skip,
         })
     }
 }
@@ -664,8 +667,22 @@ pub fn create_snapshot(
         }
     }
 
+    let skipped_package_globs = if let Some(skipped_packages) = &config.skip.skip_packages {
+        let mut globs = GlobSetBuilder::new();
+        for glob in skipped_packages {
+            let glob = Glob::new(glob)?;
+            globs.add(glob);
+        }
+        let globs = globs.build()?;
+        Some(globs)
+    } else {
+        None
+    };
+
     println!("\nFetching packages..");
     let mut dry_run_progress = Progress::new();
+    let mut total_skipped_count = 0usize;
+    let mut total_skipped_bytes = 0usize;
     for (basename, references) in packages_indices {
         let total_files = references.files.len();
         if total_files == 0 {
@@ -676,7 +693,37 @@ pub fn create_snapshot(
         }
 
         let mut fetch_progress = Progress::new();
+        let mut skipped_count = 0usize;
+        let mut skipped_bytes = 0usize;
         for package in references.files {
+            if let Some(ref sections) = &config.skip.skip_sections {
+                if sections.iter().any(|section| package.section == *section) {
+                    println!(
+                        "\tskipping {} - {}b (section '{}')",
+                        package.package, package.size, package.section
+                    );
+                    skipped_count += 1;
+                    skipped_bytes += package.size;
+                    continue;
+                }
+            }
+            if let Some(skipped_package_globs) = &skipped_package_globs {
+                let matches = skipped_package_globs.matches(&package.package);
+                if !matches.is_empty() {
+                    // safety, skipped_package_globs is set based on this
+                    let globs = config.skip.skip_packages.as_ref().unwrap();
+                    let matches: Vec<String> = matches.iter().map(|i| globs[*i].clone()).collect();
+                    println!(
+                        "\tskipping {} - {}b (package glob(s): {})",
+                        package.package,
+                        package.size,
+                        matches.join(", ")
+                    );
+                    skipped_count += 1;
+                    skipped_bytes += package.size;
+                    continue;
+                }
+            }
             let url = get_repo_url(&config.repository, &package.file);
 
             if dry_run {
@@ -728,6 +775,11 @@ pub fn create_snapshot(
         } else {
             total_progress += fetch_progress;
         }
+        if skipped_count > 0 {
+            total_skipped_count += skipped_count;
+            total_skipped_bytes += skipped_bytes;
+            println!("Skipped downloading {skipped_count} packages totalling {skipped_bytes}b");
+        }
     }
 
     if dry_run {
@@ -736,6 +788,11 @@ pub fn create_snapshot(
     } else {
         println!("\nStats: {total_progress}");
     }
+    if total_count > 0 {
+        println!(
+            "Skipped downloading {total_skipped_count} packages totalling {total_skipped_bytes}b"
+        );
+    }
 
     if !warnings.is_empty() {
         eprintln!("Warnings:");
-- 
2.30.2