From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 9310189CD4 for ; Tue, 18 Oct 2022 11:21:30 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 73131269E5 for ; Tue, 18 Oct 2022 11:21:00 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 18 Oct 2022 11:20:58 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 84E2E44A55 for ; Tue, 18 Oct 2022 11:20:58 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Tue, 18 Oct 2022 11:20:37 +0200 Message-Id: <20221018092040.860121-4-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221018092040.860121-1-f.gruenbichler@proxmox.com> References: <20221018092040.860121-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.143 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [config.rs, proxmox-offline-mirror.rs, mirror.rs] Subject: [pve-devel] [PATCH proxmox-offline-mirror 1/4] mirror: add exclusion of packages/sections X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 18 Oct 2022 09:21:30 -0000 to keep the size of mirror snapshots down by excluding unnecessary files (e.g., games data, browsers, debug packages, ..). Signed-off-by: Fabian Grünbichler --- Notes: requires proxmox-apt with 'section' field we could suggest excluding sections like 'games' in the wizard/docs.. Cargo.toml | 1 + debian/control | 2 + src/bin/proxmox-offline-mirror.rs | 4 +- src/bin/proxmox_offline_mirror_cmds/config.rs | 8 +++ src/config.rs | 40 ++++++++++++- src/mirror.rs | 59 ++++++++++++++++++- 6 files changed, 111 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76791c8..b2bb188 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ anyhow = "1.0" base64 = "0.13" bzip2 = "0.4" flate2 = "1.0.22" +globset = "0.4.8" hex = "0.4.3" lazy_static = "1.4" nix = "0.24" diff --git a/debian/control b/debian/control index 0741a7b..9fe6605 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 12), librust-base64-0.13+default-dev, librust-bzip2-0.4+default-dev, librust-flate2-1+default-dev (>= 1.0.22-~~), + librust-globset-0.4+default-dev (>= 0.4.8-~~), librust-hex-0.4+default-dev (>= 0.4.3-~~), librust-lazy-static-1+default-dev (>= 1.4-~~), librust-nix-0.24+default-dev, @@ -57,6 +58,7 @@ Depends: librust-base64-0.13+default-dev, librust-bzip2-0.4+default-dev, librust-flate2-1+default-dev (>= 1.0.22-~~), + librust-globset-0.4+default-dev (>= 0.4.8-~~), librust-hex-0.4+default-dev (>= 0.4.3-~~), librust-lazy-static-1+default-dev (>= 1.4-~~), librust-nix-0.24+default-dev, diff --git a/src/bin/proxmox-offline-mirror.rs b/src/bin/proxmox-offline-mirror.rs index 522056b..07b6ce6 100644 --- a/src/bin/proxmox-offline-mirror.rs +++ b/src/bin/proxmox-offline-mirror.rs @@ -13,7 +13,7 @@ use proxmox_offline_mirror::helpers::tty::{ read_bool_from_tty, read_selection_from_tty, read_string_from_tty, }; use proxmox_offline_mirror::{ - config::{save_config, MediaConfig, MirrorConfig}, + config::{save_config, MediaConfig, MirrorConfig, SkipConfig}, mirror, types::{ProductType, MEDIA_ID_SCHEMA, MIRROR_ID_SCHEMA}, }; @@ -387,6 +387,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result, Er base_dir: base_dir.clone(), use_subscription: None, ignore_errors: false, + skip: SkipConfig::default(), // TODO sensible default? }); } } @@ -401,6 +402,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result, Er base_dir, use_subscription, ignore_errors: false, + skip: SkipConfig::default(), }; configs.push(main_config); diff --git a/src/bin/proxmox_offline_mirror_cmds/config.rs b/src/bin/proxmox_offline_mirror_cmds/config.rs index 5ebf6d5..3ebf4ad 100644 --- a/src/bin/proxmox_offline_mirror_cmds/config.rs +++ b/src/bin/proxmox_offline_mirror_cmds/config.rs @@ -266,6 +266,14 @@ pub fn update_mirror( data.ignore_errors = ignore_errors } + if let Some(skip_packages) = update.skip.skip_packages { + data.skip.skip_packages = Some(skip_packages); + } + + if let Some(skip_sections) = update.skip.skip_sections { + data.skip.skip_sections = Some(skip_sections); + } + config.set_data(&id, "mirror", &data)?; proxmox_offline_mirror::config::save_config(&config_file, &config)?; diff --git a/src/config.rs b/src/config.rs index be8f96b..39b1193 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,6 +14,38 @@ use crate::types::{ PROXMOX_SUBSCRIPTION_KEY_SCHEMA, }; +/// Skip Configuration +#[api( + properties: { + "skip-sections": { + type: Array, + optional: true, + items: { + type: String, + description: "Section name", + }, + }, + "skip-packages": { + type: Array, + optional: true, + items: { + type: String, + description: "Package name", + }, + }, + }, +)] +#[derive(Default, Serialize, Deserialize, Updater, Clone, Debug)] +#[serde(rename_all = "kebab-case")] +pub struct SkipConfig { + /// Sections which should be skipped + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_sections: Option>, + /// Packages which should be skipped, supports globbing + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_packages: Option>, +} + #[api( properties: { id: { @@ -46,6 +78,9 @@ use crate::types::{ optional: true, default: false, }, + "skip": { + type: SkipConfig, + }, } )] #[derive(Clone, Debug, Serialize, Deserialize, Updater)] @@ -73,6 +108,9 @@ pub struct MirrorConfig { /// Whether to downgrade download errors to warnings #[serde(default)] pub ignore_errors: bool, + /// Skip package files using these criteria + #[serde(default, flatten)] + pub skip: SkipConfig, } #[api( @@ -191,7 +229,7 @@ fn init() -> SectionConfig { let mut config = SectionConfig::new(&MIRROR_ID_SCHEMA); let mirror_schema = match MirrorConfig::API_SCHEMA { - Schema::Object(ref obj_schema) => obj_schema, + Schema::AllOf(ref all_of_schema) => all_of_schema, _ => unreachable!(), }; let mirror_plugin = SectionConfigPlugin::new( diff --git a/src/mirror.rs b/src/mirror.rs index dfb4cc9..22dc716 100644 --- a/src/mirror.rs +++ b/src/mirror.rs @@ -7,12 +7,13 @@ use std::{ use anyhow::{bail, format_err, Error}; use flate2::bufread::GzDecoder; +use globset::{Glob, GlobSetBuilder}; use nix::libc; use proxmox_http::{client::sync::Client, HttpClient, HttpOptions}; use proxmox_sys::fs::file_get_contents; use crate::{ - config::{MirrorConfig, SubscriptionKey}, + config::{MirrorConfig, SkipConfig, SubscriptionKey}, convert_repo_line, pool::Pool, types::{Diff, Snapshot, SNAPSHOT_REGEX}, @@ -47,6 +48,7 @@ struct ParsedMirrorConfig { pub auth: Option, pub client: Client, pub ignore_errors: bool, + pub skip: SkipConfig, } impl TryInto for MirrorConfig { @@ -76,6 +78,7 @@ impl TryInto for MirrorConfig { auth: None, client, ignore_errors: self.ignore_errors, + skip: self.skip, }) } } @@ -664,8 +667,22 @@ pub fn create_snapshot( } } + let skipped_package_globs = if let Some(skipped_packages) = &config.skip.skip_packages { + let mut globs = GlobSetBuilder::new(); + for glob in skipped_packages { + let glob = Glob::new(glob)?; + globs.add(glob); + } + let globs = globs.build()?; + Some(globs) + } else { + None + }; + println!("\nFetching packages.."); let mut dry_run_progress = Progress::new(); + let mut total_skipped_count = 0usize; + let mut total_skipped_bytes = 0usize; for (basename, references) in packages_indices { let total_files = references.files.len(); if total_files == 0 { @@ -676,7 +693,37 @@ pub fn create_snapshot( } let mut fetch_progress = Progress::new(); + let mut skipped_count = 0usize; + let mut skipped_bytes = 0usize; for package in references.files { + if let Some(ref sections) = &config.skip.skip_sections { + if sections.iter().any(|section| package.section == *section) { + println!( + "\tskipping {} - {}b (section '{}')", + package.package, package.size, package.section + ); + skipped_count += 1; + skipped_bytes += package.size; + continue; + } + } + if let Some(skipped_package_globs) = &skipped_package_globs { + let matches = skipped_package_globs.matches(&package.package); + if !matches.is_empty() { + // safety, skipped_package_globs is set based on this + let globs = config.skip.skip_packages.as_ref().unwrap(); + let matches: Vec = matches.iter().map(|i| globs[*i].clone()).collect(); + println!( + "\tskipping {} - {}b (package glob(s): {})", + package.package, + package.size, + matches.join(", ") + ); + skipped_count += 1; + skipped_bytes += package.size; + continue; + } + } let url = get_repo_url(&config.repository, &package.file); if dry_run { @@ -728,6 +775,11 @@ pub fn create_snapshot( } else { total_progress += fetch_progress; } + if skipped_count > 0 { + total_skipped_count += skipped_count; + total_skipped_bytes += skipped_bytes; + println!("Skipped downloading {skipped_count} packages totalling {skipped_bytes}b"); + } } if dry_run { @@ -736,6 +788,11 @@ pub fn create_snapshot( } else { println!("\nStats: {total_progress}"); } + if total_count > 0 { + println!( + "Skipped downloading {total_skipped_count} packages totalling {total_skipped_bytes}b" + ); + } if !warnings.is_empty() { eprintln!("Warnings:"); -- 2.30.2