From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 908C08AB30 for ; Fri, 21 Oct 2022 11:13:20 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6E0361FA35 for ; Fri, 21 Oct 2022 11:12:50 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 21 Oct 2022 11:12:49 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 780E944AFD for ; Fri, 21 Oct 2022 11:12:49 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Fri, 21 Oct 2022 11:12:42 +0200 Message-Id: <20221021091244.3846051-1-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.142 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mirror.rs] Subject: [pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: also filter {component}/{section} X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 21 Oct 2022 09:13:20 -0000 so that a single filter 'games' covers all related sections for a stock Debian repository. this also has the side-effect that package downloads are now batched by component. Signed-off-by: Fabian Grünbichler --- src/mirror.rs | 58 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/src/mirror.rs b/src/mirror.rs index e6aca8a..ea593a9 100644 --- a/src/mirror.rs +++ b/src/mirror.rs @@ -504,6 +504,7 @@ fn convert_to_globset(config: &ParsedMirrorConfig) -> Result, Er fn fetch_binary_packages( config: &ParsedMirrorConfig, + component: &str, packages_indices: HashMap<&String, PackagesFile>, dry_run: bool, prefix: &Path, @@ -523,9 +524,13 @@ fn fetch_binary_packages( let mut fetch_progress = Progress::new(); let mut skip_count = 0usize; let mut skip_bytes = 0usize; + for package in references.files { if let Some(ref sections) = &config.skip.skip_sections { - if sections.iter().any(|section| package.section == *section) { + if sections.iter().any(|section| { + package.section == *section + || package.section == format!("{component}/{section}") + }) { println!( "\tskipping {} - {}b (section '{}')", package.package, package.size, package.section @@ -615,6 +620,7 @@ fn fetch_binary_packages( fn fetch_source_packages( config: &ParsedMirrorConfig, + component: &str, source_packages_indices: HashMap<&String, SourcesFile>, dry_run: bool, prefix: &Path, @@ -636,10 +642,10 @@ fn fetch_source_packages( let mut skip_bytes = 0usize; for package in references.source_packages { if let Some(ref sections) = &config.skip.skip_sections { - if sections - .iter() - .any(|section| package.section.as_ref() == Some(section)) - { + if sections.iter().any(|section| { + package.section.as_ref() == Some(section) + || package.section == Some(format!("{component}/{section}")) + }) { println!( "\tskipping {} - {}b (section '{}')", package.package, @@ -897,9 +903,13 @@ pub fn create_snapshot( println!(); let mut packages_size = 0_usize; - let mut packages_indices = HashMap::new(); - - let mut source_packages_indices = HashMap::new(); + let mut per_component_indices: HashMap< + String, + ( + HashMap<&String, PackagesFile>, + HashMap<&String, SourcesFile>, + ), + > = HashMap::new(); let mut failed_references = Vec::new(); for (component, references) in per_component { @@ -909,6 +919,9 @@ pub fn create_snapshot( let mut fetch_progress = Progress::new(); + let (packages_indices, source_packages_indices) = + per_component_indices.entry(component.clone()).or_default(); + for basename in references { println!("\tFetching '{basename}'.."); let files = release.files.get(basename).unwrap(); @@ -1001,17 +1014,26 @@ pub fn create_snapshot( } } - println!("\nFetching packages.."); - - fetch_binary_packages(&config, packages_indices, dry_run, prefix, &mut progress)?; + for (component, (packages_indices, source_packages_indices)) in per_component_indices { + println!("\nFetching {component} packages.."); + fetch_binary_packages( + &config, + &component, + packages_indices, + dry_run, + prefix, + &mut progress, + )?; - fetch_source_packages( - &config, - source_packages_indices, - dry_run, - prefix, - &mut progress, - )?; + fetch_source_packages( + &config, + &component, + source_packages_indices, + dry_run, + prefix, + &mut progress, + )?; + } if dry_run { println!( -- 2.30.2