From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: also filter {component}/{section}
Date: Fri, 21 Oct 2022 11:12:42 +0200 [thread overview]
Message-ID: <20221021091244.3846051-1-f.gruenbichler@proxmox.com> (raw)
so that a single filter 'games' covers all related sections for a stock
Debian repository.
this also has the side-effect that package downloads are now batched by
component.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/mirror.rs | 58 +++++++++++++++++++++++++++++++++++----------------
1 file changed, 40 insertions(+), 18 deletions(-)
diff --git a/src/mirror.rs b/src/mirror.rs
index e6aca8a..ea593a9 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -504,6 +504,7 @@ fn convert_to_globset(config: &ParsedMirrorConfig) -> Result<Option<GlobSet>, Er
fn fetch_binary_packages(
config: &ParsedMirrorConfig,
+ component: &str,
packages_indices: HashMap<&String, PackagesFile>,
dry_run: bool,
prefix: &Path,
@@ -523,9 +524,13 @@ fn fetch_binary_packages(
let mut fetch_progress = Progress::new();
let mut skip_count = 0usize;
let mut skip_bytes = 0usize;
+
for package in references.files {
if let Some(ref sections) = &config.skip.skip_sections {
- if sections.iter().any(|section| package.section == *section) {
+ if sections.iter().any(|section| {
+ package.section == *section
+ || package.section == format!("{component}/{section}")
+ }) {
println!(
"\tskipping {} - {}b (section '{}')",
package.package, package.size, package.section
@@ -615,6 +620,7 @@ fn fetch_binary_packages(
fn fetch_source_packages(
config: &ParsedMirrorConfig,
+ component: &str,
source_packages_indices: HashMap<&String, SourcesFile>,
dry_run: bool,
prefix: &Path,
@@ -636,10 +642,10 @@ fn fetch_source_packages(
let mut skip_bytes = 0usize;
for package in references.source_packages {
if let Some(ref sections) = &config.skip.skip_sections {
- if sections
- .iter()
- .any(|section| package.section.as_ref() == Some(section))
- {
+ if sections.iter().any(|section| {
+ package.section.as_ref() == Some(section)
+ || package.section == Some(format!("{component}/{section}"))
+ }) {
println!(
"\tskipping {} - {}b (section '{}')",
package.package,
@@ -897,9 +903,13 @@ pub fn create_snapshot(
println!();
let mut packages_size = 0_usize;
- let mut packages_indices = HashMap::new();
-
- let mut source_packages_indices = HashMap::new();
+ let mut per_component_indices: HashMap<
+ String,
+ (
+ HashMap<&String, PackagesFile>,
+ HashMap<&String, SourcesFile>,
+ ),
+ > = HashMap::new();
let mut failed_references = Vec::new();
for (component, references) in per_component {
@@ -909,6 +919,9 @@ pub fn create_snapshot(
let mut fetch_progress = Progress::new();
+ let (packages_indices, source_packages_indices) =
+ per_component_indices.entry(component.clone()).or_default();
+
for basename in references {
println!("\tFetching '{basename}'..");
let files = release.files.get(basename).unwrap();
@@ -1001,17 +1014,26 @@ pub fn create_snapshot(
}
}
- println!("\nFetching packages..");
-
- fetch_binary_packages(&config, packages_indices, dry_run, prefix, &mut progress)?;
+ for (component, (packages_indices, source_packages_indices)) in per_component_indices {
+ println!("\nFetching {component} packages..");
+ fetch_binary_packages(
+ &config,
+ &component,
+ packages_indices,
+ dry_run,
+ prefix,
+ &mut progress,
+ )?;
- fetch_source_packages(
- &config,
- source_packages_indices,
- dry_run,
- prefix,
- &mut progress,
- )?;
+ fetch_source_packages(
+ &config,
+ &component,
+ source_packages_indices,
+ dry_run,
+ prefix,
+ &mut progress,
+ )?;
+ }
if dry_run {
println!(
--
2.30.2
next reply other threads:[~2022-10-21 9:13 UTC|newest]
Thread overview: 4+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-10-21 9:12 Fabian Grünbichler [this message]
2022-10-21 9:12 ` [pve-devel] [PATCH proxmox-offline-mirror 2/3] mirror setup: query filters in guided mode Fabian Grünbichler
2022-10-21 9:12 ` [pve-devel] [PATCH proxmox-offline-mirror 3/3] docs: add section/package filters Fabian Grünbichler
2022-10-21 11:04 ` [pve-devel] applied-series: [PATCH proxmox-offline-mirror 1/3] mirror: also filter {component}/{section} Wolfgang Bumiller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221021091244.3846051-1-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox