From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 21549933A7 for ; Fri, 16 Sep 2022 12:36:15 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 182EB228F9 for ; Fri, 16 Sep 2022 12:35:45 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Fri, 16 Sep 2022 12:35:44 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E08D34447D for ; Fri, 16 Sep 2022 12:35:43 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Fri, 16 Sep 2022 12:35:33 +0200 Message-Id: <20220916103535.103013-2-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220916103535.103013-1-f.gruenbichler@proxmox.com> References: <20220916103535.103013-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.151 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mirror.rs] Subject: [pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: add --dry-run parameter X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 16 Sep 2022 10:36:15 -0000 in dry-run mode, creating a snapshot will download (but not persist) the Release files and any indices referenced within, but not download the package files themselves. instead, any URLs that would still need to be fetched are printed, and the statistics about to-be-fetched files and bytes is updated accordingly. Signed-off-by: Fabian Grünbichler --- src/bin/proxmox_offline_mirror_cmds/mirror.rs | 20 +++- src/mirror.rs | 108 ++++++++++++++---- 2 files changed, 102 insertions(+), 26 deletions(-) diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs index 8059c0a..0798f86 100644 --- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs +++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs @@ -28,11 +28,22 @@ use super::get_config_path; id: { schema: MIRROR_ID_SCHEMA, }, + "dry-run": { + type: bool, + optional: true, + default: false, + description: "Only fetch indices and print summary of missing package files, don't store anything.", + } }, }, )] /// Create a new repository snapshot, fetching required/missing files from original repository. -async fn create_snapshot(config: Option, id: String, _param: Value) -> Result<(), Error> { +async fn create_snapshot( + config: Option, + id: String, + dry_run: bool, + _param: Value, +) -> Result<(), Error> { let config = config.unwrap_or_else(get_config_path); let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?; @@ -61,7 +72,12 @@ async fn create_snapshot(config: Option, id: String, _param: Value) -> R None }; - proxmox_offline_mirror::mirror::create_snapshot(config, &Snapshot::now(), subscription)?; + proxmox_offline_mirror::mirror::create_snapshot( + config, + &Snapshot::now(), + subscription, + dry_run, + )?; Ok(()) } diff --git a/src/mirror.rs b/src/mirror.rs index a1fc1a0..5126393 100644 --- a/src/mirror.rs +++ b/src/mirror.rs @@ -143,6 +143,7 @@ fn fetch_release( config: &ParsedMirrorConfig, prefix: &Path, detached: bool, + dry_run: bool, ) -> Result { let (name, fetched, sig) = if detached { println!("Fetching Release/Release.gpg files"); @@ -185,6 +186,13 @@ fn fetch_release( ..Default::default() }; + if dry_run { + return Ok(FetchResult { + data: verified, + fetched: fetched.fetched, + }); + } + let locked = &config.pool.lock()?; if !locked.contains(&csums) { @@ -234,6 +242,7 @@ fn fetch_index_file( reference: &FileReference, uncompressed: Option<&FileReference>, by_hash: bool, + dry_run: bool, ) -> Result { let url = get_dist_url(&config.repository, &reference.path); let path = get_dist_path(&config.repository, prefix, &reference.path); @@ -248,6 +257,9 @@ fn fetch_index_file( .pool .get_contents(&uncompressed.checksums, config.verify)?; + if dry_run { + return Ok(FetchResult { data, fetched: 0 }); + } // Ensure they're linked at current path config.pool.lock()?.link_file(&reference.checksums, &path)?; config @@ -285,6 +297,7 @@ fn fetch_index_file( reference.size, &reference.checksums, true, + dry_run, )), }) .ok_or_else(|| format_err!("Failed to retrieve {}", reference.path))??; @@ -310,6 +323,14 @@ fn fetch_index_file( &buf[..] } }; + let res = FetchResult { + data: decompressed.to_owned(), + fetched: res.fetched, + }; + + if dry_run { + return Ok(res); + } let locked = &config.pool.lock()?; if let Some(uncompressed) = uncompressed { @@ -322,10 +343,7 @@ fn fetch_index_file( locked.link_file(&uncompressed.checksums, &uncompressed_path)?; } - Ok(FetchResult { - data: decompressed.to_owned(), - fetched: res.fetched, - }) + Ok(res) } /// Helper to fetch arbitrary files like binary packages. @@ -340,6 +358,7 @@ fn fetch_plain_file( max_size: usize, checksums: &CheckSums, need_data: bool, + dry_run: bool, ) -> Result { let locked = &config.pool.lock()?; let res = if locked.contains(checksums) { @@ -355,6 +374,11 @@ fn fetch_plain_file( fetched: 0, } } + } else if dry_run && !need_data { + FetchResult { + data: vec![], + fetched: 0, + } } else { let fetched = fetch_repo_file( &config.client, @@ -367,8 +391,10 @@ fn fetch_plain_file( fetched }; - // Ensure it's linked at current path - locked.link_file(checksums, file)?; + if !dry_run { + // Ensure it's linked at current path + locked.link_file(checksums, file)?; + } Ok(res) } @@ -433,6 +459,7 @@ pub fn create_snapshot( config: MirrorConfig, snapshot: &Snapshot, subscription: Option, + dry_run: bool, ) -> Result<(), Error> { let auth = if let Some(product) = &config.use_subscription { match subscription { @@ -477,11 +504,11 @@ pub fn create_snapshot( }; // we want both on-disk for compat reasons - let res = fetch_release(&config, prefix, true)?; + let res = fetch_release(&config, prefix, true, dry_run)?; total_progress.update(&res); let _release = parse_release(res, "Release")?; - let res = fetch_release(&config, prefix, false)?; + let res = fetch_release(&config, prefix, false, dry_run)?; total_progress.update(&res); let release = parse_release(res, "InRelease")?; @@ -591,6 +618,7 @@ pub fn create_snapshot( reference, uncompressed_ref, release.aquire_by_hash, + dry_run, ) { Ok(res) => res, Err(err) if !reference.file_type.is_package_index() => { @@ -632,6 +660,7 @@ pub fn create_snapshot( } println!("\nFetching packages.."); + let mut dry_run_progress = Progress::new(); for (basename, references) in packages_indices { let total_files = references.files.len(); if total_files == 0 { @@ -643,30 +672,61 @@ pub fn create_snapshot( let mut fetch_progress = Progress::new(); for package in references.files { - let mut full_path = PathBuf::from(prefix); - full_path.push(&package.file); - let res = fetch_plain_file( - &config, - &get_repo_url(&config.repository, &package.file), - &full_path, - package.size, - &package.checksums, - false, - )?; - fetch_progress.update(&res); + let url = get_repo_url(&config.repository, &package.file); + + if dry_run { + if config.pool.contains(&package.checksums) { + fetch_progress.update(&FetchResult { + data: vec![], + fetched: 0, + }); + } else { + println!("\t(dry-run) GET missing '{url}' ({}b)", package.size); + fetch_progress.update(&FetchResult { + data: vec![], + fetched: package.size, + }); + } + } else { + let mut full_path = PathBuf::from(prefix); + full_path.push(&package.file); + + let res = fetch_plain_file( + &config, + &url, + &full_path, + package.size, + &package.checksums, + false, + dry_run, + )?; + fetch_progress.update(&res); + } + if fetch_progress.file_count() % (max(total_files / 100, 1)) == 0 { println!("\tProgress: {fetch_progress}"); } } println!("\tProgress: {fetch_progress}"); - total_progress += fetch_progress; + if dry_run { + dry_run_progress += fetch_progress; + } else { + total_progress += fetch_progress; + } } - println!("\nStats: {total_progress}"); + if dry_run { + println!("\nDry-run Stats (indices, downloaded but not persisted):\n{total_progress}"); + println!("\nDry-run stats (packages, new == missing):\n{dry_run_progress}"); + } else { + println!("\nStats: {total_progress}"); + } - println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\""); - let locked = config.pool.lock()?; - locked.rename(prefix, Path::new(&format!("{snapshot}")))?; + if !dry_run { + println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\""); + let locked = config.pool.lock()?; + locked.rename(prefix, Path::new(&format!("{snapshot}")))?; + } Ok(()) } -- 2.30.2