From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 7C80B93196 for ; Thu, 15 Sep 2022 16:36:21 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 79ED31B376 for ; Thu, 15 Sep 2022 16:36:21 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 15 Sep 2022 16:36:20 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9B3FA4440F for ; Thu, 15 Sep 2022 16:36:20 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Thu, 15 Sep 2022 16:36:15 +0200 Message-Id: <20220915143615.1673687-1-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220915130918.727902-1-f.gruenbichler@proxmox.com> References: <20220915130918.727902-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.156 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH follow-up proxmox-offline-mirror 8/7] mirror: handle indices which are only available compressed X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Sep 2022 14:36:21 -0000 there are repositories out there that not only skip serving the uncompressed version, but not even reference it in their Release file(s). Signed-off-by: Fabian Grünbichler --- best viewed with -w, the signal-desktop repository is one such example. src/mirror.rs | 56 ++++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/mirror.rs b/src/mirror.rs index 3370ca4..f5ee48e 100644 --- a/src/mirror.rs +++ b/src/mirror.rs @@ -232,25 +232,30 @@ fn fetch_index_file( config: &ParsedMirrorConfig, prefix: &Path, reference: &FileReference, - uncompressed: &FileReference, + uncompressed: Option<&FileReference>, by_hash: bool, ) -> Result { let url = get_dist_url(&config.repository, &reference.path); let path = get_dist_path(&config.repository, prefix, &reference.path); - let uncompressed_path = get_dist_path(&config.repository, prefix, &uncompressed.path); - - if config.pool.contains(&reference.checksums) && config.pool.contains(&uncompressed.checksums) { - let data = config - .pool - .get_contents(&uncompressed.checksums, config.verify)?; - - // Ensure they're linked at current path - config.pool.lock()?.link_file(&reference.checksums, &path)?; - config - .pool - .lock()? - .link_file(&uncompressed.checksums, &uncompressed_path)?; - return Ok(FetchResult { data, fetched: 0 }); + + if let Some(uncompressed) = uncompressed { + let uncompressed_path = get_dist_path(&config.repository, prefix, &uncompressed.path); + + if config.pool.contains(&reference.checksums) + && config.pool.contains(&uncompressed.checksums) + { + let data = config + .pool + .get_contents(&uncompressed.checksums, config.verify)?; + + // Ensure they're linked at current path + config.pool.lock()?.link_file(&reference.checksums, &path)?; + config + .pool + .lock()? + .link_file(&uncompressed.checksums, &uncompressed_path)?; + return Ok(FetchResult { data, fetched: 0 }); + } } let urls = if by_hash { @@ -307,12 +312,15 @@ fn fetch_index_file( }; let locked = &config.pool.lock()?; - if !locked.contains(&uncompressed.checksums) { - locked.add_file(decompressed, &uncompressed.checksums, config.sync)?; - } + if let Some(uncompressed) = uncompressed { + if !locked.contains(&uncompressed.checksums) { + locked.add_file(decompressed, &uncompressed.checksums, config.sync)?; + } - // Ensure it's linked at current path - locked.link_file(&uncompressed.checksums, &uncompressed_path)?; + // Ensure it's linked at current path + let uncompressed_path = get_dist_path(&config.repository, prefix, &uncompressed.path); + locked.link_file(&uncompressed.checksums, &uncompressed_path)?; + } Ok(FetchResult { data: decompressed.to_owned(), @@ -566,15 +574,13 @@ pub fn create_snapshot( for basename in references { println!("\tFetching '{basename}'.."); let files = release.files.get(basename).unwrap(); - let uncompressed_ref = files - .iter() - .find(|reference| reference.path == *basename) - .ok_or_else(|| format_err!("Found derived reference without base reference."))?; + let uncompressed_ref = files.iter().find(|reference| reference.path == *basename); + let mut package_index_data = None; for reference in files { // if both compressed and uncompressed are referenced, the uncompressed file may not exist on the server - if reference == uncompressed_ref && files.len() > 1 { + if Some(reference) == uncompressed_ref && files.len() > 1 { continue; } -- 2.30.2