From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 9AE1293064 for ; Thu, 15 Sep 2022 15:10:09 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 498D219A16 for ; Thu, 15 Sep 2022 15:09:39 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 15 Sep 2022 15:09:38 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7758444298 for ; Thu, 15 Sep 2022 15:09:38 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Thu, 15 Sep 2022 15:09:16 +0200 Message-Id: <20220915130918.727902-6-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220915130918.727902-1-f.gruenbichler@proxmox.com> References: <20220915130918.727902-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.157 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH proxmox-offline-mirror 3/4] mirror: support acquiring indices by hash X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Sep 2022 13:10:09 -0000 requires proxmox-apt > 0.9.1, since earlier versions misdetect by-hash support in the release file. Signed-off-by: Fabian Grünbichler --- src/mirror.rs | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/src/mirror.rs b/src/mirror.rs index f910e6a..6cbd680 100644 --- a/src/mirror.rs +++ b/src/mirror.rs @@ -233,6 +233,7 @@ fn fetch_index_file( prefix: &Path, reference: &FileReference, uncompressed: &FileReference, + by_hash: bool, ) -> Result { let url = get_dist_url(&config.repository, &reference.path); let path = get_dist_path(&config.repository, prefix, &reference.path); @@ -252,14 +253,36 @@ fn fetch_index_file( return Ok(FetchResult { data, fetched: 0 }); } - let res = fetch_plain_file( - config, - &url, - &path, - reference.size, - &reference.checksums, - true, - )?; + let urls = if by_hash { + let mut urls = Vec::new(); + if let Some((base_url, _file_name)) = url.rsplit_once('/') { + if let Some(sha512) = reference.checksums.sha512 { + urls.push(format!("{base_url}/by-hash/SHA512/{}", hex::encode(sha512))); + } + if let Some(sha256) = reference.checksums.sha256 { + urls.push(format!("{base_url}/by-hash/SHA256/{}", hex::encode(sha256))); + } + } + urls.push(url); + urls + } else { + vec![url] + }; + + let res = urls + .iter() + .fold(None, |res, url| match res { + Some(Ok(res)) => Some(Ok(res)), + _ => Some(fetch_plain_file( + config, + &url, + &path, + reference.size, + &reference.checksums, + true, + )), + }) + .ok_or_else(|| format_err!("Failed to retrieve {}", reference.path))??; let mut buf = Vec::new(); let raw = res.data_ref(); @@ -556,7 +579,13 @@ pub fn create_snapshot( } // this will ensure the uncompressed file will be written locally - let res = match fetch_index_file(&config, prefix, reference, uncompressed_ref) { + let res = match fetch_index_file( + &config, + prefix, + reference, + uncompressed_ref, + release.aquire_by_hash, + ) { Ok(res) => res, Err(err) if !reference.file_type.is_package_index() => { eprintln!( -- 2.30.2