From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 45FF589C4F for ; Tue, 18 Oct 2022 11:21:27 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 24FF5269DA for ; Tue, 18 Oct 2022 11:20:57 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Tue, 18 Oct 2022 11:20:55 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7B95344A55 for ; Tue, 18 Oct 2022 11:20:55 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Tue, 18 Oct 2022 11:20:36 +0200 Message-Id: <20221018092040.860121-3-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20221018092040.860121-1-f.gruenbichler@proxmox.com> References: <20221018092040.860121-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.018 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LOTSOFHASH 0.25 Emails with lots of hash-like gibberish SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs] Subject: [pve-devel] [PATCH proxmox-apt 2/2] deb822: source index support X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 18 Oct 2022 09:21:27 -0000 Signed-off-by: Fabian Grünbichler --- the test file needs to be downloaded from the referenced URL and uncompressed (it's too big to send as patch). its SHA256sum is e7777c1d305f5e0a31bcf2fe26e955436986edb5c211c03a362c7d557c899349 src/deb822/mod.rs | 3 + src/deb822/release_file.rs | 2 +- src/deb822/sources_file.rs | 255 + ..._debian_dists_bullseye_main_source_Sources | 858657 +++++++++++++++ 4 files changed, 858916 insertions(+), 1 deletion(-) create mode 100644 src/deb822/sources_file.rs create mode 100644 tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources diff --git a/src/deb822/mod.rs b/src/deb822/mod.rs index 7a1bb0e..59e7c21 100644 --- a/src/deb822/mod.rs +++ b/src/deb822/mod.rs @@ -5,6 +5,9 @@ pub use release_file::{CompressionType, FileReference, FileReferenceType, Releas mod packages_file; pub use packages_file::PackagesFile; +mod sources_file; +pub use sources_file::SourcesFile; + #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] pub struct CheckSums { pub md5: Option<[u8; 16]>, diff --git a/src/deb822/release_file.rs b/src/deb822/release_file.rs index c50c095..85d3436 100644 --- a/src/deb822/release_file.rs +++ b/src/deb822/release_file.rs @@ -245,7 +245,7 @@ impl FileReferenceType { } pub fn is_package_index(&self) -> bool { - matches!(self, FileReferenceType::Packages(_, _)) + matches!(self, FileReferenceType::Packages(_, _) | FileReferenceType::Sources(_)) } } diff --git a/src/deb822/sources_file.rs b/src/deb822/sources_file.rs new file mode 100644 index 0000000..a13d84f --- /dev/null +++ b/src/deb822/sources_file.rs @@ -0,0 +1,255 @@ +use std::collections::HashMap; + +use anyhow::{bail, Error, format_err}; +use rfc822_like::de::Deserializer; +use serde::Deserialize; +use serde_json::Value; + +use super::CheckSums; +//Uploaders +// +//Homepage +// +//Version Control System (VCS) fields +// +//Testsuite +// +//Dgit +// +//Standards-Version (mandatory) +// +//Build-Depends et al +// +//Package-List (recommended) +// +//Checksums-Sha1 and Checksums-Sha256 (mandatory) +// +//Files (mandatory) + + + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct SourcesFileRaw { + pub format: String, + pub package: String, + pub binary: Option>, + pub version: String, + pub section: Option, + pub priority: Option, + pub maintainer: String, + pub uploaders: Option, + pub architecture: Option, + pub directory: String, + pub files: String, + #[serde(rename = "Checksums-Sha256")] + pub sha256: Option, + #[serde(rename = "Checksums-Sha512")] + pub sha512: Option, + #[serde(flatten)] + pub extra_fields: HashMap, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct SourcePackageEntry { + pub format: String, + pub package: String, + pub binary: Option>, + pub version: String, + pub architecture: Option, + pub section: Option, + pub priority: Option, + pub maintainer: String, + pub uploaders: Option, + pub directory: String, + pub files: HashMap, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct SourcePackageFileReference { + pub file: String, + pub size: usize, + pub checksums: CheckSums, +} + +impl SourcePackageEntry { + pub fn size(&self) -> usize { + self.files.values().map(|f| f.size).sum() + } +} + +#[derive(Debug, Default, PartialEq, Eq)] +/// A parsed representation of a Release file +pub struct SourcesFile { + pub source_packages: Vec, +} + +impl TryFrom for SourcePackageEntry { + type Error = Error; + + fn try_from(value: SourcesFileRaw) -> Result { + let mut parsed = SourcePackageEntry { + package: value.package, + binary: value.binary, + version: value.version, + architecture: value.architecture, + files: HashMap::new(), + format: value.format, + section: value.section, + priority: value.priority, + maintainer: value.maintainer, + uploaders: value.uploaders, + directory: value.directory, + }; + + for file_reference in value.files.lines() { + let (file_name, size, md5) = parse_file_reference(file_reference, 16)?; + let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()}); + entry.checksums.md5 = Some(md5.try_into().map_err(|_|format_err!("unexpected checksum length"))?); + if entry.size != size { + bail!("Size mismatch: {} != {}", entry.size, size); + } + } + + if let Some(sha256) = value.sha256 { + for line in sha256.lines() { + let (file_name, size, sha256) = parse_file_reference(line, 32)?; + let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()}); + entry.checksums.sha256 = Some(sha256.try_into().map_err(|_|format_err!("unexpected checksum length"))?); + if entry.size != size { + bail!("Size mismatch: {} != {}", entry.size, size); + } + } + }; + + if let Some(sha512) = value.sha512 { + for line in sha512.lines() { + let (file_name, size, sha512) = parse_file_reference(line, 64)?; + let entry = parsed.files.entry(file_name.clone()).or_insert_with(|| SourcePackageFileReference { file: file_name, size, checksums: CheckSums::default()}); + entry.checksums.sha512 = Some(sha512.try_into().map_err(|_|format_err!("unexpected checksum length"))?); + if entry.size != size { + bail!("Size mismatch: {} != {}", entry.size, size); + } + } + }; + + for (file_name, reference) in &parsed.files { + if !reference.checksums.is_secure() { + bail!( + "no strong checksum found for source entry '{}'", + file_name + ); + } + } + + Ok(parsed) + } +} + +impl TryFrom for SourcesFile { + type Error = Error; + + fn try_from(value: String) -> Result { + value.as_bytes().try_into() + } +} + +impl TryFrom<&[u8]> for SourcesFile { + type Error = Error; + + fn try_from(value: &[u8]) -> Result { + let deserialized = >::deserialize(Deserializer::new(value))?; + deserialized.try_into() + } +} + +impl TryFrom> for SourcesFile { + type Error = Error; + + fn try_from(value: Vec) -> Result { + let mut source_packages = Vec::with_capacity(value.len()); + for entry in value { + let entry: SourcePackageEntry = entry.try_into()?; + source_packages.push(entry); + } + + Ok(Self { source_packages }) + } +} + +fn parse_file_reference( + line: &str, + csum_len: usize, +) -> Result<(String, usize, Vec), Error> { + let mut split = line.split_ascii_whitespace(); + + let checksum = split + .next() + .ok_or_else(|| format_err!("Missing 'checksum' field."))?; + if checksum.len() > csum_len * 2 { + bail!( + "invalid checksum length: '{}', expected {} bytes", + checksum, + csum_len + ); + } + + let checksum = hex::decode(checksum)?; + + let size = split + .next() + .ok_or_else(|| format_err!("Missing 'size' field."))? + .parse::()?; + + let file = split + .next() + .ok_or_else(|| format_err!("Missing 'file name' field."))? + .to_string(); + + Ok((file, size, checksum)) +} + +#[test] +pub fn test_deb_packages_file() { + let input = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources" + )); + + let deserialized = + >::deserialize(Deserializer::new(input.as_bytes())).unwrap(); + assert_eq!(deserialized.len(), 30953); + + let parsed: SourcesFile = deserialized.try_into().unwrap(); + + assert_eq!(parsed.source_packages.len(), 30953); + + let found = parsed.source_packages.iter().find(|source| source.package == "base-files").expect("test file contains 'base-files' entry"); + assert_eq!(found.package, "base-files"); + assert_eq!(found.format, "3.0 (native)"); + assert_eq!(found.architecture.as_deref(), Some("any")); + assert_eq!(found.directory, "pool/main/b/base-files"); + assert_eq!(found.section.as_deref(), Some("admin")); + assert_eq!(found.version, "11.1+deb11u5"); + + let binary_packages = found.binary.as_ref().expect("base-files source package builds base-files binary package"); + assert_eq!(binary_packages.len(), 1); + assert_eq!(binary_packages[0], "base-files"); + + let references = &found.files; + assert_eq!(references.len(), 2); + + let dsc_file = "base-files_11.1+deb11u5.dsc"; + let dsc = references.get(dsc_file).expect("base-files source package contains 'dsc' reference"); + assert_eq!(dsc.file, dsc_file); + assert_eq!(dsc.size, 1110); + assert_eq!(dsc.checksums.md5.expect("dsc has md5 checksum"), hex::decode("741c34ac0151262a03de8d5a07bc4271").unwrap()[..]); + assert_eq!(dsc.checksums.sha256.expect("dsc has sha256 checksum"), hex::decode("c41a7f00d57759f27e6068240d1ea7ad80a9a752e4fb43850f7e86e967422bd3").unwrap()[..]); + + let tar_file = "base-files_11.1+deb11u5.tar.xz"; + let tar = references.get(tar_file).expect("base-files source package contains 'tar' reference"); + assert_eq!(tar.file, tar_file); + assert_eq!(tar.size, 65612); + assert_eq!(tar.checksums.md5.expect("tar has md5 checksum"), hex::decode("995df33642118b566a4026410e1c6aac").unwrap()[..]); + assert_eq!(tar.checksums.sha256.expect("tar has sha256 checksum"), hex::decode("31c9e5745845a73f3d5c8a7868c379d77aaca42b81194679d7ab40cc28e3a0e9").unwrap()[..]); +} \ No newline at end of file diff --git a/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources b/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources new file mode 100644 index 0000000..2b8e387 --- /dev/null +++ b/tests/deb822/sources/deb.debian.org_debian_dists_bullseye_main_source_Sources @@ -0,0 +1,1 @@ +DOWNLOAD-ME-FROM: http://snapshot.debian.org/archive/debian/20221017T212657Z/dists/bullseye/main/source/Sources.xz -- 2.30.2