From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5A45573547 for ; Fri, 18 Jun 2021 08:51:01 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 57A6A235C3 for ; Fri, 18 Jun 2021 08:51:01 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 91D30235AE for ; Fri, 18 Jun 2021 08:50:57 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7098343012 for ; Fri, 18 Jun 2021 08:50:57 +0200 (CEST) To: pbs-devel@lists.proxmox.com, =?UTF-8?Q?Fabian_Gr=c3=bcnbichler?= , PVE development discussion References: <20210611114418.28772-1-f.ebner@proxmox.com> <20210611114418.28772-7-f.ebner@proxmox.com> <1623937266.48p9vpkidm.astroid@nora.none> From: Fabian Ebner Message-ID: Date: Fri, 18 Jun 2021 08:50:55 +0200 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.11.0 MIME-Version: 1.0 In-Reply-To: <1623937266.48p9vpkidm.astroid@nora.none> Content-Type: text/plain; charset=utf-8; format=flowed Content-Language: en-US Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.491 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_ASCII_DIVIDERS 0.8 Spam that uses ascii formatting tricks KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment NICE_REPLY_A -0.254 Looks like a legit reply (A) SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com, mod.rs, check.rs, repositories.rs] Subject: Re: [pve-devel] [pbs-devel] [PATCH v6 proxmox-apt 06/11] add release_upgrade function and constants for the current and upgrade suite X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 18 Jun 2021 06:51:01 -0000 Am 17.06.21 um 16:16 schrieb Fabian Grünbichler: > On June 11, 2021 1:43 pm, Fabian Ebner wrote: >> useful for major upgrades. The stable branch can enable the upgrade, and bump >> the minor version, while the master branch will adapt to the new release and >> bump the major version. Each product can depend on the the new major version >> after branching off the stable branch, and once the release is out, its stable >> branch can depend on the new minor version. >> >> Signed-off-by: Fabian Ebner >> --- >> >> Changes from v5: >> * Make function less general (only care about the current release upgrade) >> and handle special case for security repository. >> * Make list of suite available as constants. >> * Get the current release from /etc/os-release and abort if it is not the >> same as STABLE_SUITE. >> * Add a constant UPGRADE_SUITE which can be set for the library's last >> release in the stable-X branch to enable the release_upgrade() function. >> >> .gitignore | 1 + >> src/repositories/check.rs | 57 +++++++++++++----------- >> src/repositories/mod.rs | 92 +++++++++++++++++++++++++++++++++++++++ >> tests/repositories.rs | 79 ++++++++++++++++++++++++++++++++- >> 4 files changed, 202 insertions(+), 27 deletions(-) >> >> diff --git a/.gitignore b/.gitignore >> index db6f13e..de68da9 100644 >> --- a/.gitignore >> +++ b/.gitignore >> @@ -1,6 +1,7 @@ >> Cargo.lock >> target/ >> tests/sources.list.d.actual >> +tests/sources.list.d.upgraded.actual >> tests/sources.list.d.digest >> proxmox-apt-*/ >> *proxmox-apt*.buildinfo >> diff --git a/src/repositories/check.rs b/src/repositories/check.rs >> index 585c28d..e0ec93e 100644 >> --- a/src/repositories/check.rs >> +++ b/src/repositories/check.rs >> @@ -5,8 +5,34 @@ use crate::types::{ >> APTRepositoryPackageType, >> }; >> >> +/// The (code)names of old Debian releases. >> +pub const OLD_SUITES: [&str; 7] = [ >> + "lenny", >> + "squeeze", >> + "wheezy", >> + "jessie", >> + "stretch", >> + "oldoldstable", >> + "oldstable", >> +]; >> + >> +/// The codename of the current stable Debian release. >> +pub const STABLE_SUITE: &str = "buster"; >> +/// The codename of the next stable Debian release. >> +pub const NEXT_STABLE_SUITE: &str = "bullseye"; >> + >> +/// The (code)names of new/testing Debian releases. >> +pub const NEW_SUITES: [&str; 6] = [ >> + "bookworm", >> + "trixie", >> + "testing", >> + "unstable", >> + "sid", >> + "experimental", >> +]; >> + >> /// Splits the suite into its base part and variant. >> -fn suite_variant(suite: &str) -> (&str, &str) { >> +pub fn suite_variant(suite: &str) -> (&str, &str) { >> let variants = ["-backports-sloppy", "-backports", "-updates", "/updates"]; >> >> for variant in variants.iter() { >> @@ -19,7 +45,7 @@ fn suite_variant(suite: &str) -> (&str, &str) { >> } >> >> /// Get the host part from a given URI. >> -fn host_from_uri(uri: &str) -> Option<&str> { >> +pub fn host_from_uri(uri: &str) -> Option<&str> { >> if let Some(begin) = uri.find("://") { >> let mut host = uri.split_at(begin + 3).1; >> >> @@ -145,34 +171,13 @@ impl APTRepository { >> /// Checks if old or unstable suites are configured and also that the >> /// `stable` keyword is not used. >> fn check_suites(&self, add_info: &mut dyn FnMut(String, String)) { >> - let old_suites = [ >> - "lenny", >> - "squeeze", >> - "wheezy", >> - "jessie", >> - "stretch", >> - "oldoldstable", >> - "oldstable", >> - ]; >> - >> - let next_suite = "bullseye"; >> - >> - let new_suites = [ >> - "bookworm", >> - "trixie", >> - "testing", >> - "unstable", >> - "sid", >> - "experimental", >> - ]; >> - >> if self >> .types >> .iter() >> .any(|package_type| *package_type == APTRepositoryPackageType::Deb) >> { >> for suite in self.suites.iter() { >> - if old_suites >> + if OLD_SUITES >> .iter() >> .any(|base_suite| suite_variant(suite).0 == *base_suite) >> { >> @@ -182,14 +187,14 @@ impl APTRepository { >> ); >> } >> >> - if suite_variant(suite).0 == next_suite { >> + if suite_variant(suite).0 == NEXT_STABLE_SUITE { >> add_info( >> "ignore-pre-upgrade-warning".to_string(), >> format!("suite '{}' should not be used in production!", suite), >> ); >> } >> >> - if new_suites >> + if NEW_SUITES >> .iter() >> .any(|base_suite| suite_variant(suite).0 == *base_suite) >> { >> diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs >> index 2c01011..eceede3 100644 >> --- a/src/repositories/mod.rs >> +++ b/src/repositories/mod.rs >> @@ -1,4 +1,5 @@ >> use std::collections::BTreeMap; >> +use std::io::{BufRead, BufReader}; >> use std::path::PathBuf; >> >> use anyhow::{bail, format_err, Error}; >> @@ -21,6 +22,11 @@ mod writer; >> const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list"; >> const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/"; >> >> +/// The codename of the current stable Debian release. >> +pub const STABLE_SUITE: &str = check::STABLE_SUITE; >> +/// The codename of the next stable Debian release or `None` if an upgrade is not yet possible. >> +pub const UPGRADE_SUITE: Option<&str> = None; >> + >> impl APTRepository { >> /// Crates an empty repository. >> fn new(file_type: APTRepositoryFileType) -> Self { >> @@ -265,6 +271,92 @@ pub fn repositories() -> Result<(Vec, Vec> Ok((files, errors)) >> } >> >> +/// Read the `VERSION_CODENAME` from `/etc/os-release`. >> +fn get_release_codename() -> Result { >> + let raw = std::fs::read("/etc/os-release") >> + .map_err(|err| format_err!("unable to read '/etc/os-release' - {}", err))?; >> + >> + let reader = BufReader::new(&*raw); >> + >> + for line in reader.lines() { >> + let line = line.map_err(|err| format_err!("unable to read '/etc/os-release' - {}", err))?; >> + >> + if let Some(codename) = line.strip_prefix("VERSION_CODENAME=") { >> + let codename = codename.trim_matches(&['"', '\''][..]); >> + return Ok(codename.to_string()); >> + } >> + } >> + >> + bail!("unable to parse codename from '/etc/os-release'"); >> +} >> + >> +/// For enabled repositories, replaces each occurence of the `STABLE_SUITE` with the >> +/// `UPGRADE_SUITE` suite, including variants (e.g. `-updates`). >> +/// >> +/// Returns an error if the `UPGRADE_SUITE` is currently `None`, i.e. upgrade not yet possible. >> +/// >> +/// Returns an error if the `VERSION_CODENAME` from `/etc/os-release` is not `STABLE_SUITE`. >> +/// >> +/// Also handles the special case `buster/updates` -> `bullseye-security` when the URI is >> +/// security.debian.org, but fails if there's additional URIs. >> +pub fn release_upgrade(files: &mut [APTRepositoryFile]) -> Result<(), Error> { >> + let upgrade_suite = match UPGRADE_SUITE { >> + Some(suite) => suite, >> + None => bail!("release upgrade is not yet possible"), >> + }; >> + >> + let current = get_release_codename()?; >> + >> + if current == upgrade_suite { >> + bail!("already installed '{}'", current); >> + } >> + >> + if current != STABLE_SUITE { >> + bail!( >> + "unexpected release '{}' - cannot prepare repositories for upgrade", >> + current >> + ); >> + } >> + >> + for file in files.iter_mut() { >> + for repo in file.repositories.iter_mut() { >> + if !repo.enabled { >> + continue; >> + } >> + >> + for i in 0..repo.suites.len() { >> + let suite = &repo.suites[i]; >> + >> + // FIXME special case for security repository can be removed for Debian Bookworm >> + >> + let is_security_uri = |uri| { >> + check::host_from_uri(uri).map_or(false, |host| host == "security.debian.org") > > this should probably also check for the not uncommon case of > > https://deb.debian.org/debian-security (or http) > Thanks for the info. The Debian wiki[0] only mentions security.debian.org, so I wasn't aware of that. And somebody might have a FQDN here too... [0]: https://wiki.debian.org/NewInBullseye >> + }; >> + >> + let has_security_uri = repo.uris.iter().any(|uri| is_security_uri(uri)); >> + let has_only_security_uri = repo.uris.iter().all(|uri| is_security_uri(uri)); >> + >> + if suite == "buster/updates" && has_security_uri { >> + if !has_only_security_uri { >> + bail!("cannot replace 'buster/updates' suite - multiple URIs"); >> + } >> + >> + repo.suites[i] = "bullseye-security".to_string(); >> + >> + continue; >> + } >> + >> + let (base, variant) = check::suite_variant(suite); >> + if base == STABLE_SUITE { >> + repo.suites[i] = format!("{}{}", upgrade_suite, variant); >> + } >> + } >> + } >> + } >> + >> + Ok(()) >> +} >> + >> /// Write the repositories for each file. >> /// >> /// Returns an error for each file that could not be written successfully. >> diff --git a/tests/repositories.rs b/tests/repositories.rs >> index 3919077..ee7f1a8 100644 >> --- a/tests/repositories.rs >> +++ b/tests/repositories.rs >> @@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error}; >> >> use proxmox_apt::repositories::{ >> check_repositories, common_digest, enterprise_repository_enabled, >> - no_subscription_repository_enabled, write_repositories, >> + no_subscription_repository_enabled, release_upgrade, write_repositories, >> }; >> use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo}; >> >> @@ -292,3 +292,80 @@ fn test_common_digest() -> Result<(), Error> { >> >> Ok(()) >> } >> + >> +#[test] >> +fn test_release_upgrade() -> Result<(), Error> { >> + let test_dir = std::env::current_dir()?.join("tests"); >> + let read_dir = test_dir.join("sources.list.d"); >> + let write_dir = test_dir.join("sources.list.d.upgraded.actual"); >> + let expected_dir = test_dir.join("sources.list.d.upgraded.expected"); >> + >> + if write_dir.is_dir() { >> + std::fs::remove_dir_all(&write_dir) >> + .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?; >> + } >> + >> + std::fs::create_dir_all(&write_dir) >> + .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?; >> + >> + let mut files = vec![]; >> + let mut errors = vec![]; >> + >> + for entry in std::fs::read_dir(read_dir)? { >> + let path = entry?.path(); >> + >> + match APTRepositoryFile::new(&path)? { >> + Some(mut file) => match file.parse() { >> + Ok(()) => files.push(file), >> + Err(err) => errors.push(err), >> + }, >> + None => bail!("unexpected None for '{:?}'", path), >> + } >> + } >> + >> + assert!(errors.is_empty()); >> + >> + for file in files.iter_mut() { >> + let path = PathBuf::from(&file.path); >> + let new_path = write_dir.join(path.file_name().unwrap()); >> + file.path = new_path.into_os_string().into_string().unwrap(); >> + file.digest = None; >> + } >> + >> + let res = release_upgrade(&mut files); >> + >> + // FIXME adapt test after branching off the stable-X branch! >> + assert!(res.is_err()); >> + if res.is_err() { >> + return Ok(()); >> + } >> + >> + write_repositories(&files).map_err(|err| format_err!("{:?}", err))?; >> + >> + let mut expected_count = 0; >> + >> + for entry in std::fs::read_dir(expected_dir)? { >> + expected_count += 1; >> + >> + let expected_path = entry?.path(); >> + let actual_path = write_dir.join(expected_path.file_name().unwrap()); >> + >> + let expected_contents = std::fs::read(&expected_path) >> + .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?; >> + >> + let actual_contents = std::fs::read(&actual_path) >> + .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?; >> + >> + assert_eq!( >> + expected_contents, actual_contents, >> + "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals", >> + expected_path, actual_path >> + ); >> + } >> + >> + let actual_count = std::fs::read_dir(write_dir)?.count(); >> + >> + assert_eq!(expected_count, actual_count); >> + >> + Ok(()) >> +} >> -- >> 2.20.1 >> >> >> >> _______________________________________________ >> pbs-devel mailing list >> pbs-devel@lists.proxmox.com >> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >> >> >> > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel > >