From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0CC337327D; Thu, 17 Jun 2021 16:16:48 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id EFF0F1D31C; Thu, 17 Jun 2021 16:16:47 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 504241D30E; Thu, 17 Jun 2021 16:16:44 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 14A7A441FE; Thu, 17 Jun 2021 16:16:44 +0200 (CEST) Date: Thu, 17 Jun 2021 16:16:34 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion , pve-devel@lists.proxmox.com References: <20210611114418.28772-1-f.ebner@proxmox.com> <20210611114418.28772-7-f.ebner@proxmox.com> In-Reply-To: <20210611114418.28772-7-f.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.15.0 (https://github.com/astroidmail/astroid) Message-Id: <1623937266.48p9vpkidm.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.802 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pve-devel] [pbs-devel] [PATCH v6 proxmox-apt 06/11] add release_upgrade function and constants for the current and upgrade suite X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 17 Jun 2021 14:16:48 -0000 On June 11, 2021 1:43 pm, Fabian Ebner wrote: > useful for major upgrades. The stable branch can enable the upgrade, and = bump > the minor version, while the master branch will adapt to the new release = and > bump the major version. Each product can depend on the the new major vers= ion > after branching off the stable branch, and once the release is out, its s= table > branch can depend on the new minor version. >=20 > Signed-off-by: Fabian Ebner > --- >=20 > Changes from v5: > * Make function less general (only care about the current release upg= rade) > and handle special case for security repository. > * Make list of suite available as constants. > * Get the current release from /etc/os-release and abort if it is not= the > same as STABLE_SUITE. > * Add a constant UPGRADE_SUITE which can be set for the library's las= t > release in the stable-X branch to enable the release_upgrade() func= tion. >=20 > .gitignore | 1 + > src/repositories/check.rs | 57 +++++++++++++----------- > src/repositories/mod.rs | 92 +++++++++++++++++++++++++++++++++++++++ > tests/repositories.rs | 79 ++++++++++++++++++++++++++++++++- > 4 files changed, 202 insertions(+), 27 deletions(-) >=20 > diff --git a/.gitignore b/.gitignore > index db6f13e..de68da9 100644 > --- a/.gitignore > +++ b/.gitignore > @@ -1,6 +1,7 @@ > Cargo.lock > target/ > tests/sources.list.d.actual > +tests/sources.list.d.upgraded.actual > tests/sources.list.d.digest > proxmox-apt-*/ > *proxmox-apt*.buildinfo > diff --git a/src/repositories/check.rs b/src/repositories/check.rs > index 585c28d..e0ec93e 100644 > --- a/src/repositories/check.rs > +++ b/src/repositories/check.rs > @@ -5,8 +5,34 @@ use crate::types::{ > APTRepositoryPackageType, > }; > =20 > +/// The (code)names of old Debian releases. > +pub const OLD_SUITES: [&str; 7] =3D [ > + "lenny", > + "squeeze", > + "wheezy", > + "jessie", > + "stretch", > + "oldoldstable", > + "oldstable", > +]; > + > +/// The codename of the current stable Debian release. > +pub const STABLE_SUITE: &str =3D "buster"; > +/// The codename of the next stable Debian release. > +pub const NEXT_STABLE_SUITE: &str =3D "bullseye"; > + > +/// The (code)names of new/testing Debian releases. > +pub const NEW_SUITES: [&str; 6] =3D [ > + "bookworm", > + "trixie", > + "testing", > + "unstable", > + "sid", > + "experimental", > +]; > + > /// Splits the suite into its base part and variant. > -fn suite_variant(suite: &str) -> (&str, &str) { > +pub fn suite_variant(suite: &str) -> (&str, &str) { > let variants =3D ["-backports-sloppy", "-backports", "-updates", "/u= pdates"]; > =20 > for variant in variants.iter() { > @@ -19,7 +45,7 @@ fn suite_variant(suite: &str) -> (&str, &str) { > } > =20 > /// Get the host part from a given URI. > -fn host_from_uri(uri: &str) -> Option<&str> { > +pub fn host_from_uri(uri: &str) -> Option<&str> { > if let Some(begin) =3D uri.find("://") { > let mut host =3D uri.split_at(begin + 3).1; > =20 > @@ -145,34 +171,13 @@ impl APTRepository { > /// Checks if old or unstable suites are configured and also that th= e > /// `stable` keyword is not used. > fn check_suites(&self, add_info: &mut dyn FnMut(String, String)) { > - let old_suites =3D [ > - "lenny", > - "squeeze", > - "wheezy", > - "jessie", > - "stretch", > - "oldoldstable", > - "oldstable", > - ]; > - > - let next_suite =3D "bullseye"; > - > - let new_suites =3D [ > - "bookworm", > - "trixie", > - "testing", > - "unstable", > - "sid", > - "experimental", > - ]; > - > if self > .types > .iter() > .any(|package_type| *package_type =3D=3D APTRepositoryPackag= eType::Deb) > { > for suite in self.suites.iter() { > - if old_suites > + if OLD_SUITES > .iter() > .any(|base_suite| suite_variant(suite).0 =3D=3D *bas= e_suite) > { > @@ -182,14 +187,14 @@ impl APTRepository { > ); > } > =20 > - if suite_variant(suite).0 =3D=3D next_suite { > + if suite_variant(suite).0 =3D=3D NEXT_STABLE_SUITE { > add_info( > "ignore-pre-upgrade-warning".to_string(), > format!("suite '{}' should not be used in produc= tion!", suite), > ); > } > =20 > - if new_suites > + if NEW_SUITES > .iter() > .any(|base_suite| suite_variant(suite).0 =3D=3D *bas= e_suite) > { > diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs > index 2c01011..eceede3 100644 > --- a/src/repositories/mod.rs > +++ b/src/repositories/mod.rs > @@ -1,4 +1,5 @@ > use std::collections::BTreeMap; > +use std::io::{BufRead, BufReader}; > use std::path::PathBuf; > =20 > use anyhow::{bail, format_err, Error}; > @@ -21,6 +22,11 @@ mod writer; > const APT_SOURCES_LIST_FILENAME: &str =3D "/etc/apt/sources.list"; > const APT_SOURCES_LIST_DIRECTORY: &str =3D "/etc/apt/sources.list.d/"; > =20 > +/// The codename of the current stable Debian release. > +pub const STABLE_SUITE: &str =3D check::STABLE_SUITE; > +/// The codename of the next stable Debian release or `None` if an upgra= de is not yet possible. > +pub const UPGRADE_SUITE: Option<&str> =3D None; > + > impl APTRepository { > /// Crates an empty repository. > fn new(file_type: APTRepositoryFileType) -> Self { > @@ -265,6 +271,92 @@ pub fn repositories() -> Result<(Vec, Vec Ok((files, errors)) > } > =20 > +/// Read the `VERSION_CODENAME` from `/etc/os-release`. > +fn get_release_codename() -> Result { > + let raw =3D std::fs::read("/etc/os-release") > + .map_err(|err| format_err!("unable to read '/etc/os-release' - {= }", err))?; > + > + let reader =3D BufReader::new(&*raw); > + > + for line in reader.lines() { > + let line =3D line.map_err(|err| format_err!("unable to read '/et= c/os-release' - {}", err))?; > + > + if let Some(codename) =3D line.strip_prefix("VERSION_CODENAME=3D= ") { > + let codename =3D codename.trim_matches(&['"', '\''][..]); > + return Ok(codename.to_string()); > + } > + } > + > + bail!("unable to parse codename from '/etc/os-release'"); > +} > + > +/// For enabled repositories, replaces each occurence of the `STABLE_SUI= TE` with the > +/// `UPGRADE_SUITE` suite, including variants (e.g. `-updates`). > +/// > +/// Returns an error if the `UPGRADE_SUITE` is currently `None`, i.e. up= grade not yet possible. > +/// > +/// Returns an error if the `VERSION_CODENAME` from `/etc/os-release` is= not `STABLE_SUITE`. > +/// > +/// Also handles the special case `buster/updates` -> `bullseye-security= ` when the URI is > +/// security.debian.org, but fails if there's additional URIs. > +pub fn release_upgrade(files: &mut [APTRepositoryFile]) -> Result<(), Er= ror> { > + let upgrade_suite =3D match UPGRADE_SUITE { > + Some(suite) =3D> suite, > + None =3D> bail!("release upgrade is not yet possible"), > + }; > + > + let current =3D get_release_codename()?; > + > + if current =3D=3D upgrade_suite { > + bail!("already installed '{}'", current); > + } > + > + if current !=3D STABLE_SUITE { > + bail!( > + "unexpected release '{}' - cannot prepare repositories for u= pgrade", > + current > + ); > + } > + > + for file in files.iter_mut() { > + for repo in file.repositories.iter_mut() { > + if !repo.enabled { > + continue; > + } > + > + for i in 0..repo.suites.len() { > + let suite =3D &repo.suites[i]; > + > + // FIXME special case for security repository can be rem= oved for Debian Bookworm > + > + let is_security_uri =3D |uri| { > + check::host_from_uri(uri).map_or(false, |host| host = =3D=3D "security.debian.org") this should probably also check for the not uncommon case of https://deb.debian.org/debian-security (or http) > + }; > + > + let has_security_uri =3D repo.uris.iter().any(|uri| is_s= ecurity_uri(uri)); > + let has_only_security_uri =3D repo.uris.iter().all(|uri|= is_security_uri(uri)); > + > + if suite =3D=3D "buster/updates" && has_security_uri { > + if !has_only_security_uri { > + bail!("cannot replace 'buster/updates' suite - m= ultiple URIs"); > + } > + > + repo.suites[i] =3D "bullseye-security".to_string(); > + > + continue; > + } > + > + let (base, variant) =3D check::suite_variant(suite); > + if base =3D=3D STABLE_SUITE { > + repo.suites[i] =3D format!("{}{}", upgrade_suite, va= riant); > + } > + } > + } > + } > + > + Ok(()) > +} > + > /// Write the repositories for each file. > /// > /// Returns an error for each file that could not be written successfull= y. > diff --git a/tests/repositories.rs b/tests/repositories.rs > index 3919077..ee7f1a8 100644 > --- a/tests/repositories.rs > +++ b/tests/repositories.rs > @@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error}; > =20 > use proxmox_apt::repositories::{ > check_repositories, common_digest, enterprise_repository_enabled, > - no_subscription_repository_enabled, write_repositories, > + no_subscription_repository_enabled, release_upgrade, write_repositor= ies, > }; > use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo}; > =20 > @@ -292,3 +292,80 @@ fn test_common_digest() -> Result<(), Error> { > =20 > Ok(()) > } > + > +#[test] > +fn test_release_upgrade() -> Result<(), Error> { > + let test_dir =3D std::env::current_dir()?.join("tests"); > + let read_dir =3D test_dir.join("sources.list.d"); > + let write_dir =3D test_dir.join("sources.list.d.upgraded.actual"); > + let expected_dir =3D test_dir.join("sources.list.d.upgraded.expected= "); > + > + if write_dir.is_dir() { > + std::fs::remove_dir_all(&write_dir) > + .map_err(|err| format_err!("unable to remove dir {:?} - {}",= write_dir, err))?; > + } > + > + std::fs::create_dir_all(&write_dir) > + .map_err(|err| format_err!("unable to create dir {:?} - {}", wri= te_dir, err))?; > + > + let mut files =3D vec![]; > + let mut errors =3D vec![]; > + > + for entry in std::fs::read_dir(read_dir)? { > + let path =3D entry?.path(); > + > + match APTRepositoryFile::new(&path)? { > + Some(mut file) =3D> match file.parse() { > + Ok(()) =3D> files.push(file), > + Err(err) =3D> errors.push(err), > + }, > + None =3D> bail!("unexpected None for '{:?}'", path), > + } > + } > + > + assert!(errors.is_empty()); > + > + for file in files.iter_mut() { > + let path =3D PathBuf::from(&file.path); > + let new_path =3D write_dir.join(path.file_name().unwrap()); > + file.path =3D new_path.into_os_string().into_string().unwrap(); > + file.digest =3D None; > + } > + > + let res =3D release_upgrade(&mut files); > + > + // FIXME adapt test after branching off the stable-X branch! > + assert!(res.is_err()); > + if res.is_err() { > + return Ok(()); > + } > + > + write_repositories(&files).map_err(|err| format_err!("{:?}", err))?; > + > + let mut expected_count =3D 0; > + > + for entry in std::fs::read_dir(expected_dir)? { > + expected_count +=3D 1; > + > + let expected_path =3D entry?.path(); > + let actual_path =3D write_dir.join(expected_path.file_name().unw= rap()); > + > + let expected_contents =3D std::fs::read(&expected_path) > + .map_err(|err| format_err!("unable to read {:?} - {}", expec= ted_path, err))?; > + > + let actual_contents =3D std::fs::read(&actual_path) > + .map_err(|err| format_err!("unable to read {:?} - {}", actua= l_path, err))?; > + > + assert_eq!( > + expected_contents, actual_contents, > + "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decim= als", > + expected_path, actual_path > + ); > + } > + > + let actual_count =3D std::fs::read_dir(write_dir)?.count(); > + > + assert_eq!(expected_count, actual_count); > + > + Ok(()) > +} > --=20 > 2.20.1 >=20 >=20 >=20 > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >=20 >=20 >=20