From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 6066173EEE for ; Mon, 31 May 2021 15:07:00 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 56A9B1EFBD for ; Mon, 31 May 2021 15:07:00 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id E08A81EFAE for ; Mon, 31 May 2021 15:06:58 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id ABB6842D8E for ; Mon, 31 May 2021 15:06:58 +0200 (CEST) To: pbs-devel@lists.proxmox.com, PVE development discussion References: <20210528143002.16190-1-f.ebner@proxmox.com> <20210528143002.16190-7-f.ebner@proxmox.com> From: Fabian Ebner Message-ID: <6f4d0757-275d-db92-51d4-cdffbeca2554@proxmox.com> Date: Mon, 31 May 2021 15:06:43 +0200 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.10.0 MIME-Version: 1.0 In-Reply-To: <20210528143002.16190-7-f.ebner@proxmox.com> Content-Type: text/plain; charset=utf-8; format=flowed Content-Language: en-US Content-Transfer-Encoding: 7bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.312 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment NICE_REPLY_A -0.618 Looks like a legit reply (A) SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pbs-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 31 May 2021 13:07:00 -0000 For the security repository, "buster/updates" needs to be replaced with "bullseye-security", so this is not enough. I'll add a special case for that in the next version. Am 28.05.21 um 16:29 schrieb Fabian Ebner: > useful for major upgrades. > > Signed-off-by: Fabian Ebner > --- > > New in v5. > > .gitignore | 1 + > src/repositories/check.rs | 2 +- > src/repositories/mod.rs | 23 ++++++ > tests/repositories.rs | 73 ++++++++++++++++++- > .../absolute_suite.list | 5 ++ > .../absolute_suite.sources | 5 ++ > .../bad.sources | 30 ++++++++ > .../case.sources | 16 ++++ > .../multiline.sources | 10 +++ > .../options_comment.list | 6 ++ > .../pbs-enterprise.list | 2 + > .../sources.list.d.upgraded.expected/pve.list | 15 ++++ > .../standard.list | 7 ++ > .../standard.sources | 11 +++ > 14 files changed, 204 insertions(+), 2 deletions(-) > create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.list > create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.sources > create mode 100644 tests/sources.list.d.upgraded.expected/bad.sources > create mode 100644 tests/sources.list.d.upgraded.expected/case.sources > create mode 100644 tests/sources.list.d.upgraded.expected/multiline.sources > create mode 100644 tests/sources.list.d.upgraded.expected/options_comment.list > create mode 100644 tests/sources.list.d.upgraded.expected/pbs-enterprise.list > create mode 100644 tests/sources.list.d.upgraded.expected/pve.list > create mode 100644 tests/sources.list.d.upgraded.expected/standard.list > create mode 100644 tests/sources.list.d.upgraded.expected/standard.sources > > diff --git a/.gitignore b/.gitignore > index db6f13e..de68da9 100644 > --- a/.gitignore > +++ b/.gitignore > @@ -1,6 +1,7 @@ > Cargo.lock > target/ > tests/sources.list.d.actual > +tests/sources.list.d.upgraded.actual > tests/sources.list.d.digest > proxmox-apt-*/ > *proxmox-apt*.buildinfo > diff --git a/src/repositories/check.rs b/src/repositories/check.rs > index d03d5f7..9509826 100644 > --- a/src/repositories/check.rs > +++ b/src/repositories/check.rs > @@ -6,7 +6,7 @@ use crate::types::{ > }; > > /// Splits the suite into its base part and variant. > -fn suite_variant(suite: &str) -> (&str, &str) { > +pub fn suite_variant(suite: &str) -> (&str, &str) { > let variants = ["-backports-sloppy", "-backports", "-updates", "/updates"]; > > for variant in variants.iter() { > diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs > index 2c01011..0e6c046 100644 > --- a/src/repositories/mod.rs > +++ b/src/repositories/mod.rs > @@ -21,6 +21,9 @@ mod writer; > const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list"; > const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/"; > > +pub const STABLE_SUITE: &str = "buster"; > +pub const NEXT_STABLE_SUITE: &str = "bullseye"; > + > impl APTRepository { > /// Crates an empty repository. > fn new(file_type: APTRepositoryFileType) -> Self { > @@ -265,6 +268,26 @@ pub fn repositories() -> Result<(Vec, Vec Ok((files, errors)) > } > > +/// For enabled repositories, replaces each occurence of the `current` suite with the `new` suite, > +/// including variants (e.g. `-updates`). > +pub fn replace_suite(files: &mut [APTRepositoryFile], current: &str, new: &str) { > + for file in files.iter_mut() { > + for repo in file.repositories.iter_mut() { > + if !repo.enabled { > + continue; > + } > + > + for i in 0..repo.suites.len() { > + let suite = &repo.suites[i]; > + let (base, variant) = check::suite_variant(suite); > + if base == current { > + repo.suites[i] = format!("{}{}", new, variant); > + } > + } > + } > + } > +} > + > /// Write the repositories for each file. > /// > /// Returns an error for each file that could not be written successfully. > diff --git a/tests/repositories.rs b/tests/repositories.rs > index 3919077..e2986af 100644 > --- a/tests/repositories.rs > +++ b/tests/repositories.rs > @@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error}; > > use proxmox_apt::repositories::{ > check_repositories, common_digest, enterprise_repository_enabled, > - no_subscription_repository_enabled, write_repositories, > + no_subscription_repository_enabled, replace_suite, write_repositories, > }; > use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo}; > > @@ -292,3 +292,74 @@ fn test_common_digest() -> Result<(), Error> { > > Ok(()) > } > + > +#[test] > +fn test_replace_suite() -> Result<(), Error> { > + let test_dir = std::env::current_dir()?.join("tests"); > + let read_dir = test_dir.join("sources.list.d"); > + let write_dir = test_dir.join("sources.list.d.upgraded.actual"); > + let expected_dir = test_dir.join("sources.list.d.upgraded.expected"); > + > + if write_dir.is_dir() { > + std::fs::remove_dir_all(&write_dir) > + .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?; > + } > + > + std::fs::create_dir_all(&write_dir) > + .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?; > + > + let mut files = vec![]; > + let mut errors = vec![]; > + > + for entry in std::fs::read_dir(read_dir)? { > + let path = entry?.path(); > + > + match APTRepositoryFile::new(&path)? { > + Some(mut file) => match file.parse() { > + Ok(()) => files.push(file), > + Err(err) => errors.push(err), > + }, > + None => bail!("unexpected None for '{:?}'", path), > + } > + } > + > + assert!(errors.is_empty()); > + > + for file in files.iter_mut() { > + let path = PathBuf::from(&file.path); > + let new_path = write_dir.join(path.file_name().unwrap()); > + file.path = new_path.into_os_string().into_string().unwrap(); > + file.digest = None; > + } > + > + replace_suite(&mut files, "buster", "bullseye"); > + > + write_repositories(&files).map_err(|err| format_err!("{:?}", err))?; > + > + let mut expected_count = 0; > + > + for entry in std::fs::read_dir(expected_dir)? { > + expected_count += 1; > + > + let expected_path = entry?.path(); > + let actual_path = write_dir.join(expected_path.file_name().unwrap()); > + > + let expected_contents = std::fs::read(&expected_path) > + .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?; > + > + let actual_contents = std::fs::read(&actual_path) > + .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?; > + > + assert_eq!( > + expected_contents, actual_contents, > + "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals", > + expected_path, actual_path > + ); > + } > + > + let actual_count = std::fs::read_dir(write_dir)?.count(); > + > + assert_eq!(expected_count, actual_count); > + > + Ok(()) > +} > diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.list b/tests/sources.list.d.upgraded.expected/absolute_suite.list > new file mode 100644 > index 0000000..525389c > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/absolute_suite.list > @@ -0,0 +1,5 @@ > +# From Debian Administrator's Handbook > +deb http://packages.falcot.com/ updates/ > + > +deb http://packages.falcot.com/ internal/ > + > diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.sources b/tests/sources.list.d.upgraded.expected/absolute_suite.sources > new file mode 100644 > index 0000000..51e4d56 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/absolute_suite.sources > @@ -0,0 +1,5 @@ > +# From Debian Administrator's Handbook > +Types: deb > +URIs: http://packages.falcot.com/ > +Suites: updates/ internal/ > + > diff --git a/tests/sources.list.d.upgraded.expected/bad.sources b/tests/sources.list.d.upgraded.expected/bad.sources > new file mode 100644 > index 0000000..36ff7a0 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/bad.sources > @@ -0,0 +1,30 @@ > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: sid > +Components: main contrib > + > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: lenny-backports > +Components: contrib > + > +Types: deb > +URIs: http://security.debian.org > +Suites: stretch/updates > +Components: main contrib > + > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: stable > +Components: main > + > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: bullseye > +Components: main > + > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: testing > +Components: main > + > diff --git a/tests/sources.list.d.upgraded.expected/case.sources b/tests/sources.list.d.upgraded.expected/case.sources > new file mode 100644 > index 0000000..a266594 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/case.sources > @@ -0,0 +1,16 @@ > +# comment in here > +Types: deb deb-src > +URIs: http://ftp.at.debian.org/debian > +Suites: buster-updates > +Components: main contrib > +languages: it de fr > +Enabled: false > +languages-Add: ja > +languages-Remove: de > + > +# comment in here > +Types: deb deb-src > +URIs: http://ftp.at.debian.org/debian > +Suites: bullseye > +Components: main contrib > + > diff --git a/tests/sources.list.d.upgraded.expected/multiline.sources b/tests/sources.list.d.upgraded.expected/multiline.sources > new file mode 100644 > index 0000000..d96acea > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/multiline.sources > @@ -0,0 +1,10 @@ > +# comment in here > +Types: deb deb-src > +URIs: http://ftp.at.debian.org/debian > +Suites: buster buster-updates > +Components: main contrib > +Languages: it de fr > +Enabled: false > +Languages-Add: ja > +Languages-Remove: de > + > diff --git a/tests/sources.list.d.upgraded.expected/options_comment.list b/tests/sources.list.d.upgraded.expected/options_comment.list > new file mode 100644 > index 0000000..caef5e0 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/options_comment.list > @@ -0,0 +1,6 @@ > +# comment > +deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian bullseye main contrib > + > +# non-free :( > +deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian bullseye non-free > + > diff --git a/tests/sources.list.d.upgraded.expected/pbs-enterprise.list b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list > new file mode 100644 > index 0000000..cb6e779 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list > @@ -0,0 +1,2 @@ > +deb https://enterprise.proxmox.com/debian/pbs bullseye pbs-enterprise > + > diff --git a/tests/sources.list.d.upgraded.expected/pve.list b/tests/sources.list.d.upgraded.expected/pve.list > new file mode 100644 > index 0000000..1b5b248 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/pve.list > @@ -0,0 +1,15 @@ > +deb http://ftp.debian.org/debian bullseye main contrib > + > +deb http://ftp.debian.org/debian bullseye-updates main contrib > + > +# PVE pve-no-subscription repository provided by proxmox.com, > +# NOT recommended for production use > +deb http://download.proxmox.com/debian/pve bullseye pve-no-subscription > + > +# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise > + > +deb-src https://enterprise.proxmox.com/debian/pve bullseye pve-enterprise > + > +# security updates > +deb http://security.debian.org/debian-security bullseye/updates main contrib > + > diff --git a/tests/sources.list.d.upgraded.expected/standard.list b/tests/sources.list.d.upgraded.expected/standard.list > new file mode 100644 > index 0000000..25654f1 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/standard.list > @@ -0,0 +1,7 @@ > +deb http://ftp.at.debian.org/debian bullseye main contrib > + > +deb http://ftp.at.debian.org/debian bullseye-updates main contrib > + > +# security updates > +deb http://security.debian.org bullseye/updates main contrib > + > diff --git a/tests/sources.list.d.upgraded.expected/standard.sources b/tests/sources.list.d.upgraded.expected/standard.sources > new file mode 100644 > index 0000000..f8e8620 > --- /dev/null > +++ b/tests/sources.list.d.upgraded.expected/standard.sources > @@ -0,0 +1,11 @@ > +Types: deb > +URIs: http://ftp.at.debian.org/debian > +Suites: bullseye bullseye-updates > +Components: main contrib > + > +# security updates > +Types: deb > +URIs: http://security.debian.org > +Suites: bullseye/updates > +Components: main contrib > + >