* [pve-devel] [PATCH v5 proxmox-apt 01/23] initial commit
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 02/23] add files for Debian packaging Fabian Ebner
` (21 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* clippy fixes: use 'string' instead of '&string[..]'
* update proxmox version in Cargo.toml
* tests: avoid using a variable only once, but use the string directly
.cargo/config | 5 +
.gitignore | 4 +
Cargo.toml | 23 ++
rustfmt.toml | 1 +
src/lib.rs | 3 +
src/repositories/check.rs | 47 ++++
src/repositories/file.rs | 96 +++++++
src/repositories/list_parser.rs | 171 ++++++++++++
src/repositories/mod.rs | 224 ++++++++++++++++
src/repositories/sources_parser.rs | 204 +++++++++++++++
src/repositories/writer.rs | 92 +++++++
src/types.rs | 246 ++++++++++++++++++
tests/repositories.rs | 129 +++++++++
.../absolute_suite.list | 5 +
.../absolute_suite.sources | 5 +
tests/sources.list.d.expected/case.sources | 16 ++
.../sources.list.d.expected/multiline.sources | 10 +
| 6 +
.../pbs-enterprise.list | 2 +
tests/sources.list.d.expected/pve.list | 13 +
tests/sources.list.d.expected/standard.list | 7 +
.../sources.list.d.expected/standard.sources | 11 +
tests/sources.list.d/absolute_suite.list | 3 +
tests/sources.list.d/absolute_suite.sources | 5 +
tests/sources.list.d/case.sources | 17 ++
tests/sources.list.d/multiline.sources | 11 +
| 3 +
tests/sources.list.d/pbs-enterprise.list | 1 +
tests/sources.list.d/pve.list | 10 +
tests/sources.list.d/standard.list | 6 +
tests/sources.list.d/standard.sources | 10 +
31 files changed, 1386 insertions(+)
create mode 100644 .cargo/config
create mode 100644 .gitignore
create mode 100644 Cargo.toml
create mode 100644 rustfmt.toml
create mode 100644 src/lib.rs
create mode 100644 src/repositories/check.rs
create mode 100644 src/repositories/file.rs
create mode 100644 src/repositories/list_parser.rs
create mode 100644 src/repositories/mod.rs
create mode 100644 src/repositories/sources_parser.rs
create mode 100644 src/repositories/writer.rs
create mode 100644 src/types.rs
create mode 100644 tests/repositories.rs
create mode 100644 tests/sources.list.d.expected/absolute_suite.list
create mode 100644 tests/sources.list.d.expected/absolute_suite.sources
create mode 100644 tests/sources.list.d.expected/case.sources
create mode 100644 tests/sources.list.d.expected/multiline.sources
create mode 100644 tests/sources.list.d.expected/options_comment.list
create mode 100644 tests/sources.list.d.expected/pbs-enterprise.list
create mode 100644 tests/sources.list.d.expected/pve.list
create mode 100644 tests/sources.list.d.expected/standard.list
create mode 100644 tests/sources.list.d.expected/standard.sources
create mode 100644 tests/sources.list.d/absolute_suite.list
create mode 100644 tests/sources.list.d/absolute_suite.sources
create mode 100644 tests/sources.list.d/case.sources
create mode 100644 tests/sources.list.d/multiline.sources
create mode 100644 tests/sources.list.d/options_comment.list
create mode 100644 tests/sources.list.d/pbs-enterprise.list
create mode 100644 tests/sources.list.d/pve.list
create mode 100644 tests/sources.list.d/standard.list
create mode 100644 tests/sources.list.d/standard.sources
diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000..3b5b6e4
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..24917d4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+Cargo.lock
+target/
+tests/sources.list.d.actual
+tests/sources.list.d.digest
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..24f734b
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,23 @@
+[package]
+name = "proxmox-apt"
+version = "0.1.0"
+authors = [
+ "Fabian Ebner <f.ebner@proxmox.com>",
+ "Proxmox Support Team <support@proxmox.com>",
+]
+edition = "2018"
+license = "AGPL-3"
+description = "Proxmox library for APT"
+homepage = "https://www.proxmox.com"
+
+exclude = [ "debian" ]
+
+[lib]
+name = "proxmox_apt"
+path = "src/lib.rs"
+
+[dependencies]
+anyhow = "1.0"
+openssl = "0.10"
+proxmox = { version = "0.11.5", features = [ "api-macro" ] }
+serde = { version = "1.0", features = ["derive"] }
diff --git a/rustfmt.toml b/rustfmt.toml
new file mode 100644
index 0000000..32a9786
--- /dev/null
+++ b/rustfmt.toml
@@ -0,0 +1 @@
+edition = "2018"
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 0000000..b065c0f
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,3 @@
+pub mod types;
+
+pub mod repositories;
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
new file mode 100644
index 0000000..87fbbac
--- /dev/null
+++ b/src/repositories/check.rs
@@ -0,0 +1,47 @@
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+ /// Makes sure that all basic properties of a repository are present and
+ /// not obviously invalid.
+ pub fn basic_check(&self) -> Result<(), Error> {
+ if self.types.is_empty() {
+ bail!("missing package type(s)");
+ }
+ if self.uris.is_empty() {
+ bail!("missing URI(s)");
+ }
+ if self.suites.is_empty() {
+ bail!("missing suite(s)");
+ }
+
+ for uri in self.uris.iter() {
+ if !uri.contains(':') || uri.len() < 3 {
+ bail!("invalid URI: '{}'", uri);
+ }
+ }
+
+ for suite in self.suites.iter() {
+ if !suite.ends_with('/') && self.components.is_empty() {
+ bail!("missing component(s)");
+ } else if suite.ends_with('/') && !self.components.is_empty() {
+ bail!("absolute suite '{}' does not allow component(s)", suite);
+ }
+ }
+
+ if self.file_type == APTRepositoryFileType::List {
+ if self.types.len() > 1 {
+ bail!("more than one package type");
+ }
+ if self.uris.len() > 1 {
+ bail!("more than one URI");
+ }
+ if self.suites.len() > 1 {
+ bail!("more than one suite");
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/src/repositories/file.rs b/src/repositories/file.rs
new file mode 100644
index 0000000..e264ec6
--- /dev/null
+++ b/src/repositories/file.rs
@@ -0,0 +1,96 @@
+use std::convert::TryFrom;
+use std::path::{Path, PathBuf};
+
+use anyhow::{format_err, Error};
+
+use crate::types::{APTRepositoryFile, APTRepositoryFileError, APTRepositoryFileType};
+
+impl APTRepositoryFile {
+ /// Creates a new `APTRepositoryFile` without parsing.
+ ///
+ /// If the file is hidden or the path points to a directory, `Ok(None)` is
+ /// returned, while invalid file names yield an error.
+ pub fn new<P: AsRef<Path>>(path: P) -> Result<Option<Self>, APTRepositoryFileError> {
+ let path: PathBuf = path.as_ref().to_path_buf();
+
+ let new_err = |path_string: String, err: &str| APTRepositoryFileError {
+ path: path_string,
+ error: err.to_string(),
+ };
+
+ let path_string = path
+ .clone()
+ .into_os_string()
+ .into_string()
+ .map_err(|os_string| {
+ new_err(
+ os_string.to_string_lossy().to_string(),
+ "path is not valid unicode",
+ )
+ })?;
+
+ let new_err = |err| new_err(path_string.clone(), err);
+
+ if path.is_dir() {
+ return Ok(None);
+ }
+
+ let file_name = match path.file_name() {
+ Some(file_name) => file_name
+ .to_os_string()
+ .into_string()
+ .map_err(|_| new_err("invalid path"))?,
+ None => return Err(new_err("invalid path")),
+ };
+
+ if file_name.starts_with('.') {
+ return Ok(None);
+ }
+
+ let extension = match path.extension() {
+ Some(extension) => extension
+ .to_os_string()
+ .into_string()
+ .map_err(|_| new_err("invalid path"))?,
+ None => return Err(new_err("invalid extension")),
+ };
+
+ let file_type = APTRepositoryFileType::try_from(&extension[..])
+ .map_err(|_| new_err("invalid extension"))?;
+
+ if !file_name
+ .chars()
+ .all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '-' || x == '.')
+ {
+ return Err(new_err("invalid characters in file name"));
+ }
+
+ Ok(Some(Self {
+ path: path_string,
+ file_type,
+ repositories: vec![],
+ digest: None,
+ }))
+ }
+
+ /// Check if the file exists.
+ pub fn exists(&self) -> bool {
+ PathBuf::from(&self.path).exists()
+ }
+
+ pub fn read_with_digest(&self) -> Result<(Vec<u8>, [u8; 32]), APTRepositoryFileError> {
+ let content = std::fs::read(&self.path).map_err(|err| self.err(format_err!("{}", err)))?;
+
+ let digest = openssl::sha::sha256(&content);
+
+ Ok((content, digest))
+ }
+
+ /// Create an `APTRepositoryFileError`.
+ pub fn err(&self, error: Error) -> APTRepositoryFileError {
+ APTRepositoryFileError {
+ path: self.path.clone(),
+ error: error.to_string(),
+ }
+ }
+}
diff --git a/src/repositories/list_parser.rs b/src/repositories/list_parser.rs
new file mode 100644
index 0000000..6c9f898
--- /dev/null
+++ b/src/repositories/list_parser.rs
@@ -0,0 +1,171 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::{Iterator, Peekable};
+use std::str::SplitAsciiWhitespace;
+
+use anyhow::{bail, format_err, Error};
+
+use super::APTRepositoryParser;
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+pub struct APTListFileParser<R: BufRead> {
+ input: R,
+ line_nr: usize,
+ comment: String,
+}
+
+impl<R: BufRead> APTListFileParser<R> {
+ pub fn new(reader: R) -> Self {
+ Self {
+ input: reader,
+ line_nr: 0,
+ comment: String::new(),
+ }
+ }
+
+ /// Helper to parse options from the existing token stream.
+ ///
+ /// Also returns `Ok(())` if there are no options.
+ ///
+ /// Errors when options are invalid or not closed by `']'`.
+ fn parse_options(
+ options: &mut Vec<APTRepositoryOption>,
+ tokens: &mut Peekable<SplitAsciiWhitespace>,
+ ) -> Result<(), Error> {
+ let mut option = match tokens.peek() {
+ Some(token) => {
+ match token.strip_prefix('[') {
+ Some(option) => option,
+ None => return Ok(()), // doesn't look like options
+ }
+ }
+ None => return Ok(()),
+ };
+
+ tokens.next(); // avoid reading the beginning twice
+
+ let mut finished = false;
+ loop {
+ if let Some(stripped) = option.strip_suffix(']') {
+ option = stripped;
+ if option.is_empty() {
+ break;
+ }
+ finished = true; // but still need to handle the last one
+ };
+
+ if let Some(mid) = option.find('=') {
+ let (key, mut value_str) = option.split_at(mid);
+ value_str = &value_str[1..];
+
+ if key.is_empty() {
+ bail!("option has no key: '{}'", option);
+ }
+
+ if value_str.is_empty() {
+ bail!("option has no value: '{}'", option);
+ }
+
+ let values: Vec<String> = value_str
+ .split(',')
+ .map(|value| value.to_string())
+ .collect();
+
+ options.push(APTRepositoryOption {
+ key: key.to_string(),
+ values,
+ });
+ } else if !option.is_empty() {
+ bail!("got invalid option - '{}'", option);
+ }
+
+ if finished {
+ break;
+ }
+
+ option = match tokens.next() {
+ Some(option) => option,
+ None => bail!("options not closed by ']'"),
+ }
+ }
+
+ Ok(())
+ }
+
+ /// Parse a repository or comment in one-line format.
+ ///
+ /// Commented out repositories are also detected and returned with the
+ /// `enabled` property set to `false`.
+ ///
+ /// If the line contains a repository, `self.comment` is added to the
+ /// `comment` property.
+ ///
+ /// If the line contains a comment, it is added to `self.comment`.
+ fn parse_one_line(&mut self, mut line: &str) -> Result<Option<APTRepository>, Error> {
+ line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+ // check for commented out repository first
+ if let Some(commented_out) = line.strip_prefix('#') {
+ if let Ok(Some(mut repo)) = self.parse_one_line(commented_out) {
+ repo.set_enabled(false);
+ return Ok(Some(repo));
+ }
+ }
+
+ let mut repo = APTRepository::new(APTRepositoryFileType::List);
+
+ // now handle "real" comment
+ if let Some(comment_start) = line.find('#') {
+ let (line_start, comment) = line.split_at(comment_start);
+ self.comment = format!("{}{}\n", self.comment, &comment[1..]);
+ line = line_start;
+ }
+
+ let mut tokens = line.split_ascii_whitespace().peekable();
+
+ match tokens.next() {
+ Some(package_type) => {
+ repo.types.push(package_type.try_into()?);
+ }
+ None => return Ok(None), // empty line
+ }
+
+ Self::parse_options(&mut repo.options, &mut tokens)?;
+
+ // the rest of the line is just '<uri> <suite> [<components>...]'
+ let mut tokens = tokens.map(str::to_string);
+ repo.uris
+ .push(tokens.next().ok_or_else(|| format_err!("missing URI"))?);
+ repo.suites
+ .push(tokens.next().ok_or_else(|| format_err!("missing suite"))?);
+ repo.components.extend(tokens);
+
+ repo.comment = std::mem::take(&mut self.comment);
+
+ Ok(Some(repo))
+ }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTListFileParser<R> {
+ fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error> {
+ let mut repos = vec![];
+ let mut line = String::new();
+
+ loop {
+ self.line_nr += 1;
+ line.clear();
+
+ match self.input.read_line(&mut line) {
+ Err(err) => bail!("input error - {}", err),
+ Ok(0) => break,
+ Ok(_) => match self.parse_one_line(&line) {
+ Ok(Some(repo)) => repos.push(repo),
+ Ok(None) => continue,
+ Err(err) => bail!("malformed entry on line {} - {}", self.line_nr, err),
+ },
+ }
+ }
+
+ Ok(repos)
+ }
+}
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
new file mode 100644
index 0000000..187ead3
--- /dev/null
+++ b/src/repositories/mod.rs
@@ -0,0 +1,224 @@
+use std::path::PathBuf;
+
+use anyhow::{bail, format_err, Error};
+
+use crate::types::{
+ APTRepository, APTRepositoryFile, APTRepositoryFileError, APTRepositoryFileType,
+ APTRepositoryOption,
+};
+
+mod list_parser;
+use list_parser::APTListFileParser;
+
+mod sources_parser;
+use sources_parser::APTSourcesFileParser;
+
+mod check;
+mod file;
+mod writer;
+
+const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
+const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
+
+impl APTRepository {
+ /// Crates an empty repository.
+ fn new(file_type: APTRepositoryFileType) -> Self {
+ Self {
+ types: vec![],
+ uris: vec![],
+ suites: vec![],
+ components: vec![],
+ options: vec![],
+ comment: String::new(),
+ file_type,
+ enabled: true,
+ }
+ }
+
+ /// Changes the `enabled` flag and makes sure the `Enabled` option for
+ /// `APTRepositoryPackageType::Sources` repositories is updated too.
+ fn set_enabled(&mut self, enabled: bool) {
+ self.enabled = enabled;
+
+ if self.file_type == APTRepositoryFileType::Sources {
+ let enabled_string = match enabled {
+ true => "true".to_string(),
+ false => "false".to_string(),
+ };
+ for option in self.options.iter_mut() {
+ if option.key == "Enabled" {
+ option.values = vec![enabled_string];
+ return;
+ }
+ }
+ self.options.push(APTRepositoryOption {
+ key: "Enabled".to_string(),
+ values: vec![enabled_string],
+ });
+ }
+ }
+}
+
+trait APTRepositoryParser {
+ /// Parse all repositories including the disabled ones and push them onto
+ /// the provided vector.
+ fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error>;
+}
+
+impl APTRepositoryFile {
+ /// Parses the APT repositories configured in the file on disk, including
+ /// disabled ones.
+ ///
+ /// Resets the current repositories and digest, even on failure.
+ pub fn parse(&mut self) -> Result<(), APTRepositoryFileError> {
+ self.repositories.clear();
+ self.digest = None;
+
+ let (content, digest) = self.read_with_digest()?;
+
+ let mut parser: Box<dyn APTRepositoryParser> = match self.file_type {
+ APTRepositoryFileType::List => Box::new(APTListFileParser::new(&content[..])),
+ APTRepositoryFileType::Sources => Box::new(APTSourcesFileParser::new(&content[..])),
+ };
+
+ let repos = parser.parse_repositories().map_err(|err| self.err(err))?;
+
+ for (n, repo) in repos.iter().enumerate() {
+ repo.basic_check()
+ .map_err(|err| self.err(format_err!("check for repository {} - {}", n + 1, err)))?;
+ }
+
+ self.repositories = repos;
+ self.digest = Some(digest);
+
+ Ok(())
+ }
+
+ /// Writes the repositories to the file on disk.
+ ///
+ /// If a digest is provided, checks that the current content of the file still
+ /// produces the same one.
+ pub fn write(&self) -> Result<(), APTRepositoryFileError> {
+ if let Some(digest) = self.digest {
+ if !self.exists() {
+ return Err(self.err(format_err!("digest specified, but file does not exist")));
+ }
+
+ let (_, current_digest) = self.read_with_digest()?;
+ if digest != current_digest {
+ return Err(self.err(format_err!("digest mismatch")));
+ }
+ }
+
+ let mut content = vec![];
+
+ for (n, repo) in self.repositories.iter().enumerate() {
+ repo.basic_check()
+ .map_err(|err| self.err(format_err!("check for repository {} - {}", n + 1, err)))?;
+
+ repo.write(&mut content)
+ .map_err(|err| self.err(format_err!("writing repository {} - {}", n + 1, err)))?;
+ }
+
+ let path = PathBuf::from(&self.path);
+ let dir = match path.parent() {
+ Some(dir) => dir,
+ None => return Err(self.err(format_err!("invalid path"))),
+ };
+
+ std::fs::create_dir_all(dir)
+ .map_err(|err| self.err(format_err!("unable to create parent dir - {}", err)))?;
+
+ let pid = std::process::id();
+ let mut tmp_path = path.clone();
+ tmp_path.set_extension("tmp");
+ tmp_path.set_extension(format!("{}", pid));
+
+ if let Err(err) = std::fs::write(&tmp_path, content) {
+ let _ = std::fs::remove_file(&tmp_path);
+ return Err(self.err(format_err!("writing {:?} failed - {}", path, err)));
+ }
+
+ if let Err(err) = std::fs::rename(&tmp_path, &path) {
+ let _ = std::fs::remove_file(&tmp_path);
+ return Err(self.err(format_err!("rename failed for {:?} - {}", path, err)));
+ }
+
+ Ok(())
+ }
+}
+
+/// Returns all APT repositories configured in `/etc/apt/sources.list` and
+/// in `/etc/apt/sources.list.d` including disabled repositories.
+///
+/// Returns the parsable files with their repositories and a list of errors for
+/// files that could not be read or parsed.
+///
+/// The digest is guaranteed to be set for each successfully parsed file.
+pub fn repositories() -> Result<(Vec<APTRepositoryFile>, Vec<APTRepositoryFileError>), Error> {
+ let mut files = vec![];
+ let mut errors = vec![];
+
+ let sources_list_path = PathBuf::from(APT_SOURCES_LIST_FILENAME);
+
+ let sources_list_d_path = PathBuf::from(APT_SOURCES_LIST_DIRECTORY);
+
+ match APTRepositoryFile::new(sources_list_path) {
+ Ok(Some(mut file)) => match file.parse() {
+ Ok(()) => files.push(file),
+ Err(err) => errors.push(err),
+ },
+ _ => bail!("internal error with '{}'", APT_SOURCES_LIST_FILENAME),
+ }
+
+ if !sources_list_d_path.exists() {
+ return Ok((files, errors));
+ }
+
+ if !sources_list_d_path.is_dir() {
+ errors.push(APTRepositoryFileError {
+ path: APT_SOURCES_LIST_DIRECTORY.to_string(),
+ error: "not a directory!".to_string(),
+ });
+ return Ok((files, errors));
+ }
+
+ for entry in std::fs::read_dir(sources_list_d_path)? {
+ let path = entry?.path();
+
+ match APTRepositoryFile::new(path) {
+ Ok(Some(mut file)) => match file.parse() {
+ Ok(()) => {
+ if file.digest.is_none() {
+ bail!("internal error - digest not set");
+ }
+ files.push(file);
+ }
+ Err(err) => errors.push(err),
+ },
+ Ok(None) => (),
+ Err(err) => errors.push(err),
+ }
+ }
+
+ Ok((files, errors))
+}
+
+/// Write the repositories for each file.
+///
+/// Returns an error for each file that could not be written successfully.
+pub fn write_repositories(files: &[APTRepositoryFile]) -> Result<(), Vec<APTRepositoryFileError>> {
+ let mut errors = vec![];
+
+ for file in files {
+ if let Err(err) = file.write() {
+ errors.push(err);
+ }
+ }
+
+ if !errors.is_empty() {
+ return Err(errors);
+ }
+
+ Ok(())
+}
diff --git a/src/repositories/sources_parser.rs b/src/repositories/sources_parser.rs
new file mode 100644
index 0000000..a056b8f
--- /dev/null
+++ b/src/repositories/sources_parser.rs
@@ -0,0 +1,204 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::Iterator;
+
+use anyhow::{bail, Error};
+
+use crate::types::{
+ APTRepository, APTRepositoryFileType, APTRepositoryOption, APTRepositoryPackageType,
+};
+
+use super::APTRepositoryParser;
+
+pub struct APTSourcesFileParser<R: BufRead> {
+ input: R,
+ stanza_nr: usize,
+ comment: String,
+}
+
+/// See `man sources.list` and `man deb822` for the format specification.
+impl<R: BufRead> APTSourcesFileParser<R> {
+ pub fn new(reader: R) -> Self {
+ Self {
+ input: reader,
+ stanza_nr: 1,
+ comment: String::new(),
+ }
+ }
+
+ /// Based on APT's `StringToBool` in `strutl.cc`
+ fn string_to_bool(string: &str, default: bool) -> bool {
+ let string = string.trim_matches(|c| char::is_ascii_whitespace(&c));
+ let string = string.to_lowercase();
+
+ match &string[..] {
+ "1" | "yes" | "true" | "with" | "on" | "enable" => true,
+ "0" | "no" | "false" | "without" | "off" | "disable" => false,
+ _ => default,
+ }
+ }
+
+ /// Checks if `key` is valid according to deb822
+ fn valid_key(key: &str) -> bool {
+ if key.starts_with('-') {
+ return false;
+ };
+ return key.chars().all(|c| matches!(c, '!'..='9' | ';'..='~'));
+ }
+
+ /// Try parsing a repository in stanza format from `lines`.
+ ///
+ /// Returns `Ok(None)` when no stanza can be found.
+ ///
+ /// Comments are added to `self.comments`. If a stanza can be found,
+ /// `self.comment` is added to the repository's `comment` property.
+ ///
+ /// Fully commented out stanzas are treated as comments.
+ fn parse_stanza(&mut self, lines: &str) -> Result<Option<APTRepository>, Error> {
+ let mut repo = APTRepository::new(APTRepositoryFileType::Sources);
+
+ // Values may be folded into multiple lines.
+ // Those lines have to start with a space or a tab.
+ let lines = lines.replace("\n ", " ");
+ let lines = lines.replace("\n\t", " ");
+
+ let mut got_something = false;
+
+ for line in lines.lines() {
+ let line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+ if line.is_empty() {
+ continue;
+ }
+
+ if let Some(commented_out) = line.strip_prefix('#') {
+ self.comment = format!("{}{}\n", self.comment, commented_out);
+ continue;
+ }
+
+ if let Some(mid) = line.find(':') {
+ let (key, value_str) = line.split_at(mid);
+ let value_str = &value_str[1..];
+ let key = key.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+ if key.is_empty() {
+ bail!("option has no key: '{}'", line);
+ }
+
+ if value_str.is_empty() {
+ // ignored by APT
+ eprintln!("option has no value: '{}'", line);
+ continue;
+ }
+
+ if !Self::valid_key(key) {
+ // ignored by APT
+ eprintln!("option with invalid key '{}'", key);
+ continue;
+ }
+
+ let values: Vec<String> = value_str
+ .split_ascii_whitespace()
+ .map(|value| value.to_string())
+ .collect();
+
+ match &key.to_lowercase()[..] {
+ "types" => {
+ if !repo.types.is_empty() {
+ eprintln!("key 'Types' was defined twice");
+ }
+ let mut types = Vec::<APTRepositoryPackageType>::new();
+ for package_type in values {
+ types.push((&package_type[..]).try_into()?);
+ }
+ repo.types = types;
+ }
+ "uris" => {
+ if !repo.uris.is_empty() {
+ eprintln!("key 'URIs' was defined twice");
+ }
+ repo.uris = values;
+ }
+ "suites" => {
+ if !repo.suites.is_empty() {
+ eprintln!("key 'Suites' was defined twice");
+ }
+ repo.suites = values;
+ }
+ "components" => {
+ if !repo.components.is_empty() {
+ eprintln!("key 'Components' was defined twice");
+ }
+ repo.components = values;
+ }
+ "enabled" => {
+ repo.set_enabled(Self::string_to_bool(value_str, true));
+ }
+ _ => repo.options.push(APTRepositoryOption {
+ key: key.to_string(),
+ values,
+ }),
+ }
+ } else {
+ bail!("got invalid line - '{:?}'", line);
+ }
+
+ got_something = true;
+ }
+
+ if !got_something {
+ return Ok(None);
+ }
+
+ repo.comment = std::mem::take(&mut self.comment);
+
+ Ok(Some(repo))
+ }
+
+ /// Helper function for `parse_repositories`.
+ fn try_parse_stanza(
+ &mut self,
+ lines: &str,
+ repos: &mut Vec<APTRepository>,
+ ) -> Result<(), Error> {
+ match self.parse_stanza(lines) {
+ Ok(Some(repo)) => {
+ repos.push(repo);
+ self.stanza_nr += 1;
+ }
+ Ok(None) => (),
+ Err(err) => bail!("malformed entry in stanza {} - {}", self.stanza_nr, err),
+ }
+
+ Ok(())
+ }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTSourcesFileParser<R> {
+ fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error> {
+ let mut repos = vec![];
+ let mut lines = String::new();
+
+ loop {
+ let old_length = lines.len();
+ match self.input.read_line(&mut lines) {
+ Err(err) => bail!("input error - {}", err),
+ Ok(0) => {
+ self.try_parse_stanza(&lines[..], &mut repos)?;
+ break;
+ }
+ Ok(_) => {
+ if (&lines[old_length..])
+ .trim_matches(|c| char::is_ascii_whitespace(&c))
+ .is_empty()
+ {
+ // detected end of stanza
+ self.try_parse_stanza(&lines[..], &mut repos)?;
+ lines.clear();
+ }
+ }
+ }
+ }
+
+ Ok(repos)
+ }
+}
diff --git a/src/repositories/writer.rs b/src/repositories/writer.rs
new file mode 100644
index 0000000..d9e937c
--- /dev/null
+++ b/src/repositories/writer.rs
@@ -0,0 +1,92 @@
+use std::io::Write;
+
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+ /// Writes a repository in the corresponding format followed by a blank.
+ ///
+ /// Expects that `basic_check()` for the repository was successful.
+ pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> {
+ match self.file_type {
+ APTRepositoryFileType::List => write_one_line(self, w),
+ APTRepositoryFileType::Sources => write_stanza(self, w),
+ }
+ }
+}
+
+/// Writes a repository in one-line format followed by a blank line.
+///
+/// Expects that `repo.file_type == APTRepositoryFileType::List`.
+///
+/// Expects that `basic_check()` for the repository was successful.
+fn write_one_line(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+ if repo.file_type != APTRepositoryFileType::List {
+ bail!("not a .list repository");
+ }
+
+ if !repo.comment.is_empty() {
+ for line in repo.comment.lines() {
+ writeln!(w, "#{}", line)?;
+ }
+ }
+
+ if !repo.enabled {
+ write!(w, "# ")?;
+ }
+
+ write!(w, "{} ", repo.types[0])?;
+
+ if !repo.options.is_empty() {
+ write!(w, "[ ")?;
+ repo.options
+ .iter()
+ .try_for_each(|option| write!(w, "{}={} ", option.key, option.values.join(",")))?;
+ write!(w, "] ")?;
+ };
+
+ write!(w, "{} ", repo.uris[0])?;
+ write!(w, "{} ", repo.suites[0])?;
+ writeln!(w, "{}", repo.components.join(" "))?;
+
+ writeln!(w)?;
+
+ Ok(())
+}
+
+/// Writes a single stanza followed by a blank line.
+///
+/// Expects that `repo.file_type == APTRepositoryFileType::Sources`.
+fn write_stanza(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+ if repo.file_type != APTRepositoryFileType::Sources {
+ bail!("not a .sources repository");
+ }
+
+ if !repo.comment.is_empty() {
+ for line in repo.comment.lines() {
+ writeln!(w, "#{}", line)?;
+ }
+ }
+
+ write!(w, "Types:")?;
+ repo.types
+ .iter()
+ .try_for_each(|package_type| write!(w, " {}", package_type))?;
+ writeln!(w)?;
+
+ writeln!(w, "URIs: {}", repo.uris.join(" "))?;
+ writeln!(w, "Suites: {}", repo.suites.join(" "))?;
+
+ if !repo.components.is_empty() {
+ writeln!(w, "Components: {}", repo.components.join(" "))?;
+ }
+
+ for option in repo.options.iter() {
+ writeln!(w, "{}: {}", option.key, option.values.join(" "))?;
+ }
+
+ writeln!(w)?;
+
+ Ok(())
+}
diff --git a/src/types.rs b/src/types.rs
new file mode 100644
index 0000000..bbd8e7e
--- /dev/null
+++ b/src/types.rs
@@ -0,0 +1,246 @@
+use std::convert::TryFrom;
+use std::fmt::Display;
+
+use anyhow::{bail, Error};
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum APTRepositoryFileType {
+ /// One-line-style format
+ List,
+ /// DEB822-style format
+ Sources,
+}
+
+impl TryFrom<&str> for APTRepositoryFileType {
+ type Error = Error;
+
+ fn try_from(string: &str) -> Result<Self, Error> {
+ match string {
+ "list" => Ok(APTRepositoryFileType::List),
+ "sources" => Ok(APTRepositoryFileType::Sources),
+ _ => bail!("invalid file type '{}'", string),
+ }
+ }
+}
+
+impl Display for APTRepositoryFileType {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ APTRepositoryFileType::List => write!(f, "list"),
+ APTRepositoryFileType::Sources => write!(f, "sources"),
+ }
+ }
+}
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub enum APTRepositoryPackageType {
+ /// Debian package
+ Deb,
+ /// Debian source package
+ DebSrc,
+}
+
+impl TryFrom<&str> for APTRepositoryPackageType {
+ type Error = Error;
+
+ fn try_from(string: &str) -> Result<Self, Error> {
+ match string {
+ "deb" => Ok(APTRepositoryPackageType::Deb),
+ "deb-src" => Ok(APTRepositoryPackageType::DebSrc),
+ _ => bail!("invalid package type '{}'", string),
+ }
+ }
+}
+
+impl Display for APTRepositoryPackageType {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ APTRepositoryPackageType::Deb => write!(f, "deb"),
+ APTRepositoryPackageType::DebSrc => write!(f, "deb-src"),
+ }
+ }
+}
+
+#[api(
+ properties: {
+ Key: {
+ description: "Option key.",
+ type: String,
+ },
+ Values: {
+ description: "Option values.",
+ type: Array,
+ items: {
+ description: "Value.",
+ type: String,
+ },
+ },
+ },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")] // for consistency
+/// Additional options for an APT repository.
+/// Used for both single- and mutli-value options.
+pub struct APTRepositoryOption {
+ /// Option key.
+ pub key: String,
+ /// Option value(s).
+ pub values: Vec<String>,
+}
+
+#[api(
+ properties: {
+ Types: {
+ description: "List of package types.",
+ type: Array,
+ items: {
+ type: APTRepositoryPackageType,
+ },
+ },
+ URIs: {
+ description: "List of repository URIs.",
+ type: Array,
+ items: {
+ description: "Repository URI.",
+ type: String,
+ },
+ },
+ Suites: {
+ description: "List of distributions.",
+ type: Array,
+ items: {
+ description: "Package distribution.",
+ type: String,
+ },
+ },
+ Components: {
+ description: "List of repository components.",
+ type: Array,
+ items: {
+ description: "Repository component.",
+ type: String,
+ },
+ },
+ Options: {
+ type: Array,
+ optional: true,
+ items: {
+ type: APTRepositoryOption,
+ },
+ },
+ Comment: {
+ description: "Associated comment.",
+ type: String,
+ optional: true,
+ },
+ FileType: {
+ type: APTRepositoryFileType,
+ },
+ Enabled: {
+ description: "Whether the repository is enabled or not.",
+ type: Boolean,
+ },
+ },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+/// Describes an APT repository.
+pub struct APTRepository {
+ /// List of package types.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub types: Vec<APTRepositoryPackageType>,
+
+ /// List of repository URIs.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ #[serde(rename = "URIs")]
+ pub uris: Vec<String>,
+
+ /// List of package distributions.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub suites: Vec<String>,
+
+ /// List of repository components.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub components: Vec<String>,
+
+ /// Additional options.
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ pub options: Vec<APTRepositoryOption>,
+
+ /// Associated comment.
+ #[serde(skip_serializing_if = "String::is_empty")]
+ pub comment: String,
+
+ /// Format of the defining file.
+ pub file_type: APTRepositoryFileType,
+
+ /// Whether the repository is enabled or not.
+ pub enabled: bool,
+}
+
+#[api(
+ properties: {
+ file_type: {
+ type: APTRepositoryFileType,
+ },
+ repositories: {
+ description: "List of APT repositories.",
+ type: Array,
+ items: {
+ type: APTRepository,
+ },
+ },
+ digest: {
+ description: "Digest for the content of the file.",
+ optional: true,
+ type: Array,
+ items: {
+ description: "Digest byte.",
+ type: Integer,
+ },
+ },
+ },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Represents an abstract APT repository file.
+pub struct APTRepositoryFile {
+ /// The path to the file.
+ pub path: String,
+ /// The type of the file.
+ pub file_type: APTRepositoryFileType,
+ /// List of repositories in the file.
+ pub repositories: Vec<APTRepository>,
+ /// Digest of the original contents.
+ pub digest: Option<[u8; 32]>,
+}
+
+#[api]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Error type for problems with APT repository files.
+pub struct APTRepositoryFileError {
+ /// The path to the problematic file.
+ pub path: String,
+ /// The error message.
+ pub error: String,
+}
+
+impl Display for APTRepositoryFileError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "proxmox-apt error for '{}' - {}", self.path, self.error)
+ }
+}
+
+impl std::error::Error for APTRepositoryFileError {
+ fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+ None
+ }
+}
diff --git a/tests/repositories.rs b/tests/repositories.rs
new file mode 100644
index 0000000..aca05ef
--- /dev/null
+++ b/tests/repositories.rs
@@ -0,0 +1,129 @@
+use std::path::PathBuf;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox_apt::repositories::write_repositories;
+use proxmox_apt::types::APTRepositoryFile;
+
+#[test]
+fn test_parse_write() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+ let write_dir = test_dir.join("sources.list.d.actual");
+ let expected_dir = test_dir.join("sources.list.d.expected");
+
+ if write_dir.is_dir() {
+ std::fs::remove_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+ }
+
+ std::fs::create_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+ let mut files = vec![];
+ let mut errors = vec![];
+
+ for entry in std::fs::read_dir(read_dir)? {
+ let path = entry?.path();
+
+ match APTRepositoryFile::new(&path)? {
+ Some(mut file) => match file.parse() {
+ Ok(()) => files.push(file),
+ Err(err) => errors.push(err),
+ },
+ None => bail!("unexpected None for '{:?}'", path),
+ }
+ }
+
+ assert!(errors.is_empty());
+
+ for file in files.iter_mut() {
+ let path = PathBuf::from(&file.path);
+ let new_path = write_dir.join(path.file_name().unwrap());
+ file.path = new_path.into_os_string().into_string().unwrap();
+ file.digest = None;
+ }
+
+ write_repositories(&files).map_err(|err| format_err!("{:?}", err))?;
+
+ let mut expected_count = 0;
+
+ for entry in std::fs::read_dir(expected_dir)? {
+ expected_count += 1;
+
+ let expected_path = entry?.path();
+ let actual_path = write_dir.join(expected_path.file_name().unwrap());
+
+ let expected_contents = std::fs::read(&expected_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
+
+ let actual_contents = std::fs::read(&actual_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
+
+ assert_eq!(
+ expected_contents, actual_contents,
+ "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals",
+ expected_path, actual_path
+ );
+ }
+
+ let actual_count = std::fs::read_dir(write_dir)?.count();
+
+ assert_eq!(expected_count, actual_count);
+
+ Ok(())
+}
+
+#[test]
+fn test_digest() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+ let write_dir = test_dir.join("sources.list.d.digest");
+
+ if write_dir.is_dir() {
+ std::fs::remove_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+ }
+
+ std::fs::create_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+ let path = read_dir.join("standard.list");
+
+ let mut file = APTRepositoryFile::new(&path)?.unwrap();
+ file.parse()?;
+
+ let new_path = write_dir.join(path.file_name().unwrap());
+ file.path = new_path.clone().into_os_string().into_string().unwrap();
+
+ let old_digest = file.digest.unwrap();
+ let mut files = vec![file];
+
+ // file does not exist yet...
+ assert!(files.first().unwrap().read_with_digest().is_err());
+ assert!(write_repositories(&files).is_err());
+
+ // ...but it should work if there's no digest
+ files[0].digest = None;
+ write_repositories(&files).map_err(|err| format_err!("{:?}", err))?;
+
+ // overwrite with old contents...
+ std::fs::copy(path, new_path)?;
+
+ // modify the repo
+ let mut file = files.first_mut().unwrap();
+ let mut repo = file.repositories.first_mut().unwrap();
+ repo.enabled = !repo.enabled;
+
+ // ...then it should work
+ file.digest = Some(old_digest);
+ write_repositories(&files).map_err(|err| format_err!("{:?}", err))?;
+
+ // expect a different digest, because the repo was modified
+ let (_, new_digest) = files.first().unwrap().read_with_digest()?;
+ assert_ne!(old_digest, new_digest);
+
+ assert!(write_repositories(&files).is_err());
+
+ Ok(())
+}
diff --git a/tests/sources.list.d.expected/absolute_suite.list b/tests/sources.list.d.expected/absolute_suite.list
new file mode 100644
index 0000000..525389c
--- /dev/null
+++ b/tests/sources.list.d.expected/absolute_suite.list
@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+deb http://packages.falcot.com/ updates/
+
+deb http://packages.falcot.com/ internal/
+
diff --git a/tests/sources.list.d.expected/absolute_suite.sources b/tests/sources.list.d.expected/absolute_suite.sources
new file mode 100644
index 0000000..51e4d56
--- /dev/null
+++ b/tests/sources.list.d.expected/absolute_suite.sources
@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+Types: deb
+URIs: http://packages.falcot.com/
+Suites: updates/ internal/
+
diff --git a/tests/sources.list.d.expected/case.sources b/tests/sources.list.d.expected/case.sources
new file mode 100644
index 0000000..307aab6
--- /dev/null
+++ b/tests/sources.list.d.expected/case.sources
@@ -0,0 +1,16 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster-updates
+Components: main contrib
+languages: it de fr
+Enabled: false
+languages-Add: ja
+languages-Remove: de
+
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster
+Components: main contrib
+
diff --git a/tests/sources.list.d.expected/multiline.sources b/tests/sources.list.d.expected/multiline.sources
new file mode 100644
index 0000000..d96acea
--- /dev/null
+++ b/tests/sources.list.d.expected/multiline.sources
@@ -0,0 +1,10 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+Languages: it de fr
+Enabled: false
+Languages-Add: ja
+Languages-Remove: de
+
--git a/tests/sources.list.d.expected/options_comment.list b/tests/sources.list.d.expected/options_comment.list
new file mode 100644
index 0000000..8c905c0
--- /dev/null
+++ b/tests/sources.list.d.expected/options_comment.list
@@ -0,0 +1,6 @@
+# comment
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib
+
+# non-free :(
+deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian buster non-free
+
diff --git a/tests/sources.list.d.expected/pbs-enterprise.list b/tests/sources.list.d.expected/pbs-enterprise.list
new file mode 100644
index 0000000..acb2990
--- /dev/null
+++ b/tests/sources.list.d.expected/pbs-enterprise.list
@@ -0,0 +1,2 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
+
diff --git a/tests/sources.list.d.expected/pve.list b/tests/sources.list.d.expected/pve.list
new file mode 100644
index 0000000..127a49a
--- /dev/null
+++ b/tests/sources.list.d.expected/pve.list
@@ -0,0 +1,13 @@
+deb http://ftp.debian.org/debian buster main contrib
+
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.list b/tests/sources.list.d.expected/standard.list
new file mode 100644
index 0000000..63c1b60
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.list
@@ -0,0 +1,7 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.sources b/tests/sources.list.d.expected/standard.sources
new file mode 100644
index 0000000..56ce280
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.sources
@@ -0,0 +1,11 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
+
diff --git a/tests/sources.list.d/absolute_suite.list b/tests/sources.list.d/absolute_suite.list
new file mode 100644
index 0000000..ed576bf
--- /dev/null
+++ b/tests/sources.list.d/absolute_suite.list
@@ -0,0 +1,3 @@
+# From Debian Administrator's Handbook
+deb http://packages.falcot.com/ updates/
+deb http://packages.falcot.com/ internal/
diff --git a/tests/sources.list.d/absolute_suite.sources b/tests/sources.list.d/absolute_suite.sources
new file mode 100644
index 0000000..51e4d56
--- /dev/null
+++ b/tests/sources.list.d/absolute_suite.sources
@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+Types: deb
+URIs: http://packages.falcot.com/
+Suites: updates/ internal/
+
diff --git a/tests/sources.list.d/case.sources b/tests/sources.list.d/case.sources
new file mode 100644
index 0000000..8979d0c
--- /dev/null
+++ b/tests/sources.list.d/case.sources
@@ -0,0 +1,17 @@
+tYpeS: deb deb-src
+uRis: http://ftp.at.debian.org/debian
+suiTes: buster-updates
+# comment in here
+CompOnentS: main contrib
+languages: it
+ de
+ fr
+Enabled: off
+languages-Add: ja
+languages-Remove: de
+
+types: deb deb-src
+Uris: http://ftp.at.debian.org/debian
+suites: buster
+# comment in here
+components: main contrib
diff --git a/tests/sources.list.d/multiline.sources b/tests/sources.list.d/multiline.sources
new file mode 100644
index 0000000..bdbce29
--- /dev/null
+++ b/tests/sources.list.d/multiline.sources
@@ -0,0 +1,11 @@
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+# comment in here
+Components: main contrib
+Languages: it
+ de
+ fr
+Enabled: off
+Languages-Add: ja
+Languages-Remove: de
--git a/tests/sources.list.d/options_comment.list b/tests/sources.list.d/options_comment.list
new file mode 100644
index 0000000..6b73053
--- /dev/null
+++ b/tests/sources.list.d/options_comment.list
@@ -0,0 +1,3 @@
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib # comment
+deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian buster non-free # non-free :(
+
diff --git a/tests/sources.list.d/pbs-enterprise.list b/tests/sources.list.d/pbs-enterprise.list
new file mode 100644
index 0000000..5f8763c
--- /dev/null
+++ b/tests/sources.list.d/pbs-enterprise.list
@@ -0,0 +1 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
diff --git a/tests/sources.list.d/pve.list b/tests/sources.list.d/pve.list
new file mode 100644
index 0000000..6213f72
--- /dev/null
+++ b/tests/sources.list.d/pve.list
@@ -0,0 +1,10 @@
+deb http://ftp.debian.org/debian buster main contrib
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
diff --git a/tests/sources.list.d/standard.list b/tests/sources.list.d/standard.list
new file mode 100644
index 0000000..26db887
--- /dev/null
+++ b/tests/sources.list.d/standard.list
@@ -0,0 +1,6 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
diff --git a/tests/sources.list.d/standard.sources b/tests/sources.list.d/standard.sources
new file mode 100644
index 0000000..605202e
--- /dev/null
+++ b/tests/sources.list.d/standard.sources
@@ -0,0 +1,10 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-apt 02/23] add files for Debian packaging
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 01/23] initial commit Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 03/23] add functions to check for Proxmox repositories Fabian Ebner
` (20 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
The Makefile is based on the one from Mira's conntrack series, as it already got
some review.
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* also call cargo clean with make clean
.gitignore | 5 ++++
Makefile | 65 ++++++++++++++++++++++++++++++++++++++++++++
debian/changelog | 5 ++++
debian/copyright | 16 +++++++++++
debian/debcargo.toml | 7 +++++
5 files changed, 98 insertions(+)
create mode 100644 Makefile
create mode 100644 debian/changelog
create mode 100644 debian/copyright
create mode 100644 debian/debcargo.toml
diff --git a/.gitignore b/.gitignore
index 24917d4..db6f13e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,8 @@ Cargo.lock
target/
tests/sources.list.d.actual
tests/sources.list.d.digest
+proxmox-apt-*/
+*proxmox-apt*.buildinfo
+*proxmox-apt*.tar.?z
+*proxmox-apt*.changes
+*proxmox-apt*.deb
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..42665c8
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,65 @@
+include /usr/share/dpkg/pkg-info.mk
+include /usr/share/dpkg/architecture.mk
+
+PACKAGE=proxmox-apt
+BUILDDIR ?= $(PACKAGE)-$(DEB_VERSION_UPSTREAM)
+BUILDDIR_TMP ?= $(BUILDDIR).tmp
+
+DEB=librust-$(PACKAGE)-dev_$(DEB_VERSION_UPSTREAM_REVISION)_$(DEB_BUILD_ARCH).deb
+DSC=rust-$(PACKAGE)_$(DEB_VERSION_UPSTREAM_REVISION).dsc
+
+ifeq ($(BUILD_MODE), release)
+CARGO_BUILD_ARGS += --release
+COMPILEDIR := target/release
+else
+COMPILEDIR := target/debug
+endif
+
+all: cargo-build $(SUBDIRS)
+
+.PHONY: cargo-build
+cargo-build:
+ cargo build $(CARGO_BUILD_ARGS)
+
+.PHONY: build
+build:
+ rm -rf $(BUILDDIR) $(BUILDDIR_TMP); mkdir $(BUILDDIR_TMP)
+ debcargo package \
+ --config debian/debcargo.toml \
+ --changelog-ready \
+ --no-overlay-write-back \
+ --directory $(BUILDDIR_TMP) \
+ $(PACKAGE) \
+ $(shell dpkg-parsechangelog -l debian/changelog -SVersion | sed -e 's/-.*//')
+ rm -f $(BUILDDIR_TMP)/Cargo.lock
+ find $(BUILDDIR_TMP)/debian -name "*.hint" -delete
+ mv $(BUILDDIR_TMP) $(BUILDDIR)
+
+.PHONY: deb
+deb: $(DEB)
+$(DEB): build
+ cd $(BUILDDIR); dpkg-buildpackage -b -us -uc --no-pre-clean
+ lintian $(DEB)
+
+.PHONY: dsc
+dsc: $(DSC)
+$(DSC): build
+ cd $(BUILDDIR); dpkg-buildpackage -S -us -uc -d -nc
+ lintian $(DSC)
+
+.PHONY: dinstall
+dinstall: $(DEB)
+ dpkg -i $(DEB)
+
+.PHONY: upload
+upload: $(DEB) $(DBG_DEB)
+ tar cf - $(DEB) $(DBG_DEB) | ssh -X repoman@repo.proxmox.com -- upload --product pbs,pmg,pve --dist buster --arch $(DEB_BUILD_ARCH)
+
+.PHONY: distclean
+distclean: clean
+
+.PHONY: clean
+clean:
+ cargo clean
+ rm -rf *.deb *.buildinfo *.changes *.dsc rust-$(PACKAGE)_*.tar.?z $(BUILDDIR) $(BUILDDIR_TMP)
+ find . -name '*~' -exec rm {} ';'
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..11e26ed
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+rust-proxmox-apt (0.1.0-1) unstable; urgency=medium
+
+ * Initial release.
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 18 Feb 2021 10:20:44 +0100
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..5661ef6
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,16 @@
+Copyright (C) 2021 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/debian/debcargo.toml b/debian/debcargo.toml
new file mode 100644
index 0000000..74e3854
--- /dev/null
+++ b/debian/debcargo.toml
@@ -0,0 +1,7 @@
+overlay = "."
+crate_src_path = ".."
+maintainer = "Proxmox Support Team <support@proxmox.com>"
+
+[source]
+vcs_git = "git://git.proxmox.com/git/proxmox-apt.git"
+vcs_browser = "https://git.proxmox.com/?p=proxmox-apt.git"
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-apt 03/23] add functions to check for Proxmox repositories
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 01/23] initial commit Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 02/23] add files for Debian packaging Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 04/23] add check_repositories function Fabian Ebner
` (19 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* require package type to be 'deb' for detection
src/repositories/check.rs | 60 +++++++++++++++++++++++++-
src/repositories/mod.rs | 20 +++++++++
tests/repositories.rs | 34 ++++++++++++++-
tests/sources.list.d.expected/pve.list | 2 +
tests/sources.list.d/pve.list | 2 +
5 files changed, 116 insertions(+), 2 deletions(-)
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
index 87fbbac..a682b69 100644
--- a/src/repositories/check.rs
+++ b/src/repositories/check.rs
@@ -1,6 +1,6 @@
use anyhow::{bail, Error};
-use crate::types::{APTRepository, APTRepositoryFileType};
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryPackageType};
impl APTRepository {
/// Makes sure that all basic properties of a repository are present and
@@ -44,4 +44,62 @@ impl APTRepository {
Ok(())
}
+
+ /// Checks if the repository is the no-subscription repository of the specified
+ /// Proxmox product.
+ pub fn is_no_subscription(&self, product: &str) -> bool {
+ let base_uri = "http://download.proxmox.com/debian";
+ let no_subscription_uri = format!("{}/{}", base_uri, product);
+ let no_subscription_component = format!("{}-no-subscription", product);
+
+ if !self
+ .types
+ .iter()
+ .any(|package_type| *package_type == APTRepositoryPackageType::Deb)
+ {
+ return false;
+ }
+
+ if self
+ .uris
+ .iter()
+ .any(|uri| uri.trim_end_matches('/') == no_subscription_uri)
+ {
+ return self
+ .components
+ .iter()
+ .any(|comp| *comp == no_subscription_component);
+ } else {
+ false
+ }
+ }
+
+ /// Checks if the repository is the enterprise repository of the specified
+ /// Proxmox product.
+ pub fn is_enterprise(&self, product: &str) -> bool {
+ let base_uri = "https://enterprise.proxmox.com/debian";
+ let enterprise_uri = format!("{}/{}", base_uri, product);
+ let enterprise_component = format!("{}-enterprise", product);
+
+ if !self
+ .types
+ .iter()
+ .any(|package_type| *package_type == APTRepositoryPackageType::Deb)
+ {
+ return false;
+ }
+
+ if self
+ .uris
+ .iter()
+ .any(|uri| uri.trim_end_matches('/') == enterprise_uri)
+ {
+ return self
+ .components
+ .iter()
+ .any(|comp| *comp == enterprise_component);
+ } else {
+ false
+ }
+ }
}
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
index 187ead3..b7919a9 100644
--- a/src/repositories/mod.rs
+++ b/src/repositories/mod.rs
@@ -148,6 +148,26 @@ impl APTRepositoryFile {
}
}
+/// Checks if the enterprise repository for the specified Proxmox product is
+/// configured and enabled.
+pub fn enterprise_repository_enabled(files: &[APTRepositoryFile], product: &str) -> bool {
+ files.iter().any(|file| {
+ file.repositories
+ .iter()
+ .any(|repo| repo.enabled && repo.is_enterprise(product))
+ })
+}
+
+/// Checks if the no-subscription repository for the specified Proxmox product
+/// is configured and enabled.
+pub fn no_subscription_repository_enabled(files: &[APTRepositoryFile], product: &str) -> bool {
+ files.iter().any(|file| {
+ file.repositories
+ .iter()
+ .any(|repo| repo.enabled && repo.is_no_subscription(product))
+ })
+}
+
/// Returns all APT repositories configured in `/etc/apt/sources.list` and
/// in `/etc/apt/sources.list.d` including disabled repositories.
///
diff --git a/tests/repositories.rs b/tests/repositories.rs
index aca05ef..ffb1888 100644
--- a/tests/repositories.rs
+++ b/tests/repositories.rs
@@ -2,7 +2,9 @@ use std::path::PathBuf;
use anyhow::{bail, format_err, Error};
-use proxmox_apt::repositories::write_repositories;
+use proxmox_apt::repositories::{
+ enterprise_repository_enabled, no_subscription_repository_enabled, write_repositories,
+};
use proxmox_apt::types::APTRepositoryFile;
#[test]
@@ -127,3 +129,33 @@ fn test_digest() -> Result<(), Error> {
Ok(())
}
+
+#[test]
+fn test_proxmox_repositories() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+
+ let pve_list = read_dir.join("pve.list");
+ let mut file = APTRepositoryFile::new(&pve_list)?.unwrap();
+ file.parse()?;
+
+ let files = vec![file];
+
+ assert_eq!(false, enterprise_repository_enabled(&files, "pbs"));
+ assert_eq!(false, enterprise_repository_enabled(&files, "pve"));
+ assert_eq!(false, no_subscription_repository_enabled(&files, "pmg"));
+ assert_eq!(true, no_subscription_repository_enabled(&files, "pve"));
+
+ let pbs_list = read_dir.join("pbs-enterprise.list");
+ let mut file = APTRepositoryFile::new(&pbs_list)?.unwrap();
+ file.parse()?;
+
+ let files = vec![file];
+
+ assert_eq!(true, enterprise_repository_enabled(&files, "pbs"));
+ assert_eq!(false, enterprise_repository_enabled(&files, "pve"));
+ assert_eq!(false, no_subscription_repository_enabled(&files, "pmg"));
+ assert_eq!(false, no_subscription_repository_enabled(&files, "pve"));
+
+ Ok(())
+}
diff --git a/tests/sources.list.d.expected/pve.list b/tests/sources.list.d.expected/pve.list
index 127a49a..805cef7 100644
--- a/tests/sources.list.d.expected/pve.list
+++ b/tests/sources.list.d.expected/pve.list
@@ -8,6 +8,8 @@ deb http://download.proxmox.com/debian/pve buster pve-no-subscription
# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+deb-src https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
# security updates
deb http://security.debian.org/debian-security buster/updates main contrib
diff --git a/tests/sources.list.d/pve.list b/tests/sources.list.d/pve.list
index 6213f72..2c09f15 100644
--- a/tests/sources.list.d/pve.list
+++ b/tests/sources.list.d/pve.list
@@ -6,5 +6,7 @@ deb http://ftp.debian.org/debian buster-updates main contrib
deb http://download.proxmox.com/debian/pve buster pve-no-subscription
# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+deb-src https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
# security updates
deb http://security.debian.org/debian-security buster/updates main contrib
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-apt 04/23] add check_repositories function
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (2 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 03/23] add functions to check for Proxmox repositories Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 05/23] add common_digest helper Fabian Ebner
` (18 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
which checks for bad suites and official URIs.
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* switch from a suites_is_variant (returning bool) to a suite_variant helper
(returning the base part and variant part), which can be re-used for the
replace_suite function
src/repositories/check.rs | 151 +++++++++++++++++++++-
src/repositories/mod.rs | 19 ++-
src/types.rs | 19 +++
tests/repositories.rs | 97 +++++++++++++-
tests/sources.list.d.expected/bad.sources | 30 +++++
tests/sources.list.d/bad.sources | 29 +++++
6 files changed, 341 insertions(+), 4 deletions(-)
create mode 100644 tests/sources.list.d.expected/bad.sources
create mode 100644 tests/sources.list.d/bad.sources
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
index a682b69..d03d5f7 100644
--- a/src/repositories/check.rs
+++ b/src/repositories/check.rs
@@ -1,6 +1,22 @@
use anyhow::{bail, Error};
-use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryPackageType};
+use crate::types::{
+ APTRepository, APTRepositoryFile, APTRepositoryFileType, APTRepositoryInfo,
+ APTRepositoryPackageType,
+};
+
+/// Splits the suite into its base part and variant.
+fn suite_variant(suite: &str) -> (&str, &str) {
+ let variants = ["-backports-sloppy", "-backports", "-updates", "/updates"];
+
+ for variant in variants.iter() {
+ if let Some(base) = suite.strip_suffix(variant) {
+ return (base, variant);
+ }
+ }
+
+ (suite, "")
+}
impl APTRepository {
/// Makes sure that all basic properties of a repository are present and
@@ -102,4 +118,137 @@ impl APTRepository {
false
}
}
+
+ /// Checks if old or unstable suites are configured and also that the
+ /// `stable` keyword is not used.
+ fn check_suites(&self, add_info: &mut dyn FnMut(String, String)) {
+ let old_suites = [
+ "lenny",
+ "squeeze",
+ "wheezy",
+ "jessie",
+ "stretch",
+ "oldoldstable",
+ "oldstable",
+ ];
+
+ let next_suite = "bullseye";
+
+ let new_suites = ["testing", "unstable", "sid", "experimental"];
+
+ if self
+ .types
+ .iter()
+ .any(|package_type| *package_type == APTRepositoryPackageType::Deb)
+ {
+ for suite in self.suites.iter() {
+ if old_suites
+ .iter()
+ .any(|base_suite| suite_variant(suite).0 == *base_suite)
+ {
+ add_info(
+ "warning".to_string(),
+ format!("old suite '{}' configured!", suite),
+ );
+ }
+
+ if suite_variant(suite).0 == next_suite {
+ add_info(
+ "ignore-pre-upgrade-warning".to_string(),
+ format!("suite '{}' should not be used in production!", suite),
+ );
+ }
+
+ if new_suites
+ .iter()
+ .any(|base_suite| suite_variant(suite).0 == *base_suite)
+ {
+ add_info(
+ "warning".to_string(),
+ format!("suite '{}' should not be used in production!", suite),
+ );
+ }
+
+ if suite_variant(suite).0 == "stable" {
+ add_info(
+ "warning".to_string(),
+ "use the name of the stable distribution instead of 'stable'!".to_string(),
+ );
+ }
+ }
+ }
+ }
+
+ /// Checks if an official host is configured in the repository.
+ fn check_uris(&self) -> Option<(String, String)> {
+ let official_host = |domains: &Vec<&str>| {
+ match domains.len() {
+ 3 => match domains[0] {
+ "ftp" | "deb" | "security" => domains[1] == "debian" && domains[2] == "org",
+ "download" | "enterprise" => domains[1] == "proxmox" && domains[2] == "com",
+ _ => false,
+ },
+ // ftp.*.debian.org
+ 4 => domains[0] == "ftp" && domains[2] == "debian" && domains[3] == "org",
+ _ => false,
+ }
+ };
+
+ for uri in self.uris.iter() {
+ if let Some(begin) = uri.find("://") {
+ let mut host = uri.split_at(begin + 3).1;
+ if let Some(end) = host.find('/') {
+ host = host.split_at(end).0;
+ } // otherwise assume everything belongs is the host
+
+ let domains = host.split('.').collect();
+
+ if official_host(&domains) {
+ return Some(("badge".to_string(), "official host name".to_string()));
+ }
+ }
+ }
+
+ None
+ }
+}
+
+impl APTRepositoryFile {
+ /// Checks if old or unstable suites are configured and also that the
+ /// `stable` keyword is not used.
+ pub fn check_suites(&self) -> Vec<APTRepositoryInfo> {
+ let mut infos = vec![];
+
+ for (n, repo) in self.repositories.iter().enumerate() {
+ let mut add_info = |kind, message| {
+ infos.push(APTRepositoryInfo {
+ path: self.path.clone(),
+ number: n + 1,
+ kind,
+ message,
+ })
+ };
+ repo.check_suites(&mut add_info);
+ }
+
+ infos
+ }
+
+ /// Checks for official URIs.
+ pub fn check_uris(&self) -> Vec<APTRepositoryInfo> {
+ let mut infos = vec![];
+
+ for (n, repo) in self.repositories.iter().enumerate() {
+ if let Some((kind, message)) = repo.check_uris() {
+ infos.push(APTRepositoryInfo {
+ path: self.path.clone(),
+ number: n + 1,
+ kind,
+ message,
+ });
+ }
+ }
+
+ infos
+ }
}
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
index b7919a9..c2bbc06 100644
--- a/src/repositories/mod.rs
+++ b/src/repositories/mod.rs
@@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error};
use crate::types::{
APTRepository, APTRepositoryFile, APTRepositoryFileError, APTRepositoryFileType,
- APTRepositoryOption,
+ APTRepositoryInfo, APTRepositoryOption,
};
mod list_parser;
@@ -148,6 +148,23 @@ impl APTRepositoryFile {
}
}
+/// Provides additional information about the repositories.
+///
+/// The kind of information can be:
+/// `warnings` for bad suites.
+/// `ignore-pre-upgrade-warning` when the next stable suite is configured.
+/// `badge` for official URIs.
+pub fn check_repositories(files: &[APTRepositoryFile]) -> Vec<APTRepositoryInfo> {
+ let mut infos = vec![];
+
+ for file in files.iter() {
+ infos.append(&mut file.check_suites());
+ infos.append(&mut file.check_uris());
+ }
+
+ infos
+}
+
/// Checks if the enterprise repository for the specified Proxmox product is
/// configured and enabled.
pub fn enterprise_repository_enabled(files: &[APTRepositoryFile], product: &str) -> bool {
diff --git a/src/types.rs b/src/types.rs
index bbd8e7e..057fffa 100644
--- a/src/types.rs
+++ b/src/types.rs
@@ -244,3 +244,22 @@ impl std::error::Error for APTRepositoryFileError {
None
}
}
+
+#[api]
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+/// Additional information for a repository.
+pub struct APTRepositoryInfo {
+ /// Path to the defining file.
+ #[serde(skip_serializing_if = "String::is_empty")]
+ pub path: String,
+
+ /// Number of the associated respository within the file.
+ pub number: usize,
+
+ /// Info kind (e.g. "warning")
+ pub kind: String,
+
+ /// Info message
+ pub message: String,
+}
diff --git a/tests/repositories.rs b/tests/repositories.rs
index ffb1888..9b0cd56 100644
--- a/tests/repositories.rs
+++ b/tests/repositories.rs
@@ -3,9 +3,10 @@ use std::path::PathBuf;
use anyhow::{bail, format_err, Error};
use proxmox_apt::repositories::{
- enterprise_repository_enabled, no_subscription_repository_enabled, write_repositories,
+ check_repositories, enterprise_repository_enabled, no_subscription_repository_enabled,
+ write_repositories,
};
-use proxmox_apt::types::APTRepositoryFile;
+use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo};
#[test]
fn test_parse_write() -> Result<(), Error> {
@@ -159,3 +160,95 @@ fn test_proxmox_repositories() -> Result<(), Error> {
Ok(())
}
+
+#[test]
+fn test_check_repositories() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+
+ let absolute_suite_list = read_dir.join("absolute_suite.list");
+ let mut file = APTRepositoryFile::new(&absolute_suite_list)?.unwrap();
+ file.parse()?;
+
+ let infos = check_repositories(&vec![file]);
+
+ assert_eq!(infos.is_empty(), true);
+ let pve_list = read_dir.join("pve.list");
+ let mut file = APTRepositoryFile::new(&pve_list)?.unwrap();
+ file.parse()?;
+
+ let path_string = pve_list.into_os_string().into_string().unwrap();
+
+ let mut expected_infos = vec![];
+ for n in 1..=5 {
+ expected_infos.push(APTRepositoryInfo {
+ path: path_string.clone(),
+ number: n,
+ kind: "badge".to_string(),
+ message: "official host name".to_string(),
+ });
+ }
+
+ let mut infos = check_repositories(&vec![file]);
+
+ assert_eq!(infos.sort(), expected_infos.sort());
+
+ let bad_sources = read_dir.join("bad.sources");
+ let mut file = APTRepositoryFile::new(&bad_sources)?.unwrap();
+ file.parse()?;
+
+ let path_string = bad_sources.into_os_string().into_string().unwrap();
+
+ let mut expected_infos = vec![
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 1,
+ kind: "warning".to_string(),
+ message: "suite 'sid' should not be used in production!".to_string(),
+ },
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 2,
+ kind: "warning".to_string(),
+ message: "old suite 'lenny-backports' configured!".to_string(),
+ },
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 3,
+ kind: "warning".to_string(),
+ message: "old suite 'stretch/updates' configured!".to_string(),
+ },
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 4,
+ kind: "warning".to_string(),
+ message: "use the name of the stable distribution instead of 'stable'!".to_string(),
+ },
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 5,
+ kind: "ignore-pre-upgrade-warning".to_string(),
+ message: "suite 'bullseye' should not be used in production!".to_string(),
+ },
+ APTRepositoryInfo {
+ path: path_string.clone(),
+ number: 6,
+ kind: "warning".to_string(),
+ message: "suite 'testing' should not be used in production!".to_string(),
+ },
+ ];
+ for n in 1..=6 {
+ expected_infos.push(APTRepositoryInfo {
+ path: path_string.clone(),
+ number: n,
+ kind: "badge".to_string(),
+ message: "official URI".to_string(),
+ });
+ }
+
+ let mut infos = check_repositories(&vec![file]);
+
+ assert_eq!(infos.sort(), expected_infos.sort());
+
+ Ok(())
+}
diff --git a/tests/sources.list.d.expected/bad.sources b/tests/sources.list.d.expected/bad.sources
new file mode 100644
index 0000000..36ff7a0
--- /dev/null
+++ b/tests/sources.list.d.expected/bad.sources
@@ -0,0 +1,30 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: sid
+Components: main contrib
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: lenny-backports
+Components: contrib
+
+Types: deb
+URIs: http://security.debian.org
+Suites: stretch/updates
+Components: main contrib
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: stable
+Components: main
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye
+Components: main
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: testing
+Components: main
+
diff --git a/tests/sources.list.d/bad.sources b/tests/sources.list.d/bad.sources
new file mode 100644
index 0000000..6f2524a
--- /dev/null
+++ b/tests/sources.list.d/bad.sources
@@ -0,0 +1,29 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: sid
+Components: main contrib
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: lenny-backports
+Components: contrib
+
+Types: deb
+URIs: http://security.debian.org
+Suites: stretch/updates
+Components: main contrib
+
+Suites: stable
+URIs: http://ftp.at.debian.org/debian
+Components: main
+Types: deb
+
+Suites: bullseye
+URIs: http://ftp.at.debian.org/debian
+Components: main
+Types: deb
+
+Suites: testing
+URIs: http://ftp.at.debian.org/debian
+Components: main
+Types: deb
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-apt 05/23] add common_digest helper
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (3 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 04/23] add check_repositories function Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites Fabian Ebner
` (17 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
which allows library users to detect whether they're still dealing with the same
repository configuration or not.
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
src/repositories/mod.rs | 24 ++++++++++++++++++++++
tests/repositories.rs | 44 +++++++++++++++++++++++++++++++++++++++--
2 files changed, 66 insertions(+), 2 deletions(-)
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
index c2bbc06..2c01011 100644
--- a/src/repositories/mod.rs
+++ b/src/repositories/mod.rs
@@ -1,3 +1,4 @@
+use std::collections::BTreeMap;
use std::path::PathBuf;
use anyhow::{bail, format_err, Error};
@@ -165,6 +166,29 @@ pub fn check_repositories(files: &[APTRepositoryFile]) -> Vec<APTRepositoryInfo>
infos
}
+/// Calculates a common digest for successfully parsed repository files.
+///
+/// The digest is invariant with respect to file order.
+///
+/// Files without a digest are ignored.
+pub fn common_digest(files: &[APTRepositoryFile]) -> [u8; 32] {
+ let mut digests = BTreeMap::new();
+
+ for file in files.iter() {
+ digests.insert(file.path.clone(), &file.digest);
+ }
+
+ let mut common_raw = Vec::<u8>::with_capacity(digests.len() * 32);
+ for digest in digests.values() {
+ match digest {
+ Some(digest) => common_raw.extend_from_slice(&digest[..]),
+ None => (),
+ }
+ }
+
+ openssl::sha::sha256(&common_raw[..])
+}
+
/// Checks if the enterprise repository for the specified Proxmox product is
/// configured and enabled.
pub fn enterprise_repository_enabled(files: &[APTRepositoryFile], product: &str) -> bool {
diff --git a/tests/repositories.rs b/tests/repositories.rs
index 9b0cd56..3919077 100644
--- a/tests/repositories.rs
+++ b/tests/repositories.rs
@@ -3,8 +3,8 @@ use std::path::PathBuf;
use anyhow::{bail, format_err, Error};
use proxmox_apt::repositories::{
- check_repositories, enterprise_repository_enabled, no_subscription_repository_enabled,
- write_repositories,
+ check_repositories, common_digest, enterprise_repository_enabled,
+ no_subscription_repository_enabled, write_repositories,
};
use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo};
@@ -252,3 +252,43 @@ fn test_check_repositories() -> Result<(), Error> {
Ok(())
}
+
+#[test]
+fn test_common_digest() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+
+ let pve_list = read_dir.join("pve.list");
+ let mut pve_file = APTRepositoryFile::new(&pve_list)?.unwrap();
+ pve_file.parse()?;
+
+ let standard_sources = read_dir.join("standard.sources");
+ let mut standard_file = APTRepositoryFile::new(&standard_sources)?.unwrap();
+ standard_file.parse()?;
+
+ let expected_digest = [
+ 187, 82, 67, 137, 156, 156, 49, 84, 1, 221, 240, 24, 193, 201, 247, 252, 6, 128, 137, 241,
+ 169, 176, 78, 193, 4, 190, 117, 136, 96, 28, 46, 78,
+ ];
+
+ let mut files = vec![pve_file, standard_file];
+
+ let digest = common_digest(&files);
+ assert_eq!(digest, expected_digest);
+
+ // order should be irrelevant
+ files.reverse();
+ let digest = common_digest(&files);
+ assert_eq!(digest, expected_digest);
+
+ let digest = common_digest(&files[0..=0]);
+ assert_ne!(digest, expected_digest);
+
+ let mut file = files.first_mut().unwrap();
+ file.digest = Some(expected_digest);
+
+ let digest = common_digest(&files);
+ assert_ne!(digest, expected_digest);
+
+ Ok(())
+}
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (4 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 05/23] add common_digest helper Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-31 13:06 ` [pve-devel] [pbs-devel] " Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 07/23] add UI for APT repositories Fabian Ebner
` (16 subsequent siblings)
22 siblings, 1 reply; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
useful for major upgrades.
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
.gitignore | 1 +
src/repositories/check.rs | 2 +-
src/repositories/mod.rs | 23 ++++++
tests/repositories.rs | 73 ++++++++++++++++++-
.../absolute_suite.list | 5 ++
.../absolute_suite.sources | 5 ++
.../bad.sources | 30 ++++++++
.../case.sources | 16 ++++
.../multiline.sources | 10 +++
| 6 ++
.../pbs-enterprise.list | 2 +
.../sources.list.d.upgraded.expected/pve.list | 15 ++++
.../standard.list | 7 ++
.../standard.sources | 11 +++
14 files changed, 204 insertions(+), 2 deletions(-)
create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.list
create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.sources
create mode 100644 tests/sources.list.d.upgraded.expected/bad.sources
create mode 100644 tests/sources.list.d.upgraded.expected/case.sources
create mode 100644 tests/sources.list.d.upgraded.expected/multiline.sources
create mode 100644 tests/sources.list.d.upgraded.expected/options_comment.list
create mode 100644 tests/sources.list.d.upgraded.expected/pbs-enterprise.list
create mode 100644 tests/sources.list.d.upgraded.expected/pve.list
create mode 100644 tests/sources.list.d.upgraded.expected/standard.list
create mode 100644 tests/sources.list.d.upgraded.expected/standard.sources
diff --git a/.gitignore b/.gitignore
index db6f13e..de68da9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
Cargo.lock
target/
tests/sources.list.d.actual
+tests/sources.list.d.upgraded.actual
tests/sources.list.d.digest
proxmox-apt-*/
*proxmox-apt*.buildinfo
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
index d03d5f7..9509826 100644
--- a/src/repositories/check.rs
+++ b/src/repositories/check.rs
@@ -6,7 +6,7 @@ use crate::types::{
};
/// Splits the suite into its base part and variant.
-fn suite_variant(suite: &str) -> (&str, &str) {
+pub fn suite_variant(suite: &str) -> (&str, &str) {
let variants = ["-backports-sloppy", "-backports", "-updates", "/updates"];
for variant in variants.iter() {
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
index 2c01011..0e6c046 100644
--- a/src/repositories/mod.rs
+++ b/src/repositories/mod.rs
@@ -21,6 +21,9 @@ mod writer;
const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
+pub const STABLE_SUITE: &str = "buster";
+pub const NEXT_STABLE_SUITE: &str = "bullseye";
+
impl APTRepository {
/// Crates an empty repository.
fn new(file_type: APTRepositoryFileType) -> Self {
@@ -265,6 +268,26 @@ pub fn repositories() -> Result<(Vec<APTRepositoryFile>, Vec<APTRepositoryFileEr
Ok((files, errors))
}
+/// For enabled repositories, replaces each occurence of the `current` suite with the `new` suite,
+/// including variants (e.g. `-updates`).
+pub fn replace_suite(files: &mut [APTRepositoryFile], current: &str, new: &str) {
+ for file in files.iter_mut() {
+ for repo in file.repositories.iter_mut() {
+ if !repo.enabled {
+ continue;
+ }
+
+ for i in 0..repo.suites.len() {
+ let suite = &repo.suites[i];
+ let (base, variant) = check::suite_variant(suite);
+ if base == current {
+ repo.suites[i] = format!("{}{}", new, variant);
+ }
+ }
+ }
+ }
+}
+
/// Write the repositories for each file.
///
/// Returns an error for each file that could not be written successfully.
diff --git a/tests/repositories.rs b/tests/repositories.rs
index 3919077..e2986af 100644
--- a/tests/repositories.rs
+++ b/tests/repositories.rs
@@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error};
use proxmox_apt::repositories::{
check_repositories, common_digest, enterprise_repository_enabled,
- no_subscription_repository_enabled, write_repositories,
+ no_subscription_repository_enabled, replace_suite, write_repositories,
};
use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo};
@@ -292,3 +292,74 @@ fn test_common_digest() -> Result<(), Error> {
Ok(())
}
+
+#[test]
+fn test_replace_suite() -> Result<(), Error> {
+ let test_dir = std::env::current_dir()?.join("tests");
+ let read_dir = test_dir.join("sources.list.d");
+ let write_dir = test_dir.join("sources.list.d.upgraded.actual");
+ let expected_dir = test_dir.join("sources.list.d.upgraded.expected");
+
+ if write_dir.is_dir() {
+ std::fs::remove_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+ }
+
+ std::fs::create_dir_all(&write_dir)
+ .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+ let mut files = vec![];
+ let mut errors = vec![];
+
+ for entry in std::fs::read_dir(read_dir)? {
+ let path = entry?.path();
+
+ match APTRepositoryFile::new(&path)? {
+ Some(mut file) => match file.parse() {
+ Ok(()) => files.push(file),
+ Err(err) => errors.push(err),
+ },
+ None => bail!("unexpected None for '{:?}'", path),
+ }
+ }
+
+ assert!(errors.is_empty());
+
+ for file in files.iter_mut() {
+ let path = PathBuf::from(&file.path);
+ let new_path = write_dir.join(path.file_name().unwrap());
+ file.path = new_path.into_os_string().into_string().unwrap();
+ file.digest = None;
+ }
+
+ replace_suite(&mut files, "buster", "bullseye");
+
+ write_repositories(&files).map_err(|err| format_err!("{:?}", err))?;
+
+ let mut expected_count = 0;
+
+ for entry in std::fs::read_dir(expected_dir)? {
+ expected_count += 1;
+
+ let expected_path = entry?.path();
+ let actual_path = write_dir.join(expected_path.file_name().unwrap());
+
+ let expected_contents = std::fs::read(&expected_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
+
+ let actual_contents = std::fs::read(&actual_path)
+ .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
+
+ assert_eq!(
+ expected_contents, actual_contents,
+ "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals",
+ expected_path, actual_path
+ );
+ }
+
+ let actual_count = std::fs::read_dir(write_dir)?.count();
+
+ assert_eq!(expected_count, actual_count);
+
+ Ok(())
+}
diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.list b/tests/sources.list.d.upgraded.expected/absolute_suite.list
new file mode 100644
index 0000000..525389c
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/absolute_suite.list
@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+deb http://packages.falcot.com/ updates/
+
+deb http://packages.falcot.com/ internal/
+
diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.sources b/tests/sources.list.d.upgraded.expected/absolute_suite.sources
new file mode 100644
index 0000000..51e4d56
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/absolute_suite.sources
@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+Types: deb
+URIs: http://packages.falcot.com/
+Suites: updates/ internal/
+
diff --git a/tests/sources.list.d.upgraded.expected/bad.sources b/tests/sources.list.d.upgraded.expected/bad.sources
new file mode 100644
index 0000000..36ff7a0
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/bad.sources
@@ -0,0 +1,30 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: sid
+Components: main contrib
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: lenny-backports
+Components: contrib
+
+Types: deb
+URIs: http://security.debian.org
+Suites: stretch/updates
+Components: main contrib
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: stable
+Components: main
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye
+Components: main
+
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: testing
+Components: main
+
diff --git a/tests/sources.list.d.upgraded.expected/case.sources b/tests/sources.list.d.upgraded.expected/case.sources
new file mode 100644
index 0000000..a266594
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/case.sources
@@ -0,0 +1,16 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster-updates
+Components: main contrib
+languages: it de fr
+Enabled: false
+languages-Add: ja
+languages-Remove: de
+
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye
+Components: main contrib
+
diff --git a/tests/sources.list.d.upgraded.expected/multiline.sources b/tests/sources.list.d.upgraded.expected/multiline.sources
new file mode 100644
index 0000000..d96acea
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/multiline.sources
@@ -0,0 +1,10 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+Languages: it de fr
+Enabled: false
+Languages-Add: ja
+Languages-Remove: de
+
--git a/tests/sources.list.d.upgraded.expected/options_comment.list b/tests/sources.list.d.upgraded.expected/options_comment.list
new file mode 100644
index 0000000..caef5e0
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/options_comment.list
@@ -0,0 +1,6 @@
+# comment
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian bullseye main contrib
+
+# non-free :(
+deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian bullseye non-free
+
diff --git a/tests/sources.list.d.upgraded.expected/pbs-enterprise.list b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list
new file mode 100644
index 0000000..cb6e779
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list
@@ -0,0 +1,2 @@
+deb https://enterprise.proxmox.com/debian/pbs bullseye pbs-enterprise
+
diff --git a/tests/sources.list.d.upgraded.expected/pve.list b/tests/sources.list.d.upgraded.expected/pve.list
new file mode 100644
index 0000000..1b5b248
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/pve.list
@@ -0,0 +1,15 @@
+deb http://ftp.debian.org/debian bullseye main contrib
+
+deb http://ftp.debian.org/debian bullseye-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve bullseye pve-no-subscription
+
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+deb-src https://enterprise.proxmox.com/debian/pve bullseye pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security bullseye/updates main contrib
+
diff --git a/tests/sources.list.d.upgraded.expected/standard.list b/tests/sources.list.d.upgraded.expected/standard.list
new file mode 100644
index 0000000..25654f1
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/standard.list
@@ -0,0 +1,7 @@
+deb http://ftp.at.debian.org/debian bullseye main contrib
+
+deb http://ftp.at.debian.org/debian bullseye-updates main contrib
+
+# security updates
+deb http://security.debian.org bullseye/updates main contrib
+
diff --git a/tests/sources.list.d.upgraded.expected/standard.sources b/tests/sources.list.d.upgraded.expected/standard.sources
new file mode 100644
index 0000000..f8e8620
--- /dev/null
+++ b/tests/sources.list.d.upgraded.expected/standard.sources
@@ -0,0 +1,11 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye bullseye-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: bullseye/updates
+Components: main contrib
+
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [pve-devel] [pbs-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites Fabian Ebner
@ 2021-05-31 13:06 ` Fabian Ebner
0 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-31 13:06 UTC (permalink / raw)
To: pbs-devel, PVE development discussion
For the security repository, "buster/updates" needs to be replaced with
"bullseye-security", so this is not enough. I'll add a special case for
that in the next version.
Am 28.05.21 um 16:29 schrieb Fabian Ebner:
> useful for major upgrades.
>
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>
> New in v5.
>
> .gitignore | 1 +
> src/repositories/check.rs | 2 +-
> src/repositories/mod.rs | 23 ++++++
> tests/repositories.rs | 73 ++++++++++++++++++-
> .../absolute_suite.list | 5 ++
> .../absolute_suite.sources | 5 ++
> .../bad.sources | 30 ++++++++
> .../case.sources | 16 ++++
> .../multiline.sources | 10 +++
> .../options_comment.list | 6 ++
> .../pbs-enterprise.list | 2 +
> .../sources.list.d.upgraded.expected/pve.list | 15 ++++
> .../standard.list | 7 ++
> .../standard.sources | 11 +++
> 14 files changed, 204 insertions(+), 2 deletions(-)
> create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.list
> create mode 100644 tests/sources.list.d.upgraded.expected/absolute_suite.sources
> create mode 100644 tests/sources.list.d.upgraded.expected/bad.sources
> create mode 100644 tests/sources.list.d.upgraded.expected/case.sources
> create mode 100644 tests/sources.list.d.upgraded.expected/multiline.sources
> create mode 100644 tests/sources.list.d.upgraded.expected/options_comment.list
> create mode 100644 tests/sources.list.d.upgraded.expected/pbs-enterprise.list
> create mode 100644 tests/sources.list.d.upgraded.expected/pve.list
> create mode 100644 tests/sources.list.d.upgraded.expected/standard.list
> create mode 100644 tests/sources.list.d.upgraded.expected/standard.sources
>
> diff --git a/.gitignore b/.gitignore
> index db6f13e..de68da9 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -1,6 +1,7 @@
> Cargo.lock
> target/
> tests/sources.list.d.actual
> +tests/sources.list.d.upgraded.actual
> tests/sources.list.d.digest
> proxmox-apt-*/
> *proxmox-apt*.buildinfo
> diff --git a/src/repositories/check.rs b/src/repositories/check.rs
> index d03d5f7..9509826 100644
> --- a/src/repositories/check.rs
> +++ b/src/repositories/check.rs
> @@ -6,7 +6,7 @@ use crate::types::{
> };
>
> /// Splits the suite into its base part and variant.
> -fn suite_variant(suite: &str) -> (&str, &str) {
> +pub fn suite_variant(suite: &str) -> (&str, &str) {
> let variants = ["-backports-sloppy", "-backports", "-updates", "/updates"];
>
> for variant in variants.iter() {
> diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
> index 2c01011..0e6c046 100644
> --- a/src/repositories/mod.rs
> +++ b/src/repositories/mod.rs
> @@ -21,6 +21,9 @@ mod writer;
> const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
> const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
>
> +pub const STABLE_SUITE: &str = "buster";
> +pub const NEXT_STABLE_SUITE: &str = "bullseye";
> +
> impl APTRepository {
> /// Crates an empty repository.
> fn new(file_type: APTRepositoryFileType) -> Self {
> @@ -265,6 +268,26 @@ pub fn repositories() -> Result<(Vec<APTRepositoryFile>, Vec<APTRepositoryFileEr
> Ok((files, errors))
> }
>
> +/// For enabled repositories, replaces each occurence of the `current` suite with the `new` suite,
> +/// including variants (e.g. `-updates`).
> +pub fn replace_suite(files: &mut [APTRepositoryFile], current: &str, new: &str) {
> + for file in files.iter_mut() {
> + for repo in file.repositories.iter_mut() {
> + if !repo.enabled {
> + continue;
> + }
> +
> + for i in 0..repo.suites.len() {
> + let suite = &repo.suites[i];
> + let (base, variant) = check::suite_variant(suite);
> + if base == current {
> + repo.suites[i] = format!("{}{}", new, variant);
> + }
> + }
> + }
> + }
> +}
> +
> /// Write the repositories for each file.
> ///
> /// Returns an error for each file that could not be written successfully.
> diff --git a/tests/repositories.rs b/tests/repositories.rs
> index 3919077..e2986af 100644
> --- a/tests/repositories.rs
> +++ b/tests/repositories.rs
> @@ -4,7 +4,7 @@ use anyhow::{bail, format_err, Error};
>
> use proxmox_apt::repositories::{
> check_repositories, common_digest, enterprise_repository_enabled,
> - no_subscription_repository_enabled, write_repositories,
> + no_subscription_repository_enabled, replace_suite, write_repositories,
> };
> use proxmox_apt::types::{APTRepositoryFile, APTRepositoryInfo};
>
> @@ -292,3 +292,74 @@ fn test_common_digest() -> Result<(), Error> {
>
> Ok(())
> }
> +
> +#[test]
> +fn test_replace_suite() -> Result<(), Error> {
> + let test_dir = std::env::current_dir()?.join("tests");
> + let read_dir = test_dir.join("sources.list.d");
> + let write_dir = test_dir.join("sources.list.d.upgraded.actual");
> + let expected_dir = test_dir.join("sources.list.d.upgraded.expected");
> +
> + if write_dir.is_dir() {
> + std::fs::remove_dir_all(&write_dir)
> + .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
> + }
> +
> + std::fs::create_dir_all(&write_dir)
> + .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
> +
> + let mut files = vec![];
> + let mut errors = vec![];
> +
> + for entry in std::fs::read_dir(read_dir)? {
> + let path = entry?.path();
> +
> + match APTRepositoryFile::new(&path)? {
> + Some(mut file) => match file.parse() {
> + Ok(()) => files.push(file),
> + Err(err) => errors.push(err),
> + },
> + None => bail!("unexpected None for '{:?}'", path),
> + }
> + }
> +
> + assert!(errors.is_empty());
> +
> + for file in files.iter_mut() {
> + let path = PathBuf::from(&file.path);
> + let new_path = write_dir.join(path.file_name().unwrap());
> + file.path = new_path.into_os_string().into_string().unwrap();
> + file.digest = None;
> + }
> +
> + replace_suite(&mut files, "buster", "bullseye");
> +
> + write_repositories(&files).map_err(|err| format_err!("{:?}", err))?;
> +
> + let mut expected_count = 0;
> +
> + for entry in std::fs::read_dir(expected_dir)? {
> + expected_count += 1;
> +
> + let expected_path = entry?.path();
> + let actual_path = write_dir.join(expected_path.file_name().unwrap());
> +
> + let expected_contents = std::fs::read(&expected_path)
> + .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
> +
> + let actual_contents = std::fs::read(&actual_path)
> + .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
> +
> + assert_eq!(
> + expected_contents, actual_contents,
> + "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals",
> + expected_path, actual_path
> + );
> + }
> +
> + let actual_count = std::fs::read_dir(write_dir)?.count();
> +
> + assert_eq!(expected_count, actual_count);
> +
> + Ok(())
> +}
> diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.list b/tests/sources.list.d.upgraded.expected/absolute_suite.list
> new file mode 100644
> index 0000000..525389c
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/absolute_suite.list
> @@ -0,0 +1,5 @@
> +# From Debian Administrator's Handbook
> +deb http://packages.falcot.com/ updates/
> +
> +deb http://packages.falcot.com/ internal/
> +
> diff --git a/tests/sources.list.d.upgraded.expected/absolute_suite.sources b/tests/sources.list.d.upgraded.expected/absolute_suite.sources
> new file mode 100644
> index 0000000..51e4d56
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/absolute_suite.sources
> @@ -0,0 +1,5 @@
> +# From Debian Administrator's Handbook
> +Types: deb
> +URIs: http://packages.falcot.com/
> +Suites: updates/ internal/
> +
> diff --git a/tests/sources.list.d.upgraded.expected/bad.sources b/tests/sources.list.d.upgraded.expected/bad.sources
> new file mode 100644
> index 0000000..36ff7a0
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/bad.sources
> @@ -0,0 +1,30 @@
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: sid
> +Components: main contrib
> +
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: lenny-backports
> +Components: contrib
> +
> +Types: deb
> +URIs: http://security.debian.org
> +Suites: stretch/updates
> +Components: main contrib
> +
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: stable
> +Components: main
> +
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: bullseye
> +Components: main
> +
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: testing
> +Components: main
> +
> diff --git a/tests/sources.list.d.upgraded.expected/case.sources b/tests/sources.list.d.upgraded.expected/case.sources
> new file mode 100644
> index 0000000..a266594
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/case.sources
> @@ -0,0 +1,16 @@
> +# comment in here
> +Types: deb deb-src
> +URIs: http://ftp.at.debian.org/debian
> +Suites: buster-updates
> +Components: main contrib
> +languages: it de fr
> +Enabled: false
> +languages-Add: ja
> +languages-Remove: de
> +
> +# comment in here
> +Types: deb deb-src
> +URIs: http://ftp.at.debian.org/debian
> +Suites: bullseye
> +Components: main contrib
> +
> diff --git a/tests/sources.list.d.upgraded.expected/multiline.sources b/tests/sources.list.d.upgraded.expected/multiline.sources
> new file mode 100644
> index 0000000..d96acea
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/multiline.sources
> @@ -0,0 +1,10 @@
> +# comment in here
> +Types: deb deb-src
> +URIs: http://ftp.at.debian.org/debian
> +Suites: buster buster-updates
> +Components: main contrib
> +Languages: it de fr
> +Enabled: false
> +Languages-Add: ja
> +Languages-Remove: de
> +
> diff --git a/tests/sources.list.d.upgraded.expected/options_comment.list b/tests/sources.list.d.upgraded.expected/options_comment.list
> new file mode 100644
> index 0000000..caef5e0
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/options_comment.list
> @@ -0,0 +1,6 @@
> +# comment
> +deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian bullseye main contrib
> +
> +# non-free :(
> +deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian bullseye non-free
> +
> diff --git a/tests/sources.list.d.upgraded.expected/pbs-enterprise.list b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list
> new file mode 100644
> index 0000000..cb6e779
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/pbs-enterprise.list
> @@ -0,0 +1,2 @@
> +deb https://enterprise.proxmox.com/debian/pbs bullseye pbs-enterprise
> +
> diff --git a/tests/sources.list.d.upgraded.expected/pve.list b/tests/sources.list.d.upgraded.expected/pve.list
> new file mode 100644
> index 0000000..1b5b248
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/pve.list
> @@ -0,0 +1,15 @@
> +deb http://ftp.debian.org/debian bullseye main contrib
> +
> +deb http://ftp.debian.org/debian bullseye-updates main contrib
> +
> +# PVE pve-no-subscription repository provided by proxmox.com,
> +# NOT recommended for production use
> +deb http://download.proxmox.com/debian/pve bullseye pve-no-subscription
> +
> +# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
> +
> +deb-src https://enterprise.proxmox.com/debian/pve bullseye pve-enterprise
> +
> +# security updates
> +deb http://security.debian.org/debian-security bullseye/updates main contrib
> +
> diff --git a/tests/sources.list.d.upgraded.expected/standard.list b/tests/sources.list.d.upgraded.expected/standard.list
> new file mode 100644
> index 0000000..25654f1
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/standard.list
> @@ -0,0 +1,7 @@
> +deb http://ftp.at.debian.org/debian bullseye main contrib
> +
> +deb http://ftp.at.debian.org/debian bullseye-updates main contrib
> +
> +# security updates
> +deb http://security.debian.org bullseye/updates main contrib
> +
> diff --git a/tests/sources.list.d.upgraded.expected/standard.sources b/tests/sources.list.d.upgraded.expected/standard.sources
> new file mode 100644
> index 0000000..f8e8620
> --- /dev/null
> +++ b/tests/sources.list.d.upgraded.expected/standard.sources
> @@ -0,0 +1,11 @@
> +Types: deb
> +URIs: http://ftp.at.debian.org/debian
> +Suites: bullseye bullseye-updates
> +Components: main contrib
> +
> +# security updates
> +Types: deb
> +URIs: http://security.debian.org
> +Suites: bullseye/updates
> +Components: main contrib
> +
>
^ permalink raw reply [flat|nested] 31+ messages in thread
* [pve-devel] [PATCH v5 proxmox-widget-toolkit 07/23] add UI for APT repositories
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (5 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-apt 06/23] add replace_suite function and constants for the current/next stable suites Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 08/23] APT repositories: add warnings Fabian Ebner
` (15 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* move code from activation listener into a reload() helper to be re-useable
* add a reload button
src/Makefile | 1 +
src/node/APTRepositories.js | 268 ++++++++++++++++++++++++++++++++++++
2 files changed, 269 insertions(+)
create mode 100644 src/node/APTRepositories.js
diff --git a/src/Makefile b/src/Makefile
index 9e3ad4e..9c00b98 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -67,6 +67,7 @@ JSSRC= \
window/ACMEDomains.js \
window/FileBrowser.js \
node/APT.js \
+ node/APTRepositories.js \
node/NetworkEdit.js \
node/NetworkView.js \
node/DNSEdit.js \
diff --git a/src/node/APTRepositories.js b/src/node/APTRepositories.js
new file mode 100644
index 0000000..71b141c
--- /dev/null
+++ b/src/node/APTRepositories.js
@@ -0,0 +1,268 @@
+Ext.define('apt-repolist', {
+ extend: 'Ext.data.Model',
+ fields: [
+ 'Path',
+ 'Number',
+ 'FileType',
+ 'Enabled',
+ 'Comment',
+ 'Types',
+ 'URIs',
+ 'Suites',
+ 'Components',
+ 'Options',
+ ],
+});
+
+Ext.define('Proxmox.node.APTRepositoriesErrors', {
+ extend: 'Ext.grid.GridPanel',
+
+ xtype: 'proxmoxNodeAPTRepositoriesErrors',
+
+ title: gettext('Errors'),
+
+ store: {},
+
+ viewConfig: {
+ stripeRows: false,
+ getRowClass: () => 'proxmox-invalid-row',
+ },
+
+ columns: [
+ {
+ header: gettext('File'),
+ dataIndex: 'path',
+ renderer: function(value, cell, record) {
+ return "<i class='pve-grid-fa fa fa-fw " +
+ "fa-exclamation-triangle'></i>" + value;
+ },
+ width: 350,
+ },
+ {
+ header: gettext('Error'),
+ dataIndex: 'error',
+ flex: 1,
+ },
+ ],
+});
+
+Ext.define('Proxmox.node.APTRepositoriesGrid', {
+ extend: 'Ext.grid.GridPanel',
+
+ xtype: 'proxmoxNodeAPTRepositoriesGrid',
+
+ title: gettext('APT Repositories'),
+
+ sortableColumns: false,
+
+ columns: [
+ {
+ header: gettext('Enabled'),
+ dataIndex: 'Enabled',
+ renderer: Proxmox.Utils.format_enabled_toggle,
+ width: 90,
+ },
+ {
+ header: gettext('Types'),
+ dataIndex: 'Types',
+ renderer: function(types, cell, record) {
+ return types.join(' ');
+ },
+ width: 100,
+ },
+ {
+ header: gettext('URIs'),
+ dataIndex: 'URIs',
+ renderer: function(uris, cell, record) {
+ return uris.join(' ');
+ },
+ width: 350,
+ },
+ {
+ header: gettext('Suites'),
+ dataIndex: 'Suites',
+ renderer: function(suites, cell, record) {
+ return suites.join(' ');
+ },
+ width: 130,
+ },
+ {
+ header: gettext('Components'),
+ dataIndex: 'Components',
+ renderer: function(components, cell, record) {
+ return components.join(' ');
+ },
+ width: 170,
+ },
+ {
+ header: gettext('Options'),
+ dataIndex: 'Options',
+ renderer: function(options, cell, record) {
+ if (!options) {
+ return '';
+ }
+
+ let filetype = record.data.FileType;
+ let text = '';
+
+ options.forEach(function(option) {
+ let key = option.Key;
+ if (filetype === 'list') {
+ let values = option.Values.join(',');
+ text += `${key}=${values} `;
+ } else if (filetype === 'sources') {
+ let values = option.Values.join(' ');
+ text += `${key}: ${values}<br>`;
+ } else {
+ throw "unkown file type";
+ }
+ });
+ return text;
+ },
+ flex: 1,
+ },
+ {
+ header: gettext('Comment'),
+ dataIndex: 'Comment',
+ flex: 2,
+ },
+ ],
+
+ initComponent: function() {
+ let me = this;
+
+ let store = Ext.create('Ext.data.Store', {
+ model: 'apt-repolist',
+ groupField: 'Path',
+ sorters: [
+ {
+ property: 'Number',
+ direction: 'ASC',
+ },
+ ],
+ });
+
+ let groupingFeature = Ext.create('Ext.grid.feature.Grouping', {
+ groupHeaderTpl: '{[ "File: " + values.name ]} ({rows.length} ' +
+ 'repositor{[values.rows.length > 1 ? "ies" : "y"]})',
+ enableGroupingMenu: false,
+ });
+
+ let sm = Ext.create('Ext.selection.RowModel', {});
+
+ Ext.apply(me, {
+ store: store,
+ selModel: sm,
+ features: [groupingFeature],
+ });
+
+ me.callParent();
+ },
+});
+
+Ext.define('Proxmox.node.APTRepositories', {
+ extend: 'Ext.panel.Panel',
+
+ xtype: 'proxmoxNodeAPTRepositories',
+
+ digest: undefined,
+
+ viewModel: {
+ data: {
+ errorCount: 0,
+ },
+ formulas: {
+ noErrors: (get) => get('errorCount') === 0,
+ },
+ },
+
+ items: [
+ {
+ xtype: 'proxmoxNodeAPTRepositoriesErrors',
+ name: 'repositoriesErrors',
+ hidden: true,
+ bind: {
+ hidden: '{noErrors}',
+ },
+ },
+ {
+ xtype: 'proxmoxNodeAPTRepositoriesGrid',
+ name: 'repositoriesGrid',
+ },
+ ],
+
+ tbar: [
+ {
+ xtype: 'proxmoxButton',
+ text: gettext('Reload'),
+ handler: function() {
+ let me = this;
+ me.up('proxmoxNodeAPTRepositories').reload();
+ },
+ },
+ ],
+
+ reload: function() {
+ let me = this;
+ let vm = me.getViewModel();
+ let repoGrid = me.down('proxmoxNodeAPTRepositoriesGrid');
+ let errorGrid = me.down('proxmoxNodeAPTRepositoriesErrors');
+
+ me.store.load(function(records, operation, success) {
+ let gridData = [];
+ let errors = [];
+ let digest;
+
+ if (success && records.length > 0) {
+ let data = records[0].data;
+ let files = data.files;
+ errors = data.errors;
+ digest = data.digest;
+
+ files.forEach(function(file) {
+ for (let n = 0; n < file.repositories.length; n++) {
+ let repo = file.repositories[n];
+ repo.Path = file.path;
+ repo.Number = n + 1;
+ gridData.push(repo);
+ }
+ });
+ }
+
+ me.digest = digest;
+
+ repoGrid.store.loadData(gridData);
+
+ vm.set('errorCount', errors.length);
+ errorGrid.store.loadData(errors);
+ });
+ },
+
+ listeners: {
+ activate: function() {
+ let me = this;
+ me.reload();
+ },
+ },
+
+ initComponent: function() {
+ let me = this;
+
+ if (!me.nodename) {
+ throw "no node name specified";
+ }
+
+ let store = Ext.create('Ext.data.Store', {
+ proxy: {
+ type: 'proxmox',
+ url: `/api2/json/nodes/${me.nodename}/apt/repositories`,
+ },
+ });
+
+ Ext.apply(me, { store: store });
+
+ Proxmox.Utils.monStoreErrors(me, me.store, true);
+
+ me.callParent();
+ },
+});
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-widget-toolkit 08/23] APT repositories: add warnings
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (6 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 07/23] add UI for APT repositories Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 09/23] add upgrade button Fabian Ebner
` (14 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Depends on the respective PBS/PVE patch for the checkrepositories call.
Otherwise there's an error message that the call is not implemented, but the
repository window should still work.
Changes from v4:
* Indentation fix.
* rebased on previous change
* set parameters for checkrepositories only if digest is not undefined
src/node/APTRepositories.js | 170 +++++++++++++++++++++++++++++++++++-
1 file changed, 168 insertions(+), 2 deletions(-)
diff --git a/src/node/APTRepositories.js b/src/node/APTRepositories.js
index 71b141c..5f8b2e5 100644
--- a/src/node/APTRepositories.js
+++ b/src/node/APTRepositories.js
@@ -3,6 +3,7 @@ Ext.define('apt-repolist', {
fields: [
'Path',
'Number',
+ 'OfficialHost',
'FileType',
'Enabled',
'Comment',
@@ -56,6 +57,22 @@ Ext.define('Proxmox.node.APTRepositoriesGrid', {
sortableColumns: false,
columns: [
+ {
+ header: gettext('Official'),
+ dataIndex: 'OfficialHost',
+ renderer: function(value, cell, record) {
+ let icon = (cls) => `<i class="fa fa-fw ${cls}"></i>`;
+
+ if (value === undefined || value === null) {
+ return icon('fa-question-circle-o');
+ }
+ if (!value) {
+ return icon('fa-times critical');
+ }
+ return icon('fa-check good');
+ },
+ width: 70,
+ },
{
header: gettext('Enabled'),
dataIndex: 'Enabled',
@@ -128,9 +145,90 @@ Ext.define('Proxmox.node.APTRepositoriesGrid', {
},
],
+ check_repositories: function(gridData) {
+ let me = this;
+ let panel = me.up('proxmoxNodeAPTRepositories');
+ let vm = panel.getViewModel();
+
+ let params = {};
+ if (panel.digest !== undefined) {
+ params.digest = panel.digest;
+ }
+
+ Proxmox.Utils.API2Request({
+ url: `/nodes/${me.nodename}/apt/checkrepositories`,
+ method: 'GET',
+ params: params,
+ failure: function(response, opts) {
+ me.rowBodyFeature.getAdditionalData = function() {
+ return {
+ rowBody: undefined,
+ rowBodyCls: Ext.baseCSSPrefix + 'grid-row-body-hidden',
+ };
+ };
+ me.store.loadData(gridData);
+ Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+ },
+ success: function(response, opts) {
+ const data = response.result.data;
+
+ vm.set('enterpriseRepo', data.enterprise);
+ vm.set('noSubscriptionRepo', data.nosubscription);
+
+ let warnings = {};
+ let officialHosts = {};
+
+ let addLine = function(obj, key, line) {
+ if (obj[key]) {
+ obj[key] += "\n";
+ obj[key] += line;
+ } else {
+ obj[key] = line;
+ }
+ };
+
+ for (const info of data.infos) {
+ const key = `${info.path}:${info.number}`;
+ if (info.kind === 'warning' ||
+ (info.kind === 'ignore-pre-upgrade-warning' && !me.majorUpgradeAllowed)) {
+ addLine(warnings, key, gettext('Warning') + ": " + info.message);
+ } else if (info.kind === 'badge' && info.message === 'official host name') {
+ officialHosts[key] = true;
+ }
+ }
+
+ gridData.forEach(function(record) {
+ const key = `${record.Path}:${record.Number}`;
+ record.OfficialHost = !!officialHosts[key];
+ });
+
+ me.rowBodyFeature.getAdditionalData = function(innerData, rowIndex, record, orig) {
+ let headerCt = this.view.headerCt;
+ let colspan = headerCt.getColumnCount();
+
+ const key = `${innerData.Path}:${innerData.Number}`;
+ const warning_text = warnings[key];
+
+ return {
+ rowBody: '<div style="color: red; white-space: pre-line">' +
+ Ext.String.htmlEncode(warning_text) + '</div>',
+ rowBodyCls: warning_text ? '' : Ext.baseCSSPrefix + 'grid-row-body-hidden',
+ rowBodyColspan: colspan,
+ };
+ };
+
+ me.store.loadData(gridData);
+ },
+ });
+ },
+
initComponent: function() {
let me = this;
+ if (!me.nodename) {
+ throw "no node name specified";
+ }
+
let store = Ext.create('Ext.data.Store', {
model: 'apt-repolist',
groupField: 'Path',
@@ -142,6 +240,8 @@ Ext.define('Proxmox.node.APTRepositoriesGrid', {
],
});
+ let rowBodyFeature = Ext.create('Ext.grid.feature.RowBody', {});
+
let groupingFeature = Ext.create('Ext.grid.feature.Grouping', {
groupHeaderTpl: '{[ "File: " + values.name ]} ({rows.length} ' +
'repositor{[values.rows.length > 1 ? "ies" : "y"]})',
@@ -153,7 +253,8 @@ Ext.define('Proxmox.node.APTRepositoriesGrid', {
Ext.apply(me, {
store: store,
selModel: sm,
- features: [groupingFeature],
+ rowBodyFeature: rowBodyFeature,
+ features: [groupingFeature, rowBodyFeature],
});
me.callParent();
@@ -164,19 +265,59 @@ Ext.define('Proxmox.node.APTRepositories', {
extend: 'Ext.panel.Panel',
xtype: 'proxmoxNodeAPTRepositories',
+ mixins: ['Proxmox.Mixin.CBind'],
digest: undefined,
viewModel: {
data: {
errorCount: 0,
+ subscriptionActive: '',
+ noSubscriptionRepo: '',
+ enterpriseRepo: '',
},
formulas: {
noErrors: (get) => get('errorCount') === 0,
+ mainWarning: function(get) {
+ if (get('subscriptionActive') === '' ||
+ get('noSubscriptionRepo') === '' ||
+ get('enterpriseRepo') === '') {
+ return '';
+ }
+
+ let withStyle = (msg) => "<div style='color:red;'><i class='fa fa-fw " +
+ "fa-exclamation-triangle'></i>" + gettext('Warning') + ': ' + msg + "</div>";
+
+ if (!get('subscriptionActive') && get('enterpriseRepo')) {
+ return withStyle(gettext('The enterprise repository is ' +
+ 'configured, but there is no active subscription!'));
+ }
+
+ if (get('noSubscriptionRepo')) {
+ return withStyle(gettext('The no-subscription repository is ' +
+ 'not recommended for production use!'));
+ }
+
+ if (!get('enterpriseRepo') && !get('noSubscriptionRepo')) {
+ return withStyle(gettext('Neither the enterprise repository ' +
+ 'nor the no-subscription repository is configured!'));
+ }
+
+ return '';
+ },
},
},
items: [
+ {
+ title: gettext('Warning'),
+ name: 'repositoriesMainWarning',
+ xtype: 'panel',
+ bind: {
+ title: '{mainWarning}',
+ hidden: '{!mainWarning}',
+ },
+ },
{
xtype: 'proxmoxNodeAPTRepositoriesErrors',
name: 'repositoriesErrors',
@@ -188,6 +329,10 @@ Ext.define('Proxmox.node.APTRepositories', {
{
xtype: 'proxmoxNodeAPTRepositoriesGrid',
name: 'repositoriesGrid',
+ cbind: {
+ nodename: '{nodename}',
+ majorUpgradeAllowed: '{majorUpgradeAllowed}',
+ },
},
],
@@ -202,6 +347,25 @@ Ext.define('Proxmox.node.APTRepositories', {
},
],
+ check_subscription: function() {
+ let me = this;
+ let vm = me.getViewModel();
+
+ Proxmox.Utils.API2Request({
+ url: `/nodes/${me.nodename}/subscription`,
+ method: 'GET',
+ failure: function(response, opts) {
+ Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+ },
+ success: function(response, opts) {
+ const res = response.result;
+ const subscription = !(res === null || res === undefined ||
+ !res || res.data.status.toLowerCase() !== 'active');
+ vm.set('subscriptionActive', subscription);
+ },
+ });
+ },
+
reload: function() {
let me = this;
let vm = me.getViewModel();
@@ -231,11 +395,13 @@ Ext.define('Proxmox.node.APTRepositories', {
me.digest = digest;
- repoGrid.store.loadData(gridData);
+ repoGrid.check_repositories(gridData); // loads gridData after updating it
vm.set('errorCount', errors.length);
errorGrid.store.loadData(errors);
});
+
+ me.check_subscription();
},
listeners: {
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-widget-toolkit 09/23] add upgrade button
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (7 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 08/23] APT repositories: add warnings Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 10/23] depend on new proxmox-apt crate Fabian Ebner
` (13 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
No dependency problem, because of the majorUpgradeAllowed option, which
needs to specified by the user
New in v5.
src/node/APTRepositories.js | 31 +++++++++++++++++++++++++++++++
1 file changed, 31 insertions(+)
diff --git a/src/node/APTRepositories.js b/src/node/APTRepositories.js
index 5f8b2e5..7490bbf 100644
--- a/src/node/APTRepositories.js
+++ b/src/node/APTRepositories.js
@@ -345,6 +345,37 @@ Ext.define('Proxmox.node.APTRepositories', {
me.up('proxmoxNodeAPTRepositories').reload();
},
},
+ {
+ xtype: 'proxmoxButton',
+ text: gettext('Upgrade Package Distribution'),
+ cbind: {
+ nodename: '{nodename}',
+ hidden: '{!majorUpgradeAllowed}',
+ },
+ confirmMsg: gettext('Upgrade from \'buster\' to \'bullseye\' for enabled repositories'),
+ handler: function() {
+ let me = this;
+ let panel = me.up('proxmoxNodeAPTRepositories');
+
+ let params = {};
+ if (panel.digest !== undefined) {
+ params.digest = panel.digest;
+ }
+
+ Proxmox.Utils.API2Request({
+ url: `/nodes/${me.nodename}/apt/upgraderepositories`,
+ method: 'PUT',
+ params: params,
+ failure: function(response, opts) {
+ Ext.Msg.alert(gettext('Error'), response.htmlStatus);
+ panel.reload();
+ },
+ success: function(response, opts) {
+ panel.reload();
+ },
+ });
+ },
+ },
],
check_subscription: function() {
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-backup 10/23] depend on new proxmox-apt crate
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (8 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-widget-toolkit 09/23] add upgrade button Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 11/23] api: apt: add repositories call Fabian Ebner
` (12 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
No changes from v4.
Cargo.toml | 1 +
debian/control | 1 +
2 files changed, 2 insertions(+)
diff --git a/Cargo.toml b/Cargo.toml
index 6ce778dd..31592467 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -56,6 +56,7 @@ pathpatterns = "0.1.2"
proxmox = { version = "0.11.5", features = [ "sortable-macro", "api-macro" ] }
#proxmox = { git = "git://git.proxmox.com/git/proxmox", version = "0.1.2", features = [ "sortable-macro", "api-macro" ] }
#proxmox = { path = "../proxmox/proxmox", features = [ "sortable-macro", "api-macro" ] }
+proxmox-apt = "0.1.0"
proxmox-fuse = "0.1.1"
proxmox-http = { version = "0.2.1", features = [ "client", "http-helpers", "websocket" ] }
#proxmox-http = { version = "0.2.0", path = "../proxmox/proxmox-http", features = [ "client", "http-helpers", "websocket" ] }
diff --git a/debian/control b/debian/control
index 468dd4e1..4946e06d 100644
--- a/debian/control
+++ b/debian/control
@@ -43,6 +43,7 @@ Build-Depends: debhelper (>= 11),
librust-proxmox-0.11+default-dev (>= 0.11.5-~~),
librust-proxmox-0.11+sortable-macro-dev (>= 0.11.5-~~),
librust-proxmox-acme-rs-0.2+default-dev (>= 0.2.1-~~),
+ librust-proxmox-apt-0.1+default-dev,
librust-proxmox-fuse-0.1+default-dev (>= 0.1.1-~~),
librust-proxmox-http-0.2+client-dev (>= 0.2.1-~~),
librust-proxmox-http-0.2+default-dev (>= 0.2.1-~~),
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-backup 11/23] api: apt: add repositories call
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (9 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 10/23] depend on new proxmox-apt crate Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 12/23] ui: add APT repositories Fabian Ebner
` (11 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* moved common_digest helper to proxmox-apt library
src/api2/node/apt.rs | 58 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/api2/node/apt.rs b/src/api2/node/apt.rs
index 35c9a79a..0fcdce9c 100644
--- a/src/api2/node/apt.rs
+++ b/src/api2/node/apt.rs
@@ -7,6 +7,7 @@ use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
use proxmox::api::router::{Router, SubdirMap};
use proxmox::tools::fs::{replace_file, CreateOptions};
+use proxmox_apt::types::{APTRepositoryFile, APTRepositoryFileError};
use proxmox_http::ProxyConfig;
use crate::config::node;
@@ -17,7 +18,7 @@ use crate::tools::{
subscription,
};
use crate::config::acl::{PRIV_SYS_AUDIT, PRIV_SYS_MODIFY};
-use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, UPID_SCHEMA};
+use crate::api2::types::{Authid, APTUpdateInfo, NODE_SCHEMA, PROXMOX_CONFIG_DIGEST_SCHEMA, UPID_SCHEMA};
#[api(
input: {
@@ -389,8 +390,63 @@ pub fn get_versions() -> Result<Vec<APTUpdateInfo>, Error> {
Ok(packages)
}
+#[api(
+ input: {
+ properties: {
+ node: {
+ schema: NODE_SCHEMA,
+ },
+ },
+ },
+ returns: {
+ type: Object,
+ description: "Result from parsing the APT repository files in /etc/apt/.",
+ properties: {
+ files: {
+ description: "List of parsed repository files.",
+ type: Array,
+ items: {
+ type: APTRepositoryFile,
+ }
+ },
+ errors: {
+ description: "List of problematic files.",
+ type: Array,
+ items: {
+ type: APTRepositoryFileError,
+ }
+ },
+ digest: {
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+ },
+)]
+/// Get APT repository information
+pub fn get_repositories() -> Result<Value, Error> {
+ let (files, errors) = proxmox_apt::repositories::repositories()?;
+
+ if files.len() == 0 {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ let common_digest = proxmox_apt::repositories::common_digest(&files);
+
+ let hex_digest = proxmox::tools::digest_to_hex(&common_digest);
+
+ Ok(json!({
+ "files": files,
+ "errors": errors,
+ "digest": hex_digest,
+ }))
+}
+
const SUBDIRS: SubdirMap = &[
("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
+ ("repositories", &Router::new().get(&API_METHOD_GET_REPOSITORIES)),
("update", &Router::new()
.get(&API_METHOD_APT_UPDATE_AVAILABLE)
.post(&API_METHOD_APT_UPDATE_DATABASE)
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-backup 12/23] ui: add APT repositories
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (10 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 11/23] api: apt: add repositories call Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 13/23] add check_repositories_call Fabian Ebner
` (10 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
No changes from v4.
www/ServerAdministration.js | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/www/ServerAdministration.js b/www/ServerAdministration.js
index 0d803ac4..16de687e 100644
--- a/www/ServerAdministration.js
+++ b/www/ServerAdministration.js
@@ -53,6 +53,14 @@ Ext.define('PBS.ServerAdministration', {
itemId: 'updates',
nodename: 'localhost',
},
+ {
+ xtype: 'proxmoxNodeAPTRepositories',
+ title: gettext('Repositories'),
+ iconCls: 'fa fa-files-o',
+ itemId: 'aptrepositories',
+ nodename: 'localhost',
+ majorUpgradeAllowed: false,
+ },
{
xtype: 'proxmoxJournalView',
itemId: 'logs',
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [PATCH v5 proxmox-backup 13/23] add check_repositories_call
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (11 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 12/23] ui: add APT repositories Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [RFC v5 proxmox-backup 14/23] RFC: add upgrade_repositories call Fabian Ebner
` (9 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
Changes from v4:
* use common_digest function now living in the library
* use 'infos' in the return value description to match the actual return
value key
src/api2/node/apt.rs | 68 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 67 insertions(+), 1 deletion(-)
diff --git a/src/api2/node/apt.rs b/src/api2/node/apt.rs
index 0fcdce9c..deb481a7 100644
--- a/src/api2/node/apt.rs
+++ b/src/api2/node/apt.rs
@@ -7,7 +7,7 @@ use proxmox::api::{api, RpcEnvironment, RpcEnvironmentType, Permission};
use proxmox::api::router::{Router, SubdirMap};
use proxmox::tools::fs::{replace_file, CreateOptions};
-use proxmox_apt::types::{APTRepositoryFile, APTRepositoryFileError};
+use proxmox_apt::types::{APTRepositoryFile, APTRepositoryFileError, APTRepositoryInfo};
use proxmox_http::ProxyConfig;
use crate::config::node;
@@ -390,6 +390,71 @@ pub fn get_versions() -> Result<Vec<APTUpdateInfo>, Error> {
Ok(packages)
}
+#[api(
+ input: {
+ properties: {
+ node: {
+ schema: NODE_SCHEMA,
+ },
+ digest: {
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ optional: true,
+ },
+ },
+ },
+ returns: {
+ type: Object,
+ description: "Additional sanity checks for the configured APT repositories.",
+ properties: {
+ infos: {
+ description: "Additional information/warnings for APT repositories.",
+ type: Array,
+ items: {
+ type: APTRepositoryInfo,
+ },
+ },
+ enterprise: {
+ description: "Whether the enterprise repository is enabled or not.",
+ type: Boolean,
+ },
+ nosubscription: {
+ description: "Whether the no-subscription repository is enabled or not.",
+ type: Boolean,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Privilege(&[], PRIV_SYS_AUDIT, false),
+ },
+)]
+/// Additional sanity checks for the configured APT repositories.
+pub fn check_repositories(digest: Option<String>) -> Result<Value, Error> {
+ let (files, _) = proxmox_apt::repositories::repositories()?;
+
+ if files.len() == 0 {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ if let Some(digest) = digest {
+ let expected_digest = proxmox::tools::hex_to_digest(&digest)?;
+ let current_digest = proxmox_apt::repositories::common_digest(&files);
+ crate::tools::detect_modified_configuration_file(¤t_digest, &expected_digest)?;
+ }
+
+ let infos = proxmox_apt::repositories::check_repositories(&files);
+
+ let enterprise_enabled =
+ proxmox_apt::repositories::enterprise_repository_enabled(&files, "pbs");
+ let no_subscription_enabled =
+ proxmox_apt::repositories::no_subscription_repository_enabled(&files, "pbs");
+
+ Ok(json!({
+ "infos": infos,
+ "enterprise": enterprise_enabled,
+ "nosubscription": no_subscription_enabled
+ }))
+}
+
#[api(
input: {
properties: {
@@ -446,6 +511,7 @@ pub fn get_repositories() -> Result<Value, Error> {
const SUBDIRS: SubdirMap = &[
("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
+ ("checkrepositories", &Router::new().get(&API_METHOD_CHECK_REPOSITORIES)),
("repositories", &Router::new().get(&API_METHOD_GET_REPOSITORIES)),
("update", &Router::new()
.get(&API_METHOD_APT_UPDATE_AVAILABLE)
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 proxmox-backup 14/23] RFC: add upgrade_repositories call
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (12 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [PATCH v5 proxmox-backup 13/23] add check_repositories_call Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 15/23] add cargo config Fabian Ebner
` (8 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
Should the call be root-only?
How do we want to enable/disable the call pre-/post- upgrade?
src/api2/node/apt.rs | 52 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/src/api2/node/apt.rs b/src/api2/node/apt.rs
index deb481a7..6a56a5b5 100644
--- a/src/api2/node/apt.rs
+++ b/src/api2/node/apt.rs
@@ -509,6 +509,57 @@ pub fn get_repositories() -> Result<Value, Error> {
}))
}
+#[api(
+ input: {
+ properties: {
+ node: {
+ schema: NODE_SCHEMA,
+ },
+ digest: {
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ optional: true,
+ },
+ },
+ },
+ protected: true,
+ access: {
+ permission: &Permission::Privilege(&["system"], PRIV_SYS_MODIFY, false),
+ },
+)]
+/// Upgrade the repository configuration to the next major release.
+pub fn upgrade_repositories(digest: Option<String>) -> Result<(), Error> {
+ //bail!("next major version not released yet, or current version not fully upgraded");
+
+ // TODO should it be required that all files could be parsed?
+ let (mut files, _errors) = proxmox_apt::repositories::repositories()?;
+
+ if files.len() == 0 {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ if let Some(digest) = digest {
+ let expected_digest = proxmox::tools::hex_to_digest(&digest)?;
+ let current_digest = proxmox_apt::repositories::common_digest(&files);
+ crate::tools::detect_modified_configuration_file(¤t_digest, &expected_digest)?;
+ }
+
+ proxmox_apt::repositories::replace_suite(
+ &mut files,
+ proxmox_apt::repositories::STABLE_SUITE,
+ proxmox_apt::repositories::NEXT_STABLE_SUITE,
+ );
+
+ if let Err(errors) = proxmox_apt::repositories::write_repositories(&files) {
+ let message = errors.iter().fold(
+ "Problem writing file(s):".to_string(),
+ |message, error| format!("{}\n{}", message, error),
+ );
+ bail!(message);
+ }
+
+ Ok(())
+}
+
const SUBDIRS: SubdirMap = &[
("changelog", &Router::new().get(&API_METHOD_APT_GET_CHANGELOG)),
("checkrepositories", &Router::new().get(&API_METHOD_CHECK_REPOSITORIES)),
@@ -517,6 +568,7 @@ const SUBDIRS: SubdirMap = &[
.get(&API_METHOD_APT_UPDATE_AVAILABLE)
.post(&API_METHOD_APT_UPDATE_DATABASE)
),
+ ("upgraderepositories", &Router::new().put(&API_METHOD_UPGRADE_REPOSITORIES)),
("versions", &Router::new().get(&API_METHOD_GET_VERSIONS)),
];
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-rs 15/23] add cargo config
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (13 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 proxmox-backup 14/23] RFC: add upgrade_repositories call Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-31 12:53 ` [pve-devel] [pbs-devel] " Wolfgang Bumiller
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module Fabian Ebner
` (7 subsequent siblings)
22 siblings, 1 reply; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
.cargo/config | 5 +++++
1 file changed, 5 insertions(+)
create mode 100644 .cargo/config
diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000..3b5b6e4
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [pve-devel] [pbs-devel] [RFC v5 pve-rs 15/23] add cargo config
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 15/23] add cargo config Fabian Ebner
@ 2021-05-31 12:53 ` Wolfgang Bumiller
2021-05-31 13:12 ` Fabian Ebner
0 siblings, 1 reply; 31+ messages in thread
From: Wolfgang Bumiller @ 2021-05-31 12:53 UTC (permalink / raw)
To: Fabian Ebner; +Cc: pve-devel, pbs-devel
Please don't put this on top of this repo, better start over since the
diskmanage stuff in here hasn't been touched in a while and therefore
isn't used anywhere, yet.
Also, I'm confused by the patch numbering. I'd rather have individual
repositories have their own individual numbers.
On Fri, May 28, 2021 at 04:29:54PM +0200, Fabian Ebner wrote:
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>
> New in v5.
>
> .cargo/config | 5 +++++
> 1 file changed, 5 insertions(+)
> create mode 100644 .cargo/config
>
> diff --git a/.cargo/config b/.cargo/config
> new file mode 100644
> index 0000000..3b5b6e4
> --- /dev/null
> +++ b/.cargo/config
> @@ -0,0 +1,5 @@
> +[source]
> +[source.debian-packages]
> +directory = "/usr/share/cargo/registry"
> +[source.crates-io]
> +replace-with = "debian-packages"
> --
> 2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [pve-devel] [pbs-devel] [RFC v5 pve-rs 15/23] add cargo config
2021-05-31 12:53 ` [pve-devel] [pbs-devel] " Wolfgang Bumiller
@ 2021-05-31 13:12 ` Fabian Ebner
0 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-31 13:12 UTC (permalink / raw)
To: Wolfgang Bumiller; +Cc: pve-devel, pbs-devel
Am 31.05.21 um 14:53 schrieb Wolfgang Bumiller:
> Please don't put this on top of this repo, better start over since the
> diskmanage stuff in here hasn't been touched in a while and therefore
> isn't used anywhere, yet.
>
Okay, should've paid more attention to the actual status of the
repository in hindsight.
> Also, I'm confused by the patch numbering. I'd rather have individual
> repositories have their own individual numbers.
>
I just got used to doing it like this, but don't mind changing it.
> On Fri, May 28, 2021 at 04:29:54PM +0200, Fabian Ebner wrote:
>> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
>> ---
>>
>> New in v5.
>>
>> .cargo/config | 5 +++++
>> 1 file changed, 5 insertions(+)
>> create mode 100644 .cargo/config
>>
>> diff --git a/.cargo/config b/.cargo/config
>> new file mode 100644
>> index 0000000..3b5b6e4
>> --- /dev/null
>> +++ b/.cargo/config
>> @@ -0,0 +1,5 @@
>> +[source]
>> +[source.debian-packages]
>> +directory = "/usr/share/cargo/registry"
>> +[source.crates-io]
>> +replace-with = "debian-packages"
>> --
>> 2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread
* [pve-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (14 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 15/23] add cargo config Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-31 12:55 ` [pve-devel] [pbs-devel] " Wolfgang Bumiller
2021-05-31 13:07 ` Wolfgang Bumiller
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 17/23] add files for Debian packaging Fabian Ebner
` (6 subsequent siblings)
22 siblings, 2 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
and disable diskmanage for now, whose Cargo.toml references local paths.
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
Cargo.toml | 3 +-
perlmod/Cargo.toml | 17 ++++++++++
perlmod/src/apt/mod.rs | 1 +
perlmod/src/apt/repositories.rs | 55 +++++++++++++++++++++++++++++++++
perlmod/src/lib.rs | 1 +
5 files changed, 76 insertions(+), 1 deletion(-)
create mode 100644 perlmod/Cargo.toml
create mode 100644 perlmod/src/apt/mod.rs
create mode 100644 perlmod/src/apt/repositories.rs
create mode 100644 perlmod/src/lib.rs
diff --git a/Cargo.toml b/Cargo.toml
index d16a498..0ffd4a2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,5 @@
[workspace]
members = [
- "diskmanage",
+# "diskmanage",
+ "perlmod",
]
diff --git a/perlmod/Cargo.toml b/perlmod/Cargo.toml
new file mode 100644
index 0000000..397f444
--- /dev/null
+++ b/perlmod/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "pve-rs"
+version = "0.1.0"
+authors = ["Proxmox Support Team <support@proxmox.com>"]
+edition = "2018"
+license = "AGPL-3"
+description = "Perl bindings for proxmox-apt"
+homepage = "https://www.proxmox.com"
+
+[lib]
+crate-type = [ "cdylib" ]
+
+[dependencies]
+anyhow = "1.0"
+proxmox = { version = "0.11.5" }
+proxmox-apt = "0.1.0"
+perlmod = { version = "0.4.3", features = [ "exporter" ] }
diff --git a/perlmod/src/apt/mod.rs b/perlmod/src/apt/mod.rs
new file mode 100644
index 0000000..574c1a7
--- /dev/null
+++ b/perlmod/src/apt/mod.rs
@@ -0,0 +1 @@
+mod repositories;
diff --git a/perlmod/src/apt/repositories.rs b/perlmod/src/apt/repositories.rs
new file mode 100644
index 0000000..37a62f2
--- /dev/null
+++ b/perlmod/src/apt/repositories.rs
@@ -0,0 +1,55 @@
+#[perlmod::package(name = "PVE::RS::APT::Repositories", lib = "pve_rs")]
+mod export {
+ use anyhow::{bail, Error};
+
+ use perlmod::{to_value, Value};
+
+ #[export(raw_return)]
+ fn repositories() -> Result<(Value, Value, Value), Error> {
+ let (files, errors) = proxmox_apt::repositories::repositories()?;
+
+ if files.is_empty() {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ let common_digest = proxmox_apt::repositories::common_digest(&files);
+
+ let hex_digest = proxmox::tools::digest_to_hex(&common_digest);
+
+ Ok((
+ to_value(&files)?,
+ to_value(&errors)?,
+ to_value(&hex_digest)?,
+ ))
+ }
+
+ #[export(raw_return)]
+ fn check_repositories(digest: String) -> Result<(Value, Value, Value), Error> {
+ let (files, _) = proxmox_apt::repositories::repositories()?;
+
+ if files.is_empty() {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ if !digest.is_empty() {
+ let expected_digest = proxmox::tools::hex_to_digest(&digest)?;
+ let current_digest = proxmox_apt::repositories::common_digest(&files);
+ if current_digest != expected_digest {
+ bail!("detected modified configuration - file changed by other user? Try again.");
+ }
+ }
+
+ let infos = proxmox_apt::repositories::check_repositories(&files);
+
+ let enterprise_enabled =
+ proxmox_apt::repositories::enterprise_repository_enabled(&files, "pve");
+ let no_subscription_enabled =
+ proxmox_apt::repositories::no_subscription_repository_enabled(&files, "pve");
+
+ Ok((
+ to_value(&infos)?,
+ to_value(&enterprise_enabled)?,
+ to_value(&no_subscription_enabled)?,
+ ))
+ }
+}
diff --git a/perlmod/src/lib.rs b/perlmod/src/lib.rs
new file mode 100644
index 0000000..10b3376
--- /dev/null
+++ b/perlmod/src/lib.rs
@@ -0,0 +1 @@
+pub mod apt;
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* Re: [pve-devel] [pbs-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module Fabian Ebner
@ 2021-05-31 12:55 ` Wolfgang Bumiller
2021-05-31 13:17 ` Fabian Ebner
2021-05-31 13:07 ` Wolfgang Bumiller
1 sibling, 1 reply; 31+ messages in thread
From: Wolfgang Bumiller @ 2021-05-31 12:55 UTC (permalink / raw)
To: Fabian Ebner; +Cc: pve-devel, pbs-devel
Please don't use 'perlmod' as the name of a crate which also depends on
a different 'perlmod' crate, that won't be fun for long.
Call it pve-rs ;-)
On Fri, May 28, 2021 at 04:29:55PM +0200, Fabian Ebner wrote:
> and disable diskmanage for now, whose Cargo.toml references local paths.
>
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>
> New in v5.
>
> Cargo.toml | 3 +-
> perlmod/Cargo.toml | 17 ++++++++++
> perlmod/src/apt/mod.rs | 1 +
> perlmod/src/apt/repositories.rs | 55 +++++++++++++++++++++++++++++++++
> perlmod/src/lib.rs | 1 +
> 5 files changed, 76 insertions(+), 1 deletion(-)
> create mode 100644 perlmod/Cargo.toml
> create mode 100644 perlmod/src/apt/mod.rs
> create mode 100644 perlmod/src/apt/repositories.rs
> create mode 100644 perlmod/src/lib.rs
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [pve-devel] [pbs-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module
2021-05-31 12:55 ` [pve-devel] [pbs-devel] " Wolfgang Bumiller
@ 2021-05-31 13:17 ` Fabian Ebner
0 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-31 13:17 UTC (permalink / raw)
To: Wolfgang Bumiller; +Cc: pve-devel, pbs-devel
Am 31.05.21 um 14:55 schrieb Wolfgang Bumiller:
> Please don't use 'perlmod' as the name of a crate which also depends on
> a different 'perlmod' crate, that won't be fun for long.
>
> Call it pve-rs ;-)
>
I called it perlmod, because that's what was suggested in the (outdated)
README. Should it even be a sub-crate then, or organized like pmg-rs?
> On Fri, May 28, 2021 at 04:29:55PM +0200, Fabian Ebner wrote:
>> and disable diskmanage for now, whose Cargo.toml references local paths.
>>
>> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
>> ---
>>
>> New in v5.
>>
>> Cargo.toml | 3 +-
>> perlmod/Cargo.toml | 17 ++++++++++
>> perlmod/src/apt/mod.rs | 1 +
>> perlmod/src/apt/repositories.rs | 55 +++++++++++++++++++++++++++++++++
>> perlmod/src/lib.rs | 1 +
>> 5 files changed, 76 insertions(+), 1 deletion(-)
>> create mode 100644 perlmod/Cargo.toml
>> create mode 100644 perlmod/src/apt/mod.rs
>> create mode 100644 perlmod/src/apt/repositories.rs
>> create mode 100644 perlmod/src/lib.rs
^ permalink raw reply [flat|nested] 31+ messages in thread
* Re: [pve-devel] [pbs-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module Fabian Ebner
2021-05-31 12:55 ` [pve-devel] [pbs-devel] " Wolfgang Bumiller
@ 2021-05-31 13:07 ` Wolfgang Bumiller
1 sibling, 0 replies; 31+ messages in thread
From: Wolfgang Bumiller @ 2021-05-31 13:07 UTC (permalink / raw)
To: Fabian Ebner; +Cc: pve-devel, pbs-devel
On Fri, May 28, 2021 at 04:29:55PM +0200, Fabian Ebner wrote:
> and disable diskmanage for now, whose Cargo.toml references local paths.
>
> Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
> ---
>
> New in v5.
>
> Cargo.toml | 3 +-
> perlmod/Cargo.toml | 17 ++++++++++
> perlmod/src/apt/mod.rs | 1 +
> perlmod/src/apt/repositories.rs | 55 +++++++++++++++++++++++++++++++++
> perlmod/src/lib.rs | 1 +
> 5 files changed, 76 insertions(+), 1 deletion(-)
> create mode 100644 perlmod/Cargo.toml
> create mode 100644 perlmod/src/apt/mod.rs
> create mode 100644 perlmod/src/apt/repositories.rs
> create mode 100644 perlmod/src/lib.rs
>
> diff --git a/Cargo.toml b/Cargo.toml
> index d16a498..0ffd4a2 100644
> --- a/Cargo.toml
> +++ b/Cargo.toml
> @@ -1,4 +1,5 @@
> [workspace]
> members = [
> - "diskmanage",
> +# "diskmanage",
> + "perlmod",
> ]
> diff --git a/perlmod/Cargo.toml b/perlmod/Cargo.toml
> new file mode 100644
> index 0000000..397f444
> --- /dev/null
> +++ b/perlmod/Cargo.toml
> @@ -0,0 +1,17 @@
> +[package]
> +name = "pve-rs"
> +version = "0.1.0"
> +authors = ["Proxmox Support Team <support@proxmox.com>"]
> +edition = "2018"
> +license = "AGPL-3"
> +description = "Perl bindings for proxmox-apt"
> +homepage = "https://www.proxmox.com"
> +
> +[lib]
> +crate-type = [ "cdylib" ]
> +
> +[dependencies]
> +anyhow = "1.0"
> +proxmox = { version = "0.11.5" }
> +proxmox-apt = "0.1.0"
> +perlmod = { version = "0.4.3", features = [ "exporter" ] }
> diff --git a/perlmod/src/apt/mod.rs b/perlmod/src/apt/mod.rs
> new file mode 100644
> index 0000000..574c1a7
> --- /dev/null
> +++ b/perlmod/src/apt/mod.rs
> @@ -0,0 +1 @@
> +mod repositories;
> diff --git a/perlmod/src/apt/repositories.rs b/perlmod/src/apt/repositories.rs
> new file mode 100644
> index 0000000..37a62f2
> --- /dev/null
> +++ b/perlmod/src/apt/repositories.rs
> @@ -0,0 +1,55 @@
> +#[perlmod::package(name = "PVE::RS::APT::Repositories", lib = "pve_rs")]
> +mod export {
> + use anyhow::{bail, Error};
> +
> + use perlmod::{to_value, Value};
> +
> + #[export(raw_return)]
> + fn repositories() -> Result<(Value, Value, Value), Error> {
Please also add some documentation to exported methods, especially if
you skip writing out the return type.
Ideally we'd also add some kind of doc generation to perlmod in the
future which could in theory also link to the types from the foreign
crates you're returning here, so please assume that that's already the
case ;-)
Come to think of it, since we need a `mod` statement anyway, it probably
makes sense to use `pub fn` for exported functions (since the `mod`
itself is not-`pub`).
This also means we could utilize `#![deny(missing_docs)]` inside those
`mod`s ;-)
> + let (files, errors) = proxmox_apt::repositories::repositories()?;
> +
> + if files.is_empty() {
> + bail!("no APT repository files could be parsed!");
> + }
> +
> + let common_digest = proxmox_apt::repositories::common_digest(&files);
> +
> + let hex_digest = proxmox::tools::digest_to_hex(&common_digest);
> +
> + Ok((
> + to_value(&files)?,
> + to_value(&errors)?,
> + to_value(&hex_digest)?,
> + ))
> + }
> +
> + #[export(raw_return)]
> + fn check_repositories(digest: String) -> Result<(Value, Value, Value), Error> {
^ same
^ permalink raw reply [flat|nested] 31+ messages in thread
* [pve-devel] [RFC v5 pve-rs 17/23] add files for Debian packaging
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (15 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 16/23] add perlmod crate with initial APT module Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 18/23] update .gitignore Fabian Ebner
` (5 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
adapted from pmg-rs
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
Makefile | 73 ++++++++++++++++++++++++++++++++++++++++++++++++
debian/changelog | 5 ++++
debian/compat | 1 +
debian/control | 20 +++++++++++++
debian/copyright | 16 +++++++++++
debian/rules | 7 +++++
debian/triggers | 1 +
7 files changed, 123 insertions(+)
create mode 100644 Makefile
create mode 100644 debian/changelog
create mode 100644 debian/compat
create mode 100644 debian/control
create mode 100644 debian/copyright
create mode 100755 debian/rules
create mode 100644 debian/triggers
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..a8222b1
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,73 @@
+include /usr/share/dpkg/default.mk
+
+PACKAGE=libpve-rs-perl
+
+ARCH:=$(shell dpkg-architecture -qDEB_BUILD_ARCH)
+export GITVERSION:=$(shell git rev-parse HEAD)
+
+PERL_INSTALLVENDORARCH != perl -MConfig -e 'print $$Config{installvendorarch};'
+PERL_INSTALLVENDORLIB != perl -MConfig -e 'print $$Config{installvendorlib};'
+
+MAIN_DEB=${PACKAGE}_${DEB_VERSION}_${ARCH}.deb
+DBGSYM_DEB=${PACKAGE}-dbgsym_${DEB_VERSION}_${ARCH}.deb
+DEBS=$(MAIN_DEB) $(DBGSYM_DEB)
+
+DESTDIR=
+
+PM_DIRS := \
+ PVE/RS/APT
+
+PM_FILES := \
+ PVE/RS/APT/Repositories.pm
+
+ifeq ($(BUILD_MODE), release)
+CARGO_BUILD_ARGS += --release
+endif
+
+all:
+ifneq ($(BUILD_MODE), skip)
+ cargo build $(CARGO_BUILD_ARGS)
+endif
+
+# always re-create this dir
+# but also copy the local target/ and PVE/ dirs as a build-cache
+.PHONY: build
+build:
+ rm -rf build
+ cargo build --release
+ rsync -a debian Makefile Cargo.toml Cargo.lock perlmod target PVE build/
+
+.PHONY: install
+install: target/release/libpve_rs.so
+ install -d -m755 $(DESTDIR)$(PERL_INSTALLVENDORARCH)/auto
+ install -m644 target/release/libpve_rs.so $(DESTDIR)$(PERL_INSTALLVENDORARCH)/auto/libpve_rs.so
+ install -d -m755 $(DESTDIR)$(PERL_INSTALLVENDORLIB)/PVE/RS
+ for i in $(PM_DIRS); do \
+ install -d -m755 $(DESTDIR)$(PERL_INSTALLVENDORLIB)/$$i; \
+ done
+ for i in $(PM_FILES); do \
+ install -m644 $$i $(DESTDIR)$(PERL_INSTALLVENDORLIB)/$$i; \
+ done
+
+.PHONY: deb
+deb: $(MAIN_DEB)
+$(MAIN_DEB): build
+ cd build; dpkg-buildpackage -b -us -uc --no-pre-clean
+ lintian $(DEBS)
+
+distclean: clean
+
+clean:
+ cargo clean
+ rm -rf *.deb *.dsc *.tar.gz *.buildinfo *.changes Cargo.lock build
+ find . -name '*~' -exec rm {} ';'
+
+.PHONY: dinstall
+dinstall: ${DEBS}
+ dpkg -i ${DEBS}
+
+.PHONY: upload
+upload: ${DEBS}
+ # check if working directory is clean
+ git diff --exit-code --stat && git diff --exit-code --stat --staged
+ tar cf - ${DEBS} | ssh -X repoman@repo.proxmox.com upload --product pve --dist buster
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..1ac9e56
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+libpve-rs-perl (0.1-1) unstable; urgency=medium
+
+ * Initial release.
+
+ -- Proxmox Support Team <support@proxmox.com> Thu, 27 May 2021 10:41:30 +0200
diff --git a/debian/compat b/debian/compat
new file mode 100644
index 0000000..f599e28
--- /dev/null
+++ b/debian/compat
@@ -0,0 +1 @@
+10
diff --git a/debian/control b/debian/control
new file mode 100644
index 0000000..20ab5c7
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,20 @@
+Source: libpve-rs-perl
+Section: perl
+Priority: optional
+Maintainer: Proxmox Support Team <support@proxmox.com>
+Build-Depends: debhelper (>= 10),
+ librust-perlmod-macro-dev,
+ librust-anyhow-dev,
+ librust-proxmox-apt-dev,
+ librust-perlmod-dev,
+ librust-serde-dev,
+Standards-Version: 4.3.0
+Homepage: https://www.proxmox.com
+
+Package: libpve-rs-perl
+Architecture: any
+Depends: ${perl:Depends},
+ ${shlibs:Depends},
+Description: Components of Proxmox Virtual Environment which have been ported to Rust.
+ Contains parts of Proxmox Virtual Environment which have been ported to, or
+ newly implemented in the Rust programming language.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..477c305
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,16 @@
+Copyright (C) 2020-2021 Proxmox Server Solutions GmbH
+
+This software is written by Proxmox Server Solutions GmbH <support@proxmox.com>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 0000000..97aca06
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,7 @@
+#!/usr/bin/make -f
+
+#export DH_VERBOSE=1
+export BUILD_MODE=skip
+
+%:
+ dh $@
diff --git a/debian/triggers b/debian/triggers
new file mode 100644
index 0000000..59dd688
--- /dev/null
+++ b/debian/triggers
@@ -0,0 +1 @@
+activate-noawait pve-api-updates
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-rs 18/23] update .gitignore
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (16 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 17/23] add files for Debian packaging Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 19/23] RFC: add upgrade_repositories wrapper Fabian Ebner
` (4 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
.gitignore | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/.gitignore b/.gitignore
index 39c15cc..dca7dbe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,9 @@
/target
**/*.rs.bk
Cargo.lock
+PVE/
+build/
+libpve-rs-perl-dbgsym_*.deb
+libpve-rs-perl_*.buildinfo
+libpve-rs-perl_*.changes
+libpve-rs-perl_*.deb
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-rs 19/23] RFC: add upgrade_repositories wrapper
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (17 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 18/23] update .gitignore Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-manager 20/23] api: apt: add call to list repositories Fabian Ebner
` (3 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
perlmod/src/apt/repositories.rs | 37 +++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/perlmod/src/apt/repositories.rs b/perlmod/src/apt/repositories.rs
index 37a62f2..d9d9429 100644
--- a/perlmod/src/apt/repositories.rs
+++ b/perlmod/src/apt/repositories.rs
@@ -52,4 +52,41 @@ mod export {
to_value(&no_subscription_enabled)?,
))
}
+
+ #[export]
+ fn upgrade_repositories(digest: String) -> Result<(), Error> {
+ //bail!("next major version not released yet, or current version not fully upgraded");
+
+ // TODO should it be required that all files could be parsed?
+ let (mut files, _errors) = proxmox_apt::repositories::repositories()?;
+
+ if files.is_empty() {
+ bail!("no APT repository files could be parsed!");
+ }
+
+ if !digest.is_empty() {
+ let expected_digest = proxmox::tools::hex_to_digest(&digest)?;
+ let current_digest = proxmox_apt::repositories::common_digest(&files);
+ if current_digest != expected_digest {
+ bail!("detected modified configuration - file changed by other user? Try again.");
+ }
+ }
+
+ proxmox_apt::repositories::replace_suite(
+ &mut files,
+ proxmox_apt::repositories::STABLE_SUITE,
+ proxmox_apt::repositories::NEXT_STABLE_SUITE,
+ );
+
+ if let Err(errors) = proxmox_apt::repositories::write_repositories(&files) {
+ let message = errors
+ .iter()
+ .fold("Problem writing file(s):".to_string(), |message, error| {
+ format!("{}\n{}", message, error)
+ });
+ bail!(message);
+ }
+
+ Ok(())
+ }
}
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-manager 20/23] api: apt: add call to list repositories
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (18 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-rs 19/23] RFC: add upgrade_repositories wrapper Fabian Ebner
@ 2021-05-28 14:29 ` Fabian Ebner
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 21/23] ui: add panel for listing APT repositories Fabian Ebner
` (2 subsequent siblings)
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:29 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
PVE/API2/APT.pm | 153 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 153 insertions(+)
diff --git a/PVE/API2/APT.pm b/PVE/API2/APT.pm
index fb4954e7..0e1cc260 100644
--- a/PVE/API2/APT.pm
+++ b/PVE/API2/APT.pm
@@ -19,6 +19,7 @@ use PVE::INotify;
use PVE::Exception;
use PVE::RESTHandler;
use PVE::RPCEnvironment;
+use PVE::RS::APT::Repositories;
use PVE::API2Tools;
use JSON;
@@ -66,6 +67,7 @@ __PACKAGE__->register_method({
my $res = [
{ id => 'changelog' },
+ { id => 'repositories' },
{ id => 'update' },
{ id => 'versions' },
];
@@ -478,6 +480,157 @@ __PACKAGE__->register_method({
return $data;
}});
+__PACKAGE__->register_method({
+ name => 'repositories',
+ path => 'repositories',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Get APT repository information.",
+ permissions => {
+ check => ['perm', '/nodes/{node}', [ 'Sys.Audit' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ },
+ },
+ returns => {
+ type => "object",
+ description => "Result from parsing the APT repository files in /etc/apt/.",
+ properties => {
+ files => {
+ type => "array",
+ description => "List of parsed repository files.",
+ items => {
+ type => "object",
+ properties => {
+ path => {
+ type => "string",
+ description => "Path to the problematic file.",
+ },
+ file_type => {
+ type => "string",
+ enum => [ 'list', 'sources' ],
+ description => "Format of the file.",
+ },
+ repositories => {
+ type => "array",
+ description => "The parsed repositories.",
+ items => {
+ type => "object",
+ properties => {
+ Types => {
+ type => "array",
+ description => "List of package types.",
+ items => {
+ type => "string",
+ enum => [ 'deb', 'deb-src' ],
+ },
+ },
+ URIs => {
+ description => "List of repository URIs.",
+ type => "array",
+ items => {
+ type => "string",
+ },
+ },
+ Suites => {
+ type => "array",
+ description => "List of package distribuitions",
+ items => {
+ type => "string",
+ },
+ },
+ Components => {
+ type => "array",
+ description => "List of repository components",
+ optional => 1, # not present if suite is absolute
+ items => {
+ type => "string",
+ },
+ },
+ Options => {
+ type => "array",
+ description => "Additional options",
+ optional => 1,
+ items => {
+ type => "object",
+ properties => {
+ Key => {
+ type => "string",
+ },
+ Values => {
+ type => "array",
+ items => {
+ type => "string",
+ },
+ },
+ },
+ },
+ },
+ Comment => {
+ type => "string",
+ description => "Associated comment",
+ optional => 1,
+ },
+ FileType => {
+ type => "string",
+ enum => [ 'list', 'sources' ],
+ description => "Format of the defining file.",
+ },
+ Enabled => {
+ type => "boolean",
+ description => "Whether the repository is enabled or not",
+ },
+ },
+ },
+ },
+ digest => {
+ type => "array",
+ description => "Digest of the file as bytes.",
+ items => {
+ type => "integer",
+ },
+ },
+ },
+ },
+ },
+ errors => {
+ type => "array",
+ description => "List of problematic repository files.",
+ items => {
+ type => "object",
+ properties => {
+ path => {
+ type => "string",
+ description => "Path to the problematic file.",
+ },
+ error => {
+ type => "string",
+ description => "The error message",
+ },
+ },
+ },
+ },
+ digest => {
+ type => "string",
+ description => "Common digest of all files.",
+ },
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my ($files, $errors, $digest) = PVE::RS::APT::Repositories::repositories();
+
+ return {
+ files => $files,
+ errors => $errors,
+ digest => $digest,
+ };
+ }});
+
__PACKAGE__->register_method({
name => 'versions',
path => 'versions',
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-manager 21/23] ui: add panel for listing APT repositories
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (19 preceding siblings ...)
2021-05-28 14:29 ` [pve-devel] [RFC v5 pve-manager 20/23] api: apt: add call to list repositories Fabian Ebner
@ 2021-05-28 14:30 ` Fabian Ebner
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 22/23] api: apt: add call for repository check Fabian Ebner
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 23/23] api: apt: add upgrade repos call Fabian Ebner
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:30 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
www/manager6/node/Config.js | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/www/manager6/node/Config.js b/www/manager6/node/Config.js
index 3c4079bb..50bcd184 100644
--- a/www/manager6/node/Config.js
+++ b/www/manager6/node/Config.js
@@ -247,6 +247,14 @@ Ext.define('PVE.node.Config', {
if (caps.nodes['Sys.Audit']) {
me.items.push(
+ {
+ xtype: 'proxmoxNodeAPTRepositories',
+ title: gettext('APT Repositories'),
+ iconCls: 'fa fa-files-o',
+ itemId: 'aptrepositories',
+ nodename: nodename,
+ majorUpgradeAllowed: false,
+ },
{
xtype: 'pveFirewallRules',
iconCls: 'fa fa-shield',
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-manager 22/23] api: apt: add call for repository check
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (20 preceding siblings ...)
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 21/23] ui: add panel for listing APT repositories Fabian Ebner
@ 2021-05-28 14:30 ` Fabian Ebner
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 23/23] api: apt: add upgrade repos call Fabian Ebner
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:30 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
PVE/API2/APT.pm | 76 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 76 insertions(+)
diff --git a/PVE/API2/APT.pm b/PVE/API2/APT.pm
index 0e1cc260..10f18944 100644
--- a/PVE/API2/APT.pm
+++ b/PVE/API2/APT.pm
@@ -68,6 +68,7 @@ __PACKAGE__->register_method({
my $res = [
{ id => 'changelog' },
{ id => 'repositories' },
+ { id => 'checkrepositories' },
{ id => 'update' },
{ id => 'versions' },
];
@@ -631,6 +632,81 @@ __PACKAGE__->register_method({
};
}});
+__PACKAGE__->register_method({
+ name => 'checkrepositories',
+ path => 'checkrepositories',
+ method => 'GET',
+ proxyto => 'node',
+ description => "Additional sanity checks for the configured APT repositories.",
+ permissions => {
+ check => ['perm', '/nodes/{node}', [ 'Sys.Audit' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ digest => {
+ type => "string",
+ description => "Digest to detect modifications.",
+ maxLength => 80,
+ optional => 1,
+ },
+ },
+ },
+ returns => {
+ type => "object",
+ description => "Additional sanity checks for the configured APT repositories.",
+ properties => {
+ infos => {
+ type => "array",
+ description => "Additional information/warnings for APT repositories.",
+ items => {
+ type => "object",
+ properties => {
+ path => {
+ type => "string",
+ description => "Path to the associated file.",
+ },
+ number => {
+ type => "string",
+ description => "Number of the associated repository within the file.",
+ },
+ kind => {
+ type => "string",
+ description => "Kind of the information (e.g. warning).",
+ },
+ message => {
+ type => "string",
+ description => "Information message.",
+ }
+ },
+ },
+ },
+ enterprise => {
+ type => "boolean",
+ description => "Whether the enterprise repository is enabled or not.",
+ },
+ nosubscription => {
+ type => "boolean",
+ description => "Whether the no-subscription repository is enabled or not.",
+ }
+ },
+ },
+ code => sub {
+ my ($param) = @_;
+
+ my $digest = $param->{digest} // '';
+
+ my ($infos, $enterprise, $nosubscription) =
+ PVE::RS::APT::Repositories::check_repositories($digest);
+
+ return {
+ infos => $infos,
+ enterprise => $enterprise,
+ nosubscription => $nosubscription,
+ };
+ }});
+
__PACKAGE__->register_method({
name => 'versions',
path => 'versions',
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread* [pve-devel] [RFC v5 pve-manager 23/23] api: apt: add upgrade repos call
2021-05-28 14:29 [pve-devel] [PATCH-SERIES v5] APT repositories API/UI Fabian Ebner
` (21 preceding siblings ...)
2021-05-28 14:30 ` [pve-devel] [RFC v5 pve-manager 22/23] api: apt: add call for repository check Fabian Ebner
@ 2021-05-28 14:30 ` Fabian Ebner
22 siblings, 0 replies; 31+ messages in thread
From: Fabian Ebner @ 2021-05-28 14:30 UTC (permalink / raw)
To: pve-devel, pbs-devel
Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---
New in v5.
PVE/API2/APT.pm | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/PVE/API2/APT.pm b/PVE/API2/APT.pm
index 10f18944..5164a0ab 100644
--- a/PVE/API2/APT.pm
+++ b/PVE/API2/APT.pm
@@ -70,6 +70,7 @@ __PACKAGE__->register_method({
{ id => 'repositories' },
{ id => 'checkrepositories' },
{ id => 'update' },
+ { id => 'upgraderepositories' },
{ id => 'versions' },
];
@@ -707,6 +708,41 @@ __PACKAGE__->register_method({
};
}});
+__PACKAGE__->register_method({
+ name => 'upgraderepositories',
+ path => 'upgraderepositories',
+ method => 'PUT',
+ proxyto => 'node',
+ protected => 1,
+ description => "Upgrade APT repository suites to the next stable release.",
+ permissions => {
+ check => ['perm', '/nodes/{node}', [ 'Sys.Modify' ]],
+ },
+ parameters => {
+ additionalProperties => 0,
+ properties => {
+ node => get_standard_option('pve-node'),
+ digest => {
+ type => "string",
+ description => "Digest to detect modifications.",
+ maxLength => 80,
+ optional => 1,
+ },
+ },
+ },
+ returns => { type => 'null' },
+ code => sub {
+ my ($param) = @_;
+
+ my $digest = $param->{digest} // '';
+
+ # die "PVE 7.0 is not released yet or pve-manager is not the latest version\n";
+
+ PVE::RS::APT::Repositories::upgrade_repositories($digest);
+
+ return;
+ }});
+
__PACKAGE__->register_method({
name => 'versions',
path => 'versions',
--
2.20.1
^ permalink raw reply [flat|nested] 31+ messages in thread