[pbs-devel] [PATCH v3 proxmox-apt 01/10] initial commit

[Fabian Ebner <f.ebner@proxmox.com>]

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>
---

Changes from v2:
    * incorporate Wolfgang's feedback:
        * improve warning order/structure in a few places
        * make parsing the tail for the one-line format shorter/more readable
        * use std::mem::take instead of cloning the comment
        * add write_repositories_ref_vec and repositories_from_file helpers for
          monomorphization
        * have parse_repositories take a &mut Vec and push onto it to avoid
          the need to append the result
        * improve sorting by matching with std::cmp::Ordering
        * improve offset handling for the .sources parser's main loop
        * use try_for_each and write directly to avoid collect+join
        * implement Display on types instead of From on String
        * add comment to explain why AsRef is used

 .cargo/config                                 |   5 +
 .gitignore                                    |   3 +
 Cargo.toml                                    |  22 ++
 rustfmt.toml                                  |   1 +
 src/lib.rs                                    |   3 +
 src/repositories/check.rs                     |  47 ++++
 src/repositories/list_parser.rs               | 176 ++++++++++++
 src/repositories/mod.rs                       | 256 ++++++++++++++++++
 src/repositories/sources_parser.rs            | 214 +++++++++++++++
 src/repositories/writer.rs                    |  85 ++++++
 src/types.rs                                  | 211 +++++++++++++++
 tests/repositories.rs                         |  73 +++++
 .../sources.list.d.expected/multiline.sources |   8 +
 .../options_comment.list                      |   3 +
 .../pbs-enterprise.list                       |   2 +
 tests/sources.list.d.expected/pve.list        |  13 +
 tests/sources.list.d.expected/standard.list   |   7 +
 .../sources.list.d.expected/standard.sources  |  11 +
 tests/sources.list.d/multiline.sources        |   9 +
 tests/sources.list.d/options_comment.list     |   2 +
 tests/sources.list.d/pbs-enterprise.list      |   1 +
 tests/sources.list.d/pve.list                 |  10 +
 tests/sources.list.d/standard.list            |   6 +
 tests/sources.list.d/standard.sources         |  10 +
 24 files changed, 1178 insertions(+)
 create mode 100644 .cargo/config
 create mode 100644 .gitignore
 create mode 100644 Cargo.toml
 create mode 100644 rustfmt.toml
 create mode 100644 src/lib.rs
 create mode 100644 src/repositories/check.rs
 create mode 100644 src/repositories/list_parser.rs
 create mode 100644 src/repositories/mod.rs
 create mode 100644 src/repositories/sources_parser.rs
 create mode 100644 src/repositories/writer.rs
 create mode 100644 src/types.rs
 create mode 100644 tests/repositories.rs
 create mode 100644 tests/sources.list.d.expected/multiline.sources
 create mode 100644 tests/sources.list.d.expected/options_comment.list
 create mode 100644 tests/sources.list.d.expected/pbs-enterprise.list
 create mode 100644 tests/sources.list.d.expected/pve.list
 create mode 100644 tests/sources.list.d.expected/standard.list
 create mode 100644 tests/sources.list.d.expected/standard.sources
 create mode 100644 tests/sources.list.d/multiline.sources
 create mode 100644 tests/sources.list.d/options_comment.list
 create mode 100644 tests/sources.list.d/pbs-enterprise.list
 create mode 100644 tests/sources.list.d/pve.list
 create mode 100644 tests/sources.list.d/standard.list
 create mode 100644 tests/sources.list.d/standard.sources

diff --git a/.cargo/config b/.cargo/config
new file mode 100644
index 0000000..3b5b6e4
--- /dev/null
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..0d00c41
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+Cargo.lock
+target/
+tests/sources.list.d.actual
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..a0ecc26
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "proxmox-apt"
+version = "0.1.0"
+authors = [
+    "Fabian Ebner <f.ebner@proxmox.com>",
+    "Proxmox Support Team <support@proxmox.com>",
+]
+edition = "2018"
+license = "AGPL-3"
+description = "Proxmox library for APT"
+homepage = "https://www.proxmox.com"
+
+exclude = [ "debian" ]
+
+[lib]
+name = "proxmox_apt"
+path = "src/lib.rs"
+
+[dependencies]
+anyhow = "1.0"
+proxmox = { version = "0.11.0", features = [ "api-macro" ] }
+serde = { version = "1.0", features = ["derive"] }
diff --git a/rustfmt.toml b/rustfmt.toml
new file mode 100644
index 0000000..32a9786
--- /dev/null
+++ b/rustfmt.toml
@@ -0,0 +1 @@
+edition = "2018"
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 0000000..b065c0f
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,3 @@
+pub mod types;
+
+pub mod repositories;
diff --git a/src/repositories/check.rs b/src/repositories/check.rs
new file mode 100644
index 0000000..87fbbac
--- /dev/null
+++ b/src/repositories/check.rs
@@ -0,0 +1,47 @@
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+    /// Makes sure that all basic properties of a repository are present and
+    /// not obviously invalid.
+    pub fn basic_check(&self) -> Result<(), Error> {
+        if self.types.is_empty() {
+            bail!("missing package type(s)");
+        }
+        if self.uris.is_empty() {
+            bail!("missing URI(s)");
+        }
+        if self.suites.is_empty() {
+            bail!("missing suite(s)");
+        }
+
+        for uri in self.uris.iter() {
+            if !uri.contains(':') || uri.len() < 3 {
+                bail!("invalid URI: '{}'", uri);
+            }
+        }
+
+        for suite in self.suites.iter() {
+            if !suite.ends_with('/') && self.components.is_empty() {
+                bail!("missing component(s)");
+            } else if suite.ends_with('/') && !self.components.is_empty() {
+                bail!("absolute suite '{}' does not allow component(s)", suite);
+            }
+        }
+
+        if self.file_type == APTRepositoryFileType::List {
+            if self.types.len() > 1 {
+                bail!("more than one package type");
+            }
+            if self.uris.len() > 1 {
+                bail!("more than one URI");
+            }
+            if self.suites.len() > 1 {
+                bail!("more than one suite");
+            }
+        }
+
+        Ok(())
+    }
+}
diff --git a/src/repositories/list_parser.rs b/src/repositories/list_parser.rs
new file mode 100644
index 0000000..06bb7c2
--- /dev/null
+++ b/src/repositories/list_parser.rs
@@ -0,0 +1,176 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::{Iterator, Peekable};
+use std::str::SplitAsciiWhitespace;
+
+use anyhow::{bail, format_err, Error};
+
+use super::APTRepositoryParser;
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+pub struct APTListFileParser<R: BufRead> {
+    path: String,
+    input: R,
+    line_nr: usize,
+    comment: String,
+}
+
+impl<R: BufRead> APTListFileParser<R> {
+    pub fn new(path: String, reader: R) -> Self {
+        Self {
+            path,
+            input: reader,
+            line_nr: 0,
+            comment: String::new(),
+        }
+    }
+
+    /// Helper to parse options from the existing token stream.
+    /// Also returns `Ok(())` if there are no options.
+    /// Errors when options are invalid or not closed by `']'`.
+    fn parse_options(
+        options: &mut Vec<APTRepositoryOption>,
+        tokens: &mut Peekable<SplitAsciiWhitespace>,
+    ) -> Result<(), Error> {
+        let mut option = match tokens.peek() {
+            Some(token) => {
+                match token.strip_prefix('[') {
+                    Some(option) => option,
+                    None => return Ok(()), // doesn't look like options
+                }
+            }
+            None => return Ok(()),
+        };
+
+        tokens.next(); // avoid reading the beginning twice
+
+        let mut finished = false;
+        loop {
+            if let Some(stripped) = option.strip_suffix(']') {
+                option = stripped;
+                if option.is_empty() {
+                    break;
+                }
+                finished = true; // but still need to handle the last one
+            };
+
+            if let Some(mid) = option.find('=') {
+                let (key, mut value_str) = option.split_at(mid);
+                value_str = &value_str[1..];
+
+                if key.is_empty() {
+                    bail!("option has no key: '{}'", option);
+                }
+
+                if value_str.is_empty() {
+                    bail!("option has no value: '{}'", option);
+                }
+
+                let values: Vec<String> = value_str
+                    .split(',')
+                    .map(|value| value.to_string())
+                    .collect();
+
+                options.push(APTRepositoryOption {
+                    key: key.to_string(),
+                    values,
+                });
+            } else if !option.is_empty() {
+                bail!("got invalid option - '{}'", option);
+            }
+
+            if finished {
+                break;
+            }
+
+            option = match tokens.next() {
+                Some(option) => option,
+                None => bail!("options not closed by ']'"),
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Parse a repository or comment in one-line format.
+    /// Commented out repositories are also detected and returned with the
+    /// `enabled` property set to `false`.
+    /// If the line contains a repository, `self.comment` is added to the
+    /// `comment` property.
+    /// If the line contains a comment, it is added to `self.comment`.
+    fn parse_one_line(&mut self, mut line: &str) -> Result<Option<APTRepository>, Error> {
+        line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+        // check for commented out repository first
+        if let Some(commented_out) = line.strip_prefix('#') {
+            if let Ok(Some(mut repo)) = self.parse_one_line(commented_out) {
+                repo.set_enabled(false);
+                return Ok(Some(repo));
+            }
+        }
+
+        let mut repo =
+            APTRepository::new(self.path.clone(), self.line_nr, APTRepositoryFileType::List);
+
+        // now handle "real" comment
+        if let Some(comment_start) = line.find('#') {
+            let (line_start, comment) = line.split_at(comment_start);
+            self.comment = format!("{}{}\n", self.comment, &comment[1..]);
+            line = line_start;
+        }
+
+        let mut tokens = line.split_ascii_whitespace().peekable();
+
+        match tokens.next() {
+            Some(package_type) => {
+                repo.types.push(package_type.try_into()?);
+            }
+            None => return Ok(None), // empty line
+        }
+
+        Self::parse_options(&mut repo.options, &mut tokens)?;
+
+        // the rest of the line is just '<uri> <suite> [<components>...]'
+        let mut tokens = tokens.map(str::to_string);
+        repo.uris
+            .push(tokens.next().ok_or_else(|| format_err!("missing URI"))?);
+        repo.suites
+            .push(tokens.next().ok_or_else(|| format_err!("missing suite"))?);
+        repo.components.extend(tokens);
+
+        repo.comment = std::mem::take(&mut self.comment);
+
+        Ok(Some(repo))
+    }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTListFileParser<R> {
+    fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+        let mut line = String::new();
+
+        loop {
+            self.line_nr += 1;
+            line.clear();
+
+            match self.input.read_line(&mut line) {
+                Err(err) => bail!("input error for '{}' - {}", self.path, err),
+                Ok(0) => break,
+                Ok(_) => match self.parse_one_line(&line) {
+                    Ok(Some(repo)) => {
+                        repos.push(repo);
+                        self.comment.clear();
+                    }
+                    Ok(None) => continue,
+                    Err(err) => bail!(
+                        "malformed entry in '{}' line {} - {}",
+                        self.path,
+                        self.line_nr,
+                        err,
+                    ),
+                },
+            }
+        }
+
+        Ok(())
+    }
+}
diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs
new file mode 100644
index 0000000..5c446af
--- /dev/null
+++ b/src/repositories/mod.rs
@@ -0,0 +1,256 @@
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::convert::TryFrom;
+use std::ffi::OsString;
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, format_err, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+mod list_parser;
+use list_parser::APTListFileParser;
+
+mod sources_parser;
+use sources_parser::APTSourcesFileParser;
+
+mod check;
+mod writer;
+
+const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
+const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
+
+impl APTRepository {
+    /// Crates an empty repository with infomration that is known before parsing.
+    fn new(path: String, number: usize, file_type: APTRepositoryFileType) -> Self {
+        Self {
+            types: vec![],
+            uris: vec![],
+            suites: vec![],
+            components: vec![],
+            options: vec![],
+            comment: String::new(),
+            path,
+            number,
+            file_type,
+            enabled: true,
+        }
+    }
+
+    /// Changes the `enabled` flag and makes sure the `Enabled` option for
+    /// `APTRepositoryPackageType::Sources` repositories is updated too.
+    fn set_enabled(&mut self, enabled: bool) {
+        self.enabled = enabled;
+
+        if self.file_type == APTRepositoryFileType::Sources {
+            let enabled_string = match enabled {
+                true => "true".to_string(),
+                false => "false".to_string(),
+            };
+            for option in self.options.iter_mut() {
+                if option.key == "Enabled" {
+                    option.values = vec![enabled_string];
+                    return;
+                }
+            }
+            self.options.push(APTRepositoryOption {
+                key: "Enabled".to_string(),
+                values: vec![enabled_string],
+            });
+        }
+    }
+}
+
+trait APTRepositoryParser {
+    /// Parse all repositories including the disabled ones and push them onto
+    /// the provided vector.
+    fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error>;
+}
+
+/// Helper to decide whether a file name is considered valid by APT and to
+/// extract its file type and the path as a string.
+/// Hidden files yield `Ok(None)`, while invalid file names yield an error.
+fn check_filename<P: AsRef<Path>>(
+    path: P,
+) -> Result<Option<(APTRepositoryFileType, String)>, OsString> {
+    let path: PathBuf = path.as_ref().to_path_buf();
+    let path_string = path.clone().into_os_string().into_string()?;
+
+    let file_name = match path.file_name() {
+        Some(file_name) => file_name.to_os_string().into_string()?,
+        None => return Err(OsString::from(path_string)),
+    };
+
+    // APT silently ignores hidden files
+    if file_name.starts_with('.') {
+        return Ok(None);
+    }
+
+    let extension = match path.extension() {
+        Some(extension) => extension.to_os_string().into_string()?,
+        None => return Err(OsString::from(path_string)),
+    };
+
+    let file_type = match APTRepositoryFileType::try_from(&extension[..]) {
+        Ok(file_type) => file_type,
+        _ => return Err(OsString::from(path_string)),
+    };
+
+    // APT ignores such files but issues a warning
+    if !file_name
+        .chars()
+        .all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '-' || x == '.')
+    {
+        return Err(OsString::from(path_string));
+    }
+
+    Ok(Some((file_type, path_string)))
+}
+
+/// Similar to [`repositories_from_files`], but for a single file, and adds the
+/// parsed repositories onto the provided vector instead. Another difference is
+/// that it doesn't call [`basic_check`](check::basic_check).
+fn repositories_from_file(path: &Path, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+    if !path.is_file() {
+        eprintln!("Ignoring {:?} - not a file", path);
+        return Ok(());
+    }
+
+    let file_type;
+    let path_string;
+
+    match check_filename(path) {
+        Ok(Some(res)) => {
+            file_type = res.0;
+            path_string = res.1;
+        }
+        Ok(None) => return Ok(()),
+        Err(path) => {
+            eprintln!("Ignoring {:?} - invalid file name", path);
+            return Ok(());
+        }
+    }
+
+    let contents =
+        std::fs::read(path).map_err(|err| format_err!("unable to read {:?} - {}", path, err))?;
+
+    let mut parser: Box<dyn APTRepositoryParser> = match file_type {
+        APTRepositoryFileType::List => Box::new(APTListFileParser::new(path_string, &contents[..])),
+        APTRepositoryFileType::Sources => {
+            Box::new(APTSourcesFileParser::new(path_string, &contents[..]))
+        }
+    };
+
+    parser.parse_repositories(repos)?;
+
+    Ok(())
+}
+
+/// Returns all APT repositories configured in the specified files, including
+/// disabled ones.
+/// Warns about invalid file names and some format violations, while other
+/// format violations result in an error.
+pub fn repositories_from_files<P: AsRef<Path>>(paths: &[P]) -> Result<Vec<APTRepository>, Error> {
+    let mut repos = Vec::<APTRepository>::new();
+
+    for path in paths.iter() {
+        repositories_from_file(path.as_ref(), &mut repos)?;
+    }
+
+    for repo in repos.iter() {
+        repo.basic_check().map_err(|err| {
+            format_err!("check for {}:{} failed - {}", repo.path, repo.number, err)
+        })?;
+    }
+
+    Ok(repos)
+}
+
+/// Returns all APT repositories configured in `/etc/apt/sources.list` and
+/// in `/etc/apt/sources.list.d` including disabled repositories.
+/// Warns about invalid file names and some format violations, while other
+/// format violations result in an error.
+pub fn repositories() -> Result<Vec<APTRepository>, Error> {
+    let mut paths = Vec::<PathBuf>::new();
+
+    let sources_list_path = PathBuf::from(APT_SOURCES_LIST_FILENAME);
+    if sources_list_path.is_file() {
+        paths.push(sources_list_path)
+    };
+
+    let sources_list_d_path = PathBuf::from(APT_SOURCES_LIST_DIRECTORY);
+    if sources_list_d_path.is_dir() {
+        for entry in std::fs::read_dir(sources_list_d_path)? {
+            paths.push(entry?.path());
+        }
+    }
+
+    let repos = repositories_from_files(&paths)?;
+
+    Ok(repos)
+}
+
+/// See [`write_repositories`]. Will sort the vector by the repository's file
+/// name and number.
+fn write_repositories_ref_vec(repos: &mut Vec<&APTRepository>) -> Result<(), Error> {
+    // check before writing
+    for repo in repos.iter() {
+        repo.basic_check().map_err(|err| {
+            format_err!("check for {}:{} failed - {}", repo.path, repo.number, err)
+        })?;
+    }
+
+    repos.sort_by(|a, b| match a.path.cmp(&b.path) {
+        Ordering::Equal => a.number.cmp(&b.number),
+        ord => ord,
+    });
+
+    let mut files = BTreeMap::<String, Vec<u8>>::new();
+
+    for repo in repos.iter() {
+        let raw = match files.get_mut(&repo.path) {
+            Some(raw) => raw,
+            None => {
+                files.insert(repo.path.clone(), vec![]);
+                files.get_mut(&repo.path).unwrap()
+            }
+        };
+
+        repo.write(&mut *raw)
+            .map_err(|err| format_err!("writing {}:{} failed - {}", repo.path, repo.number, err))?;
+    }
+
+    for (path, content) in files.iter() {
+        let path = PathBuf::from(path);
+        let dir = path.parent().unwrap();
+
+        std::fs::create_dir_all(dir)
+            .map_err(|err| format_err!("unable to create dir {:?} - {}", dir, err))?;
+
+        let pid = std::process::id();
+        let mut tmp_path = path.clone();
+        tmp_path.set_extension("tmp");
+        tmp_path.set_extension(format!("{}", pid));
+
+        if let Err(err) = std::fs::write(&tmp_path, &content) {
+            let _ = std::fs::remove_file(&tmp_path);
+            bail!("write failed: {}", err);
+        }
+
+        if let Err(err) = std::fs::rename(&tmp_path, &path) {
+            let _ = std::fs::remove_file(&tmp_path);
+            bail!("rename failed for {:?} - {}", path, err);
+        }
+    }
+
+    Ok(())
+}
+
+/// Write the repositories to the respective files specified by their
+/// `path` property and in the order determined by their `number` property.
+/// Does a `check::basic_check(repository)` for each repository first.
+pub fn write_repositories<A: AsRef<APTRepository>>(repos: &[A]) -> Result<(), Error> {
+    let mut repos: Vec<&APTRepository> = repos.iter().map(|repo| repo.as_ref()).collect();
+
+    write_repositories_ref_vec(&mut repos)
+}
diff --git a/src/repositories/sources_parser.rs b/src/repositories/sources_parser.rs
new file mode 100644
index 0000000..5f25d33
--- /dev/null
+++ b/src/repositories/sources_parser.rs
@@ -0,0 +1,214 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::Iterator;
+
+use anyhow::{bail, Error};
+
+use crate::types::{
+    APTRepository, APTRepositoryFileType, APTRepositoryOption, APTRepositoryPackageType,
+};
+
+use super::APTRepositoryParser;
+
+pub struct APTSourcesFileParser<R: BufRead> {
+    path: String,
+    input: R,
+    stanza_nr: usize,
+    comment: String,
+}
+
+/// See `man sources.list` and `man deb822` for the format specification.
+impl<R: BufRead> APTSourcesFileParser<R> {
+    pub fn new(path: String, reader: R) -> Self {
+        Self {
+            path,
+            input: reader,
+            stanza_nr: 1,
+            comment: String::new(),
+        }
+    }
+
+    /// Based on APT's `StringToBool` in `strutl.cc`
+    fn string_to_bool(string: &str, default: bool) -> bool {
+        let string = string.trim_matches(|c| char::is_ascii_whitespace(&c));
+        let string = string.to_lowercase();
+
+        match &string[..] {
+            "1" | "yes" | "true" | "with" | "on" | "enable" => true,
+            "0" | "no" | "false" | "without" | "off" | "disable" => false,
+            _ => default,
+        }
+    }
+
+    /// Checks if `key` is valid according to deb822
+    fn valid_key(key: &str) -> bool {
+        if key.starts_with('-') {
+            return false;
+        };
+        return key.chars().all(|c| matches!(c, '!'..='9' | ';'..='~'));
+    }
+
+    /// Try parsing a repository in stanza format from `lines`.
+    /// Returns `Ok(None)` when no stanza can be found.
+    /// Comments are added to `self.comments`. If a stanza can be found,
+    /// `self.comment` is added to the repository's `comment` property.
+    /// Fully commented out stanzas are treated as comments.
+    fn parse_stanza(&mut self, lines: &str) -> Result<Option<APTRepository>, Error> {
+        let mut repo = APTRepository::new(
+            self.path.clone(),
+            self.stanza_nr,
+            APTRepositoryFileType::Sources,
+        );
+
+        // Values may be folded into multiple lines.
+        // Those lines have to start with a space or a tab.
+        let lines = lines.replace("\n ", " ");
+        let lines = lines.replace("\n\t", " ");
+
+        let mut got_something = false;
+
+        for line in lines.lines() {
+            let line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+            if line.is_empty() {
+                continue;
+            }
+
+            if let Some(commented_out) = line.strip_prefix('#') {
+                self.comment = format!("{}{}\n", self.comment, commented_out);
+                continue;
+            }
+
+            if let Some(mid) = line.find(':') {
+                let (key, value_str) = line.split_at(mid);
+                let value_str = &value_str[1..];
+                let key = key.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+                if key.is_empty() {
+                    bail!("option has no key: '{}'", line);
+                }
+
+                if value_str.is_empty() {
+                    // ignored by APT
+                    eprintln!("option has no value: '{}'", line);
+                    continue;
+                }
+
+                if !Self::valid_key(key) {
+                    // ignored by APT
+                    eprintln!("option with invalid key '{}'", key);
+                    continue;
+                }
+
+                let values: Vec<String> = value_str
+                    .split_ascii_whitespace()
+                    .map(|value| value.to_string())
+                    .collect();
+
+                match key {
+                    "Types" => {
+                        if !repo.types.is_empty() {
+                            eprintln!("key 'Types' was defined twice");
+                        }
+                        let mut types = Vec::<APTRepositoryPackageType>::new();
+                        for package_type in values {
+                            types.push((&package_type[..]).try_into()?);
+                        }
+                        repo.types = types;
+                    }
+                    "URIs" => {
+                        if !repo.uris.is_empty() {
+                            eprintln!("key 'URIs' was defined twice");
+                        }
+                        repo.uris = values;
+                    }
+                    "Suites" => {
+                        if !repo.suites.is_empty() {
+                            eprintln!("key 'Suites' was defined twice");
+                        }
+                        repo.suites = values;
+                    }
+                    "Components" => {
+                        if !repo.components.is_empty() {
+                            eprintln!("key 'Components' was defined twice");
+                        }
+                        repo.components = values;
+                    }
+                    "Enabled" => {
+                        repo.set_enabled(Self::string_to_bool(value_str, true));
+                    }
+                    _ => repo.options.push(APTRepositoryOption {
+                        key: key.to_string(),
+                        values,
+                    }),
+                }
+            } else {
+                bail!("got invalid line - '{:?}'", line);
+            }
+
+            got_something = true;
+        }
+
+        if !got_something {
+            return Ok(None);
+        }
+
+        repo.comment = std::mem::take(&mut self.comment);
+
+        Ok(Some(repo))
+    }
+
+    /// Helper function for `parse_repositories`.
+    fn try_parse_stanza(
+        &mut self,
+        lines: &str,
+        repos: &mut Vec<APTRepository>,
+    ) -> Result<(), Error> {
+        match self.parse_stanza(&lines[..]) {
+            Ok(Some(repo)) => {
+                repos.push(repo);
+                self.comment.clear();
+                self.stanza_nr += 1;
+            }
+            Ok(None) => (),
+            Err(err) => {
+                bail!(
+                    "malformed entry in '{}' stanza {} - {}",
+                    self.path,
+                    self.stanza_nr,
+                    err,
+                );
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTSourcesFileParser<R> {
+    fn parse_repositories(&mut self, repos: &mut Vec<APTRepository>) -> Result<(), Error> {
+        let mut lines = String::new();
+
+        loop {
+            let old_length = lines.len();
+            match self.input.read_line(&mut lines) {
+                Err(err) => bail!("input error for '{}' - {}", self.path, err),
+                Ok(0) => {
+                    self.try_parse_stanza(&lines[..], repos)?;
+                    break;
+                }
+                Ok(_) => {
+                    if (&lines[old_length..])
+                        .trim_matches(|c| char::is_ascii_whitespace(&c))
+                        .is_empty()
+                    {
+                        // detected end of stanza
+                        self.try_parse_stanza(&lines[..], repos)?;
+                        lines.clear();
+                    }
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
diff --git a/src/repositories/writer.rs b/src/repositories/writer.rs
new file mode 100644
index 0000000..76ea6ea
--- /dev/null
+++ b/src/repositories/writer.rs
@@ -0,0 +1,85 @@
+use std::io::Write;
+
+use anyhow::{bail, Error};
+
+use crate::types::{APTRepository, APTRepositoryFileType};
+
+impl APTRepository {
+    /// Writes a repository in the corresponding format followed by a blank.
+    /// Expects that `basic_check()` for the repository was successful.
+    pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> {
+        match self.file_type {
+            APTRepositoryFileType::List => write_one_line(self, w),
+            APTRepositoryFileType::Sources => write_stanza(self, w),
+        }
+    }
+}
+
+/// Writes a repository in one-line format followed by a blank line.
+/// Expects that `repo.file_type == APTRepositoryFileType::List`.
+/// Expects that `basic_check()` for the repository was successful.
+fn write_one_line(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+    if repo.file_type != APTRepositoryFileType::List {
+        bail!("not a .list repository");
+    }
+
+    if !repo.comment.is_empty() {
+        for line in repo.comment.lines() {
+            writeln!(w, "#{}", line)?;
+        }
+    }
+
+    if !repo.enabled {
+        write!(w, "# ")?;
+    }
+
+    write!(w, "{} ", repo.types[0])?;
+
+    if !repo.options.is_empty() {
+        write!(w, "[ ")?;
+        repo.options
+            .iter()
+            .try_for_each(|option| write!(w, "{}={} ", option.key, option.values.join(",")))?;
+        write!(w, "] ")?;
+    };
+
+    write!(w, "{} ", repo.uris[0])?;
+    write!(w, "{} ", repo.suites[0])?;
+    writeln!(w, "{}", repo.components.join(" "))?;
+
+    writeln!(w)?;
+
+    Ok(())
+}
+
+/// Writes a single stanza followed by a blank line.
+/// Expects that `repo.file_type == APTRepositoryFileType::Sources`.
+fn write_stanza(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+    if repo.file_type != APTRepositoryFileType::Sources {
+        bail!("not a .sources repository");
+    }
+
+    if !repo.comment.is_empty() {
+        for line in repo.comment.lines() {
+            writeln!(w, "#{}", line)?;
+        }
+    }
+
+    write!(w, "Types:")?;
+    repo.types
+        .iter()
+        .try_for_each(|package_type| write!(w, " {}", package_type))?;
+    writeln!(w)?;
+
+    writeln!(w, "URIs: {}", repo.uris.join(" "))?;
+    writeln!(w, "Suites: {}", repo.suites.join(" "))?;
+    writeln!(w, "Components: {}", repo.components.join(" "))?;
+
+    for option in repo.options.iter() {
+        writeln!(w, "{}: {}", option.key, option.values.join(" "))?;
+    }
+
+    writeln!(w)?;
+
+    Ok(())
+}
diff --git a/src/types.rs b/src/types.rs
new file mode 100644
index 0000000..be69652
--- /dev/null
+++ b/src/types.rs
@@ -0,0 +1,211 @@
+use std::convert::TryFrom;
+use std::fmt::Display;
+
+use anyhow::{bail, Error};
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum APTRepositoryFileType {
+    /// One-line-style format
+    List,
+    /// DEB822-style format
+    Sources,
+}
+
+impl TryFrom<&str> for APTRepositoryFileType {
+    type Error = Error;
+
+    fn try_from(string: &str) -> Result<Self, Error> {
+        match &string[..] {
+            "list" => Ok(APTRepositoryFileType::List),
+            "sources" => Ok(APTRepositoryFileType::Sources),
+            _ => bail!("invalid file type '{}'", string),
+        }
+    }
+}
+
+impl Display for APTRepositoryFileType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            APTRepositoryFileType::List => write!(f, "list"),
+            APTRepositoryFileType::Sources => write!(f, "sources"),
+        }
+    }
+}
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub enum APTRepositoryPackageType {
+    /// Debian package
+    Deb,
+    /// Debian source package
+    DebSrc,
+}
+
+impl TryFrom<&str> for APTRepositoryPackageType {
+    type Error = Error;
+
+    fn try_from(string: &str) -> Result<Self, Error> {
+        match &string[..] {
+            "deb" => Ok(APTRepositoryPackageType::Deb),
+            "deb-src" => Ok(APTRepositoryPackageType::DebSrc),
+            _ => bail!("invalid file type '{}'", string),
+        }
+    }
+}
+
+impl Display for APTRepositoryPackageType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            APTRepositoryPackageType::Deb => write!(f, "deb"),
+            APTRepositoryPackageType::DebSrc => write!(f, "deb-src"),
+        }
+    }
+}
+
+#[api(
+    properties: {
+        Key: {
+            description: "Option key.",
+            type: String,
+        },
+        Values: {
+            description: "Option values.",
+            type: Array,
+            items: {
+                description: "Value.",
+                type: String,
+            },
+        },
+    },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")] // for consistency
+/// Additional options for an APT repository.
+/// Used for both single- and mutli-value options.
+pub struct APTRepositoryOption {
+    /// Option key.
+    pub key: String,
+    /// Option value(s).
+    pub values: Vec<String>,
+}
+
+#[api(
+    properties: {
+        Types: {
+            description: "List of package types.",
+            type: Array,
+            items: {
+                type: APTRepositoryPackageType,
+            },
+        },
+        URIs: {
+            description: "List of repository URIs.",
+            type: Array,
+            items: {
+                description: "Repository URI.",
+                type: String,
+            },
+        },
+        Suites: {
+            description: "List of distributions.",
+            type: Array,
+            items: {
+                description: "Package distribution.",
+                type: String,
+            },
+        },
+        Components: {
+            description: "List of repository components.",
+            type: Array,
+            items: {
+                description: "Repository component.",
+                type: String,
+            },
+        },
+        Options: {
+            type: Array,
+            optional: true,
+            items: {
+                type: APTRepositoryOption,
+            },
+        },
+        Comment: {
+            description: "Associated comment.",
+            type: String,
+            optional: true,
+        },
+        Path: {
+            description: "Path to the defining file.",
+            type: String,
+        },
+        Number: {
+            description: "Line or stanza number.",
+            type: Integer,
+        },
+        FileType: {
+            type: APTRepositoryFileType,
+        },
+        Enabled: {
+            description: "Whether the repository is enabled or not.",
+            type: Boolean,
+        },
+    },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+/// Describes an APT repository.
+pub struct APTRepository {
+    /// List of package types.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub types: Vec<APTRepositoryPackageType>,
+
+    /// List of repository URIs.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(rename = "URIs")]
+    pub uris: Vec<String>,
+
+    /// List of package distributions.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub suites: Vec<String>,
+
+    /// List of repository components.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub components: Vec<String>,
+
+    /// Additional options.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub options: Vec<APTRepositoryOption>,
+
+    /// Associated comment.
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub comment: String,
+
+    /// Path to the defining file.
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub path: String,
+
+    /// Line or stanza number.
+    pub number: usize,
+
+    /// Format of the defining file.
+    pub file_type: APTRepositoryFileType,
+
+    /// Whether the repository is enabled or not.
+    pub enabled: bool,
+}
+
+/// Some functions like write_repositiories can be called with either a slice of
+/// [`APTRepository`]s or a slice of references thereof. Thus, users of the
+/// crate are more flexibility in working with collections of repositories. See
+/// the test_parse_write test for an example.
+impl AsRef<APTRepository> for APTRepository {
+    fn as_ref(&self) -> &APTRepository {
+        &self
+    }
+}
diff --git a/tests/repositories.rs b/tests/repositories.rs
new file mode 100644
index 0000000..020e133
--- /dev/null
+++ b/tests/repositories.rs
@@ -0,0 +1,73 @@
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox_apt::repositories::{repositories_from_files, write_repositories};
+use proxmox_apt::types::APTRepository;
+
+#[test]
+fn test_parse_write() -> Result<(), Error> {
+    let test_dir = std::env::current_dir()?.join("tests");
+    let read_dir = test_dir.join("sources.list.d");
+    let write_dir = test_dir.join("sources.list.d.actual");
+    let expected_dir = test_dir.join("sources.list.d.expected");
+
+    let mut paths = Vec::<PathBuf>::new();
+    for entry in std::fs::read_dir(read_dir)? {
+        paths.push(entry?.path());
+    }
+
+    let repos = repositories_from_files(&paths)?;
+
+    // used to mess up the order from parsing and to check that each repo has a
+    // unique path:number
+    let mut repo_hash = HashMap::<String, APTRepository>::new();
+
+    for mut repo in repos {
+        let path = PathBuf::from(repo.path);
+        let new_path = write_dir.join(path.file_name().unwrap());
+
+        repo.path = new_path.into_os_string().into_string().unwrap();
+
+        let key = format!("{}:{}", repo.path, repo.number);
+
+        if repo_hash.insert(key.clone(), repo).is_some() {
+            bail!("key '{}' is not unique!", key);
+        }
+    }
+
+    if write_dir.is_dir() {
+        std::fs::remove_dir_all(&write_dir)
+            .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+    }
+
+    std::fs::create_dir_all(&write_dir)
+        .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+    let repos_vec: Vec<&APTRepository> = repo_hash.values().collect();
+    write_repositories(&repos_vec)?;
+
+    let mut expected_count = 0;
+
+    for entry in std::fs::read_dir(expected_dir)? {
+        expected_count += 1;
+
+        let expected_path = entry?.path();
+        let actual_path = write_dir.join(expected_path.file_name().unwrap());
+
+        let expected_contents = std::fs::read(&expected_path)
+            .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
+
+        let actual_contents = std::fs::read(&actual_path)
+            .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
+
+        assert_eq!(expected_contents, actual_contents);
+    }
+
+    let actual_count = std::fs::read_dir(write_dir)?.count();
+
+    assert_eq!(expected_count, actual_count);
+
+    Ok(())
+}
diff --git a/tests/sources.list.d.expected/multiline.sources b/tests/sources.list.d.expected/multiline.sources
new file mode 100644
index 0000000..91f53c2
--- /dev/null
+++ b/tests/sources.list.d.expected/multiline.sources
@@ -0,0 +1,8 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+Languages: it de fr
+Enabled: false
+
diff --git a/tests/sources.list.d.expected/options_comment.list b/tests/sources.list.d.expected/options_comment.list
new file mode 100644
index 0000000..f0952e4
--- /dev/null
+++ b/tests/sources.list.d.expected/options_comment.list
@@ -0,0 +1,3 @@
+# comment
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib
+
diff --git a/tests/sources.list.d.expected/pbs-enterprise.list b/tests/sources.list.d.expected/pbs-enterprise.list
new file mode 100644
index 0000000..acb2990
--- /dev/null
+++ b/tests/sources.list.d.expected/pbs-enterprise.list
@@ -0,0 +1,2 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
+
diff --git a/tests/sources.list.d.expected/pve.list b/tests/sources.list.d.expected/pve.list
new file mode 100644
index 0000000..127a49a
--- /dev/null
+++ b/tests/sources.list.d.expected/pve.list
@@ -0,0 +1,13 @@
+deb http://ftp.debian.org/debian buster main contrib
+
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.list b/tests/sources.list.d.expected/standard.list
new file mode 100644
index 0000000..63c1b60
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.list
@@ -0,0 +1,7 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
+
diff --git a/tests/sources.list.d.expected/standard.sources b/tests/sources.list.d.expected/standard.sources
new file mode 100644
index 0000000..56ce280
--- /dev/null
+++ b/tests/sources.list.d.expected/standard.sources
@@ -0,0 +1,11 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
+
diff --git a/tests/sources.list.d/multiline.sources b/tests/sources.list.d/multiline.sources
new file mode 100644
index 0000000..c3a1ff0
--- /dev/null
+++ b/tests/sources.list.d/multiline.sources
@@ -0,0 +1,9 @@
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+# comment in here
+Components: main contrib
+Languages: it
+ de
+	fr
+Enabled: off
diff --git a/tests/sources.list.d/options_comment.list b/tests/sources.list.d/options_comment.list
new file mode 100644
index 0000000..e3f4112
--- /dev/null
+++ b/tests/sources.list.d/options_comment.list
@@ -0,0 +1,2 @@
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian buster main contrib # comment
+
diff --git a/tests/sources.list.d/pbs-enterprise.list b/tests/sources.list.d/pbs-enterprise.list
new file mode 100644
index 0000000..5f8763c
--- /dev/null
+++ b/tests/sources.list.d/pbs-enterprise.list
@@ -0,0 +1 @@
+deb https://enterprise.proxmox.com/debian/pbs buster pbs-enterprise
diff --git a/tests/sources.list.d/pve.list b/tests/sources.list.d/pve.list
new file mode 100644
index 0000000..6213f72
--- /dev/null
+++ b/tests/sources.list.d/pve.list
@@ -0,0 +1,10 @@
+deb http://ftp.debian.org/debian buster main contrib
+deb http://ftp.debian.org/debian buster-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve buster pve-no-subscription
+# deb https://enterprise.proxmox.com/debian/pve buster pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security buster/updates main contrib
diff --git a/tests/sources.list.d/standard.list b/tests/sources.list.d/standard.list
new file mode 100644
index 0000000..26db887
--- /dev/null
+++ b/tests/sources.list.d/standard.list
@@ -0,0 +1,6 @@
+deb http://ftp.at.debian.org/debian buster main contrib
+
+deb http://ftp.at.debian.org/debian buster-updates main contrib
+
+# security updates
+deb http://security.debian.org buster/updates main contrib
diff --git a/tests/sources.list.d/standard.sources b/tests/sources.list.d/standard.sources
new file mode 100644
index 0000000..605202e
--- /dev/null
+++ b/tests/sources.list.d/standard.sources
@@ -0,0 +1,10 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: buster buster-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: buster/updates
+Components: main contrib
-- 
2.20.1