[pbs-devel] [PATCH v3 backup stable-2] pbs2to3: add upgrade checker binary

[Christian Ebner <c.ebner@proxmox.com>]

Adds the pbs2to3 upgrade checker with some basic checks.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
changes since v1:
 - fix suite variants split by getting rid of regex and use find instead
   (so that e.g. '*-debug' is parsed correctly)
 - ignore apt repos which are not enabled or `deb-src`
 - s/Proxmox VE/Proxmox Backup Server/
 - cargo clippy fixups + rustfmt

changes since v2:
 - fix deb suite mismatch checks for multiple files
 - use join for flatten available package upgrades list
 - refactor check_pbs_packages
 - fix s/pbs/PBS/ in log outputs
 - inline variables in format strings
 - use AsRef impl types for log_*

 Makefile                             |   1 +
 debian/proxmox-backup-server.install |   2 +
 src/bin/pbs2to3.rs                   | 598 +++++++++++++++++++++++++++
 zsh-completions/_pbs2to3             |  13 +
 4 files changed, 614 insertions(+)
 create mode 100644 src/bin/pbs2to3.rs
 create mode 100644 zsh-completions/_pbs2to3

diff --git a/Makefile b/Makefile
index b307009d..7477935d 100644
--- a/Makefile
+++ b/Makefile
@@ -19,6 +19,7 @@ USR_BIN := \
 USR_SBIN := \
 	proxmox-backup-manager \
 	proxmox-backup-debug \
+	pbs2to3
 
 # Binaries for services:
 SERVICE_BIN := \
diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install
index 76f50cd0..ebe51aae 100644
--- a/debian/proxmox-backup-server.install
+++ b/debian/proxmox-backup-server.install
@@ -11,6 +11,7 @@ usr/lib/x86_64-linux-gnu/proxmox-backup/proxmox-daily-update
 usr/lib/x86_64-linux-gnu/proxmox-backup/sg-tape-cmd
 usr/sbin/proxmox-backup-debug
 usr/sbin/proxmox-backup-manager
+usr/sbin/pbs2to3
 usr/bin/pmtx
 usr/bin/pmt
 usr/bin/proxmox-tape
@@ -39,3 +40,4 @@ usr/share/zsh/vendor-completions/_proxmox-backup-manager
 usr/share/zsh/vendor-completions/_proxmox-tape
 usr/share/zsh/vendor-completions/_pmtx
 usr/share/zsh/vendor-completions/_pmt
+usr/share/zsh/vendor-completions/_pbs2to3
diff --git a/src/bin/pbs2to3.rs b/src/bin/pbs2to3.rs
new file mode 100644
index 00000000..a3aa2816
--- /dev/null
+++ b/src/bin/pbs2to3.rs
@@ -0,0 +1,598 @@
+use std::io::Write;
+use std::path::Path;
+
+use anyhow::{format_err, Error};
+use regex::Regex;
+use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
+
+use proxmox_apt::repositories::{self, APTRepositoryFile, APTRepositoryPackageType};
+use proxmox_backup::api2::node::apt;
+
+const OLD_SUITE: &str = "bullseye";
+const NEW_SUITE: &str = "bookworm";
+const PROXMOX_BACKUP_META: &str = "proxmox-backup";
+const MIN_PBS_MAJOR: u8 = 2;
+const MIN_PBS_MINOR: u8 = 4;
+const MIN_PBS_PKGREL: u8 = 1;
+
+fn main() -> Result<(), Error> {
+    let mut checker = Checker::new();
+    checker.check_pbs_packages()?;
+    checker.check_misc()?;
+    checker.summary()?;
+    Ok(())
+}
+
+struct Checker {
+    output: ConsoleOutput,
+    upgraded: bool,
+}
+
+impl Checker {
+    pub fn new() -> Self {
+        Self {
+            output: ConsoleOutput::new(),
+            upgraded: false,
+        }
+    }
+
+    pub fn check_pbs_packages(&mut self) -> Result<(), Error> {
+        self.output
+            .print_header("CHECKING VERSION INFORMATION FOR PBS PACKAGES")?;
+
+        self.check_upgradable_packages()?;
+        let pkg_versions = apt::get_versions()?;
+        self.check_meta_package_version(&pkg_versions)?;
+        self.check_kernel_compat(&pkg_versions)?;
+        Ok(())
+    }
+
+    fn check_upgradable_packages(&mut self) -> Result<(), Error> {
+        self.output.log_info("Checking for package updates..")?;
+
+        let result = Self::get_upgradable_packages();
+        match result {
+            Err(err) => {
+                self.output.log_warn(format!("{err}"))?;
+                self.output
+                    .log_fail("unable to retrieve list of package updates!")?;
+            }
+            Ok(cache) => {
+                if cache.package_status.is_empty() {
+                    self.output.log_pass("all packages up-to-date")?;
+                } else {
+                    let pkgs = cache
+                        .package_status
+                        .iter()
+                        .map(|pkg| pkg.package.clone())
+                        .collect::<Vec<String>>()
+                        .join(", ");
+                    self.output.log_warn(format!(
+                        "updates for the following packages are available:\n      {pkgs}",
+                    ))?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn check_meta_package_version(
+        &mut self,
+        pkg_versions: &[pbs_api_types::APTUpdateInfo],
+    ) -> Result<(), Error> {
+        self.output
+            .log_info("Checking proxmox backup server package version..")?;
+
+        let pbs_meta_pkg = pkg_versions
+            .iter()
+            .find(|pkg| pkg.package.as_str() == PROXMOX_BACKUP_META);
+
+        if let Some(pbs_meta_pkg) = pbs_meta_pkg {
+            let pkg_version = Regex::new(r"^(\d+)\.(\d+)[.-](\d+)")?;
+            let captures = pkg_version.captures(&pbs_meta_pkg.old_version);
+            if let Some(captures) = captures {
+                let maj = Self::extract_version_from_captures(1, &captures)?;
+                let min = Self::extract_version_from_captures(2, &captures)?;
+                let pkgrel = Self::extract_version_from_captures(3, &captures)?;
+
+                if maj > MIN_PBS_MAJOR {
+                    self.output
+                        .log_pass(format!("Already upgraded to Proxmox Backup Server {maj}"))?;
+                    self.upgraded = true;
+                } else if maj >= MIN_PBS_MAJOR && min >= MIN_PBS_MINOR && pkgrel >= MIN_PBS_PKGREL {
+                    self.output.log_pass(format!(
+                        "'{}' has version >= {}.{}-{}",
+                        PROXMOX_BACKUP_META, MIN_PBS_MAJOR, MIN_PBS_MINOR, MIN_PBS_PKGREL,
+                    ))?;
+                } else {
+                    self.output.log_fail(format!(
+                        "'{}' package is too old, please upgrade to >= {}.{}-{}",
+                        PROXMOX_BACKUP_META, MIN_PBS_MAJOR, MIN_PBS_MINOR, MIN_PBS_PKGREL,
+                    ))?;
+                }
+            } else {
+                self.output.log_fail(format!(
+                    "could not match the '{PROXMOX_BACKUP_META}' package version, \
+                    is it installed?",
+                ))?;
+            }
+        } else {
+            self.output
+                .log_fail(format!("'{PROXMOX_BACKUP_META}' package not found!"))?;
+        }
+        Ok(())
+    }
+
+    fn check_kernel_compat(
+        &mut self,
+        pkg_versions: &[pbs_api_types::APTUpdateInfo],
+    ) -> Result<(), Error> {
+        self.output.log_info("Check running kernel version..")?;
+        let (krunning, kinstalled) = if self.upgraded {
+            (
+                Regex::new(r"^6\.(?:2\.(?:[2-9]\d+|1[6-8]|1\d\d+)|5)[^~]*$")?,
+                "pve-kernel-6.2",
+            )
+        } else {
+            (Regex::new(r"^(?:5\.(?:13|15)|6\.2)")?, "pve-kernel-5.15")
+        };
+
+        let output = std::process::Command::new("uname").arg("-r").output();
+        match output {
+            Err(_err) => self
+                .output
+                .log_fail("unable to determine running kernel version.")?,
+            Ok(ret) => {
+                let running_version = std::str::from_utf8(&ret.stdout[..ret.stdout.len() - 1])?;
+                if krunning.is_match(running_version) {
+                    if self.upgraded {
+                        self.output.log_pass(format!(
+                            "running new kernel '{running_version}' after upgrade."
+                        ))?;
+                    } else {
+                        self.output.log_pass(format!(
+                            "running kernel '{running_version}' is considered suitable for \
+                            upgrade."
+                        ))?;
+                    }
+                } else {
+                    let installed_kernel = pkg_versions
+                        .iter()
+                        .find(|pkg| pkg.package.as_str() == kinstalled);
+                    if installed_kernel.is_some() {
+                        self.output.log_warn(format!(
+                            "a suitable kernel '{kinstalled}' is installed, but an \
+                            unsuitable '{running_version}' is booted, missing reboot?!",
+                        ))?;
+                    } else {
+                        self.output.log_warn(format!(
+                            "unexpected running and installed kernel '{running_version}'.",
+                        ))?;
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn extract_version_from_captures(
+        index: usize,
+        captures: &regex::Captures,
+    ) -> Result<u8, Error> {
+        if let Some(capture) = captures.get(index) {
+            let val = capture.as_str().parse::<u8>()?;
+            Ok(val)
+        } else {
+            Ok(0)
+        }
+    }
+
+    fn check_bootloader(&mut self) -> Result<(), Error> {
+        self.output
+            .log_info("Checking bootloader configuration...")?;
+
+        // PBS packages version check needs to be run before
+        if !self.upgraded {
+            self.output
+                .log_skip("not yet upgraded, no need to check the presence of systemd-boot")?;
+        }
+
+        if !Path::new("/etc/kernel/proxmox-boot-uuids").is_file() {
+            self.output
+                .log_skip("proxmox-boot-tool not used for bootloader configuration")?;
+            return Ok(());
+        }
+
+        if !Path::new("/sys/firmware/efi").is_file() {
+            self.output
+                .log_skip("System booted in legacy-mode - no need for systemd-boot")?;
+            return Ok(());
+        }
+
+        if Path::new("/usr/share/doc/systemd-boot/changelog.Debian.gz").is_file() {
+            self.output.log_pass("systemd-boot is installed")?;
+        } else {
+            self.output.log_warn(
+                "proxmox-boot-tool is used for bootloader configuration in uefi mode \
+                 but the separate systemd-boot package, existing in Debian Bookworm \
+                 is not installed.\n\
+                 initializing new ESPs will not work unitl the package is installed.",
+            )?;
+        }
+        Ok(())
+    }
+
+    fn check_apt_repos(&mut self) -> Result<(), Error> {
+        self.output
+            .log_info("Checking for package repository suite mismatches..")?;
+
+        let mut strange_suite = false;
+        let mut mismatches = Vec::new();
+        let mut found_suite: Option<(String, String)> = None;
+
+        let (repo_files, _repo_errors, _digest) = repositories::repositories()?;
+        for repo_file in repo_files {
+            self.check_repo_file(
+                &mut found_suite,
+                &mut mismatches,
+                &mut strange_suite,
+                repo_file,
+            )?;
+        }
+
+        match (mismatches.is_empty(), strange_suite) {
+            (true, false) => self.output.log_pass("found no suite mismatch")?,
+            (true, true) => self
+                .output
+                .log_notice("found no suite mismatches, but found at least one strange suite")?,
+            (false, _) => {
+                let mut message = String::from(
+                    "Found mixed old and new packages repository suites, fix before upgrading!\
+                    \n      Mismatches:",
+                );
+                for (suite, location) in mismatches.iter() {
+                    message.push_str(
+                        format!("\n      found suite '{suite}' at '{location}'").as_str(),
+                    );
+                }
+                message.push('\n');
+                self.output.log_fail(message)?
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn check_misc(&mut self) -> Result<(), Error> {
+        self.output.print_header("MISCELLANEOUS CHECKS")?;
+        self.check_pbs_services()?;
+        self.check_time_sync()?;
+        self.check_apt_repos()?;
+        self.check_bootloader()?;
+        Ok(())
+    }
+
+    pub fn summary(&mut self) -> Result<(), Error> {
+        self.output.print_summary()
+    }
+
+    fn check_repo_file(
+        &mut self,
+        found_suite: &mut Option<(String, String)>,
+        mismatches: &mut Vec<(String, String)>,
+        strange_suite: &mut bool,
+        repo_file: APTRepositoryFile,
+    ) -> Result<(), Error> {
+        for repo in repo_file.repositories {
+            if !repo.enabled || repo.types == [APTRepositoryPackageType::DebSrc] {
+                continue;
+            }
+            for suite in &repo.suites {
+                let suite = match suite.find(&['-', '/'][..]) {
+                    Some(n) => &suite[0..n],
+                    None => suite,
+                };
+
+                if suite != OLD_SUITE && suite != NEW_SUITE {
+                    let location = repo_file.path.clone().unwrap_or_default();
+                    self.output.log_notice(format!(
+                        "found unusual suite '{suite}', neighter old '{OLD_SUITE}' nor new \
+                            '{NEW_SUITE}'..\n        Affected file {location}\n        Please \
+                            assure this is shipping compatible packages for the upgrade!"
+                    ))?;
+                    *strange_suite = true;
+                    continue;
+                }
+
+                if let Some((ref current_suite, ref current_location)) = found_suite {
+                    let location = repo_file.path.clone().unwrap_or_default();
+                    if suite != current_suite {
+                        if mismatches.is_empty() {
+                            mismatches.push((current_suite.clone(), current_location.clone()));
+                            mismatches.push((suite.to_string(), location));
+                        } else {
+                            mismatches.push((suite.to_string(), location));
+                        }
+                    }
+                } else {
+                    let location = repo_file.path.clone().unwrap_or_default();
+                    *found_suite = Some((suite.to_string(), location));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn get_systemd_unit_state(
+        &mut self,
+        unit: &str,
+    ) -> Result<(SystemdUnitState, SystemdUnitState), Error> {
+        let output = std::process::Command::new("systemctl")
+            .arg("is-enabled")
+            .arg(unit)
+            .output()
+            .map_err(|err| format_err!("failed to execute - {err}"))?;
+
+        let enabled_state = match output.stdout.as_slice() {
+            b"enabled\n" => SystemdUnitState::Enabled,
+            b"disabled\n" => SystemdUnitState::Disabled,
+            _ => SystemdUnitState::Unknown,
+        };
+
+        let output = std::process::Command::new("systemctl")
+            .arg("is-active")
+            .arg(unit)
+            .output()
+            .map_err(|err| format_err!("failed to execute - {err}"))?;
+
+        let active_state = match output.stdout.as_slice() {
+            b"active\n" => SystemdUnitState::Active,
+            b"inactive\n" => SystemdUnitState::Inactive,
+            b"failed\n" => SystemdUnitState::Failed,
+            _ => SystemdUnitState::Unknown,
+        };
+        Ok((enabled_state, active_state))
+    }
+
+    fn check_pbs_services(&mut self) -> Result<(), Error> {
+        self.output.log_info("Checking PBS daemon services..")?;
+
+        for service in ["proxmox-backup.service", "proxmox-backup-proxy.service"] {
+            match self.get_systemd_unit_state(service)? {
+                (_, SystemdUnitState::Active) => {
+                    self.output
+                        .log_pass(format!("systemd unit '{service}' is in state 'active'"))?;
+                }
+                (_, SystemdUnitState::Inactive) => {
+                    self.output.log_fail(format!(
+                        "systemd unit '{service}' is in state 'inactive'\
+                            \n    Please check the service for errors and start it.",
+                    ))?;
+                }
+                (_, SystemdUnitState::Failed) => {
+                    self.output.log_fail(format!(
+                        "systemd unit '{service}' is in state 'failed'\
+                            \n    Please check the service for errors and start it.",
+                    ))?;
+                }
+                (_, _) => {
+                    self.output.log_fail(format!(
+                        "systemd unit '{service}' is not in state 'active'\
+                            \n    Please check the service for errors and start it.",
+                    ))?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn check_time_sync(&mut self) -> Result<(), Error> {
+        self.output
+            .log_info("Checking for supported & active NTP service..")?;
+        if self.get_systemd_unit_state("systemd-timesyncd.service")?.1 == SystemdUnitState::Active {
+            self.output.log_warn(
+                "systemd-timesyncd is not the best choice for time-keeping on servers, due to only \
+                applying updates on boot.\
+                \n       While not necessary for the upgrade it's recommended to use one of:\
+                \n        * chrony (Default in new Proxmox Backup Server installations)\
+                \n        * ntpsec\
+                \n        * openntpd"
+            )?;
+        } else if self.get_systemd_unit_state("ntp.service")?.1 == SystemdUnitState::Active {
+            self.output.log_info(
+                "Debian deprecated and removed the ntp package for Bookworm, but the system \
+	            will automatically migrate to the 'ntpsec' replacement package on upgrade.",
+            )?;
+        } else if self.get_systemd_unit_state("chrony.service")?.1 == SystemdUnitState::Active
+            || self.get_systemd_unit_state("openntpd.service")?.1 == SystemdUnitState::Active
+            || self.get_systemd_unit_state("ntpsec.service")?.1 == SystemdUnitState::Active
+        {
+            self.output
+                .log_pass("Detected active time synchronisation unit")?;
+        } else {
+            self.output.log_warn(
+                "No (active) time synchronisation daemon (NTP) detected, but synchronized systems \
+                are important!",
+            )?;
+        }
+        Ok(())
+    }
+
+    fn get_upgradable_packages() -> Result<proxmox_backup::tools::apt::PkgState, Error> {
+        let cache = if let Ok(false) = proxmox_backup::tools::apt::pkg_cache_expired() {
+            if let Ok(Some(cache)) = proxmox_backup::tools::apt::read_pkg_state() {
+                cache
+            } else {
+                proxmox_backup::tools::apt::update_cache()?
+            }
+        } else {
+            proxmox_backup::tools::apt::update_cache()?
+        };
+
+        Ok(cache)
+    }
+}
+
+#[derive(PartialEq)]
+enum SystemdUnitState {
+    Active,
+    Enabled,
+    Disabled,
+    Failed,
+    Inactive,
+    Unknown,
+}
+
+#[derive(Default)]
+struct Counters {
+    pass: u64,
+    skip: u64,
+    notice: u64,
+    warn: u64,
+    fail: u64,
+}
+
+enum LogLevel {
+    Pass,
+    Info,
+    Skip,
+    Notice,
+    Warn,
+    Fail,
+}
+
+struct ConsoleOutput {
+    stream: StandardStream,
+    first_header: bool,
+    counters: Counters,
+}
+
+impl ConsoleOutput {
+    pub fn new() -> Self {
+        Self {
+            stream: StandardStream::stdout(ColorChoice::Always),
+            first_header: true,
+            counters: Counters::default(),
+        }
+    }
+
+    pub fn print_header(&mut self, message: &str) -> Result<(), Error> {
+        if !self.first_header {
+            writeln!(&mut self.stream)?;
+        }
+        self.first_header = false;
+        writeln!(&mut self.stream, "= {message} =\n")?;
+        Ok(())
+    }
+
+    pub fn set_color(&mut self, color: Color) -> Result<(), Error> {
+        self.stream
+            .set_color(ColorSpec::new().set_fg(Some(color)))?;
+        Ok(())
+    }
+
+    pub fn log_line(&mut self, level: LogLevel, message: &str) -> Result<(), Error> {
+        match level {
+            LogLevel::Pass => {
+                self.counters.pass += 1;
+                self.set_color(Color::Green)?;
+                writeln!(&mut self.stream, "PASS: {}", message)?;
+                self.set_color(Color::White)?;
+            }
+            LogLevel::Info => {
+                writeln!(&mut self.stream, "INFO: {}", message)?;
+            }
+            LogLevel::Skip => {
+                self.counters.skip += 1;
+                writeln!(&mut self.stream, "SKIP: {}", message)?;
+            }
+            LogLevel::Notice => {
+                self.counters.notice += 1;
+                writeln!(&mut self.stream, "NOTICE: {}", message)?;
+            }
+            LogLevel::Warn => {
+                self.counters.warn += 1;
+                self.set_color(Color::Yellow)?;
+                writeln!(&mut self.stream, "WARN: {}", message)?;
+                self.set_color(Color::White)?;
+            }
+            LogLevel::Fail => {
+                self.counters.fail += 1;
+                self.set_color(Color::Red)?;
+                writeln!(&mut self.stream, "FAIL: {}", message)?;
+                self.set_color(Color::White)?;
+            }
+        }
+        Ok(())
+    }
+
+    pub fn log_pass<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Pass, message.as_ref())
+    }
+
+    pub fn log_info<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Info, message.as_ref())
+    }
+
+    pub fn log_skip<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Skip, message.as_ref())
+    }
+
+    pub fn log_notice<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Notice, message.as_ref())
+    }
+
+    pub fn log_warn<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Warn, message.as_ref())
+    }
+
+    pub fn log_fail<T: AsRef<str>>(&mut self, message: T) -> Result<(), Error> {
+        self.log_line(LogLevel::Fail, message.as_ref())
+    }
+
+    pub fn print_summary(&mut self) -> Result<(), Error> {
+        self.print_header("SUMMARY")?;
+
+        let total = self.counters.fail
+            + self.counters.pass
+            + self.counters.notice
+            + self.counters.skip
+            + self.counters.warn;
+
+        self.set_color(Color::White)?;
+        writeln!(&mut self.stream, "TOTAL:     {total}")?;
+        self.set_color(Color::Green)?;
+        writeln!(&mut self.stream, "PASSED:    {}", self.counters.pass)?;
+        self.set_color(Color::White)?;
+        writeln!(&mut self.stream, "SKIPPED:   {}", self.counters.skip)?;
+        writeln!(&mut self.stream, "NOTICE:    {}", self.counters.notice)?;
+        if self.counters.warn > 0 {
+            self.set_color(Color::Yellow)?;
+            writeln!(&mut self.stream, "WARNINGS:  {}", self.counters.warn)?;
+        }
+        if self.counters.fail > 0 {
+            self.set_color(Color::Red)?;
+            writeln!(&mut self.stream, "FAILURES:  {}", self.counters.fail)?;
+        }
+        if self.counters.warn > 0 || self.counters.fail > 0 {
+            let mut color = Color::Yellow;
+            if self.counters.fail > 0 {
+                color = Color::Red;
+            }
+
+            self.set_color(color)?;
+            writeln!(
+                &mut self.stream,
+                "\nATTENTION: Please check the output for detailed information!",
+            )?;
+            if self.counters.fail > 0 {
+                writeln!(
+                    &mut self.stream,
+                    "Try to solve the problems one at a time and rerun this checklist tool again.",
+                )?;
+            }
+        }
+        self.set_color(Color::White)?;
+        Ok(())
+    }
+}
diff --git a/zsh-completions/_pbs2to3 b/zsh-completions/_pbs2to3
new file mode 100644
index 00000000..f6dc3d67
--- /dev/null
+++ b/zsh-completions/_pbs2to3
@@ -0,0 +1,13 @@
+#compdef _pbs2to3() pbs2to3
+
+function _pbs2to3() {
+    local cwords line point cmd curr prev
+    cwords=${#words[@]}
+    line=$words
+    point=${#line}
+    cmd=${words[1]}
+    curr=${words[cwords]}
+    prev=${words[cwords-1]}
+    compadd -- $(COMP_CWORD="$cwords" COMP_LINE="$line" COMP_POINT="$point" \
+        pbs2to3 bashcomplete "$cmd" "$curr" "$prev")
+}
-- 
2.30.2