[pmg-devel] [PATCH pmg-api] introduce pmg7to8 cli helper

[pmg-devel] [PATCH pmg-api] introduce pmg7to8 cli helper

[Dominik Csapak]

mostly copied from pve7to8 (without the pve specific tests) with some
notable additions to check some basic things for the pmg upgrade:
* check if the cluster is healthy
* check if the services are stopped(pre-upgrade)/started(post-upgrade)
* check if the db was upgraded (post upgrade)

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 src/Makefile           |  11 +-
 src/PMG/CLI/pmg7to8.pm | 567 +++++++++++++++++++++++++++++++++++++++++
 src/bin/pmg7to8        |   8 +
 3 files changed, 585 insertions(+), 1 deletion(-)
 create mode 100644 src/PMG/CLI/pmg7to8.pm
 create mode 100644 src/bin/pmg7to8

diff --git a/src/Makefile b/src/Makefile
index 32eac57..cab2db3 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -9,7 +9,7 @@ DOCDIR=${DESTDIR}/usr/share/doc/pmg-api/
 BASHCOMPLDIR=${DESTDIR}/usr/share/bash-completion/completions/
 
 SERVICES = pmgdaemon pmgproxy pmgtunnel pmgmirror
-CLITOOLS = pmgdb pmgconfig pmgperf pmgcm pmgqm pmgreport pmgversion pmgupgrade pmgsubscription pmgbackup
+CLITOOLS = pmgdb pmgconfig pmgperf pmgcm pmgqm pmgreport pmgversion pmgupgrade pmgsubscription pmgbackup pmg7to8
 CLISCRIPTS = pmg-smtp-filter pmgsh pmgpolicy pmgbanner pmg-system-report
 CRONSCRIPTS = pmg-hourly pmg-daily
 
@@ -223,6 +223,15 @@ clean:
 	if test -d .git; then  rm -f PMG/pmgcfg.pm; fi
 	find . -name '*~' -exec rm {} ';'
 
+pmg7to8.1: bin/pmg7to8
+	printf ".TH PMGTO8 1\n.SH NAME\npmg7to8 \- Proxmox Mail Gateway upgrade checker script for 7.3+ to current 8.x\n" > $@.tmp
+	printf ".SH DESCRIPTION\nThis tool will help you to detect common pitfalls and misconfguration\
+	 before, and during the upgrade of a Proxmox Mail Gateway system\n" >> $@.tmp
+	printf "Any failure must be addressed before the upgrade, and any waring must be addressed, \
+	 or at least carefully evaluated, if a false-positive is suspected\n" >> $@.tmp
+	printf ".SH SYNOPSIS\npmg7to8\n" >> $@.tmp
+	mv $@.tmp $@
+
 %.1: bin/%
 	rm -f $@
 	podselect $< |pod2man -n $(notdir $*) -s 1 -r ${PKGVER} -c"Proxmox Documentation" >$@.tmp
diff --git a/src/PMG/CLI/pmg7to8.pm b/src/PMG/CLI/pmg7to8.pm
new file mode 100644
index 0000000..92b330b
--- /dev/null
+++ b/src/PMG/CLI/pmg7to8.pm
@@ -0,0 +1,567 @@
+package PMG::CLI::pmg7to8;
+
+use strict;
+use warnings;
+
+use Cwd ();
+
+use PVE::INotify;
+use PVE::JSONSchema;
+use PVE::Tools qw(run_command split_list file_get_contents);
+
+use PMG::API2::APT;
+use PMG::API2::Certificates;
+use PMG::API2::Cluster;
+use PMG::RESTEnvironment;
+use PMG::Utils;
+
+use Term::ANSIColor;
+
+use PVE::CLIHandler;
+
+use base qw(PVE::CLIHandler);
+
+my $nodename = PVE::INotify::nodename();
+
+my $upgraded = 0; # set in check_pmg_packages
+
+sub setup_environment {
+    PMG::RESTEnvironment->setup_default_cli_env();
+}
+
+my ($min_pmg_major, $min_pmg_minor, $min_pmg_pkgrel) = (7, 3, 2);
+
+my $counters = {
+    pass => 0,
+    skip => 0,
+    warn => 0,
+    fail => 0,
+};
+
+my $log_line = sub {
+    my ($level, $line) = @_;
+
+    $counters->{$level}++ if defined($level) && defined($counters->{$level});
+
+    print uc($level), ': ' if defined($level);
+    print "$line\n";
+};
+
+sub log_pass {
+    print color('green');
+    $log_line->('pass', @_);
+    print color('reset');
+}
+
+sub log_info {
+    $log_line->('info', @_);
+}
+sub log_skip {
+    $log_line->('skip', @_);
+}
+sub log_warn {
+    print color('yellow');
+    $log_line->('warn', @_);
+    print color('reset');
+}
+sub log_fail {
+    print color('bold red');
+    $log_line->('fail', @_);
+    print color('reset');
+}
+
+my $print_header_first = 1;
+sub print_header {
+    my ($h) = @_;
+    print "\n" if !$print_header_first;
+    print "= $h =\n\n";
+    $print_header_first = 0;
+}
+
+my $get_systemd_unit_state = sub {
+    my ($unit, $suppress_stderr) = @_;
+
+    my $state;
+    my $filter_output = sub {
+	$state = shift;
+	chomp $state;
+    };
+
+    my %extra = (outfunc => $filter_output, noerr => 1);
+    $extra{errfunc} = sub {  } if $suppress_stderr;
+
+    eval {
+	run_command(['systemctl', 'is-enabled', "$unit"], %extra);
+	return if !defined($state);
+	run_command(['systemctl', 'is-active', "$unit"], %extra);
+    };
+
+    return $state // 'unknown';
+};
+
+my $log_systemd_unit_state = sub {
+    my ($unit, $no_fail_on_inactive) = @_;
+
+    my $log_method = \&log_warn;
+
+    my $state = $get_systemd_unit_state->($unit);
+    if ($state eq 'active') {
+	$log_method = \&log_pass;
+    } elsif ($state eq 'inactive') {
+	$log_method = $no_fail_on_inactive ? \&log_warn : \&log_fail;
+    } elsif ($state eq 'failed') {
+	$log_method = \&log_fail;
+    }
+
+    $log_method->("systemd unit '$unit' is in state '$state'");
+};
+
+my $versions;
+my $get_pkg = sub {
+    my ($pkg) = @_;
+
+    $versions = eval { PMG::API2::APT->versions({ node => $nodename }) } if !defined($versions);
+
+    if (!defined($versions)) {
+	my $msg = "unable to retrieve package version information";
+	$msg .= "- $@" if $@;
+	log_fail("$msg");
+	return undef;
+    }
+
+    my $pkgs = [ grep { $_->{Package} eq $pkg } @$versions ];
+    if (!defined $pkgs || $pkgs == 0) {
+	log_fail("unable to determine installed $pkg version.");
+	return undef;
+    } else {
+	return $pkgs->[0];
+    }
+};
+
+sub check_pmg_packages {
+    print_header("CHECKING VERSION INFORMATION FOR PMG PACKAGES");
+
+    print "Checking for package updates..\n";
+    my $updates = eval { PMG::API2::APT->list_updates({ node => $nodename }); };
+    if (!defined($updates)) {
+	log_warn("$@") if $@;
+	log_fail("unable to retrieve list of package updates!");
+    } elsif (@$updates > 0) {
+	my $pkgs = join(', ', map { $_->{Package} } @$updates);
+	log_warn("updates for the following packages are available:\n  $pkgs");
+    } else {
+	log_pass("all packages up-to-date");
+    }
+
+    print "\nChecking proxmox-mailgateway package version..\n";
+    my $pkg = 'proxmox-mailgateway';
+    my $pmg = $get_pkg->($pkg);
+    if (!defined($pmg)) {
+	print "\n$pkg not found, checking for proxmox-mailgateway-container..\n";
+	$pkg = 'proxmox-mailgateway-container';
+    }
+    if (defined(my $pmg = $get_pkg->($pkg))) {
+	# TODO: update to native version for pmg8to9
+	my $min_pmg_ver = "$min_pmg_major.$min_pmg_minor-$min_pmg_pkgrel";
+
+	my ($maj, $min, $pkgrel) = $pmg->{OldVersion} =~ m/^(\d+)\.(\d+)[.-](\d+)/;
+
+	if ($maj > $min_pmg_major) {
+	    log_pass("already upgraded to Proxmox Mail Gateway " . ($min_pmg_major + 1));
+	    $upgraded = 1;
+	} elsif ($maj >= $min_pmg_major && $min >= $min_pmg_minor && $pkgrel >= $min_pmg_pkgrel) {
+	    log_pass("$pkg package has version >= $min_pmg_ver");
+	} else {
+	    log_fail("$pkg package is too old, please upgrade to >= $min_pmg_ver!");
+	}
+
+	# FIXME: better differentiate between 6.2 from bullseye or bookworm
+	my ($krunning, $kinstalled) = (qr/6\.(?:2\.(?:[2-9]\d+|1[6-8]|1\d\d+)|5)[^~]*$/, 'pve-kernel-6.2');
+	if (!$upgraded) {
+	    # we got a few that avoided 5.15 in cluster with mixed CPUs, so allow older too
+	    ($krunning, $kinstalled) = (qr/(?:5\.(?:13|15)|6\.2)/, 'pve-kernel-5.15');
+	}
+
+	print "\nChecking running kernel version..\n";
+	my $kernel_ver = $pmg->{RunningKernel};
+	if (!defined($kernel_ver)) {
+	    log_fail("unable to determine running kernel version.");
+	} elsif ($kernel_ver =~ /^$krunning/) {
+	    if ($upgraded) {
+		log_pass("running new kernel '$kernel_ver' after upgrade.");
+	    } else {
+		log_pass("running kernel '$kernel_ver' is considered suitable for upgrade.");
+	    }
+	} elsif ($get_pkg->($kinstalled)) {
+	    # with 6.2 kernel being available in both we might want to fine-tune the check?
+	    log_warn("a suitable kernel ($kinstalled) is intalled, but an unsuitable ($kernel_ver) is booted, missing reboot?!");
+	} else {
+	    log_warn("unexpected running and installed kernel '$kernel_ver'.");
+	}
+
+	if ($upgraded && $kernel_ver =~ /^$krunning/) {
+	    my $outdated_kernel_meta_pkgs = [];
+	    for my $kernel_meta_version ('5.4', '5.11', '5.13', '5.15') {
+		my $pkg = "pve-kernel-${kernel_meta_version}";
+		if ($get_pkg->($pkg)) {
+		    push @$outdated_kernel_meta_pkgs, $pkg;
+		}
+	    }
+	    if (scalar(@$outdated_kernel_meta_pkgs) > 0) {
+		log_info(
+		    "Found outdated kernel meta-packages, taking up extra space on boot partitions.\n"
+		    ."      After a successful upgrade, you can remove them using this command:\n"
+		    ."      apt remove " . join(' ', $outdated_kernel_meta_pkgs->@*)
+		);
+	    }
+	}
+    } else {
+	log_fail("$pkg package not found!");
+    }
+}
+
+my sub check_max_length {
+    my ($raw, $max_length, $warning) = @_;
+    log_warn($warning) if defined($raw) && length($raw) > $max_length;
+}
+
+my $is_cluster = 0;
+my $cluster_healthy = 0;
+
+sub check_cluster_status {
+    log_info("Checking if the cluster nodes are in sync");
+
+    my $rpcenv = PMG::RESTEnvironment->get();
+    my $ticket = PMG::Ticket::assemble_ticket($rpcenv->get_user());
+    $rpcenv->set_ticket($ticket);
+
+    my $nodes = PMG::API2::Cluster->status({});
+    if (!scalar($nodes->@*)) {
+	log_skip("no cluster, no sync status to check");
+	$cluster_healthy = 1;
+	return;
+    }
+
+    $is_cluster = 1;
+    my $syncing = 0;
+    my $errors = 0;
+
+    for my $node ($nodes->@*) {
+	if (!$node->{insync}) {
+	    $syncing = 1;
+	}
+	if ($node->{conn_error}) {
+	    $errors = 1;
+	}
+    }
+
+    if ($errors) {
+	log_fail("Cluster not healthy, please fix the cluster before continuing");
+    } elsif ($syncing) {
+	log_warn("Cluster currently syncing.");
+    } else {
+	log_pass("Cluster healthy and in sync.");
+	$cluster_healthy = 1;
+    }
+}
+
+
+sub check_running_postgres {
+    my $version = PMG::Utils::get_pg_server_version();
+
+    my $upgraded_db = 0;
+
+    if ($upgraded) {
+	if ($version ne '15') {
+	    log_warn("Running postgres version is still 13. Please upgrade the database.");
+	} else {
+	    log_pass("Running postgres version is 15.");
+	    $upgraded_db = 1;
+	}
+    } else {
+	if ($version ne '13') {
+	    log_warn("Running postgres version '$version' is not '13', was a previous upgrade finished?");
+	} else {
+	    log_pass("Running postgres version is 13.");
+	}
+    }
+
+    return $upgraded_db;
+}
+
+sub check_services_disabled {
+    my ($upgraded_db) = @_;
+    my $unit_inactive = sub { return $get_systemd_unit_state->($_[0], 1) eq 'inactive' ? $_[0] : undef };
+
+    my $services = [qw(postfix pmg-smtp-filter pmgpolicy pmgdaemon pmgproxy)];
+
+    if ($is_cluster) {
+	push $services->@*, 'pmgmirror', 'pmgtunnel';
+    }
+
+    my $active_list = [];
+    my $inactive_list = [];
+    for my $service ($services->@*) {
+	if (!$unit_inactive->($service)) {
+	    push $active_list->@*, $service;
+	} else {
+	    push $inactive_list->@*, $service;
+	}
+    }
+
+    if (!$upgraded) {
+	if (scalar($active_list->@*) < 1) {
+	    log_pass("All services inactive.");
+	} else {
+	    my $msg = "Not all services inactive. Consider stopping and disabling them: \n    ";
+	    $msg .= join("\n    ", $active_list->@*);
+	    log_warn($msg);
+	}
+    } else {
+	if (scalar($inactive_list->@*) < 1) {
+	    log_pass("All services active.");
+	} elsif ($upgraded_db) {
+	    my $msg = "Not all services active. Consider enabling and starting them: \n    ";
+	    $msg .= join("\n    ", $inactive_list->@*);
+	    log_warn($msg);
+	} else {
+	    log_skip("Not all services upgraded, but DB was not upgraded yet.");
+	}
+    }
+}
+
+sub check_apt_repos {
+    log_info("Checking if the suite for the Debian security repository is correct..");
+
+    my $found = 0;
+
+    my $dir = '/etc/apt/sources.list.d';
+    my $in_dir = 0;
+
+    # TODO: check that (original) debian and Proxmox MG mirrors are present.
+
+    my $check_file = sub {
+	my ($file) = @_;
+
+	$file = "${dir}/${file}" if $in_dir;
+
+	my $raw = eval { PVE::Tools::file_get_contents($file) };
+	return if !defined($raw);
+	my @lines = split(/\n/, $raw);
+
+	my $number = 0;
+	for my $line (@lines) {
+	    $number++;
+
+	    next if length($line) == 0; # split would result in undef then...
+
+	    ($line) = split(/#/, $line);
+
+	    next if $line !~ m/^deb[[:space:]]/; # is case sensitive
+
+	    my $suite;
+
+	    # catch any of
+	    # https://deb.debian.org/debian-security
+	    # http://security.debian.org/debian-security
+	    # http://security.debian.org/
+	    if ($line =~ m|https?://deb\.debian\.org/debian-security/?\s+(\S*)|i) {
+		$suite = $1;
+	    } elsif ($line =~ m|https?://security\.debian\.org(?:.*?)\s+(\S*)|i) {
+		$suite = $1;
+	    } else {
+		next;
+	    }
+
+	    $found = 1;
+
+	    my $where = "in ${file}:${number}";
+	    # TODO: is this useful (for some other checks)?
+	}
+    };
+
+    $check_file->("/etc/apt/sources.list");
+
+    $in_dir = 1;
+
+    PVE::Tools::dir_glob_foreach($dir, '^.*\.list$', $check_file);
+
+    if (!$found) {
+	# only warn, it might be defined in a .sources file or in a way not caaught above
+	log_warn("No Debian security repository detected in /etc/apt/sources.list and " .
+	    "/etc/apt/sources.list.d/*.list");
+    }
+}
+
+sub check_time_sync {
+    my $unit_active = sub { return $get_systemd_unit_state->($_[0], 1) eq 'active' ? $_[0] : undef };
+
+    log_info("Checking for supported & active NTP service..");
+    if ($unit_active->('systemd-timesyncd.service')) {
+	log_warn(
+	    "systemd-timesyncd is not the best choice for time-keeping on servers, due to only applying"
+	    ." updates on boot.\n  While not necessary for the upgrade it's recommended to use one of:\n"
+	    ."    * chrony (Default in new Proxmox VE installations)\n    * ntpsec\n    * openntpd\n"
+	);
+    } elsif ($unit_active->('ntp.service')) {
+	log_info("Debian deprecated and removed the ntp package for Bookworm, but the system"
+	    ." will automatically migrate to the 'ntpsec' replacement package on upgrade.");
+    } elsif (my $active_ntp = ($unit_active->('chrony.service') || $unit_active->('openntpd.service') || $unit_active->('ntpsec.service'))) {
+	log_pass("Detected active time synchronisation unit '$active_ntp'");
+    } else {
+	log_warn(
+	    "No (active) time synchronisation daemon (NTP) detected, but synchronized systems are important,"
+	    ." especially for cluster and/or ceph!"
+	);
+    }
+}
+
+sub check_bootloader {
+    log_info("Checking bootloader configuration...");
+    if (!$upgraded) {
+	log_skip("not yet upgraded, no need to check the presence of systemd-boot");
+	return;
+    }
+
+    if (! -f "/etc/kernel/proxmox-boot-uuids") {
+	log_skip("proxmox-boot-tool not used for bootloader configuration");
+	return;
+    }
+
+    if (! -d "/sys/firmware/efi") {
+	log_skip("System booted in legacy-mode - no need for systemd-boot");
+	return;
+    }
+
+    if ( -f "/usr/share/doc/systemd-boot/changelog.Debian.gz") {
+	log_pass("systemd-boot is installed");
+    } else {
+	log_warn(
+	    "proxmox-boot-tool is used for bootloader configuration in uefi mode"
+	    . "but the separate systemd-boot package, existing in Debian Bookworm  is not installed"
+	    . "initializing new ESPs will not work until the package is installed"
+	);
+    }
+}
+
+sub check_misc {
+    print_header("MISCELLANEOUS CHECKS");
+    my $ssh_config = eval { PVE::Tools::file_get_contents('/root/.ssh/config') };
+    if (defined($ssh_config)) {
+	log_fail("Unsupported SSH Cipher configured for root in /root/.ssh/config: $1")
+	    if $ssh_config =~ /^Ciphers .*(blowfish|arcfour|3des).*$/m;
+    } else {
+	log_skip("No SSH config file found.");
+    }
+
+    check_time_sync();
+
+    my $root_free = PVE::Tools::df('/', 10);
+    log_warn("Less than 5 GB free space on root file system.")
+	if defined($root_free) && $root_free->{avail} < 5 * 1000*1000*1000;
+
+    log_info("Checking if the local node's hostname '$nodename' is resolvable..");
+    my $local_ip = eval { PVE::Network::get_ip_from_hostname($nodename) };
+    if ($@) {
+	log_warn("Failed to resolve hostname '$nodename' to IP - $@");
+    } else {
+	log_info("Checking if resolved IP is configured on local node..");
+	my $cidr = Net::IP::ip_is_ipv6($local_ip) ? "$local_ip/128" : "$local_ip/32";
+	my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+	my $ip_count = scalar(@$configured_ips);
+
+	if ($ip_count <= 0) {
+	    log_fail("Resolved node IP '$local_ip' not configured or active for '$nodename'");
+	} elsif ($ip_count > 1) {
+	    log_warn("Resolved node IP '$local_ip' active on multiple ($ip_count) interfaces!");
+	} else {
+	    log_pass("Resolved node IP '$local_ip' configured and active on single interface.");
+	}
+    }
+
+    log_info("Check node certificate's RSA key size");
+    my $certs = PMG::API2::Certificates->info({ node => $nodename });
+    my $certs_check = {
+	'rsaEncryption' => {
+	    minsize => 2048,
+	    name => 'RSA',
+	},
+	'id-ecPublicKey' => {
+	    minsize => 224,
+	    name => 'ECC',
+	},
+    };
+
+    my $certs_check_failed = 0;
+    for my $cert (@$certs) {
+	my ($type, $size, $fn) = $cert->@{qw(public-key-type public-key-bits filename)};
+
+	if (!defined($type) || !defined($size)) {
+	    log_warn("'$fn': cannot check certificate, failed to get it's type or size!");
+	}
+
+	my $check = $certs_check->{$type};
+	if (!defined($check)) {
+	    log_warn("'$fn': certificate's public key type '$type' unknown!");
+	    next;
+	}
+
+	if ($size < $check->{minsize}) {
+	    log_fail("'$fn', certificate's $check->{name} public key size is less than 2048 bit");
+	    $certs_check_failed = 1;
+	} else {
+	    log_pass("Certificate '$fn' passed Debian Busters (and newer) security level for TLS connections ($size >= 2048)");
+	}
+    }
+
+    check_apt_repos();
+    check_bootloader();
+}
+
+my sub colored_if {
+    my ($str, $color, $condition) = @_;
+    return "". ($condition ? colored($str, $color) : $str);
+}
+
+__PACKAGE__->register_method ({
+    name => 'checklist',
+    path => 'checklist',
+    method => 'GET',
+    description => 'Check (pre-/post-)upgrade conditions.',
+    parameters => {
+	additionalProperties => 0,
+	properties => {},
+    },
+    returns => { type => 'null' },
+    code => sub {
+	my ($param) = @_;
+
+	check_pmg_packages();
+	check_cluster_status();
+	my $upgraded_db = check_running_postgres();
+	check_services_disabled($upgraded_db);
+	check_misc();
+
+	print_header("SUMMARY");
+
+	my $total = 0;
+	$total += $_ for values %$counters;
+
+	print "TOTAL:    $total\n";
+	print colored("PASSED:   $counters->{pass}\n", 'green');
+	print "SKIPPED:  $counters->{skip}\n";
+	print colored_if("WARNINGS: $counters->{warn}\n", 'yellow', $counters->{warn} > 0);
+	print colored_if("FAILURES: $counters->{fail}\n", 'bold red', $counters->{fail} > 0);
+
+	if ($counters->{warn} > 0 || $counters->{fail} > 0) {
+	    my $color = $counters->{fail} > 0 ? 'bold red' : 'yellow';
+	    print colored("\nATTENTION: Please check the output for detailed information!\n", $color);
+	    print colored("Try to solve the problems one at a time and then run this checklist tool again.\n", $color) if $counters->{fail} > 0;
+	}
+
+	return undef;
+    }});
+
+our $cmddef = [ __PACKAGE__, 'checklist', [], {}];
+
+1;
diff --git a/src/bin/pmg7to8 b/src/bin/pmg7to8
new file mode 100644
index 0000000..38d8d6d
--- /dev/null
+++ b/src/bin/pmg7to8
@@ -0,0 +1,8 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use PMG::CLI::pmg7to8;
+
+PMG::CLI::pmg7to8->run_cli_handler();
-- 
2.30.2

[pmg-devel] applied: [PATCH pmg-api] introduce pmg7to8 cli helper

[Thomas Lamprecht]

Am 26/06/2023 um 14:30 schrieb Dominik Csapak:
> mostly copied from pve7to8 (without the pve specific tests) with some
> notable additions to check some basic things for the pmg upgrade:
> * check if the cluster is healthy
> * check if the services are stopped(pre-upgrade)/started(post-upgrade)
> * check if the db was upgraded (post upgrade)
> 
> Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
> ---
>  src/Makefile           |  11 +-
>  src/PMG/CLI/pmg7to8.pm | 567 +++++++++++++++++++++++++++++++++++++++++
>  src/bin/pmg7to8        |   8 +
>  3 files changed, 585 insertions(+), 1 deletion(-)
>  create mode 100644 src/PMG/CLI/pmg7to8.pm
>  create mode 100644 src/bin/pmg7to8
> 
>

applied, with a few follow-ups and a package suite mismatch check added, thanks!