[PATCH v3 storage 22/76] multipath: add a library around multipathd

[Thomas Lamprecht <t.lamprecht@proxmox.com>]

SAN scans and status reporting need map health, while iSCSI activation
must wait for a map before using its LUN. Share bounded multipathd
queries and Rust-based reply parsing between these callers.

Key maps by LUN WWID, which is stable across nodes and reboots unlike
map names and path devices, and include path groups and health.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 debian/control                  |   2 +
 src/PVE/Makefile                |   1 +
 src/PVE/Multipath.pm            | 159 ++++++++++++++++++++++++++++++++
 src/test/Makefile               |   5 +-
 src/test/run_multipath_tests.pl | 133 ++++++++++++++++++++++++++
 5 files changed, 299 insertions(+), 1 deletion(-)
 create mode 100644 src/PVE/Multipath.pm
 create mode 100755 src/test/run_multipath_tests.pl

diff --git a/debian/control b/debian/control
index 1bf02bd6..0f1f3850 100644
--- a/debian/control
+++ b/debian/control
@@ -8,6 +8,7 @@ Build-Depends: debhelper-compat (= 13),
                libpve-access-control,
                libpve-cluster-perl (>= 8.0.6),
                libpve-common-perl (>= 9.2.3),
+               libpve-rs-perl,
                librados2-perl,
                libtest-mockmodule-perl,
                libxml-libxml-perl,
@@ -39,6 +40,7 @@ Depends: bzip2,
          libpve-apiclient-perl (>= 3.1-1),
          libpve-cluster-perl (>= 8.0.6),
          libpve-common-perl (>= 9.2.3),
+         libpve-rs-perl,
          librados2-perl,
          libxml-libxml-perl,
          lvm2,
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index 9e9f6aac..91f22282 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -4,6 +4,7 @@
 install:
 	install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm
 	install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
+	install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
 	install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
 	install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
 	make -C Storage install
diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm
new file mode 100644
index 00000000..69589ac3
--- /dev/null
+++ b/src/PVE/Multipath.pm
@@ -0,0 +1,159 @@
+package PVE::Multipath;
+
+use v5.36;
+
+use PVE::Tools qw(run_command file_read_firstline);
+
+use PVE::RS::Multipath;
+
+my $MULTIPATH = '/sbin/multipath';
+my $MULTIPATHD = '/sbin/multipathd';
+
+# Reconfigure re-evaluates every path and needs more time than queries or allow-list edits.
+my $QUERY_TIMEOUT = 10;
+my $TOOL_TIMEOUT = 30;
+my $RECONFIGURE_TIMEOUT = 120;
+
+my $supported;
+
+# Cached only while true: a long-lived daemon like pvestatd must notice a later multipath-tools
+# install without a restart, so a negative result is re-checked on every call.
+sub is_supported() {
+    return $supported if $supported;
+    $supported = (-x $MULTIPATH && -x $MULTIPATHD) ? 1 : 0;
+    return $supported;
+}
+
+sub assert_supported() {
+    die "multipath-tools is not installed\n" if !is_supported();
+    return 1;
+}
+
+sub is_running() {
+    return 0 if !is_supported();
+
+    my $running = 0;
+    eval {
+        run_command(
+            [$MULTIPATHD, 'show', 'daemon'],
+            timeout => $QUERY_TIMEOUT,
+            outfunc => sub($line) {
+                $running = 1 if $line =~ m/^pid \d+ \S+/;
+            },
+            errfunc => sub($line) { },
+        );
+    };
+    return $running;
+}
+
+my sub query_maps_json() {
+    assert_supported();
+
+    my $output = '';
+    run_command(
+        [$MULTIPATHD, 'show', 'maps', 'json'],
+        timeout => $QUERY_TIMEOUT,
+        outfunc => sub($line) { $output .= "$line\n"; },
+        errfunc => sub($line) { warn "$line\n"; },
+    );
+
+    return $output;
+}
+
+sub parse_maps_json($json) {
+    return PVE::RS::Multipath::parse_maps_json($json);
+}
+
+# Expected paths include missing paths that the daemon no longer counts.
+sub derive_health($paths_active, $paths_total, $paths_expected = undef) {
+    return PVE::RS::Multipath::derive_health($paths_active, $paths_total, $paths_expected);
+}
+
+# sysfs size uses 512-byte sectors regardless of the device's logical block size.
+my sub dm_size_bytes($sysfs) {
+    return undef if !$sysfs;
+    my $sectors = file_read_firstline("/sys/block/$sysfs/size");
+    return undef if !defined($sectors) || $sectors !~ m/^\d+$/;
+    return int($sectors) * 512;
+}
+
+my sub path_transport($dev) {
+    return undef if !$dev;
+    my $link = readlink("/sys/block/$dev");
+    return undef if !$link;
+    return 'iscsi' if $link =~ m{/session\d+/};
+    return 'fc' if $link =~ m{/rport-\d+};
+    return 'sas' if $link =~ m{/end_device-};
+    return undef;
+}
+
+sub get_maps() {
+    my $maps = parse_maps_json(query_maps_json());
+
+    for my $map ($maps->@*) {
+        $map->{size} = dm_size_bytes($map->{sysfs});
+        $map->{path} = "/dev/disk/by-id/dm-uuid-mpath-$map->{wwid}"
+            if defined($map->{wwid});
+
+        my %transports;
+        for my $group ($map->{'path-groups'}->@*) {
+            for my $path ($group->{paths}->@*) {
+                $path->{transport} //= path_transport($path->{dev});
+                $transports{ $path->{transport} } = 1 if defined($path->{transport});
+            }
+        }
+        my @transports = keys %transports;
+        $map->{transport} = $transports[0] if scalar(@transports) == 1;
+    }
+
+    return $maps;
+}
+
+sub get_map_for_wwid($wwid) {
+    for my $map (get_maps()->@*) {
+        return $map if defined($map->{wwid}) && $map->{wwid} eq $wwid;
+    }
+    return undef;
+}
+
+sub path_status_by_wwid() {
+    my $res = {};
+    for my $map (get_maps()->@*) {
+        my $wwid = $map->{wwid} // next;
+
+        my $paths = [];
+        for my $path (map { $_->{paths}->@* } $map->{'path-groups'}->@*) {
+            my $entry = { device => "/dev/$path->{dev}", state => $path->{'dm-state'} // 'undef' };
+            $entry->{'checker-state'} = $path->{'check-state'} if defined($path->{'check-state'});
+            for my $key (qw(host-wwpn target-wwpn)) {
+                $entry->{$key} = $path->{$key} if defined($path->{$key});
+            }
+            push $paths->@*, $entry;
+        }
+        $res->{$wwid} = {
+            active => $map->{'paths-active'},
+            faults => $map->{'path-faults'},
+            paths => $paths,
+        };
+    }
+    return $res;
+}
+
+# An in-flight query can outlast the polling timeout.
+sub wait_for_map($wwid, $timeout = undef) {
+    # an assembled map can be used whether multipathd answers right now or not
+    my $link = "/dev/disk/by-id/dm-uuid-mpath-$wwid";
+    return { wwid => $wwid, path => $link } if -b $link;
+
+    $timeout //= 10;
+
+    my $deadline = time() + $timeout;
+    while (1) {
+        my $map = eval { get_map_for_wwid($wwid) };
+        return $map if $map;
+        return undef if time() >= $deadline;
+        sleep(1);
+    }
+}
+
+1;
diff --git a/src/test/Makefile b/src/test/Makefile
index 26b6b12e..c287ebdd 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
 all: test
 
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_lvm_status test_status_command test_disklist test_disk_api test_bwlimit test_plugin test_ovf test_volume_access test_san_api test_san_libraries
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_lvm_status test_status_command test_disklist test_disk_api test_bwlimit test_plugin test_ovf test_volume_access test_multipath test_san_api test_san_libraries
 
 test_zfspoolplugin: run_test_zfspoolplugin.pl
 	./run_test_zfspoolplugin.pl
@@ -47,6 +47,9 @@ test_ovf: run_ovf_tests.pl
 test_volume_access: run_volume_access_tests.pl
 	./run_volume_access_tests.pl
 
+test_multipath: run_multipath_tests.pl
+	./run_multipath_tests.pl
+
 test_san_api: run_san_api_tests.pl
 	./run_san_api_tests.pl
 
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
new file mode 100755
index 00000000..484e9348
--- /dev/null
+++ b/src/test/run_multipath_tests.pl
@@ -0,0 +1,133 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use Test::More;
+
+use lib ('.', '..');
+use PVE::Multipath;
+
+# A recorded `multipathd show maps json` reply with three maps exercising each
+# health state: an all-active map, a partially-failed map and an all-failed map.
+my $maps_json = <<'EOF';
+{
+   "major_version": 0,
+   "minor_version": 1,
+   "maps": [
+      {
+         "name": "mpatha",
+         "uuid": "3600140500a1b2c3d4e5f6a7b8c9d0e1f",
+         "sysfs": "dm-0",
+         "dm_st": "active",
+         "paths": 2,
+         "path_groups": [
+            {
+               "group": 1,
+               "dm_st": "active",
+               "pri": 50,
+               "paths": [
+                  { "dev": "sdb", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "target_wwpn": "0x500a098000000001" },
+                  { "dev": "sdc", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "target_wwpn": "0x500a098000000002" }
+               ]
+            }
+         ]
+      },
+      {
+         "name": "mpathb",
+         "uuid": "360014050aabbccddeeff00112233445566",
+         "sysfs": "dm-1",
+         "dm_st": "active",
+         "paths": 2,
+         "path_groups": [
+            {
+               "group": 1,
+               "dm_st": "active",
+               "pri": 10,
+               "paths": [
+                  { "dev": "sdd", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 10 }
+               ]
+            },
+            {
+               "group": 2,
+               "dm_st": "enabled",
+               "pri": 0,
+               "paths": [
+                  { "dev": "sde", "dm_st": "failed", "dev_st": "faulty", "chk_st": "faulty", "pri": 0 }
+               ]
+            }
+         ]
+      },
+      {
+         "name": "mpathc",
+         "uuid": "36001405ffffffffffffffffffffffffff",
+         "sysfs": "dm-2",
+         "dm_st": "active",
+         "paths": 1,
+         "path_groups": [
+            {
+               "group": 1,
+               "dm_st": "enabled",
+               "pri": 0,
+               "paths": [
+                  { "dev": "sdf", "dm_st": "failed", "dev_st": "faulty", "chk_st": "faulty", "pri": 0, "target_wwpn": "[undef]", "host_adapter": "[undef]" }
+               ]
+            }
+         ]
+      }
+   ]
+}
+EOF
+
+my $maps = PVE::Multipath::parse_maps_json($maps_json);
+
+is(scalar($maps->@*), 3, 'parsed all three maps');
+
+my ($a, $b, $c) = $maps->@*;
+
+# fully healthy map
+is($a->{wwid}, '3600140500a1b2c3d4e5f6a7b8c9d0e1f', 'map a wwid taken from uuid');
+is($a->{name}, 'mpatha', 'map a name');
+is($a->{sysfs}, 'dm-0', 'map a sysfs name');
+is($a->{'paths-total'}, 2, 'map a counts both paths');
+is($a->{'paths-active'}, 2, 'map a has two active paths');
+is($a->{health}, 'optimal', 'map a is optimal');
+is(scalar($a->{'path-groups'}->@*), 1, 'map a has one path group');
+is(
+    $a->{'path-groups'}->[0]->{paths}->[0]->{'target-wwpn'},
+    '0x500a098000000001',
+    'FC target wwpn is preserved',
+);
+is(
+    $a->{'path-groups'}->[0]->{paths}->[0]->{transport},
+    'fc',
+    'transport derived as fc from a target wwpn',
+);
+
+# one failed path out of two
+is($b->{'paths-total'}, 2, 'map b counts both paths across groups');
+is($b->{'paths-active'}, 1, 'map b has one active path');
+is($b->{health}, 'degraded', 'map b is degraded');
+
+# no active path left
+is($c->{'paths-total'}, 1, 'map c counts its single path');
+is($c->{'paths-active'}, 0, 'map c has no active path');
+is($c->{health}, 'failed', 'map c is failed');
+ok(
+    !defined($c->{'path-groups'}->[0]->{paths}->[0]->{'target-wwpn'}),
+    "multipathd '[undef]' target_wwpn is cleaned away (not stored)",
+);
+ok(
+    !defined($c->{'path-groups'}->[0]->{paths}->[0]->{transport}),
+    "'[undef]' target_wwpn does not imply fc transport",
+);
+
+# empty / no maps must parse to an empty list, not die
+my $empty = PVE::Multipath::parse_maps_json('{ "major_version": 0, "maps": [] }');
+is_deeply($empty, [], 'no maps parses to empty list');
+
+# malformed input must die with a clear error
+eval { PVE::Multipath::parse_maps_json('not json') };
+ok($@ =~ m/could not parse multipathd maps JSON/, 'invalid JSON raises a clear error');
+
+done_testing();
-- 
2.47.3