[RFC pve-storage 10/27] diskmanage: add helper querying multipath path state

[Dietmar Maurer <dietmar@proxmox.com>]

Only multipathd knows the health of member paths: a faulty path stays
in the device mapper table (and thus in the sysfs slave list) until
the transport removes the device, so the slave count alone cannot
detect a degraded map. Parse "multipathd show maps json" into per-map
active path counts, fault counters and per-path states, keyed by
WWID. ALUA standby paths count as active, only paths the kernel
failed are unusable.

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
 src/PVE/Diskmanage.pm                         |  57 ++++++++
 src/test/disk_tests/multipath/multipathd      | 124 ++++++++++++++++++
 .../multipath/multipathd_status_expected.json |  42 ++++++
 src/test/disklist_test.pm                     |  20 +++
 4 files changed, 243 insertions(+)
 create mode 100644 src/test/disk_tests/multipath/multipathd
 create mode 100644 src/test/disk_tests/multipath/multipathd_status_expected.json

diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm
index ab5db06..a6730a6 100644
--- a/src/PVE/Diskmanage.pm
+++ b/src/PVE/Diskmanage.pm
@@ -20,6 +20,7 @@ my $SGDISK = "/sbin/sgdisk";
 my $PVS = "/sbin/pvs";
 my $LVS = "/sbin/lvs";
 my $LSBLK = "/bin/lsblk";
+my $MULTIPATHD = "/sbin/multipathd";
 
 my sub strip_dev : prototype($) {
     my ($devpath) = @_;
@@ -912,6 +913,62 @@ sub get_multipath_disks {
     return $disklist;
 }
 
+# Queries multipathd for the path state of all multipath mapped devices, keyed by their WWID.
+# Only the daemon knows the health of the member paths; a faulty path stays in the device
+# mapper table (and thus in the sysfs slave list) until the transport removes the device.
+# Dies if multipathd is not available.
+sub get_multipath_status {
+    die "multipathd is not available\n" if !check_bin($MULTIPATHD);
+
+    my $output = '';
+    run_command(
+        [$MULTIPATHD, 'show', 'maps', 'json'],
+        outfunc => sub { $output .= "$_[0]\n"; },
+        timeout => 5,
+    );
+
+    my $parsed = decode_json($output);
+
+    my $res = {};
+    for my $map (@{ $parsed->{maps} // [] }) {
+        my $wwid = $map->{uuid} // next;
+
+        my $paths = [];
+        my $active = 0;
+        for my $group (@{ $map->{path_groups} // [] }) {
+            for my $path (@{ $group->{paths} // [] }) {
+                my $dev = $path->{dev} // next;
+                my $state = $path->{dm_st} // 'undef';
+                # ALUA standby paths are 'active' with a 'ghost' checker state, only
+                # paths the kernel actually failed count as unusable
+                $active++ if $state ne 'failed';
+
+                my $entry = { device => "/dev/$dev", state => $state };
+                $entry->{'checker-state'} = $path->{chk_st} if defined($path->{chk_st});
+                my $wwpns = {
+                    'host-wwpn' => $path->{host_wwpn},
+                    'target-wwpn' => $path->{target_wwpn},
+                };
+                for my $key (sort keys %$wwpns) {
+                    my $wwpn = $wwpns->{$key} // next;
+                    # multipathd reports '[undef]' for transports without a WWPN
+                    next if $wwpn eq '' || $wwpn eq '[undef]';
+                    $entry->{$key} = $wwpn;
+                }
+                push @$paths, $entry;
+            }
+        }
+
+        $res->{$wwid} = {
+            active => $active,
+            faults => $map->{path_faults} // 0,
+            paths => $paths,
+        };
+    }
+
+    return $res;
+}
+
 # Returns the multipath device entry if the given path refers to a multipath mapped device.
 sub lookup_multipath_map {
     my ($devpath) = @_;
diff --git a/src/test/disk_tests/multipath/multipathd b/src/test/disk_tests/multipath/multipathd
new file mode 100644
index 0000000..40243f9
--- /dev/null
+++ b/src/test/disk_tests/multipath/multipathd
@@ -0,0 +1,124 @@
+{
+   "major_version": 0,
+   "minor_version": 1,
+   "maps": [{
+      "name" : "mpatha",
+      "uuid" : "3600140500000000000000000000000001",
+      "sysfs" : "dm-0",
+      "failback" : "immediate",
+      "queueing" : "5 chk",
+      "paths" : 2,
+      "write_prot" : "rw",
+      "dm_st" : "active",
+      "features" : "1 queue_if_no_path",
+      "hwhandler" : "1 alua",
+      "action" : "",
+      "path_faults" : 0,
+      "vend" : "SANVEND ",
+      "prod" : "SANMODEL",
+      "rev" : "4.1",
+      "switch_grp" : 0,
+      "map_loads" : 1,
+      "total_q_time" : 0,
+      "q_timeouts" : 0,
+      "path_groups": [{
+         "selector" : "service-time 0",
+         "pri" : 50,
+         "dm_st" : "active",
+         "marginal_st" : "normal",
+         "group" : 1,
+         "paths": [{
+            "dev" : "sda",
+            "dev_t" : "8:0",
+            "dm_st" : "active",
+            "dev_st" : "running",
+            "chk_st" : "ready",
+            "checker" : "tur",
+            "pri" : 50,
+            "host_wwnn" : "0x20000024ff7d6a1c",
+            "target_wwnn" : "0x2000d039ea28bf00",
+            "host_wwpn" : "0x21000024ff7d6a1c",
+            "target_wwpn" : "0x2100d039ea28bf00",
+            "host_adapter" : "0000:41:00.0",
+            "lun_hex" : "0x0001000000000000",
+            "marginal_st" : "normal"
+         },
+         {
+            "dev" : "sdb",
+            "dev_t" : "8:16",
+            "dm_st" : "active",
+            "dev_st" : "running",
+            "chk_st" : "ready",
+            "checker" : "tur",
+            "pri" : 50,
+            "host_wwnn" : "0x20000024ff7d6a1d",
+            "target_wwnn" : "0x2000d039ea28bf00",
+            "host_wwpn" : "0x21000024ff7d6a1d",
+            "target_wwpn" : "0x2200d039ea28bf00",
+            "host_adapter" : "0000:41:00.1",
+            "lun_hex" : "0x0001000000000000",
+            "marginal_st" : "normal"
+         }]
+      }]
+   },
+   {
+      "name" : "mpathb",
+      "uuid" : "3600140500000000000000000000000002",
+      "sysfs" : "dm-1",
+      "failback" : "immediate",
+      "queueing" : "5 chk",
+      "paths" : 2,
+      "write_prot" : "rw",
+      "dm_st" : "active",
+      "features" : "1 queue_if_no_path",
+      "hwhandler" : "1 alua",
+      "action" : "",
+      "path_faults" : 3,
+      "vend" : "SANVEND ",
+      "prod" : "SANMODEL",
+      "rev" : "4.1",
+      "switch_grp" : 0,
+      "map_loads" : 1,
+      "total_q_time" : 0,
+      "q_timeouts" : 0,
+      "path_groups": [{
+         "selector" : "service-time 0",
+         "pri" : 50,
+         "dm_st" : "active",
+         "marginal_st" : "normal",
+         "group" : 1,
+         "paths": [{
+            "dev" : "sdc",
+            "dev_t" : "8:32",
+            "dm_st" : "active",
+            "dev_st" : "running",
+            "chk_st" : "ready",
+            "checker" : "tur",
+            "pri" : 50,
+            "host_wwnn" : "0x20000024ff7d6a1c",
+            "target_wwnn" : "0x2000d039ea28bf00",
+            "host_wwpn" : "0x21000024ff7d6a1c",
+            "target_wwpn" : "0x2100d039ea28bf00",
+            "host_adapter" : "0000:41:00.0",
+            "lun_hex" : "0x0002000000000000",
+            "marginal_st" : "normal"
+         },
+         {
+            "dev" : "sdd",
+            "dev_t" : "8:48",
+            "dm_st" : "failed",
+            "dev_st" : "running",
+            "chk_st" : "faulty",
+            "checker" : "tur",
+            "pri" : 0,
+            "host_wwnn" : "0x20000024ff7d6a1d",
+            "target_wwnn" : "0x2000d039ea28bf00",
+            "host_wwpn" : "0x21000024ff7d6a1d",
+            "target_wwpn" : "0x2200d039ea28bf00",
+            "host_adapter" : "0000:41:00.1",
+            "lun_hex" : "0x0002000000000000",
+            "marginal_st" : "normal"
+         }]
+      }]
+   }]
+}
diff --git a/src/test/disk_tests/multipath/multipathd_status_expected.json b/src/test/disk_tests/multipath/multipathd_status_expected.json
new file mode 100644
index 0000000..d57e620
--- /dev/null
+++ b/src/test/disk_tests/multipath/multipathd_status_expected.json
@@ -0,0 +1,42 @@
+{
+    "3600140500000000000000000000000001": {
+        "active": 2,
+        "faults": 0,
+        "paths": [
+            {
+                "device": "/dev/sda",
+                "state": "active",
+                "checker-state": "ready",
+                "host-wwpn": "0x21000024ff7d6a1c",
+                "target-wwpn": "0x2100d039ea28bf00"
+            },
+            {
+                "device": "/dev/sdb",
+                "state": "active",
+                "checker-state": "ready",
+                "host-wwpn": "0x21000024ff7d6a1d",
+                "target-wwpn": "0x2200d039ea28bf00"
+            }
+        ]
+    },
+    "3600140500000000000000000000000002": {
+        "active": 1,
+        "faults": 3,
+        "paths": [
+            {
+                "device": "/dev/sdc",
+                "state": "active",
+                "checker-state": "ready",
+                "host-wwpn": "0x21000024ff7d6a1c",
+                "target-wwpn": "0x2100d039ea28bf00"
+            },
+            {
+                "device": "/dev/sdd",
+                "state": "failed",
+                "checker-state": "faulty",
+                "host-wwpn": "0x21000024ff7d6a1d",
+                "target-wwpn": "0x2200d039ea28bf00"
+            }
+        ]
+    }
+}
diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm
index d2ea3b2..e804485 100644
--- a/src/test/disklist_test.pm
+++ b/src/test/disklist_test.pm
@@ -68,6 +68,8 @@ sub mocked_run_command {
                 $content = '{}';
             }
             @$outputlines = split(/\n/, $content);
+        } elsif ($cmd->[0] =~ m/multipathd/i) {
+            @$outputlines = split(/\n/, read_test_file('multipathd'));
         } else {
             die "unexpected run_command call: '@$cmd', aborting\n";
         }
@@ -257,6 +259,16 @@ sub test_disk_list {
             like($@, qr/not a valid local disk/, 'unknown device mapper device should fail');
         }
 
+        if (-f "disk_tests/$testcasedir/multipathd_status_expected.json") {
+            my $status = PVE::Diskmanage::get_multipath_status();
+            my $expected_status =
+                decode_json(read_test_file('multipathd_status_expected.json'));
+
+            print Dumper($status) if $print;
+            $testcount++;
+            is_deeply($status, $expected_status, 'multipath status should be the same');
+        }
+
         done_testing($testcount);
     };
 }
@@ -279,6 +291,14 @@ $diskmanage_module->mock('get_sysdir_size' => \&mocked_get_sysdir_size);
 print("\tMocked get_sysdir_size\n");
 $diskmanage_module->mock('assert_blockdev' => sub { return 1; });
 print("\tMocked assert_blockdev\n");
+$diskmanage_module->mock(
+    'check_bin' => sub {
+        my ($path) = @_;
+        return 1 if $path =~ m/multipathd/;
+        return &{ $diskmanage_module->original('check_bin') }($path);
+    },
+);
+print("\tMocked check_bin\n");
 $diskmanage_module->mock(
     'dir_is_empty' => sub {
         my $val = shift;
-- 
2.47.3