[PATCH v3 storage 23/76] diskmanage: collect native NVMe multipath path state

[Thomas Lamprecht <t.lamprecht@proxmox.com>]

From: Dietmar Maurer <dietmar@proxmox.com>

NVMe over Fabrics usually uses native multipath, whose paths are
unknown to multipathd. Collect controller state and address, plus
per-path namespace ANA state, from sysfs and group them by head
namespace. This allows uniform health reporting for both multipath
stacks. Local single-controller devices have no per-path entries and
are omitted.

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
[TL: drop redundant comments]
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 src/PVE/Diskmanage.pm                         | 60 +++++++++++++++++++
 .../nvme-subsystem/nvme-subsys0/nvme3/address |  1 +
 .../nvme-subsys0/nvme3/nvme2c3n1/ana_state    |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme3/state   |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme4/address |  1 +
 .../nvme-subsys0/nvme4/nvme2c4n1/ana_state    |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme4/state   |  1 +
 .../nvme-subsystem/nvme-subsys1/nvme0/address |  1 +
 .../nvme-subsystem/nvme-subsys1/nvme0/state   |  1 +
 .../multipath/nvme_path_status_expected.json  | 19 ++++++
 src/test/disklist_test.pm                     | 16 +++++
 11 files changed, 103 insertions(+)
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
 create mode 100644 src/test/disk_tests/multipath/nvme_path_status_expected.json

diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm
index caad0fee..a7eb2b21 100644
--- a/src/PVE/Diskmanage.pm
+++ b/src/PVE/Diskmanage.pm
@@ -974,6 +974,66 @@ sub is_multipath_device {
     return $uuid =~ m/^(?:part\d+-)?mpath-/ ? 1 : 0;
 }
 
+# Native multipath creates per-path namespace children nvmeXcYnZ below controllers. Local
+# single-controller devices lack these children and are omitted.
+sub get_nvme_path_status {
+    my $res = {};
+
+    dir_glob_foreach(
+        '/sys/class/nvme-subsystem',
+        'nvme-subsys\d+',
+        sub {
+            my ($subsys) = @_;
+            my $subsysdir = "/sys/class/nvme-subsystem/$subsys";
+
+            dir_glob_foreach(
+                $subsysdir,
+                'nvme\d+',
+                sub {
+                    my ($ctrl) = @_;
+                    my $ctrldir = "$subsysdir/$ctrl";
+                    my $state = file_read_firstline("$ctrldir/state") // return;
+                    my $address = file_read_firstline("$ctrldir/address");
+
+                    dir_glob_foreach(
+                        $ctrldir,
+                        'nvme\d+c\d+n\d+',
+                        sub {
+                            my ($pathdev) = @_;
+                            return if $pathdev !~ m/^nvme(\d+)c\d+n(\d+)$/;
+                            my $head = "nvme$1n$2";
+
+                            my $entry = { device => "/dev/$ctrl", state => $state };
+                            $entry->{address} = $address
+                                if defined($address) && $address ne '';
+                            my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state");
+                            $entry->{'ana-state'} = $ana if defined($ana);
+
+                            # 'change' is transient; paths in 'inaccessible' or 'persistent-loss'
+                            # cannot serve IO even on a live controller
+                            my $usable = $state eq 'live'
+                                && (!defined($ana)
+                                    || $ana eq 'optimized'
+                                    || $ana eq 'non-optimized');
+
+                            my $status = $res->{$head} //= { active => 0, paths => [] };
+                            $status->{active}++ if $usable;
+                            push @{ $status->{paths} }, $entry;
+                        },
+                    );
+                },
+            );
+        },
+    );
+
+    for my $status (values %$res) {
+        @{ $status->{paths} } =
+            sort { $a->{device} cmp $b->{device} } @{ $status->{paths} };
+    }
+
+    return $res;
+}
+
 sub lookup_multipath_map {
     my ($devpath) = @_;
 
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
new file mode 100644
index 00000000..803fab3e
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
@@ -0,0 +1 @@
+traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
new file mode 100644
index 00000000..e48c58f5
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
@@ -0,0 +1 @@
+optimized
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
new file mode 100644
index 00000000..e23fe645
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
new file mode 100644
index 00000000..36618744
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
@@ -0,0 +1 @@
+traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
new file mode 100644
index 00000000..436c48ee
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
@@ -0,0 +1 @@
+inaccessible
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
new file mode 100644
index 00000000..e23fe645
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
new file mode 100644
index 00000000..f99fcb14
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
@@ -0,0 +1 @@
+0000:02:00.0
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
new file mode 100644
index 00000000..e23fe645
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme_path_status_expected.json b/src/test/disk_tests/multipath/nvme_path_status_expected.json
new file mode 100644
index 00000000..62611b3f
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme_path_status_expected.json
@@ -0,0 +1,19 @@
+{
+    "nvme2n1": {
+        "active": 1,
+        "paths": [
+            {
+                "device": "/dev/nvme3",
+                "state": "live",
+                "address": "traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171",
+                "ana-state": "optimized"
+            },
+            {
+                "device": "/dev/nvme4",
+                "state": "live",
+                "address": "traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171",
+                "ana-state": "inaccessible"
+            }
+        ]
+    }
+}
diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm
index 21823daa..c6fb620c 100644
--- a/src/test/disklist_test.pm
+++ b/src/test/disklist_test.pm
@@ -111,6 +111,7 @@ sub mocked_file_read_firstline {
     my $originalsub = $diskmanage_module->original('file_read_firstline');
 
     $path =~ s|^/sys/block|disk_tests/$testcasedir|;
+    $path =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|;
 
     return &$originalsub($path);
 }
@@ -120,6 +121,12 @@ sub mocked_dir_glob_foreach {
 
     my $lines = [];
 
+    # nvme subsystem trees are provided as directories, glob them like production code
+    if ($dir =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|) {
+        my $originalsub = $diskmanage_module->original('dir_glob_foreach');
+        return &$originalsub($dir, $regex, $sub);
+    }
+
     # read lines in from file
     if ($dir =~ m{^/sys/block$}) {
         @$lines = split(/\n/, read_test_file('disklist'));
@@ -267,6 +274,15 @@ sub test_disk_list {
             is($@, '', 'the SAN opt-in accepts an unused iSCSI LUN');
         }
 
+        if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") {
+            my $status = PVE::Diskmanage::get_nvme_path_status();
+            my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json'));
+
+            print Dumper($status) if $print;
+            $testcount++;
+            is_deeply($status, $expected_status, 'nvme path status should be the same');
+        }
+
         done_testing($testcount);
     };
 }
-- 
2.47.3