[RFC pve-storage 11/27] diskmanage: add helper querying NVMe native multipath path state

[Dietmar Maurer <dietmar@proxmox.com>]

NVMe over fabrics namespaces are usually handled by native NVMe
multipath instead of device mapper, so multipathd knows nothing
about them. The kernel exposes the equivalent information in sysfs:
a per-path namespace entry below each controller of a shared
subsystem, with the controller state and address and the ANA state
of the namespace on that path. Collect it per head namespace, so
path health can be reported uniformly for both multipath stacks.
Local single-controller devices have no per-path entries and are
not reported.

Signed-off-by: Dietmar Maurer <dietmar@proxmox.com>
---
 src/PVE/Diskmanage.pm                         | 63 +++++++++++++++++++
 .../nvme-subsystem/nvme-subsys0/nvme3/address |  1 +
 .../nvme-subsys0/nvme3/nvme2c3n1/ana_state    |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme3/state   |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme4/address |  1 +
 .../nvme-subsys0/nvme4/nvme2c4n1/ana_state    |  1 +
 .../nvme-subsystem/nvme-subsys0/nvme4/state   |  1 +
 .../nvme-subsystem/nvme-subsys1/nvme0/address |  1 +
 .../nvme-subsystem/nvme-subsys1/nvme0/state   |  1 +
 .../multipath/nvme_path_status_expected.json  | 19 ++++++
 src/test/disklist_test.pm                     | 16 +++++
 11 files changed, 106 insertions(+)
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
 create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
 create mode 100644 src/test/disk_tests/multipath/nvme_path_status_expected.json

diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm
index a6730a6..706f696 100644
--- a/src/PVE/Diskmanage.pm
+++ b/src/PVE/Diskmanage.pm
@@ -969,6 +969,69 @@ sub get_multipath_status {
     return $res;
 }
 
+# Queries the path state of NVMe namespaces handled by native NVMe multipath, keyed by the
+# kernel device name of the head namespace (for example nvme0n1). The kernel creates a
+# per-path namespace entry (nvmeXcYnZ) below each controller of a shared subsystem; local
+# single-controller devices have none and are not reported.
+sub get_nvme_path_status {
+    my $res = {};
+
+    dir_glob_foreach(
+        '/sys/class/nvme-subsystem',
+        'nvme-subsys\d+',
+        sub {
+            my ($subsys) = @_;
+            my $subsysdir = "/sys/class/nvme-subsystem/$subsys";
+
+            dir_glob_foreach(
+                $subsysdir,
+                'nvme\d+',
+                sub {
+                    my ($ctrl) = @_;
+                    my $ctrldir = "$subsysdir/$ctrl";
+                    my $state = file_read_firstline("$ctrldir/state") // return;
+                    my $address = file_read_firstline("$ctrldir/address");
+
+                    dir_glob_foreach(
+                        $ctrldir,
+                        'nvme\d+c\d+n\d+',
+                        sub {
+                            my ($pathdev) = @_;
+                            return if $pathdev !~ m/^nvme(\d+)c\d+n(\d+)$/;
+                            my $head = "nvme$1n$2";
+
+                            my $entry = { device => "/dev/$ctrl", state => $state };
+                            $entry->{address} = $address
+                                if defined($address) && $address ne '';
+                            my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state");
+                            $entry->{'ana-state'} = $ana if defined($ana);
+
+                            # 'change' is transient, inaccessible and persistent-loss
+                            # paths cannot serve IO even on a live controller
+                            my $usable = $state eq 'live'
+                                && (!defined($ana)
+                                    || $ana eq 'optimized'
+                                    || $ana eq 'non-optimized');
+
+                            my $status = $res->{$head} //= { active => 0, paths => [] };
+                            $status->{active}++ if $usable;
+                            push @{ $status->{paths} }, $entry;
+                        },
+                    );
+                },
+            );
+        },
+    );
+
+    # readdir order is not deterministic
+    for my $status (values %$res) {
+        @{ $status->{paths} } =
+            sort { $a->{device} cmp $b->{device} } @{ $status->{paths} };
+    }
+
+    return $res;
+}
+
 # Returns the multipath device entry if the given path refers to a multipath mapped device.
 sub lookup_multipath_map {
     my ($devpath) = @_;
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
new file mode 100644
index 0000000..803fab3
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address
@@ -0,0 +1 @@
+traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
new file mode 100644
index 0000000..e48c58f
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state
@@ -0,0 +1 @@
+optimized
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
new file mode 100644
index 0000000..e23fe64
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
new file mode 100644
index 0000000..3661874
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address
@@ -0,0 +1 @@
+traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
new file mode 100644
index 0000000..436c48e
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state
@@ -0,0 +1 @@
+inaccessible
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
new file mode 100644
index 0000000..e23fe64
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
new file mode 100644
index 0000000..f99fcb1
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address
@@ -0,0 +1 @@
+0000:02:00.0
diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
new file mode 100644
index 0000000..e23fe64
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state
@@ -0,0 +1 @@
+live
diff --git a/src/test/disk_tests/multipath/nvme_path_status_expected.json b/src/test/disk_tests/multipath/nvme_path_status_expected.json
new file mode 100644
index 0000000..62611b3
--- /dev/null
+++ b/src/test/disk_tests/multipath/nvme_path_status_expected.json
@@ -0,0 +1,19 @@
+{
+    "nvme2n1": {
+        "active": 1,
+        "paths": [
+            {
+                "device": "/dev/nvme3",
+                "state": "live",
+                "address": "traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171",
+                "ana-state": "optimized"
+            },
+            {
+                "device": "/dev/nvme4",
+                "state": "live",
+                "address": "traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171",
+                "ana-state": "inaccessible"
+            }
+        ]
+    }
+}
diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm
index e804485..0b090f8 100644
--- a/src/test/disklist_test.pm
+++ b/src/test/disklist_test.pm
@@ -113,6 +113,7 @@ sub mocked_file_read_firstline {
     my $originalsub = $diskmanage_module->original('file_read_firstline');
 
     $path =~ s|^/sys/block|disk_tests/$testcasedir|;
+    $path =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|;
 
     return &$originalsub($path);
 }
@@ -122,6 +123,12 @@ sub mocked_dir_glob_foreach {
 
     my $lines = [];
 
+    # nvme subsystem trees are provided as directories, glob them like production code
+    if ($dir =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|) {
+        my $originalsub = $diskmanage_module->original('dir_glob_foreach');
+        return &$originalsub($dir, $regex, $sub);
+    }
+
     # read lines in from file
     if ($dir =~ m{^/sys/block$}) {
         @$lines = split(/\n/, read_test_file('disklist'));
@@ -269,6 +276,15 @@ sub test_disk_list {
             is_deeply($status, $expected_status, 'multipath status should be the same');
         }
 
+        if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") {
+            my $status = PVE::Diskmanage::get_nvme_path_status();
+            my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json'));
+
+            print Dumper($status) if $print;
+            $testcount++;
+            is_deeply($status, $expected_status, 'nvme path status should be the same');
+        }
+
         done_testing($testcount);
     };
 }
-- 
2.47.3