From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id D9BEC1FF0ED for ; Fri, 31 Jul 2026 12:23:59 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 05FB42172A; Fri, 31 Jul 2026 12:22:08 +0200 (CEST) From: Dietmar Maurer To: pve-devel@lists.proxmox.com Subject: [RFC pve-storage 11/27] diskmanage: add helper querying NVMe native multipath path state Date: Fri, 31 Jul 2026 12:21:40 +0200 Message-ID: <20260731102156.3947857-12-dietmar@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260731102156.3947857-1-dietmar@proxmox.com> References: <20260731102156.3947857-1-dietmar@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 2 AWL -0.231 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Message-ID-Hash: XEJKOVN3RUTMWFZ4M3IET2AFJAQZZ3BG X-Message-ID-Hash: XEJKOVN3RUTMWFZ4M3IET2AFJAQZZ3BG X-MailFrom: dietmar@zilli.proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: NVMe over fabrics namespaces are usually handled by native NVMe multipath instead of device mapper, so multipathd knows nothing about them. The kernel exposes the equivalent information in sysfs: a per-path namespace entry below each controller of a shared subsystem, with the controller state and address and the ANA state of the namespace on that path. Collect it per head namespace, so path health can be reported uniformly for both multipath stacks. Local single-controller devices have no per-path entries and are not reported. Signed-off-by: Dietmar Maurer --- src/PVE/Diskmanage.pm | 63 +++++++++++++++++++ .../nvme-subsystem/nvme-subsys0/nvme3/address | 1 + .../nvme-subsys0/nvme3/nvme2c3n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme3/state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/address | 1 + .../nvme-subsys0/nvme4/nvme2c4n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/state | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/address | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/state | 1 + .../multipath/nvme_path_status_expected.json | 19 ++++++ src/test/disklist_test.pm | 16 +++++ 11 files changed, 106 insertions(+) create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state create mode 100644 src/test/disk_tests/multipath/nvme_path_status_expected.json diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index a6730a6..706f696 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -969,6 +969,69 @@ sub get_multipath_status { return $res; } +# Queries the path state of NVMe namespaces handled by native NVMe multipath, keyed by the +# kernel device name of the head namespace (for example nvme0n1). The kernel creates a +# per-path namespace entry (nvmeXcYnZ) below each controller of a shared subsystem; local +# single-controller devices have none and are not reported. +sub get_nvme_path_status { + my $res = {}; + + dir_glob_foreach( + '/sys/class/nvme-subsystem', + 'nvme-subsys\d+', + sub { + my ($subsys) = @_; + my $subsysdir = "/sys/class/nvme-subsystem/$subsys"; + + dir_glob_foreach( + $subsysdir, + 'nvme\d+', + sub { + my ($ctrl) = @_; + my $ctrldir = "$subsysdir/$ctrl"; + my $state = file_read_firstline("$ctrldir/state") // return; + my $address = file_read_firstline("$ctrldir/address"); + + dir_glob_foreach( + $ctrldir, + 'nvme\d+c\d+n\d+', + sub { + my ($pathdev) = @_; + return if $pathdev !~ m/^nvme(\d+)c\d+n(\d+)$/; + my $head = "nvme$1n$2"; + + my $entry = { device => "/dev/$ctrl", state => $state }; + $entry->{address} = $address + if defined($address) && $address ne ''; + my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state"); + $entry->{'ana-state'} = $ana if defined($ana); + + # 'change' is transient, inaccessible and persistent-loss + # paths cannot serve IO even on a live controller + my $usable = $state eq 'live' + && (!defined($ana) + || $ana eq 'optimized' + || $ana eq 'non-optimized'); + + my $status = $res->{$head} //= { active => 0, paths => [] }; + $status->{active}++ if $usable; + push @{ $status->{paths} }, $entry; + }, + ); + }, + ); + }, + ); + + # readdir order is not deterministic + for my $status (values %$res) { + @{ $status->{paths} } = + sort { $a->{device} cmp $b->{device} } @{ $status->{paths} }; + } + + return $res; +} + # Returns the multipath device entry if the given path refers to a multipath mapped device. sub lookup_multipath_map { my ($devpath) = @_; diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address new file mode 100644 index 0000000..803fab3 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address @@ -0,0 +1 @@ +traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state new file mode 100644 index 0000000..e48c58f --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state @@ -0,0 +1 @@ +optimized diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state new file mode 100644 index 0000000..e23fe64 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address new file mode 100644 index 0000000..3661874 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address @@ -0,0 +1 @@ +traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state new file mode 100644 index 0000000..436c48e --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state @@ -0,0 +1 @@ +inaccessible diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state new file mode 100644 index 0000000..e23fe64 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address new file mode 100644 index 0000000..f99fcb1 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address @@ -0,0 +1 @@ +0000:02:00.0 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state new file mode 100644 index 0000000..e23fe64 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme_path_status_expected.json b/src/test/disk_tests/multipath/nvme_path_status_expected.json new file mode 100644 index 0000000..62611b3 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme_path_status_expected.json @@ -0,0 +1,19 @@ +{ + "nvme2n1": { + "active": 1, + "paths": [ + { + "device": "/dev/nvme3", + "state": "live", + "address": "traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "optimized" + }, + { + "device": "/dev/nvme4", + "state": "live", + "address": "traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "inaccessible" + } + ] + } +} diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm index e804485..0b090f8 100644 --- a/src/test/disklist_test.pm +++ b/src/test/disklist_test.pm @@ -113,6 +113,7 @@ sub mocked_file_read_firstline { my $originalsub = $diskmanage_module->original('file_read_firstline'); $path =~ s|^/sys/block|disk_tests/$testcasedir|; + $path =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|; return &$originalsub($path); } @@ -122,6 +123,12 @@ sub mocked_dir_glob_foreach { my $lines = []; + # nvme subsystem trees are provided as directories, glob them like production code + if ($dir =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|) { + my $originalsub = $diskmanage_module->original('dir_glob_foreach'); + return &$originalsub($dir, $regex, $sub); + } + # read lines in from file if ($dir =~ m{^/sys/block$}) { @$lines = split(/\n/, read_test_file('disklist')); @@ -269,6 +276,15 @@ sub test_disk_list { is_deeply($status, $expected_status, 'multipath status should be the same'); } + if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") { + my $status = PVE::Diskmanage::get_nvme_path_status(); + my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json')); + + print Dumper($status) if $print; + $testcount++; + is_deeply($status, $expected_status, 'nvme path status should be the same'); + } + done_testing($testcount); }; } -- 2.47.3