From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 7C85B1FF0AF for ; Fri, 06 Nov 2026 01:52:41 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 6A74E218AE; Fri, 06 Nov 2026 01:51:01 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v3 storage 23/76] diskmanage: collect native NVMe multipath path state Date: Fri, 6 Nov 2026 00:30:58 +0100 Message-ID: <20261105233336.2178064-24-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261105233336.2178064-1-t.lamprecht@proxmox.com> References: <20261105233336.2178064-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793926178766 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.329 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: SQYJATRLXHWDF3RP2DKHKAPDUOP6LPWA X-Message-ID-Hash: SQYJATRLXHWDF3RP2DKHKAPDUOP6LPWA X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: From: Dietmar Maurer NVMe over Fabrics usually uses native multipath, whose paths are unknown to multipathd. Collect controller state and address, plus per-path namespace ANA state, from sysfs and group them by head namespace. This allows uniform health reporting for both multipath stacks. Local single-controller devices have no per-path entries and are omitted. Signed-off-by: Dietmar Maurer [TL: drop redundant comments] Signed-off-by: Thomas Lamprecht --- src/PVE/Diskmanage.pm | 60 +++++++++++++++++++ .../nvme-subsystem/nvme-subsys0/nvme3/address | 1 + .../nvme-subsys0/nvme3/nvme2c3n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme3/state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/address | 1 + .../nvme-subsys0/nvme4/nvme2c4n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/state | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/address | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/state | 1 + .../multipath/nvme_path_status_expected.json | 19 ++++++ src/test/disklist_test.pm | 16 +++++ 11 files changed, 103 insertions(+) create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state create mode 100644 src/test/disk_tests/multipath/nvme_path_status_expected.json diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index caad0fee..a7eb2b21 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -974,6 +974,66 @@ sub is_multipath_device { return $uuid =~ m/^(?:part\d+-)?mpath-/ ? 1 : 0; } +# Native multipath creates per-path namespace children nvmeXcYnZ below controllers. Local +# single-controller devices lack these children and are omitted. +sub get_nvme_path_status { + my $res = {}; + + dir_glob_foreach( + '/sys/class/nvme-subsystem', + 'nvme-subsys\d+', + sub { + my ($subsys) = @_; + my $subsysdir = "/sys/class/nvme-subsystem/$subsys"; + + dir_glob_foreach( + $subsysdir, + 'nvme\d+', + sub { + my ($ctrl) = @_; + my $ctrldir = "$subsysdir/$ctrl"; + my $state = file_read_firstline("$ctrldir/state") // return; + my $address = file_read_firstline("$ctrldir/address"); + + dir_glob_foreach( + $ctrldir, + 'nvme\d+c\d+n\d+', + sub { + my ($pathdev) = @_; + return if $pathdev !~ m/^nvme(\d+)c\d+n(\d+)$/; + my $head = "nvme$1n$2"; + + my $entry = { device => "/dev/$ctrl", state => $state }; + $entry->{address} = $address + if defined($address) && $address ne ''; + my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state"); + $entry->{'ana-state'} = $ana if defined($ana); + + # 'change' is transient; paths in 'inaccessible' or 'persistent-loss' + # cannot serve IO even on a live controller + my $usable = $state eq 'live' + && (!defined($ana) + || $ana eq 'optimized' + || $ana eq 'non-optimized'); + + my $status = $res->{$head} //= { active => 0, paths => [] }; + $status->{active}++ if $usable; + push @{ $status->{paths} }, $entry; + }, + ); + }, + ); + }, + ); + + for my $status (values %$res) { + @{ $status->{paths} } = + sort { $a->{device} cmp $b->{device} } @{ $status->{paths} }; + } + + return $res; +} + sub lookup_multipath_map { my ($devpath) = @_; diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address new file mode 100644 index 00000000..803fab3e --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address @@ -0,0 +1 @@ +traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state new file mode 100644 index 00000000..e48c58f5 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state @@ -0,0 +1 @@ +optimized diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address new file mode 100644 index 00000000..36618744 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address @@ -0,0 +1 @@ +traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state new file mode 100644 index 00000000..436c48ee --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state @@ -0,0 +1 @@ +inaccessible diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address new file mode 100644 index 00000000..f99fcb14 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address @@ -0,0 +1 @@ +0000:02:00.0 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme_path_status_expected.json b/src/test/disk_tests/multipath/nvme_path_status_expected.json new file mode 100644 index 00000000..62611b3f --- /dev/null +++ b/src/test/disk_tests/multipath/nvme_path_status_expected.json @@ -0,0 +1,19 @@ +{ + "nvme2n1": { + "active": 1, + "paths": [ + { + "device": "/dev/nvme3", + "state": "live", + "address": "traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "optimized" + }, + { + "device": "/dev/nvme4", + "state": "live", + "address": "traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "inaccessible" + } + ] + } +} diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm index 21823daa..c6fb620c 100644 --- a/src/test/disklist_test.pm +++ b/src/test/disklist_test.pm @@ -111,6 +111,7 @@ sub mocked_file_read_firstline { my $originalsub = $diskmanage_module->original('file_read_firstline'); $path =~ s|^/sys/block|disk_tests/$testcasedir|; + $path =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|; return &$originalsub($path); } @@ -120,6 +121,12 @@ sub mocked_dir_glob_foreach { my $lines = []; + # nvme subsystem trees are provided as directories, glob them like production code + if ($dir =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|) { + my $originalsub = $diskmanage_module->original('dir_glob_foreach'); + return &$originalsub($dir, $regex, $sub); + } + # read lines in from file if ($dir =~ m{^/sys/block$}) { @$lines = split(/\n/, read_test_file('disklist')); @@ -267,6 +274,15 @@ sub test_disk_list { is($@, '', 'the SAN opt-in accepts an unused iSCSI LUN'); } + if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") { + my $status = PVE::Diskmanage::get_nvme_path_status(); + my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json')); + + print Dumper($status) if $print; + $testcount++; + is_deeply($status, $expected_status, 'nvme path status should be the same'); + } + done_testing($testcount); }; } -- 2.47.3