From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 6F5731FF0B7 for ; Fri, 13 Nov 2026 04:03:35 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 47D9C217AB; Fri, 13 Nov 2026 04:02:05 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v4 storage 05/36] diskmanage: collect native NVMe multipath path state Date: Fri, 13 Nov 2026 04:00:23 +0100 Message-ID: <20261113030146.1559030-6-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com> References: <20261113030146.1559030-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794538913779 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.249 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: C2KLW2CTFSJ4B5BD2PI5YHBHIGJQXROT X-Message-ID-Hash: C2KLW2CTFSJ4B5BD2PI5YHBHIGJQXROT X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: From: Dietmar Maurer NVMe over Fabrics usually uses native multipath, whose paths multipathd does not know. Collect the controller state and address and the ANA state of each path from sysfs, grouped by namespace, so the SAN LUN inventory can report path health for both multipath stacks. Devices with a single local controller have no per-path entries and are left out. Signed-off-by: Dietmar Maurer [TL: drop redundant comments] Signed-off-by: Thomas Lamprecht --- src/PVE/Diskmanage.pm | 60 +++++++++++++++++++ .../nvme-subsystem/nvme-subsys0/nvme3/address | 1 + .../nvme-subsys0/nvme3/nvme2c3n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme3/state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/address | 1 + .../nvme-subsys0/nvme4/nvme2c4n1/ana_state | 1 + .../nvme-subsystem/nvme-subsys0/nvme4/state | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/address | 1 + .../nvme-subsystem/nvme-subsys1/nvme0/state | 1 + .../multipath/nvme_path_status_expected.json | 19 ++++++ src/test/disklist_test.pm | 16 +++++ 11 files changed, 103 insertions(+) create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address create mode 100644 src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state create mode 100644 src/test/disk_tests/multipath/nvme_path_status_expected.json diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index 5043a83d..d5251095 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -816,6 +816,66 @@ sub get_disks { return $disklist; } +# Native multipath creates per-path namespace children nvmeXcYnZ below controllers. Local +# single-controller devices lack these children and are omitted. +sub get_nvme_path_status { + my $res = {}; + + dir_glob_foreach( + '/sys/class/nvme-subsystem', + 'nvme-subsys\d+', + sub { + my ($subsys) = @_; + my $subsysdir = "/sys/class/nvme-subsystem/$subsys"; + + dir_glob_foreach( + $subsysdir, + 'nvme\d+', + sub { + my ($ctrl) = @_; + my $ctrldir = "$subsysdir/$ctrl"; + my $state = file_read_firstline("$ctrldir/state") // return; + my $address = file_read_firstline("$ctrldir/address"); + + dir_glob_foreach( + $ctrldir, + 'nvme\d+c\d+n\d+', + sub { + my ($pathdev) = @_; + return if $pathdev !~ m/^nvme(\d+)c\d+n(\d+)$/; + my $head = "nvme$1n$2"; + + my $entry = { device => "/dev/$ctrl", state => $state }; + $entry->{address} = $address + if defined($address) && $address ne ''; + my $ana = file_read_firstline("$ctrldir/$pathdev/ana_state"); + $entry->{'ana-state'} = $ana if defined($ana); + + # 'change' is transient; paths in 'inaccessible' or 'persistent-loss' + # cannot serve IO even on a live controller + my $usable = $state eq 'live' + && (!defined($ana) + || $ana eq 'optimized' + || $ana eq 'non-optimized'); + + my $status = $res->{$head} //= { active => 0, paths => [] }; + $status->{active}++ if $usable; + push @{ $status->{paths} }, $entry; + }, + ); + }, + ); + }, + ); + + for my $status (values %$res) { + @{ $status->{paths} } = + sort { $a->{device} cmp $b->{device} } @{ $status->{paths} }; + } + + return $res; +} + sub get_partnum { my ($part_path) = @_; diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address new file mode 100644 index 00000000..803fab3e --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/address @@ -0,0 +1 @@ +traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state new file mode 100644 index 00000000..e48c58f5 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/nvme2c3n1/ana_state @@ -0,0 +1 @@ +optimized diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme3/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address new file mode 100644 index 00000000..36618744 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/address @@ -0,0 +1 @@ +traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state new file mode 100644 index 00000000..436c48ee --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/nvme2c4n1/ana_state @@ -0,0 +1 @@ +inaccessible diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys0/nvme4/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address new file mode 100644 index 00000000..f99fcb14 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/address @@ -0,0 +1 @@ +0000:02:00.0 diff --git a/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state new file mode 100644 index 00000000..e23fe645 --- /dev/null +++ b/src/test/disk_tests/multipath/nvme-subsystem/nvme-subsys1/nvme0/state @@ -0,0 +1 @@ +live diff --git a/src/test/disk_tests/multipath/nvme_path_status_expected.json b/src/test/disk_tests/multipath/nvme_path_status_expected.json new file mode 100644 index 00000000..62611b3f --- /dev/null +++ b/src/test/disk_tests/multipath/nvme_path_status_expected.json @@ -0,0 +1,19 @@ +{ + "nvme2n1": { + "active": 1, + "paths": [ + { + "device": "/dev/nvme3", + "state": "live", + "address": "traddr=192.168.3.209,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "optimized" + }, + { + "device": "/dev/nvme4", + "state": "live", + "address": "traddr=192.168.3.109,trsvcid=4420,src_addr=192.168.2.171", + "ana-state": "inaccessible" + } + ] + } +} diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm index bd70dc33..0d34c0db 100644 --- a/src/test/disklist_test.pm +++ b/src/test/disklist_test.pm @@ -103,6 +103,7 @@ sub mocked_file_read_firstline { my $originalsub = $diskmanage_module->original('file_read_firstline'); $path =~ s|^/sys/block|disk_tests/$testcasedir|; + $path =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|; return &$originalsub($path); } @@ -112,6 +113,12 @@ sub mocked_dir_glob_foreach { my $lines = []; + # nvme subsystem trees are provided as directories, glob them like production code + if ($dir =~ s|^/sys/class/nvme-subsystem|disk_tests/$testcasedir/nvme-subsystem|) { + my $originalsub = $diskmanage_module->original('dir_glob_foreach'); + return &$originalsub($dir, $regex, $sub); + } + # read lines in from file if ($dir =~ m{^/sys/block$}) { @$lines = split(/\n/, read_test_file('disklist')); @@ -230,6 +237,15 @@ sub test_disk_list { ok($san_disks->{sda}, 'SAN inventory opts into iSCSI'); } + if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") { + my $status = PVE::Diskmanage::get_nvme_path_status(); + my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json')); + + print Dumper($status) if $print; + $testcount++; + is_deeply($status, $expected_status, 'nvme path status should be the same'); + } + done_testing($testcount); }; }