From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id DC1161FF0B7 for ; Fri, 13 Nov 2026 04:32:34 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 9FA3021CAA; Fri, 13 Nov 2026 04:22:16 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH storage 09/17] multipath: report NVMe over Fabrics namespaces in the cluster status Date: Fri, 13 Nov 2026 04:04:37 +0100 Message-ID: <20261113030541.1591528-10-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030541.1591528-1-t.lamprecht@proxmox.com> References: <20261113030541.1591528-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794539148452 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.232 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: U5Q7YK7PGMLPPY2BXF5XFYP5RU4BM5D7 X-Message-ID-Hash: U5Q7YK7PGMLPPY2BXF5XFYP5RU4BM5D7 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The cluster-wide status lists the LUNs that multipathd handles on each node. The namespaces of an NVMe over Fabrics storage use the native multipath of the kernel and were missing from it, so a lost path or a node without access to a namespace only showed in the LUN list of that node. Let each node report the namespaces of the subsystems such a storage names, with the path counts its LUN list already shows, and expect them on the nodes of the storage like the LUNs of an iSCSI storage. They do not depend on multipathd, so a node without it reports them as well and only the state of its maps stays unknown. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Multipath.pm | 6 ++ src/PVE/Multipath/Status.pm | 58 +++++++++++++++--- src/test/run_multipath_api_tests.pl | 91 ++++++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 10 deletions(-) diff --git a/src/PVE/API2/Multipath.pm b/src/PVE/API2/Multipath.pm index 1d7f5707..82d14c9f 100644 --- a/src/PVE/API2/Multipath.pm +++ b/src/PVE/API2/Multipath.pm @@ -121,6 +121,12 @@ __PACKAGE__->register_method({ wwid => { type => 'string', description => 'The LUN WWID.' }, explicit => { type => 'boolean', description => 'An explicit entry exists.' }, + native => { + type => 'boolean', + optional => 1, + description => 'An NVMe namespace, which uses the native' + . ' multipath of the kernel.', + }, name => { type => 'string', optional => 1 }, 'used-by-storage' => { type => 'string', optional => 1 }, 'multipath-storage' => { diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm index 9dc8e2cf..092d26a6 100644 --- a/src/PVE/Multipath/Status.pm +++ b/src/PVE/Multipath/Status.pm @@ -3,6 +3,7 @@ package PVE::Multipath::Status; use v5.36; use JSON qw(decode_json); use PVE::Cluster; +use PVE::Diskmanage; use PVE::INotify; use PVE::Multipath; use PVE::Multipath::Config; @@ -21,6 +22,39 @@ sub worst_state(@states) { return $worst; } +# Namespaces use the native multipath of the kernel, so their state does not depend on multipathd. +my sub namespace_luns($pv_vgs) { + require PVE::Storage; + my $cfg = PVE::Storage::config(); + my $status = PVE::Diskmanage::get_nvme_path_status(); + my %luns; + for my $sid (sort keys $cfg->{ids}->%*) { + my $scfg = $cfg->{ids}->{$sid}; + next if $scfg->{type} ne 'nvmeof'; + my $expected = eval { scalar(PVE::NVMeOF::storage_paths($scfg)->@*) }; + my $namespaces = PVE::NVMeOF::namespaces($scfg->{subsysnqn}); + for my $name (sort keys %$namespaces) { + my $wwid = $namespaces->{$name}->{wwid}; + next if $luns{$wwid}; + my $paths = $status->{$name} // { active => 0, paths => [] }; + my $total = scalar($paths->{paths}->@*); + my $lun = $luns{$wwid} = { + wwid => $wwid, + native => 1, + size => $namespaces->{$name}->{size}, + storage => $sid, + 'paths-total' => $total, + 'paths-active' => $paths->{active}, + state => PVE::Multipath::derive_health($paths->{active}, $total, $expected), + }; + $lun->{'paths-expected'} = $expected if defined($expected); + my $vg = $pv_vgs->{"/dev/$name"}; + $lun->{vgname} = $vg if defined($vg); + } + } + return [map { $luns{$_} } sort keys %luns]; +} + sub publish($report) { my $node = PVE::INotify::nodename(); my $now = time(); @@ -51,11 +85,11 @@ sub update_node_status() { require PVE::NVMeOF; my $host = eval { PVE::NVMeOF::host_identity() }; $report->{'nvme-host-nqn'} = $host->{nqn} if $host; + require PVE::Storage::LVMPlugin; + my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups(); eval { die "multipath-tools is not installed\n" if !PVE::Multipath::is_supported(); die "multipathd is not running\n" if !PVE::Multipath::is_running(); - require PVE::Storage::LVMPlugin; - my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups(); for my $map (PVE::Multipath::get_maps()->@*) { my $wwid = $map->{wwid} // next; my $info = $apply->{wwids}->{$wwid} // {}; @@ -82,8 +116,11 @@ sub update_node_status() { if (my $error = $@) { chomp $error; $report->{error} = $error; - delete $report->{luns}; + $report->{luns} = []; } + my $namespaces = eval { namespace_luns($pv_vgs) }; + warn "multipath: cannot query NVMe namespaces - $@" if $@; + push $report->{luns}->@*, @$namespaces if $namespaces; eval { publish($report) }; warn "multipath: cannot publish node status - $@" if $@; } @@ -96,7 +133,7 @@ sub cluster_status() { my $all_nodes = PVE::Cluster::get_nodelist(); my $members = PVE::Cluster::get_members(); my $raw = PVE::Cluster::get_node_kv($KEY); - my (%nodes, %luns, %expected, %observed, %vg_owner); + my (%nodes, %luns, %expected, %observed, %vg_owner, %lists); my $storage_nodes = sub($scfg) { return $scfg->{nodes} && $scfg->{nodes}->%* ? [keys $scfg->{nodes}->%*] : $all_nodes; }; @@ -119,11 +156,12 @@ sub cluster_status() { my $scfg = $storage->{ids}->{$sid}; $vg_owner{ $scfg->{vgname} } //= $sid if defined($scfg->{vgname}); next if $scfg->{type} ne 'lvm'; - my ($wwid) = ($scfg->{base} // '') =~ /(?:^|\.)dm-uuid-mpath-(\S+)$/; + my ($base, $volname) = PVE::Storage::parse_volume_id($scfg->{base} // '', 1); + my ($kind, $wwid) = ($volname // '') =~ /(?:^|\.)(dm-uuid-mpath|nvme)-(\S+)$/; next if !defined($wwid); $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $luns{$wwid}->{native} = 1 if $kind eq 'nvme'; $luns{$wwid}->{'used-by-storage'} //= $sid; - my ($base) = $scfg->{base} =~ /^([^:]+):/; $luns{$wwid}->{'multipath-storage'} //= $base if $requests_maps->($base); $expected{$wwid}->{$_} = 1 for $storage_nodes->($scfg)->@*; } @@ -140,11 +178,13 @@ sub cluster_status() { for grep { defined($report->{$_}) } qw(error apply-error iscsi-initiator-name nvme-host-nqn); $entry->{state} = defined($report->{error}) ? 'unavailable' : 'ok'; - next if $entry->{state} ne 'ok'; + # an unavailable node lists its namespaces only, or nothing + $lists{$node} = 1 if $report->{luns}; for my $lun (($report->{luns} // [])->@*) { my $wwid = $lun->{wwid} // next; my $row = $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $row->{native} = 1 if $lun->{native}; $row->{$_} //= $lun->{$_} for grep { defined($lun->{$_}) } qw(name size); my $owner = defined($lun->{vgname}) ? $vg_owner{ $lun->{vgname} } : undef; $row->{'used-by-storage'} //= $owner if defined($owner); @@ -176,8 +216,8 @@ sub cluster_status() { if ($observed{$wwid}->{$node}) { $row->{nodes}->{$node} = $observed{$wwid}->{$node}; } elsif ($expected{$wwid}->{$node}) { - $row->{nodes}->{$node} = - { state => $nodes{$node}->{state} eq 'ok' ? 'missing' : 'unknown' }; + my $listed = $nodes{$node}->{state} eq 'ok' || ($row->{native} && $lists{$node}); + $row->{nodes}->{$node} = { state => $listed ? 'missing' : 'unknown' }; } } $row->{state} = worst_state(map { $_->{state} } values $row->{nodes}->%*); diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl index 9bf44733..bd62f756 100755 --- a/src/test/run_multipath_api_tests.pl +++ b/src/test/run_multipath_api_tests.pl @@ -215,7 +215,12 @@ subtest 'one entry point applies configuration and reports unavailable reasons' my $status = Test::MockModule->new('PVE::Multipath::Status'); my $mp = Test::MockModule->new('PVE::Multipath'); my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + my $storage = Test::MockModule->new('PVE::Storage'); + my $disks = Test::MockModule->new('PVE::Diskmanage'); my ($report, @order); + my $ids = {}; + $storage->mock(config => sub { { ids => $ids } }); + $disks->mock(get_nvme_path_status => sub { { nvme1n1 => { active => 1, paths => [{}] } } }); $gen->mock(sync => sub { push @order, 'apply'; { error => 'reload failed' } }); $status->mock(publish => sub { push @order, 'publish'; $report = $_[0] }); $iscsi->mock(read_initiator_info => sub { { name => 'iqn.2025-01.example:node' } }); @@ -249,6 +254,38 @@ subtest 'one entry point applies configuration and reports unavailable reasons' is($report->{luns}->[0]->{'paths-expected'}, 3, 'expected count reported'); is($report->{luns}->[0]->{state}, 'degraded', 'fewer paths than expected'); ok(!exists($report->{supported}), 'no separate capability state'); + + $ids->{nvme} = { + type => 'nvmeof', + subsysnqn => 'nqn.test:sub', + 'nvme-portals' => '192.0.2.1,192.0.2.2', + }; + $nvme->mock( + namespaces => sub($nqn) { + return $nqn eq 'nqn.test:sub' + ? { nvme1n1 => { wwid => 'uuid.1', size => 4096 } } + : {}; + }, + ); + my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); + $lvm->mock(cached_pv_volume_groups => sub { { '/dev/nvme1n1' => 'vgns' } }); + my $namespace = { + wwid => 'uuid.1', + native => 1, + size => 4096, + storage => 'nvme', + vgname => 'vgns', + 'paths-total' => 1, + 'paths-active' => 1, + 'paths-expected' => 2, + state => 'degraded', + }; + PVE::Multipath::Status::update_node_status(); + is_deeply($report->{luns}->[1], $namespace, 'namespace with fewer paths than portals'); + $mp->mock(is_running => sub { 0 }); + PVE::Multipath::Status::update_node_status(); + is($report->{error}, 'multipathd is not running', 'stopped daemon still reported'); + is_deeply($report->{luns}, [$namespace], 'namespaces do not need multipathd'); }; subtest 'API returns final node and LUN states' => sub { @@ -285,12 +322,31 @@ subtest 'API returns final node and LUN states' => sub { 'paths-active' => 2, storage => 'scoped', }, + { + wwid => 'uuid.1', + native => 1, + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + storage => 'nvme', + }, ], }), n2 => encode_json({ time => $NOW, luns => [], 'nvme-host-nqn' => 'nqn.test:n2' }), n3 => encode_json({ time => $NOW - 181, luns => [] }), n4 => encode_json({ time => $NOW, luns => [] }), - n5 => encode_json({ time => $NOW, error => 'multipath-tools is not installed' }), + n5 => encode_json({ + time => $NOW, + error => 'multipath-tools is not installed', + luns => [{ + wwid => 'uuid.1', + native => 1, + state => 'degraded', + 'paths-total' => 2, + 'paths-active' => 1, + storage => 'nvme', + }], + }), ); $cluster->mock(get_node_kv => sub { \%raw }); $config->mock(read_config => sub { { ids => { '3600a' => { type => 'wwid' } } } }); @@ -306,6 +362,8 @@ subtest 'API returns final node and LUN states' => sub { gone => { type => 'iscsi', multipath => 1, nodes => { departed => 1 } }, lvmgone => { type => 'lvm', base => 'gone:0.0.0.dm-uuid-mpath-3600e' }, scoped => { type => 'iscsi', multipath => 1, nodes => { n2 => 1 } }, + nvme => { type => 'nvmeof', subsysnqn => 'nqn.test:sub' }, + lvmns => { type => 'lvm', base => 'nvme:nvme-uuid.2', nodes => { n5 => 1 } }, }, } }, ); @@ -345,6 +403,37 @@ subtest 'API returns final node and LUN states' => sub { 'the nodes of a shared LVM expect the map', ); is($lun{'3600d'}->{'used-by-storage'}, 'shared', 'shared LVM attributed by volume group'); + my $namespace = $lun{'uuid.1'}; + ok($namespace->{native}, 'namespace marked as native'); + ok(!exists($namespace->{'multipath-storage'}), 'a namespace has no requested map'); + is_deeply( + { map { $_ => $namespace->{nodes}->{$_}->{state} } keys $namespace->{nodes}->%* }, + { + n1 => 'optimal', + n2 => 'missing', + n3 => 'unknown', + n4 => 'unknown', + n5 => 'degraded', + }, + 'namespace state of a node without multipathd counts', + ); + is($lun->{nodes}->{n5}->{state}, 'unknown', 'no map state from a node without multipathd'); + is_deeply( + $lun{'uuid.2'}, + { + wwid => 'uuid.2', + explicit => 0, + native => 1, + 'used-by-storage' => 'lvmns', + state => 'missing', + nodes => { n5 => { state => 'missing' } }, + }, + 'base namespace that a node without multipathd does not report', + ); + $raw{n5} = encode_json({ time => $NOW, error => 'too many LUNs to report' }); + $res = PVE::API2::Multipath->status({}); + %lun = map { $_->{wwid} => $_ } $res->{luns}->@*; + is($lun{'uuid.2'}->{nodes}->{n5}->{state}, 'unknown', 'node without a list'); eval { PVE::JSONSchema::validate( $res,