From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 448691FF0B2 for ; Sun, 22 Nov 2026 02:28:19 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id C8B0D216C3; Sun, 22 Nov 2026 02:27:07 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v3 storage 09/18] multipath: report NVMe over Fabrics namespaces in the cluster status Date: Sun, 22 Nov 2026 02:25:46 +0100 Message-ID: <20261122012605.1848824-10-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261122012605.1848824-1-t.lamprecht@proxmox.com> References: <20261122012605.1848824-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795310771987 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.145 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: Y3MWIAQYNUJX65MJEC4SCGP4CBWMIMTN X-Message-ID-Hash: Y3MWIAQYNUJX65MJEC4SCGP4CBWMIMTN X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The cluster-wide status lists the LUNs that multipathd handles on each node. The namespaces of an NVMe over Fabrics storage use the native multipath of the kernel and were missing from it, so a lost path or a node without access to a namespace only showed in the LUN list of that node. Let each node report the namespaces of the subsystems such a storage names, with the path counts its LUN list already shows, and expect them on the nodes of the storage like the LUNs of an iSCSI storage. They do not depend on multipathd, so a node without it reports them as well and only the state of its maps stays unknown. A node that cannot list its namespaces marks its report, so that they show as unknown there instead of missing. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/SAN/Cluster.pm | 17 +++- src/PVE/Multipath/Status.pm | 67 ++++++++++++++-- src/test/run_multipath_api_tests.pl | 120 +++++++++++++++++++++++++++- 3 files changed, 191 insertions(+), 13 deletions(-) diff --git a/src/PVE/API2/SAN/Cluster.pm b/src/PVE/API2/SAN/Cluster.pm index ac372d65..4d7f0011 100644 --- a/src/PVE/API2/SAN/Cluster.pm +++ b/src/PVE/API2/SAN/Cluster.pm @@ -63,7 +63,8 @@ my $lun_node_schema = { minimum => 0, optional => 1, description => 'The number of active paths of the map whose checker state is ready,' - . ' ghost, or not reported.', + . ' ghost, or not reported. For an NVMe namespace, the number of its paths over a' + . ' live controller whose ANA state is optimized, non-optimized, or not reported.', }, 'paths-total' => { type => 'integer', @@ -77,7 +78,8 @@ my $lun_node_schema = { optional => 1, description => 'The number of paths the LUN should have on the node: the highest of' . ' the count of its multipath entry and, for each managed iSCSI storage of the' - . ' LUN, the number of its portals.', + . ' LUN, the number of its portals. For an NVMe namespace, the number of portals' + . ' of its NVMe over Fabrics storage.', }, }, }; @@ -165,6 +167,12 @@ __PACKAGE__->register_method({ wwid => { type => 'string', description => 'The LUN WWID.' }, explicit => { type => 'boolean', description => 'An explicit entry exists.' }, + native => { + type => 'boolean', + optional => 1, + description => 'An NVMe namespace, which uses the native' + . ' multipath of the kernel.', + }, name => { type => 'string', optional => 1, @@ -177,8 +185,9 @@ __PACKAGE__->register_method({ optional => 1, description => 'The storage that uses the LUN: as its base volume or through' - . ' the volume group on it, or as an iSCSI storage that serves' - . ' the LUNs of its target as images.', + . ' the volume group on it, or as an iSCSI or NVMe over Fabrics' + . ' storage that serves the LUNs of its target or subsystem as' + . ' images.', }, 'multipath-storage' => { type => 'string', diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm index 1b85e108..baae32a6 100644 --- a/src/PVE/Multipath/Status.pm +++ b/src/PVE/Multipath/Status.pm @@ -9,6 +9,7 @@ use MIME::Base64 qw(decode_base64 encode_base64); use PVE::Cluster; use PVE::INotify; +use PVE::Diskmanage; use PVE::Multipath; use PVE::Multipath::Config; use PVE::Multipath::Generator; @@ -37,6 +38,47 @@ sub worst_state(@states) { return $worst; } +# Namespaces use the native multipath of the kernel, so their state does not depend on multipathd. +my sub namespace_luns($pv_vgs) { + require PVE::Storage; + my $cfg = PVE::Storage::config(); + my $node = PVE::INotify::nodename(); + my @storages = grep { + my $scfg = $cfg->{ids}->{$_}; + $scfg->{type} eq 'nvmeof' + && !$scfg->{disable} + && (!$scfg->{nodes} || $scfg->{nodes}->{$node}) + } sort keys $cfg->{ids}->%*; + # reading the paths walks all NVMe namespaces of the node, also those of other plugins + return [] if !@storages; + my $status = PVE::Diskmanage::get_nvme_path_status(); + my %luns; + for my $sid (@storages) { + my $scfg = $cfg->{ids}->{$sid}; + my $expected = eval { scalar(PVE::NVMeOF::storage_paths($scfg)->@*) }; + my $namespaces = PVE::NVMeOF::namespaces($scfg->{subsysnqn}); + for my $name (sort keys %$namespaces) { + my $wwid = $namespaces->{$name}->{wwid}; + next if $luns{$wwid}; + my $paths = $status->{$name} // { active => 0, paths => [] }; + my $total = scalar($paths->{paths}->@*); + my $lun = $luns{$wwid} = { + wwid => $wwid, + native => 1, + size => $namespaces->{$name}->{size}, + storages => [$sid], + 'paths-total' => $total, + 'paths-active' => $paths->{active}, + state => PVE::Multipath::derive_health($paths->{active}, $total, $expected), + }; + $lun->{'paths-expected'} = $expected if defined($expected); + my $vg = ($pv_vgs // {})->{"/dev/$name"}; + $lun->{vgname} = $vg if defined($vg); + } + } + return [map { $luns{$_} } sort keys %luns]; +} + sub publish($report) { my $node = PVE::INotify::nodename(); my $now = time(); @@ -68,11 +110,11 @@ sub update_node_status() { require PVE::NVMeOF; my $host = eval { PVE::NVMeOF::host_identity() }; $report->{'nvme-host-nqn'} = $host->{nqn} if $host; + require PVE::Storage::LVMPlugin; + my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups(); eval { die "multipath-tools is not installed\n" if !PVE::Multipath::is_supported(); die "multipathd is not running or does not respond\n" if !PVE::Multipath::is_running(); - require PVE::Storage::LVMPlugin; - my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups(); # A node can have many maps that something else set up, for example a storage plugin with # one LUN per guest disk. Only the maps that the node manages, that are the base volume of a # storage or hold its volume group get a row, the others are counted. @@ -134,8 +176,15 @@ sub update_node_status() { if (my $error = $@) { chomp $error; $report->{error} = $error; - delete $report->{luns}; + $report->{luns} = []; } + my $namespaces = eval { namespace_luns($pv_vgs) }; + if (my $error = $@) { + warn "multipath: cannot query NVMe namespaces - $error"; + # a consumer must not take the namespaces as gone + $report->{'namespaces-unknown'} = 1; + } + push $report->{luns}->@*, @$namespaces if $namespaces; eval { publish($report) }; warn "multipath: cannot publish node status - $@" if $@; return $report; @@ -152,7 +201,7 @@ sub cluster_status() { my %reports = map { $_ => decode_report($raw->{$_}) // {} } @$all_nodes; my %reported = map { $_->{wwid} => 1 } grep { defined($_->{wwid}) } map { ($_->{luns} // [])->@* } values %reports; - my (%nodes, %luns, %expected, %observed, %vg_owner, %disabled_owner); + my (%nodes, %luns, %expected, %observed, %vg_owner, %lists, %disabled_owner); # the nodes that list a map because the configuration names it for them my (%managed, %unhinted); my $storage_nodes = sub($scfg) { @@ -178,7 +227,6 @@ sub cluster_status() { $vg_owner{ $scfg->{vgname} } //= $sid if defined($scfg->{vgname}); next if $scfg->{type} ne 'lvm'; my $base = PVE::Multipath::parse_base_volume($scfg->{base}) // next; - next if $base->{kind} eq 'nvme'; # a namespace, which has no map # A base volume by its SCSI name is a map only if its iSCSI storage requests maps. The name # can also be another by-id link of the LUN, so only a WWID that a node reports counts. my $map = $requests_maps->($base->{storage}) && $reported{ $base->{wwid} }; @@ -190,6 +238,7 @@ sub cluster_status() { next; } $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $luns{$wwid}->{native} = 1 if $base->{kind} eq 'nvme'; $luns{$wwid}->{'used-by-storage'} //= $sid; $managed{$wwid}->{$_} = 1 for @$all_nodes; $luns{$wwid}->{'multipath-storage'} //= $base->{storage} @@ -206,11 +255,13 @@ sub cluster_status() { $entry->{$_} = $report->{$_} for grep { defined($report->{$_}) } @keys; $entry->{state} = defined($report->{error}) ? 'unavailable' : 'ok'; $unhinted{$node} = 1 if $report->{'vgs-unknown'}; - next if $entry->{state} ne 'ok'; + # the node lists its namespaces, also if it is unavailable for its maps + $lists{$node} = 1 if $report->{luns} && !$report->{'namespaces-unknown'}; for my $lun (($report->{luns} // [])->@*) { my $wwid = $lun->{wwid} // next; my $row = $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $row->{native} = 1 if $lun->{native}; $row->{$_} //= $lun->{$_} for grep { defined($lun->{$_}) } qw(name size); my $owner = defined($lun->{vgname}) ? $vg_owner{ $lun->{vgname} } : undef; $row->{'used-by-storage'} //= $owner if defined($owner); @@ -244,10 +295,10 @@ sub cluster_status() { if ($observed{$wwid}->{$node}) { $row->{nodes}->{$node} = $observed{$wwid}->{$node}; } elsif ($expected{$wwid}->{$node}) { - my $listed = $nodes{$node}->{state} eq 'ok'; + my $listed = $row->{native} ? $lists{$node} : $nodes{$node}->{state} eq 'ok'; # without the hint file of LVM a node leaves a map out that it only knows through # the volume group of a storage - $listed = 0 if $unhinted{$node} && !$managed{$wwid}->{$node}; + $listed = 0 if $unhinted{$node} && !$managed{$wwid}->{$node} && !$row->{native}; $row->{nodes}->{$node} = { state => $listed ? 'missing' : 'unknown' }; } } diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl index a91332df..85ddce7b 100755 --- a/src/test/run_multipath_api_tests.pl +++ b/src/test/run_multipath_api_tests.pl @@ -686,7 +686,12 @@ subtest 'one entry point applies configuration and reports unavailable reasons' my $status = Test::MockModule->new('PVE::Multipath::Status'); my $mp = Test::MockModule->new('PVE::Multipath'); my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + my $storage = Test::MockModule->new('PVE::Storage'); + my $disks = Test::MockModule->new('PVE::Diskmanage'); my ($report, @order); + my $ids = {}; + $storage->mock(config => sub { { ids => $ids } }); + $disks->mock(get_nvme_path_status => sub { { nvme1n1 => { active => 1, paths => [{}] } } }); $gen->mock(sync => sub { push @order, 'apply'; { error => 'reload failed' } }); $status->mock(publish => sub { push @order, 'publish'; $report = $_[0] }); $iscsi->mock(read_initiator_info => sub { { name => 'iqn.2025-01.example:node' } }); @@ -728,6 +733,58 @@ subtest 'one entry point applies configuration and reports unavailable reasons' is($report->{luns}->[0]->{'paths-expected'}, 3, 'expected count reported'); is($report->{luns}->[0]->{state}, 'degraded', 'fewer paths than expected'); ok(!exists($report->{supported}), 'the report has no separate "supported" field'); + + $ids->{nvme} = { + type => 'nvmeof', + subsysnqn => 'nqn.test:sub', + portals => '192.0.2.1,192.0.2.2', + }; + $nvme->mock( + namespaces => sub($nqn) { + return $nqn eq 'nqn.test:sub' + ? { nvme1n1 => { wwid => 'uuid.1', size => 4096 } } + : {}; + }, + ); + my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); + $lvm->mock(cached_pv_volume_groups => sub { { '/dev/nvme1n1' => 'vgns' } }); + my $namespace = { + wwid => 'uuid.1', + native => 1, + size => 4096, + storages => ['nvme'], + vgname => 'vgns', + 'paths-total' => 1, + 'paths-active' => 1, + 'paths-expected' => 2, + state => 'degraded', + }; + PVE::Multipath::Status::update_node_status(); + is_deeply($report->{luns}->[1], $namespace, 'namespace with fewer paths than portals'); + $mp->mock(is_running => sub { 0 }); + PVE::Multipath::Status::update_node_status(); + is( + $report->{error}, + 'multipathd is not running or does not respond', + 'stopped daemon still reported', + ); + is_deeply($report->{luns}, [$namespace], 'namespaces do not need multipathd'); + ok(!$report->{'namespaces-unknown'}, 'which the node could list'); + { + local $ids->{nvme}->{disable} = 1; + $disks->mock(get_nvme_path_status => sub { die "walked the NVMe paths\n" }); + PVE::Multipath::Status::update_node_status(); + is_deeply($report->{luns}, [], 'a disabled storage reports no namespaces'); + ok(!$report->{'namespaces-unknown'}, 'and reads no NVMe paths for them'); + $disks->mock( + get_nvme_path_status => sub { { nvme1n1 => { active => 1, paths => [{}] } } }); + } + $nvme->mock(namespaces => sub { die "sysfs gone\n" }); + { + local $SIG{__WARN__} = sub { }; + PVE::Multipath::Status::update_node_status(); + } + ok($report->{'namespaces-unknown'}, 'a failed namespace query marks the report'); }; subtest 'API returns final node and LUN states' => sub { @@ -764,12 +821,31 @@ subtest 'API returns final node and LUN states' => sub { 'paths-active' => 2, storages => ['scoped'], }, + { + wwid => 'uuid.1', + native => 1, + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + storages => ['nvme'], + }, ], }), n2 => encode_report({ time => $NOW, luns => [], 'nvme-host-nqn' => 'nqn.test:n2' }), n3 => encode_report({ time => $NOW - 181, luns => [] }), n4 => encode_report({ time => $NOW, luns => [] }), - n5 => encode_report({ time => $NOW, error => 'multipath-tools is not installed' }), + n5 => encode_report({ + time => $NOW, + error => 'multipath-tools is not installed', + luns => [{ + wwid => 'uuid.1', + native => 1, + state => 'degraded', + 'paths-total' => 2, + 'paths-active' => 1, + storages => ['nvme'], + }], + }), ); $cluster->mock(get_node_kv => sub { \%raw }); my $entries = { '3600a' => { type => 'wwid' } }; @@ -787,6 +863,8 @@ subtest 'API returns final node and LUN states' => sub { { type => 'iscsi', 'managed-multipath' => 1, nodes => { departed => 1 } }, lvmgone => { type => 'lvm', base => 'gone:0.0.0.dm-uuid-mpath-3600e' }, scoped => { type => 'iscsi', 'managed-multipath' => 1, nodes => { n2 => 1 } }, + nvme => { type => 'nvmeof', subsysnqn => 'nqn.test:sub' }, + lvmns => { type => 'lvm', base => 'nvme:nvme-uuid.2', nodes => { n5 => 1 } }, }, } }, ); @@ -848,6 +926,46 @@ subtest 'API returns final node and LUN states' => sub { 'an entry for other nodes does not change that', ); } + my $namespace = $lun{'uuid.1'}; + ok($namespace->{native}, 'namespace marked as native'); + ok(!exists($namespace->{'multipath-storage'}), 'a namespace has no requested map'); + is_deeply( + { map { $_ => $namespace->{nodes}->{$_}->{state} } keys $namespace->{nodes}->%* }, + { + n1 => 'optimal', + n2 => 'missing', + n3 => 'unknown', + n4 => 'unknown', + n5 => 'degraded', + }, + 'namespace state of a node without multipathd counts', + ); + is($lun->{nodes}->{n5}->{state}, 'unknown', 'no map state from a node without multipathd'); + is_deeply( + $lun{'uuid.2'}, + { + wwid => 'uuid.2', + explicit => 0, + native => 1, + 'used-by-storage' => 'lvmns', + state => 'missing', + nodes => { n5 => { state => 'missing' } }, + }, + 'base namespace that a node without multipathd does not report', + ); + $raw{n5} = encode_report({ time => $NOW, error => 'too many LUNs to report' }); + $res = PVE::API2::SAN::Cluster->status({}); + %lun = map { $_->{wwid} => $_ } $res->{luns}->@*; + is($lun{'uuid.2'}->{nodes}->{n5}->{state}, 'unknown', 'node without a list'); + { + local $raw{n2} = encode_report({ time => $NOW, luns => [], 'namespaces-unknown' => 1 }); + my %unlisted = map { $_->{wwid} => $_ } PVE::API2::SAN::Cluster->status({})->{luns}->@*; + is( + $unlisted{'uuid.1'}->{nodes}->{n2}->{state}, + 'unknown', + 'a node that could not query its namespaces does not miss them', + ); + } eval { PVE::JSONSchema::validate( $res,