From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 208311FF0E1 for ; Mon, 16 Nov 2026 02:27:24 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 35FE6219F8; Mon, 16 Nov 2026 02:25:06 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v5 storage 15/36] api: multipath: add the cluster-wide health status Date: Mon, 16 Nov 2026 02:23:24 +0100 Message-ID: <20261116012355.3187417-16-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com> References: <20261116012355.3187417-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794792242099 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.207 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: SHDO4AZ2WORVU4YYQK3HR6MW5T4NWP3Y X-Message-ID-Hash: SHDO4AZ2WORVU4YYQK3HR6MW5T4NWP3Y X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Combine the reports of the nodes into one view: the state of each node (ok, multipath unavailable with the reason, or no report within three minutes) and, per LUN, the state of its map on each node with the worst one as summary. A LUN is listed if a node reports it, if it has an explicit entry or if an LVM storage uses it as base volume. The entry of a node has the number of maps that it does not report. A node that left maps out without knowing their volume groups counts as unknown instead of missing for a LUN that only a volume group relates to a storage on that node. A node should have a LUN if an explicit entry, the iSCSI storage that requests the map or a shared LVM storage on it covers the node. One that does not report it counts as missing, one without a current report as unknown. Storage admins can read it as well, as they need the initiator names for the array. Each LUN also names the enabled iSCSI storage with multipath that requests its map on one of its nodes, so that a map without an explicit entry can be told apart from one that nothing in Proxmox VE asked for. A node reports the number of paths it expects, so that one with fewer paths can show how many are missing. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Multipath.pm | 96 ++++++++++++++++++- src/PVE/Multipath/Status.pm | 122 ++++++++++++++++++++++++ src/test/run_multipath_api_tests.pl | 142 +++++++++++++++++++++++++++- 3 files changed, 357 insertions(+), 3 deletions(-) diff --git a/src/PVE/API2/Multipath.pm b/src/PVE/API2/Multipath.pm index 4951b1d7..40614cae 100644 --- a/src/PVE/API2/Multipath.pm +++ b/src/PVE/API2/Multipath.pm @@ -10,6 +10,7 @@ use PVE::RESTHandler; use PVE::Tools qw(extract_param); use PVE::Multipath::Config; +use PVE::Multipath::Status; use base qw(PVE::RESTHandler); @@ -60,7 +61,100 @@ __PACKAGE__->register_method({ links => [{ rel => 'child', href => "{name}" }], }, code => sub { - return [map { +{ name => $_ } } qw(overrides wwid)]; + return [map { +{ name => $_ } } qw(overrides status wwid)]; + }, +}); + +my $state_schema = { + type => 'string', + enum => ['optimal', 'degraded', 'failed', 'missing', 'unknown'], +}; +my $lun_node_schema = { + type => 'object', + additionalProperties => 0, + properties => { + state => $state_schema, + name => { type => 'string', optional => 1 }, + map { $_ => { type => 'integer', minimum => 0, optional => 1 } } + qw(size paths-active paths-total paths-expected), + }, +}; +my $node_schema = { + type => 'object', + additionalProperties => 0, + properties => { + state => { type => 'string', enum => ['ok', 'unavailable', 'no-report'] }, + time => { type => 'integer', optional => 1 }, + 'other-maps' => { + type => 'integer', + minimum => 1, + optional => 1, + description => 'The number of multipath maps that the node leaves out of its LUN' + . ' list. It lists the maps that Proxmox VE manages and the ones it knows to hold' + . ' the volume group of a storage.', + }, + map { $_ => { type => 'string', optional => 1 } } + qw(error apply-error iscsi-initiator-name), + }, +}; + +__PACKAGE__->register_method({ + name => 'status', + path => 'status', + method => 'GET', + description => "Report the multipath health of each LUN on each node, and the worst" + . " state among the nodes that should see or report the LUN.", + # Storage administrators need path health and initiator names to configure SAN target ACLs. + permissions => { + check => [ + 'or', ['perm', '/', ['Sys.Audit']], ['perm', '/storage', ['Datastore.Allocate']], + ], + }, + parameters => { + additionalProperties => 0, + properties => {}, + }, + returns => { + type => 'object', + properties => { + luns => { + type => 'array', + description => + "One entry per LUN, identified by its WWID, with the state of its" + . " map on each node and the overall cluster state.", + items => { + type => 'object', + additionalProperties => 0, + properties => { + wwid => { type => 'string', description => 'The LUN WWID.' }, + explicit => + { type => 'boolean', description => 'An explicit entry exists.' }, + name => { type => 'string', optional => 1 }, + 'used-by-storage' => { type => 'string', optional => 1 }, + 'multipath-storage' => { + type => 'string', + optional => 1, + description => 'The storage that requests the map of the LUN.', + }, + size => { type => 'integer', minimum => 0, optional => 1 }, + state => $state_schema, + nodes => { + type => 'object', + description => "Map health keyed by node name.", + additionalProperties => $lun_node_schema, + }, + }, + }, + }, + nodes => { + type => 'object', + description => "Available node reports, keyed by node name.", + additionalProperties => $node_schema, + }, + }, + }, + code => sub { + return PVE::Multipath::Status::cluster_status(); }, }); diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm index f926e960..ae6859e4 100644 --- a/src/PVE/Multipath/Status.pm +++ b/src/PVE/Multipath/Status.pm @@ -8,11 +8,22 @@ use PVE::Cluster; use PVE::INotify; use PVE::Multipath; +use PVE::Multipath::Config; use PVE::Multipath::Generator; my $json = JSON->new->utf8->canonical; my $KEY = 'multipath'; +sub worst_state(@states) { + my %rank = (unknown => -1, optimal => 0, degraded => 1, missing => 2, failed => 3); + my $worst = 'unknown'; + for my $state (@states) { + $worst = $state + if defined($state) && exists($rank{$state}) && $rank{$state} > $rank{$worst}; + } + return $worst; +} + sub publish($report) { my $node = PVE::INotify::nodename(); my $now = time(); @@ -80,6 +91,8 @@ sub update_node_status() { $lun->{size} = $map->{size} if defined($map->{size}); $lun->{name} = $map->{name} if defined($map->{name}) && $map->{name} ne $wwid; $lun->{storage} = $info->{storages}->[0] if $info->{storages}; + $lun->{'paths-expected'} = $info->{'paths-expected'} + if defined($info->{'paths-expected'}); $lun->{vgname} = $vg if defined($vg); push $report->{luns}->@*, $lun; } @@ -95,4 +108,113 @@ sub update_node_status() { warn "multipath: cannot publish node status - $@" if $@; } +sub cluster_status() { + require PVE::Storage; + + my $cfg = PVE::Multipath::Config::read_config(); + my $storage = PVE::Storage::config(); + my $all_nodes = PVE::Cluster::get_nodelist(); + my $members = PVE::Cluster::get_members(); + my $raw = PVE::Cluster::get_node_kv($KEY); + my (%nodes, %luns, %expected, %observed, %vg_owner); + # the nodes that list a map because the configuration names it for them + my (%managed, %unhinted); + my $storage_nodes = sub($scfg) { + return $scfg->{nodes} && $scfg->{nodes}->%* ? [keys $scfg->{nodes}->%*] : $all_nodes; + }; + # only an enabled iSCSI storage with the option requests the maps of its LUNs, on its nodes + my %current = map { $_ => 1 } @$all_nodes; + my $requests_maps = sub($sid, $node = undef) { + my $scfg = defined($sid) ? $storage->{ids}->{$sid} : undef; + return 0 + if !$scfg + || $scfg->{type} ne 'iscsi' + || !$scfg->{'managed-multipath'} + || $scfg->{disable}; + my @nodes = grep { $current{$_} } $storage_nodes->($scfg)->@*; + return defined($node) ? !!grep({ $_ eq $node } @nodes) : !!@nodes; + }; + for my $wwid (PVE::Multipath::Config::wwid_list($cfg)->@*) { + my $entry = $cfg->{ids}->{$wwid}; + $luns{$wwid} = { wwid => $wwid, explicit => 1 }; + $luns{$wwid}->{name} = $entry->{alias} if defined($entry->{alias}); + $expected{$wwid}->{$_} = $managed{$wwid}->{$_} = 1 for $storage_nodes->($entry)->@*; + } + for my $sid (sort keys $storage->{ids}->%*) { + my $scfg = $storage->{ids}->{$sid}; + $vg_owner{ $scfg->{vgname} } //= $sid if defined($scfg->{vgname}); + next if $scfg->{type} ne 'lvm'; + my ($wwid) = ($scfg->{base} // '') =~ /(?:^|\.)dm-uuid-mpath-(\S+)$/; + next if !defined($wwid); + $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $luns{$wwid}->{'used-by-storage'} //= $sid; + $managed{$wwid}->{$_} = 1 for @$all_nodes; + my ($base) = $scfg->{base} =~ /^([^:]+):/; + $luns{$wwid}->{'multipath-storage'} //= $base if $requests_maps->($base); + $expected{$wwid}->{$_} = 1 for $storage_nodes->($scfg)->@*; + } + for my $node (sort @$all_nodes) { + my $report = eval { decode_json($raw->{$node} // '') }; + $report = {} if ref($report) ne 'HASH'; + my $entry = $nodes{$node} = { state => 'no-report' }; + $entry->{time} = $report->{time} if defined($report->{time}); + next + if (exists($members->{$node}) && !$members->{$node}->{online}) + || !defined($report->{time}) + || time() - $report->{time} > 180; + $entry->{$_} = $report->{$_} + for grep { defined($report->{$_}) } + qw(error apply-error iscsi-initiator-name other-maps); + $entry->{state} = defined($report->{error}) ? 'unavailable' : 'ok'; + $unhinted{$node} = 1 if $report->{'vgs-unknown'}; + next if $entry->{state} ne 'ok'; + + for my $lun (($report->{luns} // [])->@*) { + my $wwid = $lun->{wwid} // next; + my $row = $luns{$wwid} //= { wwid => $wwid, explicit => 0 }; + $row->{$_} //= $lun->{$_} for grep { defined($lun->{$_}) } qw(name size); + my $owner = defined($lun->{vgname}) ? $vg_owner{ $lun->{vgname} } : undef; + $row->{'used-by-storage'} //= $owner if defined($owner); + # a shared LVM storage on the map needs it on its nodes, also without a base volume + my $lvm = defined($owner) ? $storage->{ids}->{$owner} : undef; + if ($lvm && $lvm->{type} eq 'lvm' && $lvm->{shared} && !$lvm->{disable}) { + $expected{$wwid}->{$_} = 1 for $storage_nodes->($lvm)->@*; + } + my $sid = $lun->{storage}; + if (defined($sid) && (my $scfg = $storage->{ids}->{$sid})) { + $row->{'multipath-storage'} //= $sid if $requests_maps->($sid, $node); + $row->{'used-by-storage'} //= $sid + if PVE::Storage::san_storage_serves_images($scfg); + $expected{$wwid}->{$_} = 1 + for !$scfg->{disable} ? $storage_nodes->($scfg)->@* : (); + $managed{$wwid}->{$_} = 1 + for $requests_maps->($sid) ? $storage_nodes->($scfg)->@* : (); + } + $observed{$wwid}->{$node} = { + state => worst_state($lun->{state}), + map { $_ => $lun->{$_} } + grep { defined($lun->{$_}) } + qw(name size paths-active paths-total paths-expected), + }; + } + } + for my $wwid (sort keys %luns) { + my $row = $luns{$wwid}; + $row->{nodes} = {}; + for my $node (sort @$all_nodes) { + if ($observed{$wwid}->{$node}) { + $row->{nodes}->{$node} = $observed{$wwid}->{$node}; + } elsif ($expected{$wwid}->{$node}) { + my $listed = $nodes{$node}->{state} eq 'ok'; + # without the hint file of LVM a node may have left the map out, if only a volume + # group relates it to a storage there + $listed = 0 if $unhinted{$node} && !$managed{$wwid}->{$node}; + $row->{nodes}->{$node} = { state => $listed ? 'missing' : 'unknown' }; + } + } + $row->{state} = worst_state(map { $_->{state} } values $row->{nodes}->%*); + } + return { nodes => \%nodes, luns => [map { $luns{$_} } sort keys %luns] }; +} + 1; diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl index 9a668905..766a2e38 100755 --- a/src/test/run_multipath_api_tests.pl +++ b/src/test/run_multipath_api_tests.pl @@ -8,7 +8,7 @@ BEGIN { } use lib ('.', '..'); use File::Temp qw(tempdir); -use JSON qw(decode_json); +use JSON qw(encode_json decode_json); use Storable qw(dclone); use Test::More; use Test::MockModule; @@ -45,7 +45,7 @@ subtest 'explicit entries are ordinary configuration writes' => sub { } is_deeply( PVE::API2::Multipath->index({}), - [map { { name => $_ } } qw(overrides wwid)], + [map { { name => $_ } } qw(overrides status wwid)], 'index', ); }; @@ -300,7 +300,145 @@ subtest 'one entry point applies configuration and reports unavailable reasons' sub { [{ wwid => '3600a', name => 'data', 'paths-active' => 1, 'paths-total' => 2 }] }); PVE::Multipath::Status::update_node_status(); is($report->{luns}->[0]->{state}, 'degraded', 'all maps while the wanted ones are unknown'); + ok(!exists($report->{luns}->[0]->{'paths-expected'}), 'no expected count without a source'); + $gen->mock(sync => sub { { wwids => { '3600a' => { 'paths-expected' => 3 } } } }); + $mp->mock(get_maps => + sub { [{ wwid => '3600a', name => 'data', 'paths-active' => 2, 'paths-total' => 2 }] }); + PVE::Multipath::Status::update_node_status(); + is($report->{luns}->[0]->{'paths-expected'}, 3, 'expected count reported'); + is($report->{luns}->[0]->{state}, 'degraded', 'fewer paths than expected'); ok(!exists($report->{supported}), 'no separate capability state'); }; +subtest 'API returns final node and LUN states' => sub { + my $cluster = Test::MockModule->new('PVE::Cluster'); + my $config = Test::MockModule->new('PVE::Multipath::Config'); + my $storage = Test::MockModule->new('PVE::Storage'); + $cluster->mock(get_nodelist => sub { [qw(n1 n2 n3 n4 n5)] }); + $cluster->mock(get_members => sub { { n4 => { online => 0 } } }); + my %raw = ( + n1 => encode_json({ + time => $NOW, + luns => [ + { + wwid => '3600a', + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + 'paths-expected' => 2, + storage => 'iscsi', + vgname => 'vgfc', + }, + { + wwid => '3600d', + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + storage => 'off', + vgname => 'vgshared', + }, + { + wwid => '3600f', + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + storage => 'scoped', + }, + ], + }), + n2 => encode_json({ time => $NOW, luns => [] }), + n3 => encode_json({ time => $NOW - 181, luns => [] }), + n4 => encode_json({ time => $NOW, luns => [] }), + n5 => encode_json({ time => $NOW, error => 'multipath-tools is not installed' }), + ); + $cluster->mock(get_node_kv => sub { \%raw }); + my $entries = { '3600a' => { type => 'wwid' } }; + $config->mock(read_config => sub { { ids => $entries } }); + $storage->mock( + config => sub { { + ids => { + iscsi => { type => 'iscsi', 'managed-multipath' => 1 }, + lvm => { type => 'lvm', base => 'iscsi:0.0.0.dm-uuid-mpath-3600b' }, + fc => { type => 'lvm', vgname => 'vgfc' }, + off => { type => 'iscsi' }, + lvmoff => { type => 'lvm', base => 'off:0.0.0.dm-uuid-mpath-3600c' }, + shared => { type => 'lvm', vgname => 'vgshared', shared => 1 }, + gone => + { type => 'iscsi', 'managed-multipath' => 1, nodes => { departed => 1 } }, + lvmgone => { type => 'lvm', base => 'gone:0.0.0.dm-uuid-mpath-3600e' }, + scoped => { type => 'iscsi', 'managed-multipath' => 1, nodes => { n2 => 1 } }, + }, + } }, + ); + my $res = PVE::API2::Multipath->status({}); + is_deeply( + [map { $res->{nodes}->{$_}->{state} } qw(n1 n2 n3 n4 n5)], + [qw(ok ok no-report no-report unavailable)], + 'final node states', + ); + is($res->{nodes}->{n3}->{time}, $NOW - 181, 'old report time retained'); + my $lun = $res->{luns}->[0]; + is($lun->{state}, 'missing', 'worst known state'); + is($lun->{nodes}->{n1}->{state}, 'optimal', 'observed healthy map'); + is($lun->{nodes}->{n2}->{state}, 'missing', 'expected map absent from current report'); + is($lun->{nodes}->{n3}->{state}, 'unknown', 'stale observation'); + is($lun->{'used-by-storage'}, 'fc', 'storage attributed through the reported volume group'); + is($res->{luns}->[1]->{wwid}, '3600b', 'LVM base included without observation'); + is($res->{luns}->[1]->{'used-by-storage'}, 'lvm', 'LVM consumer attributed'); + is($lun->{'multipath-storage'}, 'iscsi', 'reported storage requests the map'); + is($res->{luns}->[1]->{'multipath-storage'}, 'iscsi', 'base storage requests the map'); + is($lun->{nodes}->{n1}->{'paths-expected'}, 2, 'expected path count of the node'); + my %lun = map { $_->{wwid} => $_ } $res->{luns}->@*; + ok( + !exists($lun{'3600c'}->{'multipath-storage'}), + 'a base without multipath requests nothing', + ); + ok(!exists($lun{'3600d'}->{'multipath-storage'}), 'neither does a reported one'); + ok( + !exists($lun{'3600e'}->{'multipath-storage'}), + 'a base for no current node requests nothing', + ); + ok(!exists($lun{'3600f'}->{'multipath-storage'}), 'a storage requests on its nodes only'); + is( + $lun{'3600d'}->{nodes}->{n2}->{state}, + 'missing', + 'the nodes of a shared LVM expect the map', + ); + is($lun{'3600d'}->{'used-by-storage'}, 'shared', 'shared LVM attributed by volume group'); + { + local $raw{n2} = encode_json({ time => $NOW, luns => [], 'vgs-unknown' => 1 }); + my %unhinted = map { $_->{wwid} => $_ } PVE::API2::Multipath->status({})->{luns}->@*; + is( + $unhinted{'3600d'}->{nodes}->{n2}->{state}, + 'unknown', + 'a node without LVM hints cannot tell for a map that only a volume group relates', + ); + is($unhinted{'3600a'}->{nodes}->{n2}->{state}, 'missing', 'but for one with an entry'); + is( + $unhinted{'3600f'}->{nodes}->{n2}->{state}, + 'missing', + 'and for one that an iSCSI storage requests there', + ); + local $entries->{'3600d'} = { type => 'wwid', nodes => { n1 => 1 } }; + %unhinted = map { $_->{wwid} => $_ } PVE::API2::Multipath->status({})->{luns}->@*; + is( + $unhinted{'3600d'}->{nodes}->{n2}->{state}, + 'unknown', + 'an entry for other nodes does not change that', + ); + } + eval { + PVE::JSONSchema::validate( + $res, + PVE::API2::Multipath->map_method_by_name('status')->{returns}, + ); + }; + is($@, '', 'result satisfies the API schema'); + is( + PVE::Multipath::Status::worst_state(qw(unknown optimal failed degraded missing)), + 'failed', + 'one ranking', + ); +}; + done_testing();