From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 97A841FF0B7 for ; Fri, 13 Nov 2026 04:11:46 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id E999F219F0; Fri, 13 Nov 2026 04:05:05 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v4 storage 13/36] multipath: publish per-node status Date: Fri, 13 Nov 2026 04:00:31 +0100 Message-ID: <20261113030146.1559030-14-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com> References: <20261113030146.1559030-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794538914345 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.242 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: FNIKOAELJTO4UAZEKP3LBTKUJOKK4SZO X-Message-ID-Hash: FNIKOAELJTO4UAZEKP3LBTKUJOKK4SZO X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Path health differs between nodes. Add the function that the status daemon will call on each node: it applies the multipath configuration and publishes the state of the node's maps in the cluster key-value store, for the cluster-wide status. A report has the WWID, state and path counts of each map, the iSCSI initiator name of the node and the error of a failed configuration run, or the reason why multipath is not usable. An unchanged report is sent again once a minute, so that readers can tell a stale one. The initiator name, which an admin needs for the access rules on the storage array, comes from a new reader in the iSCSI plugin, which the node iSCSI API will use as well. Signed-off-by: Thomas Lamprecht --- src/PVE/Makefile | 1 + src/PVE/Multipath/Status.pm | 75 +++++++++++++++++++ src/PVE/Storage/ISCSIPlugin.pm | 16 ++++ src/PVE/Storage/LVMPlugin.pm | 11 ++- src/test/run_multipath_api_tests.pl | 109 ++++++++++++++++++++++++++++ src/test/run_san_library_tests.pl | 31 ++++++++ 6 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 src/PVE/Multipath/Status.pm diff --git a/src/PVE/Makefile b/src/PVE/Makefile index 87d5c41f..13fecbf8 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -9,6 +9,7 @@ install: install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm install -D -m 0644 Multipath/Generator.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Generator.pm install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm + install -D -m 0644 Multipath/Status.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Status.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm make -C Storage install diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm new file mode 100644 index 00000000..91b699c4 --- /dev/null +++ b/src/PVE/Multipath/Status.pm @@ -0,0 +1,75 @@ +package PVE::Multipath::Status; + +use v5.36; +use JSON qw(decode_json); +use PVE::Cluster; +use PVE::INotify; +use PVE::Multipath; +use PVE::Multipath::Generator; + +my $json = JSON->new->utf8->canonical; +my $KEY = 'multipath'; + +sub publish($report) { + my $node = PVE::INotify::nodename(); + my $now = time(); + my $data = { %$report, time => $now }; + my $encoded = $json->encode($data); + if (length($encoded) >= 32 * 1024) { + $data = { time => $now, error => 'too many LUNs to report' }; + $encoded = $json->encode($data); + } + my $raw = PVE::Cluster::get_node_kv($KEY, $node); + my $previous = eval { decode_json($raw->{$node} // '') }; + if (ref($previous) eq 'HASH') { + my $age = $now - ($previous->{time} // 0); + $previous->{time} = $now; + return if $age >= 0 && $age < 60 && $json->encode($previous) eq $encoded; + } + PVE::Cluster::broadcast_node_kv($KEY, $encoded); +} + +sub update_node_status() { + my $apply = PVE::Multipath::Generator::sync(); + my $report = { luns => [] }; + $report->{'apply-error'} = $apply->{error} if defined($apply->{error}); + require PVE::Storage::ISCSIPlugin; + my $initiator = eval { PVE::Storage::ISCSIPlugin::read_initiator_info() }; + $report->{'iscsi-initiator-name'} = $initiator->{name} + if $initiator && defined($initiator->{name}); + eval { + die "multipath-tools is not installed\n" if !PVE::Multipath::is_supported(); + die "multipathd is not running\n" if !PVE::Multipath::is_running(); + require PVE::Storage::LVMPlugin; + my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups(); + for my $map (PVE::Multipath::get_maps()->@*) { + my $wwid = $map->{wwid} // next; + my $info = $apply->{wwids}->{$wwid} // {}; + my $lun = { + wwid => $wwid, + 'paths-total' => $map->{'paths-total'}, + 'paths-active' => $map->{'paths-active'}, + state => PVE::Multipath::derive_health( + $map->{'paths-active'}, + $map->{'paths-total'}, + $info->{'paths-expected'}, + ), + }; + $lun->{size} = $map->{size} if defined($map->{size}); + $lun->{name} = $map->{name} if defined($map->{name}) && $map->{name} ne $wwid; + $lun->{storage} = $info->{storages}->[0] if $info->{storages}; + my $vg = defined($map->{sysfs}) ? $pv_vgs->{"/dev/$map->{sysfs}"} : undef; + $lun->{vgname} = $vg if defined($vg); + push $report->{luns}->@*, $lun; + } + }; + if (my $error = $@) { + chomp $error; + $report->{error} = $error; + delete $report->{luns}; + } + eval { publish($report) }; + warn "multipath: cannot publish node status - $@" if $@; +} + +1; diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm index 3bddcda6..d3ae56a4 100644 --- a/src/PVE/Storage/ISCSIPlugin.pm +++ b/src/PVE/Storage/ISCSIPlugin.pm @@ -21,8 +21,24 @@ use base qw(PVE::Storage::Plugin); # iscsi helper function my $ISCSIADM = '/usr/bin/iscsiadm'; +our $INITIATOR_NAME_FILE = '/etc/iscsi/initiatorname.iscsi'; our $SYSFS = '/sys'; +sub read_initiator_info { + my $content = eval { PVE::Tools::file_get_contents($INITIATOR_NAME_FILE, 64 * 1024) }; + return undef if !defined($content); + + my $res = {}; + for my $line (split(/\n/, $content)) { + if ($line =~ m/^\s*InitiatorName\s*=\s*(\S+)\s*$/) { + $res->{name} = $1; + } elsif ($line =~ m/^\s*InitiatorAlias\s*=\s*(.+?)\s*$/) { + $res->{alias} = $1; + } + } + return defined($res->{name}) ? $res : undef; +} + my $found_iscsi_adm_exe; my sub assert_iscsi_support { my ($noerr) = @_; diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm index ccae074a..ee64b782 100644 --- a/src/PVE/Storage/LVMPlugin.pm +++ b/src/PVE/Storage/LVMPlugin.pm @@ -17,7 +17,7 @@ use PVE::Format qw(render_bytes render_duration); use PVE::INotify; use PVE::JSONSchema qw(get_standard_option); use PVE::RESTEnvironment qw(log_warn); -use PVE::Tools qw(run_command file_read_firstline trim); +use PVE::Tools qw(run_command file_get_contents file_read_firstline trim); use PVE::Storage::Common; use PVE::Storage::Plugin; @@ -138,6 +138,15 @@ sub lvm_pv_info { return $pvinfo; } +sub cached_pv_volume_groups { + my $data = eval { file_get_contents('/run/lvm/hints') } // ''; + my $res = {}; + while ($data =~ m/^scan:(\/dev\/\S+) pvid:\S+ devn:\d+:\d+ vg:([a-zA-Z0-9+_.-]+)$/mg) { + $res->{ abs_path($1) // $1 } = $2 if $2 ne '-'; + } + return $res; +} + sub clear_first_sector { my ($dev) = shift; diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl index 82aaa5f0..7bed4266 100755 --- a/src/test/run_multipath_api_tests.pl +++ b/src/test/run_multipath_api_tests.pl @@ -8,9 +8,11 @@ BEGIN { } use lib ('.', '..'); use File::Temp qw(tempdir); +use JSON qw(decode_json); use Test::More; use Test::MockModule; use PVE::Multipath::Generator; +use PVE::Storage; subtest 'add-only generation and failed reload throttle' => sub { my $dir = tempdir('multipath-test-XXXX', DIR => '.', CLEANUP => 1); @@ -90,4 +92,111 @@ subtest 'storage LUNs are the identified paths of the storage target' => sub { ); }; +subtest 'an unconfigured node does not write drop-ins or reload' => sub { + my $dir = tempdir('multipath-empty-XXXX', DIR => '.', CLEANUP => 1); + local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults"; + local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides"; + local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases"; + local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending"; + my $config = Test::MockModule->new('PVE::Multipath::Config'); + $config->mock(read_config => sub { { ids => {} } }); + $config->mock(read_overrides => sub { '' }); + my $storage = Test::MockModule->new('PVE::Storage'); + my $identity = Test::MockModule->new('PVE::Multipath::Identity'); + $identity->mock(collect => sub { die "unexpected identity collection\n" }); + my $mp = Test::MockModule->new('PVE::Multipath'); + $mp->mock(is_supported => sub { 1 }); + $mp->mock(is_running => sub { 1 }); + $mp->mock(get_maps => sub { [] }); + $mp->mock(list_etc_multipath_wwids => sub { [] }); + my $generator = Test::MockModule->new('PVE::Multipath::Generator'); + my @changes; + + for my $function (qw(file_set_contents make_path)) { + $generator->mock($function => sub { push @changes, $function }); + } + for my $function (qw(add_wwid reconfigure)) { + $mp->mock($function => sub { push @changes, $function }); + } + my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + $iscsi->mock(read_initiator_info => sub { {} }); + my $status = Test::MockModule->new('PVE::Multipath::Status'); + my $report; + $status->mock(publish => sub { $report = $_[0] }); + for my $ids ({}, { plain => { type => 'iscsi', target => 'iqn.test' } }) { + $storage->mock(config => sub { { ids => $ids } }); + PVE::Multipath::Status::update_node_status(); + ok(!exists($report->{error}), 'running daemon reports successfully'); + ok(!exists($report->{'apply-error'}), 'unconfigured application succeeds'); + is_deeply(\@changes, [], 'no write, mkdir, allow-list addition or reload'); + is_deeply([glob("$dir/*")], [], 'no generated files or pending marker'); + } +}; + +subtest 'node reports use read-back, refresh and a bounded error report' => sub { + my $cluster = Test::MockModule->new('PVE::Cluster'); + my $node = Test::MockModule->new('PVE::INotify'); + $node->mock(nodename => sub { 'n1' }); + my (%raw, @sent); + $cluster->mock(get_node_kv => sub { \%raw }); + $cluster->mock( + broadcast_node_kv => sub($key, $data) { + is($key, 'multipath', 'one node key'); + push @sent, decode_json($data); + $raw{n1} = $data; + }, + ); + my $report = { + luns => [{ + wwid => '3600a', + name => 'data', + state => 'optimal', + 'paths-total' => 2, + 'paths-active' => 2, + }], + }; + PVE::Multipath::Status::publish($report); + $NOW += 59; + PVE::Multipath::Status::publish($report); + is(scalar(@sent), 1, 'same report younger than 60 seconds is skipped'); + $NOW++; + PVE::Multipath::Status::publish($report); + is(scalar(@sent), 2, 'report refreshed at 60 seconds'); + %raw = (); + PVE::Multipath::Status::publish($report); + is(scalar(@sent), 3, 'lost pmxcfs value is republished without local send state'); + PVE::Multipath::Status::publish({ luns => [('x' x 33000)] }); + is($sent[-1]->{error}, 'too many LUNs to report', 'oversized report is an error'); + ok(!exists($sent[-1]->{luns}), 'not an empty or partial LUN report'); +}; + +subtest 'one entry point applies configuration and reports unavailable reasons' => sub { + my $gen = Test::MockModule->new('PVE::Multipath::Generator'); + my $status = Test::MockModule->new('PVE::Multipath::Status'); + my $mp = Test::MockModule->new('PVE::Multipath'); + my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + my ($report, @order); + $gen->mock(sync => sub { push @order, 'apply'; { error => 'reload failed' } }); + $status->mock(publish => sub { push @order, 'publish'; $report = $_[0] }); + $iscsi->mock(read_initiator_info => sub { { name => 'iqn.2025-01.example:node' } }); + $mp->mock(is_supported => sub { 0 }); + PVE::Multipath::Status::update_node_status(); + is($report->{error}, 'multipath-tools is not installed', 'missing tools reason only'); + is_deeply(\@order, [qw(apply publish)], 'single call owns ordering'); + is($report->{'apply-error'}, 'reload failed', 'apply failure is separate'); + $mp->mock(is_supported => sub { 1 }); + $mp->mock(is_running => sub { 0 }); + PVE::Multipath::Status::update_node_status(); + is($report->{error}, 'multipathd is not running', 'stopped daemon reason only'); + $mp->mock(is_running => sub { 1 }); + $mp->mock(get_maps => sub { die "bad JSON\n" }); + PVE::Multipath::Status::update_node_status(); + is($report->{error}, 'bad JSON', 'query error without unavailable prefix'); + $mp->mock(get_maps => + sub { [{ wwid => '3600a', name => 'data', 'paths-active' => 1, 'paths-total' => 2 }] }); + PVE::Multipath::Status::update_node_status(); + is($report->{luns}->[0]->{state}, 'degraded', 'maps without PVE entries are included'); + ok(!exists($report->{supported}), 'no separate capability state'); +}; + done_testing(); diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl index 2ec2a651..6a93e868 100755 --- a/src/test/run_san_library_tests.pl +++ b/src/test/run_san_library_tests.pl @@ -50,4 +50,35 @@ subtest 'omitted optional parameters retain explicit undef behavior' => sub { } }; +subtest 'LVM hints are optional cached display metadata' => sub { + require PVE::Storage::LVMPlugin; + my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); + my $contents; + $lvm->mock( + file_get_contents => sub { + is($_[0], '/run/lvm/hints', 'only the existing LVM cache is read'); + die "No such file\n" if !defined($contents); + return $contents; + }, + ); + $lvm->mock(abs_path => sub { $_[0] eq '/dev/disk/by-id/test' ? '/dev/dm-0' : $_[0] }); + $lvm->mock(run_command => sub { die "unexpected device probe\n" }); + for my $case ( + [undef, {}, 'missing file'], + ['', {}, 'empty file'], + ["scan=/dev/sda pvid=abc devn=8:0 vg=data\n", {}, 'different format'], + ["scan:/dev/sda pvid:abc devn:8:0 vg:-\n", {}, 'orphan PV'], + ["scan:/dev/sda pvid:abc devn:8:0 vg:[unknown]\n", {}, 'unknown VG'], + ["scan:/dev/sda pvid:abc devn:8:0 vg:data\n", { '/dev/sda' => 'data' }, 'cached VG'], + [ + "scan:/dev/disk/by-id/test pvid:abc devn:253:0 vg:data\n", + { '/dev/dm-0' => 'data' }, + 'resolved device name', + ], + ) { + $contents = $case->[0]; + is_deeply(PVE::Storage::LVMPlugin::cached_pv_volume_groups(), $case->[1], $case->[2]); + } +}; + done_testing();