From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 5EFBC1FF0E1 for ; Mon, 16 Nov 2026 02:26:35 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 067F52190F; Mon, 16 Nov 2026 02:25:03 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v5 storage 12/36] multipath: apply the configuration on every node Date: Mon, 16 Nov 2026 02:23:21 +0100 Message-ID: <20261116012355.3187417-13-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com> References: <20261116012355.3187417-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794792241907 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.209 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: K6C3UEQYBHDVOGRIAMFXH7HN6YLKGWLU X-Message-ID-Hash: K6C3UEQYBHDVOGRIAMFXH7HN6YLKGWLU X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Each node needs the cluster-wide configuration as local multipath.conf drop-ins and the WWIDs of its LUNs in the allow-list of multipath: the explicit entries valid for the node and the LUNs it sees through sessions of enabled iSCSI storages with the multipath option. The generated defaults set 'auto_resize grow_only', so that a map follows a grown LUN. Missing WWIDs are added; nothing is ever removed. multipathd is reloaded only after a change, and a failed reload is retried after a minute. Nothing calls this yet. Signed-off-by: Thomas Lamprecht --- src/PVE/Makefile | 1 + src/PVE/Multipath.pm | 19 ++++- src/PVE/Multipath/Generator.pm | 114 ++++++++++++++++++++++++++++ src/PVE/Multipath/Identity.pm | 2 +- src/PVE/Storage/ISCSIPlugin.pm | 16 ++++ src/test/Makefile | 8 +- src/test/run_multipath_api_tests.pl | 94 +++++++++++++++++++++++ src/test/run_multipath_tests.pl | 12 +++ src/test/run_san_library_tests.pl | 6 ++ src/test/run_san_taint_tests.pl | 34 +++++++++ 10 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 src/PVE/Multipath/Generator.pm create mode 100755 src/test/run_multipath_api_tests.pl create mode 100755 src/test/run_san_taint_tests.pl diff --git a/src/PVE/Makefile b/src/PVE/Makefile index 53347442..87d5c41f 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -7,6 +7,7 @@ install: install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm + install -D -m 0644 Multipath/Generator.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Generator.pm install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm index 1caadee1..ec417381 100644 --- a/src/PVE/Multipath.pm +++ b/src/PVE/Multipath.pm @@ -5,7 +5,7 @@ use v5.36; use Encode qw(decode encode); use JSON qw(decode_json); -use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach); +use PVE::Tools qw(run_command file_get_contents file_read_firstline dir_glob_foreach); use PVE::Diskmanage; use PVE::Multipath::Config; @@ -169,4 +169,21 @@ sub wait_for_map($wwid, $timeout = 10) { } } +sub list_etc_multipath_wwids() { + return [] if !-e '/etc/multipath/wwids'; + return PVE::Multipath::Config::parse_wwids(file_get_contents('/etc/multipath/wwids')); +} + +sub add_wwid($wwid) { + assert_supported(); + my ($id) = $wwid =~ /^($PVE::Multipath::Config::WWID_PATTERN)\z/; + die "invalid WWID '$wwid'\n" if !defined($id); + run_command([$MULTIPATH, '-a', '--', $id], timeout => 10); +} + +sub reconfigure() { + assert_supported(); + run_command([$MULTIPATHD, 'reconfigure'], timeout => 30); +} + 1; diff --git a/src/PVE/Multipath/Generator.pm b/src/PVE/Multipath/Generator.pm new file mode 100644 index 00000000..df050c0c --- /dev/null +++ b/src/PVE/Multipath/Generator.pm @@ -0,0 +1,114 @@ +package PVE::Multipath::Generator; + +use v5.36; + +use File::Path qw(make_path); + +use PVE::INotify; +use PVE::Tools qw(file_get_contents file_set_contents); + +use PVE::Multipath; +use PVE::Multipath::Config; +use PVE::Multipath::Identity; + +# Local drop-ins let multipathd start before pmxcfs is available. +our $DEFAULTS_DROPIN = '/etc/multipath/conf.d/pve-defaults.conf'; +our $OVERRIDES_DROPIN = '/etc/multipath/conf.d/pve-overrides.conf'; +our $ALIASES_DROPIN = '/etc/multipath/conf.d/pve-aliases.conf'; +our $RECONFIGURE_PENDING = '/run/pve-multipath.reconfigure-pending'; +my $failed_at; + +sub derived_wwids($cfg, $ids = undef) { + require PVE::Storage; + my @storages = grep { + my $scfg = $cfg->{ids}->{$_}; + $scfg->{type} eq 'iscsi' + && $scfg->{'managed-multipath'} + && PVE::Storage::storage_check_enabled($cfg, $_, undef, 1) + } sort keys $cfg->{ids}->%*; + if (!@storages) { + return {}; + } + $ids //= PVE::Multipath::Identity::collect( + targets => { map { $cfg->{ids}->{$_}->{target} => 1 } @storages }); + return { map { $_ => PVE::Storage::ISCSIPlugin->multipath_lun_wwids($cfg->{ids}->{$_}, $ids) } + @storages }; +} + +my sub pending() { file_set_contents($RECONFIGURE_PENDING, '') if !-e $RECONFIGURE_PENDING } +my sub write_dropin($path, $content) { + my $old = -e $path ? file_get_contents($path) : ''; + return if $old eq $content; + pending(); + if (length($content)) { + make_path($path =~ s!/[^/]+$!!r); + file_set_contents($path, $content); + } else { + unlink($path) or die "cannot remove '$path' - $!\n"; + } +} + +sub regenerate($cfg = undef, $overrides = undef, $derived = undef) { + $cfg //= PVE::Multipath::Config::read_config(); + $overrides //= PVE::Multipath::Config::read_overrides(); + $derived //= {}; + my $node = PVE::INotify::nodename(); + my @explicit = PVE::Multipath::Config::wwids_for_node($cfg, $node)->@*; + my %wanted = map { + my $count = $cfg->{ids}->{$_}->{'paths-expected'}; + $_ => defined($count) ? { 'paths-expected' => $count } : {} + } @explicit; + for my $sid (sort keys %$derived) { + for my $wwid (sort keys $derived->{$sid}->{wwids}->%*) { + push $wanted{$wwid}->{storages}->@*, $sid; + my $expected = $derived->{$sid}->{wwids}->{$wwid}->{'paths-expected'}; + $wanted{$wwid}->{'paths-expected'} = $expected + if defined($expected) && $expected > ($wanted{$wwid}->{'paths-expected'} // 0); + } + } + eval { + my $in_use = @explicit || keys(%$derived) || length($overrides); + write_dropin( + $DEFAULTS_DROPIN, + $in_use ? PVE::Multipath::Config::generate_managed_conf() : '', + ); + write_dropin( + $OVERRIDES_DROPIN, + length($overrides) ? "# Managed by Proxmox VE.\n$overrides" : '', + ); + write_dropin( + $ALIASES_DROPIN, + @explicit ? PVE::Multipath::Config::build_multipaths_block($cfg, $node) : '', + ); + my %current = map { $_ => 1 } PVE::Multipath::list_etc_multipath_wwids()->@*; + for my $wwid (sort keys %wanted) { + next if $current{$wwid}; + pending(); + PVE::Multipath::add_wwid($wwid); + } + if (-e $RECONFIGURE_PENDING) { + die "multipath reload pending after a failed reload\n" + if defined($failed_at) && time() - $failed_at < 60; + eval { PVE::Multipath::reconfigure() }; + if (my $error = $@) { + $failed_at = time(); + die $error; + } + unlink($RECONFIGURE_PENDING) or die "cannot clear multipath reload marker - $!\n"; + $failed_at = undef; + } + }; + my $error = $@; + return { wwids => \%wanted, $error ? (error => $error) : () }; +} + +sub sync() { + return {} if !PVE::Multipath::is_supported(); + my $res = eval { + require PVE::Storage; + regenerate(undef, undef, derived_wwids(PVE::Storage::config())); + }; + return $res // { error => "$@" }; +} + +1; diff --git a/src/PVE/Multipath/Identity.pm b/src/PVE/Multipath/Identity.pm index bc40c67b..8669e7dd 100644 --- a/src/PVE/Multipath/Identity.pm +++ b/src/PVE/Multipath/Identity.pm @@ -29,7 +29,7 @@ sub collect(%param) { ($wwid) = $data =~ m/^E:ID_SERIAL=(.+)$/m; } return if !defined($wwid) || $wwid !~ $PVE::Multipath::Config::WWID_RE; - $paths{$dev} = { wwid => $wwid }; + $paths{$dev} = { wwid => $wwid, $session ? (target => $session->{target}) : () }; }, ); return { diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm index fc4d6988..2d23b8b3 100644 --- a/src/PVE/Storage/ISCSIPlugin.pm +++ b/src/PVE/Storage/ISCSIPlugin.pm @@ -347,6 +347,22 @@ sub iscsi_device_list { return $res; } +# Each portal of the connection storage contributes one expected path. +sub multipath_lun_wwids { + my ($class, $scfg, $ids) = @_; + + my $res = { wwids => {} }; + my $target = $scfg->{target}; + + my $portals = iscsi_portals($target, $scfg->{portal}); + for my $path (values $ids->{paths}->%*) { + next if ($path->{target} // '') ne $target; + $res->{wwids}->{ $path->{wwid} } = { 'paths-expected' => scalar(@$portals) }; + } + + return $res; +} + # Configuration sub type { diff --git a/src/test/Makefile b/src/test/Makefile index c3632e05..a758e593 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -59,3 +59,9 @@ test_lun_identity: run_lun_identity_tests.pl test_lvm_base_activation: run_test_lvm_base_activation.pl ./run_test_lvm_base_activation.pl + +test_multipath_api: run_multipath_api_tests.pl + ./run_multipath_api_tests.pl + +test_san_taint: run_san_taint_tests.pl + perl -T ./run_san_taint_tests.pl diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl new file mode 100755 index 00000000..e0c5622d --- /dev/null +++ b/src/test/run_multipath_api_tests.pl @@ -0,0 +1,94 @@ +#!/usr/bin/perl + +use v5.36; +our $NOW = 1000; + +BEGIN { + *CORE::GLOBAL::time = sub { $NOW } +} +use lib ('.', '..'); +use File::Temp qw(tempdir); +use Test::More; +use Test::MockModule; +use PVE::Multipath::Generator; + +subtest 'add-only generation and failed reload throttle' => sub { + my $dir = tempdir('multipath-test-XXXX', DIR => '.', CLEANUP => 1); + local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults"; + local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides"; + local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases"; + local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending"; + my $node = Test::MockModule->new('PVE::INotify'); + $node->mock(nodename => sub { 'n1' }); + my $mp = Test::MockModule->new('PVE::Multipath'); + my (%listed, @added, $reloads, $fail); + $mp->mock(list_etc_multipath_wwids => sub { [sort keys %listed] }); + $mp->mock(add_wwid => sub { $listed{ $_[0] } = 1; push @added, $_[0] }); + $mp->mock(reconfigure => sub { $reloads++; die "reload failed\n" if $fail }); + my $cfg = { ids => { '3600a' => { type => 'wwid', alias => 'data' } } }; + my $res = PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply(\@added, ['3600a'], 'missing WWID added'); + is_deeply($res->{wwids}->{'3600a'}, {}, 'no expected path count without one in the entry'); + is($reloads, 1, 'changed config reloaded'); + $res = PVE::Multipath::Generator::regenerate( + { ids => { '3600a' => { $cfg->{ids}->{'3600a'}->%*, 'paths-expected' => 2 } } }, + '', + {}, + ); + is($res->{wwids}->{'3600a'}->{'paths-expected'}, 2, 'expected path count of the entry'); + is($reloads, 1, 'unchanged config not reloaded'); + PVE::Multipath::Generator::regenerate({ ids => {} }, '', {}); + ok($listed{'3600a'}, 'removed entry remains allow-listed'); + $fail = 1; + my $failed = PVE::Multipath::Generator::regenerate($cfg, '', {}); + like($failed->{error}, qr/reload failed/, 'reload error returned'); + ok(exists($failed->{wwids}->{'3600a'}), 'apply failure retains wanted LUNs'); + ok(-e "$dir/pending", 'failed reload remains pending'); + my $count = $reloads; + $NOW += 59; + $failed = PVE::Multipath::Generator::regenerate($cfg, '', {}); + like($failed->{error}, qr/reload pending/, 'pending reload remains visible'); + is($reloads, $count, 'no retry before 60 seconds'); + $NOW++; + $fail = 0; + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is($reloads, $count + 1, 'retry at 60 seconds'); + ok(!-e "$dir/pending", 'successful reload clears pending marker'); +}; + +subtest 'storage LUNs are the identified paths of the storage target' => sub { + my $topology = Test::MockModule->new('PVE::ISCSI::Topology'); + $topology->mock( + device_map => sub { + return { map { $_ => { target => $_ eq 'sdc' ? 'iqn.other' : 'iqn.san' } } + qw(sda sdb sdc) }; + }, + ); + my $identity = Test::MockModule->new('PVE::Multipath::Identity'); + $identity->mock(dir_glob_foreach => sub { $_[2]->($_) for qw(sda sdb sdc sdd) }); + my $mp = Test::MockModule->new('PVE::Multipath'); + my %wwid = (sda => '3600a', sdb => '3600a', sdc => '3600b', sdd => '3600c'); + $mp->mock( + query_path_identities => sub { + return { map { $_ => { wwid => $wwid{$_} } } keys %wwid }; + }, + ); + my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + $iscsi->mock(iscsi_portals => sub { ['192.0.2.1:3260', '192.0.2.2:3260'] }); + my $cfg = + { ids => { san => { type => 'iscsi', target => 'iqn.san', 'managed-multipath' => 1 } } }; + my $expected = { san => { wwids => { '3600a' => { 'paths-expected' => 2 } } } }; + is_deeply( + PVE::Multipath::Generator::derived_wwids($cfg), + $expected, + 'one WWID per LUN of the target, one expected path per portal', + ); + my $all = PVE::Multipath::Identity::collect(all => 1); + is_deeply( + PVE::Multipath::Generator::derived_wwids($cfg, $all), + $expected, + 'paths of other targets and transports in a full collection are left out', + ); +}; + +done_testing(); diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl index ae44e962..c8800c38 100755 --- a/src/test/run_multipath_tests.pl +++ b/src/test/run_multipath_tests.pl @@ -146,7 +146,19 @@ subtest 'one owner for map identity and membership' => sub { subtest 'short commands and link-only assembly wait' => sub { my $mp = Test::MockModule->new('PVE::Multipath'); my @calls; + $mp->mock(is_supported => sub { 1 }); $mp->mock(run_command => sub($cmd, %opts) { push @calls, [$cmd, $opts{timeout}] }); + PVE::Multipath::add_wwid('3600a'); + PVE::Multipath::reconfigure(); + is_deeply( + \@calls, + [ + [['/sbin/multipath', '-a', '--', '3600a'], 10], + [['/sbin/multipathd', 'reconfigure'], 30], + ], + 'loop commands have short timeouts', + ); + @calls = (); ok( !PVE::Multipath::wait_for_map('nonexistent-test-lun', 0), 'missing by-id block link times out', diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl index 819b9fc3..2ec2a651 100755 --- a/src/test/run_san_library_tests.pl +++ b/src/test/run_san_library_tests.pl @@ -13,6 +13,7 @@ my %registered; my $cluster = Test::MockModule->new('PVE::Cluster'); $cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] }); require PVE::Multipath::Config; +require PVE::Multipath::Generator; subtest 'CFS callbacks preserve filenames, data and method receivers' => sub { my ($parse, $write) = $registered{'multipath.cfg'}->@*; @@ -31,6 +32,11 @@ subtest 'CFS callbacks preserve filenames, data and method receivers' => sub { subtest 'omitted optional parameters retain explicit undef behavior' => sub { my $config = Test::MockModule->new('PVE::Multipath::Config'); + $config->mock(read_config => sub { die "configuration unavailable\n" }); + for my $args ([], [undef, undef, undef]) { + eval { PVE::Multipath::Generator::regenerate(@$args) }; + is($@, "configuration unavailable\n", 'regeneration reads the default configuration'); + } $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" }); for my $lock ( \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides, diff --git a/src/test/run_san_taint_tests.pl b/src/test/run_san_taint_tests.pl new file mode 100755 index 00000000..35b11cd7 --- /dev/null +++ b/src/test/run_san_taint_tests.pl @@ -0,0 +1,34 @@ +#!/usr/bin/perl -T + +use v5.36; +use lib ('.', '..'); +use File::Temp qw(tempfile); +use Scalar::Util qw(tainted); +use Test::MockModule; +use Test::More; +use PVE::Multipath; +use PVE::Tools; + +$ENV{PATH} = '/usr/sbin:/usr/bin:/sbin:/bin'; +delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; +my ($fh, $path) = tempfile('multipath-taint-XXXX', DIR => '.', UNLINK => 1); +print $fh '3600a'; +close($fh); +my $wwid = PVE::Tools::file_get_contents($path); +ok(tainted($wwid), 'WWID read from file is tainted'); +my $mp = Test::MockModule->new('PVE::Multipath'); +$mp->mock(is_supported => sub { 1 }); +$mp->mock( + run_command => sub($cmd, %opts) { + is_deeply($cmd, ['/sbin/multipath', '-a', '--', '3600a'], 'WWID add command'); + ok($opts{timeout}, 'command is bounded'); + PVE::Tools::run_command(['/usr/bin/true', @$cmd]); + }, +); +PVE::Multipath::add_wwid($wwid); +for my $bad ('-bad', '1:2', '../dev', "3600a\n") { + eval { PVE::Multipath::add_wwid($bad) }; + like($@, qr/invalid WWID/, 'ambiguous identity refused'); +} + +done_testing();