From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 6AE331FF0DA for ; Thu, 19 Nov 2026 20:53:37 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3FFF721EBD; Thu, 19 Nov 2026 20:48:43 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v8 storage 12/43] multipath: apply the configuration on every node Date: Thu, 19 Nov 2026 20:46:17 +0100 Message-ID: <20261119194655.3163258-13-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261119194655.3163258-1-t.lamprecht@proxmox.com> References: <20261119194655.3163258-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795117621800 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.150 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: LRLDCFP5B3LKOK4B2FTCR2ZHOOXFHY6V X-Message-ID-Hash: LRLDCFP5B3LKOK4B2FTCR2ZHOOXFHY6V X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Each node needs the cluster-wide configuration as local multipath.conf drop-ins. It also needs the WWIDs of its LUNs in the allow-list of multipath: the explicit entries valid for the node and the LUNs it sees through sessions of enabled iSCSI storages with the multipath option. Storages of other types, also those of third-party plugins, need explicit entries for their LUNs instead; a plugin method for this can still be added later. Missing WWIDs are added, but none is ever removed. A WWID that multipath refuses, for example a blacklisted one, is reported as an error and holds up neither the other WWIDs nor the reload. It is tried again after five minutes, or sooner if a drop-in changed, and the error names at most four such WWIDs and counts the others, so that the node report stays small. The generated defaults set 'auto_resize grow_only', so that a map follows a grown LUN. multipath only knows this setting for all maps of a node. So once a node has something to apply, maps that Proxmox VE does not manage also grow when all their paths have grown. 'auto_resize never' in a defaults section of the overrides file keeps the resize manual, as multipath reads that file after the generated defaults. A drop-in with a generated name whose first line is not the generated one is left alone. If the node needs that file, applying fails until it is renamed, and no file is changed before that check. Nothing calls this yet. Signed-off-by: Thomas Lamprecht --- src/PVE/Makefile | 1 + src/PVE/Multipath.pm | 19 ++- src/PVE/Multipath/Generator.pm | 156 ++++++++++++++++++ src/PVE/Multipath/Identity.pm | 2 +- src/PVE/Storage/ISCSIPlugin.pm | 16 ++ src/test/Makefile | 8 +- src/test/run_multipath_api_tests.pl | 247 ++++++++++++++++++++++++++++ src/test/run_multipath_tests.pl | 12 ++ src/test/run_san_library_tests.pl | 6 + src/test/run_san_taint_tests.pl | 34 ++++ 10 files changed, 498 insertions(+), 3 deletions(-) create mode 100644 src/PVE/Multipath/Generator.pm create mode 100755 src/test/run_multipath_api_tests.pl create mode 100755 src/test/run_san_taint_tests.pl diff --git a/src/PVE/Makefile b/src/PVE/Makefile index 53347442..87d5c41f 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -7,6 +7,7 @@ install: install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm + install -D -m 0644 Multipath/Generator.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Generator.pm install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm index 1caadee1..ec417381 100644 --- a/src/PVE/Multipath.pm +++ b/src/PVE/Multipath.pm @@ -5,7 +5,7 @@ use v5.36; use Encode qw(decode encode); use JSON qw(decode_json); -use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach); +use PVE::Tools qw(run_command file_get_contents file_read_firstline dir_glob_foreach); use PVE::Diskmanage; use PVE::Multipath::Config; @@ -169,4 +169,21 @@ sub wait_for_map($wwid, $timeout = 10) { } } +sub list_etc_multipath_wwids() { + return [] if !-e '/etc/multipath/wwids'; + return PVE::Multipath::Config::parse_wwids(file_get_contents('/etc/multipath/wwids')); +} + +sub add_wwid($wwid) { + assert_supported(); + my ($id) = $wwid =~ /^($PVE::Multipath::Config::WWID_PATTERN)\z/; + die "invalid WWID '$wwid'\n" if !defined($id); + run_command([$MULTIPATH, '-a', '--', $id], timeout => 10); +} + +sub reconfigure() { + assert_supported(); + run_command([$MULTIPATHD, 'reconfigure'], timeout => 30); +} + 1; diff --git a/src/PVE/Multipath/Generator.pm b/src/PVE/Multipath/Generator.pm new file mode 100644 index 00000000..203c6de5 --- /dev/null +++ b/src/PVE/Multipath/Generator.pm @@ -0,0 +1,156 @@ +package PVE::Multipath::Generator; + +use v5.36; + +use File::Path qw(make_path); + +use PVE::INotify; +use PVE::Tools qw(file_get_contents file_set_contents); + +use PVE::Multipath; +use PVE::Multipath::Config; +use PVE::Multipath::Identity; + +# Local drop-ins let multipathd start before pmxcfs is available. +our $DEFAULTS_DROPIN = '/etc/multipath/conf.d/pve-defaults.conf'; +our $OVERRIDES_DROPIN = '/etc/multipath/conf.d/pve-overrides.conf'; +our $ALIASES_DROPIN = '/etc/multipath/conf.d/pve-aliases.conf'; +our $RECONFIGURE_PENDING = '/run/pve-multipath.reconfigure-pending'; +my ($failed_at, $failed_error); +my %refused; # WWID => [time, error] of the last refusal + +sub derived_wwids($cfg, $ids = undef) { + require PVE::Storage; + my @storages = grep { + my $scfg = $cfg->{ids}->{$_}; + $scfg->{type} eq 'iscsi' + && $scfg->{'managed-multipath'} + && PVE::Storage::storage_check_enabled($cfg, $_, undef, 1) + } sort keys $cfg->{ids}->%*; + if (!@storages) { + return {}; + } + $ids //= PVE::Multipath::Identity::collect( + targets => { map { $cfg->{ids}->{$_}->{target} => 1 } @storages }); + return { map { $_ => PVE::Storage::ISCSIPlugin->multipath_lun_wwids($cfg->{ids}->{$_}, $ids) } + @storages }; +} + +my sub pending() { file_set_contents($RECONFIGURE_PENDING, '') if !-e $RECONFIGURE_PENDING } +my sub dropin_changes($path, $content) { + my $old = -e $path ? file_get_contents($path) : ''; + return 0 if $old eq $content; + # a file of this name that an administrator wrote stays + if (length($old) && $old !~ m/\A# Managed by Proxmox VE\.\n/) { + die "cannot replace '$path' - it was not written by Proxmox VE, rename it\n" + if length($content); + return 0; + } + return 1; +} +my sub write_dropin($path, $content) { + pending(); + if (length($content)) { + make_path($path =~ s!/[^/]+$!!r); + file_set_contents($path, $content); + } else { + unlink($path) or die "cannot remove '$path' - $!\n"; + } +} + +sub regenerate($cfg = undef, $overrides = undef, $derived = undef) { + $cfg //= PVE::Multipath::Config::read_config(); + $overrides //= PVE::Multipath::Config::read_overrides(); + $derived //= {}; + my $node = PVE::INotify::nodename(); + my @explicit = PVE::Multipath::Config::wwids_for_node($cfg, $node)->@*; + my %wanted = map { + my $count = $cfg->{ids}->{$_}->{'paths-expected'}; + $_ => defined($count) ? { 'paths-expected' => $count } : {} + } @explicit; + for my $sid (sort keys %$derived) { + for my $wwid (sort keys $derived->{$sid}->{wwids}->%*) { + push $wanted{$wwid}->{storages}->@*, $sid; + my $expected = $derived->{$sid}->{wwids}->{$wwid}->{'paths-expected'}; + $wanted{$wwid}->{'paths-expected'} = $expected + if defined($expected) && $expected > ($wanted{$wwid}->{'paths-expected'} // 0); + } + } + my @added; + eval { + my $in_use = @explicit || keys(%$derived) || length($overrides); + my @dropins = ( + [$DEFAULTS_DROPIN, $in_use ? PVE::Multipath::Config::generate_managed_conf() : ''], + [ + $OVERRIDES_DROPIN, + length($overrides) ? "# Managed by Proxmox VE.\n$overrides" : '', + ], + [ + $ALIASES_DROPIN, + @explicit ? PVE::Multipath::Config::build_multipaths_block($cfg, $node) : '', + ], + ); + # check every drop-in before one changes, as a restart of multipathd loads what is there + my @changes = grep { dropin_changes($_->@*) } @dropins; + write_dropin($_->@*) for @changes; + my %current = map { $_ => 1 } PVE::Multipath::list_etc_multipath_wwids()->@*; + # A WWID that multipath refuses, for example a blacklisted one, must not hold up the others + # or the reload, nor cause a reload on every pass. Retry it after five minutes, or after a + # drop-in changed, which can lift the refusal. + %refused = () if @changes; + delete @refused{ grep { !$wanted{$_} || $current{$_} } keys %refused }; + my @failed; + for my $wwid (sort keys %wanted) { + next if $current{$wwid}; + my $age = $refused{$wwid} ? time() - $refused{$wwid}->[0] : undef; + if (defined($age) && $age >= 0 && $age < 300) { + push @failed, $refused{$wwid}->[1]; + next; + } + eval { PVE::Multipath::add_wwid($wwid) }; + if (my $err = $@) { + # a multipath that hangs for one WWID would for the next one as well, and the + # summary of refused WWIDs must not hide that + if ($err =~ /got timeout/) { + unshift(@failed, $err); + last; + } + push @failed, $err; + $refused{$wwid} = [time(), $err]; + next; + } + pending(); + push @added, $wwid; + } + if (-e $RECONFIGURE_PENDING) { + die "multipath reload pending after a failed reload - $failed_error\n" + if defined($failed_at) && time() >= $failed_at && time() - $failed_at < 60; + eval { PVE::Multipath::reconfigure() }; + if (my $error = $@) { + $failed_at = time(); + chomp($failed_error = $error); + die $error; + } + unlink($RECONFIGURE_PENDING) or die "cannot clear multipath reload marker - $!\n"; + $failed_at = undef; + } + if (@failed > 4) { + my $more = @failed - 3; + splice(@failed, 3, $more, "and $more more WWIDs could not be added\n"); + } + die join('', @failed) if @failed; + }; + my $error = $@; + return { wwids => \%wanted, added => \@added, $error ? (error => $error) : () }; +} + +sub sync() { + return {} if !PVE::Multipath::is_supported(); + my $res = eval { + require PVE::Storage; + regenerate(undef, undef, derived_wwids(PVE::Storage::config())); + }; + return $res // { error => "$@" }; +} + +1; diff --git a/src/PVE/Multipath/Identity.pm b/src/PVE/Multipath/Identity.pm index bc40c67b..8669e7dd 100644 --- a/src/PVE/Multipath/Identity.pm +++ b/src/PVE/Multipath/Identity.pm @@ -29,7 +29,7 @@ sub collect(%param) { ($wwid) = $data =~ m/^E:ID_SERIAL=(.+)$/m; } return if !defined($wwid) || $wwid !~ $PVE::Multipath::Config::WWID_RE; - $paths{$dev} = { wwid => $wwid }; + $paths{$dev} = { wwid => $wwid, $session ? (target => $session->{target}) : () }; }, ); return { diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm index 88a43062..08c766df 100644 --- a/src/PVE/Storage/ISCSIPlugin.pm +++ b/src/PVE/Storage/ISCSIPlugin.pm @@ -347,6 +347,22 @@ sub iscsi_device_list { return $res; } +# Each portal of the connection storage contributes one expected path. +sub multipath_lun_wwids { + my ($class, $scfg, $ids) = @_; + + my $res = { wwids => {} }; + my $target = $scfg->{target}; + + my $portals = iscsi_portals($target, $scfg->{portal}); + for my $path (values $ids->{paths}->%*) { + next if ($path->{target} // '') ne $target; + $res->{wwids}->{ $path->{wwid} } = { 'paths-expected' => scalar(@$portals) }; + } + + return $res; +} + # Configuration sub type { diff --git a/src/test/Makefile b/src/test/Makefile index 72e306f7..bfef56d6 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_san_libraries test_lun_identity test_lvm_base_activation +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -65,3 +65,9 @@ test_lun_identity: run_lun_identity_tests.pl test_lvm_base_activation: run_test_lvm_base_activation.pl ./run_test_lvm_base_activation.pl + +test_multipath_api: run_multipath_api_tests.pl + ./run_multipath_api_tests.pl + +test_san_taint: run_san_taint_tests.pl + perl -T ./run_san_taint_tests.pl diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl new file mode 100755 index 00000000..4ba6d0e3 --- /dev/null +++ b/src/test/run_multipath_api_tests.pl @@ -0,0 +1,247 @@ +#!/usr/bin/perl + +use v5.36; +our $NOW; + +BEGIN { + $NOW = 1000; + *CORE::GLOBAL::time = sub { $NOW } +} +use lib ('.', '..'); +use File::Temp qw(tempdir); +use Test::More; +use Test::MockModule; +use PVE::Multipath::Generator; + +subtest 'add-only generation and failed reload throttle' => sub { + my $dir = tempdir('multipath-test-XXXX', DIR => '.', CLEANUP => 1); + local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults"; + local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides"; + local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases"; + local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending"; + my $node = Test::MockModule->new('PVE::INotify'); + $node->mock(nodename => sub { 'n1' }); + my $mp = Test::MockModule->new('PVE::Multipath'); + my (%listed, @added, $reloads, $fail); + $mp->mock(list_etc_multipath_wwids => sub { [sort keys %listed] }); + $mp->mock(add_wwid => sub { $listed{ $_[0] } = 1; push @added, $_[0] }); + $mp->mock(reconfigure => sub { $reloads++; die "reload failed\n" if $fail }); + my $cfg = { ids => { '3600a' => { type => 'wwid', alias => 'data' } } }; + my $res = PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply(\@added, ['3600a'], 'missing WWID added'); + is_deeply($res->{wwids}->{'3600a'}, {}, 'no expected path count without one in the entry'); + is($reloads, 1, 'changed config reloaded'); + $res = PVE::Multipath::Generator::regenerate( + { ids => { '3600a' => { $cfg->{ids}->{'3600a'}->%*, 'paths-expected' => 2 } } }, + '', + {}, + ); + is($res->{wwids}->{'3600a'}->{'paths-expected'}, 2, 'expected path count of the entry'); + is($reloads, 1, 'unchanged config not reloaded'); + PVE::Multipath::Generator::regenerate({ ids => {} }, '', {}); + ok($listed{'3600a'}, 'removed entry remains allow-listed'); + $fail = 1; + my $failed = PVE::Multipath::Generator::regenerate($cfg, '', {}); + like($failed->{error}, qr/reload failed/, 'reload error returned'); + ok(exists($failed->{wwids}->{'3600a'}), 'apply failure retains wanted LUNs'); + ok(-e "$dir/pending", 'failed reload remains pending'); + my $count = $reloads; + $NOW += 59; + $failed = PVE::Multipath::Generator::regenerate($cfg, '', {}); + like( + $failed->{error}, + qr/^multipath reload pending after a failed reload - reload failed$/, + 'pending reload remains visible with its error', + ); + is($reloads, $count, 'no retry before 60 seconds'); + $NOW -= 3600; + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is($reloads, ++$count, 'unless the clock went back'); + $NOW += 59; + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is($reloads, $count, 'which restarts the wait'); + $NOW++; + $fail = 0; + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is($reloads, $count + 1, 'retry at 60 seconds'); + ok(!-e "$dir/pending", 'successful reload clears pending marker'); + $NOW += 3600; +}; + +subtest 'a WWID that multipath refuses does not hold up the others' => sub { + my $dir = tempdir('multipath-refused-XXXX', DIR => '.', CLEANUP => 1); + local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults"; + local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides"; + local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases"; + local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending"; + my $node = Test::MockModule->new('PVE::INotify'); + $node->mock(nodename => sub { 'n1' }); + my $mp = Test::MockModule->new('PVE::Multipath'); + my (%listed, %refuse, @tried, $reloads); + $mp->mock(list_etc_multipath_wwids => sub { [sort keys %listed] }); + $mp->mock( + add_wwid => sub($wwid) { + push @tried, $wwid; + die "command 'multipath -a -- $wwid' failed: exit code 1\n" if $refuse{$wwid}; + $listed{$wwid} = 1; + }, + ); + $mp->mock(reconfigure => sub { $reloads++ }); + %refuse = ('3600b' => 1); + my $cfg = { ids => { map { $_ => { type => 'wwid' } } qw(3600a 3600b 3600c) } }; + my $res = PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply($res->{added}, [qw(3600a 3600c)], 'the other WWIDs are added'); + is($reloads, 1, 'and loaded with one reload'); + ok(!-e "$dir/pending", 'which clears the marker'); + like($res->{error}, qr/-- 3600b' failed/, 'the refused WWID is reported'); + @tried = (); + $res = PVE::Multipath::Generator::regenerate($cfg, '', {}); + like($res->{error}, qr/-- 3600b' failed/, 'on every pass'); + is_deeply(\@tried, [], 'but not tried again at once'); + is($reloads, 1, 'nor reloaded again'); + $NOW += 300; + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply(\@tried, ['3600b'], 'only after five minutes'); + PVE::Multipath::Generator::regenerate($cfg, "defaults {\n}\n", {}); + is_deeply(\@tried, ['3600b', '3600b'], 'or after a drop-in changed'); + $NOW -= 3600; + PVE::Multipath::Generator::regenerate($cfg, "defaults {\n}\n", {}); + is_deeply(\@tried, ['3600b', '3600b', '3600b'], 'or after the clock went back'); + $NOW += 3600; + # derived WWIDs leave the drop-ins alone, whose change would also lift a refusal + my $none = { ids => {} }; + my $with = { s1 => { wwids => { '3600a' => {}, '3600b' => {} } } }; + PVE::Multipath::Generator::regenerate($none, '', $with); + PVE::Multipath::Generator::regenerate($none, '', { s1 => { wwids => { '3600a' => {} } } }); + @tried = (); + PVE::Multipath::Generator::regenerate($none, '', $with); + is_deeply(\@tried, ['3600b'], 'or after a pass that did not want it'); + + %refuse = map { ("3601$_" => 1) } 1 .. 5; + my $many = { ids => { map { ("3601$_" => { type => 'wwid' }) } 1 .. 5 } }; + $res = PVE::Multipath::Generator::regenerate($many, '', {}); + is( + $res->{error}, + join('', map { "command 'multipath -a -- 3601$_' failed: exit code 1\n" } 1 .. 3) + . "and 2 more WWIDs could not be added\n", + 'the error names the first three refused WWIDs and counts the others', + ); + %refuse = map { ("3602$_" => 1) } 1 .. 4; + my $four = { ids => { map { ("3602$_" => { type => 'wwid' }) } 1 .. 4 } }; + $res = PVE::Multipath::Generator::regenerate($four, '', {}); + is( + $res->{error}, + join('', map { "command 'multipath -a -- 3602$_' failed: exit code 1\n" } 1 .. 4), + 'up to four refused WWIDs are all named', + ); + + $mp->mock( + add_wwid => sub($wwid) { + push @tried, $wwid; + die "command 'multipath -a -- $wwid' failed: got timeout\n"; + }, + ); + %listed = (); + @tried = (); + $res = PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply(\@tried, ['3600a'], 'a timeout ends the adds of the pass'); + like($res->{error}, qr/got timeout/, 'and is reported'); + PVE::Multipath::Generator::regenerate($cfg, '', {}); + is_deeply(\@tried, ['3600a', '3600a'], 'the WWID is tried again on the next pass'); + + $mp->mock( + add_wwid => sub($wwid) { + die "command 'multipath -a -- $wwid' failed: got timeout\n" if $wwid eq '3709'; + die "command 'multipath -a -- $wwid' failed: exit code 1\n"; + }, + ); + my $mixed = + { ids => { map { ($_ => { type => 'wwid' }) } qw(37001 37002 37003 37004 3709) } }; + $res = PVE::Multipath::Generator::regenerate($mixed, '', {}); + like($res->{error}, qr/\A[^\n]*-- 3709' failed: got timeout\n/, 'a timeout is named first'); + like($res->{error}, qr/and 2 more WWIDs/, 'before the summary of the refused WWIDs'); +}; + +subtest 'drop-ins of the same name that an administrator wrote stay' => sub { + my $dir = tempdir('multipath-foreign-XXXX', DIR => '.', CLEANUP => 1); + local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults"; + local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides"; + local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases"; + local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending"; + my $mp = Test::MockModule->new('PVE::Multipath'); + my $reloads = 0; + $mp->mock(list_etc_multipath_wwids => sub { [] }); + $mp->mock(add_wwid => sub { }); + $mp->mock(reconfigure => sub { $reloads++ }); + my $own = "defaults {\n\tfind_multipaths strict\n}\n"; + my $admin = "devices {\n\tdevice {\n\t\tvendor \"NETAPP\"\n\t}\n}\n"; + PVE::Tools::file_set_contents("$dir/overrides", $admin); + + my $near = "# Not managed by Proxmox VE\nmultipaths {\n}\n"; + PVE::Tools::file_set_contents("$dir/aliases", $near); + my $res = PVE::Multipath::Generator::regenerate({ ids => {} }, '', {}); + is(PVE::Tools::file_get_contents("$dir/overrides"), $admin, 'kept on an unconfigured node'); + is(PVE::Tools::file_get_contents("$dir/aliases"), $near, 'also with a similar first line'); + ok(!$res->{error} && !$reloads, 'without error or reload'); + my $edit = "# Managed by Proxmox VE, edited by hand\nmultipaths {\n}\n"; + PVE::Tools::file_set_contents("$dir/aliases", $edit); + PVE::Multipath::Generator::regenerate({ ids => {} }, '', {}); + is(PVE::Tools::file_get_contents("$dir/aliases"), $edit, 'also with an edited first line'); + # an outdated file of Proxmox VE that the next pass would remove + my $stale = "# Managed by Proxmox VE.\nmultipaths {\n}\n"; + PVE::Tools::file_set_contents("$dir/aliases", $stale); + $res = PVE::Multipath::Generator::regenerate({ ids => {} }, $own, {}); + is(PVE::Tools::file_get_contents("$dir/overrides"), $admin, 'kept when overrides are set'); + like($res->{error}, qr/not written by Proxmox VE, rename it/, 'with an error'); + is($reloads, 0, 'and no reload'); + ok(!-e "$dir/defaults", 'no other drop-in is written'); + is(PVE::Tools::file_get_contents("$dir/aliases"), $stale, 'or removed'); + ok(!-e "$dir/pending", 'or marked for a reload'); + + unlink("$dir/overrides"); + $res = PVE::Multipath::Generator::regenerate({ ids => {} }, $own, {}); + like( + PVE::Tools::file_get_contents("$dir/overrides"), + qr/^# Managed by Proxmox VE\./, + 'written once the name is free', + ); + $res = PVE::Multipath::Generator::regenerate({ ids => {} }, '', {}); + ok(!-e "$dir/overrides" && !$res->{error}, 'removed again when no longer needed'); +}; + +subtest 'storage LUNs are the identified paths of the storage target' => sub { + my $topology = Test::MockModule->new('PVE::ISCSI::Topology'); + $topology->mock( + device_map => sub { + return { map { $_ => { target => $_ eq 'sdc' ? 'iqn.other' : 'iqn.san' } } + qw(sda sdb sdc) }; + }, + ); + my $identity = Test::MockModule->new('PVE::Multipath::Identity'); + $identity->mock(dir_glob_foreach => sub { $_[2]->($_) for qw(sda sdb sdc sdd) }); + my $mp = Test::MockModule->new('PVE::Multipath'); + my %wwid = (sda => '3600a', sdb => '3600a', sdc => '3600b', sdd => '3600c'); + $mp->mock( + query_path_identities => sub { + return { map { $_ => { wwid => $wwid{$_} } } keys %wwid }; + }, + ); + my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); + $iscsi->mock(iscsi_portals => sub { ['192.0.2.1:3260', '192.0.2.2:3260'] }); + my $cfg = + { ids => { san => { type => 'iscsi', target => 'iqn.san', 'managed-multipath' => 1 } } }; + my $expected = { san => { wwids => { '3600a' => { 'paths-expected' => 2 } } } }; + is_deeply( + PVE::Multipath::Generator::derived_wwids($cfg), + $expected, + 'one WWID per LUN of the target, one expected path per portal', + ); + my $all = PVE::Multipath::Identity::collect(all => 1); + is_deeply( + PVE::Multipath::Generator::derived_wwids($cfg, $all), + $expected, + 'paths of other targets and transports in a full collection are left out', + ); +}; + +done_testing(); diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl index 4688f817..9ed591cc 100755 --- a/src/test/run_multipath_tests.pl +++ b/src/test/run_multipath_tests.pl @@ -171,7 +171,19 @@ subtest 'map identity from the dm UUID and map members' => sub { subtest 'short commands and link-only assembly wait' => sub { my $mp = Test::MockModule->new('PVE::Multipath'); my @calls; + $mp->mock(is_supported => sub { 1 }); $mp->mock(run_command => sub($cmd, %opts) { push @calls, [$cmd, $opts{timeout}] }); + PVE::Multipath::add_wwid('3600a'); + PVE::Multipath::reconfigure(); + is_deeply( + \@calls, + [ + [['/sbin/multipath', '-a', '--', '3600a'], 10], + [['/sbin/multipathd', 'reconfigure'], 30], + ], + 'status loop commands have timeouts', + ); + @calls = (); ok( !PVE::Multipath::wait_for_map('nonexistent-test-lun', 0), 'missing by-id block link times out', diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl index eab6570f..c4293eab 100755 --- a/src/test/run_san_library_tests.pl +++ b/src/test/run_san_library_tests.pl @@ -13,6 +13,7 @@ my %registered; my $cluster = Test::MockModule->new('PVE::Cluster'); $cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] }); require PVE::Multipath::Config; +require PVE::Multipath::Generator; subtest 'CFS callbacks preserve filenames, data and method receivers' => sub { my ($parse, $write) = $registered{'multipath.cfg'}->@*; @@ -31,6 +32,11 @@ subtest 'CFS callbacks preserve filenames, data and method receivers' => sub { subtest 'omitted optional arguments behave like undef' => sub { my $config = Test::MockModule->new('PVE::Multipath::Config'); + $config->mock(read_config => sub { die "configuration unavailable\n" }); + for my $args ([], [undef, undef, undef]) { + eval { PVE::Multipath::Generator::regenerate(@$args) }; + is($@, "configuration unavailable\n", 'regeneration reads the default configuration'); + } $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" }); for my $lock ( \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides, diff --git a/src/test/run_san_taint_tests.pl b/src/test/run_san_taint_tests.pl new file mode 100755 index 00000000..35b11cd7 --- /dev/null +++ b/src/test/run_san_taint_tests.pl @@ -0,0 +1,34 @@ +#!/usr/bin/perl -T + +use v5.36; +use lib ('.', '..'); +use File::Temp qw(tempfile); +use Scalar::Util qw(tainted); +use Test::MockModule; +use Test::More; +use PVE::Multipath; +use PVE::Tools; + +$ENV{PATH} = '/usr/sbin:/usr/bin:/sbin:/bin'; +delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; +my ($fh, $path) = tempfile('multipath-taint-XXXX', DIR => '.', UNLINK => 1); +print $fh '3600a'; +close($fh); +my $wwid = PVE::Tools::file_get_contents($path); +ok(tainted($wwid), 'WWID read from file is tainted'); +my $mp = Test::MockModule->new('PVE::Multipath'); +$mp->mock(is_supported => sub { 1 }); +$mp->mock( + run_command => sub($cmd, %opts) { + is_deeply($cmd, ['/sbin/multipath', '-a', '--', '3600a'], 'WWID add command'); + ok($opts{timeout}, 'command is bounded'); + PVE::Tools::run_command(['/usr/bin/true', @$cmd]); + }, +); +PVE::Multipath::add_wwid($wwid); +for my $bad ('-bad', '1:2', '../dev', "3600a\n") { + eval { PVE::Multipath::add_wwid($bad) }; + like($@, qr/invalid WWID/, 'ambiguous identity refused'); +} + +done_testing();