From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 9B9D11FF0E1 for ; Mon, 16 Nov 2026 02:26:10 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3B267218E0; Mon, 16 Nov 2026 02:24:46 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v5 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Date: Mon, 16 Nov 2026 02:23:19 +0100 Message-ID: <20261116012355.3187417-11-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com> References: <20261116012355.3187417-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794792241778 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.210 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: BGLHSAPPKQAJMODGL22YACVM3P65JTRP X-Message-ID-Hash: BGLHSAPPKQAJMODGL22YACVM3P65JTRP X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: With the new 'managed-multipath' option, an iSCSI storage names its volumes after the multipath map of the LUN (dm-uuid-mpath-), as the scsi- name can resolve to a single path while the map has not assembled yet. Activating such a volume waits up to ten seconds for the map, and an LVM storage on top only accepts a map as base volume. The option is not named 'multipath': third-party storage plugins define a property of that name already, and a property that two plugins define stops the storage layer from loading. The WWID is the one multipathd reports for a path, or the ID_SERIAL udev property of the disk if multipathd is not available. Adding the LUNs to the multipath allow-list of the nodes follows separately. Signed-off-by: Thomas Lamprecht --- src/PVE/Makefile | 1 + src/PVE/Multipath.pm | 36 +++++++++++ src/PVE/Multipath/Identity.pm | 44 +++++++++++++ src/PVE/Storage/ISCSIPlugin.pm | 41 +++++++++++- src/PVE/Storage/LVMPlugin.pm | 7 ++- src/test/Makefile | 8 ++- src/test/run_iscsi_activation_tests.pl | 86 ++++++++++++++++++++++++++ src/test/run_lun_identity_tests.pl | 48 ++++++++++++++ src/test/run_multipath_tests.pl | 12 ++++ 9 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 src/PVE/Multipath/Identity.pm create mode 100755 src/test/run_iscsi_activation_tests.pl create mode 100755 src/test/run_lun_identity_tests.pl diff --git a/src/PVE/Makefile b/src/PVE/Makefile index 597ad7a1..53347442 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -7,6 +7,7 @@ install: install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm + install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm make -C Storage install diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm index 84c4901e..1caadee1 100644 --- a/src/PVE/Multipath.pm +++ b/src/PVE/Multipath.pm @@ -8,6 +8,7 @@ use JSON qw(decode_json); use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach); use PVE::Diskmanage; +use PVE::Multipath::Config; my $MULTIPATH = '/sbin/multipath'; my $MULTIPATHD = '/sbin/multipathd'; @@ -117,6 +118,31 @@ sub parse_maps_json($json) { return \@maps; } +sub parse_path_identities($text) { + my @lines = grep { length } split(/\n/, $text // ''); + return {} if @lines == 1 && $lines[0] eq 'ok'; + my $res = {}; + for my $line (@lines) { + my ($dev, $wwid) = split(/\|/, $line, 2); + return undef if !defined($wwid); + next if $dev !~ /^sd[a-z]+$/ || $wwid !~ $PVE::Multipath::Config::WWID_RE; + $res->{$dev} = { wwid => $wwid }; + } + return $res; +} + +sub query_path_identities() { + assert_supported(); + my $text = ''; + run_command( + [$MULTIPATHD, 'show', 'paths', 'raw', 'format', '%d|%w'], + timeout => 5, + outfunc => sub($line) { $text .= "$line\n" }, + errfunc => sub($line) { }, + ); + return parse_path_identities($text) // die "multipathd returned an invalid path list\n"; +} + sub get_maps() { assert_supported(); my $json = ''; @@ -133,4 +159,14 @@ sub get_maps() { return $maps; } +sub wait_for_map($wwid, $timeout = 10) { + my $link = "/dev/disk/by-id/dm-uuid-mpath-$wwid"; + my $deadline = time() + ($timeout // 10); + while (1) { + return 1 if -b $link; + return undef if time() >= $deadline; + sleep(1); + } +} + 1; diff --git a/src/PVE/Multipath/Identity.pm b/src/PVE/Multipath/Identity.pm new file mode 100644 index 00000000..bc40c67b --- /dev/null +++ b/src/PVE/Multipath/Identity.pm @@ -0,0 +1,44 @@ +package PVE::Multipath::Identity; + +use v5.36; + +use PVE::Tools qw(dir_glob_foreach file_get_contents file_read_firstline); + +use PVE::ISCSI::Topology; +use PVE::Multipath; +use PVE::Multipath::Config; + +our $SYSFS = '/sys'; +our $UDEV_DATA = '/run/udev/data'; + +sub collect(%param) { + my $devices = PVE::ISCSI::Topology::device_map($SYSFS); + my $daemon = eval { PVE::Multipath::query_path_identities() }; + my %paths; + dir_glob_foreach( + $SYSFS . '/class/block', + 'sd[a-z]+', + sub($dev, @) { + my $session = $devices->{$dev}; + return if !$param{all} && (!$session || !$param{targets}->{ $session->{target} }); + my $wwid = $daemon ? $daemon->{$dev}->{wwid} : undef; + # multipathd does not list every disk, for example not the ones it has blacklisted + if (!defined($wwid)) { + my $devt = file_read_firstline("$SYSFS/class/block/$dev/dev") // return; + my $data = eval { file_get_contents("$UDEV_DATA/b$devt") } // ''; + ($wwid) = $data =~ m/^E:ID_SERIAL=(.+)$/m; + } + return if !defined($wwid) || $wwid !~ $PVE::Multipath::Config::WWID_RE; + $paths{$dev} = { wwid => $wwid }; + }, + ); + return { + paths => \%paths, + }; +} + +sub path_wwid($ids, $name) { + return $ids->{paths}->{$name}->{wwid}; +} + +1; diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm index 70c0e9dc..fc4d6988 100644 --- a/src/PVE/Storage/ISCSIPlugin.pm +++ b/src/PVE/Storage/ISCSIPlugin.pm @@ -10,6 +10,7 @@ use IO::File; use PVE::JSONSchema qw(get_standard_option); use PVE::Multipath; +use PVE::Multipath::Identity; use PVE::Storage::Plugin; use PVE::ISCSI::Topology; use PVE::Tools @@ -292,11 +293,15 @@ sub load_stable_scsi_paths { return $stable_paths; } +# The LUNs of the iSCSI sessions, { target => { volname => info } }. With $map_names a LUN is named +# after its multipath map, see PVE::Multipath::Identity, and only the LUNs of $target are listed. sub iscsi_device_list { + my ($map_names, $target) = @_; my $res = {}; my $stable_paths = load_stable_scsi_paths(); + my $ids = $map_names ? PVE::Multipath::Identity::collect(targets => { $target => 1 }) : undef; my $device_map = PVE::ISCSI::Topology::device_map($SYSFS); @@ -304,6 +309,8 @@ sub iscsi_device_list { my $info = $device_map->{$bdev}; my ($channel, $id, $lun) = $info->@{qw(channel id lun)}; + my $path_dev = $bdev; + #check multipath if (-d "$SYSFS/block/$bdev/holders") { my $multipathdev = dir_glob_regex("$SYSFS/block/$bdev/holders", '[A-Za-z]\S*'); @@ -311,6 +318,15 @@ sub iscsi_device_list { } my $blockdev = $stable_paths->{$bdev}; + # scsi- can name a member before assembly; dm-uuid-mpath- only names the map. + # Managed volumes keep the map name even while assembly is pending. + if ($map_names) { + my $wwid = + $info->{target} eq $target + ? PVE::Multipath::Identity::path_wwid($ids, $path_dev) + : undef; + $blockdev = defined($wwid) ? "dm-uuid-mpath-$wwid" : undef; + } next if !$blockdev; my $size = file_read_firstline("$SYSFS/block/$bdev/size"); @@ -356,6 +372,11 @@ sub properties { type => 'string', format => 'pve-storage-portal-dns', }, + 'managed-multipath' => { + description => "Manage multipath access to the storage's LUNs and include them in" + . " the cluster-wide path-health monitoring.", + type => 'boolean', + }, }; } @@ -367,6 +388,7 @@ sub options { disable => { optional => 1 }, content => { optional => 1 }, bwlimit => { optional => 1 }, + 'managed-multipath' => { optional => 1 }, }; } @@ -437,13 +459,15 @@ sub list_images { my $res = []; - $cache->{iscsi_devices} = iscsi_device_list() if !$cache->{iscsi_devices}; + my $cache_key = + $scfg->{'managed-multipath'} ? "iscsi_devices_mpath:$scfg->{target}" : 'iscsi_devices'; + $cache->{$cache_key} //= iscsi_device_list($scfg->{'managed-multipath'}, $scfg->{target}); # we have no owner for iscsi devices my $target = $scfg->{target}; - if (my $dat = $cache->{iscsi_devices}->{$target}) { + if (my $dat = $cache->{$cache_key}->{$target}) { foreach my $volname (keys %$dat) { @@ -614,6 +638,19 @@ $resolve_virtual_devices = sub { sub activate_volume { my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_; + if ($scfg->{'managed-multipath'}) { + my ($wwid) = $volname =~ m/\.dm-uuid-mpath-(\S+)$/; + if (defined($wwid) && !-b "/dev/disk/by-id/dm-uuid-mpath-$wwid") { + # Activation can race map assembly at login or boot. Only wait for a LUN that is there. + my $ids = PVE::Multipath::Identity::collect(targets => { $scfg->{target} => 1 }); + die "LUN with WWID '$wwid' is not visible on this node\n" + if !grep { $_->{wwid} eq $wwid } values $ids->{paths}->%*; + PVE::Multipath::wait_for_map($wwid) + or die "multipath map for WWID '$wwid' did not assemble in time" + . " - check the multipath status of this node\n"; + } + } + my $path = $class->filesystem_path($scfg, $volname, $snapname); my $real_path = Cwd::realpath($path); die "failed to get realpath for '$path': $!\n" if !$real_path; diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm index e576bd74..78538111 100644 --- a/src/PVE/Storage/LVMPlugin.pm +++ b/src/PVE/Storage/LVMPlugin.pm @@ -1087,11 +1087,16 @@ sub on_add_hook { die "unsupported base type '$basecfg->{type}'" if $basecfg->{type} ne 'iscsi'; + # Managed multipath requires the coalesced map, never a transient single-path PV. + die "base volume '$volname' is no multipath volume of storage '$baseid'\n" + if $basecfg->{'managed-multipath'} && $volname !~ m/\.dm-uuid-mpath-\S+$/; + my $path = PVE::Storage::path($cfg, $base); PVE::Storage::activate_storage($cfg, $baseid); - # Activating the base volume verifies that it belongs to its target. + # the base's activate_volume checks target membership and, with managed multipath, waits + # for the map to assemble before the VG is created on it PVE::Storage::activate_volumes($cfg, [$base]); # a failing hook does not remove a volume group it created, so check the device first diff --git a/src/test/Makefile b/src/test/Makefile index 21f51441..389aac31 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -40,7 +40,8 @@ test_volume_access: run_volume_access_tests.pl test_remote_command: run_remote_command_tests.pl ./run_remote_command_tests.pl -test_iscsi: run_iscsi_discovery_tests.pl run_iscsi_rescan_tests.pl run_iscsi_device_list_tests.pl run_iscsi_session_tests.pl run_iscsi_volume_tests.pl +test_iscsi: run_iscsi_activation_tests.pl run_iscsi_discovery_tests.pl run_iscsi_rescan_tests.pl run_iscsi_device_list_tests.pl run_iscsi_session_tests.pl run_iscsi_volume_tests.pl + ./run_iscsi_activation_tests.pl ./run_iscsi_discovery_tests.pl ./run_iscsi_rescan_tests.pl ./run_iscsi_device_list_tests.pl @@ -52,3 +53,6 @@ test_multipath: run_multipath_tests.pl test_san_libraries: run_san_library_tests.pl ./run_san_library_tests.pl + +test_lun_identity: run_lun_identity_tests.pl + ./run_lun_identity_tests.pl diff --git a/src/test/run_iscsi_activation_tests.pl b/src/test/run_iscsi_activation_tests.pl new file mode 100755 index 00000000..414d0e04 --- /dev/null +++ b/src/test/run_iscsi_activation_tests.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Test::MockModule; +use Test::More; + +use lib ('.', '..'); +use PVE::Storage::ISCSIPlugin; + +my $plugin = Test::MockModule->new('PVE::Storage::ISCSIPlugin'); +my $identity = Test::MockModule->new('PVE::Multipath::Identity'); +my $mp = Test::MockModule->new('PVE::Multipath'); +my $cwd = Test::MockModule->new('Cwd'); +my ($resolved, $waits, $ready, $visible) = (undef, 0, 0, undef); +$identity->mock( + collect => sub { + die "unexpected identity query\n" if !defined($visible); + return { paths => { map { $_ => { wwid => '3600a' } } @$visible } }; + }, +); +$mp->mock( + wait_for_map => sub { + $waits++; + is($_[0], '3600a', 'map name supplies the WWID'); + return $ready; + }, +); +$cwd->mock(realpath => sub { $resolved }); +$plugin->mock( + filesystem_path => sub { '/dev/disk/by-id/test' }, + file_read_firstline => sub { 'iqn.test' }, + run_command => sub { + my ($cmd, %opts) = @_; + is($cmd->[0], 'udevadm', 'normal activation validates target membership'); + $opts{outfunc}->('/devices/platform/host1/session1/target1:0:0/1:0:0:1/block/sda'); + }, +); + +for my $multipath (0, 1) { + for my $case ([undef, qr/failed to get realpath/], ['/dev/null', qr/not a block device/]) { + $resolved = $case->[0]; + my $warnings = ''; + local $SIG{__WARN__} = sub { $warnings .= $_[0] }; + eval { + PVE::Storage::ISCSIPlugin->activate_volume( + 'san', + { target => 'iqn.test', 'managed-multipath' => $multipath }, + '0.0.1.scsi-legacy', + ); + }; + like($@, $case->[1], "plain name uses normal validation with multipath=$multipath"); + is($warnings, '', 'no compatibility warning'); + is($waits, 0, 'no wait for a plain name'); + } +} + +my $scfg = { target => 'iqn.test', 'managed-multipath' => 1 }; +$visible = []; +eval { PVE::Storage::ISCSIPlugin->activate_volume('san', $scfg, '0.0.1.dm-uuid-mpath-3600a') }; +like($@, qr/is not visible on this node/, 'no wait for a LUN without a path on the node'); +is($waits, 0, 'absent LUN fails at once'); + +$visible = ['sda', 'sdb']; +eval { PVE::Storage::ISCSIPlugin->activate_volume('san', $scfg, '0.0.1.dm-uuid-mpath-3600a') }; +like($@, qr/did not assemble in time/, 'map-named timeout is fatal'); +is($waits, 1, 'map-named volume waits once'); + +$ready = 1; +$resolved = undef; +eval { PVE::Storage::ISCSIPlugin->activate_volume('san', $scfg, '0.0.1.dm-uuid-mpath-3600a') }; +like($@, qr/failed to get realpath/, 'map link is resolved after waiting'); +is($waits, 2, 'one additional wait'); + +SKIP: { + my ($blockdev) = grep { -b $_ } map { s!^/sys/block/!/dev/!r } glob('/sys/block/*'); + skip 'block device metadata required for successful activation', 3 if !$blockdev; + $resolved = $blockdev; + my $before = $waits; + eval { PVE::Storage::ISCSIPlugin->activate_volume('san', $scfg, '0.0.1.scsi-legacy') }; + is($@, '', 'plain name activates through the normal target check'); + is($waits, $before, 'successful plain activation also has no map wait'); +} + +done_testing(); diff --git a/src/test/run_lun_identity_tests.pl b/src/test/run_lun_identity_tests.pl new file mode 100755 index 00000000..91a4fac7 --- /dev/null +++ b/src/test/run_lun_identity_tests.pl @@ -0,0 +1,48 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use lib qw(..); +use Test::More; +use Test::MockModule; +use PVE::Multipath::Identity; + +my $mp = Test::MockModule->new('PVE::Multipath'); +my $id = Test::MockModule->new('PVE::Multipath::Identity'); +my $topology = Test::MockModule->new('PVE::ISCSI::Topology'); +$topology->mock(sessions => sub { {} }); +$id->mock( + dir_glob_foreach => sub { $_[2]->($_) for qw(sda sdb sdc) }, + file_read_firstline => sub { '8:0' }, + file_get_contents => sub { "E:ID_SERIAL=3600udev\n" }, +); +my $answer = { sda => { wwid => '3600daemon' }, sdb => { wwid => '[undef]' } }; +$mp->mock(query_path_identities => sub { return $answer if $answer; die "unavailable\n" }); +my $ids = PVE::Multipath::Identity::collect(all => 1); +is( + PVE::Multipath::Identity::path_wwid($ids, 'sda'), + '3600daemon', + 'daemon identity is authoritative', +); +ok(!defined(PVE::Multipath::Identity::path_wwid($ids, 'sdb')), 'unknown identity skipped'); +is( + PVE::Multipath::Identity::path_wwid($ids, 'sdc'), + '3600udev', + 'udev identity for a path the daemon does not list', +); +$answer = undef; +$ids = PVE::Multipath::Identity::collect(all => 1); +is( + PVE::Multipath::Identity::path_wwid($ids, 'sda'), + '3600udev', + 'udev fallback when daemon cannot answer', +); +is_deeply( + PVE::Multipath::parse_path_identities("sda|3600one\nsdb|[undef]\n"), + { sda => { wwid => '3600one' } }, + 'path-list parser skips unknown identities', +); +is_deeply(PVE::Multipath::parse_path_identities('ok'), {}, 'empty daemon reply'); +ok(!defined(PVE::Multipath::parse_path_identities('timeout')), 'invalid daemon reply'); + +done_testing(); diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl index db356cde..ae44e962 100755 --- a/src/test/run_multipath_tests.pl +++ b/src/test/run_multipath_tests.pl @@ -3,6 +3,7 @@ use v5.36; use lib ('.', '..'); use Test::More; +use Test::MockModule; use PVE::Multipath; use PVE::Multipath::Config; use PVE::Tools qw(file_get_contents); @@ -142,4 +143,15 @@ subtest 'one owner for map identity and membership' => sub { ); }; +subtest 'short commands and link-only assembly wait' => sub { + my $mp = Test::MockModule->new('PVE::Multipath'); + my @calls; + $mp->mock(run_command => sub($cmd, %opts) { push @calls, [$cmd, $opts{timeout}] }); + ok( + !PVE::Multipath::wait_for_map('nonexistent-test-lun', 0), + 'missing by-id block link times out', + ); + is_deeply(\@calls, [], 'waiting never queries the daemon'); +}; + done_testing();