From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id E8A4D1FF0DA for ; Thu, 19 Nov 2026 20:50:25 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 73E6721AFB; Thu, 19 Nov 2026 20:48:28 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v8 storage 07/43] multipath: add a library around multipathd Date: Thu, 19 Nov 2026 20:46:12 +0100 Message-ID: <20261119194655.3163258-8-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261119194655.3163258-1-t.lamprecht@proxmox.com> References: <20261119194655.3163258-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795117621400 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.154 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: BQWN6RYRMYKSN3B6XB76HANOQ6LQNEJ7 X-Message-ID-Hash: BQWN6RYRMYKSN3B6XB76HANOQ6LQNEJ7 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The SAN LUN inventory, the per-node status and the iSCSI plugin need the multipath maps of a node and the state of their paths. Add a module that reads the maps and their member devices from sysfs and the path states from multipathd, with a timeout for the queries. Maps are identified by the WWID of their LUN, which is the same on all nodes. The iSCSI plugin already uses the module here, to recognize a map. multipath-tools is only recommended, as storages that do not use multipath work without it. Signed-off-by: Thomas Lamprecht --- debian/control | 3 +- src/PVE/Makefile | 1 + src/PVE/Multipath.pm | 136 +++++++++++++++++++++++++++ src/PVE/Storage/ISCSIPlugin.pm | 3 +- src/test/Makefile | 5 +- src/test/multipath-three-states.json | 52 ++++++++++ src/test/run_multipath_tests.pl | 106 +++++++++++++++++++++ 7 files changed, 303 insertions(+), 3 deletions(-) create mode 100644 src/PVE/Multipath.pm create mode 100644 src/test/multipath-three-states.json create mode 100755 src/test/run_multipath_tests.pl diff --git a/debian/control b/debian/control index 4abf489c..d10dfb23 100644 --- a/debian/control +++ b/debian/control @@ -54,7 +54,8 @@ Depends: bzip2, zstd, ${misc:Depends}, ${perl:Depends}, -Recommends: pve-esxi-import-tools (>= 0.6.0), +Recommends: multipath-tools, + pve-esxi-import-tools (>= 0.6.0), skopeo, zfsutils-linux (>= 2.4.1-pve2), zfs-zed, diff --git a/src/PVE/Makefile b/src/PVE/Makefile index bd2ddfb9..db1c340d 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -5,6 +5,7 @@ install: install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm + install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm make -C Storage install diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm new file mode 100644 index 00000000..84c4901e --- /dev/null +++ b/src/PVE/Multipath.pm @@ -0,0 +1,136 @@ +package PVE::Multipath; + +use v5.36; + +use Encode qw(decode encode); +use JSON qw(decode_json); + +use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach); + +use PVE::Diskmanage; + +my $MULTIPATH = '/sbin/multipath'; +my $MULTIPATHD = '/sbin/multipathd'; + +sub parse_dm_uuid($uuid) { + my ($part, $wwid) = ($uuid // '') =~ /^(?:part([1-9][0-9]*)-)?mpath-(\S+)$/; + return undef if !defined($wwid); + return { wwid => $wwid, defined($part) ? (partition => int($part)) : () }; +} + +sub map_info() { + my $maps = {}; + dir_glob_foreach( + '/sys/block', + 'dm-\\d+', + sub($dev, @) { + my $info = parse_dm_uuid(file_read_firstline("/sys/block/$dev/dm/uuid")); + return if !$info || $info->{partition}; + my $name = file_read_firstline("/sys/block/$dev/dm/name") // return; + my @slaves; + dir_glob_foreach( + "/sys/block/$dev/slaves", + '[^.].*', + sub($slave, @) { push @slaves, $slave }, + ); + $maps->{$dev} = { %$info, name => $name, slaves => [sort @slaves] }; + }, + ); + return $maps; +} + +sub member_map($maps) { + my $members = {}; + for my $dev (sort keys %$maps) { + $members->{$_} = $dev for $maps->{$dev}->{slaves}->@*; + } + return $members; +} + +sub is_supported() { return -x $MULTIPATH && -x $MULTIPATHD ? 1 : 0 } +sub assert_supported() { die "multipath-tools is not installed\n" if !is_supported() } + +sub is_running() { + return 0 if !is_supported(); + my $running = 0; + eval { + run_command( + [$MULTIPATHD, 'show', 'daemon'], + timeout => 10, + outfunc => sub($line) { $running = 1 if $line =~ /^pid \d+ \S+/ }, + errfunc => sub($line) { }, + ); + }; + return $running; +} + +my sub string_value($value) { + return undef if !defined($value) || $value eq '' || $value eq '[undef]'; + die "invalid string in multipathd output\n" if ref($value); + return $value; +} + +sub path_is_usable($path) { + return ($path->{'dm-state'} // '') eq 'active' + && (!defined($path->{'check-state'}) || $path->{'check-state'} =~ /^(?:ready|ghost)$/); +} + +sub derive_health($active, $total, $expected = undef) { + return 'failed' if !$total || !$active; + return 'degraded' if $active < $total || $total < ($expected // 0); + return 'optimal'; +} + +sub parse_maps_json($json) { + $json = decode('UTF-8', $json) if !utf8::is_utf8($json); + my $data = decode_json(encode('UTF-8', $json)); + die "invalid multipathd maps output\n" + if ref($data) ne 'HASH' || ref($data->{maps} // []) ne 'ARRAY'; + my @maps; + for my $raw (($data->{maps} // [])->@*) { + die "invalid multipathd map\n" if ref($raw) ne 'HASH'; + my $map = { paths => [], 'paths-total' => 0, 'paths-active' => 0 }; + for my $pair ([uuid => 'wwid'], [name => 'name'], [sysfs => 'sysfs']) { + my $value = string_value($raw->{ $pair->[0] }); + $map->{ $pair->[1] } = $value if defined($value); + } + for my $group (($raw->{path_groups} // [])->@*) { + for my $rawpath (($group->{paths} // [])->@*) { + my $path = { dev => string_value($rawpath->{dev}) // '' }; + for my $pair ( + [dm_st => 'dm-state'], + [chk_st => 'check-state'], + [host_wwpn => 'host-wwpn'], + [target_wwpn => 'target-wwpn'], + ) { + my $value = string_value($rawpath->{ $pair->[0] }); + next if !defined($value) || ($pair->[0] =~ /wwpn$/ && $value eq '[unknown]'); + $path->{ $pair->[1] } = $value; + } + push $map->{paths}->@*, $path; + $map->{'paths-total'}++; + $map->{'paths-active'}++ if path_is_usable($path); + } + } + push @maps, $map; + } + return \@maps; +} + +sub get_maps() { + assert_supported(); + my $json = ''; + run_command( + [$MULTIPATHD, 'show', 'maps', 'json'], + timeout => 10, + outfunc => sub($line) { $json .= "$line\n" }, + ); + my $maps = parse_maps_json($json); + for my $map (@$maps) { + $map->{size} = PVE::Diskmanage::get_sysdir_size("/sys/block/$map->{sysfs}") + if $map->{sysfs}; + } + return $maps; +} + +1; diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm index cec58afb..1e1ee4cb 100644 --- a/src/PVE/Storage/ISCSIPlugin.pm +++ b/src/PVE/Storage/ISCSIPlugin.pm @@ -10,6 +10,7 @@ use IO::File; use PVE::ISCSI::Topology; use PVE::JSONSchema qw(get_standard_option); +use PVE::Multipath; use PVE::Storage::Plugin; use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach $IPV4RE $IPV6RE); @@ -588,7 +589,7 @@ $resolve_virtual_devices = sub { my $uuid = file_read_firstline("/sys/$dev/dm/uuid") // ''; my ($name) = $dev =~ m!([^/]+)$!; die "'$name' is neither a multipath map nor a partition of one\n" - if $uuid !~ m/^(?:part\d+-)?mpath-\S+$/; + if !PVE::Multipath::parse_dm_uuid($uuid); dir_glob_foreach( "/sys/$dev/slaves", diff --git a/src/test/Makefile b/src/test/Makefile index b2676762..3f388a07 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -52,3 +52,6 @@ test_sensitive_properties: run_sensitive_properties_tests.pl test_import_stream: run_import_stream_tests.pl ./run_import_stream_tests.pl + +test_multipath: run_multipath_tests.pl + ./run_multipath_tests.pl diff --git a/src/test/multipath-three-states.json b/src/test/multipath-three-states.json new file mode 100644 index 00000000..b39b123f --- /dev/null +++ b/src/test/multipath-three-states.json @@ -0,0 +1,52 @@ +{ + "major_version": 0, + "minor_version": 1, + "maps": [ + { + "name": "mpatha", + "uuid": "3600140500a1b2c3d4e5f6a7b8c9d0e1f", + "sysfs": "dm-0", + "dm_st": "active", + "paths": 2, + "path_faults": 0, + "path_groups": [{ + "group": 1, + "dm_st": "active", + "pri": 50, + "paths": [ + { "dev": "sdb", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1c", "target_wwpn": "0x500a098000000001" }, + { "dev": "sdc", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 50, "host_wwpn": "0x21000024ff7d6a1d", "target_wwpn": "0x500a098000000002" } + ] + }] + }, + { + "name": "mpathb", + "uuid": "360014050aabbccddeeff00112233445566", + "sysfs": "dm-1", + "dm_st": "active", + "paths": 2, + "path_faults": 3, + "path_groups": [ + { "group": 1, "dm_st": "active", "pri": 10, "paths": [ + { "dev": "sdd", "dm_st": "active", "dev_st": "running", "chk_st": "ready", "pri": 10 } + ] }, + { "group": 2, "dm_st": "enabled", "pri": 0, "paths": [ + { "dev": "sde", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0 } + ] } + ] + }, + { + "name": "mpathc", + "uuid": "36001405ffffffffffffffffffffffffff", + "sysfs": "dm-2", + "dm_st": "active", + "paths": 1, + "path_groups": [{ + "group": 1, + "dm_st": "enabled", + "pri": 0, + "paths": [{ "dev": "sdf", "dm_st": "failed", "dev_st": "offline", "chk_st": "faulty", "pri": 0, "target_wwpn": "[undef]", "host_adapter": "[undef]" }] + }] + } + ] +} diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl new file mode 100755 index 00000000..6a6142f2 --- /dev/null +++ b/src/test/run_multipath_tests.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +use v5.36; +use lib ('.', '..'); +use Test::More; +use PVE::Multipath; +use PVE::Tools qw(file_get_contents); + +subtest 'multipathd JSON fixtures' => sub { + my $maps = + PVE::Multipath::parse_maps_json(file_get_contents('multipath-three-states.json')); + is_deeply( + [map { [$_->{'paths-active'}, $_->{'paths-total'}] } @$maps], + [[2, 2], [1, 2], [0, 1]], + 'fixture path counts', + ); + is_deeply( + [ + map { PVE::Multipath::derive_health($_->{'paths-active'}, $_->{'paths-total'}) } + @$maps + ], + [qw(optimal degraded failed)], + 'health is derived by consumers', + ); + is_deeply( + [sort keys $maps->[0]->%*], + [qw(name paths paths-active paths-total sysfs wwid)], + 'only consumed map fields are parsed', + ); + is_deeply( + [sort keys $maps->[0]->{paths}->[0]->%*], + [qw(check-state dev dm-state host-wwpn target-wwpn)], + 'only consumed path fields are parsed', + ); + is($maps->[0]->{paths}->[0]->{'target-wwpn'}, '0x500a098000000001', 'FC target port'); + my $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [ + { "group": 1, "pri": 50, "paths": [ { "dev": "sda", "dm_st": "active", "chk_st": "ready" } ] }, + { "group": 2, "pri": 10, "paths": [ { "dev": "sdb", "dm_st": "active", "chk_st": "ghost" } ] } + ] } ] }'; + my $map = PVE::Multipath::parse_maps_json($json)->[0]; + is($map->{'paths-active'}, 2, 'ALUA ghost path is usable'); + is_deeply( + [map { $_->{dev} } $map->{paths}->@*], + ['sda', 'sdb'], + 'groups flatten into one path list', + ); + $json =~ s/"ghost"/"faulty"/; + is( + PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'}, + 1, + 'faulty path is unusable', + ); + $json =~ s/"ready"/"faulty"/; + is(PVE::Multipath::parse_maps_json($json)->[0]->{'paths-active'}, 0, 'no usable paths'); + $json = '{ "maps": [ { "uuid": "3600a", "path_faults": "7", "path_groups": [ + { "group": "1", "pri": "50", "paths": [ { "dev": "sda", "dm_st": "active", "pri": "50" } ] } + ] } ] }'; + $map = PVE::Multipath::parse_maps_json($json)->[0]; + is_deeply( + $map, + { + wwid => '3600a', + 'paths-active' => 1, + 'paths-total' => 1, + paths => [{ dev => 'sda', 'dm-state' => 'active' }], + }, + 'numeric strings in unused fields are ignored', + ); + $json = '{ "maps": [ { "uuid": "3600a", "path_groups": [ { "group": 1, "pri": 50, "paths": [ + { "dev": "sda", "dm_st": "active", "host_wwpn": "[unknown]", "target_wwpn": "[unknown]" } + ] } ] } ] }'; + my $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0]; + ok(!exists($path->{'host-wwpn'}), 'unknown WWPN is omitted'); + $json =~ s/\[unknown\]/[undef]/g; + $path = PVE::Multipath::parse_maps_json($json)->[0]->{paths}->[0]; + ok(!exists($path->{'target-wwpn'}), 'undefined WWPN is omitted'); + is( + PVE::Multipath::parse_maps_json("{\"maps\":[{\"name\":\"lun\xff\"}]}")->[0]->{name}, + "lun\x{fffd}", + 'invalid UTF-8 alias is replaced, not a query failure', + ); + is_deeply(PVE::Multipath::parse_maps_json('{}'), [], 'empty response'); + is_deeply(PVE::Multipath::parse_maps_json('{"maps":[]}'), [], 'empty list'); + eval { PVE::Multipath::parse_maps_json('not json') }; + ok($@, 'invalid JSON fails'); + is(PVE::Multipath::derive_health(1, 1, 2), 'degraded', 'removed expected path is degraded'); + is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed'); +}; + +subtest 'map identity from the dm UUID and map members' => sub { + is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map'); + is_deeply( + PVE::Multipath::parse_dm_uuid('part2-mpath-3600a'), + { wwid => '3600a', partition => 2 }, + 'partition of map', + ); + ok(!PVE::Multipath::parse_dm_uuid('LVM-abc'), 'LV is not a multipath map'); + ok(!PVE::Multipath::parse_dm_uuid(undef), 'no identity does not invent a map'); + is_deeply( + PVE::Multipath::member_map({ 'dm-0' => { slaves => ['sdb', 'sdc'] } }), + { sdb => 'dm-0', sdc => 'dm-0' }, + 'each member device maps to its map', + ); +}; + +done_testing();