From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id CAB141FF0DA for ; Thu, 19 Nov 2026 20:49:43 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 24DC621694; Thu, 19 Nov 2026 20:48:25 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v8 storage 09/43] multipath: add a cluster-wide configuration Date: Thu, 19 Nov 2026 20:46:14 +0100 Message-ID: <20261119194655.3163258-10-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261119194655.3163258-1-t.lamprecht@proxmox.com> References: <20261119194655.3163258-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795117621569 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.153 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 6IHTW2WBGFQD2V4QH4GIVUHW7DJSI7FU X-Message-ID-Hash: 6IHTW2WBGFQD2V4QH4GIVUHW7DJSI7FU X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A multipath configuration kept per node can drift, so that some nodes use a LUN through its map and others through a single path. Keep it in the cluster file system instead: /etc/pve/multipath.cfg has one entry per LUN WWID with an optional map alias, node restriction, no_path_retry setting and expected number of paths, and /etc/pve/multipath-overrides.conf takes multipath.conf sections for the settings an array needs. The expected number of paths is for LUNs without a storage that knows it, such as Fibre Channel ones: a node with fewer paths than expected is degraded, even after the kernel has removed a lost path. Nothing applies the configuration on the nodes yet. Signed-off-by: Thomas Lamprecht --- src/PVE/Makefile | 1 + src/PVE/Multipath/Config.pm | 188 ++++++++++++++++++++++++++++++ src/test/Makefile | 5 +- src/test/run_multipath_tests.pl | 64 ++++++++++ src/test/run_san_library_tests.pl | 47 ++++++++ 5 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 src/PVE/Multipath/Config.pm create mode 100755 src/test/run_san_library_tests.pl diff --git a/src/PVE/Makefile b/src/PVE/Makefile index db1c340d..597ad7a1 100644 --- a/src/PVE/Makefile +++ b/src/PVE/Makefile @@ -6,6 +6,7 @@ install: install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm + install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm make -C Storage install diff --git a/src/PVE/Multipath/Config.pm b/src/PVE/Multipath/Config.pm new file mode 100644 index 00000000..ae0ab81b --- /dev/null +++ b/src/PVE/Multipath/Config.pm @@ -0,0 +1,188 @@ +package PVE::Multipath::Config; + +use v5.36; + +use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file); +use PVE::JSONSchema qw(get_standard_option); +use PVE::SectionConfig; +use PVE::Tools; + +use base qw(PVE::SectionConfig); + +# multipath interprets a leading '-' as an option and major:minor as a device number. +our $WWID_PATTERN = '(?!-|[0-9]+:-?[0-9])[a-zA-Z0-9._:-]+'; +our $WWID_RE = qr/^(?:$WWID_PATTERN)\z/; + +my $properties = { + alias => { + type => 'string', + pattern => '[a-zA-Z0-9][a-zA-Z0-9._-]*', + maxLength => 64, + description => 'The name of the multipath map.', + optional => 1, + }, + nodes => get_standard_option( + 'pve-node-list', + { + description => 'The nodes that should assemble the map, or all nodes when unset.', + optional => 1, + }, + ), + 'no-path-retry' => { + type => 'string', + pattern => '(?:queue|fail|[0-9]+)', + description => 'The action or number of retries when all paths are down.', + optional => 1, + }, + 'paths-expected' => { + type => 'integer', + minimum => 1, + description => 'The number of paths each node should have to the LUN. A node with fewer' + . ' paths reports the LUN as degraded.', + optional => 1, + }, +}; +my $defaultData = { + propertyList => { + type => { description => 'The section type.' }, + id => { + type => 'string', + pattern => $WWID_PATTERN, + maxLength => 128, + description => 'The LUN WWID.', + }, + %$properties, + }, +}; +sub private($class) { return $defaultData } +sub type($class) { return 'wwid' } + +sub options($class) { + return { map { $_ => { optional => 1 } } keys %$properties }; +} + +sub wwid_api_schema() { + return { map { $_ => { $properties->{$_}->%* } } keys %$properties }; +} + +sub parse_section_header($class, $line) { + my ($type, $id, $errmsg, $config) = $class->SUPER::parse_section_header($line); + return undef if !defined($type); + $errmsg //= "invalid WWID '$id'" if $id !~ $WWID_RE; + return ($type, $id, $errmsg, $config); +} + +sub decode_value($class, $type, $key, $value) { + if ($key eq 'nodes') { + my @nodes = PVE::Tools::split_list($value); + return { map { $_ => 1 } grep { PVE::JSONSchema::pve_verify_node_name($_) } @nodes }; + } + return $value; +} + +sub encode_value($class, $type, $key, $value) { + return join(',', sort keys %$value) if $key eq 'nodes'; + return $value; +} +__PACKAGE__->register(); +__PACKAGE__->init(); + +sub check_overrides($text) { + my $allowed = { map { $_ => 1 } qw(devices overrides defaults blacklist blacklist_exceptions) }; + my $depth = 0; + for my $line (split(/\n/, $text // '')) { + # multipath ends a word at a quoted string and at a character outside of ASCII + $line =~ s/"[^"]*"/ /g; + $line =~ s/[^\x00-\x7f]/ /g; + $line =~ s/[#!].*//; + # multipath takes the first word of a line as section name, also without a '{' after it + die "multipath overrides: configure LUNs in multipath.cfg, not a multipaths block\n" + if $line =~ /^\s*multipaths\b/; + # multipath wants '{' to end a line and '}' alone on one, it ignores text after a '{' + die "multipath overrides: '{' has to end its line - quote a value that contains braces\n" + if $line =~ /\{\s*\S/; + die "multipath overrides: '}' has to stand alone on its line\n" + if $line =~ /\}/ && $line !~ /^\s*\}\s*$/; + while ($line =~ /(\w+)\s*\{|([{}])/g) { + if (defined($1)) { + my $keyword = $1; + # at any depth, as multipath can nest the sections differently than counted here + die "multipath overrides: configure LUNs in multipath.cfg, not a multipaths block\n" + if $keyword eq 'multipaths'; + die "multipath overrides: unknown section '$keyword'\n" + if !$depth && !$allowed->{$keyword}; + $depth++; + } elsif ($2 eq '{') { + die "multipath overrides: '{' has to follow its section name on the same line\n"; + } else { + die "multipath overrides: unbalanced braces\n" if !$depth; + $depth--; + } + } + } + die "multipath overrides: unbalanced braces\n" if $depth; +} + +sub format_overrides($text) { + $text //= ''; + $text =~ s/\s+$//; + return length($text) ? "$text\n" : ''; +} + +cfs_register_file( + 'multipath.cfg', + sub($filename, $data) { __PACKAGE__->parse_config($filename, $data) }, + sub($filename, $data) { __PACKAGE__->write_config($filename, $data) }, +); +cfs_register_file( + 'multipath-overrides.conf', + sub($filename, $data) { $data // '' }, + sub($filename, $data) { format_overrides($data) }, +); + +sub read_config() { return cfs_read_file('multipath.cfg') } +sub write_cluster_config($cfg) { cfs_write_file('multipath.cfg', $cfg) } +sub read_overrides() { return cfs_read_file('multipath-overrides.conf') } +sub write_overrides($text) { cfs_write_file('multipath-overrides.conf', $text) } + +sub lock_config($code, $errmsg = undef) { + cfs_lock_file('multipath.cfg', undef, $code); + die defined($errmsg) ? "$errmsg - $@" : $@ if $@; +} + +sub lock_overrides($code, $errmsg = undef) { + cfs_lock_file('multipath-overrides.conf', undef, $code); + die defined($errmsg) ? "$errmsg - $@" : $@ if $@; +} + +sub generate_managed_conf() { + return "# Managed by Proxmox VE.\ndefaults {\n\tauto_resize grow_only\n}\n"; +} + +sub parse_wwids($text) { + return [map { m{^/($WWID_PATTERN)/\s*$} ? $1 : () } split(/\n/, $text // '')]; +} + +sub wwid_list($cfg) { + return [sort keys $cfg->{ids}->%*]; +} + +sub wwids_for_node($cfg, $node) { + return [grep { my $nodes = $cfg->{ids}->{$_}->{nodes}; !$nodes || !%$nodes || $nodes->{$node} } + wwid_list($cfg)->@*]; +} + +sub build_multipaths_block($cfg, $node) { + my $out = "# Managed by Proxmox VE.\nmultipaths {\n"; + for my $wwid (wwids_for_node($cfg, $node)->@*) { + my $entry = $cfg->{ids}->{$wwid}; + $out .= "\tmultipath {\n\t\twwid $wwid\n"; + $out .= "\t\talias $entry->{alias}\n" if defined($entry->{alias}); + $out .= "\t\tno_path_retry $entry->{'no-path-retry'}\n" + if defined($entry->{'no-path-retry'}); + $out .= "\t}\n"; + } + return "$out}\n"; +} + +1; diff --git a/src/test/Makefile b/src/test/Makefile index 3f388a07..a267c7ba 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_san_libraries test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -55,3 +55,6 @@ test_import_stream: run_import_stream_tests.pl test_multipath: run_multipath_tests.pl ./run_multipath_tests.pl + +test_san_libraries: run_san_library_tests.pl + ./run_san_library_tests.pl diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl index 6a6142f2..62acc0f4 100755 --- a/src/test/run_multipath_tests.pl +++ b/src/test/run_multipath_tests.pl @@ -4,6 +4,7 @@ use v5.36; use lib ('.', '..'); use Test::More; use PVE::Multipath; +use PVE::Multipath::Config; use PVE::Tools qw(file_get_contents); subtest 'multipathd JSON fixtures' => sub { @@ -87,6 +88,69 @@ subtest 'multipathd JSON fixtures' => sub { is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed'); }; +subtest 'WWID entries, generated configuration and overrides check' => sub { + my $cfg = PVE::Multipath::Config->parse_config( + 'multipath.cfg', + "wwid: 3600a\n\talias data\n\tnodes n1\n\tno-path-retry fail\n\tpaths-expected 2\n", + ); + is_deeply(PVE::Multipath::Config::wwid_list($cfg), ['3600a'], 'WWID is listed'); + is_deeply(PVE::Multipath::Config::wwids_for_node($cfg, 'n2'), [], 'node restriction'); + is_deeply( + [sort keys PVE::Multipath::Config::wwid_api_schema()->%*], + [qw(alias no-path-retry nodes paths-expected)], + 'only four options', + ); + like( + PVE::Multipath::Config::generate_managed_conf(), + qr/auto_resize grow_only/, + 'generated defaults set auto_resize', + ); + like( + PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'), + qr/no_path_retry fail/, + 'per-LUN option', + ); + unlike( + PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'), + qr/expected/, + 'the expected path count is not a multipath option', + ); + is_deeply( + PVE::Multipath::Config::parse_wwids("# header\n/3600a/\n/-bad/\n"), + ['3600a'], + 'allow-list parser', + ); + eval { PVE::Multipath::Config::check_overrides('multipaths { }') }; + like($@, qr/configure LUNs/, 'LUN configuration belongs in explicit entries'); + eval { PVE::Multipath::Config::check_overrides("multipaths\ndefaults {\n}\n") }; + like($@, qr/configure LUNs/, 'also as a bare section name, which multipath still enters'); + # multipath reads the byte as space and closes 'devices' with the '}' of an unknown section + my $nested = "devices {\n\tbogus {\n\t}\n\x80multipaths {\n\t}\n}\n"; + eval { PVE::Multipath::Config::check_overrides($nested) }; + like($@, qr/configure LUNs/, 'and at any depth'); + eval { PVE::Multipath::Config::check_overrides("\x80multipaths\ndefaults {\n}\n") }; + like($@, qr/configure LUNs/, 'also behind a byte outside of ASCII'); + $nested =~ s/\x80/\x{e9}/; + eval { PVE::Multipath::Config::check_overrides($nested) }; + like($@, qr/configure LUNs/, 'or a letter outside of ASCII'); + eval { PVE::Multipath::Config::check_overrides("multipaths\"x\"y\ndefaults {\n}\n") }; + like($@, qr/configure LUNs/, 'or with a quoted string glued to it'); + eval { PVE::Multipath::Config::check_overrides("devices\n{\n}\n") }; + like($@, qr/'\{' has to follow its section name/, 'nor a brace on a line of its own'); + eval { + PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no\n}\n"); + }; + is($@, '', 'raw defaults override is allowed'); + eval { PVE::Multipath::Config::check_overrides('defaults { user_friendly_names no }') }; + like($@, qr/'\{' has to end its line/, 'but not on one line, multipath ignores the rest'); + eval { PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no }\n") }; + like($@, qr/'\}' has to stand alone on its line/, 'nor with a closing brace after a value'); + eval { + PVE::Multipath::Config::check_overrides("blacklist {\n\tdevnode ^nvme[0-9]{1,3}\n}\n"); + }; + like($@, qr/quote a value that contains braces/, 'a value with braces needs quotes'); +}; + subtest 'map identity from the dm UUID and map members' => sub { is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map'); is_deeply( diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl new file mode 100755 index 00000000..eab6570f --- /dev/null +++ b/src/test/run_san_library_tests.pl @@ -0,0 +1,47 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Test::MockModule; +use Test::More; + +use lib ('.', '..'); +use PVE::Cluster; + +my %registered; +my $cluster = Test::MockModule->new('PVE::Cluster'); +$cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] }); +require PVE::Multipath::Config; + +subtest 'CFS callbacks preserve filenames, data and method receivers' => sub { + my ($parse, $write) = $registered{'multipath.cfg'}->@*; + my $cfg = $parse->('multipath.cfg', undef); + is_deeply($cfg->{ids}, {}, 'an absent configuration parses as empty'); + $cfg = $parse->('multipath.cfg', "wwid: 3600abc\n\talias san-lun\n\tnodes n2,n1\n"); + is_deeply($cfg->{ids}->{'3600abc'}->{nodes}, { n1 => 1, n2 => 1 }, 'nodes are decoded'); + my $again = $parse->('multipath.cfg', $write->('multipath.cfg', $cfg)); + is_deeply($again->{ids}, $cfg->{ids}, 'sections survive the registered writer and parser'); + ($parse, $write) = $registered{'multipath-overrides.conf'}->@*; + is($parse->('multipath-overrides.conf', undef), '', 'absent overrides parse as empty'); + is($write->('multipath-overrides.conf', undef), '', 'undefined overrides write as empty'); + is($write->('multipath-overrides.conf', "devices { } \n"), "devices { }\n", + 'writer trims'); +}; + +subtest 'omitted optional arguments behave like undef' => sub { + my $config = Test::MockModule->new('PVE::Multipath::Config'); + $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" }); + for my $lock ( + \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides, + ) { + for my $args ([], [undef]) { + eval { + $lock->(sub { die "unexpected callback\n" }, @$args); + }; + is($@, "lock unavailable\n", 'an omitted error prefix leaves the error intact'); + } + } +}; + +done_testing();