From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v7 storage 09/43] multipath: add a cluster-wide configuration
Date: Wed, 18 Nov 2026 21:25:52 +0100 [thread overview]
Message-ID: <20261118202646.3017286-10-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261118202646.3017286-1-t.lamprecht@proxmox.com>
A multipath configuration kept per node can drift, so that some nodes
use a LUN through its map and others through a single path. Keep it
in the cluster file system instead: /etc/pve/multipath.cfg has one
entry per LUN WWID with an optional map alias, node restriction,
no_path_retry setting and expected number of paths, and
/etc/pve/multipath-overrides.conf takes multipath.conf sections for
the settings an array needs.
The expected number of paths is for LUNs without a storage that knows
it, such as Fibre Channel ones: a node with fewer paths than expected
is degraded, even after the kernel has removed a lost path.
Nothing applies the configuration on the nodes yet.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/Makefile | 1 +
src/PVE/Multipath/Config.pm | 182 ++++++++++++++++++++++++++++++
src/test/Makefile | 5 +-
src/test/run_multipath_tests.pl | 45 ++++++++
src/test/run_san_library_tests.pl | 47 ++++++++
5 files changed, 279 insertions(+), 1 deletion(-)
create mode 100644 src/PVE/Multipath/Config.pm
create mode 100755 src/test/run_san_library_tests.pl
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index db1c340d..597ad7a1 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -6,6 +6,7 @@ install:
install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
+ install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath/Config.pm b/src/PVE/Multipath/Config.pm
new file mode 100644
index 00000000..1111a8a7
--- /dev/null
+++ b/src/PVE/Multipath/Config.pm
@@ -0,0 +1,182 @@
+package PVE::Multipath::Config;
+
+use v5.36;
+
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::SectionConfig;
+use PVE::Tools;
+
+use base qw(PVE::SectionConfig);
+
+# multipath interprets a leading '-' as an option and major:minor as a device number.
+our $WWID_PATTERN = '(?!-|[0-9]+:-?[0-9])[a-zA-Z0-9._:-]+';
+our $WWID_RE = qr/^(?:$WWID_PATTERN)\z/;
+
+my $properties = {
+ alias => {
+ type => 'string',
+ pattern => '[a-zA-Z0-9][a-zA-Z0-9._-]*',
+ maxLength => 64,
+ description => 'The name of the multipath map.',
+ optional => 1,
+ },
+ nodes => get_standard_option(
+ 'pve-node-list',
+ {
+ description => 'The nodes that should assemble the map, or all nodes when unset.',
+ optional => 1,
+ },
+ ),
+ 'no-path-retry' => {
+ type => 'string',
+ pattern => '(?:queue|fail|[0-9]+)',
+ description => 'The action or number of retries when all paths are down.',
+ optional => 1,
+ },
+ 'paths-expected' => {
+ type => 'integer',
+ minimum => 1,
+ description => 'The number of paths each node should have to the LUN. A node with fewer'
+ . ' paths reports the LUN as degraded.',
+ optional => 1,
+ },
+};
+my $defaultData = {
+ propertyList => {
+ type => { description => 'The section type.' },
+ id => {
+ type => 'string',
+ pattern => $WWID_PATTERN,
+ maxLength => 128,
+ description => 'The LUN WWID.',
+ },
+ %$properties,
+ },
+};
+sub private($class) { return $defaultData }
+sub type($class) { return 'wwid' }
+
+sub options($class) {
+ return { map { $_ => { optional => 1 } } keys %$properties };
+}
+
+sub wwid_api_schema() {
+ return { map { $_ => { $properties->{$_}->%* } } keys %$properties };
+}
+
+sub parse_section_header($class, $line) {
+ my ($type, $id, $errmsg, $config) = $class->SUPER::parse_section_header($line);
+ return undef if !defined($type);
+ $errmsg //= "invalid WWID '$id'" if $id !~ $WWID_RE;
+ return ($type, $id, $errmsg, $config);
+}
+
+sub decode_value($class, $type, $key, $value) {
+ if ($key eq 'nodes') {
+ my @nodes = PVE::Tools::split_list($value);
+ return { map { $_ => 1 } grep { PVE::JSONSchema::pve_verify_node_name($_) } @nodes };
+ }
+ return $value;
+}
+
+sub encode_value($class, $type, $key, $value) {
+ return join(',', sort keys %$value) if $key eq 'nodes';
+ return $value;
+}
+__PACKAGE__->register();
+__PACKAGE__->init();
+
+sub check_overrides($text) {
+ my $allowed = { map { $_ => 1 } qw(devices overrides defaults blacklist blacklist_exceptions) };
+ my $depth = 0;
+ for my $line (split(/\n/, $text // '')) {
+ $line =~ s/"[^"]*"//g;
+ $line =~ s/[#!].*//;
+ while ($line =~ /(\w+)\s*\{|([{}])/g) {
+ if (defined($1)) {
+ my $keyword = $1;
+ die "multipath overrides: configure LUNs in multipath.cfg, not a multipaths block\n"
+ if $keyword eq 'multipaths';
+ die "multipath overrides: unknown section '$keyword'\n"
+ if !$depth && !$allowed->{$keyword};
+ $depth++;
+ } elsif ($2 eq '{') {
+ $depth++;
+ } else {
+ die "multipath overrides: unbalanced braces\n" if !$depth;
+ $depth--;
+ }
+ }
+ # multipath wants '{' to end a line and '}' alone on one, it ignores text after a '{'
+ die "multipath overrides: '{' has to end its line, quote a value with braces\n"
+ if $line =~ /\{\s*\S/;
+ die "multipath overrides: '}' has to stand alone on its line\n"
+ if $line =~ /\}/ && $line !~ /^\s*\}\s*$/;
+ }
+ die "multipath overrides: unbalanced braces\n" if $depth;
+}
+
+sub format_overrides($text) {
+ $text //= '';
+ $text =~ s/\s+$//;
+ return length($text) ? "$text\n" : '';
+}
+
+cfs_register_file(
+ 'multipath.cfg',
+ sub($filename, $data) { __PACKAGE__->parse_config($filename, $data) },
+ sub($filename, $data) { __PACKAGE__->write_config($filename, $data) },
+);
+cfs_register_file(
+ 'multipath-overrides.conf',
+ sub($filename, $data) { $data // '' },
+ sub($filename, $data) { format_overrides($data) },
+);
+
+sub read_config() { return cfs_read_file('multipath.cfg') }
+sub write_cluster_config($cfg) { cfs_write_file('multipath.cfg', $cfg) }
+sub read_overrides() { return cfs_read_file('multipath-overrides.conf') }
+sub write_overrides($text) { cfs_write_file('multipath-overrides.conf', $text) }
+
+sub lock_config($code, $errmsg = undef) {
+ cfs_lock_file('multipath.cfg', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub lock_overrides($code, $errmsg = undef) {
+ cfs_lock_file('multipath-overrides.conf', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub generate_managed_conf() {
+ return "# Managed by Proxmox VE.\ndefaults {\n\tauto_resize grow_only\n}\n";
+}
+
+sub parse_wwids($text) {
+ return [map { m{^/($WWID_PATTERN)/\s*$} ? $1 : () } split(/\n/, $text // '')];
+}
+
+sub wwid_list($cfg) {
+ return [sort keys $cfg->{ids}->%*];
+}
+
+sub wwids_for_node($cfg, $node) {
+ return [grep { my $nodes = $cfg->{ids}->{$_}->{nodes}; !$nodes || !%$nodes || $nodes->{$node} }
+ wwid_list($cfg)->@*];
+}
+
+sub build_multipaths_block($cfg, $node) {
+ my $out = "# Managed by Proxmox VE.\nmultipaths {\n";
+ for my $wwid (wwids_for_node($cfg, $node)->@*) {
+ my $entry = $cfg->{ids}->{$wwid};
+ $out .= "\tmultipath {\n\t\twwid $wwid\n";
+ $out .= "\t\talias $entry->{alias}\n" if defined($entry->{alias});
+ $out .= "\t\tno_path_retry $entry->{'no-path-retry'}\n"
+ if defined($entry->{'no-path-retry'});
+ $out .= "\t}\n";
+ }
+ return "$out}\n";
+}
+
+1;
diff --git a/src/test/Makefile b/src/test/Makefile
index 3f388a07..a267c7ba 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
all: test
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_san_libraries
test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -55,3 +55,6 @@ test_import_stream: run_import_stream_tests.pl
test_multipath: run_multipath_tests.pl
./run_multipath_tests.pl
+
+test_san_libraries: run_san_library_tests.pl
+ ./run_san_library_tests.pl
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
index b1248ad0..4a499ab5 100755
--- a/src/test/run_multipath_tests.pl
+++ b/src/test/run_multipath_tests.pl
@@ -4,6 +4,7 @@ use v5.36;
use lib ('.', '..');
use Test::More;
use PVE::Multipath;
+use PVE::Multipath::Config;
use PVE::Tools qw(file_get_contents);
subtest 'multipathd JSON fixtures' => sub {
@@ -87,6 +88,50 @@ subtest 'multipathd JSON fixtures' => sub {
is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed');
};
+subtest 'one section type and fixed baseline' => sub {
+ my $cfg = PVE::Multipath::Config->parse_config(
+ 'multipath.cfg',
+ "wwid: 3600a\n\talias data\n\tnodes n1\n\tno-path-retry fail\n\tpaths-expected 2\n",
+ );
+ is_deeply(PVE::Multipath::Config::wwid_list($cfg), ['3600a'], 'WWID is listed');
+ is_deeply(PVE::Multipath::Config::wwids_for_node($cfg, 'n2'), [], 'node restriction');
+ is_deeply(
+ [sort keys PVE::Multipath::Config::wwid_api_schema()->%*],
+ [qw(alias no-path-retry nodes paths-expected)],
+ 'only four options',
+ );
+ like(
+ PVE::Multipath::Config::generate_managed_conf(),
+ qr/auto_resize grow_only/,
+ 'fixed baseline',
+ );
+ like(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/no_path_retry fail/,
+ 'per-LUN option',
+ );
+ unlike(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/expected/,
+ 'the expected path count is not a multipath option',
+ );
+ is_deeply(
+ PVE::Multipath::Config::parse_wwids("# header\n/3600a/\n/-bad/\n"),
+ ['3600a'],
+ 'allow-list parser',
+ );
+ eval { PVE::Multipath::Config::check_overrides('multipaths { }') };
+ like($@, qr/configure LUNs/, 'LUN configuration belongs in explicit entries');
+ eval {
+ PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no\n}\n");
+ };
+ is($@, '', 'raw defaults override is allowed');
+ eval { PVE::Multipath::Config::check_overrides('defaults { user_friendly_names no }') };
+ like($@, qr/'\{' has to end its line/, 'but not on one line, multipath ignores the rest');
+ eval { PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no }\n") };
+ like($@, qr/'\}' has to stand alone on its line/, 'nor with a closing brace after a value');
+};
+
subtest 'one owner for map identity and membership' => sub {
is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map');
is_deeply(
diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl
new file mode 100755
index 00000000..819b9fc3
--- /dev/null
+++ b/src/test/run_san_library_tests.pl
@@ -0,0 +1,47 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use Test::MockModule;
+use Test::More;
+
+use lib ('.', '..');
+use PVE::Cluster;
+
+my %registered;
+my $cluster = Test::MockModule->new('PVE::Cluster');
+$cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] });
+require PVE::Multipath::Config;
+
+subtest 'CFS callbacks preserve filenames, data and method receivers' => sub {
+ my ($parse, $write) = $registered{'multipath.cfg'}->@*;
+ my $cfg = $parse->('multipath.cfg', undef);
+ is_deeply($cfg->{ids}, {}, 'an absent configuration parses as empty');
+ $cfg = $parse->('multipath.cfg', "wwid: 3600abc\n\talias san-lun\n\tnodes n2,n1\n");
+ is_deeply($cfg->{ids}->{'3600abc'}->{nodes}, { n1 => 1, n2 => 1 }, 'nodes are decoded');
+ my $again = $parse->('multipath.cfg', $write->('multipath.cfg', $cfg));
+ is_deeply($again->{ids}, $cfg->{ids}, 'sections survive the registered writer and parser');
+ ($parse, $write) = $registered{'multipath-overrides.conf'}->@*;
+ is($parse->('multipath-overrides.conf', undef), '', 'absent overrides parse as empty');
+ is($write->('multipath-overrides.conf', undef), '', 'undefined overrides write as empty');
+ is($write->('multipath-overrides.conf', "devices { } \n"), "devices { }\n",
+ 'writer trims');
+};
+
+subtest 'omitted optional parameters retain explicit undef behavior' => sub {
+ my $config = Test::MockModule->new('PVE::Multipath::Config');
+ $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" });
+ for my $lock (
+ \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides,
+ ) {
+ for my $args ([], [undef]) {
+ eval {
+ $lock->(sub { die "unexpected callback\n" }, @$args);
+ };
+ is($@, "lock unavailable\n", 'an omitted error prefix leaves the error intact');
+ }
+ }
+};
+
+done_testing();
next prev parent reply other threads:[~2026-11-18 20:28 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 20:25 [PATCH v7 cluster,storage,docs,manager 00/43] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 cluster 01/43] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 02/43] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 03/43] diskmanage: qualify NVMe over Fabrics transport Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 04/43] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 05/43] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 06/43] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 07/43] multipath: add a library around multipathd Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 08/43] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-18 20:25 ` Thomas Lamprecht [this message]
2026-11-18 20:25 ` [PATCH v7 storage 10/43] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 11/43] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 12/43] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 13/43] multipath: publish per-node status Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 14/43] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 15/43] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 16/43] diskmanage: look up the requested disks by name Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 17/43] diskmanage: read the udev data of all disks with one call Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 18/43] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 19/43] pvesm: expose the " Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 20/43] api: add node-level iSCSI status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 21/43] api: scan: name the portal of an iSCSI path Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 22/43] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 23/43] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 24/43] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 25/43] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 26/43] api: scan: report the process that has a SAN LUN open Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 docs 27/43] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 docs 28/43] storage: multipath: describe the use shown for an open LUN Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 29/43] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 30/43] report: add the managed multipath configuration Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 31/43] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 32/43] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 33/43] ui: dc: add a SAN view Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 34/43] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 35/43] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 36/43] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 37/43] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 38/43] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 39/43] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 40/43] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 41/43] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 42/43] ui: san: show the guest or process that has a LUN open Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 43/43] ui: node: san: add a column for the target or subsystem of a LUN Thomas Lamprecht
2026-11-19 19:49 ` superseded: [PATCH v7 cluster,storage,docs,manager 00/43] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261118202646.3017286-10-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox