From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v7 storage 09/43] multipath: add a cluster-wide configuration
Date: Wed, 18 Nov 2026 21:25:52 +0100 [thread overview]
Message-ID: <20261118202646.3017286-10-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261118202646.3017286-1-t.lamprecht@proxmox.com>
A multipath configuration kept per node can drift, so that some nodes
use a LUN through its map and others through a single path. Keep it
in the cluster file system instead: /etc/pve/multipath.cfg has one
entry per LUN WWID with an optional map alias, node restriction,
no_path_retry setting and expected number of paths, and
/etc/pve/multipath-overrides.conf takes multipath.conf sections for
the settings an array needs.
The expected number of paths is for LUNs without a storage that knows
it, such as Fibre Channel ones: a node with fewer paths than expected
is degraded, even after the kernel has removed a lost path.
Nothing applies the configuration on the nodes yet.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/Makefile | 1 +
src/PVE/Multipath/Config.pm | 182 ++++++++++++++++++++++++++++++
src/test/Makefile | 5 +-
src/test/run_multipath_tests.pl | 45 ++++++++
src/test/run_san_library_tests.pl | 47 ++++++++
5 files changed, 279 insertions(+), 1 deletion(-)
create mode 100644 src/PVE/Multipath/Config.pm
create mode 100755 src/test/run_san_library_tests.pl
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index db1c340d..597ad7a1 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -6,6 +6,7 @@ install:
install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
+ install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath/Config.pm b/src/PVE/Multipath/Config.pm
new file mode 100644
index 00000000..1111a8a7
--- /dev/null
+++ b/src/PVE/Multipath/Config.pm
@@ -0,0 +1,182 @@
+package PVE::Multipath::Config;
+
+use v5.36;
+
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::SectionConfig;
+use PVE::Tools;
+
+use base qw(PVE::SectionConfig);
+
+# multipath interprets a leading '-' as an option and major:minor as a device number.
+our $WWID_PATTERN = '(?!-|[0-9]+:-?[0-9])[a-zA-Z0-9._:-]+';
+our $WWID_RE = qr/^(?:$WWID_PATTERN)\z/;
+
+my $properties = {
+ alias => {
+ type => 'string',
+ pattern => '[a-zA-Z0-9][a-zA-Z0-9._-]*',
+ maxLength => 64,
+ description => 'The name of the multipath map.',
+ optional => 1,
+ },
+ nodes => get_standard_option(
+ 'pve-node-list',
+ {
+ description => 'The nodes that should assemble the map, or all nodes when unset.',
+ optional => 1,
+ },
+ ),
+ 'no-path-retry' => {
+ type => 'string',
+ pattern => '(?:queue|fail|[0-9]+)',
+ description => 'The action or number of retries when all paths are down.',
+ optional => 1,
+ },
+ 'paths-expected' => {
+ type => 'integer',
+ minimum => 1,
+ description => 'The number of paths each node should have to the LUN. A node with fewer'
+ . ' paths reports the LUN as degraded.',
+ optional => 1,
+ },
+};
+my $defaultData = {
+ propertyList => {
+ type => { description => 'The section type.' },
+ id => {
+ type => 'string',
+ pattern => $WWID_PATTERN,
+ maxLength => 128,
+ description => 'The LUN WWID.',
+ },
+ %$properties,
+ },
+};
+sub private($class) { return $defaultData }
+sub type($class) { return 'wwid' }
+
+sub options($class) {
+ return { map { $_ => { optional => 1 } } keys %$properties };
+}
+
+sub wwid_api_schema() {
+ return { map { $_ => { $properties->{$_}->%* } } keys %$properties };
+}
+
+sub parse_section_header($class, $line) {
+ my ($type, $id, $errmsg, $config) = $class->SUPER::parse_section_header($line);
+ return undef if !defined($type);
+ $errmsg //= "invalid WWID '$id'" if $id !~ $WWID_RE;
+ return ($type, $id, $errmsg, $config);
+}
+
+sub decode_value($class, $type, $key, $value) {
+ if ($key eq 'nodes') {
+ my @nodes = PVE::Tools::split_list($value);
+ return { map { $_ => 1 } grep { PVE::JSONSchema::pve_verify_node_name($_) } @nodes };
+ }
+ return $value;
+}
+
+sub encode_value($class, $type, $key, $value) {
+ return join(',', sort keys %$value) if $key eq 'nodes';
+ return $value;
+}
+__PACKAGE__->register();
+__PACKAGE__->init();
+
+sub check_overrides($text) {
+ my $allowed = { map { $_ => 1 } qw(devices overrides defaults blacklist blacklist_exceptions) };
+ my $depth = 0;
+ for my $line (split(/\n/, $text // '')) {
+ $line =~ s/"[^"]*"//g;
+ $line =~ s/[#!].*//;
+ while ($line =~ /(\w+)\s*\{|([{}])/g) {
+ if (defined($1)) {
+ my $keyword = $1;
+ die "multipath overrides: configure LUNs in multipath.cfg, not a multipaths block\n"
+ if $keyword eq 'multipaths';
+ die "multipath overrides: unknown section '$keyword'\n"
+ if !$depth && !$allowed->{$keyword};
+ $depth++;
+ } elsif ($2 eq '{') {
+ $depth++;
+ } else {
+ die "multipath overrides: unbalanced braces\n" if !$depth;
+ $depth--;
+ }
+ }
+ # multipath wants '{' to end a line and '}' alone on one, it ignores text after a '{'
+ die "multipath overrides: '{' has to end its line, quote a value with braces\n"
+ if $line =~ /\{\s*\S/;
+ die "multipath overrides: '}' has to stand alone on its line\n"
+ if $line =~ /\}/ && $line !~ /^\s*\}\s*$/;
+ }
+ die "multipath overrides: unbalanced braces\n" if $depth;
+}
+
+sub format_overrides($text) {
+ $text //= '';
+ $text =~ s/\s+$//;
+ return length($text) ? "$text\n" : '';
+}
+
+cfs_register_file(
+ 'multipath.cfg',
+ sub($filename, $data) { __PACKAGE__->parse_config($filename, $data) },
+ sub($filename, $data) { __PACKAGE__->write_config($filename, $data) },
+);
+cfs_register_file(
+ 'multipath-overrides.conf',
+ sub($filename, $data) { $data // '' },
+ sub($filename, $data) { format_overrides($data) },
+);
+
+sub read_config() { return cfs_read_file('multipath.cfg') }
+sub write_cluster_config($cfg) { cfs_write_file('multipath.cfg', $cfg) }
+sub read_overrides() { return cfs_read_file('multipath-overrides.conf') }
+sub write_overrides($text) { cfs_write_file('multipath-overrides.conf', $text) }
+
+sub lock_config($code, $errmsg = undef) {
+ cfs_lock_file('multipath.cfg', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub lock_overrides($code, $errmsg = undef) {
+ cfs_lock_file('multipath-overrides.conf', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub generate_managed_conf() {
+ return "# Managed by Proxmox VE.\ndefaults {\n\tauto_resize grow_only\n}\n";
+}
+
+sub parse_wwids($text) {
+ return [map { m{^/($WWID_PATTERN)/\s*$} ? $1 : () } split(/\n/, $text // '')];
+}
+
+sub wwid_list($cfg) {
+ return [sort keys $cfg->{ids}->%*];
+}
+
+sub wwids_for_node($cfg, $node) {
+ return [grep { my $nodes = $cfg->{ids}->{$_}->{nodes}; !$nodes || !%$nodes || $nodes->{$node} }
+ wwid_list($cfg)->@*];
+}
+
+sub build_multipaths_block($cfg, $node) {
+ my $out = "# Managed by Proxmox VE.\nmultipaths {\n";
+ for my $wwid (wwids_for_node($cfg, $node)->@*) {
+ my $entry = $cfg->{ids}->{$wwid};
+ $out .= "\tmultipath {\n\t\twwid $wwid\n";
+ $out .= "\t\talias $entry->{alias}\n" if defined($entry->{alias});
+ $out .= "\t\tno_path_retry $entry->{'no-path-retry'}\n"
+ if defined($entry->{'no-path-retry'});
+ $out .= "\t}\n";
+ }
+ return "$out}\n";
+}
+
+1;
diff --git a/src/test/Makefile b/src/test/Makefile
index 3f388a07..a267c7ba 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
all: test
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_multipath test_san_libraries
test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -55,3 +55,6 @@ test_import_stream: run_import_stream_tests.pl
test_multipath: run_multipath_tests.pl
./run_multipath_tests.pl
+
+test_san_libraries: run_san_library_tests.pl
+ ./run_san_library_tests.pl
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
index b1248ad0..4a499ab5 100755
--- a/src/test/run_multipath_tests.pl
+++ b/src/test/run_multipath_tests.pl
@@ -4,6 +4,7 @@ use v5.36;
use lib ('.', '..');
use Test::More;
use PVE::Multipath;
+use PVE::Multipath::Config;
use PVE::Tools qw(file_get_contents);
subtest 'multipathd JSON fixtures' => sub {
@@ -87,6 +88,50 @@ subtest 'multipathd JSON fixtures' => sub {
is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed');
};
+subtest 'one section type and fixed baseline' => sub {
+ my $cfg = PVE::Multipath::Config->parse_config(
+ 'multipath.cfg',
+ "wwid: 3600a\n\talias data\n\tnodes n1\n\tno-path-retry fail\n\tpaths-expected 2\n",
+ );
+ is_deeply(PVE::Multipath::Config::wwid_list($cfg), ['3600a'], 'WWID is listed');
+ is_deeply(PVE::Multipath::Config::wwids_for_node($cfg, 'n2'), [], 'node restriction');
+ is_deeply(
+ [sort keys PVE::Multipath::Config::wwid_api_schema()->%*],
+ [qw(alias no-path-retry nodes paths-expected)],
+ 'only four options',
+ );
+ like(
+ PVE::Multipath::Config::generate_managed_conf(),
+ qr/auto_resize grow_only/,
+ 'fixed baseline',
+ );
+ like(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/no_path_retry fail/,
+ 'per-LUN option',
+ );
+ unlike(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/expected/,
+ 'the expected path count is not a multipath option',
+ );
+ is_deeply(
+ PVE::Multipath::Config::parse_wwids("# header\n/3600a/\n/-bad/\n"),
+ ['3600a'],
+ 'allow-list parser',
+ );
+ eval { PVE::Multipath::Config::check_overrides('multipaths { }') };
+ like($@, qr/configure LUNs/, 'LUN configuration belongs in explicit entries');
+ eval {
+ PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no\n}\n");
+ };
+ is($@, '', 'raw defaults override is allowed');
+ eval { PVE::Multipath::Config::check_overrides('defaults { user_friendly_names no }') };
+ like($@, qr/'\{' has to end its line/, 'but not on one line, multipath ignores the rest');
+ eval { PVE::Multipath::Config::check_overrides("defaults {\n\tuser_friendly_names no }\n") };
+ like($@, qr/'\}' has to stand alone on its line/, 'nor with a closing brace after a value');
+};
+
subtest 'one owner for map identity and membership' => sub {
is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map');
is_deeply(
diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl
new file mode 100755
index 00000000..819b9fc3
--- /dev/null
+++ b/src/test/run_san_library_tests.pl
@@ -0,0 +1,47 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use Test::MockModule;
+use Test::More;
+
+use lib ('.', '..');
+use PVE::Cluster;
+
+my %registered;
+my $cluster = Test::MockModule->new('PVE::Cluster');
+$cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] });
+require PVE::Multipath::Config;
+
+subtest 'CFS callbacks preserve filenames, data and method receivers' => sub {
+ my ($parse, $write) = $registered{'multipath.cfg'}->@*;
+ my $cfg = $parse->('multipath.cfg', undef);
+ is_deeply($cfg->{ids}, {}, 'an absent configuration parses as empty');
+ $cfg = $parse->('multipath.cfg', "wwid: 3600abc\n\talias san-lun\n\tnodes n2,n1\n");
+ is_deeply($cfg->{ids}->{'3600abc'}->{nodes}, { n1 => 1, n2 => 1 }, 'nodes are decoded');
+ my $again = $parse->('multipath.cfg', $write->('multipath.cfg', $cfg));
+ is_deeply($again->{ids}, $cfg->{ids}, 'sections survive the registered writer and parser');
+ ($parse, $write) = $registered{'multipath-overrides.conf'}->@*;
+ is($parse->('multipath-overrides.conf', undef), '', 'absent overrides parse as empty');
+ is($write->('multipath-overrides.conf', undef), '', 'undefined overrides write as empty');
+ is($write->('multipath-overrides.conf', "devices { } \n"), "devices { }\n",
+ 'writer trims');
+};
+
+subtest 'omitted optional parameters retain explicit undef behavior' => sub {
+ my $config = Test::MockModule->new('PVE::Multipath::Config');
+ $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" });
+ for my $lock (
+ \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides,
+ ) {
+ for my $args ([], [undef]) {
+ eval {
+ $lock->(sub { die "unexpected callback\n" }, @$args);
+ };
+ is($@, "lock unavailable\n", 'an omitted error prefix leaves the error intact');
+ }
+ }
+};
+
+done_testing();
next prev parent reply other threads:[~2026-11-18 20:28 UTC|newest]
Thread overview: 45+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 20:25 [PATCH v7 cluster,storage,docs,manager 00/43] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 cluster 01/43] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 02/43] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 03/43] diskmanage: qualify NVMe over Fabrics transport Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 04/43] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 05/43] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 06/43] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 07/43] multipath: add a library around multipathd Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 08/43] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-18 20:25 ` Thomas Lamprecht [this message]
2026-11-18 20:25 ` [PATCH v7 storage 10/43] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 11/43] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 12/43] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 13/43] multipath: publish per-node status Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 14/43] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 15/43] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-18 20:25 ` [PATCH v7 storage 16/43] diskmanage: look up the requested disks by name Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 17/43] diskmanage: read the udev data of all disks with one call Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 18/43] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 19/43] pvesm: expose the " Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 20/43] api: add node-level iSCSI status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 21/43] api: scan: name the portal of an iSCSI path Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 22/43] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 23/43] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 24/43] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 25/43] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 storage 26/43] api: scan: report the process that has a SAN LUN open Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 docs 27/43] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 docs 28/43] storage: multipath: describe the use shown for an open LUN Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 29/43] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 30/43] report: add the managed multipath configuration Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 31/43] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 32/43] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 33/43] ui: dc: add a SAN view Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 34/43] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 35/43] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 36/43] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 37/43] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 38/43] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 39/43] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 40/43] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 41/43] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 42/43] ui: san: show the guest or process that has a LUN open Thomas Lamprecht
2026-11-18 20:26 ` [PATCH v7 manager 43/43] ui: node: san: add a column for the target or subsystem of a LUN Thomas Lamprecht
2026-11-19 19:49 ` superseded: [PATCH v7 cluster,storage,docs,manager 00/43] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261118202646.3017286-10-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.