From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v5 storage 09/36] multipath: add a cluster-wide configuration
Date: Mon, 16 Nov 2026 02:23:18 +0100 [thread overview]
Message-ID: <20261116012355.3187417-10-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com>
A multipath configuration kept per node can drift, so that some nodes
use a LUN through its map and others through a single path. Keep it
in the cluster file system instead: /etc/pve/multipath.cfg has one
entry per LUN WWID with an optional map alias, node restriction,
no_path_retry setting and expected number of paths, and
/etc/pve/multipath-overrides.conf takes multipath.conf sections for
the settings an array needs.
The expected number of paths is for LUNs without a storage that knows
it, such as Fibre Channel ones: a node with fewer paths than expected
is degraded, also after the kernel removed a lost path.
Nothing applies the configuration on the nodes yet.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/Makefile | 1 +
src/PVE/Multipath/Config.pm | 178 ++++++++++++++++++++++++++++++
src/test/Makefile | 5 +-
src/test/run_multipath_tests.pl | 39 +++++++
src/test/run_san_library_tests.pl | 47 ++++++++
5 files changed, 269 insertions(+), 1 deletion(-)
create mode 100644 src/PVE/Multipath/Config.pm
create mode 100755 src/test/run_san_library_tests.pl
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index db1c340d..597ad7a1 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -6,6 +6,7 @@ install:
install -D -m 0644 Diskmanage.pm ${DESTDIR}${PERLDIR}/PVE/Diskmanage.pm
install -D -m 0644 ISCSI/Topology.pm ${DESTDIR}${PERLDIR}/PVE/ISCSI/Topology.pm
install -D -m 0644 Multipath.pm ${DESTDIR}${PERLDIR}/PVE/Multipath.pm
+ install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath/Config.pm b/src/PVE/Multipath/Config.pm
new file mode 100644
index 00000000..667ad2b8
--- /dev/null
+++ b/src/PVE/Multipath/Config.pm
@@ -0,0 +1,178 @@
+package PVE::Multipath::Config;
+
+use v5.36;
+
+use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::SectionConfig;
+use PVE::Tools;
+
+use base qw(PVE::SectionConfig);
+
+# multipath interprets a leading '-' as an option and major:minor as a device number.
+our $WWID_PATTERN = '(?!-|[0-9]+:-?[0-9])[a-zA-Z0-9._:-]+';
+our $WWID_RE = qr/^(?:$WWID_PATTERN)\z/;
+
+my $properties = {
+ alias => {
+ type => 'string',
+ pattern => '[a-zA-Z0-9][a-zA-Z0-9._-]*',
+ maxLength => 64,
+ description => 'The name of the multipath map.',
+ optional => 1,
+ },
+ nodes => get_standard_option(
+ 'pve-node-list',
+ {
+ description => 'The nodes that should assemble the map, or all nodes when unset.',
+ optional => 1,
+ },
+ ),
+ 'no-path-retry' => {
+ type => 'string',
+ pattern => '(?:queue|fail|[0-9]+)',
+ description => 'The action or number of retries when all paths are down.',
+ optional => 1,
+ },
+ 'paths-expected' => {
+ type => 'integer',
+ minimum => 1,
+ description => 'The number of paths each node should have to the LUN. A node with fewer'
+ . ' paths reports the LUN as degraded.',
+ optional => 1,
+ },
+};
+my $defaultData = {
+ propertyList => {
+ type => { description => 'The section type.' },
+ id => {
+ type => 'string',
+ pattern => $WWID_PATTERN,
+ maxLength => 128,
+ description => 'The LUN WWID.',
+ },
+ %$properties,
+ },
+};
+sub private($class) { return $defaultData }
+sub type($class) { return 'wwid' }
+
+sub options($class) {
+ return { map { $_ => { optional => 1 } } keys %$properties };
+}
+
+sub wwid_api_schema() {
+ return { map { $_ => { $properties->{$_}->%* } } keys %$properties };
+}
+
+sub parse_section_header($class, $line) {
+ my ($type, $id, $errmsg, $config) = $class->SUPER::parse_section_header($line);
+ return undef if !defined($type);
+ $errmsg //= "invalid WWID '$id'" if $id !~ $WWID_RE;
+ return ($type, $id, $errmsg, $config);
+}
+
+sub decode_value($class, $type, $key, $value) {
+ return {
+ map { $_ => 1 }
+ grep { PVE::JSONSchema::pve_verify_node_name($_) } PVE::Tools::split_list($value)
+ }
+ if $key eq 'nodes';
+ return $value;
+}
+
+sub encode_value($class, $type, $key, $value) {
+ return join(',', sort keys %$value) if $key eq 'nodes';
+ return $value;
+}
+__PACKAGE__->register();
+__PACKAGE__->init();
+
+sub check_overrides($text) {
+ my $allowed = { map { $_ => 1 } qw(devices overrides defaults blacklist blacklist_exceptions) };
+ my $depth = 0;
+ for my $line (split(/\n/, $text // '')) {
+ $line =~ s/"[^"]*"//g;
+ $line =~ s/[#!].*//;
+ while ($line =~ /(\w+)\s*\{|([{}])/g) {
+ if (defined($1)) {
+ my $keyword = $1;
+ die "multipath overrides: configure LUNs in multipath.cfg, not a multipaths block\n"
+ if $keyword eq 'multipaths';
+ die "multipath overrides: unknown section '$keyword'\n"
+ if !$depth && !$allowed->{$keyword};
+ $depth++;
+ } elsif ($2 eq '{') {
+ $depth++;
+ } else {
+ die "multipath overrides: unbalanced braces\n" if !$depth;
+ $depth--;
+ }
+ }
+ }
+ die "multipath overrides: unbalanced braces\n" if $depth;
+}
+
+sub format_overrides($text) {
+ $text //= '';
+ $text =~ s/\s+$//;
+ return length($text) ? "$text\n" : '';
+}
+
+cfs_register_file(
+ 'multipath.cfg',
+ sub($filename, $data) { __PACKAGE__->parse_config($filename, $data) },
+ sub($filename, $data) { __PACKAGE__->write_config($filename, $data) },
+);
+cfs_register_file(
+ 'multipath-overrides.conf',
+ sub($filename, $data) { $data // '' },
+ sub($filename, $data) { format_overrides($data) },
+);
+
+sub read_config() { return cfs_read_file('multipath.cfg') }
+sub write_cluster_config($cfg) { cfs_write_file('multipath.cfg', $cfg) }
+sub read_overrides() { return cfs_read_file('multipath-overrides.conf') }
+sub write_overrides($text) { cfs_write_file('multipath-overrides.conf', $text) }
+
+sub lock_config($code, $errmsg = undef) {
+ cfs_lock_file('multipath.cfg', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub lock_overrides($code, $errmsg = undef) {
+ cfs_lock_file('multipath-overrides.conf', undef, $code);
+ die defined($errmsg) ? "$errmsg - $@" : $@ if $@;
+}
+
+sub generate_managed_conf() {
+ return "# Managed by Proxmox VE.\ndefaults {\n\tauto_resize grow_only\n}\n";
+}
+
+sub parse_wwids($text) {
+ return [map { m{^/($WWID_PATTERN)/\s*$} ? $1 : () } split(/\n/, $text // '')];
+}
+
+sub wwid_list($cfg) {
+ return [sort keys $cfg->{ids}->%*];
+}
+
+sub wwids_for_node($cfg, $node) {
+ return [grep { my $nodes = $cfg->{ids}->{$_}->{nodes}; !$nodes || !%$nodes || $nodes->{$node} }
+ wwid_list($cfg)->@*];
+}
+
+sub build_multipaths_block($cfg, $node) {
+ my $out = "# Managed by Proxmox VE.\nmultipaths {\n";
+ for my $wwid (wwids_for_node($cfg, $node)->@*) {
+ my $entry = $cfg->{ids}->{$wwid};
+ $out .= "\tmultipath {\n\t\twwid $wwid\n";
+ $out .= "\t\talias $entry->{alias}\n" if defined($entry->{alias});
+ $out .= "\t\tno_path_retry $entry->{'no-path-retry'}\n"
+ if defined($entry->{'no-path-retry'});
+ $out .= "\t}\n";
+ }
+ return "$out}\n";
+}
+
+1;
diff --git a/src/test/Makefile b/src/test/Makefile
index 3098bb15..21f51441 100644
--- a/src/test/Makefile
+++ b/src/test/Makefile
@@ -1,6 +1,6 @@
all: test
-test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath
+test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries
test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -49,3 +49,6 @@ test_iscsi: run_iscsi_discovery_tests.pl run_iscsi_rescan_tests.pl run_iscsi_dev
test_multipath: run_multipath_tests.pl
./run_multipath_tests.pl
+
+test_san_libraries: run_san_library_tests.pl
+ ./run_san_library_tests.pl
diff --git a/src/test/run_multipath_tests.pl b/src/test/run_multipath_tests.pl
index b1248ad0..db356cde 100755
--- a/src/test/run_multipath_tests.pl
+++ b/src/test/run_multipath_tests.pl
@@ -4,6 +4,7 @@ use v5.36;
use lib ('.', '..');
use Test::More;
use PVE::Multipath;
+use PVE::Multipath::Config;
use PVE::Tools qw(file_get_contents);
subtest 'multipathd JSON fixtures' => sub {
@@ -87,6 +88,44 @@ subtest 'multipathd JSON fixtures' => sub {
is(PVE::Multipath::derive_health(0, 0), 'failed', 'empty map is failed');
};
+subtest 'one section type and fixed baseline' => sub {
+ my $cfg = PVE::Multipath::Config->parse_config(
+ 'multipath.cfg',
+ "wwid: 3600a\n\talias data\n\tnodes n1\n\tno-path-retry fail\n\tpaths-expected 2\n",
+ );
+ is_deeply(PVE::Multipath::Config::wwid_list($cfg), ['3600a'], 'WWID is listed');
+ is_deeply(PVE::Multipath::Config::wwids_for_node($cfg, 'n2'), [], 'node restriction');
+ is_deeply(
+ [sort keys PVE::Multipath::Config::wwid_api_schema()->%*],
+ [qw(alias no-path-retry nodes paths-expected)],
+ 'only four options',
+ );
+ like(
+ PVE::Multipath::Config::generate_managed_conf(),
+ qr/auto_resize grow_only/,
+ 'fixed baseline',
+ );
+ like(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/no_path_retry fail/,
+ 'per-LUN option',
+ );
+ unlike(
+ PVE::Multipath::Config::build_multipaths_block($cfg, 'n1'),
+ qr/expected/,
+ 'the expected path count is not a multipath option',
+ );
+ is_deeply(
+ PVE::Multipath::Config::parse_wwids("# header\n/3600a/\n/-bad/\n"),
+ ['3600a'],
+ 'allow-list parser',
+ );
+ eval { PVE::Multipath::Config::check_overrides('multipaths { }') };
+ like($@, qr/configure LUNs/, 'LUN configuration belongs in explicit entries');
+ eval { PVE::Multipath::Config::check_overrides('defaults { user_friendly_names no }') };
+ is($@, '', 'raw defaults override is allowed');
+};
+
subtest 'one owner for map identity and membership' => sub {
is_deeply(PVE::Multipath::parse_dm_uuid('mpath-3600a'), { wwid => '3600a' }, 'whole map');
is_deeply(
diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl
new file mode 100755
index 00000000..819b9fc3
--- /dev/null
+++ b/src/test/run_san_library_tests.pl
@@ -0,0 +1,47 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use Test::MockModule;
+use Test::More;
+
+use lib ('.', '..');
+use PVE::Cluster;
+
+my %registered;
+my $cluster = Test::MockModule->new('PVE::Cluster');
+$cluster->mock(cfs_register_file => sub { $registered{ $_[0] } = [@_[1, 2]] });
+require PVE::Multipath::Config;
+
+subtest 'CFS callbacks preserve filenames, data and method receivers' => sub {
+ my ($parse, $write) = $registered{'multipath.cfg'}->@*;
+ my $cfg = $parse->('multipath.cfg', undef);
+ is_deeply($cfg->{ids}, {}, 'an absent configuration parses as empty');
+ $cfg = $parse->('multipath.cfg', "wwid: 3600abc\n\talias san-lun\n\tnodes n2,n1\n");
+ is_deeply($cfg->{ids}->{'3600abc'}->{nodes}, { n1 => 1, n2 => 1 }, 'nodes are decoded');
+ my $again = $parse->('multipath.cfg', $write->('multipath.cfg', $cfg));
+ is_deeply($again->{ids}, $cfg->{ids}, 'sections survive the registered writer and parser');
+ ($parse, $write) = $registered{'multipath-overrides.conf'}->@*;
+ is($parse->('multipath-overrides.conf', undef), '', 'absent overrides parse as empty');
+ is($write->('multipath-overrides.conf', undef), '', 'undefined overrides write as empty');
+ is($write->('multipath-overrides.conf', "devices { } \n"), "devices { }\n",
+ 'writer trims');
+};
+
+subtest 'omitted optional parameters retain explicit undef behavior' => sub {
+ my $config = Test::MockModule->new('PVE::Multipath::Config');
+ $config->mock(cfs_lock_file => sub { $@ = "lock unavailable\n" });
+ for my $lock (
+ \&PVE::Multipath::Config::lock_config, \&PVE::Multipath::Config::lock_overrides,
+ ) {
+ for my $args ([], [undef]) {
+ eval {
+ $lock->(sub { die "unexpected callback\n" }, @$args);
+ };
+ is($@, "lock unavailable\n", 'an omitted error prefix leaves the error intact');
+ }
+ }
+};
+
+done_testing();
next prev parent reply other threads:[~2026-11-16 1:25 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 1:23 [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 cluster 01/36] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 02/36] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 03/36] diskmanage: qualify NVMe over fabrics transport Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 04/36] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 05/36] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 06/36] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 07/36] multipath: add a library around multipathd Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 08/36] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-16 1:23 ` Thomas Lamprecht [this message]
2026-11-16 1:23 ` [PATCH v5 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 11/36] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 12/36] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 13/36] multipath: publish per-node status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 14/36] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 15/36] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 16/36] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 17/36] pvesm: expose the " Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 18/36] api: add node-level iSCSI status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 19/36] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 21/36] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 docs 23/36] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 24/36] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 25/36] report: add the managed multipath configuration Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 26/36] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 27/36] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 28/36] ui: dc: add a SAN view Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 29/36] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 30/36] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 31/36] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 32/36] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 33/36] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 34/36] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 35/36] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-16 1:23 ` [PATCH v5 manager 36/36] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-18 8:11 ` superseded: [PATCH v5 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116012355.3187417-10-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.