From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v4 storage 13/36] multipath: publish per-node status
Date: Fri, 13 Nov 2026 04:00:31 +0100 [thread overview]
Message-ID: <20261113030146.1559030-14-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com>
Path health differs between nodes. Add the function that the status
daemon will call on each node: it applies the multipath configuration
and publishes the state of the node's maps in the cluster key-value
store, for the cluster-wide status.
A report has the WWID, state and path counts of each map, the iSCSI
initiator name of the node and the error of a failed configuration
run, or the reason why multipath is not usable. An unchanged report
is sent again once a minute, so that readers can tell a stale one.
The initiator name, which an admin needs for the access rules on the
storage array, comes from a new reader in the iSCSI plugin, which the
node iSCSI API will use as well.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/Makefile | 1 +
src/PVE/Multipath/Status.pm | 75 +++++++++++++++++++
src/PVE/Storage/ISCSIPlugin.pm | 16 ++++
src/PVE/Storage/LVMPlugin.pm | 11 ++-
src/test/run_multipath_api_tests.pl | 109 ++++++++++++++++++++++++++++
src/test/run_san_library_tests.pl | 31 ++++++++
6 files changed, 242 insertions(+), 1 deletion(-)
create mode 100644 src/PVE/Multipath/Status.pm
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index 87d5c41f..13fecbf8 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -9,6 +9,7 @@ install:
install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
install -D -m 0644 Multipath/Generator.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Generator.pm
install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm
+ install -D -m 0644 Multipath/Status.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Status.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm
new file mode 100644
index 00000000..91b699c4
--- /dev/null
+++ b/src/PVE/Multipath/Status.pm
@@ -0,0 +1,75 @@
+package PVE::Multipath::Status;
+
+use v5.36;
+use JSON qw(decode_json);
+use PVE::Cluster;
+use PVE::INotify;
+use PVE::Multipath;
+use PVE::Multipath::Generator;
+
+my $json = JSON->new->utf8->canonical;
+my $KEY = 'multipath';
+
+sub publish($report) {
+ my $node = PVE::INotify::nodename();
+ my $now = time();
+ my $data = { %$report, time => $now };
+ my $encoded = $json->encode($data);
+ if (length($encoded) >= 32 * 1024) {
+ $data = { time => $now, error => 'too many LUNs to report' };
+ $encoded = $json->encode($data);
+ }
+ my $raw = PVE::Cluster::get_node_kv($KEY, $node);
+ my $previous = eval { decode_json($raw->{$node} // '') };
+ if (ref($previous) eq 'HASH') {
+ my $age = $now - ($previous->{time} // 0);
+ $previous->{time} = $now;
+ return if $age >= 0 && $age < 60 && $json->encode($previous) eq $encoded;
+ }
+ PVE::Cluster::broadcast_node_kv($KEY, $encoded);
+}
+
+sub update_node_status() {
+ my $apply = PVE::Multipath::Generator::sync();
+ my $report = { luns => [] };
+ $report->{'apply-error'} = $apply->{error} if defined($apply->{error});
+ require PVE::Storage::ISCSIPlugin;
+ my $initiator = eval { PVE::Storage::ISCSIPlugin::read_initiator_info() };
+ $report->{'iscsi-initiator-name'} = $initiator->{name}
+ if $initiator && defined($initiator->{name});
+ eval {
+ die "multipath-tools is not installed\n" if !PVE::Multipath::is_supported();
+ die "multipathd is not running\n" if !PVE::Multipath::is_running();
+ require PVE::Storage::LVMPlugin;
+ my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups();
+ for my $map (PVE::Multipath::get_maps()->@*) {
+ my $wwid = $map->{wwid} // next;
+ my $info = $apply->{wwids}->{$wwid} // {};
+ my $lun = {
+ wwid => $wwid,
+ 'paths-total' => $map->{'paths-total'},
+ 'paths-active' => $map->{'paths-active'},
+ state => PVE::Multipath::derive_health(
+ $map->{'paths-active'},
+ $map->{'paths-total'},
+ $info->{'paths-expected'},
+ ),
+ };
+ $lun->{size} = $map->{size} if defined($map->{size});
+ $lun->{name} = $map->{name} if defined($map->{name}) && $map->{name} ne $wwid;
+ $lun->{storage} = $info->{storages}->[0] if $info->{storages};
+ my $vg = defined($map->{sysfs}) ? $pv_vgs->{"/dev/$map->{sysfs}"} : undef;
+ $lun->{vgname} = $vg if defined($vg);
+ push $report->{luns}->@*, $lun;
+ }
+ };
+ if (my $error = $@) {
+ chomp $error;
+ $report->{error} = $error;
+ delete $report->{luns};
+ }
+ eval { publish($report) };
+ warn "multipath: cannot publish node status - $@" if $@;
+}
+
+1;
diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm
index 3bddcda6..d3ae56a4 100644
--- a/src/PVE/Storage/ISCSIPlugin.pm
+++ b/src/PVE/Storage/ISCSIPlugin.pm
@@ -21,8 +21,24 @@ use base qw(PVE::Storage::Plugin);
# iscsi helper function
my $ISCSIADM = '/usr/bin/iscsiadm';
+our $INITIATOR_NAME_FILE = '/etc/iscsi/initiatorname.iscsi';
our $SYSFS = '/sys';
+sub read_initiator_info {
+ my $content = eval { PVE::Tools::file_get_contents($INITIATOR_NAME_FILE, 64 * 1024) };
+ return undef if !defined($content);
+
+ my $res = {};
+ for my $line (split(/\n/, $content)) {
+ if ($line =~ m/^\s*InitiatorName\s*=\s*(\S+)\s*$/) {
+ $res->{name} = $1;
+ } elsif ($line =~ m/^\s*InitiatorAlias\s*=\s*(.+?)\s*$/) {
+ $res->{alias} = $1;
+ }
+ }
+ return defined($res->{name}) ? $res : undef;
+}
+
my $found_iscsi_adm_exe;
my sub assert_iscsi_support {
my ($noerr) = @_;
diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index ccae074a..ee64b782 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -17,7 +17,7 @@ use PVE::Format qw(render_bytes render_duration);
use PVE::INotify;
use PVE::JSONSchema qw(get_standard_option);
use PVE::RESTEnvironment qw(log_warn);
-use PVE::Tools qw(run_command file_read_firstline trim);
+use PVE::Tools qw(run_command file_get_contents file_read_firstline trim);
use PVE::Storage::Common;
use PVE::Storage::Plugin;
@@ -138,6 +138,15 @@ sub lvm_pv_info {
return $pvinfo;
}
+sub cached_pv_volume_groups {
+ my $data = eval { file_get_contents('/run/lvm/hints') } // '';
+ my $res = {};
+ while ($data =~ m/^scan:(\/dev\/\S+) pvid:\S+ devn:\d+:\d+ vg:([a-zA-Z0-9+_.-]+)$/mg) {
+ $res->{ abs_path($1) // $1 } = $2 if $2 ne '-';
+ }
+ return $res;
+}
+
sub clear_first_sector {
my ($dev) = shift;
diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl
index 82aaa5f0..7bed4266 100755
--- a/src/test/run_multipath_api_tests.pl
+++ b/src/test/run_multipath_api_tests.pl
@@ -8,9 +8,11 @@ BEGIN {
}
use lib ('.', '..');
use File::Temp qw(tempdir);
+use JSON qw(decode_json);
use Test::More;
use Test::MockModule;
use PVE::Multipath::Generator;
+use PVE::Storage;
subtest 'add-only generation and failed reload throttle' => sub {
my $dir = tempdir('multipath-test-XXXX', DIR => '.', CLEANUP => 1);
@@ -90,4 +92,111 @@ subtest 'storage LUNs are the identified paths of the storage target' => sub {
);
};
+subtest 'an unconfigured node does not write drop-ins or reload' => sub {
+ my $dir = tempdir('multipath-empty-XXXX', DIR => '.', CLEANUP => 1);
+ local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults";
+ local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides";
+ local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases";
+ local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending";
+ my $config = Test::MockModule->new('PVE::Multipath::Config');
+ $config->mock(read_config => sub { { ids => {} } });
+ $config->mock(read_overrides => sub { '' });
+ my $storage = Test::MockModule->new('PVE::Storage');
+ my $identity = Test::MockModule->new('PVE::Multipath::Identity');
+ $identity->mock(collect => sub { die "unexpected identity collection\n" });
+ my $mp = Test::MockModule->new('PVE::Multipath');
+ $mp->mock(is_supported => sub { 1 });
+ $mp->mock(is_running => sub { 1 });
+ $mp->mock(get_maps => sub { [] });
+ $mp->mock(list_etc_multipath_wwids => sub { [] });
+ my $generator = Test::MockModule->new('PVE::Multipath::Generator');
+ my @changes;
+
+ for my $function (qw(file_set_contents make_path)) {
+ $generator->mock($function => sub { push @changes, $function });
+ }
+ for my $function (qw(add_wwid reconfigure)) {
+ $mp->mock($function => sub { push @changes, $function });
+ }
+ my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin');
+ $iscsi->mock(read_initiator_info => sub { {} });
+ my $status = Test::MockModule->new('PVE::Multipath::Status');
+ my $report;
+ $status->mock(publish => sub { $report = $_[0] });
+ for my $ids ({}, { plain => { type => 'iscsi', target => 'iqn.test' } }) {
+ $storage->mock(config => sub { { ids => $ids } });
+ PVE::Multipath::Status::update_node_status();
+ ok(!exists($report->{error}), 'running daemon reports successfully');
+ ok(!exists($report->{'apply-error'}), 'unconfigured application succeeds');
+ is_deeply(\@changes, [], 'no write, mkdir, allow-list addition or reload');
+ is_deeply([glob("$dir/*")], [], 'no generated files or pending marker');
+ }
+};
+
+subtest 'node reports use read-back, refresh and a bounded error report' => sub {
+ my $cluster = Test::MockModule->new('PVE::Cluster');
+ my $node = Test::MockModule->new('PVE::INotify');
+ $node->mock(nodename => sub { 'n1' });
+ my (%raw, @sent);
+ $cluster->mock(get_node_kv => sub { \%raw });
+ $cluster->mock(
+ broadcast_node_kv => sub($key, $data) {
+ is($key, 'multipath', 'one node key');
+ push @sent, decode_json($data);
+ $raw{n1} = $data;
+ },
+ );
+ my $report = {
+ luns => [{
+ wwid => '3600a',
+ name => 'data',
+ state => 'optimal',
+ 'paths-total' => 2,
+ 'paths-active' => 2,
+ }],
+ };
+ PVE::Multipath::Status::publish($report);
+ $NOW += 59;
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 1, 'same report younger than 60 seconds is skipped');
+ $NOW++;
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 2, 'report refreshed at 60 seconds');
+ %raw = ();
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 3, 'lost pmxcfs value is republished without local send state');
+ PVE::Multipath::Status::publish({ luns => [('x' x 33000)] });
+ is($sent[-1]->{error}, 'too many LUNs to report', 'oversized report is an error');
+ ok(!exists($sent[-1]->{luns}), 'not an empty or partial LUN report');
+};
+
+subtest 'one entry point applies configuration and reports unavailable reasons' => sub {
+ my $gen = Test::MockModule->new('PVE::Multipath::Generator');
+ my $status = Test::MockModule->new('PVE::Multipath::Status');
+ my $mp = Test::MockModule->new('PVE::Multipath');
+ my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin');
+ my ($report, @order);
+ $gen->mock(sync => sub { push @order, 'apply'; { error => 'reload failed' } });
+ $status->mock(publish => sub { push @order, 'publish'; $report = $_[0] });
+ $iscsi->mock(read_initiator_info => sub { { name => 'iqn.2025-01.example:node' } });
+ $mp->mock(is_supported => sub { 0 });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'multipath-tools is not installed', 'missing tools reason only');
+ is_deeply(\@order, [qw(apply publish)], 'single call owns ordering');
+ is($report->{'apply-error'}, 'reload failed', 'apply failure is separate');
+ $mp->mock(is_supported => sub { 1 });
+ $mp->mock(is_running => sub { 0 });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'multipathd is not running', 'stopped daemon reason only');
+ $mp->mock(is_running => sub { 1 });
+ $mp->mock(get_maps => sub { die "bad JSON\n" });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'bad JSON', 'query error without unavailable prefix');
+ $mp->mock(get_maps =>
+ sub { [{ wwid => '3600a', name => 'data', 'paths-active' => 1, 'paths-total' => 2 }] });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{luns}->[0]->{state}, 'degraded', 'maps without PVE entries are included');
+ ok(!exists($report->{supported}), 'no separate capability state');
+};
+
done_testing();
diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl
index 2ec2a651..6a93e868 100755
--- a/src/test/run_san_library_tests.pl
+++ b/src/test/run_san_library_tests.pl
@@ -50,4 +50,35 @@ subtest 'omitted optional parameters retain explicit undef behavior' => sub {
}
};
+subtest 'LVM hints are optional cached display metadata' => sub {
+ require PVE::Storage::LVMPlugin;
+ my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin');
+ my $contents;
+ $lvm->mock(
+ file_get_contents => sub {
+ is($_[0], '/run/lvm/hints', 'only the existing LVM cache is read');
+ die "No such file\n" if !defined($contents);
+ return $contents;
+ },
+ );
+ $lvm->mock(abs_path => sub { $_[0] eq '/dev/disk/by-id/test' ? '/dev/dm-0' : $_[0] });
+ $lvm->mock(run_command => sub { die "unexpected device probe\n" });
+ for my $case (
+ [undef, {}, 'missing file'],
+ ['', {}, 'empty file'],
+ ["scan=/dev/sda pvid=abc devn=8:0 vg=data\n", {}, 'different format'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:-\n", {}, 'orphan PV'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:[unknown]\n", {}, 'unknown VG'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:data\n", { '/dev/sda' => 'data' }, 'cached VG'],
+ [
+ "scan:/dev/disk/by-id/test pvid:abc devn:253:0 vg:data\n",
+ { '/dev/dm-0' => 'data' },
+ 'resolved device name',
+ ],
+ ) {
+ $contents = $case->[0];
+ is_deeply(PVE::Storage::LVMPlugin::cached_pv_volume_groups(), $case->[1], $case->[2]);
+ }
+};
+
done_testing();
next prev parent reply other threads:[~2026-11-13 3:11 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 3:00 [PATCH v4 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 cluster 01/36] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 02/36] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 03/36] diskmanage: qualify NVMe over fabrics transport Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 04/36] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 05/36] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 06/36] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 07/36] multipath: add a library around multipathd Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 08/36] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 09/36] multipath: add a cluster-wide configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 11/36] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 12/36] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-13 3:00 ` Thomas Lamprecht [this message]
2026-11-13 3:00 ` [PATCH v4 storage 14/36] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 15/36] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 16/36] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 17/36] pvesm: expose the " Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 18/36] api: add node-level iSCSI status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 19/36] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 21/36] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 docs 23/36] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 24/36] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 25/36] report: add the managed multipath configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 26/36] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 27/36] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 28/36] ui: dc: add a SAN view Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 29/36] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 30/36] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 31/36] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 32/36] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 33/36] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 34/36] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 35/36] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 36/36] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-16 1:28 ` superseded: [PATCH v4 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113030146.1559030-14-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.