From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH v4 storage 13/36] multipath: publish per-node status
Date: Fri, 13 Nov 2026 04:00:31 +0100 [thread overview]
Message-ID: <20261113030146.1559030-14-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com>
Path health differs between nodes. Add the function that the status
daemon will call on each node: it applies the multipath configuration
and publishes the state of the node's maps in the cluster key-value
store, for the cluster-wide status.
A report has the WWID, state and path counts of each map, the iSCSI
initiator name of the node and the error of a failed configuration
run, or the reason why multipath is not usable. An unchanged report
is sent again once a minute, so that readers can tell a stale one.
The initiator name, which an admin needs for the access rules on the
storage array, comes from a new reader in the iSCSI plugin, which the
node iSCSI API will use as well.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/Makefile | 1 +
src/PVE/Multipath/Status.pm | 75 +++++++++++++++++++
src/PVE/Storage/ISCSIPlugin.pm | 16 ++++
src/PVE/Storage/LVMPlugin.pm | 11 ++-
src/test/run_multipath_api_tests.pl | 109 ++++++++++++++++++++++++++++
src/test/run_san_library_tests.pl | 31 ++++++++
6 files changed, 242 insertions(+), 1 deletion(-)
create mode 100644 src/PVE/Multipath/Status.pm
diff --git a/src/PVE/Makefile b/src/PVE/Makefile
index 87d5c41f..13fecbf8 100644
--- a/src/PVE/Makefile
+++ b/src/PVE/Makefile
@@ -9,6 +9,7 @@ install:
install -D -m 0644 Multipath/Config.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Config.pm
install -D -m 0644 Multipath/Generator.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Generator.pm
install -D -m 0644 Multipath/Identity.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Identity.pm
+ install -D -m 0644 Multipath/Status.pm ${DESTDIR}${PERLDIR}/PVE/Multipath/Status.pm
install -D -m 0644 CephConfig.pm ${DESTDIR}${PERLDIR}/PVE/CephConfig.pm
install -D -m 0644 GuestImport.pm ${DESTDIR}${PERLDIR}/PVE/GuestImport.pm
make -C Storage install
diff --git a/src/PVE/Multipath/Status.pm b/src/PVE/Multipath/Status.pm
new file mode 100644
index 00000000..91b699c4
--- /dev/null
+++ b/src/PVE/Multipath/Status.pm
@@ -0,0 +1,75 @@
+package PVE::Multipath::Status;
+
+use v5.36;
+use JSON qw(decode_json);
+use PVE::Cluster;
+use PVE::INotify;
+use PVE::Multipath;
+use PVE::Multipath::Generator;
+
+my $json = JSON->new->utf8->canonical;
+my $KEY = 'multipath';
+
+sub publish($report) {
+ my $node = PVE::INotify::nodename();
+ my $now = time();
+ my $data = { %$report, time => $now };
+ my $encoded = $json->encode($data);
+ if (length($encoded) >= 32 * 1024) {
+ $data = { time => $now, error => 'too many LUNs to report' };
+ $encoded = $json->encode($data);
+ }
+ my $raw = PVE::Cluster::get_node_kv($KEY, $node);
+ my $previous = eval { decode_json($raw->{$node} // '') };
+ if (ref($previous) eq 'HASH') {
+ my $age = $now - ($previous->{time} // 0);
+ $previous->{time} = $now;
+ return if $age >= 0 && $age < 60 && $json->encode($previous) eq $encoded;
+ }
+ PVE::Cluster::broadcast_node_kv($KEY, $encoded);
+}
+
+sub update_node_status() {
+ my $apply = PVE::Multipath::Generator::sync();
+ my $report = { luns => [] };
+ $report->{'apply-error'} = $apply->{error} if defined($apply->{error});
+ require PVE::Storage::ISCSIPlugin;
+ my $initiator = eval { PVE::Storage::ISCSIPlugin::read_initiator_info() };
+ $report->{'iscsi-initiator-name'} = $initiator->{name}
+ if $initiator && defined($initiator->{name});
+ eval {
+ die "multipath-tools is not installed\n" if !PVE::Multipath::is_supported();
+ die "multipathd is not running\n" if !PVE::Multipath::is_running();
+ require PVE::Storage::LVMPlugin;
+ my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups();
+ for my $map (PVE::Multipath::get_maps()->@*) {
+ my $wwid = $map->{wwid} // next;
+ my $info = $apply->{wwids}->{$wwid} // {};
+ my $lun = {
+ wwid => $wwid,
+ 'paths-total' => $map->{'paths-total'},
+ 'paths-active' => $map->{'paths-active'},
+ state => PVE::Multipath::derive_health(
+ $map->{'paths-active'},
+ $map->{'paths-total'},
+ $info->{'paths-expected'},
+ ),
+ };
+ $lun->{size} = $map->{size} if defined($map->{size});
+ $lun->{name} = $map->{name} if defined($map->{name}) && $map->{name} ne $wwid;
+ $lun->{storage} = $info->{storages}->[0] if $info->{storages};
+ my $vg = defined($map->{sysfs}) ? $pv_vgs->{"/dev/$map->{sysfs}"} : undef;
+ $lun->{vgname} = $vg if defined($vg);
+ push $report->{luns}->@*, $lun;
+ }
+ };
+ if (my $error = $@) {
+ chomp $error;
+ $report->{error} = $error;
+ delete $report->{luns};
+ }
+ eval { publish($report) };
+ warn "multipath: cannot publish node status - $@" if $@;
+}
+
+1;
diff --git a/src/PVE/Storage/ISCSIPlugin.pm b/src/PVE/Storage/ISCSIPlugin.pm
index 3bddcda6..d3ae56a4 100644
--- a/src/PVE/Storage/ISCSIPlugin.pm
+++ b/src/PVE/Storage/ISCSIPlugin.pm
@@ -21,8 +21,24 @@ use base qw(PVE::Storage::Plugin);
# iscsi helper function
my $ISCSIADM = '/usr/bin/iscsiadm';
+our $INITIATOR_NAME_FILE = '/etc/iscsi/initiatorname.iscsi';
our $SYSFS = '/sys';
+sub read_initiator_info {
+ my $content = eval { PVE::Tools::file_get_contents($INITIATOR_NAME_FILE, 64 * 1024) };
+ return undef if !defined($content);
+
+ my $res = {};
+ for my $line (split(/\n/, $content)) {
+ if ($line =~ m/^\s*InitiatorName\s*=\s*(\S+)\s*$/) {
+ $res->{name} = $1;
+ } elsif ($line =~ m/^\s*InitiatorAlias\s*=\s*(.+?)\s*$/) {
+ $res->{alias} = $1;
+ }
+ }
+ return defined($res->{name}) ? $res : undef;
+}
+
my $found_iscsi_adm_exe;
my sub assert_iscsi_support {
my ($noerr) = @_;
diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm
index ccae074a..ee64b782 100644
--- a/src/PVE/Storage/LVMPlugin.pm
+++ b/src/PVE/Storage/LVMPlugin.pm
@@ -17,7 +17,7 @@ use PVE::Format qw(render_bytes render_duration);
use PVE::INotify;
use PVE::JSONSchema qw(get_standard_option);
use PVE::RESTEnvironment qw(log_warn);
-use PVE::Tools qw(run_command file_read_firstline trim);
+use PVE::Tools qw(run_command file_get_contents file_read_firstline trim);
use PVE::Storage::Common;
use PVE::Storage::Plugin;
@@ -138,6 +138,15 @@ sub lvm_pv_info {
return $pvinfo;
}
+sub cached_pv_volume_groups {
+ my $data = eval { file_get_contents('/run/lvm/hints') } // '';
+ my $res = {};
+ while ($data =~ m/^scan:(\/dev\/\S+) pvid:\S+ devn:\d+:\d+ vg:([a-zA-Z0-9+_.-]+)$/mg) {
+ $res->{ abs_path($1) // $1 } = $2 if $2 ne '-';
+ }
+ return $res;
+}
+
sub clear_first_sector {
my ($dev) = shift;
diff --git a/src/test/run_multipath_api_tests.pl b/src/test/run_multipath_api_tests.pl
index 82aaa5f0..7bed4266 100755
--- a/src/test/run_multipath_api_tests.pl
+++ b/src/test/run_multipath_api_tests.pl
@@ -8,9 +8,11 @@ BEGIN {
}
use lib ('.', '..');
use File::Temp qw(tempdir);
+use JSON qw(decode_json);
use Test::More;
use Test::MockModule;
use PVE::Multipath::Generator;
+use PVE::Storage;
subtest 'add-only generation and failed reload throttle' => sub {
my $dir = tempdir('multipath-test-XXXX', DIR => '.', CLEANUP => 1);
@@ -90,4 +92,111 @@ subtest 'storage LUNs are the identified paths of the storage target' => sub {
);
};
+subtest 'an unconfigured node does not write drop-ins or reload' => sub {
+ my $dir = tempdir('multipath-empty-XXXX', DIR => '.', CLEANUP => 1);
+ local $PVE::Multipath::Generator::DEFAULTS_DROPIN = "$dir/defaults";
+ local $PVE::Multipath::Generator::OVERRIDES_DROPIN = "$dir/overrides";
+ local $PVE::Multipath::Generator::ALIASES_DROPIN = "$dir/aliases";
+ local $PVE::Multipath::Generator::RECONFIGURE_PENDING = "$dir/pending";
+ my $config = Test::MockModule->new('PVE::Multipath::Config');
+ $config->mock(read_config => sub { { ids => {} } });
+ $config->mock(read_overrides => sub { '' });
+ my $storage = Test::MockModule->new('PVE::Storage');
+ my $identity = Test::MockModule->new('PVE::Multipath::Identity');
+ $identity->mock(collect => sub { die "unexpected identity collection\n" });
+ my $mp = Test::MockModule->new('PVE::Multipath');
+ $mp->mock(is_supported => sub { 1 });
+ $mp->mock(is_running => sub { 1 });
+ $mp->mock(get_maps => sub { [] });
+ $mp->mock(list_etc_multipath_wwids => sub { [] });
+ my $generator = Test::MockModule->new('PVE::Multipath::Generator');
+ my @changes;
+
+ for my $function (qw(file_set_contents make_path)) {
+ $generator->mock($function => sub { push @changes, $function });
+ }
+ for my $function (qw(add_wwid reconfigure)) {
+ $mp->mock($function => sub { push @changes, $function });
+ }
+ my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin');
+ $iscsi->mock(read_initiator_info => sub { {} });
+ my $status = Test::MockModule->new('PVE::Multipath::Status');
+ my $report;
+ $status->mock(publish => sub { $report = $_[0] });
+ for my $ids ({}, { plain => { type => 'iscsi', target => 'iqn.test' } }) {
+ $storage->mock(config => sub { { ids => $ids } });
+ PVE::Multipath::Status::update_node_status();
+ ok(!exists($report->{error}), 'running daemon reports successfully');
+ ok(!exists($report->{'apply-error'}), 'unconfigured application succeeds');
+ is_deeply(\@changes, [], 'no write, mkdir, allow-list addition or reload');
+ is_deeply([glob("$dir/*")], [], 'no generated files or pending marker');
+ }
+};
+
+subtest 'node reports use read-back, refresh and a bounded error report' => sub {
+ my $cluster = Test::MockModule->new('PVE::Cluster');
+ my $node = Test::MockModule->new('PVE::INotify');
+ $node->mock(nodename => sub { 'n1' });
+ my (%raw, @sent);
+ $cluster->mock(get_node_kv => sub { \%raw });
+ $cluster->mock(
+ broadcast_node_kv => sub($key, $data) {
+ is($key, 'multipath', 'one node key');
+ push @sent, decode_json($data);
+ $raw{n1} = $data;
+ },
+ );
+ my $report = {
+ luns => [{
+ wwid => '3600a',
+ name => 'data',
+ state => 'optimal',
+ 'paths-total' => 2,
+ 'paths-active' => 2,
+ }],
+ };
+ PVE::Multipath::Status::publish($report);
+ $NOW += 59;
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 1, 'same report younger than 60 seconds is skipped');
+ $NOW++;
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 2, 'report refreshed at 60 seconds');
+ %raw = ();
+ PVE::Multipath::Status::publish($report);
+ is(scalar(@sent), 3, 'lost pmxcfs value is republished without local send state');
+ PVE::Multipath::Status::publish({ luns => [('x' x 33000)] });
+ is($sent[-1]->{error}, 'too many LUNs to report', 'oversized report is an error');
+ ok(!exists($sent[-1]->{luns}), 'not an empty or partial LUN report');
+};
+
+subtest 'one entry point applies configuration and reports unavailable reasons' => sub {
+ my $gen = Test::MockModule->new('PVE::Multipath::Generator');
+ my $status = Test::MockModule->new('PVE::Multipath::Status');
+ my $mp = Test::MockModule->new('PVE::Multipath');
+ my $iscsi = Test::MockModule->new('PVE::Storage::ISCSIPlugin');
+ my ($report, @order);
+ $gen->mock(sync => sub { push @order, 'apply'; { error => 'reload failed' } });
+ $status->mock(publish => sub { push @order, 'publish'; $report = $_[0] });
+ $iscsi->mock(read_initiator_info => sub { { name => 'iqn.2025-01.example:node' } });
+ $mp->mock(is_supported => sub { 0 });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'multipath-tools is not installed', 'missing tools reason only');
+ is_deeply(\@order, [qw(apply publish)], 'single call owns ordering');
+ is($report->{'apply-error'}, 'reload failed', 'apply failure is separate');
+ $mp->mock(is_supported => sub { 1 });
+ $mp->mock(is_running => sub { 0 });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'multipathd is not running', 'stopped daemon reason only');
+ $mp->mock(is_running => sub { 1 });
+ $mp->mock(get_maps => sub { die "bad JSON\n" });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{error}, 'bad JSON', 'query error without unavailable prefix');
+ $mp->mock(get_maps =>
+ sub { [{ wwid => '3600a', name => 'data', 'paths-active' => 1, 'paths-total' => 2 }] });
+ PVE::Multipath::Status::update_node_status();
+ is($report->{luns}->[0]->{state}, 'degraded', 'maps without PVE entries are included');
+ ok(!exists($report->{supported}), 'no separate capability state');
+};
+
done_testing();
diff --git a/src/test/run_san_library_tests.pl b/src/test/run_san_library_tests.pl
index 2ec2a651..6a93e868 100755
--- a/src/test/run_san_library_tests.pl
+++ b/src/test/run_san_library_tests.pl
@@ -50,4 +50,35 @@ subtest 'omitted optional parameters retain explicit undef behavior' => sub {
}
};
+subtest 'LVM hints are optional cached display metadata' => sub {
+ require PVE::Storage::LVMPlugin;
+ my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin');
+ my $contents;
+ $lvm->mock(
+ file_get_contents => sub {
+ is($_[0], '/run/lvm/hints', 'only the existing LVM cache is read');
+ die "No such file\n" if !defined($contents);
+ return $contents;
+ },
+ );
+ $lvm->mock(abs_path => sub { $_[0] eq '/dev/disk/by-id/test' ? '/dev/dm-0' : $_[0] });
+ $lvm->mock(run_command => sub { die "unexpected device probe\n" });
+ for my $case (
+ [undef, {}, 'missing file'],
+ ['', {}, 'empty file'],
+ ["scan=/dev/sda pvid=abc devn=8:0 vg=data\n", {}, 'different format'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:-\n", {}, 'orphan PV'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:[unknown]\n", {}, 'unknown VG'],
+ ["scan:/dev/sda pvid:abc devn:8:0 vg:data\n", { '/dev/sda' => 'data' }, 'cached VG'],
+ [
+ "scan:/dev/disk/by-id/test pvid:abc devn:253:0 vg:data\n",
+ { '/dev/dm-0' => 'data' },
+ 'resolved device name',
+ ],
+ ) {
+ $contents = $case->[0];
+ is_deeply(PVE::Storage::LVMPlugin::cached_pv_volume_groups(), $case->[1], $case->[2]);
+ }
+};
+
done_testing();
next prev parent reply other threads:[~2026-11-13 3:11 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 3:00 [PATCH v4 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 cluster 01/36] cfs: observe the multipath configuration files Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 02/36] diskmanage: collect disk transport type from lsblk Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 03/36] diskmanage: qualify NVMe over fabrics transport Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 04/36] diskmanage: optionally include iSCSI session devices in disk enumeration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 05/36] diskmanage: collect native NVMe multipath path state Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 06/36] iscsi: factor out session device map from device list Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 07/36] multipath: add a library around multipathd Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 08/36] diskmanage: add helper to list multipath devices Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 09/36] multipath: add a cluster-wide configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 10/36] iscsi: add an option to manage storage LUNs with multipath Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 11/36] lvm: activate a missing base volume before using the volume group Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 12/36] multipath: apply the configuration on every node Thomas Lamprecht
2026-11-13 3:00 ` Thomas Lamprecht [this message]
2026-11-13 3:00 ` [PATCH v4 storage 14/36] api: add endpoints for the cluster-wide multipath configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 15/36] api: multipath: add the cluster-wide health status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 16/36] api: scan: add a SAN LUN inventory Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 17/36] pvesm: expose the " Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 18/36] api: add node-level iSCSI status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 19/36] disks: lvm: drop unused imports Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 21/36] disks: lvm: refuse to destroy the volume group of a shared storage Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 docs 23/36] storage: document the cluster-wide multipath management Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 24/36] pvestatd: apply the multipath configuration and report the node status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 25/36] report: add the managed multipath configuration Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 26/36] api: cluster: add the multipath endpoints Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 27/36] api: nodes: expose the iSCSI status Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 28/36] ui: dc: add a SAN view Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 29/36] ui: storage: iscsi: add the multipath option Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 30/36] ui: lvm: enable shared when a base storage is selected Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 31/36] ui: node: add SAN LUNs panel Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 32/36] ui: node: san: wipe a LUN that no storage uses Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 33/36] ui: node: san: show iSCSI targets and sessions Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 34/36] ui: dc: san: add a guided setup for shared LVM on iSCSI Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 35/36] ui: dc: san: add a guided setup for shared LVM on Fibre Channel Thomas Lamprecht
2026-11-13 3:00 ` [PATCH v4 manager 36/36] ui: storage: point the iSCSI and LVM add dialogs to the SAN view Thomas Lamprecht
2026-11-16 1:28 ` superseded: [PATCH v4 cluster,storage,docs,manager 00/36] SAN: cluster-wide multipath, SAN views and a guided setup for shared LVM Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113030146.1559030-14-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox