From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 024011FF0E1 for ; Mon, 16 Nov 2026 02:28:54 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id CCC10218AF; Mon, 16 Nov 2026 02:26:05 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v5 storage 16/36] api: scan: add a SAN LUN inventory Date: Mon, 16 Nov 2026 02:23:25 +0100 Message-ID: <20261116012355.3187417-17-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116012355.3187417-1-t.lamprecht@proxmox.com> References: <20261116012355.3187417-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794792242164 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.054 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: CZURBG7NGMBL6HJ3JGBKLWXUCLW4SOAF X-Message-ID-Hash: CZURBG7NGMBL6HJ3JGBKLWXUCLW4SOAF X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Add /nodes/{node}/scan/san-luns, which lists the LUNs a node sees over iSCSI, Fibre Channel or NVMe over Fabrics, one row per WWID with the paths folded into their multipath map. The node disk list has neither the maps nor a relation to volume groups and storages. A row has the device to use, its size and transport, how multipath handles the LUN, the health and state of its paths, and the known usage with the volume group and storage. The usage comes from cached data only, as probing a LUN whose paths are down can block. The paths of a map are left out before the disk list queries udev for each device. A node with a thousand LUNs has several thousand of them, and the views poll this list every few seconds. Co-authored-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Storage/Scan.pm | 202 ++++++++++++++++++ src/PVE/Diskmanage.pm | 7 + src/PVE/Multipath.pm | 24 +++ src/PVE/Storage/Makefile | 3 +- src/PVE/Storage/SAN.pm | 197 +++++++++++++++++ src/test/Makefile | 8 +- src/test/disk_tests/multipath/multipathd | 124 +++++++++++ .../multipath/multipathd_status_expected.json | 40 ++++ src/test/disklist_test.pm | 15 ++ src/test/run_san_api_tests.pl | 24 +++ src/test/run_san_lun_scan_tests.pl | 197 +++++++++++++++++ 11 files changed, 839 insertions(+), 2 deletions(-) create mode 100644 src/PVE/Storage/SAN.pm create mode 100644 src/test/disk_tests/multipath/multipathd create mode 100644 src/test/disk_tests/multipath/multipathd_status_expected.json create mode 100755 src/test/run_san_api_tests.pl create mode 100755 src/test/run_san_lun_scan_tests.pl diff --git a/src/PVE/API2/Storage/Scan.pm b/src/PVE/API2/Storage/Scan.pm index a58bdd80..b4273f5c 100644 --- a/src/PVE/API2/Storage/Scan.pm +++ b/src/PVE/API2/Storage/Scan.pm @@ -12,6 +12,7 @@ use PVE::SafeSyslog; use PVE::Storage::LVMPlugin; use PVE::Storage::LvmThinPlugin; use PVE::Storage::PBSPlugin; +use PVE::Storage::SAN; use PVE::Storage; use base qw(PVE::RESTHandler); @@ -49,6 +50,7 @@ __PACKAGE__->register_method({ { method => 'lvm' }, { method => 'nfs' }, { method => 'pbs' }, + { method => 'san-luns' }, { method => 'zfs' }, ]; @@ -373,6 +375,206 @@ __PACKAGE__->register_method({ }, }); +__PACKAGE__->register_method({ + name => 'sanlunscan', + path => 'san-luns', + method => 'GET', + description => "List SAN LUNs with their paths and cached usage.", + protected => 1, + proxyto => "node", + permissions => { + check => [ + 'or', + ['perm', '/nodes/{node}', ['Sys.Audit']], + ['perm', '/storage', ['Datastore.Allocate']], + ], + }, + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + }, + }, + returns => { + type => 'array', + items => { + type => "object", + properties => { + devpath => { + description => "The device path.", + type => 'string', + }, + size => { + description => "The device size in bytes.", + type => 'integer', + renderer => 'bytes', + optional => 1, + }, + transport => { + description => + "The device transport, for example 'sata', 'sas', 'usb', 'fc'," + . " 'iscsi', 'nvme', 'nvme-fc', or 'nvme-tcp'.", + type => 'string', + optional => 1, + }, + vendor => { type => 'string', optional => 1 }, + model => { type => 'string', optional => 1 }, + serial => { type => 'string', optional => 1 }, + state => { + description => + 'The health of the multipath LUN when path status is available.', + type => 'string', + enum => ['optimal', 'degraded', 'failed'], + optional => 1, + }, + 'paths-total' => { + description => "The number of paths of a multipath mapped device or of" + . " an NVMe namespace attached through multiple controllers.", + type => 'integer', + optional => 1, + }, + 'paths-active' => { + description => + "The number of active paths whose checker state is ready, ghost," + . " or not reported. Native NVMe paths count when their controller is live" + . " and their ANA state is optimized, non-optimized, or not reported.", + type => 'integer', + optional => 1, + }, + 'paths-expected' => { + description => "The number of paths the LUN should have according to the" + . " storage configuration, one per portal for iSCSI; absent when it is" + . " not known.", + type => 'integer', + optional => 1, + }, + 'path-list' => { + description => "The member paths of the device with their state.", + type => 'array', + optional => 1, + items => { + type => 'object', + properties => { + device => { + description => "The path block device, or the NVMe" + . " controller device the path uses.", + type => 'string', + }, + state => { + description => "The device mapper state of the path (for" + . " example active or failed) or the state of the NVMe" + . " controller (for example live or connecting).", + type => 'string', + }, + 'checker-state' => { + description => "The state reported by the multipathd path" + . " checker, for example ready, ghost or faulty.", + type => 'string', + optional => 1, + }, + 'ana-state' => { + description => "The Asymmetric Namespace Access state of" + . " the namespace on this path, for example optimized" + . " or inaccessible.", + type => 'string', + optional => 1, + }, + 'host-wwpn' => { + description => "The WWPN of the host (initiator) port of" + . " Fibre Channel attached paths.", + type => 'string', + optional => 1, + }, + 'target-wwpn' => { + description => "The WWPN of the target port of Fibre" + . " Channel attached paths.", + type => 'string', + optional => 1, + }, + address => { + description => "The address of the NVMe controller the" + . " path uses.", + type => 'string', + optional => 1, + }, + }, + }, + }, + usage => { + description => "The usage indicated by cached device metadata.", + type => 'string', + enum => ['unused', 'lvm', 'other'], + }, + vgname => { + description => "The name of the LVM volume group the device belongs to.", + type => 'string', + optional => 1, + }, + 'by-id' => { + description => + "A path under /dev/disk/by-id that encodes the device's identity" + . " and has been verified to resolve to this device. Absent when no such" + . " link could be verified.", + type => 'string', + optional => 1, + }, + 'used-by-storage' => { + description => "The storage that uses the LUN: through the volume group" + . " on it, or as an iSCSI storage that serves the LUNs of its target as" + . " images.", + type => 'string', + optional => 1, + }, + wwid => { + description => "The LUN WWID.", + type => 'string', + }, + name => { + description => 'The map alias, when different from its WWID.', + type => 'string', + optional => 1, + }, + multipath => { + description => "How the node manages multipath for the LUN: 'manual' for a" + . " map with an explicit entry, 'storage' for the map of an iSCSI storage" + . " with the 'managed-multipath' option, 'unmanaged' for any other map," + . " 'pending' while a map that an entry, a storage or the allow-list of" + . " the node requests does not exist yet, 'none' without a map or" + . " request, 'native' for the kernel-native multipath of an NVMe" + . " namespace.", + type => 'string', + enum => ['none', 'pending', 'storage', 'manual', 'unmanaged', 'native'], + }, + 'multipath-storage' => { + description => + "The multipath-enabled storage the LUN belongs to, whether or" + . " not it also has an explicit multipath entry.", + type => 'string', + optional => 1, + }, + 'iscsi-target' => { + description => "The IQN of the iSCSI target the LUN is attached through.", + type => 'string', + optional => 1, + }, + 'iscsi-lun' => { + description => "The number of the LUN on its iSCSI target.", + type => 'integer', + optional => 1, + }, + detail => { + description => "The cached signature or usage when usage is 'other'.", + type => 'string', + optional => 1, + }, + }, + }, + }, + code => sub { + return PVE::Storage::SAN::scan_luns(PVE::Storage::config()); + }, +}); + __PACKAGE__->register_method({ name => 'zfsscan', path => 'zfs', diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index f1a5cd18..8670165c 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -525,6 +525,13 @@ sub get_nvme_transport { return $transport; } +# SAS does not distinguish local from remote attachments, so it is not classified as remote. +sub is_remote_transport { + my ($transport) = @_; + + return ($transport // '') =~ m/^(?:fc|fcoe|iscsi|nvme-(?:fc|tcp|rdma))$/; +} + my sub is_ssdlike { my ($type) = @_; return $type eq 'ssd' || $type eq 'nvme'; diff --git a/src/PVE/Multipath.pm b/src/PVE/Multipath.pm index ec417381..027e6a53 100644 --- a/src/PVE/Multipath.pm +++ b/src/PVE/Multipath.pm @@ -2,7 +2,9 @@ package PVE::Multipath; use v5.36; +use Cwd qw(abs_path); use Encode qw(decode encode); +use File::Basename qw(basename); use JSON qw(decode_json); use PVE::Tools qw(run_command file_get_contents file_read_firstline dir_glob_foreach); @@ -19,6 +21,12 @@ sub parse_dm_uuid($uuid) { return { wwid => $wwid, defined($part) ? (partition => int($part)) : () }; } +sub device_map($device) { + my $name = basename(abs_path($device) // $device); + return undef if $name !~ /^dm-\d+$/; + return parse_dm_uuid(file_read_firstline("/sys/block/$name/dm/uuid")); +} + sub map_info() { my $maps = {}; dir_glob_foreach( @@ -159,6 +167,22 @@ sub get_maps() { return $maps; } +sub path_status_by_wwid() { + my $res = {}; + for my $map (get_maps()->@*) { + my $wwid = $map->{wwid} // next; + my @paths; + for my $path ($map->{paths}->@*) { + my $row = { device => "/dev/$path->{dev}", state => $path->{'dm-state'} // 'undef' }; + $row->{'checker-state'} = $path->{'check-state'} if defined($path->{'check-state'}); + $row->{$_} = $path->{$_} for grep { defined($path->{$_}) } qw(host-wwpn target-wwpn); + push @paths, $row; + } + $res->{$wwid} = { active => $map->{'paths-active'}, paths => \@paths }; + } + return $res; +} + sub wait_for_map($wwid, $timeout = 10) { my $link = "/dev/disk/by-id/dm-uuid-mpath-$wwid"; my $deadline = time() + ($timeout // 10); diff --git a/src/PVE/Storage/Makefile b/src/PVE/Storage/Makefile index a67dc25f..a2df83c0 100644 --- a/src/PVE/Storage/Makefile +++ b/src/PVE/Storage/Makefile @@ -14,7 +14,8 @@ SOURCES= \ PBSPlugin.pm \ BTRFSPlugin.pm \ LvmThinPlugin.pm \ - ESXiPlugin.pm + ESXiPlugin.pm \ + SAN.pm .PHONY: install install: diff --git a/src/PVE/Storage/SAN.pm b/src/PVE/Storage/SAN.pm new file mode 100644 index 00000000..5e742523 --- /dev/null +++ b/src/PVE/Storage/SAN.pm @@ -0,0 +1,197 @@ +package PVE::Storage::SAN; + +use v5.36; + +use Cwd qw(abs_path); +use File::Basename qw(basename); + +use PVE::INotify; +use PVE::Tools qw(file_read_firstline dir_glob_foreach); + +use PVE::Diskmanage; +use PVE::ISCSI::Topology; +use PVE::Multipath; +use PVE::Multipath::Config; +use PVE::Multipath::Generator; +use PVE::Multipath::Identity; +use PVE::Storage; +use PVE::Storage::LVMPlugin; + +sub scan_luns($cfg) { + my $ids = PVE::Multipath::Identity::collect(all => 1); + my $map_info = PVE::Multipath::map_info(); + my $members = PVE::Multipath::member_map($map_info); + # A map stands for its paths, of which a node with many LUNs has thousands. Leave them out + # before the disk list queries udev for each device. + my @names; + dir_glob_foreach( + '/sys/block', + '(?:sd[a-z]+|nvme\d+n\d+)', + sub($name, @) { push @names, $name if !$members->{$name} }, + ); + my $disks = PVE::Diskmanage::get_disks(\@names, 1, 0, 1); + my $maps = PVE::Diskmanage::get_multipath_disks($map_info); + my $sessions = PVE::ISCSI::Topology::device_map(); + my $status = eval { PVE::Multipath::path_status_by_wwid() } // {}; + my $nvme = PVE::Diskmanage::get_nvme_path_status(); + my $pv_vgs = PVE::Storage::LVMPlugin::cached_pv_volume_groups() // {}; + my $mpcfg = PVE::Multipath::Config::read_config(); + my %explicit = map { $_ => 1 } + PVE::Multipath::Config::wwids_for_node($mpcfg, PVE::INotify::nodename())->@*; + my %listed = map { $_ => 1 } PVE::Multipath::list_etc_multipath_wwids()->@*; + my $derived = PVE::Multipath::Generator::derived_wwids($cfg, $ids); + my (%storage, %expected, %vg_owner, %target_owner, %base_owner); + + for my $sid (sort keys %$derived) { + for my $wwid (keys $derived->{$sid}->{wwids}->%*) { + $storage{$wwid} //= $sid; + my $count = $derived->{$sid}->{wwids}->{$wwid}->{'paths-expected'}; + $expected{$wwid} = $count if defined($count) && $count > ($expected{$wwid} // 0); + } + } + for my $wwid (keys %explicit) { + my $count = $mpcfg->{ids}->{$wwid}->{'paths-expected'}; + $expected{$wwid} = $count if defined($count) && $count > ($expected{$wwid} // 0); + } + for my $sid (sort keys $cfg->{ids}->%*) { + my $scfg = $cfg->{ids}->{$sid}; + $vg_owner{ $scfg->{vgname} } //= $sid if defined($scfg->{vgname}); + $target_owner{ $scfg->{target} } //= $sid + if defined($scfg->{target}) && PVE::Storage::san_storage_serves_images($scfg); + $base_owner{$1} //= $sid if ($scfg->{base} // '') =~ m/(?:^|\.)dm-uuid-mpath-(.+)$/; + } + + my %rows; + my $add = sub($disk, $map = 0) { + my $name = basename(abs_path($disk->{devpath}) // $disk->{devpath}); + return + if !is_san_device( + $disk->{devpath}, + { + map => $map, + transport => $disk->{transport}, + session => $sessions->{$name}, + }, + ); + my $wwid = $map ? $disk->{wwn} : PVE::Multipath::Identity::path_wwid($ids, $name); + $wwid //= file_read_firstline("/sys/class/block/$name/wwid") if $name =~ /^nvme/; + return if !defined($wwid) || !length($wwid); + + my $row = $rows{$wwid} //= { + wwid => $wwid, + devpath => $disk->{devpath}, + size => $disk->{size}, + usage => 'unused', + multipath => $listed{$wwid} + || $explicit{$wwid} + || $storage{$wwid} ? 'pending' : 'none', + }; + my $link = stable_path($disk->{devpath}, $wwid, $map); + # The device and its by-id name come from one disk. A map replaces a single path. + if ($map || (defined($link) && !$row->{'by-id'})) { + delete $row->{'by-id'}; + $row->{devpath} = $disk->{devpath}; + $row->{size} = $disk->{size}; + $row->{'by-id'} = $link if defined($link); + } + if ($map) { + my $alias = basename($disk->{devpath}); + $row->{name} = $alias if $alias ne $wwid; + $row->{multipath} = + $explicit{$wwid} ? 'manual' + : $storage{$wwid} ? 'storage' + : 'unmanaged'; + } + $row->{multipath} = 'native' if $name =~ /^nvme\d+n\d+$/; + for my $key (qw(transport vendor model serial)) { + my $value = $disk->{$key}; + $row->{$key} //= $value if defined($value) && $value ne 'unknown'; + } + for my $member (map { basename($_) } ($disk->{slaves} // [$disk->{devpath}])->@*) { + my $session = $sessions->{$member} // next; + $row->{'iscsi-target'} //= $session->{target}; + $row->{'iscsi-lun'} //= $session->{lun}; + $row->{transport} = 'iscsi'; + } + my $used = $disk->{used}; + if (defined($used)) { + if ($used eq 'LVM' || $used eq 'LVM2_member') { + $row->{usage} = 'lvm'; + delete $row->{detail}; + } elsif ($row->{usage} ne 'lvm') { + $row->{usage} = 'other'; + $row->{detail} = $used; + } + } + my $vg = $pv_vgs->{ abs_path($disk->{devpath}) // $disk->{devpath} }; + if (defined($vg)) { + $row->{vgname} //= $vg; + $row->{usage} = 'lvm'; + delete $row->{detail}; + } + my $paths = $map ? $status->{$wwid} : $nvme->{$name}; + if ($paths) { + $row->{'path-list'} = $paths->{paths}; + $row->{'paths-total'} = scalar($paths->{paths}->@*); + $row->{'paths-active'} = $paths->{active}; + } elsif ($map) { + $row->{'paths-total'} = $disk->{paths}; + } + $row->{'multipath-storage'} = $storage{$wwid} if defined($storage{$wwid}); + # multipath entries and iSCSI portals are for maps, not for native NVMe multipath + $row->{'paths-expected'} = $expected{$wwid} + if defined($expected{$wwid}) && $row->{multipath} ne 'native'; + }; + for my $name (sort keys %$disks) { + my $disk = $disks->{$name}; + next if $members->{$name}; + $add->($disk); + } + $add->($maps->{$_}, 1) for sort keys %$maps; + for my $row (values %rows) { + if (defined($row->{'paths-active'})) { + $row->{state} = PVE::Multipath::derive_health( + $row->{'paths-active'}, + $row->{'paths-total'}, + $row->{'paths-expected'}, + ); + } + my $owner = $base_owner{ $row->{wwid} } // $vg_owner{ $row->{vgname} // '' } + // $target_owner{ $row->{'iscsi-target'} // '' }; + $row->{'used-by-storage'} = $owner if defined($owner); + } + return [map { $rows{$_} } sort keys %rows]; +} + +sub stable_path($device, $wwid, $map = 0) { + my $prefix = $map ? 'dm-uuid-mpath' : basename($device) =~ /^nvme/ ? 'nvme' : 'scsi'; + my ($link) = "/dev/disk/by-id/$prefix-$wwid" =~ m{\A(/dev/disk/by-id/[A-Za-z0-9_.:+-]+)\z}; + return undef if !defined($link); + return $link if (abs_path($link) // '') eq (abs_path($device) // $device); + return undef; +} + +sub is_san_device($device, $info = undef) { + if (!$info) { + my $path = abs_path($device) // $device; + return 0 if PVE::Diskmanage::is_partition($path); + my $name = basename($path); + my $transport; + if ($name =~ /^nvme/) { + $transport = 'nvme-' . (PVE::Diskmanage::get_nvme_transport("/sys/block/$name") // ''); + } elsif ((readlink("/sys/block/$name") // '') =~ m{/rport-\d+}) { + $transport = 'fc'; + } + $info = { + map => PVE::Multipath::device_map($path), + transport => $transport, + session => PVE::Diskmanage::is_iscsi_device($name), + }; + } + return + $info->{map} + || $info->{session} + || PVE::Diskmanage::is_remote_transport($info->{transport}) ? 1 : 0; +} + +1; diff --git a/src/test/Makefile b/src/test/Makefile index a758e593..6348c61e 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_san_lun_scan test_san_api test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -65,3 +65,9 @@ test_multipath_api: run_multipath_api_tests.pl test_san_taint: run_san_taint_tests.pl perl -T ./run_san_taint_tests.pl + +test_san_lun_scan: run_san_lun_scan_tests.pl + ./run_san_lun_scan_tests.pl + +test_san_api: run_san_api_tests.pl + ./run_san_api_tests.pl diff --git a/src/test/disk_tests/multipath/multipathd b/src/test/disk_tests/multipath/multipathd new file mode 100644 index 00000000..40243f91 --- /dev/null +++ b/src/test/disk_tests/multipath/multipathd @@ -0,0 +1,124 @@ +{ + "major_version": 0, + "minor_version": 1, + "maps": [{ + "name" : "mpatha", + "uuid" : "3600140500000000000000000000000001", + "sysfs" : "dm-0", + "failback" : "immediate", + "queueing" : "5 chk", + "paths" : 2, + "write_prot" : "rw", + "dm_st" : "active", + "features" : "1 queue_if_no_path", + "hwhandler" : "1 alua", + "action" : "", + "path_faults" : 0, + "vend" : "SANVEND ", + "prod" : "SANMODEL", + "rev" : "4.1", + "switch_grp" : 0, + "map_loads" : 1, + "total_q_time" : 0, + "q_timeouts" : 0, + "path_groups": [{ + "selector" : "service-time 0", + "pri" : 50, + "dm_st" : "active", + "marginal_st" : "normal", + "group" : 1, + "paths": [{ + "dev" : "sda", + "dev_t" : "8:0", + "dm_st" : "active", + "dev_st" : "running", + "chk_st" : "ready", + "checker" : "tur", + "pri" : 50, + "host_wwnn" : "0x20000024ff7d6a1c", + "target_wwnn" : "0x2000d039ea28bf00", + "host_wwpn" : "0x21000024ff7d6a1c", + "target_wwpn" : "0x2100d039ea28bf00", + "host_adapter" : "0000:41:00.0", + "lun_hex" : "0x0001000000000000", + "marginal_st" : "normal" + }, + { + "dev" : "sdb", + "dev_t" : "8:16", + "dm_st" : "active", + "dev_st" : "running", + "chk_st" : "ready", + "checker" : "tur", + "pri" : 50, + "host_wwnn" : "0x20000024ff7d6a1d", + "target_wwnn" : "0x2000d039ea28bf00", + "host_wwpn" : "0x21000024ff7d6a1d", + "target_wwpn" : "0x2200d039ea28bf00", + "host_adapter" : "0000:41:00.1", + "lun_hex" : "0x0001000000000000", + "marginal_st" : "normal" + }] + }] + }, + { + "name" : "mpathb", + "uuid" : "3600140500000000000000000000000002", + "sysfs" : "dm-1", + "failback" : "immediate", + "queueing" : "5 chk", + "paths" : 2, + "write_prot" : "rw", + "dm_st" : "active", + "features" : "1 queue_if_no_path", + "hwhandler" : "1 alua", + "action" : "", + "path_faults" : 3, + "vend" : "SANVEND ", + "prod" : "SANMODEL", + "rev" : "4.1", + "switch_grp" : 0, + "map_loads" : 1, + "total_q_time" : 0, + "q_timeouts" : 0, + "path_groups": [{ + "selector" : "service-time 0", + "pri" : 50, + "dm_st" : "active", + "marginal_st" : "normal", + "group" : 1, + "paths": [{ + "dev" : "sdc", + "dev_t" : "8:32", + "dm_st" : "active", + "dev_st" : "running", + "chk_st" : "ready", + "checker" : "tur", + "pri" : 50, + "host_wwnn" : "0x20000024ff7d6a1c", + "target_wwnn" : "0x2000d039ea28bf00", + "host_wwpn" : "0x21000024ff7d6a1c", + "target_wwpn" : "0x2100d039ea28bf00", + "host_adapter" : "0000:41:00.0", + "lun_hex" : "0x0002000000000000", + "marginal_st" : "normal" + }, + { + "dev" : "sdd", + "dev_t" : "8:48", + "dm_st" : "failed", + "dev_st" : "running", + "chk_st" : "faulty", + "checker" : "tur", + "pri" : 0, + "host_wwnn" : "0x20000024ff7d6a1d", + "target_wwnn" : "0x2000d039ea28bf00", + "host_wwpn" : "0x21000024ff7d6a1d", + "target_wwpn" : "0x2200d039ea28bf00", + "host_adapter" : "0000:41:00.1", + "lun_hex" : "0x0002000000000000", + "marginal_st" : "normal" + }] + }] + }] +} diff --git a/src/test/disk_tests/multipath/multipathd_status_expected.json b/src/test/disk_tests/multipath/multipathd_status_expected.json new file mode 100644 index 00000000..df5a24da --- /dev/null +++ b/src/test/disk_tests/multipath/multipathd_status_expected.json @@ -0,0 +1,40 @@ +{ + "3600140500000000000000000000000001": { + "active": 2, + "paths": [ + { + "device": "/dev/sda", + "state": "active", + "checker-state": "ready", + "host-wwpn": "0x21000024ff7d6a1c", + "target-wwpn": "0x2100d039ea28bf00" + }, + { + "device": "/dev/sdb", + "state": "active", + "checker-state": "ready", + "host-wwpn": "0x21000024ff7d6a1d", + "target-wwpn": "0x2200d039ea28bf00" + } + ] + }, + "3600140500000000000000000000000002": { + "active": 1, + "paths": [ + { + "device": "/dev/sdc", + "state": "active", + "checker-state": "ready", + "host-wwpn": "0x21000024ff7d6a1c", + "target-wwpn": "0x2100d039ea28bf00" + }, + { + "device": "/dev/sdd", + "state": "failed", + "checker-state": "faulty", + "host-wwpn": "0x21000024ff7d6a1d", + "target-wwpn": "0x2200d039ea28bf00" + } + ] + } +} diff --git a/src/test/disklist_test.pm b/src/test/disklist_test.pm index caa60e79..2f22fae8 100644 --- a/src/test/disklist_test.pm +++ b/src/test/disklist_test.pm @@ -70,6 +70,8 @@ sub mocked_run_command { $content = '{}'; } @$outputlines = split(/\n/, $content); + } elsif ($cmd->[0] =~ m/multipathd/i) { + @$outputlines = split(/\n/, read_test_file('multipathd')); } else { die "unexpected run_command call: '@$cmd', aborting\n"; } @@ -264,6 +266,16 @@ sub test_disk_list { ok($san_disks->{sda}, 'SAN inventory opts into iSCSI'); } + if (-f "disk_tests/$testcasedir/multipathd_status_expected.json") { + my $status = PVE::Multipath::path_status_by_wwid(); + my $expected_status = + decode_json(read_test_file('multipathd_status_expected.json')); + + print Dumper($status) if $print; + $testcount++; + is_deeply($status, $expected_status, 'multipath status should be the same'); + } + if (-f "disk_tests/$testcasedir/nvme_path_status_expected.json") { my $status = PVE::Diskmanage::get_nvme_path_status(); my $expected_status = decode_json(read_test_file('nvme_path_status_expected.json')); @@ -296,8 +308,11 @@ print("\tMocked get_sysdir_size\n"); $diskmanage_module->mock('assert_blockdev' => sub { return 1; }); print("\tMocked assert_blockdev\n"); $multipath_module = Test::MockModule->new('PVE::Multipath', no_auto => 1); +$multipath_module->mock('run_command' => \&mocked_run_command); +$multipath_module->mock('is_supported' => sub { return 1; }); $multipath_module->mock('dir_glob_foreach' => \&mocked_dir_glob_foreach); $multipath_module->mock('file_read_firstline' => \&mocked_file_read_firstline); +print("\tMocked PVE::Multipath run_command and is_supported\n"); $diskmanage_module->mock( 'dir_is_empty' => sub { my $val = shift; diff --git a/src/test/run_san_api_tests.pl b/src/test/run_san_api_tests.pl new file mode 100755 index 00000000..4bd7d2d7 --- /dev/null +++ b/src/test/run_san_api_tests.pl @@ -0,0 +1,24 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Test::More; +use lib qw(. ..); +use PVE::API2::Storage::Scan; + +for my $case (['PVE::API2::Storage::Scan', 'sanlunscan']) { + my ($class, @names) = @$case; + for my $name (@names) { + is_deeply( + $class->map_method_by_name($name)->{permissions}->{check}, + [ + 'or', + ['perm', '/nodes/{node}', ['Sys.Audit']], + ['perm', '/storage', ['Datastore.Allocate']], + ], + "$class $name read permission", + ); + } +} + +done_testing(); diff --git a/src/test/run_san_lun_scan_tests.pl b/src/test/run_san_lun_scan_tests.pl new file mode 100755 index 00000000..b9e128cc --- /dev/null +++ b/src/test/run_san_lun_scan_tests.pl @@ -0,0 +1,197 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use lib qw(..); +use Test::More; +use Test::MockModule; + +use PVE::Storage::SAN; +use PVE::API2::Storage::Scan; + +my $disk = Test::MockModule->new('PVE::Diskmanage'); +my $mp = Test::MockModule->new('PVE::Multipath'); +my $id = Test::MockModule->new('PVE::Multipath::Identity'); +my $config = Test::MockModule->new('PVE::Multipath::Config'); +my $storage = Test::MockModule->new('PVE::Storage'); +my $iscsi = Test::MockModule->new('PVE::ISCSI::Topology'); +my $scan = Test::MockModule->new('PVE::Storage::SAN'); +my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); +my $hints; +$lvm->mock(file_get_contents => sub { $hints // die "No such file\n" }); +$lvm->mock(abs_path => sub { $_[0] }); +$lvm->mock(run_command => sub { die "unexpected LVM device probe\n" }); + +my $disks = { + sda => { devpath => '/dev/sda', size => 100, transport => 'sata' }, + sdb => { devpath => '/dev/sdb', size => 200, transport => 'iscsi' }, + sdc => { devpath => '/dev/sdc', size => 200, transport => 'iscsi', used => 'LVM2_member' }, + sdd => + { devpath => '/dev/sdd', size => 300, transport => 'fc', multipath => '/dev/mapper/lun' }, + sde => { devpath => '/dev/sde', size => 400, transport => 'fc' }, +}; +my $maps = { + 'dm-0' => { + devpath => '/dev/mapper/lun', + size => 300, + wwn => '3600map', + paths => 2, + slaves => ['/dev/sdd', '/dev/sdf'], + transport => 'fc', + }, +}; +$id->mock(collect => sub { {} }); +$id->mock(path_wwid => sub { $_[1] =~ /^(sdb|sdc)$/ ? '3600plain' : undef }); +$config->mock(read_config => sub { { ids => { '3600map' => { type => 'wwid' } } } }); +my $generator = Test::MockModule->new('PVE::Multipath::Generator'); +$generator->mock(derived_wwids => sub { {} }); +$iscsi->mock( + device_map => sub { + return { map { $_ => { target => 'iqn.test', lun => 1 } } qw(sdb sdc) }; + }, +); +my @listed; +$scan->redefine( + dir_glob_foreach => sub { + my ($dir, $regex, $code) = @_; + return PVE::Tools::dir_glob_foreach($dir, $regex, $code) if $dir ne '/sys/block'; + $code->($_) for grep { /^$regex$/ } qw(sda sdb sdc sdd sde sdf dm-0 loop0 nvme0n1); + }, +); +$disk->mock( + get_disks => sub { + ok($_[3], 'SAN inventory opts into iSCSI'); + @listed = sort $_[0]->@*; + return $disks; + }, + get_multipath_disks => sub { $maps }, + get_nvme_path_status => sub { {} }, + run_command => sub { die 'unexpected device probe' }, +); +$mp->mock(list_etc_multipath_wwids => sub { [] }); +$mp->mock( + map_info => sub { + return { + map { + $_ => { + wwid => $maps->{$_}->{wwn}, + slaves => [map { s{^/dev/}{}r } $maps->{$_}->{slaves}->@*], + } + } keys %$maps + }; + }, +); +$mp->mock( + path_status_by_wwid => sub { + return { + '3600map' => { + active => 1, + paths => [ + { device => '/dev/sdd', state => 'active', 'checker-state' => 'ghost' }, + { device => '/dev/sdf', state => 'failed', 'checker-state' => 'faulty' }, + ], + }, + }; + }, +); +$scan->mock(abs_path => sub { $_[0] }); +$scan->mock(stable_path => sub { '/dev/disk/by-id/scsi-' . $_[1] }); +my $cfg = + { ids => { direct => { type => 'iscsi', target => 'iqn.test', content => { images => 1 } } } }; +my $rows = PVE::Storage::SAN::scan_luns($cfg); +is(scalar(@$rows), 2, 'only identified SAN LUNs, one row per identity'); +is_deeply( + \@listed, + [qw(nvme0n1 sda sdb sdc sde)], + 'the paths of a map are left out of the disk list', +); +my %row = map { $_->{wwid} => $_ } @$rows; +is($row{'3600plain'}->{usage}, 'lvm', 'positive usage from any path survives folding'); +is($row{'3600plain'}->{'used-by-storage'}, 'direct', 'a storage serving the LUNs uses them'); +{ + local $cfg->{ids}->{direct}->{content} = { none => 1 }; + my %plain = map { $_->{wwid} => $_ } PVE::Storage::SAN::scan_luns($cfg)->@*; + is($plain{'3600plain'}->{'used-by-storage'}, undef, 'a connection storage does not'); +} +ok(!exists($row{'3600plain'}->{state}), 'single path devices have no multipath health'); +is($row{'3600map'}->{state}, 'degraded', 'final state counts usable paths'); +is($row{'3600map'}->{multipath}, 'manual', 'explicit map'); +is($row{'3600map'}->{devpath}, '/dev/mapper/lun', 'map is the selected device'); +my $schema = PVE::API2::Storage::Scan->map_method_by_name('sanlunscan')->{returns}; +eval { PVE::JSONSchema::validate($rows, $schema) }; +is($@, '', 'rows match the public schema'); +is($row{'3600map'}->{name}, 'lun', 'map alias is exposed'); +$maps->{'dm-0'}->{devpath} = '/dev/mapper/3600map'; +$rows = PVE::Storage::SAN::scan_luns($cfg); +my ($unnamed) = grep { $_->{wwid} eq '3600map' } @$rows; +ok(!exists($unnamed->{name}), 'WWID is not repeated as an alias'); +$maps->{'dm-0'}->{devpath} = '/dev/mapper/lun'; + +{ + my $healthy = { + '3600map' => { + active => 2, + paths => [map { { device => $_, state => 'active' } } qw(/dev/sdd /dev/sdf)], + }, + }; + $mp->mock(path_status_by_wwid => sub { $healthy }); + my ($map) = grep { $_->{wwid} eq '3600map' } PVE::Storage::SAN::scan_luns($cfg)->@*; + is($map->{state}, 'optimal', 'all paths usable without an expected count'); + $config->mock( + read_config => + sub { { ids => { '3600map' => { type => 'wwid', 'paths-expected' => 3 } } } }, + ); + ($map) = grep { $_->{wwid} eq '3600map' } PVE::Storage::SAN::scan_luns($cfg)->@*; + is($map->{'paths-expected'}, 3, 'expected count of the entry'); + is($map->{state}, 'degraded', 'fewer paths than the entry expects'); + $config->mock(read_config => sub { { ids => { '3600map' => { type => 'wwid' } } } }); +} + +$mp->mock(path_status_by_wwid => sub { {} }); +$rows = PVE::Storage::SAN::scan_luns($cfg); +my ($unobserved) = grep { $_->{wwid} eq '3600map' } @$rows; +is($unobserved->{'paths-total'}, 2, 'map membership supplies total without daemon status'); +ok(!exists($unobserved->{'paths-active'}), 'no invented active count'); +ok(!exists($unobserved->{state}), 'no false failed state'); + +$cfg->{ids}->{data} = { type => 'lvm', vgname => 'data' }; +$hints = "hints_version: 2.1\nscan:/dev/mapper/lun pvid:abc devn:253:0 vg:data\n"; +$rows = PVE::Storage::SAN::scan_luns($cfg); +my ($cached) = grep { $_->{wwid} eq '3600map' } @$rows; +is($cached->{vgname}, 'data', 'hints supply the display VG'); +is($cached->{usage}, 'lvm', 'hints also supply cached usage evidence'); +is($cached->{'used-by-storage'}, 'data', 'VG allows storage attribution'); +for my $contents (undef, '', "scan=/dev/mapper/lun vg=data\n") { + $hints = $contents; + $rows = PVE::Storage::SAN::scan_luns($cfg); + is(scalar(@$rows), 2, 'missing or unrecognized hints do not lose LUNs'); + my ($uncached) = grep { $_->{wwid} eq '3600map' } @$rows; + ok(!exists($uncached->{vgname}), 'VG name is absent without a hint'); + ok(!exists($uncached->{'used-by-storage'}), 'VG-derived attribution is also absent'); + is($uncached->{usage}, 'unused', 'usage falls back to other cached evidence'); +} + +$disks->{sdc}->{used} = 'ext4'; +$maps = {}; +$rows = PVE::Storage::SAN::scan_luns($cfg); +is($rows->[0]->{usage}, 'other', 'foreign data from a sibling path is retained'); +is($rows->[0]->{detail}, 'ext4', 'signature detail'); +$scan->mock(stable_path => sub { $_[0] eq '/dev/sdc' ? '/dev/disk/by-id/scsi-3600plain' : undef }); +$rows = PVE::Storage::SAN::scan_luns($cfg); +is($rows->[0]->{'by-id'}, '/dev/disk/by-id/scsi-3600plain', 'by-id link of the second path'); +is($rows->[0]->{devpath}, '/dev/sdc', 'the row names the path that its by-id link resolves to'); + +$disks = { nvme0n1 => { devpath => '/dev/nvme0n1', size => 100, transport => 'nvme-fc' } }; +$scan->mock(file_read_firstline => sub { 'eui.0123456789abcdef' }); +$scan->unmock('stable_path'); +$scan->mock(abs_path => sub { '/dev/nvme0n1' }); +$rows = PVE::Storage::SAN::scan_luns($cfg); +is($rows->[0]->{multipath}, 'native', 'NVMe uses kernel-native multipath'); +is($rows->[0]->{'by-id'}, '/dev/disk/by-id/nvme-eui.0123456789abcdef', 'NVMe stable name'); +eval { PVE::JSONSchema::validate($rows, $schema) }; +is($@, '', 'native row matches the API schema'); + +$disk->mock(is_partition => sub { 1 }); +ok(!PVE::Storage::SAN::is_san_device('/dev/sdb1'), 'a partition is never a LUN'); + +done_testing();