From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 3BE851FF0B7 for ; Fri, 13 Nov 2026 04:21:40 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 499A721CB2; Fri, 13 Nov 2026 04:05:43 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v4 storage 20/36] disks: lvm: allow volume groups on SAN LUNs and multipath maps Date: Fri, 13 Nov 2026 04:00:38 +0100 Message-ID: <20261113030146.1559030-21-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com> References: <20261113030146.1559030-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794538914876 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.235 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: E7Q4ELGHEEKKOVHGDLKKMW76DFYKMRGJ X-Message-ID-Hash: E7Q4ELGHEEKKOVHGDLKKMW76DFYKMRGJ X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The node disk API creates volume groups on local disks only. It takes a Fibre Channel LUN for one, but not a multipath map, and adds the storage as local to the node. Accept the LUNs of the SAN inventory and add their storage as shared and without node restriction. Refuse an existing storage ID for them: adding the node to a storage with the same volume group name is meant for local volume groups and would combine two different volume groups in one shared storage. Initializing a LUN, here or as base volume of a new LVM storage, now takes a cluster-wide lock per WWID and refuses a LUN that a storage references through its volume group, as base volume or as iSCSI target serving images. Co-authored-by: Dietmar Maurer Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Disks/LVM.pm | 56 +++--- src/PVE/Storage/LVMPlugin.pm | 74 +++++++- src/PVE/Storage/SAN.pm | 97 +++++++++- src/test/Makefile | 8 +- src/test/disk_api_test.pm | 235 ++++++++++++++++++++++++ src/test/run_disk_api_tests.pl | 11 ++ src/test/run_san_nvme_tests.pl | 91 +++++++++ src/test/run_san_taint_tests.pl | 26 +++ src/test/run_test_lvm_base_conflicts.pl | 232 ++++++++++++++++------- src/test/run_test_lvm_create.pl | 14 ++ 10 files changed, 746 insertions(+), 98 deletions(-) create mode 100644 src/test/disk_api_test.pm create mode 100755 src/test/run_disk_api_tests.pl create mode 100755 src/test/run_san_nvme_tests.pl diff --git a/src/PVE/API2/Disks/LVM.pm b/src/PVE/API2/Disks/LVM.pm index 6101743b..a0ab6d69 100644 --- a/src/PVE/API2/Disks/LVM.pm +++ b/src/PVE/API2/Disks/LVM.pm @@ -4,6 +4,7 @@ use strict; use warnings; use PVE::Storage::LVMPlugin; +use PVE::Storage::SAN; use PVE::Diskmanage; use PVE::JSONSchema qw(get_standard_option); use PVE::API2::Storage::Config; @@ -130,7 +131,8 @@ __PACKAGE__->register_method({ name => get_standard_option('pve-storage-id'), device => { type => 'string', - description => 'The block device you want to create the volume group on', + description => 'The block device you want to create the volume group on.' + . ' Multipath mapped devices are supported.', }, add_storage => { description => "Configure storage using the Volume Group", @@ -152,48 +154,54 @@ __PACKAGE__->register_method({ my $node = $param->{node}; $dev = PVE::Diskmanage::verify_blockdev_path($dev); - PVE::Diskmanage::assert_disk_unused($dev); + my $san = PVE::Storage::SAN::is_san_device($dev); + PVE::Diskmanage::assert_disk_unused($dev) if !$san; my $storage_params = { type => 'lvm', vgname => $name, storage => $name, content => 'rootdir,images', - shared => 0, - nodes => $node, + shared => $san ? 1 : 0, + $san ? () : (nodes => $node), }; my $verify_params = [qw(vgname)]; if ($param->{add_storage}) { $rpcenv->check($user, "/storage", ['Datastore.Allocate']); + # only local volume groups of the same name may share a storage that exists already + die "storage ID '$name' already defined\n" + if $san && PVE::Storage::storage_config(PVE::Storage::config(), $name, 1); + # reserve the name and add as disabled, will be enabled below if creation works out PVE::API2::Storage::Config->create_or_update( $name, $node, $storage_params, $verify_params, 1, ); } + my $create_group = sub { + PVE::Diskmanage::assert_disk_unused($dev) if !$san; + die "volume group with name '${name}' already exists on node '${node}'\n" + if PVE::Storage::LVMPlugin::lvm_vgs()->{$name}; + + if (PVE::Diskmanage::is_partition($dev)) { + eval { PVE::Diskmanage::change_parttype($dev, '8E00'); }; + warn $@ if $@; + } + + PVE::Storage::LVMPlugin::lvm_create_volume_group($dev, $name, $san); + + PVE::Diskmanage::udevadm_trigger($dev); + + if ($param->{add_storage}) { + PVE::API2::Storage::Config->create_or_update( + $name, $node, $storage_params, $verify_params, + ); + } + }; my $worker = sub { - PVE::Diskmanage::locked_disk_action(sub { - PVE::Diskmanage::assert_disk_unused($dev); - die "volume group with name '${name}' already exists on node '${node}'\n" - if PVE::Storage::LVMPlugin::lvm_vgs()->{$name}; - - if (PVE::Diskmanage::is_partition($dev)) { - eval { PVE::Diskmanage::change_parttype($dev, '8E00'); }; - warn $@ if $@; - } - - PVE::Storage::LVMPlugin::lvm_create_volume_group($dev, $name); - - PVE::Diskmanage::udevadm_trigger($dev); - - if ($param->{add_storage}) { - PVE::API2::Storage::Config->create_or_update( - $name, $node, $storage_params, $verify_params, - ); - } - }); + $san ? $create_group->() : PVE::Diskmanage::locked_disk_action($create_group); }; return $rpcenv->fork_worker('lvmcreate', $name, $user, $worker); diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm index ee64b782..c9e52b4c 100644 --- a/src/PVE/Storage/LVMPlugin.pm +++ b/src/PVE/Storage/LVMPlugin.pm @@ -16,6 +16,7 @@ use PVE::Exception qw(raise_param_exc); use PVE::Format qw(render_bytes render_duration); use PVE::INotify; use PVE::JSONSchema qw(get_standard_option); +use PVE::Multipath; use PVE::RESTEnvironment qw(log_warn); use PVE::Tools qw(run_command file_get_contents file_read_firstline trim); @@ -147,6 +148,37 @@ sub cached_pv_volume_groups { return $res; } +sub lun_volume_groups { + my ($devices) = @_; + my $output = ''; + run_command( + [ + '/sbin/pvs', + '--nohints', + '--readonly', + '--devices', + join(',', @$devices), + '--reportformat', + 'json', + '--options', + 'pv_name,vg_name', + ], + timeout => 10, + outfunc => sub { $output .= "$_[0]\n" }, + ); + my $data = decode_json($output); + my $pvs = $data->{report}->[0]->{pv}; + die "unable to inspect volume groups - invalid pvs report\n" if ref($pvs) ne 'ARRAY'; + my %vgs; + for my $pv (@$pvs) { + my $vg = $pv->{vg_name}; + die "unable to inspect volume groups - missing volume group metadata\n" + if !defined($vg) || $vg eq '[unknown]'; + $vgs{$vg} = 1 if length($vg); + } + return [sort keys %vgs]; +} + sub clear_first_sector { my ($dev) = shift; @@ -157,10 +189,16 @@ sub clear_first_sector { } } -sub lvm_create_volume_group { - my ($device, $vgname, $shared, $guard) = @_; - +my sub create_volume_group { + my ($device, $vgname, $lun, $guard) = @_; $device = PVE::Diskmanage::verify_blockdev_path($device); + my $fh; + if ($lun) { + sysopen($fh, $lun->{'by-id'}, O_RDWR) or die "cannot open '$device' - $!\n"; + } + my $check_name = sub { + PVE::Storage::SAN::assert_same_device($lun, $fh) if $lun; + }; my $refuse_data = sub { my (@types) = @_; die "device '$device' contains data (" @@ -178,7 +216,7 @@ sub lvm_create_volume_group { if !defined($size) || $size !~ m/^\d+$/ || !$size; die "unable to probe '$device' - missing device mapper identity\n" if basename($device) =~ m/^dm-\d+$/ && !defined($uuid); - if (defined($uuid) && $uuid =~ m/^mpath-/) { + if (PVE::Multipath::parse_dm_uuid($uuid)) { # A queueing map without a running path can stall even a read-only probe. my $running = 0; PVE::Tools::dir_glob_foreach( @@ -247,26 +285,48 @@ sub lvm_create_volume_group { } $refuse_data->(keys %signatures) if %signatures; + # Only initializing the device needs the checks, an existing volume group on it can be added. $guard->{check_storage}->($device) if $guard && $guard->{check_storage}; - clear_first_sector($device); # else pvcreate fails + PVE::Storage::SAN::assert_lun_unreferenced( + PVE::Storage::config(), + $lun, + $guard ? $guard->{exempt} : undef, + ) if $lun; + + $check_name->(); + clear_first_sector($lun ? '/proc/' . $$ . '/fd/' . fileno($fh) : $device); # we use --metadatasize 250k, which reseults in "pe_start = 512" # so pe_start is aligned on a 128k boundary (advantage for SSDs) my $cmd = ['/sbin/pvcreate', '--metadatasize', '250k', $device]; - run_command($cmd, errmsg => "pvcreate '$device' error"); + $check_name->(); + run_command($cmd, errmsg => "pvcreate '$device' error", $lun ? (timeout => 30) : ()); $cmd = ['/sbin/vgcreate', $vgname, $device]; # push @$cmd, '-c', 'y' if $shared; # we do not use this yet + $check_name->(); run_command( $cmd, + $lun ? (timeout => 30) : (), errmsg => "vgcreate $vgname $device error", errfunc => $ignore_no_medium_warnings, outfunc => $ignore_no_medium_warnings, ); } +sub lvm_create_volume_group { + my ($device, $vgname, $shared, $guard) = @_; + require PVE::Storage::SAN; + my $lun = PVE::Storage::SAN::lun_for_device($device); + return create_volume_group($device, $vgname, undef, $guard) if !$lun; + PVE::Storage::SAN::with_lun_lock( + $lun, + sub { create_volume_group($device, $vgname, $lun, $guard) }, + ); +} + sub lvm_destroy_volume_group { my ($vgname) = @_; @@ -1130,7 +1190,7 @@ sub on_add_hook { $path, $scfg->{vgname}, $scfg->{shared}, - { check_storage => $check_target }, + { check_storage => $check_target, exempt => $storeid }, ); } diff --git a/src/PVE/Storage/SAN.pm b/src/PVE/Storage/SAN.pm index f13d9633..210c3a78 100644 --- a/src/PVE/Storage/SAN.pm +++ b/src/PVE/Storage/SAN.pm @@ -5,6 +5,7 @@ use v5.36; use Cwd qw(abs_path); use File::Basename qw(basename); +use PVE::Cluster; use PVE::INotify; use PVE::Diskmanage; @@ -15,7 +16,7 @@ use PVE::Multipath::Config; use PVE::Multipath::Identity; use PVE::Storage; use PVE::Storage::LVMPlugin; -use PVE::Tools qw(file_read_firstline); +use PVE::Tools qw(file_read_firstline dir_glob_foreach); sub scan_luns($cfg) { my $ids = PVE::Multipath::Identity::collect(all => 1); @@ -186,4 +187,98 @@ sub is_san_device($device, $info = undef) { || PVE::Diskmanage::is_remote_transport($info->{transport}) ? 1 : 0; } +sub lun_for_device($device) { + $device = PVE::Diskmanage::verify_blockdev_path($device); + return undef if !is_san_device($device); + my $rows = scan_luns(PVE::Storage::config()); + for my $lun (@$rows) { + next if (abs_path($lun->{devpath}) // '') ne $device; + die "cannot use '$device' - no stable name under /dev/disk/by-id\n" if !$lun->{'by-id'}; + # read from sysfs, but used in command lines and in the name of the cluster lock + ($lun->{devpath}) = $lun->{devpath} =~ m{\A(/dev/[A-Za-z0-9_.:+/-]+)\z} + or die "cannot use '$device' - unexpected device path\n"; + ($lun->{wwid}) = $lun->{wwid} =~ m/\A([A-Za-z0-9_.:+=-]+)\z/ + or die "cannot use '$device' - unexpected WWID\n"; + return $lun; + } + die "cannot use '$device' - select an identified LUN or its multipath map\n"; +} + +sub lun_partitions($lun) { + my $device = PVE::Diskmanage::verify_blockdev_path($lun->{devpath}); + my $name = basename($device); + my @parts; + dir_glob_foreach( + "/sys/class/block/$name", + "\Q$name\E.+", + sub($part, @) { + my ($path) = "/dev/$part" =~ m{\A(/dev/[A-Za-z0-9_.+-]+)\z}; + push @parts, $path if defined($path) && -e "/sys/class/block/$name/$part/partition"; + }, + ); + dir_glob_foreach( + "/sys/class/block/$name/holders", + 'dm-\\d+', + sub($part, @) { + my $map = PVE::Multipath::parse_dm_uuid( + file_read_firstline("/sys/class/block/$part/dm/uuid")); + my ($path) = "/dev/$part" =~ m{\A(/dev/dm-\d+)\z}; + push @parts, $path + if defined($path) && $map && $map->{partition} && $map->{wwid} eq $lun->{wwid}; + }, + ); + return \@parts; +} + +sub assert_lun_unreferenced($cfg, $lun, $exempt = undef) { + my %vgs = map { $_ => 1 } PVE::Storage::LVMPlugin::lun_volume_groups( + [$lun->{devpath}, lun_partitions($lun)->@*], + )->@*; + # the paths of a LUN can belong to more than one target + my %targets = map { $_->{target} => 1 } + grep { $_->{wwid} eq $lun->{wwid} && defined($_->{target}) } + values PVE::Multipath::Identity::collect(all => 1)->{paths}->%*; + for my $sid (sort keys $cfg->{ids}->%*) { + next if defined($exempt) && $sid eq $exempt; + my $scfg = $cfg->{ids}->{$sid}; + my $type = $scfg->{type}; + if (($type eq 'lvm' || $type eq 'lvmthin') && $vgs{ $scfg->{vgname} // '' }) { + die "cannot use '$lun->{devpath}' - volume group is used by storage '$sid'\n"; + } + if ( + defined($scfg->{target}) + && $targets{ $scfg->{target} } + && PVE::Storage::san_storage_serves_images($scfg) + ) { + die "cannot use '$lun->{devpath}' - storage '$sid' serves images from this target\n"; + } + next if $type ne 'lvm' || !$scfg->{base}; + my (undef, $volname) = PVE::Storage::Plugin::parse_volume_id($scfg->{base}); + my ($wwid) = $volname =~ m/(?:^|\.)(?:dm-uuid-mpath|scsi|nvme)-(.+)$/; + my $matches = defined($wwid) && $wwid eq $lun->{wwid}; + if (!$matches) { + my $path = eval { abs_path(PVE::Storage::path($cfg, $scfg->{base})) }; + my $device = abs_path($lun->{devpath}); + $matches = defined($path) && defined($device) && $path eq $device; + } + die "cannot use '$lun->{devpath}' - storage '$sid' uses this LUN as its base volume\n" + if $matches; + } + return; +} + +# Serializes writes to a LUN cluster-wide. The code sees the current storage configuration. +sub with_lun_lock($lun, $code) { + my $key = $lun->{wwid} =~ s/([^A-Za-z0-9_.:-])/sprintf('=%02x', ord($1))/ger; + PVE::Cluster::cfs_lock_domain("san-$key", undef, $code); + die $@ if $@; +} + +sub assert_same_device($lun, $fh) { + my @held = stat($fh); + my @named = stat($lun->{'by-id'}); + die "cannot use '$lun->{devpath}' - its by-id name changed\n" + if !@held || !@named || $held[6] != $named[6]; +} + 1; diff --git a/src/test/Makefile b/src/test/Makefile index 6348c61e..e453951a 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_san_lun_scan test_san_api +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_multipath test_san_libraries test_lun_identity test_lvm_base_activation test_multipath_api test_san_taint test_san_lun_scan test_san_api test_disk_api test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -69,5 +69,9 @@ test_san_taint: run_san_taint_tests.pl test_san_lun_scan: run_san_lun_scan_tests.pl ./run_san_lun_scan_tests.pl -test_san_api: run_san_api_tests.pl +test_san_api: run_san_api_tests.pl run_san_nvme_tests.pl ./run_san_api_tests.pl + ./run_san_nvme_tests.pl + +test_disk_api: run_disk_api_tests.pl + ./run_disk_api_tests.pl diff --git a/src/test/disk_api_test.pm b/src/test/disk_api_test.pm new file mode 100644 index 00000000..32ab5c80 --- /dev/null +++ b/src/test/disk_api_test.pm @@ -0,0 +1,235 @@ +package PVE::API2::Disks::Test; + +use strict; +use warnings; + +use lib qw(..); + +use PVE::API2::Disks; +use PVE::API2::Disks::LVM; +use PVE::Diskmanage; +use PVE::Storage; +use PVE::Storage::LVMPlugin; + +use Test::MockModule; +use Test::More; + +my @transport_cases = ( + ['none', undef], + ['unknown', 'unknown'], + ['ata', 'ata'], + ['sata', 'sata'], + ['sas', 'sas'], + ['spi', 'spi'], + ['sbp', 'sbp'], + ['usb', 'usb'], + ['fc', 'fc'], + ['fcoe', 'fcoe'], + ['iscsi', 'iscsi'], + ['nvme', 'nvme'], + ['virtio', 'virtio'], + ['nvme-fc', 'nvme-fc'], + ['nvme-tcp', 'nvme-tcp'], + ['nvme-rdma', 'nvme-rdma'], + ['nvme-loop', 'nvme-loop'], +); + +my $disks = {}; +my @all; +for my $case (@transport_cases) { + my ($name, $transport) = $case->@*; + my $devpath = "/dev/test-$name"; + my $entry = { + devpath => $devpath, + gpt => 0, + osdid => -1, + }; + $entry->{transport} = $transport if defined($transport); + $disks->{$name} = $entry; + push @all, $devpath; +} + +my $diskmanage_module = Test::MockModule->new('PVE::Diskmanage', no_auto => 1); +$diskmanage_module->mock('get_disks' => sub { return $disks }); + +my $list_handler = PVE::API2::Disks->map_method_by_name('list')->{code}; +my $list_devpaths = sub { + my ($param) = @_; + my $result = $list_handler->({ node => 'test-node', skipsmart => 1, $param->%* }); + return [sort map { $_->{devpath} } $result->@*]; +}; + +is_deeply( + $list_devpaths->({}), [sort @all], 'the disk list uses the default disk inventory', +); + +is_deeply( + $list_devpaths->({ type => 'unused' }), + [sort @all], + 'unused disk selection preserves transport visibility', +); + +$diskmanage_module->mock( + 'get_disks' => sub { + return { + 'fc-disk' => { + devpath => '/dev/fc-disk', + gpt => 0, + osdid => -1, + transport => 'fc', + }, + 'fc-partition' => { + devpath => '/dev/fc-disk1', + gpt => 0, + osdid => -1, + parent => '/dev/fc-disk', + }, + 'loop-disk' => { + devpath => '/dev/loop-disk', + gpt => 0, + osdid => -1, + transport => 'nvme-loop', + }, + 'loop-partition' => { + devpath => '/dev/loop-disk1', + gpt => 0, + osdid => -1, + parent => '/dev/loop-disk', + }, + }; + }, +); + +is_deeply( + $list_devpaths->({ 'include-partitions' => 1 }), + ['/dev/fc-disk', '/dev/fc-disk1', '/dev/loop-disk', '/dev/loop-disk1'], + 'the plain list keeps FC disks and their partitions', +); + +is_deeply( + $list_devpaths->({ 'include-partitions' => 1, type => 'unused' }), + ['/dev/fc-disk', '/dev/fc-disk1', '/dev/loop-disk', '/dev/loop-disk1'], + 'unused partitions preserve transport visibility', +); + +subtest 'volume group creation leaves SAN locking to the LVM helper' => sub { + my $storage = Test::MockModule->new('PVE::Storage'); + my $disk = $diskmanage_module; + my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); + my $rpc = Test::MockModule->new('PVE::RPCEnvironment'); + my (@stack, @created); + my $lock = sub { + my ($name, $code) = @_; + push @stack, $name; + my $res = eval { $code->() }; + my $err = $@; + pop @stack; + die $err if $err; + return $res; + }; + $storage->mock(lock_storage_config => sub { $lock->('storage.cfg', $_[0]) }); + $disk->mock(locked_disk_action => sub { $lock->('disk', $_[0]) }); + $disk->mock(verify_blockdev_path => sub { $_[0] }); + $disk->mock(assert_disk_unused => sub { }); + $disk->mock(is_partition => sub { 0 }); + $disk->mock(udevadm_trigger => sub { }); + $lvm->mock(lvm_vgs => sub { {} }); + $lvm->mock( + lvm_create_volume_group => sub { + is(scalar(@_), 3, 'disk API does not exempt a storage'); + push @created, [$_[0], join('>', @stack)]; + }, + ); + { + + package MultipathTestRPC; + sub get_user { 'root@pam' } + sub check { 1 } + sub fork_worker { $_[4]->(); return 'test-upid' } + } + $rpc->mock(get => sub { bless {}, 'MultipathTestRPC' }); + + my ($is_map, $is_san) = (1, 0); + my $san = Test::MockModule->new('PVE::Storage::SAN'); + $san->mock(is_san_device => sub { $is_map || $is_san }); + PVE::API2::Disks::LVM->create({ node => 'n1', name => 'vg1', device => '/dev/dm-3' }); + ($is_map, $is_san) = (0, 1); + PVE::API2::Disks::LVM->create({ node => 'n1', name => 'vg2', device => '/dev/sdc' }); + ($is_map, $is_san) = (0, 0); + PVE::API2::Disks::LVM->create({ node => 'n1', name => 'vg3', device => '/dev/sdb' }); + is_deeply( + \@created, + [ + ['/dev/dm-3', ''], ['/dev/sdc', ''], ['/dev/sdb', 'disk'], + ], + 'SAN writes use no local disk or storage configuration lock', + ); + my $config_api = Test::MockModule->new('PVE::API2::Storage::Config'); + my @params; + $config_api->mock(create_or_update => sub { push @params, $_[3] }); + my $cfg = + { ids => { other => { type => 'lvm', vgname => 'other', nodes => { n2 => 1 } } } }; + $storage->mock(config => sub { $cfg }); + $is_san = 1; + PVE::API2::Disks::LVM->create({ + node => 'n1', + name => 'vg4', + device => '/dev/sdc', + add_storage => 1, + }); + is($params[-1]->{shared}, 1, 'SAN storage is shared'); + ok(!exists($params[-1]->{nodes}), 'SAN storage covers all nodes'); + @created = (); + eval { + PVE::API2::Disks::LVM->create({ + node => 'n1', + name => 'other', + device => '/dev/sdc', + add_storage => 1, + }); + }; + like($@, qr/already defined/, 'SAN volume group does not join the storage of another node'); + is_deeply(\@created, [], 'existing storage ID prevents initialization'); + $is_san = 0; + PVE::API2::Disks::LVM->create({ + node => 'n1', + name => 'vg5', + device => '/dev/sdb', + add_storage => 1, + }); + is($params[-1]->{shared}, 0, 'local storage is not shared'); + is($params[-1]->{nodes}, 'n1', 'local storage remains node-restricted'); + @created = (); + $disk->mock(assert_disk_unused => sub { die "device is already in use\n" }); + eval { + PVE::API2::Disks::LVM->create({ + node => 'n1', + name => 'vg', + device => '/dev/sdc', + }); + }; + like($@, qr/already in use/, 'disk API rejects a used inventory entry'); + is_deeply(\@created, [], 'used inventory prevents initialization'); + $disk->mock(assert_disk_unused => sub { }); + $lvm->mock(lvm_vgs => sub { { vg => {} } }); + eval { + PVE::API2::Disks::LVM->create({ + node => 'n1', + name => 'vg', + device => '/dev/sdc', + }); + }; + like( + $@, + qr/volume group with name 'vg' already exists/, + 'stale inventory does not permit an existing VG', + ); + is_deeply(\@created, [], 'existing VG prevents initialization'); + $disk->unmock($_) + for qw( + locked_disk_action verify_blockdev_path assert_disk_unused is_partition + udevadm_trigger + ); +}; + +done_testing(); diff --git a/src/test/run_disk_api_tests.pl b/src/test/run_disk_api_tests.pl new file mode 100755 index 00000000..75fc98c6 --- /dev/null +++ b/src/test/run_disk_api_tests.pl @@ -0,0 +1,11 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use TAP::Harness; + +my $harness = TAP::Harness->new({ verbosity => -2 }); +my $res = $harness->runtests("disk_api_test.pm"); + +exit -1 if !$res || $res->{failed} || $res->{parse_errors}; diff --git a/src/test/run_san_nvme_tests.pl b/src/test/run_san_nvme_tests.pl new file mode 100755 index 00000000..b28efe7d --- /dev/null +++ b/src/test/run_san_nvme_tests.pl @@ -0,0 +1,91 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +our ($backing, @opens); + +BEGIN { + *CORE::GLOBAL::sysopen = sub (*$$;$) { + push @opens, $_[1]; + my $path = $_[1] =~ m{^/dev/disk/by-id/nvme-} ? $backing : $_[1]; + return CORE::sysopen($_[0], $path, $_[2]); + }; +} +use lib qw(..); +use File::Temp qw(tempfile); +use Test::More; +use Test::MockModule; +use PVE::Storage::SAN; + +my ($fh, $file) = tempfile('san-nvme-XXXX', DIR => '.', UNLINK => 1); +close($fh); +$backing = $file; +my $device = '/dev/nvme0n1'; +my $link = '/dev/disk/by-id/nvme-eui.0123456789abcdef'; +my $lun = + { devpath => $device, 'by-id' => $link, wwid => 'eui.0123456789abcdef', multipath => 'native' }; +my $san = Test::MockModule->new('PVE::Storage::SAN'); +my $disk = Test::MockModule->new('PVE::Diskmanage'); +my $mp = Test::MockModule->new('PVE::Multipath'); +my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); +my $storage = Test::MockModule->new('PVE::Storage'); +my $cluster = Test::MockModule->new('PVE::Cluster'); +my $cfg = { ids => {} }; +my @order; +$storage->mock(config => sub { $cfg }); +$disk->mock(verify_blockdev_path => sub { $device }); +$disk->mock(is_partition => sub { 0 }); +$disk->mock(get_nvme_transport => sub { 'fc' }); +$disk->mock(is_iscsi_device => sub { 0 }); +$disk->mock(is_mounted => sub { 0 }); +$disk->mock(has_holder => sub { 0 }); +$mp->mock(device_map => sub { undef }); +$san->mock(abs_path => sub { $device }); +$san->mock(scan_luns => sub { [$lun] }); +$san->mock(lun_partitions => sub { [] }); +$san->mock(assert_same_device => sub { ok(defined(fileno($_[1])), 'held descriptor checked') }); +$lvm->mock(lun_volume_groups => sub { push @order, 'check'; [] }); +$cluster->mock( + cfs_lock_domain => sub { + is($_[0], 'san-eui.0123456789abcdef', 'native namespace uses the LUN lock'); + push @order, 'lock'; + $_[2]->(); + push @order, 'unlock'; + $@ = ''; + }, +); +is(PVE::Storage::SAN::stable_path($device, $lun->{wwid}), $link, 'native stable path'); +is_deeply(PVE::Storage::SAN::lun_for_device($link), $lun, 'native row selected for writes'); +$disk->mock(is_partition => sub { 1 }); +is(PVE::Storage::SAN::lun_for_device($link), undef, 'a partition is handled as a local device'); +$disk->mock(is_partition => sub { 0 }); +$cfg->{ids}->{data} = { type => 'lvm', base => 'fabric:nvme-eui.0123456789abcdef' }; +eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun) }; +like($@, qr/base volume/, 'native base reference refuses'); +$cfg->{ids} = {}; +@order = @opens = (); +$lvm->mock(file_read_firstline => sub { $_[0] =~ m{/size$} ? 1024 : undef }); +$lvm->mock(lvm_pv_info => sub { undef }); +$lvm->mock( + clear_first_sector => sub { push @order, 'clear'; ok(-e $_[0], 'VG clear uses held fd') }); +$lvm->mock( + run_command => sub { + my ($cmd, %opts) = @_; + if ($cmd->[0] eq 'wipefs') { + $opts{outfunc}->('{"signatures":[]}'); + } else { + is($cmd->[-1], $device, 'VG command uses the checked device'); + push @order, $cmd->[0]; + } + }, +); +PVE::Storage::LVMPlugin::lvm_create_volume_group($link, 'data', 1); +is_deeply( + \@order, + [qw(lock check clear /sbin/pvcreate /sbin/vgcreate unlock)], + 'native VG sequencing', +); +is_deeply(\@opens, [$link], 'VG creation holds the native by-id device'); + +done_testing(); diff --git a/src/test/run_san_taint_tests.pl b/src/test/run_san_taint_tests.pl index 35b11cd7..0ccd9e2f 100755 --- a/src/test/run_san_taint_tests.pl +++ b/src/test/run_san_taint_tests.pl @@ -31,4 +31,30 @@ for my $bad ('-bad', '1:2', '../dev', "3600a\n") { like($@, qr/invalid WWID/, 'ambiguous identity refused'); } +# the LUN that write operations get must be usable in command lines and lock names +use PVE::Storage::SAN; +my $san = Test::MockModule->new('PVE::Storage::SAN'); +my $tainted_dev = PVE::Tools::file_get_contents('/proc/self/comm') =~ s/.*//sr . '/dev/null'; +ok(tainted($tainted_dev), 'device path derived from a file is tainted'); +$san->mock( + is_san_device => sub { 1 }, + scan_luns => sub { + return [{ + devpath => $tainted_dev, + wwid => $wwid, + 'by-id' => '/dev/null', + usage => 'unused', + multipath => 'none', + }]; + }, +); +my $disks = Test::MockModule->new('PVE::Diskmanage'); +$disks->mock(verify_blockdev_path => sub { '/dev/null' }); +my $storage = Test::MockModule->new('PVE::Storage'); +$storage->mock(config => sub { { ids => {} } }); +my $lun = PVE::Storage::SAN::lun_for_device('/dev/null'); +ok(!tainted($lun->{devpath}), 'LUN device path is untainted'); +ok(!tainted($lun->{wwid}), 'LUN WWID is untainted'); +is($lun->{wwid}, '3600a', 'LUN WWID is unchanged'); + done_testing(); diff --git a/src/test/run_test_lvm_base_conflicts.pl b/src/test/run_test_lvm_base_conflicts.pl index 26362a74..beb38a4e 100755 --- a/src/test/run_test_lvm_base_conflicts.pl +++ b/src/test/run_test_lvm_base_conflicts.pl @@ -2,78 +2,182 @@ use strict; use warnings; - -use Test::MockModule; +use lib qw(..); use Test::More; +use Test::MockModule; +use PVE::Storage::SAN; -use lib ('.', '..'); -use PVE::Storage; -use PVE::Storage::LVMPlugin; - -my $storage = Test::MockModule->new('PVE::Storage'); +my $scan = Test::MockModule->new('PVE::Storage::SAN'); my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); -my ($cfg, $created, $existing); -$storage->mock( - config => sub { $cfg }, - path => sub { '/dev/sda' }, - activate_storage => sub { }, - activate_volumes => sub { }, -); -$lvm->mock( - lvm_create_volume_group => sub { - my ($device, $vgname, $shared, $guard) = @_; - return if $existing; - $guard->{check_storage}->($device); - $created++; - }, - assert_discard_supported => sub { }, -); - -for my $case ( - ['iscsi', {}, 0], - ['iscsi', { images => 1 }, 1], - ['iscsidirect', {}, 1], - ['zfs', {}, 1], - ['vendor', { images => 1 }, 1], -) { - my ($type, $content, $conflict) = @$case; - $created = 0; - $cfg = { - ids => { - base => { type => 'iscsi', target => 'iqn.test', content => { none => 1 } }, - other => { - type => $type, - target => 'iqn.test', - content => $content, - disable => 1, - nodes => { elsewhere => 1 }, +my $storage = Test::MockModule->new('PVE::Storage'); +my $cluster = Test::MockModule->new('PVE::Cluster'); +my $lun = { wwid => '3600+abc', devpath => '/dev/dm-0', 'iscsi-target' => 'iqn.test' }; +# the LUN is exported through two targets, its row names only the first +my $identity = Test::MockModule->new('PVE::Multipath::Identity'); +$identity->mock( + collect => sub { + { + paths => { + sdb => { wwid => '3600+abc', target => 'iqn.test' }, + sdc => { wwid => '3600+abc', target => 'iqn.second' }, + sdd => { wwid => '3600+other', target => 'iqn.other' }, }, + }; + }, +); +my $vgs = []; +$scan->mock(lun_partitions => sub { [] }); +$lvm->mock(lun_volume_groups => sub { $vgs }); +$lvm->mock(cached_pv_volume_groups => sub { die "write check must not use cached PV metadata\n" }); +my $cfg = { ids => {} }; +$storage->mock(config => sub { $cfg }); +my $check = sub { + eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun, @_) }; + return $@; +}; +is($check->(), '', 'unreferenced LUN accepted'); +for my $type (qw(lvm lvmthin)) { + $vgs = ['data']; + $cfg->{ids} = { + data => { + type => $type, + vgname => 'data', + shared => 1, + disable => 1, + nodes => { other => 1 }, }, }; - eval { - PVE::Storage::LVMPlugin->on_add_hook( - 'test', - { base => 'base:0.0.1.scsi-test', vgname => 'test' }, - ); - }; - if ($conflict) { - like($@, qr/storage 'other' serves disk images from this target/, "$type conflict refused"); - like($@, qr/set its content to 'none'/, "$type: remedy named for iSCSI only") - if $type eq 'iscsi'; - } else { - is($@, '', 'connection-only storage permits initialization'); + for my $cached_vg (undef, 'unrelated') { + $lun->{vgname} = $cached_vg; + $lun->{usage} = 'unused'; + like($check->(), qr/used by storage 'data'/, "$type refusal ignores display VG and usage"); } - is($created, $conflict ? 0 : 1, 'no writes on conflicting target'); + is($check->('data'), '', 'the storage being added can be exempted'); +} +delete $lun->{vgname}; +delete $lun->{usage}; +$vgs = []; +$cfg->{ids} = { direct => { type => 'iscsi', target => 'iqn.test', content => { images => 1 } } }; +like($check->(), qr/storage 'direct' serves images from this target$/, 'direct image use refuses'); +$cfg->{ids}->{direct}->{target} = 'iqn.second'; +like($check->(), qr/storage 'direct' serves images/, 'also through another target of the LUN'); +$cfg->{ids}->{direct}->{target} = 'iqn.other'; +is($check->(), '', 'a target of another LUN is no reference'); +$cfg->{ids}->{direct}->{target} = 'iqn.test'; +$cfg->{ids}->{direct}->{content} = { none => 1 }; +is($check->(), '', 'connection-only storage permits use'); - $existing = 1; - eval { - PVE::Storage::LVMPlugin->on_add_hook( - 'test', - { base => 'base:0.0.1.scsi-test', vgname => 'test' }, - ); - }; - is($@, '', "$type: an existing volume group on the base volume can be added"); - $existing = 0; +for my $type (qw(iscsidirect zfs)) { + local $cfg->{ids}->{other} = { type => $type, target => 'iqn.test', content => {} }; + like( + $check->(), + qr/storage 'other' serves images from this target$/, + "$type storage of the target refuses", + ); } +$cfg->{ids}->{data} = { type => 'lvm', base => 'direct:0.0.1.dm-uuid-mpath-3600+abc' }; +like($check->(), qr/base volume/, 'base reference refuses'); +$cfg->{ids}->{data}->{base} = 'direct:0.0.1.scsi-3600+abc'; +like($check->(), qr/base volume/, 'scsi WWID in base name refuses'); +$cfg->{ids}->{data}->{base} = 'direct:0.0.1.scsi-legacy'; +$storage->mock(path => sub { '/dev/disk/by-id/legacy' }); +$scan->mock(abs_path => sub { '/dev/dm-0' }); +like($check->(), qr/base volume/, 'resolved alias to the LUN refuses'); +$scan->mock(abs_path => sub { $_[0] eq $lun->{devpath} ? '/dev/dm-0' : '/dev/dm-1' }); +is($check->(), '', 'base on another device is no match'); +$scan->mock(abs_path => sub { undef }); +is($check->(), '', 'unresolvable base is no match'); +$storage->mock(path => sub { die "base unavailable\n" }); +is($check->(), '', 'unavailable base is no match'); +ok(!PVE::Storage::LVMPlugin->can('create_volume_group'), 'creation implementation is lexical'); +$cfg->{ids} = {}; +my @order; +$cluster->mock( + cfs_lock_domain => sub { + is($_[0], 'san-3600=2babc', 'lock name escapes unsafe characters without collisions'); + push @order, 'lock'; + $_[2]->(); + push @order, 'unlock'; + $@ = ''; + }, +); +PVE::Storage::SAN::with_lun_lock($lun, sub { push @order, 'write' }); +is_deeply(\@order, [qw(lock write unlock)], 'the code runs under the lock of the LUN'); + +# volume group creation checks the references only when it has to initialize the LUN +my $disks = Test::MockModule->new('PVE::Diskmanage'); +$disks->mock( + verify_blockdev_path => sub { $_[0] }, + is_mounted => sub { undef }, + has_holder => sub { undef }, +); +my ($signatures, $pvinfo) = ('', undef); +$lun->{'by-id'} = '/dev/null'; +$scan->mock( + lun_for_device => sub { $lun }, + assert_same_device => sub { }, + assert_lun_unreferenced => sub { push @order, 'check'; die "referenced\n" }, +); +$lvm->mock( + file_read_firstline => sub { $_[0] =~ m{/size$} ? '2048' : undef }, + lvm_pv_info => sub { $pvinfo }, + clear_first_sector => sub { push @order, 'clear' }, + run_command => sub { + my ($cmd, %param) = @_; + return $param{outfunc}->(qq({"signatures":[$signatures]})) if $cmd->[0] eq 'wipefs'; + push @order, $cmd->[0]; + }, +); +@order = (); +$signatures = '{"type":"LVM2_member","offset":"0x218"}'; +$pvinfo = { vgname => 'data', in_use => '1' }; +eval { + PVE::Storage::LVMPlugin::lvm_create_volume_group( + '/dev/null', + 'data', + 1, + { exempt => 'new' }, + ); +}; +is($@, '', 'an existing volume group can be added whatever references its LUN'); +is_deeply(\@order, [qw(lock unlock)], 'without reference check or write'); +eval { + PVE::Storage::LVMPlugin::lvm_create_volume_group( + '/dev/null', + 'other', + 1, + { exempt => 'new' }, + ); +}; +like($@, qr/already used by volume group 'data'/, 'another volume group on the LUN refuses'); +@order = (); +($signatures, $pvinfo) = ('', undef); +eval { + PVE::Storage::LVMPlugin::lvm_create_volume_group( + '/dev/null', + 'data', + 1, + { exempt => 'new' }, + ); +}; +like($@, qr/referenced/, 'initializing a referenced LUN refuses'); +is_deeply(\@order, [qw(lock check)], 'the check precedes the first write'); +$scan->mock(assert_lun_unreferenced => + sub { is($_[2], 'new', 'new storage exempt'); push @order, 'check' }); +@order = (); +eval { + PVE::Storage::LVMPlugin::lvm_create_volume_group( + '/dev/null', + 'data', + 1, + { exempt => 'new' }, + ); +}; +is($@, '', 'an unreferenced empty LUN is initialized'); +is_deeply( + \@order, + [qw(lock check clear /sbin/pvcreate /sbin/vgcreate unlock)], + 'checked once, then written under the lock', +); done_testing(); diff --git a/src/test/run_test_lvm_create.pl b/src/test/run_test_lvm_create.pl index d4be25b0..4d94ea9c 100755 --- a/src/test/run_test_lvm_create.pl +++ b/src/test/run_test_lvm_create.pl @@ -10,9 +10,12 @@ use Test::More; use lib ('.', '..'); use PVE::Storage::LVMPlugin; +use PVE::Storage::SAN; my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); my $disk = Test::MockModule->new('PVE::Diskmanage'); +my $san = Test::MockModule->new('PVE::Storage::SAN'); +$san->mock(lun_for_device => sub { undef }); my $tools = Test::MockModule->new('PVE::Tools'); my $inotify = Test::MockModule->new('PVE::INotify'); $inotify->mock(nodename => sub { 'testnode' }); @@ -170,6 +173,17 @@ like( qr/contains data \(gpt\)/, 'a partition table of a map is data, kpartx may not map it', ); +reset_case(); +($signatures, $partitioned) = ([{ type => 'PMBR' }, { type => 'gpt' }], 1); +$san->mock( + lun_for_device => + sub { { wwid => '3600abc', devpath => '/dev/sda', 'by-id' => '/dev/null' } }, + with_lun_lock => sub { $_[1]->() }, +); +like(create(), qr/contains data \(PMBR, gpt\)/, 'a partitioned LUN that is no map is data too'); +is_deeply(\@writes, [], 'partitioned LUN: no writes'); +$san->unmock('with_lun_lock'); +$san->mock(lun_for_device => sub { undef }); my $uuid = 'aaaaaa-bbbb-cccc-dddd-eeee-ffff-gggggg'; for my $vg ('test', 'other', 'unknown', '') {