From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id B6E511FF0B7 for ; Fri, 13 Nov 2026 04:17:53 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 1EB8221BA8; Fri, 13 Nov 2026 04:05:17 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH v4 storage 22/36] disks: allow wiping a SAN LUN that no storage uses Date: Fri, 13 Nov 2026 04:00:40 +0100 Message-ID: <20261113030146.1559030-23-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261113030146.1559030-1-t.lamprecht@proxmox.com> References: <20261113030146.1559030-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794538915048 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.237 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: ZOTXYIYOMRP5FKL3QE5B3E7AHKAAPT43 X-Message-ID-Hash: ZOTXYIYOMRP5FKL3QE5B3E7AHKAAPT43 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A recycled LUN can hold old data, for example the partition table of a VMFS datastore, which the volume group creation refuses. The disk wipe of a node wipes any block device that is not mounted and has no holder. So it refuses a multipath map whose partitions are mapped, but wipes a LUN that is only in use on other nodes. For a LUN of the SAN inventory, run the wipe under the cluster-wide lock of the LUN and only if no storage references it. Besides the mount and holder checks of the local wipe, it removes the partition mappings of a multipath map and writes through an exclusively opened descriptor. The device parameter now accepts the characters of by-id and mapper names, so that a LUN can be given by those. LVM keeps listing a PV that was wiped by other means in its hint file until two more of its commands ran, and the SAN inventory takes the volume group names from that file. Remove the file after the wipe and let one LVM command write a new one, so that the LUN list of the node does not show the wiped LUN with its old volume group. Other devices, including kernel partitions of a LUN, are wiped as before. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Disks.pm | 29 +++++-- src/PVE/Diskmanage.pm | 28 +++++-- src/PVE/Storage/LVMPlugin.pm | 7 ++ src/PVE/Storage/SAN.pm | 46 ++++++++++- src/test/Makefile | 3 +- src/test/run_san_nvme_tests.pl | 16 ++++ src/test/run_san_wipe_tests.pl | 137 +++++++++++++++++++++++++++++++++ 7 files changed, 251 insertions(+), 15 deletions(-) create mode 100755 src/test/run_san_wipe_tests.pl diff --git a/src/PVE/API2/Disks.pm b/src/PVE/API2/Disks.pm index 980ba7ff..13f7c005 100644 --- a/src/PVE/API2/Disks.pm +++ b/src/PVE/API2/Disks.pm @@ -9,6 +9,8 @@ use HTTP::Status qw(:constants); use PVE::Diskmanage; use PVE::JSONSchema qw(get_standard_option); use PVE::SafeSyslog; +use PVE::Storage::LVMPlugin; +use PVE::Storage::SAN; use PVE::Tools qw(run_command); use PVE::API2::Disks::Directory; @@ -296,7 +298,7 @@ __PACKAGE__->register_method({ disk => { type => 'string', description => "Block device name", - pattern => '^/dev/[a-zA-Z0-9\/]+$', + pattern => '^/dev/[a-zA-Z0-9._:+/-]+$', }, }, }, @@ -306,18 +308,31 @@ __PACKAGE__->register_method({ my $disk = PVE::Diskmanage::verify_blockdev_path($param->{disk}); - my $mounted = PVE::Diskmanage::is_mounted($disk); - die "disk/partition '${mounted}' is mounted\n" if $mounted; + my $lun = PVE::Storage::SAN::lun_for_device($disk); + if (!$lun) { + my $mounted = PVE::Diskmanage::is_mounted($disk); + die "disk/partition '${mounted}' is mounted\n" if $mounted; - my $held = PVE::Diskmanage::has_holder($disk); - die "disk/partition '${held}' has a holder\n" if $held; + my $held = PVE::Diskmanage::has_holder($disk); + die "disk/partition '${held}' has a holder\n" if $held; + } my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $worker = sub { - PVE::Diskmanage::wipe_blockdev($disk); - PVE::Diskmanage::udevadm_trigger($disk); + if ($lun) { + PVE::Storage::SAN::with_lun_lock( + $lun, + sub { PVE::Storage::SAN::wipe_lun($lun) }, + ); + PVE::Diskmanage::udevadm_trigger($disk, PVE::Storage::SAN::lun_paths($lun)->@*); + run_command(['udevadm', 'settle', '--timeout=30'], timeout => 35); + PVE::Storage::LVMPlugin::refresh_cached_pv_volume_groups(); + } else { + PVE::Diskmanage::wipe_blockdev($disk); + PVE::Diskmanage::udevadm_trigger($disk); + } }; my $basename = basename($disk); # avoid '/' in the ID diff --git a/src/PVE/Diskmanage.pm b/src/PVE/Diskmanage.pm index bef401af..d6340d32 100644 --- a/src/PVE/Diskmanage.pm +++ b/src/PVE/Diskmanage.pm @@ -1119,7 +1119,7 @@ sub change_parttype { # If called with a partition, also sets the partition type to 0x83 'Linux filesystem'. # Expected to be called with a result of verify_blockdev_path(). sub wipe_blockdev { - my ($devpath) = @_; + my ($devpath, $writepath) = @_; my $devname = basename($devpath); my $dev_size = PVE::Tools::file_get_contents("/sys/class/block/$devname/size"); @@ -1138,23 +1138,39 @@ sub wipe_blockdev { my ($part) = @_; push $to_wipe->@*, "/dev/${part}" if -b "/dev/${part}"; }, - ); + ) if !defined($writepath); if (scalar($to_wipe->@*) > 0) { print "found child partitions to wipe: " . join(', ', $to_wipe->@*) . "\n"; } - push $to_wipe->@*, $devpath; # put actual device last + push $to_wipe->@*, $writepath // $devpath; # put actual device last print "wiping block device ${devpath}\n"; - run_command(['wipefs', '--all', $to_wipe->@*], errmsg => "error wiping '${devpath}'"); + # The caller's exclusive descriptor checks use, so wipefs must not claim the device again. + run_command( + ['wipefs', '--all', defined($writepath) ? ('--force') : (), $to_wipe->@*], + errmsg => "error wiping '${devpath}'", + defined($writepath) ? (timeout => 10) : (), + ); + my $target = $writepath // $devpath; run_command( - ['dd', 'if=/dev/zero', "of=${devpath}", 'bs=1M', 'conv=fdatasync', "count=${count}"], + [ + 'dd', + 'if=/dev/zero', + "of=${target}", + 'bs=1M', + 'conv=fdatasync', + defined($writepath) + ? ('iflag=count_bytes', 'count=' . ($count * 1024 * 1024)) + : "count=${count}", + ], errmsg => "error wiping '${devpath}'", + defined($writepath) ? (timeout => 60) : (), ); - if (is_partition($devpath)) { + if (!defined($writepath) && is_partition($devpath)) { eval { change_parttype($devpath, '8300'); }; warn $@ if $@; } diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm index c9e52b4c..4e885429 100644 --- a/src/PVE/Storage/LVMPlugin.pm +++ b/src/PVE/Storage/LVMPlugin.pm @@ -148,6 +148,13 @@ sub cached_pv_volume_groups { return $res; } +# LVM keeps listing a PV that was wiped by other means in its hint file until two more LVM commands +# ran. It writes a new file when there is none. +sub refresh_cached_pv_volume_groups { + unlink '/run/lvm/hints'; + lvm_vgs(); +} + sub lun_volume_groups { my ($devices) = @_; my $output = ''; diff --git a/src/PVE/Storage/SAN.pm b/src/PVE/Storage/SAN.pm index 210c3a78..bba3c128 100644 --- a/src/PVE/Storage/SAN.pm +++ b/src/PVE/Storage/SAN.pm @@ -4,6 +4,7 @@ use v5.36; use Cwd qw(abs_path); use File::Basename qw(basename); +use Fcntl qw(O_RDWR O_EXCL); use PVE::Cluster; use PVE::INotify; @@ -16,7 +17,7 @@ use PVE::Multipath::Config; use PVE::Multipath::Identity; use PVE::Storage; use PVE::Storage::LVMPlugin; -use PVE::Tools qw(file_read_firstline dir_glob_foreach); +use PVE::Tools qw(file_read_firstline dir_glob_foreach run_command); sub scan_luns($cfg) { my $ids = PVE::Multipath::Identity::collect(all => 1); @@ -281,4 +282,47 @@ sub assert_same_device($lun, $fh) { if !@held || !@named || $held[6] != $named[6]; } +# The SCSI paths of the LUN. Without a map, udev keeps the signatures it has seen for each of them. +sub lun_paths($lun) { + my $paths = PVE::Multipath::Identity::collect(all => 1)->{paths}; + return [ + map { m/^(sd[a-z]+)$/ ? "/dev/$1" : () } + grep { $paths->{$_}->{wwid} eq $lun->{wwid} } sort keys %$paths + ]; +} + +sub wipe_lun($lun) { + assert_lun_unreferenced(PVE::Storage::config(), $lun); + my $device = PVE::Diskmanage::verify_blockdev_path($lun->{'by-id'}); + my $parts = lun_partitions($lun); + my %parts = map { basename($_) => 1 } @$parts; + dir_glob_foreach( + '/sys/class/block/' . basename($device) . '/holders', + '[^.].*', + sub($name, @) { + die "cannot wipe '$device' - another block device uses it\n" if !$parts{$name}; + }, + ); + for my $path ($device, @$parts) { + die "cannot wipe '$device' - '$path' is mounted\n" if PVE::Diskmanage::is_mounted($path); + next if $path eq $device; + die "cannot wipe '$device' - another block device uses '$path'\n" + if PVE::Diskmanage::has_holder($path); + } + sysopen(my $fh, $lun->{'by-id'}, O_RDWR) or die "cannot open '$device' - $!\n"; + for my $part (@$parts) { + assert_same_device($lun, $fh); + run_command(['wipefs', '--all', $part], timeout => 10); + if ($part =~ m{^/dev/dm-}) { + assert_same_device($lun, $fh); + run_command(['dmsetup', 'remove', '--retry', $part], timeout => 10); + } + } + assert_same_device($lun, $fh); + close($fh); + sysopen($fh, $lun->{'by-id'}, O_RDWR | O_EXCL) or die "cannot claim '$device' - $!\n"; + my $path = '/proc/' . $$ . '/fd/' . fileno($fh); + PVE::Diskmanage::wipe_blockdev($device, $path); +} + 1; diff --git a/src/test/Makefile b/src/test/Makefile index e453951a..c9bcea8f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -69,8 +69,9 @@ test_san_taint: run_san_taint_tests.pl test_san_lun_scan: run_san_lun_scan_tests.pl ./run_san_lun_scan_tests.pl -test_san_api: run_san_api_tests.pl run_san_nvme_tests.pl +test_san_api: run_san_api_tests.pl run_san_wipe_tests.pl run_san_nvme_tests.pl ./run_san_api_tests.pl + ./run_san_wipe_tests.pl ./run_san_nvme_tests.pl test_disk_api: run_disk_api_tests.pl diff --git a/src/test/run_san_nvme_tests.pl b/src/test/run_san_nvme_tests.pl index b28efe7d..c2f3c65e 100755 --- a/src/test/run_san_nvme_tests.pl +++ b/src/test/run_san_nvme_tests.pl @@ -31,6 +31,7 @@ my $mp = Test::MockModule->new('PVE::Multipath'); my $lvm = Test::MockModule->new('PVE::Storage::LVMPlugin'); my $storage = Test::MockModule->new('PVE::Storage'); my $cluster = Test::MockModule->new('PVE::Cluster'); +my $tools = Test::MockModule->new('PVE::Tools'); my $cfg = { ids => {} }; my @order; $storage->mock(config => sub { $cfg }); @@ -44,6 +45,7 @@ $mp->mock(device_map => sub { undef }); $san->mock(abs_path => sub { $device }); $san->mock(scan_luns => sub { [$lun] }); $san->mock(lun_partitions => sub { [] }); +$san->mock(dir_glob_foreach => sub { }); $san->mock(assert_same_device => sub { ok(defined(fileno($_[1])), 'held descriptor checked') }); $lvm->mock(lun_volume_groups => sub { push @order, 'check'; [] }); $cluster->mock( @@ -64,6 +66,20 @@ $cfg->{ids}->{data} = { type => 'lvm', base => 'fabric:nvme-eui.0123456789abcdef eval { PVE::Storage::SAN::assert_lun_unreferenced($cfg, $lun) }; like($@, qr/base volume/, 'native base reference refuses'); $cfg->{ids} = {}; +@order = @opens = (); +$tools->mock(file_get_contents => sub { 1024 }); +$disk->mock( + run_command => sub { + my ($cmd, %opts) = @_; + push @order, $cmd->[0]; + my ($path) = map { /^of=(.*)$/ ? $1 : m{^(/proc/\d+/fd/\d+)$} ? $1 : () } @$cmd; + ok(defined($path) && -e $path, 'native wipe writes through the held descriptor'); + }, +); +PVE::Storage::SAN::with_lun_lock($lun, sub { PVE::Storage::SAN::wipe_lun($lun) }); +is_deeply(\@order, [qw(lock check wipefs dd unlock)], 'native wipe sequencing'); +is_deeply(\@opens, [$link, $link], 'native wipe opens its by-id name'); + @order = @opens = (); $lvm->mock(file_read_firstline => sub { $_[0] =~ m{/size$} ? 1024 : undef }); $lvm->mock(lvm_pv_info => sub { undef }); diff --git a/src/test/run_san_wipe_tests.pl b/src/test/run_san_wipe_tests.pl new file mode 100755 index 00000000..9fffe294 --- /dev/null +++ b/src/test/run_san_wipe_tests.pl @@ -0,0 +1,137 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +our @open_modes; + +BEGIN { + *CORE::GLOBAL::sysopen = sub (*$$;$) { + push @open_modes, $_[2]; + return @_ == 4 + ? CORE::sysopen($_[0], $_[1], $_[2], $_[3]) + : CORE::sysopen($_[0], $_[1], $_[2]); + }; +} + +use lib qw(..); +use Fcntl qw(O_RDWR O_EXCL); +use File::Temp qw(tempfile); +use Test::MockModule; +use Test::More; +use PVE::Storage::SAN; + +my ($file, $name) = tempfile('san-wipe-XXXX', DIR => '.', UNLINK => 1); +close($file); +my $lun = { devpath => '/dev/dm-0', 'by-id' => $name, wwid => '3600a' }; +my $san = Test::MockModule->new('PVE::Storage::SAN'); +my $disk = Test::MockModule->new('PVE::Diskmanage'); +my $tools = Test::MockModule->new('PVE::Tools'); +my (@commands, @parts, @parttypes); +my $sectors = 1024 * 1024; +my $run = sub { + my ($cmd, %opts) = @_; + push @commands, [$cmd, \%opts]; + if ($cmd->[0] eq 'dd' || ($cmd->[0] eq 'wipefs' && grep { $_ eq '--force' } @$cmd)) { + my ($path) = map { /^of=(.*)$/ ? $1 : m{^(/proc/\d+/fd/\d+)$} ? $1 : () } @$cmd; + ok(defined($path) && -e $path, 'whole-device command uses a live descriptor'); + is((stat($path))[1], (stat($name))[1], 'descriptor still names the opened device'); + ok($open_modes[-1] & O_EXCL, 'exclusive descriptor held for whole-device commands'); + } +}; +$san->mock(run_command => $run); +my $referenced = 0; +my $storage = Test::MockModule->new('PVE::Storage'); +$storage->mock(config => sub { { ids => {} } }); +$san->mock(assert_lun_unreferenced => sub { die "referenced\n" if $referenced }); +$san->mock(lun_partitions => sub { [@parts] }); +$san->mock(dir_glob_foreach => sub { }); +$disk->mock(verify_blockdev_path => sub { '/dev/dm-0' }); +$disk->mock(is_mounted => sub { 0 }); +$disk->mock(has_holder => sub { 0 }); +$disk->mock(run_command => $run); +$disk->mock(dir_glob_foreach => sub { die "partition enumeration after exclusive open\n" }); +$disk->mock(is_partition => sub { die "partition check after exclusive open\n" }); +$tools->mock(file_get_contents => sub { $sectors }); + +$referenced = 1; +@commands = @open_modes = (); +eval { PVE::Storage::SAN::wipe_lun($lun) }; +like($@, qr/referenced/, 'a referenced LUN is refused'); +is_deeply([\@commands, \@open_modes], [[], []], 'before the device is opened or written'); +$referenced = 0; + +for my $count (0, 2, 10) { + @parts = map { "/dev/dm-$_" } 1 .. $count; + @commands = @open_modes = (); + PVE::Storage::SAN::wipe_lun($lun); + is_deeply(\@open_modes, [O_RDWR, O_RDWR | O_EXCL], 'one exclusive whole-device open'); + my $fdpath = $commands[-2]->[0]->[-1]; + is_deeply( + $commands[-2]->[0], + ['wipefs', '--all', '--force', $fdpath], + 'force avoids a second exclusive claim', + ); + is($commands[-1]->[1]->{timeout}, 60, 'dd has 60 seconds for fdatasync'); + ok(grep({ $_ eq 'count=209715200' } $commands[-1]->[0]->@*), 'wipe capped at 200 MiB'); + + for my $i (0 .. $#parts) { + is_deeply( + $commands[2 * $i]->[0], + ['wipefs', '--all', $parts[$i]], + 'partition signatures first', + ); + is_deeply( + $commands[2 * $i + 1]->[0], + ['dmsetup', 'remove', '--retry', $parts[$i]], + 'remove partition map with retry', + ); + } + is($_->[1]->{timeout}, 10, 'partition command has a fixed timeout') + for @commands[0 .. $#commands - 2]; + ok(!-e $fdpath, 'descriptor released on return'); +} + +@parts = (); +@commands = (); +$sectors = 1; +PVE::Storage::SAN::wipe_lun($lun); +ok(grep({ $_ eq 'count=512' } $commands[-1]->[0]->@*), 'small device uses a byte count'); + +$disk->mock(dir_glob_foreach => sub { }); +$disk->mock(is_partition => sub { 1 }); +$disk->mock(change_parttype => sub { push @parttypes, [@_] }); +$disk->mock(run_command => sub { my ($cmd, %opts) = @_; push @commands, [$cmd, \%opts] }); +@commands = (); +$sectors = 204800; +PVE::Diskmanage::wipe_blockdev('/dev/sda1'); +is_deeply( + [map { $_->[0] } @commands], + [ + ['wipefs', '--all', '/dev/sda1'], + ['dd', 'if=/dev/zero', 'of=/dev/sda1', 'bs=1M', 'conv=fdatasync', 'count=100'], + ], + 'local disk command lines unchanged', +); +ok(!grep({ exists($_->[1]->{timeout}) } @commands), 'local disk timeout behavior unchanged'); +is_deeply(\@parttypes, [['/dev/sda1', '8300']], 'local partition type still updated'); + +my $identity = Test::MockModule->new('PVE::Multipath::Identity'); +$identity->mock( + collect => sub { + { + paths => { + sdb => { wwid => '3600a' }, + sdc => { wwid => '3600b' }, + sdd => { wwid => '3600a' }, + }, + }; + }, +); +is_deeply( + PVE::Storage::SAN::lun_paths($lun), + ['/dev/sdb', '/dev/sdd'], + 'all paths of the LUN are refreshed after a wipe', +); + +done_testing();