From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 5A0B81FF0DE for ; Thu, 19 Nov 2026 15:13:51 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 4FCF6216A8; Thu, 19 Nov 2026 15:13:20 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pve-devel@lists.proxmox.com Subject: [PATCH storage v2 2/4] lvm plugin: avoid name collisions with volumes pending removal Date: Thu, 19 Nov 2026 15:13:00 +0100 Message-ID: <20261119141302.600137-3-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261119141302.600137-1-m.koeppl@proxmox.com> References: <20261119141302.600137-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795097588302 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.179 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: QCJMOCU2UWU2IPGUFUC2I2CN3ZM2CRQI X-Message-ID-Hash: QCJMOCU2UWU2IPGUFUC2I2CN3ZM2CRQI X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: l.sichert@proxmox.com X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: With saferemove or discard on removal, a freed volume is renamed to del- and wiped by a worker in the background. If a volume with the same name is freed again before that worker has finished, the rename fails because del- still exists (e.g. when destroying a VM right after rolling back a qcow2 snapshot). If del- is taken, rename the volume to the first free del-- instead. The name is chosen with the storage lock held, so it cannot race with other removals. If del- is free, it is used as before. The numbered names still contain the volume name, so they keep its disk number reserved for new allocations, like del-. The cleanup worker and the recovery of a failed rollback now use that name. Signed-off-by: Michael Köppl --- Notes: Changing this might be too much regression potential for the fix in 3/4. This is based on the feedback of the automated LLM review pointing out that snapshot delection could collide with an outstanding rollback cleanup in v1 of this series, leaving behind a locked guest. So the choice is either only applying the pve-common patch and the actual fix for 8268 and dropping the rest, or avoiding the naming overlap as done here. There might be other ways to do this, but I wanted to keep the potential for regressions as small as possible. src/PVE/Storage/LVMPlugin.pm | 61 ++++++---- src/test/Makefile | 5 +- src/test/run_test_lvm_cleanup_name.pl | 165 ++++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 21 deletions(-) create mode 100755 src/test/run_test_lvm_cleanup_name.pl diff --git a/src/PVE/Storage/LVMPlugin.pm b/src/PVE/Storage/LVMPlugin.pm index 3cdab9e4..e12145bb 100644 --- a/src/PVE/Storage/LVMPlugin.pm +++ b/src/PVE/Storage/LVMPlugin.pm @@ -465,7 +465,7 @@ my sub lock_storage_with_acquire_retry { } my sub rename_after_failed_cleanup { - my ($class, $scfg, $storeid, $vg, $name) = @_; + my ($class, $scfg, $storeid, $vg, $del_name) = @_; eval { my $failed_name; @@ -482,7 +482,7 @@ my sub rename_after_failed_cleanup { my $existing = $lvs->{$vg} // {}; my $prefix = 'failed-'; - my $suffix = "-del-$name"; + my $suffix = "-$del_name"; my $last_fail = max( -1, @@ -493,17 +493,17 @@ my sub rename_after_failed_cleanup { $failed_name = $prefix . ($last_fail + 1) . $suffix; - my $cmd = ['/sbin/lvrename', $vg, "del-$name", $failed_name]; + my $cmd = ['/sbin/lvrename', $vg, $del_name, $failed_name]; run_command( $cmd, - errmsg => "lvrename '$vg/del-$name' to '$vg/$failed_name' error", + errmsg => "lvrename '$vg/$del_name' to '$vg/$failed_name' error", ); - print "renamed '$vg/del-$name' to '$vg/$failed_name'\n"; + print "renamed '$vg/$del_name' to '$vg/$failed_name'\n"; }, ); }; if (my $rename_err = $@) { - print STDERR "ERROR: unable to rename '$vg/del-$name': $rename_err"; + print STDERR "ERROR: unable to rename '$vg/$del_name': $rename_err"; } } @@ -526,10 +526,23 @@ my sub parse_saferemove_throughput { return $number * $multiplier; } +# Freeing a volume again while an earlier one with the same name is still being cleaned up, e.g. +# after two rollbacks in a row, must not collide with the earlier one's name. +my sub cleanup_name { + my ($existing, $name) = @_; + + return "del-$name" if !$existing->{"del-$name"}; + + my $n = 1; + $n++ while $existing->{"del-$n-$name"}; + return "del-$n-$name"; +} + my sub free_lvm_volumes_locked { - my ($class, $scfg, $storeid, $volnames) = @_; + my ($class, $scfg, $storeid, $volnames, $cleanup_names) = @_; my $vg = $scfg->{vgname}; + my $del_names = {}; my $on_remove_opts = {}; if ($scfg->{'on-volume-remove'}) { @@ -759,7 +772,8 @@ my sub free_lvm_volumes_locked { my $total_cleanup_errors = 0; for my $name (@$volnames) { - my $lvmpath = "/dev/$vg/del-$name"; + my $del_name = $del_names->{$name}; + my $lvmpath = "/dev/$vg/$del_name"; my $discard_action; if ($scfg->{saferemove} && $on_remove_opts->{discard}) { @@ -790,7 +804,7 @@ my sub free_lvm_volumes_locked { }; if (my $activation_err = $@) { print STDERR "ERROR: $activation_err"; - eval { rename_after_failed_cleanup($class, $scfg, $storeid, $vg, $name) }; + eval { rename_after_failed_cleanup($class, $scfg, $storeid, $vg, $del_name) }; $total_cleanup_errors += 1; next; } @@ -802,18 +816,18 @@ my sub free_lvm_volumes_locked { $storeid, $scfg, sub { - my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$name"]; - run_command($cmd, errmsg => "lvremove '$vg/del-$name' error"); + my $cmd = ['/sbin/lvremove', '-f', "$vg/$del_name"]; + run_command($cmd, errmsg => "lvremove '$vg/$del_name' error"); }, ); }; if (my $cleanup_err = $@) { print STDERR "ERROR: cleanup failed for lv $name: $cleanup_err"; - eval { rename_after_failed_cleanup($class, $scfg, $storeid, $vg, $name) }; + eval { rename_after_failed_cleanup($class, $scfg, $storeid, $vg, $del_name) }; $total_cleanup_errors += 1; next; } - print "successfully removed volume $name ($vg/del-$name)\n"; + print "successfully removed volume $name ($vg/$del_name)\n"; } if ($total_cleanup_errors != 0) { my $number_of_vols = scalar @$volnames; @@ -829,11 +843,15 @@ my sub free_lvm_volumes_locked { my $cmd = ['/sbin/lvchange', '-aln', "$vg/$name"]; run_command($cmd, errmsg => "can't deactivate LV '$vg/$name'"); } + my $existing = lvm_list_volumes($vg)->{$vg} // {}; for my $name (@$volnames) { # avoid long running task, so we only rename here - my $cmd = ['/sbin/lvrename', $vg, $name, "del-$name"]; + my $del_name = cleanup_name($existing, $name); + my $cmd = ['/sbin/lvrename', $vg, $name, $del_name]; run_command($cmd, errmsg => "lvrename '$vg/$name' error"); + $del_names->{$name} = $del_name; } + $cleanup_names->%* = $del_names->%* if $cleanup_names; return $cleanup_worker; } else { for my $name (@$volnames) { @@ -1450,10 +1468,10 @@ my sub alloc_snap_image { } my sub free_snap_image_locked { - my ($class, $storeid, $scfg, $volname, $snap) = @_; + my ($class, $storeid, $scfg, $volname, $snap, $cleanup_names) = @_; my $snap_volname = get_snap_name($class, $volname, $snap); - return free_lvm_volumes_locked($class, $scfg, $storeid, [$snap_volname]); + return free_lvm_volumes_locked($class, $scfg, $storeid, [$snap_volname], $cleanup_names); } # Must be called with the storage lock held. @@ -1858,19 +1876,22 @@ my sub volume_snapshot_rollback_locked { die "can't rollback snapshot for '$format' volume\n" if $format ne 'qcow2'; - my $cleanup_worker = - eval { free_snap_image_locked($class, $storeid, $scfg, $volname, 'current'); }; + my $cleanup_names = {}; + my $cleanup_worker = eval { + free_snap_image_locked($class, $storeid, $scfg, $volname, 'current', $cleanup_names); + }; die "error deleting snapshot $snap $@\n" if $@; eval { alloc_snap_image($class, $storeid, $scfg, $volname, $snap) }; if (my $err = $@) { my $original_state = ''; if ($cleanup_worker) { # rename original image back - eval { lvrename($scfg, "del-${volname}", $volname) }; + my $del_name = $cleanup_names->{$volname}; + eval { lvrename($scfg, $del_name, $volname) }; if ($@) { warn $@; # no cleanup worker is started, so the original stays available under that name - $original_state = " (original volume kept as 'del-${volname}')"; + $original_state = " (original volume kept as '$del_name')"; } else { $original_state = ' (original volume restored)'; } diff --git a/src/test/Makefile b/src/test/Makefile index b2676762..10f0d9b5 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -1,6 +1,6 @@ all: test -test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream +test: test_iscsi test_zfspoolplugin test_lvmplugin test_lvm_create test_lvm_lock_retry test_lvm_cleanup_name test_lvm_base_volume test_lvm_hints test_disklist test_bwlimit test_plugin test_ovf test_volume_access test_remote_command test_sensitive_properties test_import_stream test_zfspoolplugin: run_test_zfspoolplugin.pl ./run_test_zfspoolplugin.pl @@ -16,6 +16,9 @@ test_lvm_create: run_test_lvm_create.pl run_test_lvm_base_conflicts.pl test_lvm_lock_retry: run_test_lvm_lock_retry.pl ./run_test_lvm_lock_retry.pl +test_lvm_cleanup_name: run_test_lvm_cleanup_name.pl + ./run_test_lvm_cleanup_name.pl + test_lvm_base_volume: run_test_lvm_base_volume.pl ./run_test_lvm_base_volume.pl diff --git a/src/test/run_test_lvm_cleanup_name.pl b/src/test/run_test_lvm_cleanup_name.pl new file mode 100755 index 00000000..964de004 --- /dev/null +++ b/src/test/run_test_lvm_cleanup_name.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl + +# Unit test for freeing a volume while an earlier one with the same name is still pending removal. + +use strict; +use warnings; + +use lib '..'; +use Test::More; +use Test::MockModule; + +use PVE::Storage::LVMPlugin; + +my $CLASS = 'PVE::Storage::LVMPlugin'; +my $vg = 'test-vg'; +my $scfg = { type => 'lvm', vgname => $vg, saferemove => 1 }; +my $volname = 'vm-100-disk-0.qcow2'; +my $snapvolname = 'snap_vm-100-disk-0_snap1.qcow2'; + +my $lvs; # names of the existing LVs +my @renames; +my @activated; +my @removed; +my $workers; + +my $lvm_mock = Test::MockModule->new('PVE::Storage::LVMPlugin'); +$lvm_mock->redefine( + cluster_lock_storage => sub { + my ($class, $storeid, $shared, $timeout, $func, @param) = @_; + return $func->(@param); + }, + lvm_vgs => sub { + return { $vg => {} }; + }, + lvm_list_volumes => sub { + return { $vg => { map { $_ => {} } keys %$lvs } }; + }, + run_command => sub { + my ($cmd) = @_; + if ($cmd->[0] eq '/sbin/lvrename') { + my (undef, undef, $from, $to) = @$cmd; + die "Logical Volume \"$from\" not found in volume group \"$vg\"\n" + if !$lvs->{$from}; + die "Logical Volume \"$to\" already exists in volume group \"$vg\"\n" + if $lvs->{$to}; + push @renames, "$from -> $to"; + delete $lvs->{$from}; + $lvs->{$to} = 1; + } elsif ($cmd->[0] eq '/sbin/lvremove') { + my $name = $cmd->[-1] =~ s|^\Q$vg\E/||r; + die "Failed to find logical volume \"$vg/$name\"\n" if !$lvs->{$name}; + push @removed, $name; + delete $lvs->{$name}; + } elsif ($cmd->[0] eq '/sbin/lvchange' && $cmd->[1] eq '-aly') { + push @activated, $cmd->[2]; + } + return 0; + }, + activate_volume => sub { }, + # no volume has a device mapper link, so the cleanup worker cannot zero out + abs_path => sub { return undef }, + volume_snapshot_info => sub { + return { + snap1 => { file => '/dev/null', volname => $snapvolname, child => 'current' }, + current => { file => '/dev/null', volname => $volname, parent => 'snap1' }, + }; + }, +); + +# makes allocating the new current volume of a rollback fail +my $plugin_mock = Test::MockModule->new('PVE::Storage::Plugin'); +$plugin_mock->redefine(file_size_info => sub { return undef }); + +my $rpcenv_mock = Test::MockModule->new('PVE::RPCEnvironment'); +$rpcenv_mock->redefine( + get => sub { return bless({}, 'PVE::RPCEnvironment') }, + get_user => sub { return 'root@pam' }, + fork_worker => sub { $workers++; return 'UPID:test' }, +); + +my sub setup { + my (@names) = @_; + $lvs = { map { $_ => 1 } @names }; + @renames = (); + @activated = (); + @removed = (); + $workers = 0; +} + +# keep plugin progress and errors out of the test output +my sub quietly { + my ($code) = @_; + local *STDOUT; + local *STDERR; + open(STDOUT, '>', '/dev/null') or die "cannot open /dev/null - $!\n"; + open(STDERR, '>', '/dev/null') or die "cannot open /dev/null - $!\n"; + my $ok = eval { $code->(); 1 }; + my $err = $@; + return ($ok, $err); +} + +# the previous current volume replaced by a rollback is still pending removal +setup($volname, $snapvolname, "del-$volname"); +my ($ok, $err) = + quietly(sub { $CLASS->volume_snapshot_delete($scfg, 'mystore', $volname, 'snap1', 0) }); +ok($ok, 'deleting the snapshot right after a rollback succeeds') or diag($err); +is_deeply( + \@renames, + ["$volname -> del-1-$volname", "$snapvolname -> $volname"], + 'frees the current volume under a numbered name', +); +ok($lvs->{"del-$volname"}, 'volume replaced by the rollback is left alone'); +is($workers, 1, 'started a cleanup worker'); + +setup($volname, $snapvolname, "del-$volname"); +($ok, $err) = + quietly(sub { $CLASS->volume_snapshot_rollback($scfg, 'mystore', $volname, 'snap1') }); +ok(!$ok, 'second rollback fails when the new volume cannot be allocated'); +like($err, qr/original volume restored/, 'reports the original volume as restored'); +is_deeply( + \@renames, + ["$volname -> del-1-$volname", "del-1-$volname -> $volname"], + 'restores the volume it renamed, not the one of the earlier rollback', +); +ok($lvs->{"del-$volname"}, 'volume replaced by the earlier rollback is left alone'); +is($workers, 0, 'started no cleanup worker'); + +my $rawvolname = 'vm-100-disk-1'; + +# only a numbered volume is still pending removal, so del- is free +setup($rawvolname, "del-1-$rawvolname"); +my $discard_scfg = { type => 'lvm', vgname => $vg, 'on-volume-remove' => 'discard=1' }; +my $worker = $CLASS->free_image('mystore', $discard_scfg, $rawvolname, 0, 'raw'); +is_deeply(\@renames, ["$rawvolname -> del-$rawvolname"], 'renamed to del- if that is free'); +($ok, $err) = quietly($worker); +ok($ok, 'cleanup worker succeeds') or diag($err); +is_deeply(\@activated, ["/dev/$vg/del-$rawvolname"], 'cleanup worker activates its own volume'); +is_deeply(\@removed, ["del-$rawvolname"], 'cleanup worker removes its own volume'); +ok($lvs->{"del-1-$rawvolname"}, 'volume pending removal is left alone'); + +# zeroing out fails without a device mapper link, so the cleanup worker renames its volume +setup($rawvolname, "del-$rawvolname", "del-1-$rawvolname"); +$worker = $CLASS->free_image('mystore', $scfg, $rawvolname, 0, 'raw'); +is_deeply( + \@renames, + ["$rawvolname -> del-2-$rawvolname"], + 'renamed to the first free numbered name', +); +@renames = (); +($ok, $err) = quietly($worker); +ok(!$ok, 'cleanup worker fails'); +like($err, qr/cleanup failed for 1 out of 1 volumes/, 'cleanup worker reports the failed cleanup'); +is_deeply(\@activated, ["/dev/$vg/del-2-$rawvolname"], 'cleanup worker activates its own volume'); +is_deeply( + \@renames, + ["del-2-$rawvolname -> failed-0-del-2-$rawvolname"], + 'failed cleanup renames its own volume', +); +is_deeply(\@removed, [], 'failed cleanup removes no volume'); +ok( + $lvs->{"del-$rawvolname"} && $lvs->{"del-1-$rawvolname"}, + 'volumes pending removal are left alone', +); + +done_testing(); -- 2.47.3