From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 4B65C1FF0AA for ; Tue, 17 Nov 2026 23:42:01 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3EE9121813; Tue, 17 Nov 2026 23:40:55 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH container v2 24/36] test: remote migration: check the cleanup of the target after failures Date: Tue, 17 Nov 2026 23:34:34 +0100 Message-ID: <20261117223945.862914-25-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261117223945.862914-1-t.lamprecht@proxmox.com> References: <20261117223945.862914-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794955192410 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.198 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: FR7H2GYYMNZ746GHFNRNS6J2UKHHGUHY X-Message-ID-Hash: FR7H2GYYMNZ746GHFNRNS6J2UKHHGUHY X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A config left behind on the target of a failed remote migration blocks its VMID there, and with guest identities it would also keep the uid of the container in the target cluster. Check with a mocked tunnel that the source asks the target to clean up if the tunnel versions do not match or the container does not stop for a restart migration. Check on the target that it removes its reservation if the tunnel closes before the config arrived, keeps a config that arrived, and logs the error of a failed tunnel command. Signed-off-by: Thomas Lamprecht --- changes v1 -> v2: * was 'migrate: remote: clean up the target on a tunnel version mismatch', its fix got applied separately together with the cleanup after a failed shutdown * more tests: failed shutdown of a restart migration, tunnel that closes before the config arrived (with and without a running volume transfer), logging of a failed tunnel command src/test/lifecycle_tests.pl | 164 ++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/src/test/lifecycle_tests.pl b/src/test/lifecycle_tests.pl index 235169f..930fa4f 100755 --- a/src/test/lifecycle_tests.pl +++ b/src/test/lifecycle_tests.pl @@ -1558,6 +1558,170 @@ subtest 'restore over a container of another guest needs an override' => sub { $lxc_module->mock(delete_ifaces_ipams_ips => sub { }); }; +subtest 'remote restart migration cleans up the target if the container does not stop' => sub { + reset_configs(); + my $raw = existing_config('ctime=1500000000,rtime=1500000000000,uid=existingexistingexis'); + write_raw_config(200, $raw); + + my $log = ''; + my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate'); + $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" }); + + my $cleanup; + my $tunnel_module = Test::MockModule->new('PVE::Tunnel'); + $tunnel_module->mock( + fork_websocket_tunnel => sub { + return { version => $PVE::LXC::Migrate::WS_TUNNEL_VERSION, age => 0 }; + }, + write_tunnel => sub { return {} }, + finish_tunnel => sub { (undef, $cleanup) = @_ }, + ); + $lxc_module->mock( + check_running => sub { return 1 }, + vm_stop => sub { die "container did not stop\n" }, + ); + + my $err = run_task(sub { + PVE::LXC::Migrate->migrate( + 'remote-node', + '192.0.2.1', + 200, + { + remote => { vmid => 300, conn => { host => '192.0.2.1', port => 8006 } }, + restart => 1, + }, + ); + }); + $lxc_module->mock(check_running => sub { return 0 }); + $lxc_module->unmock('vm_stop'); + + like($err, qr/migration aborted/, 'migration fails'); + like($log, qr/^err: .*container did not stop/m, 'failed shutdown is reported'); + is($cleanup, 1, 'target is asked to clean up'); + is(read_raw_config(200), $raw, 'config of the source is unchanged'); +}; + +subtest 'migration tunnel logs failed commands and cleans up after an early end' => sub { + reset_configs(); + my ($err, $replies) = run_mtunnel(300); + like($err, qr/mtunnel exited unexpectedly/, 'tunnel that ends without a command fails'); + like( + $task_log, + qr/^WARN: tunnel closed before the config arrived, cleaning up$/m, + 'cleanup is logged', + ); + ok(!defined(read_raw_config(300)), 'reserved config is removed'); + + reset_configs(); + my $broken = $INCOMING_CONFIG =~ s/^rootfs: local:/rootfs: iso:/mr; + ($err, $replies) = run_mtunnel(300, { cmd => 'config', conf => $broken }); + like($err, qr/mtunnel exited unexpectedly/, 'tunnel that ends after a failed config fails'); + ok(!$replies->[0]->{success}, 'config command fails'); + like( + $task_log, + qr/^WARN: failed to handle 'config' command - /m, + 'failed command is logged', + ); + ok(!defined(read_raw_config(300)), 'reserved config is removed after a failed config'); + + reset_configs(); + ($err, $replies) = run_mtunnel(300, { cmd => 'config', conf => $INCOMING_CONFIG }); + like($err, qr/mtunnel exited unexpectedly/, 'tunnel that ends after the config fails'); + unlike($task_log, qr/cleaning up/, 'no cleanup once the config arrived'); + is(PVE::LXC::Config->load_config(300)->{lock}, 'migrate', 'config that arrived is kept'); + + # the volume of a transfer is only known once the transfer is done + my $import_cmd = { + cmd => 'disk-import', + volname => 'vm-300-disk-0', + format => 'raw', + export_formats => 'raw+size', + storage => 'local', + }; + my ($queries, $ends_after, $freed) = (0, 2, []); + my $storage_tunnel_module = Test::MockModule->new('PVE::StorageTunnel'); + $storage_tunnel_module->mock( + handle_disk_import => sub { + my ($state) = @_; + $state->{disk_import} = { pid => 1 }; + return {}; + }, + handle_query_disk_import => sub { + my ($state) = @_; + return { status => 'pending' } if ++$queries < $ends_after; + delete $state->{disk_import}; + $state->{cleanup}->{volumes}->{'local:300/vm-300-disk-0.raw'} = 1; + return { status => 'complete' }; + }, + ); + $storage_module->mock(vdisk_free => sub { push $freed->@*, $_[1] }); + + reset_configs(); + ($err, $replies) = run_mtunnel(300, $import_cmd); + ok($replies->[0]->{success}, 'volume transfer starts') or diag(explain($replies)); + is_deeply( + $freed, + ['local:300/vm-300-disk-0.raw'], + 'volume of a transfer that ended is freed', + ); + ok(!defined(read_raw_config(300)), 'reserved config is removed after the transfer ended'); + + reset_configs(); + ($queries, $ends_after, $freed) = (0, 100, []); + ($err, $replies) = run_mtunnel(300, $import_cmd); + like( + $task_log, + qr/^WARN: cleanup failed - a volume transfer is still running$/m, + 'transfer that does not end is reported', + ); + is_deeply($freed, [], 'nothing is freed while a transfer runs'); + ok(defined(read_raw_config(300)), 'reserved config is kept while a transfer runs'); + $storage_module->unmock('vdisk_free'); +}; + +subtest 'remote migration cleans up a target with an incompatible tunnel version' => sub { + reset_configs(); + my $raw = existing_config('ctime=1500000000,rtime=1500000000000,uid=existingexistingexis'); + write_raw_config(200, $raw); + + my $log = ''; + my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate'); + $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" }); + + my $remote_version = { version => $PVE::LXC::Migrate::WS_TUNNEL_VERSION - 1, age => 0 }; + my ($commands, $cleanup) = (0); + my $tunnel_module = Test::MockModule->new('PVE::Tunnel'); + $tunnel_module->mock( + fork_websocket_tunnel => sub { return $remote_version }, + write_tunnel => sub { $commands++ }, + finish_tunnel => sub { (undef, $cleanup) = @_ }, + ); + + my $migrate = sub { + ($log, $commands, $cleanup) = ('', 0); + return run_task(sub { + PVE::LXC::Migrate->migrate( + 'remote-node', + '192.0.2.1', + 200, + { remote => { vmid => 300, conn => { host => '192.0.2.1', port => 8006 } } }, + ); + }); + }; + + like($migrate->(), qr/migration aborted/, 'migration to an older target fails'); + like($log, qr/^err: .*Remote tunnel endpoint too old/m, 'target is reported as too old'); + is($cleanup, 1, 'older target is asked to clean up'); + is($commands, 0, 'nothing else is sent to the target'); + is(read_raw_config(200), $raw, 'config of the source is unchanged'); + + $remote_version = { version => $PVE::LXC::Migrate::WS_TUNNEL_VERSION + 1, age => 0 }; + like($migrate->(), qr/migration aborted/, 'migration to an incompatible target fails'); + like($log, qr/^err: .*Remote tunnel endpoint not compatible/m, 'target is reported'); + is($cleanup, 1, 'incompatible target is asked to clean up'); + is(read_raw_config(200), $raw, 'config of the source is unchanged'); +}; + remove_tree($WORKDIR); done_testing(); -- 2.47.3