From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 3D7C51FF0E1 for ; Mon, 16 Nov 2026 01:48:53 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 5B832216E9; Mon, 16 Nov 2026 01:48:21 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server 16/34] api: remote migration: take over the guest uid from the source Date: Mon, 16 Nov 2026 01:46:28 +0100 Message-ID: <20261116004703.2908294-17-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116004703.2908294-1-t.lamprecht@proxmox.com> References: <20261116004703.2908294-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794790029708 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.218 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: STPNE3EYXD7SN3IG4E4RCCXKO75ZIVVT X-Message-ID-Hash: STPNE3EYXD7SN3IG4E4RCCXKO75ZIVVT X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A VM migrated to another cluster should keep its uid, unless another guest there already uses it. Take over the uid of the source through the cluster-wide check of pve-guest-common after the incoming config is written. Otherwise, the VM keeps the uid assigned when its VMID was reserved. The other members of 'meta', including the creation information, come from the source as before. Fail the config command if taking over the uid raises an error, for example because the cluster-wide lock cannot be taken. Finishing a migration that deletes the source would otherwise lose the uid because of a transient failure. Update the lock state of the tunnel first, so that the cleanup after the failure finds the target config. The existing reply timeout of two minutes covers the wait for the lock. Report the outcome in the task logs of both sides and in the reply to the config command. Older sources ignore the added field. An outcome that the source does not know, as a newer target could report, only produces a warning, because the target accepted the config at that point. The existing lock check rejects a repeated config command, so the registration time cannot be reset. Bump the tunnel version, so that older targets are refused before disks are allocated or transferred, and not only when their strict config parser rejects the identity. Keep accepting the previous tunnel version from older sources. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/Qemu.pm | 20 ++- src/PVE/QemuMigrate.pm | 9 +- src/test/run_guest_identity_tests.pl | 197 +++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 5 deletions(-) diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm index 395dc334..0ac5e14a 100644 --- a/src/PVE/API2/Qemu.pm +++ b/src/PVE/API2/Qemu.pm @@ -6867,7 +6867,7 @@ __PACKAGE__->register_method({ PVE::Cluster::check_cfs_quorum(); my $lock = 'create'; - eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); }; + my $reservation = eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, $lock); }; raise_param_exc({ vmid => "unable to create empty VM config - $@" }) if $@; @@ -7006,7 +7006,7 @@ __PACKAGE__->register_method({ # bump/bump for opt-in changes return { api => $PVE::QemuMigrate::WS_TUNNEL_VERSION, - age => 0, + age => 1, }; }, 'config' => sub { @@ -7039,6 +7039,7 @@ __PACKAGE__->register_method({ my $conf_fn = "incoming/qemu-server/$state->{vmid}.conf"; my $new_conf = PVE::QemuServer::parse_vm_config($conf_fn, $params->{conf}, 1); + my $source_identity = PVE::QemuConfig->get_identity($new_conf); delete $new_conf->{lock}; delete $new_conf->{digest}; @@ -7093,7 +7094,20 @@ __PACKAGE__->register_method({ $state->{lock} = 'migrate'; - return; + # only after the state knows the new lock, so that 'quit' can clean up if + # adopting fails + my $identity = eval { + PVE::GuestIdentity::adopt( + 'PVE::QemuConfig', + $state->{vmid}, + $source_identity && $source_identity->{uid}, + $reservation, + ); + }; + die "unable to adopt the uid of the source guest - $@" if $@; + print PVE::GuestIdentity::adopt_result_message($identity, 'source') . "\n"; + + return { identity => $identity }; }, 'bwlimit' => sub { my ($params) = @_; diff --git a/src/PVE/QemuMigrate.pm b/src/PVE/QemuMigrate.pm index 22f192e6..a2b897d8 100644 --- a/src/PVE/QemuMigrate.pm +++ b/src/PVE/QemuMigrate.pm @@ -14,6 +14,7 @@ use PVE::Cmd qw(build_ssh_command); use PVE::Format qw(render_bytes); use PVE::Firewall::Helpers; use PVE::GuestHelpers qw(safe_boolean_ne safe_string_ne); +use PVE::GuestIdentity; use PVE::INotify; use PVE::JSONSchema; use PVE::RPCEnvironment; @@ -45,7 +46,8 @@ use PVE::AbstractMigrate; use base qw(PVE::AbstractMigrate); # compared against remote end's minimum version -our $WS_TUNNEL_VERSION = 2; +# 3: the target accepts a guest identity in the config and takes over its uid +our $WS_TUNNEL_VERSION = 3; sub fork_tunnel { my ($self, $ssh_forward_info) = @_; @@ -953,7 +955,10 @@ sub phase1_remote { 'firewall-config' => $fw_conf_str, }; - PVE::Tunnel::write_tunnel($self->{tunnel}, 120, 'config', $params); + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 120, 'config', $params); + if (my $identity = $res->{identity}) { + $self->log(PVE::GuestIdentity::remote_adopt_result_log($identity)); + } } sub phase1_cleanup { diff --git a/src/test/run_guest_identity_tests.pl b/src/test/run_guest_identity_tests.pl index 3429fca6..b201594a 100755 --- a/src/test/run_guest_identity_tests.pl +++ b/src/test/run_guest_identity_tests.pl @@ -9,6 +9,9 @@ use warnings; use lib qw(.. .); use File::Temp qw(tempdir); +use IO::Handle; +use JSON; +use Socket; use Test::MockModule; use Test::More; @@ -16,6 +19,7 @@ use PVE::API2::Qemu; use PVE::JSONSchema; use PVE::QemuConfig; use PVE::QemuConfig::NoWrite; +use PVE::QemuMigrate; use PVE::QemuServer; use PVE::QemuServer::MetaInfo; use PVE::RPCEnvironment; @@ -333,4 +337,197 @@ subtest 'importovf' => sub { ); }; +# Runs the migration tunnel endpoint for $vmid with the commands @$commands from the source side +# and returns the decoded replies, the task output and the error of the task. +sub run_mtunnel { + my ($vmid, $commands) = @_; + + socketpair(my $client, my $server, AF_UNIX, SOCK_STREAM, PF_UNSPEC) + or die "socketpair failed - $!\n"; + $client->autoflush(1); + + print $client encode_json($_) . "\n" for $commands->@*; + shutdown($client, 1); + + my $rpcenv_module = Test::MockModule->new('PVE::RPCEnvironment'); + $rpcenv_module->mock( + fork_worker => sub { + my ($self, $type, $id, $user, $code) = @_; + $code->('UPID:test'); + return 'UPID:test'; + }, + ); + my $access_control_module = Test::MockModule->new('PVE::AccessControl'); + $access_control_module->mock(assemble_tunnel_ticket => sub { return 'ticket'; }); + my $socket_module = Test::MockModule->new('IO::Socket::UNIX'); + $socket_module->mock(new => sub { return bless { conn => $server }, 'TestListenSocket'; }); + + my ($output, $err) = ('', ''); + { + open(my $out_fh, '>', \$output); + my $oldfh = select($out_fh); + eval { PVE::API2::Qemu->mtunnel({ node => 'localhost', vmid => $vmid }) }; + $err = $@; + select($oldfh); + close($out_fh); + } + close($server); + + my $replies = []; + while (defined(my $line = <$client>)) { + push $replies->@*, decode_json($line); + } + + return ($replies, $output, $err); +} + +package TestListenSocket { + + sub accept { + my ($self) = @_; + return $self->{conn}; + } +} + +subtest 'remote migration, target side' => sub { + QemuServerTest::FakeCFS::reset(); + + my $cluster_module = Test::MockModule->new('PVE::Cluster'); + $cluster_module->mock(check_cfs_quorum => sub { return 1; }); + my $storage_module = Test::MockModule->new('PVE::Storage'); + $storage_module->mock(config => sub { return {}; }); + my $replication_config_module = Test::MockModule->new('PVE::ReplicationConfig'); + $replication_config_module->mock( + new => sub { return bless {}, 'PVE::ReplicationConfig'; }, + check_for_existing_jobs => sub { return; }, + ); + my $qemu_server_module = Test::MockModule->new('PVE::QemuServer'); + $qemu_server_module->mock(check_running => sub { return 0; }); + + my $source_conf = "cores: 2\nmemory: 1024\nmeta: creation-qemu=9.2.0,ctime=1600000000," + . "rtime=1600000000000,uid=$UID_A\nname: source\n"; + my $config_cmd = { cmd => 'config', conf => $source_conf }; + my $quit_cmd = { cmd => 'quit', cleanup => 0 }; + + $QemuServerTest::FakeCFS::now_ms = 1800000000000; + my ($replies, $output, $err) = run_mtunnel(100, [$config_cmd, $config_cmd, $quit_cmd]); + is($err, '', 'tunnel exits normally'); + + ok($replies->[0]->{success}, 'config command succeeds'); + is_deeply( + $replies->[0]->{identity}, + { result => 'adopted', uid => $UID_A, requested => $UID_A }, + 'config command returns the adopted identity', + ); + like( + $output, + qr/^guest identity: adopted uid $UID_A from the source guest$/m, + 'target task logs it', + ); + ok(!$replies->[1]->{success}, 'repeated config command fails'); + is_deeply( + load_meta(100), + { + 'creation-qemu' => '9.2.0', + ctime => 1600000000, + rtime => 1800000000000, + uid => $UID_A, + }, + 'source uid with rtime of the first config command, other members from the source', + ); + ok($replies->[2]->{success}, 'tunnel quits'); + + QemuServerTest::FakeCFS::reset(); + $QemuServerTest::FakeCFS::other_guests->{300} = "rtime=1,uid=$UID_A"; + ($replies) = run_mtunnel(101, [$config_cmd, $quit_cmd]); + is_deeply( + $replies->[0]->{identity}, + { + result => 'in-use', + uid => generated_uid(1), + requested => $UID_A, + 'in-use-by' => 300, + }, + 'uid in use by another guest is not adopted', + ); + is(published_identity(101)->{uid}, generated_uid(1), 'new uid stays'); + + QemuServerTest::FakeCFS::reset(); + $QemuServerTest::FakeCFS::lock_domain_error = + "cfs-lock 'domain-guest-identity' error: no quorum!\n"; + ($replies, $output, $err) = + run_mtunnel(103, [$config_cmd, { cmd => 'quit', cleanup => 1 }]); + is($err, '', 'tunnel exits normally after the failed config command'); + ok(!$replies->[0]->{success}, 'config command fails if adopting fails'); + is( + $replies->[0]->{msg}, + "failed to handle 'config' command - unable to adopt the uid of the source guest" + . " - cfs-lock 'domain-guest-identity' error: no quorum!\n", + 'error explains the failure', + ); + ok($replies->[1]->{success}, 'cleanup on quit succeeds'); + ok(!defined(configs()->{103}), 'config removed by the cleanup'); + + QemuServerTest::FakeCFS::reset(); + $config_cmd->{conf} =~ s/,rtime=\d+,uid=\w+//; + ($replies) = run_mtunnel(102, [$config_cmd, $quit_cmd]); + is($replies->[0]->{identity}->{result}, 'skipped', 'source without uid'); + is(published_identity(102)->{uid}, generated_uid(1), 'new uid stays'); +}; + +subtest 'remote migration, source side' => sub { + QemuServerTest::FakeCFS::reset(); + + # the source reads the firewall config of the VM from /etc/pve, use a VMID the host lacks + my $vmid = 999999999; + configs()->{$vmid} = "cores: 2\nmemory: 1024\nmeta: rtime=1600000000000,uid=$UID_A\n"; + + my ($sent, $identity); + my $tunnel_module = Test::MockModule->new('PVE::Tunnel'); + $tunnel_module->mock( + write_tunnel => sub { + my ($tunnel, $timeout, $command, $params) = @_; + die "unexpected command '$command'\n" if $command ne 'config'; + $sent = $params->{conf}; + return { success => 1, identity => $identity }; + }, + ); + + my $self = bless { + opts => { remote => {} }, + local_volumes => {}, + volume_map => {}, + storecfg => {}, + tunnel => {}, + }, + 'PVE::QemuMigrate'; + + my $phase1 = sub { + my $output = ''; + open(my $out_fh, '>', \$output); + my $oldfh = select($out_fh); + eval { $self->phase1_remote($vmid) }; + my $err = $@; + select($oldfh); + close($out_fh); + return ($err, $output); + }; + + $identity = { result => 'adopted', uid => $UID_A, requested => $UID_A }; + my ($err, $output) = $phase1->(); + is($err, '', 'config sent'); + like($sent, qr/^meta: rtime=1600000000000,uid=$UID_A$/m, 'source config sent with its uid'); + like( + $output, + qr/guest identity: adopted uid $UID_A from the source guest$/m, + 'source task logs the outcome', + ); + + # a target with a newer version has already taken over the config at this point + $identity = { result => 'future', uid => $UID_A, requested => $UID_A }; + ($err, $output) = $phase1->(); + is($err, '', 'unknown outcome does not fail the migration'); + like($output, qr/unknown result of guest identity adoption 'future'$/m, 'it is logged'); +}; + done_testing(); -- 2.47.3