From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id C86A91FF0E1 for ; Mon, 16 Nov 2026 01:50:43 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 17F6B21937; Mon, 16 Nov 2026 01:48:38 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH container 24/34] api: remote migration: take over the guest uid from the source Date: Mon, 16 Nov 2026 01:46:36 +0100 Message-ID: <20261116004703.2908294-25-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261116004703.2908294-1-t.lamprecht@proxmox.com> References: <20261116004703.2908294-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794790030198 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.213 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: UHJXDBLN3NJIW2KNQATJ5DBK7JTMEWJ5 X-Message-ID-Hash: UHJXDBLN3NJIW2KNQATJ5DBK7JTMEWJ5 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A container migrated to another cluster should keep its uid, unless another guest there already uses it. Take over the uid of the source after the incoming config is written, while holding the config lock. Otherwise, or if the source has no uid, the container keeps the uid assigned when its VMID was reserved. The registration time is the time of the migration. Unlike for a restore, an error while taking over the uid fails the config command and aborts the migration. The source is still intact then and can clean up the target, while finishing the migration, and possibly deleting the source, would lose the uid because of a transient failure. Raise the timeout for the reply to the config command to 60 seconds, to cover the wait for the cluster-wide lock. Return the outcome, so that the task logs of both sides report it. An outcome that the source does not know, as a newer target could report, only produces a warning, because the target accepted the config at that point. The existing lock check keeps a successful config command from being repeated, so the registration time cannot be reset. An error while taking over the uid happens after the lock transition and leads to abort and cleanup, not to a retry of the command. Bump the tunnel version, so that older targets are refused before volumes are transferred, and not only when their strict parser rejects 'meta'. Keep accepting the previous version from older sources. Signed-off-by: Thomas Lamprecht --- src/PVE/API2/LXC.pm | 18 ++- src/PVE/LXC/Migrate.pm | 19 ++- src/test/lifecycle_tests.pl | 255 ++++++++++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+), 5 deletions(-) diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm index 06049d7..60b8802 100644 --- a/src/PVE/API2/LXC.pm +++ b/src/PVE/API2/LXC.pm @@ -3305,7 +3305,7 @@ __PACKAGE__->register_method({ my $socket_addr = "/run/pve/ct-$vmid.mtunnel"; my $lock = 'create'; - eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock); }; + my $reservation = eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock) }; raise_param_exc({ vmid => "unable to create empty CT config - $@" }) if $@; @@ -3372,7 +3372,7 @@ __PACKAGE__->register_method({ # bump/bump for opt-in changes return { api => $PVE::LXC::Migrate::WS_TUNNEL_VERSION, - age => 0, + age => 1, }; }, 'config' => sub { @@ -3410,6 +3410,7 @@ __PACKAGE__->register_method({ my $unprivileged = delete $new_conf->{unprivileged}; my $arch = delete $new_conf->{arch}; + my $source_identity = PVE::LXC::Config->get_identity($new_conf); # read-only for the update below, the source's values are kept as they are my $meta = delete $new_conf->{meta}; @@ -3465,7 +3466,18 @@ __PACKAGE__->register_method({ $state->{lock} = 'migrate'; - return; + # Failing the command keeps the guest on the source, instead of letting the + # migration finish with a new identity. The source cleans up the target then. + my $uid = $source_identity && $source_identity->{uid}; + my $identity = eval { + PVE::GuestIdentity::adopt( + 'PVE::LXC::Config', $state->{vmid}, $uid, $reservation, + ); + }; + die "unable to adopt the uid of the source guest - $@" if $@; + print PVE::GuestIdentity::adopt_result_message($identity, 'source') . "\n"; + + return { identity => $identity }; }, 'bwlimit' => sub { my ($params) = @_; diff --git a/src/PVE/LXC/Migrate.pm b/src/PVE/LXC/Migrate.pm index 1882e57..21aedf4 100644 --- a/src/PVE/LXC/Migrate.pm +++ b/src/PVE/LXC/Migrate.pm @@ -7,6 +7,7 @@ use File::Basename; use File::Copy; # fixme: remove use PVE::Cluster; +use PVE::GuestIdentity; use PVE::INotify; use PVE::JSONSchema; use PVE::Replication; @@ -25,7 +26,8 @@ use PVE::AbstractMigrate; use base qw(PVE::AbstractMigrate); # compared against remote end's minimum version -our $WS_TUNNEL_VERSION = 2; +# 3: the target accepts a guest identity in the config and takes over its uid +our $WS_TUNNEL_VERSION = 3; sub lock_vm { my ($self, $vmid, $code, @param) = @_; @@ -471,7 +473,9 @@ sub phase1 { 'firewall-config' => $fw_conf_str, }; - PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params); + # the target takes the cluster-wide guest identity lock in addition to its config writes + my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 60, 'config', $params); + $self->log_target_identity($res && $res->{identity}); } else { # transfer replication state before moving config $self->transfer_replication_state() if $rep_volumes; @@ -483,6 +487,17 @@ sub phase1 { $self->{conf_migrated} = 1; } +# Logs the outcome of taking over the guest identity of the source on the target of a remote +# migration, as reported by the target. +sub log_target_identity { + my ($self, $identity) = @_; + + return if !$identity; + + my ($level, $msg) = PVE::GuestIdentity::remote_adopt_result_log($identity); + $self->log($level, "target CT $self->{opts}->{remote}->{vmid}: $msg"); +} + sub phase1_cleanup { my ($self, $vmid, $err) = @_; diff --git a/src/test/lifecycle_tests.pl b/src/test/lifecycle_tests.pl index e3a86a8..e2fdb9f 100755 --- a/src/test/lifecycle_tests.pl +++ b/src/test/lifecycle_tests.pl @@ -36,6 +36,7 @@ use PVE::CLI::pct; use PVE::LXC; use PVE::LXC::Config; use PVE::LXC::Create; +use PVE::LXC::Migrate; my $WORKDIR = 'lifecycle-working'; my $NODENAME = PVE::INotify::nodename(); @@ -101,6 +102,11 @@ my $storage_cfg = { shared => 1, content => { rootdir => 1, images => 1 }, }, + iso => { + type => 'dir', + path => '/mnt/iso', + content => { iso => 1 }, + }, }, }; @@ -1592,6 +1598,255 @@ subtest 'remote migration cleans up a target with an incompatible tunnel version is(read_raw_config(200), $raw, 'config of the source is unchanged'); }; +subtest 'remote migration takes over the guest identity of the source' => sub { + my $source_meta = 'ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso'; + my $incoming = $INCOMING_CONFIG =~ s/^meta: .*$/meta: $source_meta/mr; + $now_ms = 1_700_000_000_000; + + my $migrate = sub { + my ($name, $setup, $conf, $expected_result, $expected_uid) = @_; + + reset_configs(); + $setup->(); + my ($err, $replies) = run_mtunnel( + 300, { cmd => 'config', conf => $conf }, { cmd => 'quit', cleanup => 0 }, + ); + is($err, '', "$name: tunnel exits normally") or diag(explain($replies), $task_log); + ok($replies->[0]->{success}, "$name: config command succeeds"); + + my $identity = $replies->[0]->{identity}; + is($identity->{result}, $expected_result, "$name: result is returned"); + my $published = identity_of(300); + is($identity->{uid}, $published->{uid}, "$name: returned uid is the published one"); + if (defined($expected_uid)) { + is($published->{uid}, $expected_uid, "$name: uid"); + } else { + isnt( + $published->{uid}, + 'sourcesourcesourceso', + "$name: uid is not the one of the source", + ); + } + is($published->{rtime}, $now_ms, "$name: rtime is the time of the migration"); + is(meta_of(300)->{ctime}, 1600000000, "$name: ctime of the source is kept"); + like( + $task_log, + qr/^guest identity: .*$identity->{uid}/m, + "$name: result is logged on the target", + ); + + return $identity; + }; + + $migrate->('free uid', sub { }, $incoming, 'adopted', 'sourcesourcesourceso'); + $migrate->( + 'uid used by another container', + sub { write_raw_config(150, existing_config('uid=sourcesourcesourceso')) }, + $incoming, + 'in-use', + ); + $migrate->( + 'uid used by a VM', + sub { write_raw_vm_config(151, "memory: 512\nmeta: uid=sourcesourcesourceso\n") }, + $incoming, + 'in-use', + ); + $migrate->('source without identity', sub { }, $INCOMING_CONFIG, 'skipped'); + + my $real_adopt = \&PVE::GuestIdentity::adopt; + my ($lock, $locked); + $identity_module->mock( + adopt => sub { + my ($class, $vmid) = @_; + $lock = PVE::LXC::Config->load_config($vmid)->{lock}; + open(my $fh, '>>', "$WORKDIR/lock/pve-config-$vmid.lock") + or die "open failed - $!\n"; + $locked = !flock($fh, LOCK_EX | LOCK_NB); + close($fh); + return $real_adopt->(@_); + }, + ); + $migrate->('order of the writes', sub { }, $incoming, 'adopted', 'sourcesourcesourceso'); + $identity_module->unmock('adopt'); + is($lock, 'migrate', 'identity is taken over after the final config write'); + ok($locked, 'config lock is held while taking over the identity'); + + reset_configs(); + $domain_lock_error = "cfs-lock 'domain-guest-identity' error: got lock request timeout\n"; + my ($err, $replies) = run_mtunnel( + 300, { cmd => 'config', conf => $incoming }, { cmd => 'quit', cleanup => 1 }, + ); + $domain_lock_error = undef; + is($err, '', 'identity lock failure: tunnel exits normally') or diag($task_log); + ok(!$replies->[0]->{success}, 'identity lock failure: config command fails'); + is( + $replies->[0]->{msg}, + "failed to handle 'config' command - unable to adopt the uid of the source guest -" + . " cfs-lock 'domain-guest-identity' error: got lock request timeout\n", + 'identity lock failure: error is returned', + ); + ok($replies->[1]->{success}, 'identity lock failure: cleanup succeeds'); + ok(!defined(read_raw_config(300)), 'identity lock failure: target config is removed'); + + reset_configs(); + my $broken = $incoming =~ s/^rootfs: local:/rootfs: iso:/mr; + ($err, $replies) = run_mtunnel( + 300, + { cmd => 'config', conf => $broken }, + { cmd => 'config', conf => $incoming }, + { cmd => 'config', conf => $incoming }, + { cmd => 'quit', cleanup => 0 }, + ); + is($err, '', 'repeated config: tunnel exits normally'); + ok(!$replies->[0]->{success}, 'failing config command fails'); + is($replies->[1]->{identity}->{result}, 'adopted', 'retry takes over the uid'); + ok(!$replies->[2]->{success}, 'config command after success fails'); + is_deeply( + identity_of(300), + { uid => 'sourcesourcesourceso', rtime => $now_ms }, + 'identity is published once', + ); +}; + +subtest 'remote migration logs the guest identity on the source' => sub { + my $source_raw = <<~'EOF'; + arch: amd64 + hostname: source + memory: 512 + meta: ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso + ostype: debian + unprivileged: 1 + EOF + reset_configs(); + write_raw_config(987, $source_raw); + + my $log = ''; + my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate'); + $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" }); + + my ($sent_conf, $config_timeout, $reply); + my $tunnel_module = Test::MockModule->new('PVE::Tunnel'); + $tunnel_module->mock( + write_tunnel => sub { + my ($tunnel, $timeout, $command, $params) = @_; + die "unexpected command '$command'\n" if $command ne 'config'; + ($sent_conf, $config_timeout) = ($params->{conf}, $timeout); + return $reply; + }, + ); + + my $phase1 = sub { + my $self = bless { + opts => { remote => { vmid => 300 } }, + vmid => 987, + node => 'remote-node', + nodeip => '192.0.2.1', + storecfg => $storage_cfg, + vmconf => PVE::LXC::Config->load_config(987), + tunnel => {}, + }, + 'PVE::LXC::Migrate'; + $log = ''; + return run_task(sub { $self->phase1(987) }); + }; + + $reply = { + success => 1, + identity => { + result => 'adopted', + uid => 'sourcesourcesourceso', + requested => 'sourcesourcesourceso', + }, + }; + is($phase1->(), '', 'phase 1 succeeds') or diag($log, $task_log); + like($sent_conf, qr/^meta: .*uid=sourcesourcesourceso/m, 'identity is sent to the target'); + is($config_timeout, 60, 'source waits for the identity lock of the target'); + my $adopted = + 'target CT 300: guest identity: adopted uid sourcesourcesourceso from the source guest'; + like($log, qr/^info: \Q$adopted\E$/m, 'result of the target is logged'); + is(identity_of(987)->{uid}, 'sourcesourcesourceso', 'source keeps its uid'); + + PVE::LXC::Config->remove_lock(987, 'migrate'); + $reply = { success => 1 }; + is($phase1->(), '', 'phase 1 with a target without guest identity succeeds'); + unlike($log, qr/guest identity/, 'nothing is logged about the identity'); + PVE::LXC::Config->remove_lock(987, 'migrate'); + + # a target with a newer version has already taken over the config at this point + $reply = + { success => 1, identity => { result => 'future', uid => 'sourcesourcesourceso' } }; + is($phase1->(), '', 'phase 1 with an outcome unknown here succeeds'); + like( + $log, + qr/^warn: target CT 300: unknown result of guest identity adoption 'future'$/m, + 'unknown outcome is a warning', + ); + PVE::LXC::Config->remove_lock(987, 'migrate'); +}; + +subtest 'remote migration keeps the source if the target fails to take over the identity' => sub { + reset_configs(); + write_raw_config(987, <<~'EOF'); + arch: amd64 + hostname: source + memory: 512 + meta: ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso + ostype: debian + unprivileged: 1 + EOF + + my $log = ''; + my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate'); + $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" }); + + my $cleanup; + my $tunnel_module = Test::MockModule->new('PVE::Tunnel'); + $tunnel_module->mock( + fork_websocket_tunnel => sub { return { version => 3, age => 1 } }, + write_tunnel => sub { + my ($tunnel, $timeout, $command) = @_; + die "error - tunnel command '$command' failed - failed to handle 'config' command" + . " - unable to adopt the uid of the source guest - cfs-lock" + . " 'domain-guest-identity' error: got lock request timeout\n"; + }, + finish_tunnel => sub { (undef, $cleanup) = @_ }, + ); + + my ($running, $started, $destroyed) = (1, 0, 0); + my $destroy_ct = \&PVE::LXC::destroy_lxc_container; + $lxc_module->mock( + check_running => sub { return $running }, + vm_stop => sub { $running = 0 }, + vm_start => sub { $started = 1 }, + destroy_lxc_container => sub { $destroyed = 1 }, + ); + + my $err = run_task(sub { + PVE::LXC::Migrate->migrate( + 'remote-node', + '192.0.2.1', + 987, + { + remote => { vmid => 300, conn => { host => '192.0.2.1', port => 8006 } }, + restart => 1, + delete => 1, + }, + ); + }); + + $lxc_module->mock(check_running => sub { return 0 }, destroy_lxc_container => $destroy_ct); + $lxc_module->unmock('vm_stop', 'vm_start'); + + like($err, qr/migration aborted/, 'migration fails'); + like($log, qr/^err: .*lock request timeout/m, 'error of the target is logged'); + is($cleanup, 1, 'target gets cleaned up'); + ok(!$destroyed, 'source is not destroyed'); + ok($started, 'source is started again'); + my $conf = PVE::LXC::Config->load_config(987); + ok(!defined($conf->{lock}), 'migrate lock of the source is removed'); + is(identity_of(987)->{uid}, 'sourcesourcesourceso', 'source keeps its uid'); +}; + remove_tree($WORKDIR); done_testing(); -- 2.47.3