From: Thomas Lamprecht <t.lamprecht@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [PATCH container 24/34] api: remote migration: take over the guest uid from the source
Date: Mon, 16 Nov 2026 01:46:36 +0100 [thread overview]
Message-ID: <20261116004703.2908294-25-t.lamprecht@proxmox.com> (raw)
In-Reply-To: <20261116004703.2908294-1-t.lamprecht@proxmox.com>
A container migrated to another cluster should keep its uid, unless
another guest there already uses it. Take over the uid of the source
after the incoming config is written, while holding the config lock.
Otherwise, or if the source has no uid, the container keeps the uid
assigned when its VMID was reserved. The registration time is the time
of the migration.
Unlike for a restore, an error while taking over the uid fails the
config command and aborts the migration. The source is still intact
then and can clean up the target, while finishing the migration, and
possibly deleting the source, would lose the uid because of a transient
failure. Raise the timeout for the reply to the config command to 60
seconds, to cover the wait for the cluster-wide lock.
Return the outcome, so that the task logs of both sides report it. An
outcome that the source does not know, as a newer target could report,
only produces a warning, because the target accepted the config at that
point. The existing lock check keeps a successful config command from
being repeated, so the registration time cannot be reset. An error
while taking over the uid happens after the lock transition and leads
to abort and cleanup, not to a retry of the command.
Bump the tunnel version, so that older targets are refused before
volumes are transferred, and not only when their strict parser rejects
'meta'. Keep accepting the previous version from older sources.
Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
src/PVE/API2/LXC.pm | 18 ++-
src/PVE/LXC/Migrate.pm | 19 ++-
src/test/lifecycle_tests.pl | 255 ++++++++++++++++++++++++++++++++++++
3 files changed, 287 insertions(+), 5 deletions(-)
diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm
index 06049d7..60b8802 100644
--- a/src/PVE/API2/LXC.pm
+++ b/src/PVE/API2/LXC.pm
@@ -3305,7 +3305,7 @@ __PACKAGE__->register_method({
my $socket_addr = "/run/pve/ct-$vmid.mtunnel";
my $lock = 'create';
- eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock); };
+ my $reservation = eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock) };
raise_param_exc({ vmid => "unable to create empty CT config - $@" })
if $@;
@@ -3372,7 +3372,7 @@ __PACKAGE__->register_method({
# bump/bump for opt-in changes
return {
api => $PVE::LXC::Migrate::WS_TUNNEL_VERSION,
- age => 0,
+ age => 1,
};
},
'config' => sub {
@@ -3410,6 +3410,7 @@ __PACKAGE__->register_method({
my $unprivileged = delete $new_conf->{unprivileged};
my $arch = delete $new_conf->{arch};
+ my $source_identity = PVE::LXC::Config->get_identity($new_conf);
# read-only for the update below, the source's values are kept as they are
my $meta = delete $new_conf->{meta};
@@ -3465,7 +3466,18 @@ __PACKAGE__->register_method({
$state->{lock} = 'migrate';
- return;
+ # Failing the command keeps the guest on the source, instead of letting the
+ # migration finish with a new identity. The source cleans up the target then.
+ my $uid = $source_identity && $source_identity->{uid};
+ my $identity = eval {
+ PVE::GuestIdentity::adopt(
+ 'PVE::LXC::Config', $state->{vmid}, $uid, $reservation,
+ );
+ };
+ die "unable to adopt the uid of the source guest - $@" if $@;
+ print PVE::GuestIdentity::adopt_result_message($identity, 'source') . "\n";
+
+ return { identity => $identity };
},
'bwlimit' => sub {
my ($params) = @_;
diff --git a/src/PVE/LXC/Migrate.pm b/src/PVE/LXC/Migrate.pm
index 1882e57..21aedf4 100644
--- a/src/PVE/LXC/Migrate.pm
+++ b/src/PVE/LXC/Migrate.pm
@@ -7,6 +7,7 @@ use File::Basename;
use File::Copy; # fixme: remove
use PVE::Cluster;
+use PVE::GuestIdentity;
use PVE::INotify;
use PVE::JSONSchema;
use PVE::Replication;
@@ -25,7 +26,8 @@ use PVE::AbstractMigrate;
use base qw(PVE::AbstractMigrate);
# compared against remote end's minimum version
-our $WS_TUNNEL_VERSION = 2;
+# 3: the target accepts a guest identity in the config and takes over its uid
+our $WS_TUNNEL_VERSION = 3;
sub lock_vm {
my ($self, $vmid, $code, @param) = @_;
@@ -471,7 +473,9 @@ sub phase1 {
'firewall-config' => $fw_conf_str,
};
- PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params);
+ # the target takes the cluster-wide guest identity lock in addition to its config writes
+ my $res = PVE::Tunnel::write_tunnel($self->{tunnel}, 60, 'config', $params);
+ $self->log_target_identity($res && $res->{identity});
} else {
# transfer replication state before moving config
$self->transfer_replication_state() if $rep_volumes;
@@ -483,6 +487,17 @@ sub phase1 {
$self->{conf_migrated} = 1;
}
+# Logs the outcome of taking over the guest identity of the source on the target of a remote
+# migration, as reported by the target.
+sub log_target_identity {
+ my ($self, $identity) = @_;
+
+ return if !$identity;
+
+ my ($level, $msg) = PVE::GuestIdentity::remote_adopt_result_log($identity);
+ $self->log($level, "target CT $self->{opts}->{remote}->{vmid}: $msg");
+}
+
sub phase1_cleanup {
my ($self, $vmid, $err) = @_;
diff --git a/src/test/lifecycle_tests.pl b/src/test/lifecycle_tests.pl
index e3a86a8..e2fdb9f 100755
--- a/src/test/lifecycle_tests.pl
+++ b/src/test/lifecycle_tests.pl
@@ -36,6 +36,7 @@ use PVE::CLI::pct;
use PVE::LXC;
use PVE::LXC::Config;
use PVE::LXC::Create;
+use PVE::LXC::Migrate;
my $WORKDIR = 'lifecycle-working';
my $NODENAME = PVE::INotify::nodename();
@@ -101,6 +102,11 @@ my $storage_cfg = {
shared => 1,
content => { rootdir => 1, images => 1 },
},
+ iso => {
+ type => 'dir',
+ path => '/mnt/iso',
+ content => { iso => 1 },
+ },
},
};
@@ -1592,6 +1598,255 @@ subtest 'remote migration cleans up a target with an incompatible tunnel version
is(read_raw_config(200), $raw, 'config of the source is unchanged');
};
+subtest 'remote migration takes over the guest identity of the source' => sub {
+ my $source_meta = 'ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso';
+ my $incoming = $INCOMING_CONFIG =~ s/^meta: .*$/meta: $source_meta/mr;
+ $now_ms = 1_700_000_000_000;
+
+ my $migrate = sub {
+ my ($name, $setup, $conf, $expected_result, $expected_uid) = @_;
+
+ reset_configs();
+ $setup->();
+ my ($err, $replies) = run_mtunnel(
+ 300, { cmd => 'config', conf => $conf }, { cmd => 'quit', cleanup => 0 },
+ );
+ is($err, '', "$name: tunnel exits normally") or diag(explain($replies), $task_log);
+ ok($replies->[0]->{success}, "$name: config command succeeds");
+
+ my $identity = $replies->[0]->{identity};
+ is($identity->{result}, $expected_result, "$name: result is returned");
+ my $published = identity_of(300);
+ is($identity->{uid}, $published->{uid}, "$name: returned uid is the published one");
+ if (defined($expected_uid)) {
+ is($published->{uid}, $expected_uid, "$name: uid");
+ } else {
+ isnt(
+ $published->{uid},
+ 'sourcesourcesourceso',
+ "$name: uid is not the one of the source",
+ );
+ }
+ is($published->{rtime}, $now_ms, "$name: rtime is the time of the migration");
+ is(meta_of(300)->{ctime}, 1600000000, "$name: ctime of the source is kept");
+ like(
+ $task_log,
+ qr/^guest identity: .*$identity->{uid}/m,
+ "$name: result is logged on the target",
+ );
+
+ return $identity;
+ };
+
+ $migrate->('free uid', sub { }, $incoming, 'adopted', 'sourcesourcesourceso');
+ $migrate->(
+ 'uid used by another container',
+ sub { write_raw_config(150, existing_config('uid=sourcesourcesourceso')) },
+ $incoming,
+ 'in-use',
+ );
+ $migrate->(
+ 'uid used by a VM',
+ sub { write_raw_vm_config(151, "memory: 512\nmeta: uid=sourcesourcesourceso\n") },
+ $incoming,
+ 'in-use',
+ );
+ $migrate->('source without identity', sub { }, $INCOMING_CONFIG, 'skipped');
+
+ my $real_adopt = \&PVE::GuestIdentity::adopt;
+ my ($lock, $locked);
+ $identity_module->mock(
+ adopt => sub {
+ my ($class, $vmid) = @_;
+ $lock = PVE::LXC::Config->load_config($vmid)->{lock};
+ open(my $fh, '>>', "$WORKDIR/lock/pve-config-$vmid.lock")
+ or die "open failed - $!\n";
+ $locked = !flock($fh, LOCK_EX | LOCK_NB);
+ close($fh);
+ return $real_adopt->(@_);
+ },
+ );
+ $migrate->('order of the writes', sub { }, $incoming, 'adopted', 'sourcesourcesourceso');
+ $identity_module->unmock('adopt');
+ is($lock, 'migrate', 'identity is taken over after the final config write');
+ ok($locked, 'config lock is held while taking over the identity');
+
+ reset_configs();
+ $domain_lock_error = "cfs-lock 'domain-guest-identity' error: got lock request timeout\n";
+ my ($err, $replies) = run_mtunnel(
+ 300, { cmd => 'config', conf => $incoming }, { cmd => 'quit', cleanup => 1 },
+ );
+ $domain_lock_error = undef;
+ is($err, '', 'identity lock failure: tunnel exits normally') or diag($task_log);
+ ok(!$replies->[0]->{success}, 'identity lock failure: config command fails');
+ is(
+ $replies->[0]->{msg},
+ "failed to handle 'config' command - unable to adopt the uid of the source guest -"
+ . " cfs-lock 'domain-guest-identity' error: got lock request timeout\n",
+ 'identity lock failure: error is returned',
+ );
+ ok($replies->[1]->{success}, 'identity lock failure: cleanup succeeds');
+ ok(!defined(read_raw_config(300)), 'identity lock failure: target config is removed');
+
+ reset_configs();
+ my $broken = $incoming =~ s/^rootfs: local:/rootfs: iso:/mr;
+ ($err, $replies) = run_mtunnel(
+ 300,
+ { cmd => 'config', conf => $broken },
+ { cmd => 'config', conf => $incoming },
+ { cmd => 'config', conf => $incoming },
+ { cmd => 'quit', cleanup => 0 },
+ );
+ is($err, '', 'repeated config: tunnel exits normally');
+ ok(!$replies->[0]->{success}, 'failing config command fails');
+ is($replies->[1]->{identity}->{result}, 'adopted', 'retry takes over the uid');
+ ok(!$replies->[2]->{success}, 'config command after success fails');
+ is_deeply(
+ identity_of(300),
+ { uid => 'sourcesourcesourceso', rtime => $now_ms },
+ 'identity is published once',
+ );
+};
+
+subtest 'remote migration logs the guest identity on the source' => sub {
+ my $source_raw = <<~'EOF';
+ arch: amd64
+ hostname: source
+ memory: 512
+ meta: ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso
+ ostype: debian
+ unprivileged: 1
+ EOF
+ reset_configs();
+ write_raw_config(987, $source_raw);
+
+ my $log = '';
+ my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate');
+ $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" });
+
+ my ($sent_conf, $config_timeout, $reply);
+ my $tunnel_module = Test::MockModule->new('PVE::Tunnel');
+ $tunnel_module->mock(
+ write_tunnel => sub {
+ my ($tunnel, $timeout, $command, $params) = @_;
+ die "unexpected command '$command'\n" if $command ne 'config';
+ ($sent_conf, $config_timeout) = ($params->{conf}, $timeout);
+ return $reply;
+ },
+ );
+
+ my $phase1 = sub {
+ my $self = bless {
+ opts => { remote => { vmid => 300 } },
+ vmid => 987,
+ node => 'remote-node',
+ nodeip => '192.0.2.1',
+ storecfg => $storage_cfg,
+ vmconf => PVE::LXC::Config->load_config(987),
+ tunnel => {},
+ },
+ 'PVE::LXC::Migrate';
+ $log = '';
+ return run_task(sub { $self->phase1(987) });
+ };
+
+ $reply = {
+ success => 1,
+ identity => {
+ result => 'adopted',
+ uid => 'sourcesourcesourceso',
+ requested => 'sourcesourcesourceso',
+ },
+ };
+ is($phase1->(), '', 'phase 1 succeeds') or diag($log, $task_log);
+ like($sent_conf, qr/^meta: .*uid=sourcesourcesourceso/m, 'identity is sent to the target');
+ is($config_timeout, 60, 'source waits for the identity lock of the target');
+ my $adopted =
+ 'target CT 300: guest identity: adopted uid sourcesourcesourceso from the source guest';
+ like($log, qr/^info: \Q$adopted\E$/m, 'result of the target is logged');
+ is(identity_of(987)->{uid}, 'sourcesourcesourceso', 'source keeps its uid');
+
+ PVE::LXC::Config->remove_lock(987, 'migrate');
+ $reply = { success => 1 };
+ is($phase1->(), '', 'phase 1 with a target without guest identity succeeds');
+ unlike($log, qr/guest identity/, 'nothing is logged about the identity');
+ PVE::LXC::Config->remove_lock(987, 'migrate');
+
+ # a target with a newer version has already taken over the config at this point
+ $reply =
+ { success => 1, identity => { result => 'future', uid => 'sourcesourcesourceso' } };
+ is($phase1->(), '', 'phase 1 with an outcome unknown here succeeds');
+ like(
+ $log,
+ qr/^warn: target CT 300: unknown result of guest identity adoption 'future'$/m,
+ 'unknown outcome is a warning',
+ );
+ PVE::LXC::Config->remove_lock(987, 'migrate');
+};
+
+subtest 'remote migration keeps the source if the target fails to take over the identity' => sub {
+ reset_configs();
+ write_raw_config(987, <<~'EOF');
+ arch: amd64
+ hostname: source
+ memory: 512
+ meta: ctime=1600000000,rtime=1600000000000,uid=sourcesourcesourceso
+ ostype: debian
+ unprivileged: 1
+ EOF
+
+ my $log = '';
+ my $migrate_module = Test::MockModule->new('PVE::AbstractMigrate');
+ $migrate_module->mock(log => sub { $log .= "$_[1]: $_[2]\n" });
+
+ my $cleanup;
+ my $tunnel_module = Test::MockModule->new('PVE::Tunnel');
+ $tunnel_module->mock(
+ fork_websocket_tunnel => sub { return { version => 3, age => 1 } },
+ write_tunnel => sub {
+ my ($tunnel, $timeout, $command) = @_;
+ die "error - tunnel command '$command' failed - failed to handle 'config' command"
+ . " - unable to adopt the uid of the source guest - cfs-lock"
+ . " 'domain-guest-identity' error: got lock request timeout\n";
+ },
+ finish_tunnel => sub { (undef, $cleanup) = @_ },
+ );
+
+ my ($running, $started, $destroyed) = (1, 0, 0);
+ my $destroy_ct = \&PVE::LXC::destroy_lxc_container;
+ $lxc_module->mock(
+ check_running => sub { return $running },
+ vm_stop => sub { $running = 0 },
+ vm_start => sub { $started = 1 },
+ destroy_lxc_container => sub { $destroyed = 1 },
+ );
+
+ my $err = run_task(sub {
+ PVE::LXC::Migrate->migrate(
+ 'remote-node',
+ '192.0.2.1',
+ 987,
+ {
+ remote => { vmid => 300, conn => { host => '192.0.2.1', port => 8006 } },
+ restart => 1,
+ delete => 1,
+ },
+ );
+ });
+
+ $lxc_module->mock(check_running => sub { return 0 }, destroy_lxc_container => $destroy_ct);
+ $lxc_module->unmock('vm_stop', 'vm_start');
+
+ like($err, qr/migration aborted/, 'migration fails');
+ like($log, qr/^err: .*lock request timeout/m, 'error of the target is logged');
+ is($cleanup, 1, 'target gets cleaned up');
+ ok(!$destroyed, 'source is not destroyed');
+ ok($started, 'source is started again');
+ my $conf = PVE::LXC::Config->load_config(987);
+ ok(!defined($conf->{lock}), 'migrate lock of the source is removed');
+ is(identity_of(987)->{uid}, 'sourcesourcesourceso', 'source keeps its uid');
+};
+
remove_tree($WORKDIR);
done_testing();
--
2.47.3
next prev parent reply other threads:[~2026-11-16 0:50 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-16 0:46 [PATCH cluster/guest-common/qemu-server/container/manager/docs 00/34] fix #7846: guest identity (uid) and cluster ID Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 01/34] pmxcfs: reject changes to the cluster ID file from user space Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 02/34] pmxcfs: generate a cluster ID on standalone nodes Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 03/34] pmxcfs: dfsm: add a callback that runs once all members are in sync Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 04/34] pmxcfs: quorum: pass the corosync membership to an optional listener Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 05/34] pmxcfs: dcdb: generate the cluster ID once the cluster is in sync Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 06/34] cluster: add helper to read the cluster ID Thomas Lamprecht
2026-11-16 0:46 ` [PATCH cluster 07/34] pmxcfs: allow removing the cluster ID file in forced local mode Thomas Lamprecht
2026-11-16 0:46 ` [PATCH guest-common 08/34] guest identity: add helpers for the guest uid and registration time Thomas Lamprecht
2026-11-17 10:47 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH guest-common 09/34] abstract config: assign a guest identity and keep it on config writes Thomas Lamprecht
2026-11-17 10:47 ` Dominik Csapak
2026-11-17 12:47 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH guest-common 10/34] abstract config: let the VMID reservation return the previous state Thomas Lamprecht
2026-11-16 0:46 ` [PATCH guest-common 11/34] guest identity: allow taking over a uid that is unused in the cluster Thomas Lamprecht
2026-11-17 13:14 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH qemu-server 12/34] api: restore: release the VMID if a restore to an unused VMID fails Thomas Lamprecht
2026-11-16 0:46 ` [PATCH qemu-server 13/34] api: migrate, clone: refuse targets that would drop the guest identity Thomas Lamprecht
2026-11-16 0:46 ` [PATCH qemu-server 14/34] fix #7846: config: add a guest identity to the 'meta' property Thomas Lamprecht
2026-11-16 0:46 ` [PATCH qemu-server 15/34] restore: take over the guest uid from the backup where possible Thomas Lamprecht
2026-11-17 15:17 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH qemu-server 16/34] api: remote migration: take over the guest uid from the source Thomas Lamprecht
2026-11-17 15:26 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH qemu-server 17/34] api: status: return the guest uid Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 18/34] config: add read-only meta property with the creation time Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 19/34] fix #7846: config: add a guest identity to the 'meta' property Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 20/34] api: migrate, clone: refuse targets that would drop the guest identity Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 21/34] restore: take over the guest uid from the backup where possible Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 22/34] restore: refuse overwriting a container with another guest's backup Thomas Lamprecht
2026-11-16 0:46 ` [PATCH container 23/34] migrate: remote: clean up the target on a tunnel version mismatch Thomas Lamprecht
2026-11-16 0:46 ` Thomas Lamprecht [this message]
2026-11-16 0:46 ` [PATCH container 25/34] api: status: return the guest uid Thomas Lamprecht
2026-11-16 0:46 ` [PATCH manager 26/34] api: cluster resources: add guest uid and registration time Thomas Lamprecht
2026-11-17 15:30 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH manager 27/34] api: cluster status: add cluster ID Thomas Lamprecht
2026-11-16 0:46 ` [PATCH manager 28/34] ui: restore: allow overwriting a guest with a backup of another guest Thomas Lamprecht
2026-11-16 0:46 ` [PATCH manager 29/34] ui: migrate: refuse target nodes that would drop the guest identity Thomas Lamprecht
2026-11-16 0:46 ` [PATCH manager 30/34] ui: guest summary: show " Thomas Lamprecht
2026-11-17 15:44 ` Dominik Csapak
2026-11-16 0:46 ` [PATCH docs 31/34] vzdump: describe the guest identity of VMs and containers Thomas Lamprecht
2026-11-16 0:46 ` [PATCH docs 32/34] pvecm: document the cluster ID and how to get a new one Thomas Lamprecht
2026-11-16 0:46 ` [PATCH docs 33/34] qm: configuration: describe the read-only meta property Thomas Lamprecht
2026-11-16 0:46 ` [PATCH docs 34/34] pct: " Thomas Lamprecht
2026-11-17 10:47 ` [PATCH cluster/guest-common/qemu-server/container/manager/docs 00/34] fix #7846: guest identity (uid) and cluster ID Dominik Csapak
2026-11-17 11:29 ` Thomas Lamprecht
2026-11-17 11:55 ` Dominik Csapak
2026-11-17 12:47 ` Thomas Lamprecht
2026-11-17 13:54 ` Dominik Csapak
2026-11-17 15:55 ` Dominik Csapak
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116004703.2908294-25-t.lamprecht@proxmox.com \
--to=t.lamprecht@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.