From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 780811FF0C0 for ; Wed, 18 Nov 2026 16:44:18 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id DDDFE21867; Wed, 18 Nov 2026 16:43:30 +0100 (CET) From: Jonas Theisen To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server 11/15] migrate: block migration to nodes with incompatible TSC for re-enlightenment Date: Wed, 18 Nov 2026 16:40:42 +0100 Message-ID: <20261118154207.424260-12-j.theisen@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118154207.424260-1-j.theisen@proxmox.com> References: <20261118154207.424260-1-j.theisen@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795016561595 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.085 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: RLC57YYMJPCXPQIBZXLHQHCIIUQZLUAE X-Message-ID-Hash: RLC57YYMJPCXPQIBZXLHQHCIIUQZLUAE X-MailFrom: j.theisen@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A VM with 'hv-reenlightenment' runs with the TSC frequency of the host it was started on. QEMU on the target node can only use it if KVM there supports TSC scaling, or if the target's TSC frequency is within 250 ppm, which KVM accepts without scaling. QEMU hard-codes this bound [0], matching the default of KVM's 'tsc_tolerance_ppm' module parameter [1]. Otherwise the target instance fails to start and the migration is aborted only after its preparation, for example after local disks were already allocated on the target. Broadcast the TSC frequency and scaling support as part of the node's CPU information, and check this upfront. Target nodes that do not broadcast this information yet are not blocked, but a warning is logged, as for the other checks. [0]: https://gitlab.com/qemu-project/qemu/-/blob/v11.1.0/target/i386/kvm/kvm.c#L886 [1]: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/x86/kvm/x86.c?h=v7.0#n164 Signed-off-by: Jonas Theisen --- src/PVE/API2/Qemu.pm | 8 +- src/PVE/QemuMigrate/Helpers.pm | 30 +++++ src/PVE/QemuServer/Helpers.pm | 12 ++ src/test/run_qemu_migrate_tests.pl | 183 +++++++++++++++++++++++++++++ 4 files changed, 231 insertions(+), 2 deletions(-) diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm index 5e14a1bb..17911368 100644 --- a/src/PVE/API2/Qemu.pm +++ b/src/PVE/API2/Qemu.pm @@ -5528,8 +5528,12 @@ __PACKAGE__->register_method({ type => { type => 'string', description => "The check that failed.", - enum => - ['cpu-vendor', 'host-cpu-flags', 'kept-cpu-flags'], + enum => [ + 'cpu-vendor', + 'host-cpu-flags', + 'kept-cpu-flags', + 'tsc-frequency', + ], }, message => { type => 'string', diff --git a/src/PVE/QemuMigrate/Helpers.pm b/src/PVE/QemuMigrate/Helpers.pm index 37fa10d4..df23d605 100644 --- a/src/PVE/QemuMigrate/Helpers.pm +++ b/src/PVE/QemuMigrate/Helpers.pm @@ -323,6 +323,36 @@ my $target_cpu_checks = [ . format_flag_list(\@missing); }, }, + { + type => 'tsc-frequency', + severity => 'error', + needs => ['tsc-scaling'], + applies => sub { + my ($cpu) = @_; + my $props = $cpu->{props}; + return PVE::JSONSchema::parse_boolean($props->{'hv-reenlightenment'} // 'off') + && $props->{'tsc-frequency'}; + }, + check => sub { + my ($cpu, $source, $target) = @_; + + return if $target->{'tsc-scaling'}; + + my $vm_khz = int($cpu->{props}->{'tsc-frequency'} / 1000); + my $requirement = "'hv-reenlightenment' requires TSC scaling support or a TSC" + . " frequency within 250 ppm of the VM's $vm_khz kHz"; + + my $target_khz = $target->{'tsc-khz'} + or return "$requirement, but the node's TSC frequency is unknown"; + + # without TSC scaling, KVM can only set frequencies within 250 ppm, which QEMU + # considers to be within NTP correction bounds + my $tolerance = int($target_khz * 250 / 1_000_000); + return if abs($vm_khz - $target_khz) <= $tolerance; + + return "$requirement, but the node has $target_khz kHz without TSC scaling"; + }, + }, ]; # Returns the issues of the target node, and whether all checks that can prevent the migration diff --git a/src/PVE/QemuServer/Helpers.pm b/src/PVE/QemuServer/Helpers.pm index 098d71eb..8b5f0cd8 100644 --- a/src/PVE/QemuServer/Helpers.pm +++ b/src/PVE/QemuServer/Helpers.pm @@ -426,6 +426,10 @@ encoded as JSON. The returned hash contains: =item C: the CPU model name, for example C +=item C: the TSC frequency in kHz that KVM uses for new vCPUs + +=item C: whether KVM can run vCPUs with a different TSC frequency + =back Properties that cannot be determined are missing. Consumers of the broadcast information must @@ -443,6 +447,14 @@ sub get_host_cpu_info { $info->{vendor} = $cpuinfo->{vendor} if $known->($cpuinfo->{vendor}); $info->{'model-name'} = $cpuinfo->{model} if $known->($cpuinfo->{model}); + my $tsc_info = eval { get_host_tsc_info() }; + warn "unable to query TSC information - $@" if $@; + + if ($tsc_info) { + $info->{'tsc-khz'} = $tsc_info->{khz} if defined($tsc_info->{khz}); + $info->{'tsc-scaling'} = $tsc_info->{scaling} ? 1 : 0; + } + return $info; } diff --git a/src/test/run_qemu_migrate_tests.pl b/src/test/run_qemu_migrate_tests.pl index 57aa59b1..538a55f3 100755 --- a/src/test/run_qemu_migrate_tests.pl +++ b/src/test/run_qemu_migrate_tests.pl @@ -1944,6 +1944,189 @@ my $tests = [ }, }, }, + { + name => '1033_running_hv_reenlightenment_tsc_within_bounds', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + cpu_info => { + nodes => { + pve1 => { vendor => 'GenuineIntel', 'tsc-khz' => 3000500, 'tsc-scaling' => 0 }, + }, + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'kvm64,flags=+hv-reenlightenment', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'kvm64,flags=+hv-reenlightenment', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + }, + }, + { + name => '1033_running_hv_reenlightenment_tsc_scaling', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + cpu_info => { + nodes => { + pve1 => { vendor => 'GenuineIntel', 'tsc-khz' => 2400000, 'tsc-scaling' => 1 }, + }, + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'kvm64,flags=+hv-reenlightenment', + }, + expected_calls => $default_expected_calls_online, + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'kvm64,flags=+hv-reenlightenment', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + }, + }, + { + name => '1033_running_hv_reenlightenment_tsc_mismatch', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + cpu_info => { + nodes => { + pve1 => { vendor => 'GenuineIntel', 'tsc-khz' => 2400000, 'tsc-scaling' => 0 }, + }, + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'kvm64,flags=+hv-reenlightenment', + }, + expected_calls => {}, + expect_die => "can't migrate VM to 'pve1' - incompatible CPU: 'hv-reenlightenment'" + . " requires TSC scaling support or a TSC frequency within 250 ppm of the VM's" + . " 2999999 kHz, but the node has 2400000 kHz without TSC scaling", + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'kvm64,flags=+hv-reenlightenment', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + }, + }, + { + name => '1033_running_hv_reenlightenment_tsc_unknown', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + cpu_info => { + nodes => { + pve1 => { vendor => 'GenuineIntel', 'tsc-scaling' => 0 }, + }, + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'kvm64,flags=+hv-reenlightenment', + }, + expected_calls => {}, + expect_die => "can't migrate VM to 'pve1' - incompatible CPU: 'hv-reenlightenment'" + . " requires TSC scaling support or a TSC frequency within 250 ppm of the VM's" + . " 2999999 kHz, but the node's TSC frequency is unknown", + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'kvm64,flags=+hv-reenlightenment', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + }, + }, + { + name => '1033_running_hv_reenlightenment_target_without_tsc_info', + target => 'pve1', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + cpu_info => { + nodes => { + pve1 => { vendor => 'GenuineIntel' }, + }, + 'kvm-flags' => { + pve0 => 'fpu hv-reenlightenment sse', + pve1 => 'fpu hv-reenlightenment sse', + }, + }, + opts => { + online => 1, + }, + config_patch => { + cpu => 'kvm64,flags=+hv-reenlightenment', + }, + expected_calls => $default_expected_calls_online, + expect_log => [ + "unable to check CPU compatibility of node 'pve1' - CPU information not available", + ], + expected => { + source_volids => {}, + target_volids => {}, + vm_config => get_patched_config(1033, { + cpu => 'kvm64,flags=+hv-reenlightenment', + }), + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + runningcpu => 'kvm64,+hv-reenlightenment,tsc-frequency=2999999000', + }, + }, + }, { name => '105_replicated_to_non_replication_target', target => 'pve1', -- 2.47.3