From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id D9DFF1FF0AD for ; Sun, 20 Sep 2026 18:26:40 +0200 (CEST) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 84AC2214C3; Sun, 20 Sep 2026 18:26:38 +0200 (CEST) ARC-Seal: i=1; a=rsa-sha256; s=arcselector10001; d=microsoft.com; cv=none; b=F0axOZCVDTj8Bs5/dVajZUiQQNXqlJRgEPCnqhMTAf7ju4UIYu8Z7r7X3Gic7va/KY0ts9ocSsF9dsGVFotJSqnEhie8iMDPtiO46dTKjA8rfaLulL3VvBpjD+s1pofUPCF9UWP7Zy/MoFbs9DjgnWdwnKzAXmn9yZVWyhBjpW20fCGOQZZ1R4R/ggXXDpxUkZAeZ5q4/HFMdBSV74iLYFBUgwH/oUHLAbG+iY7hfu9F/bWBU0rv4uZpnkQeCjxf4jHor+PZuFfJpSx1fEyRKDTUkaJObyQ0ySV7Ms8O3EPDZgP1Qd0TjlcCERPverZwEuvg4T/6Fd+wmSBFoV9Xqg== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; s=arcselector10001; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; bh=KVQRi13oCWhDhc8EpwfVjAuQB6G3s5qmesZPGsovM7Y=; b=RDAjditwDQCLZ5kyeiPfK4w0gcAT2T4k+/E+p+VFAWEFpoOQgnStvFYrBeghlF+aBgcrDpLiPfWYdOf43mO128+oq44dom1scyQJQH/A8PK38IFc3UcjNWmEvyt0dz/sdjB4EKPM0TrFqEhEB0vAhxdnTw+TFGgPMASZIC50fD4kXxmCaQ9WCxEaT3gCq+phvtZqshVYyDw+UZRczwaQpkaai6Mw1fVp0DxZzvQMTGY18f85cVeON0ntC8Jxl2A6YeEp7R3fp7NnLIBFolbARA5ciTu8QvfzmVzOUWsQqsjvOM8FfQHfD9FClDrA9PBj5bmwQV2onsazFnTP1FKSPw== ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=pass smtp.mailfrom=ryomherold.dk; dmarc=pass action=none header.from=ryomherold.dk; dkim=pass header.d=ryomherold.dk; arc=none DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=RyomHerold.dk; s=selector2; h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; bh=KVQRi13oCWhDhc8EpwfVjAuQB6G3s5qmesZPGsovM7Y=; b=eEqdCUiuywcuR3hw5ras3OnCd/eLzyiL+i1sMYcKbhYnyb1fdOcAQ19iqtYHMMb8KrwQdnMzhRqVkFyWLvWiiG13YnWk4yf3SREYJcDW3Qn1dhr+b4tdtbq+bacK6u+MNaS5xQnU52LDW3pP/fzvS//pd56BfK078LlsySGxM047vjqMeQ12nOVADDKc4+PK8b6Sy53l3PU1c0F2zscdT1IJIzXL7xFlhmrssGC598YlfM4QFBWcdiZTcbmsW1hphi+rjsUkBURG5clSmKHBqcUKWAsRz7xhmZdH8K+HWPHrlAYj3y3ON3bBKmQeilr5wD/H5WlTCGC2NKEFHJg6zQ== Authentication-Results: dkim=none (message not signed) header.d=none;dmarc=none action=none header.from=RyomHerold.dk; From: Michael Ryom To: pve-devel@lists.proxmox.com Subject: [PATCH qemu-server] migrate: escalate cpu throttling and abort when RAM migration cannot converge Date: Sun, 20 Sep 2026 18:26:23 +0200 Message-ID: <20260920162623.575803-1-Michael@RyomHerold.dk> X-Mailer: git-send-email 2.43.0 Content-Transfer-Encoding: 8bit Content-Type: text/plain X-ClientProxiedBy: FR4P281CA0305.DEUP281.PROD.OUTLOOK.COM (2603:10a6:d10:f6::14) To AS8PR10MB7231.EURPRD10.PROD.OUTLOOK.COM (2603:10a6:20b:619::17) MIME-Version: 1.0 X-MS-PublicTrafficType: Email X-MS-TrafficTypeDiagnostic: AS8PR10MB7231:EE_|AM0PR10MB883391:EE_ X-MS-Office365-Filtering-Correlation-Id: 514d620a-dcf1-4369-bfd9-08df1733ee71 X-MS-Exchange-SenderADCheck: 1 X-MS-Exchange-AntiSpam-Relay: 0 X-Microsoft-Antispam: BCL:0;ARA:13230040|23010399003|376014|1800799024|366016|10067099003|3023799007|6133799003|18002099003|56012099006; X-Microsoft-Antispam-Message-Info: 0yyHhCKVr9dYm6q88rmPj0Pbb/IALpFrPlsw1WzKcdIMpdtM3bpd+/ARnaYqREG0swp6wag8U7+r1stqd0GAMb6jTChBJRv13i15xolIeq+V92JcYPi3FiksYekRB7A+iQ80X9m3u+y/S58jzMr2kO4cwS2/qLZRvPoKoqcF12zKFDDiZZW2iiPvV4nFuocyufyiDaHF38lG50Zr48ohLaOFMwesmTl9JzMI6+VoE58WXvYPjLsn2EBEVjhaDyPdBEcXhhaEKAnxK4mlQAjocQE3AsyfZMvzc8A91cmB/e+6mku0rwzlxPOideU9v3hBuExl76eNuY7PTYYgp+9TIqnL264gONW+GEf1EEilCF5/FZjUricP3gItY/2mP8nd2pvfgFZTse7qFjpLQxK/DAAPjicvqfBbjWDUrofnUoig2AW35KF51wuzSDQ81IhYQF3NVATYF4Xmdrgd54UaQoEwIe+7EiQVqaOq+T92TEawwL3cpuMLEURLqHX4iKrAknSOG70RlITOcqBQNOxsomyt2NLKAjbdt5qXFiQ7u0pVlaUHHNnN9qQR4XseHPor30NxECn0JCYgqu6HgtqjmjylXNMEARtteM6M/NmXjgyQX+yIQ3iw5ZSUc9I4eEm53vDI8vTQsRaEh7kaBIoqSfb88dWpzQ5qMN5j9SpNTtI= X-Forefront-Antispam-Report: CIP:255.255.255.255;CTRY:;LANG:en;SCL:1;SRV:;IPV:NLI;SFV:NSPM;H:AS8PR10MB7231.EURPRD10.PROD.OUTLOOK.COM;PTR:;CAT:NONE;SFS:(13230040)(23010399003)(376014)(1800799024)(366016)(10067099003)(3023799007)(6133799003)(18002099003)(56012099006);DIR:OUT;SFP:1102; X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1 X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?w91ErDiE8Nl7MGvjJcPrdHKYmepWhA7IU0f280FmfmwcawKhtt9pKP4lP+dY?= =?us-ascii?Q?Cf7qth6VFgyQ0PDJZqha4Dnl9M9q5bjIfFo5eRFq7DzuK7r6gtKNRbIV0QI8?= =?us-ascii?Q?QvXWlUmhzq+9TWQHcD4pNpOajZDVJg2lYYdLZW+oLA5IvSmAIGb53AqBqvGh?= =?us-ascii?Q?h2V7IfhVKvafoHQGCwIqthBl2zw3wSqLy/kr8ii/VUbuY4JUWurIqOaMo4yY?= =?us-ascii?Q?S7HmX5+R1WvZwrYUKnMItBANhLExcp8eJ8kVKDVuQnqhajXCaY4UouvIzl6x?= =?us-ascii?Q?Bu2pQ2TlFLo0ooQJVOckq4qtl1aJci8Q8s05mF6irBqePkuI22IReoxgceJz?= =?us-ascii?Q?BV5TfzOWBdqby5SUnaDEU1ikFYoLgYSPoqVrXLs17p0WAjGQ4lGNdMQLvJQl?= =?us-ascii?Q?eC4su3QoH2ysHPhnoKw5U7q0mjcriwO/CT0BU6Mqh+kvQulDRDojZj3IprRu?= =?us-ascii?Q?BgRG66mDjBJds8XkvW/oYDZbGA9oTXq57cn6WFvb38bpT0/Y/rkVa08iriMj?= =?us-ascii?Q?+Anot0JvHRDjoCeH0QzFLEk+Cx/m0BJaBEpF0E3VJLBJbmJJyGabbEeFuZlz?= =?us-ascii?Q?x2I4YHB2MjXAHPUSExEsMItagDXoJ2XksNn09/So75X390vHPzK4VcSWPbth?= =?us-ascii?Q?B5HNBBGMfvQcdOTr9Fcd9itBVg/oPU05FIEDYyER9hTtHHwgfbx4yM5Xfmq9?= =?us-ascii?Q?6oBC1HyKW7P1sMZgSqAZSmFuFyzYhX1IGmUr+GDYHx2nUheTi4YJTNO3zBFQ?= =?us-ascii?Q?vv9YW9ak/HY247+o2m5xd/1RP15y60GEwUDkkN/kQIbwtDzGFQaAdtA2N8+u?= =?us-ascii?Q?7rMfh+DmUlfejU2uAjBG3p50LXVku/YtpXLEhadHYWeP2ReE84TsWnGyWMre?= =?us-ascii?Q?tCaMZampezlnPeHa3WxKB7+H5iMUtPzUG5SlegY8/snLCJgDcNmeGdCBHHh2?= =?us-ascii?Q?VzdhCV4M3nkeVgjLhi2EfdxCUMGYU5Qqs14SeOwsByY0J7ItKFf5oiP+0SUG?= =?us-ascii?Q?Jqp6694wgmLl39XAmrGsn5FYFb4j+fPM7o1QOT5YFuDlAlWvLwPTXHtfn8lh?= =?us-ascii?Q?W0+lTKIemkfUDgoiyUcNh31jWRDZwUVo/tiDuTezccWcX+uPO/fOaiqaMSlO?= =?us-ascii?Q?OGlRjGnq4MLr9SiE4HAZ4bJZg4DRZpod4/X6ghzbRrEErGS253JdrEBnVvmZ?= =?us-ascii?Q?nocM0+RgAcdwJnuXS6Gj+Bv3A78C3VNV+GLXiXgMVARgN53kjnpfIW56YXmj?= =?us-ascii?Q?N345he2lg0JtD6MaGAFvqCAx8kFTmGN3bwO8SAk2MZplfqQh1AkeDK4mhHrF?= =?us-ascii?Q?z5SColYt6SUU5XvlnI7Oid40hVWEq99awIKbEDfGRPFWLtPjJhbsHgPgLvn2?= =?us-ascii?Q?7FttB2jXz2PDxt9Bty5nY9UEeiWZ+D2QLdntIgx6swKLzDnEbZ8GmUIsux5V?= =?us-ascii?Q?U7pEQRu3ZQT0RKiPzhpOemN2Qw1BFf8ePiDEeEyYmv+hFwd701xiQyP3avAB?= =?us-ascii?Q?1tv1WYo9qhYHuTQcKuukvqRwKPVwID1R49pjJErDJU2CwNucW7M3LaDcfMsb?= =?us-ascii?Q?GrDhStvcEpkf2FdhHA1zjq0w1QE/wnt8ZlG2G9hN1KKNCDvUi+9dpQwcXDrM?= =?us-ascii?Q?rAYgeSQTCOLIXMOJPY8dxiSav3bJugw+LZ3Xy7VaqG6MbmtJyCc3CMWCrCPG?= =?us-ascii?Q?Cpwbb39X/hG4MaY8Rbjih69QlFHNvmmRmzXMPk8N6oryHsGv2BdXA5cqWdsZ?= =?us-ascii?Q?sTDICADTrg=3D=3D?= X-OriginatorOrg: RyomHerold.dk X-MS-Exchange-CrossTenant-Network-Message-Id: 514d620a-dcf1-4369-bfd9-08df1733ee71 X-MS-Exchange-CrossTenant-AuthSource: AS8PR10MB7231.EURPRD10.PROD.OUTLOOK.COM X-MS-Exchange-CrossTenant-AuthAs: Internal X-MS-Exchange-CrossTenant-OriginalArrivalTime: 20 Sep 2026 16:26:33.5713 (UTC) X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted X-MS-Exchange-CrossTenant-Id: 26a2499e-4646-4888-a695-c64736a56807 X-MS-Exchange-CrossTenant-MailboxType: HOSTED X-MS-Exchange-CrossTenant-UserPrincipalName: nFMyTSMH+wk97zFOMTYqFIObOZfdRgAsidZBk5RUwtU5IWjtwPkdzlRRUNtDeyLQguf3c1teaVcIWo6gna7OTg== X-MS-Exchange-Transport-CrossTenantHeadersStamped: AM0PR10MB883391 X-SPAM-LEVEL: Spam detection results: 0 DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy RCVD_IN_DNSWL_NONE -0.0001 Sender listed at https://www.dnswl.org/, no trust SPF_HELO_PASS -0.001 SPF: HELO matches SPF record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: PS4WXMGQ4RPP27SARNPJMNAQAM74ZGMV X-Message-ID-Hash: PS4WXMGQ4RPP27SARNPJMNAQAM74ZGMV X-MailFrom: Michael@RyomHerold.dk X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Michael Ryom X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: When a guest dirties its memory faster than the RAM migration can transfer it, the migration runs for an unbounded time: QEMU's auto-converge only throttles the guest's vCPUs in small steps per migration iteration, and the status loop merely appends "VM dirties lots of memory" to the progress log while auto-increasing the downtime limit. Observed live: a 24 GiB guest with ~430 MiB/s transfer speed and a 600-760 MiB/s dirty rate had transferred 62 GiB (2.6 full passes over the guest memory) after 2.5 minutes, with the downtime limit already escalated to 800 ms, and only stopped because it was cancelled manually. Meanwhile the migration saturates the migration network and, for a rebalance migration issued by the HA manager, occupies the serialized motion pipeline indefinitely. Every full additional pass over the guest memory means the guest dirtied all of it again before the transfer could finish. Use that as a robust convergence measure (the reported dirty-pages-rate only pulses during sync phases) and act in two stages: - after 1.5 passes, escalate the auto-converge parameters (cpu-throttle-increment 20, max-cpu-throttle 99), so the guest is throttled harder and faster to force convergence at the cost of guest cpu time; - after 4 passes, i.e. if even the escalated throttling does not lead to convergence, abort the migration with a clear error instead of transferring indefinitely. The factors are constants for now and could become migration options later. For rebalance migrations, the HA manager backs off failed motions, so an aborted non-converging migration is not simply retried immediately. Add a regression test simulating a non-converging RAM migration, which checks that the throttle escalation is attempted and the migration is aborted afterwards. Signed-off-by: Michael Ryom --- src/PVE/QemuMigrate.pm | 38 +++++++++++++++++++++++ src/test/MigrationTest/QemuMigrateMock.pm | 21 +++++++++++++ src/test/run_qemu_migrate_tests.pl | 29 +++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/src/PVE/QemuMigrate.pm b/src/PVE/QemuMigrate.pm index 8da6f15d..c6a6799c 100644 --- a/src/PVE/QemuMigrate.pm +++ b/src/PVE/QemuMigrate.pm @@ -1176,6 +1176,14 @@ sub phase2_start_remote_cluster { my $migrate_downtime_max = 2000 * 1000; # as defined in QEMU's migration/options.c +# when the RAM migration has transferred more than this factor of the guest +# memory without completing, the guest dirties memory faster than it can be +# transferred; escalate the auto-converge cpu throttling to force convergence +my $migrate_throttle_boost_factor = 1.5; +# ... and when even that does not lead to convergence, abort the migration +# instead of transferring indefinitely +my $migrate_abort_factor = 4; + my sub cap_migrate_downtime { my ($self, $migrate_downtime) = @_; @@ -1442,6 +1450,8 @@ sub phase2 { my $err_count = 0; my $lastrem = undef; my $downtimecounter = 0; + my $cpu_throttle_boosted = 0; + while (1) { $i++; my $avglstat = $last_mem_transferred ? $last_mem_transferred / $i : 0; @@ -1560,6 +1570,34 @@ sub phase2 { $self->log('info', "xbzrle: $msg") if $should_log; } + # every full additional pass over the guest memory means the + # guest dirtied all of it again before the transfer could finish + if ($total > 0 && $mem_transferred > $migrate_abort_factor * $total) { + my $passes = sprintf("%.1f", $mem_transferred / $total); + die "aborting migration - no convergence: transferred ${passes}x the guest" + . " memory without completing, the guest dirties its memory faster than" + . " it can be transferred\n"; + } elsif ( + $total > 0 + && !$cpu_throttle_boosted + && $mem_transferred > $migrate_throttle_boost_factor * $total + ) { + $cpu_throttle_boosted = 1; + $self->log( + 'info', + "migration does not converge - escalating the auto-converge cpu" + . " throttling of the guest to force convergence", + ); + eval { + mon_cmd( + $vmid, "migrate-set-parameters", + 'cpu-throttle-increment' => 20, + 'max-cpu-throttle' => 99, + ); + }; + $self->log('info', "migrate-set-parameters error: $@") if $@; + } + if (($lastrem && $rem > $lastrem) || ($rem == 0)) { $downtimecounter++; } diff --git a/src/test/MigrationTest/QemuMigrateMock.pm b/src/test/MigrationTest/QemuMigrateMock.pm index 3a483817..9ecdd502 100644 --- a/src/test/MigrationTest/QemuMigrateMock.pm +++ b/src/test/MigrationTest/QemuMigrateMock.pm @@ -31,6 +31,7 @@ my $test_vmid = $migrate_params->{vmid}; my $test_target = $migrate_params->{target}; my $test_opts = $migrate_params->{opts}; my $current_log = ''; +my $query_migrate_calls = 0; my $vm_stop_executed = 0; @@ -102,10 +103,30 @@ $qemu_migrate_module->mock( return; } elsif ($command eq 'query-migrate') { return { status => 'failed' } if $fail_config->{'query-migrate'}; + if ($fail_config->{'query-migrate-nonconverging'}) { + $query_migrate_calls++; + my $gib = 1024 * 1024 * 1024; + # the guest dirties its memory faster than the transfer speed, + # so the transferred amount keeps growing by more than a full + # pass over the guest memory without the migration finishing + return { + status => 'active', + ram => { + total => 4 * $gib, + transferred => $query_migrate_calls * 7 * $gib, + remaining => 2 * $gib, + 'page-size' => 4096, + 'pages-per-second' => 102400, # 400 MiB/s + 'dirty-pages-rate' => 179200, # 700 MiB/s + }, + }; + } return { status => 'completed' }; } elsif ($command eq 'migrate') { return; } elsif ($command eq 'migrate-set-parameters') { + delete $expected_calls->{'migrate-set-parameters-throttle'} + if defined($params{'cpu-throttle-increment'}); return; } elsif ($command eq 'migrate_cancel') { return; diff --git a/src/test/run_qemu_migrate_tests.pl b/src/test/run_qemu_migrate_tests.pl index 05eed1d9..861011ea 100755 --- a/src/test/run_qemu_migrate_tests.pl +++ b/src/test/run_qemu_migrate_tests.pl @@ -717,6 +717,35 @@ my $tests = [ }, }, }, + { + name => '1033_running_nonconverging', + target => 'pve2', + vmid => 1033, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + opts => { + online => 1, + }, + fail_config => { + 'query-migrate-nonconverging' => 1, + }, + # the cpu throttle escalation must have been attempted before aborting + expected_calls => { + 'migrate-set-parameters-throttle' => 1, + }, + expect_die => 'online migrate failure - aborting migration - no convergence', + expected => { + source_volids => {}, + target_volids => {}, + vm_config => $vm_configs->{1033}, + vm_status => { + running => 1, + runningmachine => 'pc-q35-5.0+pve0', + }, + }, + }, { name => '4567_targetstorage_dirotherdir', target => 'pve1', -- 2.47.3