From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id 5B6611FF0C0 for ; Wed, 04 Nov 2026 20:08:39 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id AFC1D219DF; Wed, 04 Nov 2026 20:05:29 +0100 (CET) From: =?UTF-8?q?Michael=20K=C3=B6ppl?= To: pve-devel@lists.proxmox.com Subject: [PATCH e2e-tests 37/38] tests: ha: verify moving guests off a node for maintenance Date: Wed, 4 Nov 2026 20:03:44 +0100 Message-ID: <20261104190345.2573464-38-m.koeppl@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261104190345.2573464-1-m.koeppl@proxmox.com> References: <20261104190345.2573464-1-m.koeppl@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1793819036950 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.219 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: DBSPYDEJSV3SOCJV2LMQVOY7LZBSHVRV X-Message-ID-Hash: DBSPYDEJSV3SOCJV2LMQVOY7LZBSHVRV X-MailFrom: m.koeppl@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Put a node with an HA-managed VM into HA maintenance mode and check that the HA stack moves the VM to another node, and back once the maintenance mode is disabled. HA maintenance mode is only available through ha-manager on the node. Signed-off-by: Michael Köppl --- tests/pve_node_maintenance.pl | 136 ++++++++++++++++++++++++++++++++++ tests/testcases.ron | 11 +++ 2 files changed, 147 insertions(+) create mode 100755 tests/pve_node_maintenance.pl diff --git a/tests/pve_node_maintenance.pl b/tests/pve_node_maintenance.pl new file mode 100755 index 0000000..6673761 --- /dev/null +++ b/tests/pve_node_maintenance.pl @@ -0,0 +1,136 @@ +#!/usr/bin/perl + +=head1 NAME + +C - Test moving HA-managed guests off a node in maintenance mode. + +=head1 DESCRIPTION + +An HA-managed VM is started on the first node. Putting the first node into HA +maintenance mode must make the HA stack move the VM to another node, where it +keeps running. Taking the node out of maintenance mode must move the VM back. + +=cut + +use strict; +use warnings; + +use Test::More; + +use lib '.'; +use Proxmox::Test::Helper qw(get_pve_instance); +use Proxmox::Test::Testcase; +use Proxmox::Test::Util qw(poll_until); + +my $pve_a = get_pve_instance('pve1-cluster'); +my $node_a = $pve_a->get_nodename(); +my $client = $pve_a->client(); + +my $ha_vmid; + +my $resource = sub { + my ($id) = @_; + my $resources = $client->get('/cluster/resources', { type => 'vm' }); + my ($entry) = grep { ($_->{vmid} // 0) == $id } $resources->@*; + return $entry // {}; +}; + +my $wait_for_guest = sub { + my ($id, $node, $timeout) = @_; + poll_until( + sub { + my $entry = $resource->($id); + return ($entry->{node} // '') eq $node && ($entry->{status} // '') eq 'running'; + }, + timeout => $timeout, + interval => 5, + description => "guest $id to run on node $node", + ); +}; + +my $lrm_status = sub { + my ($node) = @_; + my ($entry) = grep { $_->{type} eq 'lrm' && $_->{node} eq $node } + $client->get('/cluster/ha/status/current', {})->@*; + return $entry ? $entry->{status} : ''; +}; + +my $set_maintenance = sub { + my ($mode) = @_; + $pve_a->run_command("ha-manager crm-command node-maintenance $mode $node_a"); +}; + +my $destroy = sub { + my ($type, $id) = @_; + my $node = $resource->($id)->{node}; + $pve_a->destroy_guest_on($node, $type, $id) if defined($node); +}; + +Proxmox::Test::Testcase->new('pve_node_maintenance')->with([{}])->step( + 'start HA-managed VM', + sub { + $ha_vmid = $client->get('/cluster/nextid'); + my $upid = $client->post( + "/nodes/$node_a/qemu", + { vmid => $ha_vmid, name => 'vm-ha-maintenance', memory => 256 }, + ); + $pve_a->task_ok($upid, "created VM $ha_vmid on node $node_a") + or die "creating VM $ha_vmid failed\n"; + + $client->post('/cluster/ha/resources', { sid => "vm:$ha_vmid", state => 'started' }); + $pve_a->wait_for_ha_manager(); + + $wait_for_guest->($ha_vmid, $node_a, 120); + pass("HA started VM $ha_vmid on node $node_a"); + }, + sub { + eval { $client->delete("/cluster/ha/resources/vm:$ha_vmid", {}) }; + $destroy->('qemu', $ha_vmid) if defined($ha_vmid); + }, +)->step( + 'enable maintenance mode', + sub { + $set_maintenance->('enable'); + + poll_until( + sub { $lrm_status->($node_a) =~ m/\bmaintenance mode\b/ }, + timeout => 120, + interval => 5, + description => "node $node_a to enter maintenance mode", + ); + pass("node $node_a is in maintenance mode"); + + poll_until( + sub { + my $entry = $resource->($ha_vmid); + return ($entry->{node} // $node_a) ne $node_a + && ($entry->{status} // '') eq 'running'; + }, + timeout => 240, + interval => 5, + description => "HA to move VM $ha_vmid off node $node_a", + ); + pass("HA moved VM $ha_vmid to node " . $resource->($ha_vmid)->{node}); + }, + sub { + eval { $set_maintenance->('disable') }; + }, +)->step( + 'disable maintenance mode', + sub { + $set_maintenance->('disable'); + + poll_until( + sub { $lrm_status->($node_a) !~ m/\bmaintenance mode\b/ }, + timeout => 120, + interval => 5, + description => "node $node_a to leave maintenance mode", + ); + pass("node $node_a is no longer in maintenance mode"); + + $wait_for_guest->($ha_vmid, $node_a, 240); + pass("HA moved VM $ha_vmid back to node $node_a"); + }, +)->run(); + +done_testing(); diff --git a/tests/testcases.ron b/tests/testcases.ron index e0d622c..0e4cea0 100644 --- a/tests/testcases.ron +++ b/tests/testcases.ron @@ -293,6 +293,17 @@ TestcaseConfig( ], tag: "ha", ), + Testcase( + name: "pve_node_maintenance", + run: "./tests/pve_node_maintenance.pl", + timeout: 900, + instances: [ + "pve1-cluster", + "pve2-cluster", + "pve3-cluster", + ], + tag: "ha", + ), Testcase( name: "pbs_s3_datastore", run: "./tests/pbs/test_pbs_s3_datastore.pl", -- 2.47.3