From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 92B471FF0C0 for ; Wed, 18 Nov 2026 12:09:17 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id D25EA21793; Wed, 18 Nov 2026 12:07:52 +0100 (CET) From: Thomas Lamprecht To: pve-devel@lists.proxmox.com Subject: [PATCH manager 12/15] api: bulk migrate: allow a per-guest target node mapping Date: Wed, 18 Nov 2026 12:05:19 +0100 Message-ID: <20261118110705.595989-13-t.lamprecht@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20261118110705.595989-1-t.lamprecht@proxmox.com> References: <20261118110705.595989-1-t.lamprecht@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1795000030951 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.175 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: CPLLXHPSV5JYIZ64APFQI5ARU7S7L6I4 X-Message-ID-Hash: CPLLXHPSV5JYIZ64APFQI5ARU7S7L6I4 X-MailFrom: t.lamprecht@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: The bulk migration can only send all guests to the same node, which does not fit spreading guests over the cluster, for example to apply a rebalance proposal. Add a 'target-map' parameter with per-guest target nodes that take precedence over 'target'. Without 'vms' and 'target', only the mapped guests are migrated. Signed-off-by: Thomas Lamprecht --- PVE/API2/Cluster/BulkAction/Guest.pm | 100 ++++++++++++- test/BulkMigrate_test.pl | 204 +++++++++++++++++++++++++++ test/Makefile | 6 +- 3 files changed, 303 insertions(+), 7 deletions(-) create mode 100755 test/BulkMigrate_test.pl diff --git a/PVE/API2/Cluster/BulkAction/Guest.pm b/PVE/API2/Cluster/BulkAction/Guest.pm index a55ea2ea6..ae6186324 100644 --- a/PVE/API2/Cluster/BulkAction/Guest.pm +++ b/PVE/API2/Cluster/BulkAction/Guest.pm @@ -11,12 +11,52 @@ use PVE::JSONSchema qw(get_standard_option); use PVE::RESTEnvironment qw(log_warn); use PVE::RPCEnvironment; use PVE::Storage; +use PVE::Tools; use PVE::UPID; use PVE::API2::Nodes; use base qw(PVE::RESTHandler); +PVE::JSONSchema::register_format('pve-vmid-node-pair', \&pve_verify_vmid_node_pair); + +sub pve_verify_vmid_node_pair { + my ($pair, $noerr) = @_; + + if ($pair =~ m/^(\d+):(.+)$/) { + my ($vmid, $node) = ($1, $2); + return $pair + if eval { + PVE::JSONSchema::check_format('pve-vmid', $vmid); + PVE::JSONSchema::check_format('pve-node', $node); + 1; + }; + } + + return undef if $noerr; + die "value '$pair' is not a 'vmid:node' pair\n"; +} + +# Parse a 'pve-vmid-node-pair-list' into a { vmid => node } hash. +sub parse_target_map { + my ($target_map) = @_; + + my $map = {}; + return $map if !defined($target_map); + + for my $entry (PVE::Tools::split_list($target_map)) { + my ($vmid, $node) = $entry =~ m/^(\d+):(.+)$/ + or raise_param_exc({ 'target-map' => "value '$entry' is not a 'vmid:node' pair" }); + raise_param_exc({ 'target-map' => "duplicate entry for guest '$vmid'" }) + if defined($map->{$vmid}); + raise_param_exc({ 'target-map' => "no such cluster node '$node'" }) + if !PVE::Cluster::check_node_exists($node, 1); + $map->{$vmid} = $node; + } + + return $map; +} + __PACKAGE__->register_method({ name => 'index', path => '', @@ -711,7 +751,23 @@ __PACKAGE__->register_method({ items => get_standard_option('pve-vmid'), optional => 1, }, - target => get_standard_option('pve-node', { description => "Target node." }), + target => get_standard_option( + 'pve-node', + { + description => + "Default target node for guests without an entry in 'target-map'.", + optional => 1, + }, + ), + 'target-map' => { + description => + "Per-guest target nodes, taking precedence over 'target'. If neither" + . " 'vms' nor 'target' is given, only the listed guests are migrated.", + type => 'string', + format => 'pve-vmid-node-pair-list', + typetext => ':[,:...]', + optional => 1, + }, online => { type => 'boolean', description => "Enable live migration for VMs and restart migration for CTs.", @@ -740,7 +796,6 @@ __PACKAGE__->register_method({ maximum => 64, }, # TODO: - # Node mappings, not just a single target node! # Failure resolution mode (fail, warn, retry?) # mode-limits (offline only, suspend only, ?) # filter (tags, name, ?) @@ -756,15 +811,46 @@ __PACKAGE__->register_method({ my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); + my $target_map = parse_target_map($param->{'target-map'}); + my $default_target = $param->{target}; + + raise_param_exc({ target => "neither 'target' nor 'target-map' given" }) + if !defined($default_target) && !%$target_map; + + # without a default target only the mapped guests can be migrated, so limit the action to + # them rather than walking, and skipping, every other guest in the cluster + $param->{vms} = [sort { $a <=> $b } keys %$target_map] + if !defined($default_target) && !($param->{vms} && $param->{vms}->@*); + my ($vmlist, $vmlist_string) = extract_vmlist($param); check_guest_permissions($rpcenv, $authuser, $vmlist, ['VM.Migrate']); + my %in_scope = map { $_ => 1 } $vmlist->@*; + if (%in_scope) { + for my $vmid (sort { $a <=> $b } keys %$target_map) { + raise_param_exc({ 'target-map' => "guest '$vmid' is not in the 'vms' list" }) + if !$in_scope{$vmid}; + } + } + + if (!defined($default_target)) { + my @missing = grep { !defined($target_map->{$_}) } $vmlist->@*; + raise_param_exc({ + 'target-map' => "no target node for guest(s) " . join(', ', @missing), + }) + if @missing; + } + my $code = sub { my $list = PVE::API2::Nodes::Nodeinfo::get_filtered_vmlist(undef, $vmlist_string, 1, 1); - print_start_action($vmlist, "Migrating", "to $param->{target}"); + print_start_action( + $vmlist, + "Migrating", + %$target_map ? "to their target nodes" : "to $default_target", + ); my $start_task = sub { my ($api_client, $vmid, $guest) = @_; @@ -773,13 +859,15 @@ __PACKAGE__->register_method({ my $type = $guest->{type}; my $type_text = get_type_text($type); - if ($node eq $param->{target}) { + my $target = $target_map->{$vmid} // $default_target; + + if ($node eq $target) { log_warn("$type_text $vmid already on $node, skipping.\n"); return 1; } my $params = { - target => $param->{target}, + target => $target, }; if ($type eq 'lxc') { @@ -791,7 +879,7 @@ __PACKAGE__->register_method({ } my $url = "/nodes/$node/$type/$vmid/migrate"; - print "Migrating $type_text $vmid\n"; + print "Migrating $type_text $vmid to $target\n"; return $api_client->post($url, $params); }; diff --git a/test/BulkMigrate_test.pl b/test/BulkMigrate_test.pl new file mode 100755 index 000000000..bee292f9a --- /dev/null +++ b/test/BulkMigrate_test.pl @@ -0,0 +1,204 @@ +#!/usr/bin/perl + +use v5.36; + +use lib ('.', '..'); + +use Test::MockModule; +use Test::More; + +use PVE::JSONSchema; +use PVE::RPCEnvironment; +use PVE::Tools; + +use PVE::API2::Cluster::BulkAction::Guest; + +my $nodes = { map { $_ => 1 } qw(node1 node2 node3) }; + +my $guests = { + 100 => { type => 'qemu', node => 'node1' }, + 101 => { type => 'qemu', node => 'node1' }, + 102 => { type => 'lxc', node => 'node2' }, + 103 => { type => 'qemu', node => 'node3' }, +}; + +# Per user, the ACL paths on which the user has 'VM.Migrate', the only privilege the bulk migration +# checks itself; the per-guest migrate API calls check their own permissions. +my $migrate_acl = {}; + +# What the bulk migration worker did in the current test, see bulk_migrate(). +my $worker; + +my $cluster_mock = Test::MockModule->new('PVE::Cluster'); +$cluster_mock->redefine( + check_node_exists => sub($node, $noerr = undef) { + return 1 if $nodes->{$node}; + return undef if $noerr; + die "no such cluster node '$node'\n"; + }, +); + +my $nodeinfo_mock = Test::MockModule->new('PVE::API2::Nodes::Nodeinfo', no_auto => 1); +$nodeinfo_mock->redefine( + get_filtered_vmlist => sub($nodename, $vmfilter, $templates, $ha_managed) { + $worker->{filter} = $vmfilter; + my @vmids = defined($vmfilter) ? PVE::Tools::split_list($vmfilter) : keys %$guests; + return { map { $_ => { $guests->{$_}->%* } } grep { $guests->{$_} } @vmids }; + }, +); + +# Run the per-guest step directly instead of starting and polling tasks through the API client, and +# record each migration it starts. +my $guest_mock = Test::MockModule->new('PVE::API2::Cluster::BulkAction::Guest'); +$guest_mock->redefine( + get_max_workers => sub($param, $default = undef) { return 1 }, + log_warn => sub($message) { push $worker->{warnings}->@*, $message }, + handle_task_foreach_guest => sub($startlist, $max_workers, $start_task, $check_task) { + my $client = bless {}, 'FakeAPIClient'; + for my $order (sort { $a <=> $b } keys %$startlist) { + my $list = $startlist->{$order}; + $start_task->($client, $_, $list->{$_}) for sort { $a <=> $b } keys %$list; + } + return []; + }, +); + +package FakeAPIClient { + + sub post($self, $url, $param) { + my ($vmid) = $url =~ m!^/nodes/[^/]+/(?:qemu|lxc)/(\d+)/migrate$! + or die "unexpected API call '$url'\n"; + $worker->{migrations}->{$vmid} = $param->{target}; + return "UPID:migrate:$vmid"; + } +} + +my $rpcenv_mock = Test::MockModule->new('PVE::RPCEnvironment'); +$rpcenv_mock->redefine( + permissions => sub($self, $user, $path) { + return { 'VM.Migrate' => 1 } + if $user eq 'root@pam' || grep { $_ eq $path } ($migrate_acl->{$user} // [])->@*; + return {}; + }, + fork_worker => sub($self, $type, $id, $user, $code) { + $worker->{code} = $code; + return "UPID:$type"; + }, +); + +my $rpcenv = PVE::RPCEnvironment->init('cli'); + +# Call the bulk migration the way the API server does and run the worker it forks. Returns what the +# worker did: the guest filter it applied, the started migrations as { vmid => target }, its log +# output and warnings. Dies like the API call if the request is rejected. +sub bulk_migrate($user, $param) { + $worker = { migrations => {}, warnings => [] }; + + my ($handler, $info) = + PVE::API2::Cluster::BulkAction::Guest->find_handler('POST', 'migrate', {}); + + my $call_param = {%$param}; + PVE::JSONSchema::validate($call_param, $info->{parameters}); + + $rpcenv->set_user($user); + $rpcenv->check_api2_permissions($info->{permissions}, $user, $call_param); + + $handler->handle($info, $call_param); + die "no worker started\n" if !$worker->{code}; + + open(my $log_fh, '>', \$worker->{log}) or die "cannot capture worker log - $!\n"; + my $previous_fh = select($log_fh); + eval { $worker->{code}->() }; + my $err = $@; + select($previous_fh); + die "worker failed - $err" if $err; + + return $worker; +} + +# The exception a rejected bulk migration raises, or undef if it is accepted. +sub bulk_migrate_error($user, $param) { + eval { bulk_migrate($user, $param) }; + return $@ || undef; +} + +subtest 'single target node' => sub { + my $run = bulk_migrate('root@pam', { vms => [100, 102], target => 'node3' }); + is_deeply($run->{migrations}, { 100 => 'node3', 102 => 'node3' }, 'guests go to target'); + like($run->{log}, qr/^Migrating guests to node3: 100, 102$/m, 'log names the target node'); +}; + +subtest 'per-guest target nodes' => sub { + my $param = { vms => [100, 101, 103], target => 'node2' }; + $param->{'target-map'} = '101:node3,103:node3'; + + my $run = bulk_migrate('root@pam', $param); + my $expected = { 100 => 'node2', 101 => 'node3' }; + is_deeply($run->{migrations}, $expected, "'target-map' takes precedence over 'target'"); + like( + join('', $run->{warnings}->@*), + qr/VM 103 already on node3, skipping/, + 'guest already on its target node is skipped', + ); + + # whitespace and semicolons separate list entries like everywhere else + $param = { vms => [100, 101], 'target-map' => '100:node2; 101:node3' }; + $run = bulk_migrate('root@pam', $param); + $expected = { 100 => 'node2', 101 => 'node3' }; + is_deeply($run->{migrations}, $expected, 'all list separators are accepted'); + + # with a default target, omitting 'vms' still means all guests + $run = bulk_migrate('root@pam', { target => 'node3', 'target-map' => '100:node2' }); + ok(!defined($run->{filter}), "with 'target' and without 'vms' all guests are considered"); + $expected = { 100 => 'node2', 101 => 'node3', 102 => 'node3' }; + is_deeply($run->{migrations}, $expected, 'unmapped guests go to the default target'); +}; + +subtest 'only mapped guests without vms and target' => sub { + # without a default target the other guests could only be skipped, one warning each + my $run = bulk_migrate('root@pam', { 'target-map' => '102:node3,100:node2' }); + is($run->{filter}, '100,102', 'action is limited to the mapped guests'); + is_deeply($run->{migrations}, { 100 => 'node2', 102 => 'node3' }, 'mapped guests migrate'); + is_deeply($run->{warnings}, [], 'no warnings for the other guests'); + + # as the mapped guests are the explicit selection, permissions on them suffice + $migrate_acl = { 'op@pve' => ['/vms/100', '/vms/102'] }; + $run = bulk_migrate('op@pve', { 'target-map' => '100:node2,102:node3' }); + is_deeply($run->{migrations}, { 100 => 'node2', 102 => 'node3' }, 'no privilege on /'); + + my $err = bulk_migrate_error('op@pve', { 'target-map' => '100:node2,101:node3' }); + is($err->{code}, 403, "mapped guest without 'VM.Migrate' is rejected"); +}; + +subtest 'invalid target maps are rejected' => sub { + my $not_a_pair = qr/not a 'vmid:node' pair/; + + # name, 'target-map', expected error, 'vms' as list (default: 100) + my $cases = [ + ['bare node', 'node2', $not_a_pair], + ['bare guest id', '100', $not_a_pair], + ['pair without node', '100:', $not_a_pair], + ['invalid guest id', '0100:node2', $not_a_pair], + ['invalid node name', '100:-x', $not_a_pair], + ['duplicate entry', '100:node2,100:node3', qr/duplicate entry for guest '100'/], + ['unknown node', '100:nodeX', qr/no such cluster node 'nodeX'/], + ['guest not in vms', '100:node2,101:node3', qr/guest '101' is not in the 'vms' list/], + ['guest without target', '100:node2', qr/no target node for guest\(s\) 101/, '100,101'], + ]; + + for my $case (@$cases) { + my ($name, $target_map, $message, $vms) = @$case; + my $param = { vms => [split(/,/, $vms // '100')], 'target-map' => $target_map }; + + my $err = bulk_migrate_error('root@pam', $param); + is($err->{code}, 400, "$name: rejected"); + like($err->{errors}->{'target-map'}, $message, "$name: error message"); + ok(!$worker->{code}, "$name: no worker started"); + } + + my $err = bulk_migrate_error('root@pam', { vms => [100] }); + is($err->{code}, 400, 'neither target nor target map: rejected'); + like($err->{errors}->{target}, qr/neither 'target' nor 'target-map'/, 'error names both'); +}; + +done_testing(); diff --git a/test/Makefile b/test/Makefile index 5073c1bfc..20c00d624 100644 --- a/test/Makefile +++ b/test/Makefile @@ -5,7 +5,7 @@ all: export PERLLIB=.. .PHONY: check -check: test-replication test-balloon test-vzdump test-osd test-ceph-health test-ceph-services test-ceph-auth test-ceph-key-migration test-ceph-lockbox-migration test-custom-cpu-models test-resource-scheduling test-pvesh test-report +check: test-replication test-balloon test-vzdump test-osd test-ceph-health test-ceph-services test-ceph-auth test-ceph-key-migration test-ceph-lockbox-migration test-custom-cpu-models test-resource-scheduling test-bulk-migrate test-pvesh test-report .PHONY: test-balloon test-balloon: @@ -61,6 +61,10 @@ test-custom-cpu-models: test-resource-scheduling: ./ResourceScheduling_test.pl +.PHONY: test-bulk-migrate +test-bulk-migrate: + ./BulkMigrate_test.pl + .PHONY: test-pvesh test-pvesh: ./pvesh_test.pl -- 2.47.3