From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 265ADEE4A for ; Thu, 28 Sep 2023 01:30:26 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id EDB7DFF30 for ; Thu, 28 Sep 2023 01:29:55 +0200 (CEST) Received: from bastionodiso.odiso.net (bastionodiso.odiso.net [185.151.191.93]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 28 Sep 2023 01:29:53 +0200 (CEST) Received: from kvmformation3.odiso.net (formationkvm3.odiso.net [10.3.94.12]) by bastionodiso.odiso.net (Postfix) with ESMTP id 6956D811F; Thu, 28 Sep 2023 01:29:47 +0200 (CEST) Received: by kvmformation3.odiso.net (Postfix, from userid 0) id 55ADD18051; Thu, 28 Sep 2023 01:29:47 +0200 (CEST) From: Alexandre Derumier To: pve-devel@lists.proxmox.com Date: Thu, 28 Sep 2023 01:29:45 +0200 Message-Id: <20230927232945.44215-3-aderumier@odiso.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20230927232945.44215-1-aderumier@odiso.com> References: <20230927232945.44215-1-aderumier@odiso.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.030 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy HEADER_FROM_DIFFERENT_DOMAINS 0.249 From and EnvelopeFrom 2nd level mail domains are different KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [qemu.pm, qemumigrate.pm, qm.pm] Subject: [pve-devel] [PATCH v3 qemu-server 2/2] remote-migration: add target-cpu && target-reboot params X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 27 Sep 2023 23:30:26 -0000 This patch add support for remote migration when target cpu model is different. target-reboot param need to be defined to allow migration whens source vm is online. When defined, only the live storage migration is done, and instead to transfert memory, we cleanly shutdown source vm and restart the target vm. (like a virtual reboot between source/dest) --- PVE/API2/Qemu.pm | 28 +++++++++++++++++++++++++++- PVE/CLI/qm.pm | 11 +++++++++++ PVE/QemuMigrate.pm | 31 +++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index 774b0c7..e1cefba 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -4586,6 +4586,17 @@ __PACKAGE__->register_method({ optional => 1, default => 0, }, + 'target-cpu' => { + optional => 1, + description => "Target Emulated CPU model. For online migration, this require target-reboot option", + type => 'string', + format => 'pve-vm-cpu-conf', + }, + 'target-reboot' => { + type => 'boolean', + description => "For online migration , don't migrate memory, only storage. Then, the source vm is shutdown and the target vm is restarted.", + optional => 1, + }, 'target-storage' => get_standard_option('pve-targetstorage', { completion => \&PVE::QemuServer::complete_migration_storage, optional => 0, @@ -4664,9 +4675,12 @@ __PACKAGE__->register_method({ my $is_replicated = $repl_conf->check_for_existing_jobs($source_vmid, 1); die "cannot remote-migrate replicated VM\n" if $is_replicated; + my $target_cpu = extract_param($param, 'target-cpu'); + my $target_reboot = extract_param($param, 'target-reboot'); + if (PVE::QemuServer::check_running($source_vmid)) { die "can't migrate running VM without --online\n" if !$param->{online}; - + die "can't migrate running VM without --target-reboot when target cpu is different" if $target_cpu && !$target_reboot; } else { warn "VM isn't running. Doing offline migration instead.\n" if $param->{online}; $param->{online} = 0; @@ -4683,11 +4697,14 @@ __PACKAGE__->register_method({ raise_param_exc({ 'target-bridge' => "failed to parse bridge map: $@" }) if $@; + die "remote migration requires explicit storage mapping!\n" if $storagemap->{identity}; $param->{storagemap} = $storagemap; $param->{bridgemap} = $bridgemap; + $param->{targetcpu} = $target_cpu; + $param->{targetreboot} = $target_reboot; $param->{remote} = { conn => $conn_args, # re-use fingerprint for tunnel client => $api_client, @@ -5732,6 +5749,15 @@ __PACKAGE__->register_method({ PVE::QemuServer::nbd_stop($state->{vmid}); return; }, + 'restart' => sub { + PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); + my $info = PVE::QemuServer::vm_start_nolock( + $state->{storecfg}, + $state->{vmid}, + $state->{conf}, + ); + return; + }, 'resume' => sub { if (PVE::QemuServer::Helpers::vm_running_locally($state->{vmid})) { PVE::QemuServer::vm_resume($state->{vmid}, 1, 1); diff --git a/PVE/CLI/qm.pm b/PVE/CLI/qm.pm index b17b4fe..9d89cfe 100755 --- a/PVE/CLI/qm.pm +++ b/PVE/CLI/qm.pm @@ -189,6 +189,17 @@ __PACKAGE__->register_method({ optional => 1, default => 0, }, + 'target-cpu' => { + optional => 1, + description => "Target Emulated CPU model. For online migration, this require target-reboot option", + type => 'string', + format => 'pve-vm-cpu-conf', + }, + 'target-reboot' => { + type => 'boolean', + description => "For online migration , don't migrate memory, only storage. Then, the source vm is shutdown and the target vm is restarted.", + optional => 1, + }, 'target-storage' => get_standard_option('pve-targetstorage', { completion => \&PVE::QemuServer::complete_migration_storage, optional => 0, diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm index 5ea78a7..0eaa73d 100644 --- a/PVE/QemuMigrate.pm +++ b/PVE/QemuMigrate.pm @@ -729,6 +729,11 @@ sub cleanup_bitmaps { sub live_migration { my ($self, $vmid, $migrate_uri, $spice_port) = @_; + if($self->{opts}->{targetreboot}){ + $self->log('info', "target reboot - skip live migration."); + return; + } + my $conf = $self->{vmconf}; $self->log('info', "starting online/live migration on $migrate_uri"); @@ -993,6 +998,7 @@ sub phase1_remote { my $remote_conf = PVE::QemuConfig->load_config($vmid); PVE::QemuConfig->update_volume_ids($remote_conf, $self->{volume_map}); + $remote_conf->{cpu} = $self->{opts}->{targetcpu}; my $bridges = map_bridges($remote_conf, $self->{opts}->{bridgemap}); for my $target (keys $bridges->%*) { for my $nic (keys $bridges->{$target}->%*) { @@ -1356,7 +1362,14 @@ sub phase2 { # finish block-job with block-job-cancel, to disconnect source VM from NBD # to avoid it trying to re-establish it. We are in blockjob ready state, # thus, this command changes to it to blockjob complete (see qapi docs) - eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, 'cancel'); }; + my $finish_cmd = "cancel"; + if ($self->{opts}->{targetreboot}) { + # no live migration. + # finish block-job with block-job-complete, the source will switch to remote NDB + # then we cleanly stop the source vm at phase3 + $finish_cmd = "complete"; + } + eval { PVE::QemuServer::qemu_drive_mirror_monitor($vmid, undef, $self->{storage_migration_jobs}, $finish_cmd); }; if (my $err = $@) { die "Failed to complete storage migration: $err\n"; } @@ -1573,7 +1586,17 @@ sub phase3_cleanup { }; # always stop local VM with nocheck, since config is moved already - eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1); }; + my $shutdown_timeout = undef; + my $shutdown = undef; + my $force_stop = undef; + if ($self->{opts}->{targetreboot}) { + $shutdown_timeout = 180; + $shutdown = 1; + $force_stop = 1; + $self->log('info', "clean shutdown of source vm."); + } + + eval { PVE::QemuServer::vm_stop($self->{storecfg}, $vmid, 1, 1, $shutdown_timeout, $shutdown, $force_stop); }; if (my $err = $@) { $self->log('err', "stopping vm failed - $err"); $self->{errors} = 1; @@ -1607,6 +1630,10 @@ sub phase3_cleanup { # clear migrate lock if ($tunnel && $tunnel->{version} >= 2) { PVE::Tunnel::write_tunnel($tunnel, 10, "unlock"); + if ($self->{opts}->{targetreboot}) { + $self->log('info', "restart target vm."); + PVE::Tunnel::write_tunnel($tunnel, 10, 'restart'); + } PVE::Tunnel::finish_tunnel($tunnel); } else { -- 2.39.2