From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 45D2172B91 for ; Tue, 13 Apr 2021 14:17:35 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 31B812AA8C for ; Tue, 13 Apr 2021 14:17:05 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 28ACA2AA26 for ; Tue, 13 Apr 2021 14:17:01 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E726B45A77 for ; Tue, 13 Apr 2021 14:17:00 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Tue, 13 Apr 2021 14:16:38 +0200 Message-Id: <20210413121640.3602975-21-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210413121640.3602975-1-f.gruenbichler@proxmox.com> References: <20210413121640.3602975-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.026 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [qemuserver.pm, qemumigrate.pm] Subject: [pve-devel] [PATCH qemu-server 5/7] migrate: refactor remote VM/tunnel start X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 13 Apr 2021 12:17:35 -0000 no semantic changes intended, except for: - no longer passing the main migration UNIX socket to SSH twice for forwarding - dropping the 'unix:' prefix in start_remote_tunnel's timeout error message Signed-off-by: Fabian Grünbichler --- PVE/QemuMigrate.pm | 154 +++++++++++++++++++++++++++------------------ PVE/QemuServer.pm | 32 +++++----- 2 files changed, 110 insertions(+), 76 deletions(-) diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm index eb95762..5d44c51 100644 --- a/PVE/QemuMigrate.pm +++ b/PVE/QemuMigrate.pm @@ -204,19 +204,24 @@ sub finish_tunnel { die $err if $err; } +# tunnel_info: +# proto: unix (secure) or tcp (insecure/legacy compat) +# addr: IP or UNIX socket path +# port: optional TCP port +# unix_sockets: additional UNIX socket paths to forward sub start_remote_tunnel { - my ($self, $raddr, $rport, $ruri, $unix_socket_info) = @_; + my ($self, $tunnel_info) = @_; my $nodename = PVE::INotify::nodename(); my $migration_type = $self->{opts}->{migration_type}; if ($migration_type eq 'secure') { - if ($ruri =~ /^unix:/) { - my $ssh_forward_info = ["$raddr:$raddr"]; - $unix_socket_info->{$raddr} = 1; + if ($tunnel_info->{proto} eq 'unix') { + my $ssh_forward_info = []; - my $unix_sockets = [ keys %$unix_socket_info ]; + my $unix_sockets = [ keys %{$tunnel_info->{unix_sockets}} ]; + push @$unix_sockets, $tunnel_info->{addr}; for my $sock (@$unix_sockets) { push @$ssh_forward_info, "$sock:$sock"; unlink $sock; @@ -243,23 +248,23 @@ sub start_remote_tunnel { if ($unix_socket_try > 100) { $self->{errors} = 1; $self->finish_tunnel($self->{tunnel}); - die "Timeout, migration socket $ruri did not get ready"; + die "Timeout, migration socket $tunnel_info->{addr} did not get ready"; } $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); - } elsif ($ruri =~ /^tcp:/) { + } elsif ($tunnel_info->{proto} eq 'tcp') { my $ssh_forward_info = []; - if ($raddr eq "localhost") { + if ($tunnel_info->{addr} eq "localhost") { # for backwards compatibility with older qemu-server versions my $pfamily = PVE::Tools::get_host_address_family($nodename); my $lport = PVE::Tools::next_migrate_port($pfamily); - push @$ssh_forward_info, "$lport:localhost:$rport"; + push @$ssh_forward_info, "$lport:localhost:$tunnel_info->{rporyt}"; } $self->{tunnel} = $self->fork_tunnel($ssh_forward_info); } else { - die "unsupported protocol in migration URI: $ruri\n"; + die "unsupported protocol in migration URI: $tunnel_info->{proto}\n"; } } else { #fork tunnel for insecure migration, to send faster commands like resume @@ -737,48 +742,36 @@ sub phase1_cleanup { } -sub phase2 { - my ($self, $vmid) = @_; +sub phase2_start_local_cluster { + my ($self, $vmid, $params) = @_; my $conf = $self->{vmconf}; + my $start = $params->{start_params}; + my $migrate = $params->{migrate_opts}; $self->log('info', "starting VM $vmid on remote node '$self->{node}'"); - my $raddr; - my $rport; - my $ruri; # the whole migration dst. URI (protocol:address[:port]) - my $nodename = PVE::INotify::nodename(); + my $tunnel_info = {}; ## start on remote node my $cmd = [@{$self->{rem_ssh}}]; - my $spice_ticket; - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { - my $res = mon_cmd($vmid, 'query-spice'); - $spice_ticket = $res->{ticket}; - } + push @$cmd, 'qm', 'start', $vmid, '--skiplock'; + push @$cmd, '--migratedfrom', $migrate->{migratedfrom}; - push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename; + push @$cmd, '--migration_type', $migrate->{type}; - my $migration_type = $self->{opts}->{migration_type}; + push @$cmd, '--migration_network', $migrate->{network} + if $migrate->{migration_network}; - push @$cmd, '--migration_type', $migration_type; + push @$cmd, '--stateuri', $start->{statefile}; - push @$cmd, '--migration_network', $self->{opts}->{migration_network} - if $self->{opts}->{migration_network}; - - if ($migration_type eq 'insecure') { - push @$cmd, '--stateuri', 'tcp'; - } else { - push @$cmd, '--stateuri', 'unix'; + if ($start->{forcemachine}) { + push @$cmd, '--machine', $start->{forcemachine}; } - if ($self->{forcemachine}) { - push @$cmd, '--machine', $self->{forcemachine}; - } - - if ($self->{forcecpu}) { - push @$cmd, '--force-cpu', $self->{forcecpu}; + if ($start->{forcecpu}) { + push @$cmd, '--force-cpu', $start->{forcecpu}; } if ($self->{online_local_volumes}) { @@ -786,12 +779,9 @@ sub phase2 { } my $spice_port; - my $unix_socket_info = {}; - # version > 0 for unix socket support - my $nbd_protocol_version = 1; # TODO change to 'spice_ticket: \n' in 7.0 - my $input = $spice_ticket ? "$spice_ticket\n" : "\n"; - $input .= "nbd_protocol_version: $nbd_protocol_version\n"; + my $input = $migrate->{spice_ticket} ? "$migrate->{spice_ticket}\n" : "\n"; + $input .= "nbd_protocol_version: $migrate->{nbd_proto_version}\n"; my $number_of_online_replicated_volumes = 0; @@ -811,20 +801,20 @@ sub phase2 { my $exitcode = PVE::Tools::run_command($cmd, input => $input, outfunc => sub { my $line = shift; - if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { - $raddr = $1; - $rport = int($2); - $ruri = "tcp:$raddr:$rport"; + if ($line =~ m/^migration listens on (tcp):(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { + $tunnel_info->{addr} = $2; + $tunnel_info->{port} = int($3); + $tunnel_info->{proto} = $1; } - elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) { - $raddr = $1; - die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2; - $ruri = "unix:$raddr"; + elsif ($line =~ m!^migration listens on (unix):(/run/qemu-server/(\d+)\.migrate)$!) { + $tunnel_info->{addr} = $2; + die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $3; + $tunnel_info->{proto} = $1; } elsif ($line =~ m/^migration listens on port (\d+)$/) { - $raddr = "localhost"; - $rport = int($1); - $ruri = "tcp:$raddr:$rport"; + $tunnel_info->{addr} = "localhost"; + $tunnel_info->{port} = int($1); + $tunnel_info->{proto} = "tcp"; } elsif ($line =~ m/^spice listens on port (\d+)$/) { $spice_port = int($1); @@ -849,7 +839,7 @@ sub phase2 { $self->{stopnbd} = 1; $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr; $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri; - $unix_socket_info->{$nbd_unix_addr} = 1; + $tunnel_info->{unix_sockets}->{$nbd_unix_addr} = 1; } elsif ($line =~ m/^re-using replicated volume: (\S+) - (.*)$/) { my $drive = $1; my $volid = $2; @@ -864,14 +854,58 @@ sub phase2 { die "remote command failed with exit code $exitcode\n" if $exitcode; - die "unable to detect remote migration address\n" if !$raddr; + die "unable to detect remote migration address\n" if !$tunnel_info->{addr} || !$tunnel_info->{proto}; if (scalar(keys %$target_replicated_volumes) != $number_of_online_replicated_volumes) { die "number of replicated disks on source and target node do not match - target node too old?\n" } + return ($tunnel_info, $spice_port); +} + +sub phase2 { + my ($self, $vmid) = @_; + + my $conf = $self->{vmconf}; + + # version > 0 for unix socket support + my $nbd_protocol_version = 1; + + my $spice_ticket; + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { + my $res = mon_cmd($vmid, 'query-spice'); + $spice_ticket = $res->{ticket}; + } + + my $migration_type = $self->{opts}->{migration_type}; + my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix'; + + my $params = { + start_params => { + statefile => $state_uri, + forcemachine => $self->{forcemachine}, + forcecpu => $self->{forcecpu}, + skiplock => 1, + }, + migrate_opts => { + spice_ticket => $spice_ticket, + type => $migration_type, + network => $self->{opts}->{migration_network}, + storagemap => $self->{opts}->{storagemap}, + migratedfrom => PVE::INotify::nodename(), + nbd_proto_version => $nbd_protocol_version, + nbd => $self->{nbd}, + }, + }; + + my ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params); + $self->log('info', "start remote tunnel"); - $self->start_remote_tunnel($raddr, $rport, $ruri, $unix_socket_info); + $self->start_remote_tunnel($tunnel_info); + + my $migrate_uri = "$tunnel_info->{proto}:$tunnel_info->{addr}"; + $migrate_uri .= ":$tunnel_info->{port}" + if defined($tunnel_info->{port}); my $start = time(); @@ -908,7 +942,7 @@ sub phase2 { } } - $self->log('info', "starting online/live migration on $ruri"); + $self->log('info', "starting online/live migration on $migrate_uri"); $self->{livemigration} = 1; # load_defaults @@ -981,12 +1015,12 @@ sub phase2 { } - $self->log('info', "start migrate command to $ruri"); + $self->log('info', "start migrate command to $migrate_uri"); eval { - mon_cmd($vmid, "migrate", uri => $ruri); + mon_cmd($vmid, "migrate", uri => $migrate_uri); }; my $merr = $@; - $self->log('info', "migrate uri => $ruri failed: $merr") if $merr; + $self->log('info', "migrate uri => $migrate_uri failed: $merr") if $merr; my $lstat = 0; my $usleep = 1000000; diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index d323d3d..a131fc8 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -5141,10 +5141,10 @@ sub vm_start_nolock { return $migration_ip; }; - my $migrate_uri; if ($statefile) { if ($statefile eq 'tcp') { - my $localip = "localhost"; + my $migrate = $res->{migrate} = { proto => 'tcp' }; + $migrate->{addr} = "localhost"; my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg'); my $nodename = nodename(); @@ -5157,26 +5157,26 @@ sub vm_start_nolock { } if ($migration_type eq 'insecure') { - $localip = $get_migration_ip->($nodename); - $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip); + $migrate->{addr} = $get_migration_ip->($nodename); + $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr}); } my $pfamily = PVE::Tools::get_host_address_family($nodename); - my $migrate_port = PVE::Tools::next_migrate_port($pfamily); - $migrate_uri = "tcp:${localip}:${migrate_port}"; - push @$cmd, '-incoming', $migrate_uri; + $migrate->{port} = PVE::Tools::next_migrate_port($pfamily); + $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}"; + push @$cmd, '-incoming', $migrate->{uri}; push @$cmd, '-S'; } elsif ($statefile eq 'unix') { # should be default for secure migrations as a ssh TCP forward # tunnel is not deterministic reliable ready and fails regurarly # to set up in time, so use UNIX socket forwards - my $socket_addr = "/run/qemu-server/$vmid.migrate"; - unlink $socket_addr; + my $migrate = $res->{migrate} = { proto => 'unix' }; + $migrate->{addr} = "/run/qemu-server/$vmid.migrate"; + unlink $migrate->{addr}; - $migrate_uri = "unix:$socket_addr"; - - push @$cmd, '-incoming', $migrate_uri; + $migrate->{uri} = "unix:$migrate->{addr}"; + push @$cmd, '-incoming', $migrate->{uri}; push @$cmd, '-S'; } elsif (-e $statefile) { @@ -5297,10 +5297,9 @@ sub vm_start_nolock { die "start failed: $err"; } - print "migration listens on $migrate_uri\n" if $migrate_uri; - $res->{migrate_uri} = $migrate_uri; - - if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') { + if (defined($res->{migrate})) { + print "migration listens on $res->{migrate}->{uri}\n"; + } elsif ($statefile) { eval { mon_cmd($vmid, "cont"); }; warn $@ if $@; } @@ -5315,6 +5314,7 @@ sub vm_start_nolock { my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate"; mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } ); $migrate_storage_uri = "nbd:unix:$socket_path"; + $res->{migrate}->{unix_sockets} = [$socket_path]; } else { my $nodename = nodename(); my $localip = $get_migration_ip->($nodename); -- 2.20.1