From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id DDD7F60F00 for ; Thu, 3 Feb 2022 13:42:53 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E1919269DF for ; Thu, 3 Feb 2022 13:42:22 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C68A3269D2 for ; Thu, 3 Feb 2022 13:42:21 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 89A2A44BC2 for ; Thu, 3 Feb 2022 13:42:21 +0100 (CET) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Thu, 3 Feb 2022 13:41:34 +0100 Message-Id: <20220203124143.1931377-14-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220203124143.1931377-1-f.gruenbichler@proxmox.com> References: <20220203124143.1931377-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.210 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [qemumigrate.pm, qemumigratemock.pm] Subject: [pve-devel] [PATCH v4 qemu-server 06/11] migrate: move tunnel-helpers to pve-guest-common X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 03 Feb 2022 12:42:53 -0000 besides the log calls these don't need any parts of the migration state, so let's make them generic and re-use them for container migration and replication in the future. Signed-off-by: Fabian Grünbichler --- Notes: v4: switch log to use two-parameter logging sub like migration new in v3, requires bumped libpve-guest-common-perl PVE/QemuMigrate.pm | 183 ++------------------------ test/MigrationTest/QemuMigrateMock.pm | 28 ++-- 2 files changed, 26 insertions(+), 185 deletions(-) diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm index 3d052421..b8e5b942 100644 --- a/PVE/QemuMigrate.pm +++ b/PVE/QemuMigrate.pm @@ -18,6 +18,7 @@ use PVE::ReplicationConfig; use PVE::ReplicationState; use PVE::Storage; use PVE::Tools; +use PVE::Tunnel; use PVE::QemuConfig; use PVE::QemuServer::CPUConfig; @@ -30,180 +31,16 @@ use PVE::QemuServer; use PVE::AbstractMigrate; use base qw(PVE::AbstractMigrate); -sub fork_command_pipe { - my ($self, $cmd) = @_; - - my $reader = IO::File->new(); - my $writer = IO::File->new(); - - my $orig_pid = $$; - - my $cpid; - - eval { $cpid = open2($reader, $writer, @$cmd); }; - - my $err = $@; - - # catch exec errors - if ($orig_pid != $$) { - $self->log('err', "can't fork command pipe\n"); - POSIX::_exit(1); - kill('KILL', $$); - } - - die $err if $err; - - return { writer => $writer, reader => $reader, pid => $cpid }; -} - -sub finish_command_pipe { - my ($self, $cmdpipe, $timeout) = @_; - - my $cpid = $cmdpipe->{pid}; - return if !defined($cpid); - - my $writer = $cmdpipe->{writer}; - my $reader = $cmdpipe->{reader}; - - $writer->close(); - $reader->close(); - - my $collect_child_process = sub { - my $res = waitpid($cpid, WNOHANG); - if (defined($res) && ($res == $cpid)) { - delete $cmdpipe->{cpid}; - return 1; - } else { - return 0; - } - }; - - if ($timeout) { - for (my $i = 0; $i < $timeout; $i++) { - return if &$collect_child_process(); - sleep(1); - } - } - - $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n"); - kill(15, $cpid); - - # wait again - for (my $i = 0; $i < 10; $i++) { - return if &$collect_child_process(); - sleep(1); - } - - $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n"); - kill 9, $cpid; - sleep 1; - - $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n") - if !&$collect_child_process(); -} - -sub read_tunnel { - my ($self, $tunnel, $timeout) = @_; - - $timeout = 60 if !defined($timeout); - - my $reader = $tunnel->{reader}; - - my $output; - eval { - PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; }); - }; - die "reading from tunnel failed: $@\n" if $@; - - chomp $output; - - return $output; -} - -sub write_tunnel { - my ($self, $tunnel, $timeout, $command) = @_; - - $timeout = 60 if !defined($timeout); - - my $writer = $tunnel->{writer}; - - eval { - PVE::Tools::run_with_timeout($timeout, sub { - print $writer "$command\n"; - $writer->flush(); - }); - }; - die "writing to tunnel failed: $@\n" if $@; - - if ($tunnel->{version} && $tunnel->{version} >= 1) { - my $res = eval { $self->read_tunnel($tunnel, 10); }; - die "no reply to command '$command': $@\n" if $@; - - if ($res eq 'OK') { - return; - } else { - die "tunnel replied '$res' to command '$command'\n"; - } - } -} - sub fork_tunnel { my ($self, $ssh_forward_info) = @_; - my @localtunnelinfo = (); - foreach my $addr (@$ssh_forward_info) { - push @localtunnelinfo, '-L', $addr; - } - - my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ]; - - my $tunnel = $self->fork_command_pipe($cmd); - - eval { - my $helo = $self->read_tunnel($tunnel, 60); - die "no reply\n" if !$helo; - die "no quorum on target node\n" if $helo =~ m/^no quorum$/; - die "got strange reply from mtunnel ('$helo')\n" - if $helo !~ m/^tunnel online$/; + my $cmd = ['/usr/sbin/qm', 'mtunnel']; + my $log = sub { + my ($level, $msg) = @_; + $self->log($level, $msg); }; - my $err = $@; - - eval { - my $ver = $self->read_tunnel($tunnel, 10); - if ($ver =~ /^ver (\d+)$/) { - $tunnel->{version} = $1; - $self->log('info', "ssh tunnel $ver\n"); - } else { - $err = "received invalid tunnel version string '$ver'\n" if !$err; - } - }; - - if ($err) { - $self->finish_command_pipe($tunnel); - die "can't open migration tunnel - $err"; - } - return $tunnel; -} - -sub finish_tunnel { - my ($self, $tunnel) = @_; - - eval { $self->write_tunnel($tunnel, 30, 'quit'); }; - my $err = $@; - - $self->finish_command_pipe($tunnel, 30); - - if (my $unix_sockets = $tunnel->{unix_sockets}) { - # ssh does not clean up on local host - my $cmd = ['rm', '-f', @$unix_sockets]; - PVE::Tools::run_command($cmd); - - # .. and just to be sure check on remote side - unshift @{$cmd}, @{$self->{rem_ssh}}; - PVE::Tools::run_command($cmd); - } - die $err if $err; + return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log); } sub start_remote_tunnel { @@ -244,7 +81,7 @@ sub start_remote_tunnel { } if ($unix_socket_try > 100) { $self->{errors} = 1; - $self->finish_tunnel($self->{tunnel}); + PVE::Tunnel::finish_tunnel($self->{tunnel}); die "Timeout, migration socket $ruri did not get ready"; } $self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets); @@ -1254,7 +1091,7 @@ sub phase2_cleanup { if ($self->{tunnel}) { - eval { finish_tunnel($self, $self->{tunnel}); }; + eval { PVE::Tunnel::finish_tunnel($self->{tunnel}); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; @@ -1312,7 +1149,7 @@ sub phase3_cleanup { # config moved and nbd server stopped - now we can resume vm on target if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) { eval { - $self->write_tunnel($tunnel, 30, "resume $vmid"); + PVE::Tunnel::write_tunnel($tunnel, 30, "resume $vmid"); }; if (my $err = $@) { $self->log('err', $err); @@ -1339,7 +1176,7 @@ sub phase3_cleanup { # close tunnel on successful migration, on error phase2_cleanup closed it if ($tunnel) { - eval { finish_tunnel($self, $tunnel); }; + eval { PVE::Tunnel::finish_tunnel($tunnel); }; if (my $err = $@) { $self->log('err', $err); $self->{errors} = 1; diff --git a/test/MigrationTest/QemuMigrateMock.pm b/test/MigrationTest/QemuMigrateMock.pm index 8e0b7d09..f2c02819 100644 --- a/test/MigrationTest/QemuMigrateMock.pm +++ b/test/MigrationTest/QemuMigrateMock.pm @@ -51,12 +51,26 @@ $MigrationTest::Shared::qemu_config_module->mock( }, ); -my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate"); -$qemu_migrate_module->mock( +my $tunnel_module = Test::MockModule->new("PVE::Tunnel"); +$tunnel_module->mock( finish_tunnel => sub { delete $expected_calls->{'finish_tunnel'}; return; }, + write_tunnel => sub { + my ($tunnel, $timeout, $command) = @_; + + if ($command =~ m/^resume (\d+)$/) { + my $vmid = $1; + die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid; + return; + } + die "write_tunnel (mocked) - implement me: $command\n"; + }, +); + +my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate"); +$qemu_migrate_module->mock( fork_tunnel => sub { die "fork_tunnel (mocked) - implement me\n"; # currently no call should lead here }, @@ -73,16 +87,6 @@ $qemu_migrate_module->mock( version => 1, }; }, - write_tunnel => sub { - my ($self, $tunnel, $timeout, $command) = @_; - - if ($command =~ m/^resume (\d+)$/) { - my $vmid = $1; - die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid; - return; - } - die "write_tunnel (mocked) - implement me: $command\n"; - }, log => sub { my ($self, $level, $message) = @_; $current_log .= "$level: $message\n"; -- 2.30.2