From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 2AAFB620F8 for ; Wed, 19 Jan 2022 15:31:02 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1E4B619DEB for ; Wed, 19 Jan 2022 15:30:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 9FF9619DDB for ; Wed, 19 Jan 2022 15:30:30 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 71376460BD for ; Wed, 19 Jan 2022 15:30:30 +0100 (CET) Date: Wed, 19 Jan 2022 15:30:23 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Fabian Ebner , pve-devel@lists.proxmox.com References: <20211222135257.3242938-1-f.gruenbichler@proxmox.com> <20211222135257.3242938-3-f.gruenbichler@proxmox.com> <47e7d41f-e328-d9fa-25b7-f7585de8ce5b@proxmox.com> In-Reply-To: <<47e7d41f-e328-d9fa-25b7-f7585de8ce5b@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.15.0 (https://github.com/astroidmail/astroid) Message-Id: <1642590268.dgcsphrfs6.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.217 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pve-devel] [PATCH v3 guest-common 2/3] add tunnel helper module X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 19 Jan 2022 14:31:02 -0000 On January 3, 2022 1:30 pm, Fabian Ebner wrote: > A few nits inline. >=20 > Am 12/22/21 um 14:52 schrieb Fabian Gr=C3=BCnbichler: >> lifted from PVE::QemuMigrate, abstracting away use-case specific data. >>=20 >> Signed-off-by: Fabian Gr=C3=BCnbichler >> --- >> src/Makefile | 1 + >> debian/control | 1 + >> src/PVE/Tunnel.pm | 356 ++++++++++++++++++++++++++++++++++++++++++++++ >> 3 files changed, 358 insertions(+) >> create mode 100644 src/PVE/Tunnel.pm >>=20 >> diff --git a/src/Makefile b/src/Makefile >> index 0298d3f..d82162c 100644 >> --- a/src/Makefile >> +++ b/src/Makefile >> @@ -12,6 +12,7 @@ install: PVE >> install -m 0644 PVE/ReplicationConfig.pm ${PERL5DIR}/PVE/ >> install -m 0644 PVE/ReplicationState.pm ${PERL5DIR}/PVE/ >> install -m 0644 PVE/Replication.pm ${PERL5DIR}/PVE/ >> + install -m 0644 PVE/Tunnel.pm ${PERL5DIR}/PVE/ >> install -d ${PERL5DIR}/PVE/VZDump >> install -m 0644 PVE/VZDump/Plugin.pm ${PERL5DIR}/PVE/VZDump/ >> install -m 0644 PVE/VZDump/Common.pm ${PERL5DIR}/PVE/VZDump/ >> diff --git a/debian/control b/debian/control >> index 4c246d0..73c28bd 100644 >> --- a/debian/control >> +++ b/debian/control >> @@ -16,6 +16,7 @@ Depends: libpve-cluster-perl, >> libpve-common-perl (>=3D 4.0-89), >> libpve-storage-perl (>=3D 7.0-14), >> pve-cluster, >> + proxmox-websocket-tunnel, >> ${misc:Depends}, >> ${perl:Depends}, >> Breaks: libpve-common-perl (<< 4.0-89), >> diff --git a/src/PVE/Tunnel.pm b/src/PVE/Tunnel.pm >> new file mode 100644 >> index 0000000..bbd1169 >> --- /dev/null >> +++ b/src/PVE/Tunnel.pm >> @@ -0,0 +1,356 @@ >> +package PVE::Tunnel; >> + >> +use strict; >> +use warnings; >> + >> +use IO::Pipe; >> +use IPC::Open2; >> +use JSON qw(encode_json decode_json); >> +use POSIX qw( WNOHANG ); >> +use Storable qw(dclone); >> +use URI::Escape; >> + >> +use PVE::APIClient::LWP; >> +use PVE::Tools; >> + >> +my $finish_command_pipe =3D sub { >> + my ($cmdpipe, $timeout) =3D @_; >> + >> + my $cpid =3D $cmdpipe->{pid}; >> + return if !defined($cpid); >> + >> + my $writer =3D $cmdpipe->{writer}; >> + my $reader =3D $cmdpipe->{reader}; >> + >> + $writer->close(); >> + $reader->close(); >> + >> + my $collect_child_process =3D sub { >> + my $res =3D waitpid($cpid, WNOHANG); >> + if (defined($res) && ($res =3D=3D $cpid)) { >> + delete $cmdpipe->{cpid}; >> + return 1; >> + } else { >> + return 0; >> + } >> + }; >=20 > style nit: white-space error >=20 >> + >> + if ($timeout) { >> + for (my $i =3D 0; $i < $timeout; $i++) { >> + return if &$collect_child_process(); >> + sleep(1); >> + } >> + } >> + >> + $cmdpipe->{log}->("tunnel still running - terminating now with SIGT= ERM\n"); >> + kill(15, $cpid); >> + >> + # wait again >> + for (my $i =3D 0; $i < 10; $i++) { >> + return if &$collect_child_process(); >> + sleep(1); >> + } >> + >> + $cmdpipe->{log}->("tunnel still running - terminating now with SIGK= ILL\n"); >> + kill 9, $cpid; >> + sleep 1; >> + >> + $cmdpipe->{log}->("tunnel child process (PID $cpid) couldn't be col= lected\n") >> + if !&$collect_child_process(); >> +}; >> + >> +sub read_tunnel { >> + my ($tunnel, $timeout) =3D @_; >> + >> + $timeout =3D 60 if !defined($timeout); >> + >> + my $reader =3D $tunnel->{reader}; >> + >> + my $output; >> + eval { >> + PVE::Tools::run_with_timeout($timeout, sub { $output =3D <$reader>; })= ; >> + }; >> + die "reading from tunnel failed: $@\n" if $@; >> + >> + chomp $output if defined($output); >> + >> + return $output; >> +} >> + >> +sub write_tunnel { >> + my ($tunnel, $timeout, $command, $params) =3D @_; >> + >> + $timeout =3D 60 if !defined($timeout); >> + >> + my $writer =3D $tunnel->{writer}; >> + >> + if ($tunnel->{version} && $tunnel->{version} >=3D 2) { >> + my $object =3D defined($params) ? dclone($params) : {}; >> + $object->{cmd} =3D $command; >> + >> + $command =3D eval { JSON::encode_json($object) }; >> + >> + die "failed to encode command as JSON - $@\n" >> + if $@; >> + } >> + >> + eval { >> + PVE::Tools::run_with_timeout($timeout, sub { >> + print $writer "$command\n"; >> + $writer->flush(); >> + }); >> + }; >> + die "writing to tunnel failed: $@\n" if $@; >> + >> + if ($tunnel->{version} && $tunnel->{version} >=3D 1) { >> + my $res =3D eval { read_tunnel($tunnel, $timeout); }; >> + die "no reply to command '$command': $@\n" if $@; >> + >> + if ($tunnel->{version} =3D=3D 1) { >> + if ($res eq 'OK') { >> + return; >> + } else { >> + die "tunnel replied '$res' to command '$command'\n"; >> + } >> + } else { >> + my $parsed =3D eval { JSON::decode_json($res) }; >> + die "failed to decode tunnel reply '$res' (command '$command') - $= @\n" >> + if $@; >> + >> + if (!$parsed->{success}) { >> + if (defined($parsed->{msg})) { >> + die "error - tunnel command '$command' failed - $parsed->{msg}\n"= ; >> + } else { >> + die "error - tunnel command '$command' failed\n"; >> + } >> + } >> + >> + return $parsed; >> + } >> + } >> +} >> + >> +sub fork_ssh_tunnel { >> + my ($rem_ssh, $cmd, $ssh_forward_info, $log) =3D @_; >> + >> + my @localtunnelinfo =3D (); >> + foreach my $addr (@$ssh_forward_info) { >> + push @localtunnelinfo, '-L', $addr; >> + } >> + >> + my $full_cmd =3D [@$rem_ssh, '-o ExitOnForwardFailure=3Dyes', @loca= ltunnelinfo, @$cmd]; >> + >> + my $reader =3D IO::File->new(); >> + my $writer =3D IO::File->new(); >> + >> + my $orig_pid =3D $$; >> + >> + my $cpid; >> + >> + eval { $cpid =3D open2($reader, $writer, @$full_cmd); }; >> + >> + my $err =3D $@; >> + >> + # catch exec errors >> + if ($orig_pid !=3D $$) { >> + $log->("can't fork command pipe, aborting\n"); >> + POSIX::_exit(1); >> + kill('KILL', $$); >> + } >> + >> + die $err if $err; >> + >> + my $tunnel =3D { >> + writer =3D> $writer, >> + reader =3D> $reader, >> + pid =3D> $cpid, >> + rem_ssh =3D> $rem_ssh, >> + log =3D> $log, >> + }; >> + >> + eval { >> + my $helo =3D read_tunnel($tunnel, 60); >> + die "no reply\n" if !$helo; >> + die "no quorum on target node\n" if $helo =3D~ m/^no quorum$/; >> + die "got strange reply from tunnel ('$helo')\n" >> + if $helo !~ m/^tunnel online$/; >> + }; >> + $err =3D $@; >> + >> + eval { >> + my $ver =3D read_tunnel($tunnel, 10); >> + if ($ver =3D~ /^ver (\d+)$/) { >> + $tunnel->{version} =3D $1; >> + $log->('info', "ssh tunnel $ver\n"); >=20 > Should pass only the message like the other calls to $log (unless the=20 > function is intended to behave differently when multiple parameters are=20 > used, but in qemu-server 06/10 that doesn't happen). What are the=20 > reasons for not sticking to the two-parameter log interface? no real reason other than 'let's make it more simple' (all the 'err'=20 level does is prefix the lines with 'ERROR: '). but we can add it back=20 in if we want? replication uses a simple logfunc, so we need to adapt one to the other=20 in any case: - either migration only using 'info' (current) - or replication wrapping its logfunc and throwing away the level/adding it= =20 as prefix (alternative) the alternative is less 'lossy', so maybe I'll switch to that.. >=20 >> + } else { >> + $err =3D "received invalid tunnel version string '$ver'\n" if !$er= r; >> + } >> + }; >> + >> + if ($err) { >> + $finish_command_pipe->($tunnel); >> + die "can't open tunnel - $err"; >> + } >> + return $tunnel; >> +} >> + >> +sub forward_unix_socket { >> + my ($tunnel, $local, $remote) =3D @_; >> + >> + my $params =3D dclone($tunnel->{params}); >> + $params->{unix} =3D $local; >> + $params->{url} =3D $params->{url} ."socket=3D".uri_escape($remote).= "&"; >> + $params->{ticket} =3D { path =3D> $remote }; >> + >> + my $cmd =3D encode_json({ >> + control =3D> JSON::true, >> + cmd =3D> 'forward', >> + data =3D> $params, >> + }); >> + >> + my $writer =3D $tunnel->{writer}; >> + $tunnel->{forwarded}->{$local} =3D $remote; >> + eval { >> + unlink $local; >> + PVE::Tools::run_with_timeout(15, sub { >> + print $writer "$cmd\n"; >> + $writer->flush(); >> + }); >> + }; >> + die "failed to write forwarding command - $@\n" if $@; >> + >> + read_tunnel($tunnel); >> +} >> + >> +sub fork_websocket_tunnel { >> + my ($conn, $url, $req_params, $tunnel_params, $log) =3D @_; >> + >> + if (my $apitoken =3D $conn->{apitoken}) { >> + $tunnel_params->{headers} =3D [["Authorization", "$apitoken"]]; >> + } else { >> + die "can't connect to remote host without credentials\n"; >> + } >> + >> + if (my $fps =3D $conn->{cached_fingerprints}) { >> + $tunnel_params->{fingerprint} =3D (keys %$fps)[0]; >> + } >> + >> + my $api_client =3D PVE::APIClient::LWP->new(%$conn); >> + >> + my $res =3D $api_client->post( >> + $url, >> + $req_params, >> + ); >> + >> + $log->("remote: started migration tunnel worker '$res->{upid}'"); >=20 > Nit: still mentions migration, as is done... thanks, removed >=20 >> + >> + my $websocket_url =3D $tunnel_params->{url}; >> + >> + $tunnel_params->{url} .=3D "?ticket=3D".uri_escape($res->{ticket}); >> + $tunnel_params->{url} .=3D "&socket=3D".uri_escape($res->{socket}); >> + >> + my $reader =3D IO::Pipe->new(); >> + my $writer =3D IO::Pipe->new(); >> + >> + my $cpid =3D fork(); >> + if ($cpid) { >> + $writer->writer(); >> + $reader->reader(); >> + my $tunnel =3D { writer =3D> $writer, reader =3D> $reader, pid =3D> $c= pid }; >> + >> + eval { >> + my $writer =3D $tunnel->{writer}; >> + my $cmd =3D encode_json({ >> + control =3D> JSON::true, >> + cmd =3D> 'connect', >> + data =3D> $tunnel_params, >> + }); >> + >> + eval { >> + PVE::Tools::run_with_timeout(15, sub { >> + print {$writer} "$cmd\n"; >> + $writer->flush(); >> + }); >> + }; >> + die "failed to write tunnel connect command - $@\n" if $@; >> + }; >> + die "failed to connect via WS: $@\n" if $@; >> + >> + my $err; >> + eval { >> + my $writer =3D $tunnel->{writer}; >> + my $cmd =3D encode_json({ >> + cmd =3D> 'version', >> + }); >> + >> + eval { >> + PVE::Tools::run_with_timeout(15, sub { >> + print {$writer} "$cmd\n"; >> + $writer->flush(); >> + }); >> + }; >> + $err =3D "failed to write tunnel version command - $@\n" if $@; >> + my $res =3D read_tunnel($tunnel, 10); >> + $res =3D JSON::decode_json($res); >> + my $version =3D $res->{api}; >> + >> + if ($version =3D~ /^(\d+)$/) { >> + $tunnel->{version} =3D $1; >> + $tunnel->{age} =3D $res->{age}; >> + } else { >> + $err =3D "received invalid tunnel version string '$version'\n" if !$e= rr; >> + } >> + }; >> + $err =3D $@ if !$err; >> + >> + if ($err) { >> + $finish_command_pipe->($tunnel); >=20 > Nit: Here, $tunnel->{log} is still undef, but finish_command_pipe might=20 > try to use it. ack, since we already have $log anyway we can just set it from the=20 start, like in fork_ssh_tunnel. >=20 >> + die "can't open migration tunnel - $err"; >=20 > ...here. removed as well. >=20 >> + } >> + >> + $tunnel_params->{url} =3D "$websocket_url?"; # reset ticket and socket >> + >> + $tunnel->{params} =3D $tunnel_params; # for forwarding >> + $tunnel->{log} =3D $log; >> + >> + return $tunnel; >> + } else { >> + eval { >> + $writer->reader(); >> + $reader->writer(); >> + PVE::Tools::run_command( >> + ['proxmox-websocket-tunnel'], >> + input =3D> "<&".fileno($writer), >> + output =3D> ">&".fileno($reader), >> + errfunc =3D> sub { my $line =3D shift; print "tunnel: $line\n"; }, >> + ); >> + }; >> + warn "CMD websocket tunnel died: $@\n" if $@; >> + exit 0; >> + } >> +} >> + >> +sub finish_tunnel { >> + my ($tunnel, $cleanup) =3D @_; >> + >> + $cleanup =3D $cleanup ? 1 : 0; >> + >> + eval { write_tunnel($tunnel, 30, 'quit', { cleanup =3D> $cleanup })= ; }; >> + my $err =3D $@; >> + >> + $finish_command_pipe->($tunnel, 30); >> + >> + if (my $unix_sockets =3D $tunnel->{unix_sockets}) { >> + # ssh does not clean up on local host >> + my $cmd =3D ['rm', '-f', @$unix_sockets]; >> + PVE::Tools::run_command($cmd); >> + >> + # .. and just to be sure check on remote side >> + if ($tunnel->{rem_ssh}) { >> + unshift @{$cmd}, @{$tunnel->{rem_ssh}}; >> + PVE::Tools::run_command($cmd); >> + } >> + } >> + >> + die $err if $err; >> +} >=20