From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH v5 qemu-server 06/11] migrate: move tunnel-helpers to pve-guest-common
Date: Wed, 9 Feb 2022 14:07:42 +0100 [thread overview]
Message-ID: <20220209130750.902245-14-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20220209130750.902245-1-f.gruenbichler@proxmox.com>
besides the log calls these don't need any parts of the migration state,
so let's make them generic and re-use them for container migration and
replication in the future.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
Notes:
v4: switch log to use two-parameter logging sub like migration
new in v3, requires bumped libpve-guest-common-perl
PVE/QemuMigrate.pm | 183 ++------------------------
test/MigrationTest/QemuMigrateMock.pm | 28 ++--
2 files changed, 26 insertions(+), 185 deletions(-)
diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm
index a00c2459..104e62ce 100644
--- a/PVE/QemuMigrate.pm
+++ b/PVE/QemuMigrate.pm
@@ -18,6 +18,7 @@ use PVE::ReplicationConfig;
use PVE::ReplicationState;
use PVE::Storage;
use PVE::Tools;
+use PVE::Tunnel;
use PVE::QemuConfig;
use PVE::QemuServer::CPUConfig;
@@ -30,180 +31,16 @@ use PVE::QemuServer;
use PVE::AbstractMigrate;
use base qw(PVE::AbstractMigrate);
-sub fork_command_pipe {
- my ($self, $cmd) = @_;
-
- my $reader = IO::File->new();
- my $writer = IO::File->new();
-
- my $orig_pid = $$;
-
- my $cpid;
-
- eval { $cpid = open2($reader, $writer, @$cmd); };
-
- my $err = $@;
-
- # catch exec errors
- if ($orig_pid != $$) {
- $self->log('err', "can't fork command pipe\n");
- POSIX::_exit(1);
- kill('KILL', $$);
- }
-
- die $err if $err;
-
- return { writer => $writer, reader => $reader, pid => $cpid };
-}
-
-sub finish_command_pipe {
- my ($self, $cmdpipe, $timeout) = @_;
-
- my $cpid = $cmdpipe->{pid};
- return if !defined($cpid);
-
- my $writer = $cmdpipe->{writer};
- my $reader = $cmdpipe->{reader};
-
- $writer->close();
- $reader->close();
-
- my $collect_child_process = sub {
- my $res = waitpid($cpid, WNOHANG);
- if (defined($res) && ($res == $cpid)) {
- delete $cmdpipe->{cpid};
- return 1;
- } else {
- return 0;
- }
- };
-
- if ($timeout) {
- for (my $i = 0; $i < $timeout; $i++) {
- return if &$collect_child_process();
- sleep(1);
- }
- }
-
- $self->log('info', "ssh tunnel still running - terminating now with SIGTERM\n");
- kill(15, $cpid);
-
- # wait again
- for (my $i = 0; $i < 10; $i++) {
- return if &$collect_child_process();
- sleep(1);
- }
-
- $self->log('info', "ssh tunnel still running - terminating now with SIGKILL\n");
- kill 9, $cpid;
- sleep 1;
-
- $self->log('err', "ssh tunnel child process (PID $cpid) couldn't be collected\n")
- if !&$collect_child_process();
-}
-
-sub read_tunnel {
- my ($self, $tunnel, $timeout) = @_;
-
- $timeout = 60 if !defined($timeout);
-
- my $reader = $tunnel->{reader};
-
- my $output;
- eval {
- PVE::Tools::run_with_timeout($timeout, sub { $output = <$reader>; });
- };
- die "reading from tunnel failed: $@\n" if $@;
-
- chomp $output;
-
- return $output;
-}
-
-sub write_tunnel {
- my ($self, $tunnel, $timeout, $command) = @_;
-
- $timeout = 60 if !defined($timeout);
-
- my $writer = $tunnel->{writer};
-
- eval {
- PVE::Tools::run_with_timeout($timeout, sub {
- print $writer "$command\n";
- $writer->flush();
- });
- };
- die "writing to tunnel failed: $@\n" if $@;
-
- if ($tunnel->{version} && $tunnel->{version} >= 1) {
- my $res = eval { $self->read_tunnel($tunnel, 10); };
- die "no reply to command '$command': $@\n" if $@;
-
- if ($res eq 'OK') {
- return;
- } else {
- die "tunnel replied '$res' to command '$command'\n";
- }
- }
-}
-
sub fork_tunnel {
my ($self, $ssh_forward_info) = @_;
- my @localtunnelinfo = ();
- foreach my $addr (@$ssh_forward_info) {
- push @localtunnelinfo, '-L', $addr;
- }
-
- my $cmd = [@{$self->{rem_ssh}}, '-o ExitOnForwardFailure=yes', @localtunnelinfo, '/usr/sbin/qm', 'mtunnel' ];
-
- my $tunnel = $self->fork_command_pipe($cmd);
-
- eval {
- my $helo = $self->read_tunnel($tunnel, 60);
- die "no reply\n" if !$helo;
- die "no quorum on target node\n" if $helo =~ m/^no quorum$/;
- die "got strange reply from mtunnel ('$helo')\n"
- if $helo !~ m/^tunnel online$/;
+ my $cmd = ['/usr/sbin/qm', 'mtunnel'];
+ my $log = sub {
+ my ($level, $msg) = @_;
+ $self->log($level, $msg);
};
- my $err = $@;
-
- eval {
- my $ver = $self->read_tunnel($tunnel, 10);
- if ($ver =~ /^ver (\d+)$/) {
- $tunnel->{version} = $1;
- $self->log('info', "ssh tunnel $ver\n");
- } else {
- $err = "received invalid tunnel version string '$ver'\n" if !$err;
- }
- };
-
- if ($err) {
- $self->finish_command_pipe($tunnel);
- die "can't open migration tunnel - $err";
- }
- return $tunnel;
-}
-
-sub finish_tunnel {
- my ($self, $tunnel) = @_;
-
- eval { $self->write_tunnel($tunnel, 30, 'quit'); };
- my $err = $@;
-
- $self->finish_command_pipe($tunnel, 30);
-
- if (my $unix_sockets = $tunnel->{unix_sockets}) {
- # ssh does not clean up on local host
- my $cmd = ['rm', '-f', @$unix_sockets];
- PVE::Tools::run_command($cmd);
-
- # .. and just to be sure check on remote side
- unshift @{$cmd}, @{$self->{rem_ssh}};
- PVE::Tools::run_command($cmd);
- }
- die $err if $err;
+ return PVE::Tunnel::fork_ssh_tunnel($self->{rem_ssh}, $cmd, $ssh_forward_info, $log);
}
sub start_remote_tunnel {
@@ -244,7 +81,7 @@ sub start_remote_tunnel {
}
if ($unix_socket_try > 100) {
$self->{errors} = 1;
- $self->finish_tunnel($self->{tunnel});
+ PVE::Tunnel::finish_tunnel($self->{tunnel});
die "Timeout, migration socket $ruri did not get ready";
}
$self->{tunnel}->{unix_sockets} = $unix_sockets if (@$unix_sockets);
@@ -1254,7 +1091,7 @@ sub phase2_cleanup {
if ($self->{tunnel}) {
- eval { finish_tunnel($self, $self->{tunnel}); };
+ eval { PVE::Tunnel::finish_tunnel($self->{tunnel}); };
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
@@ -1312,7 +1149,7 @@ sub phase3_cleanup {
# config moved and nbd server stopped - now we can resume vm on target
if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 1) {
eval {
- $self->write_tunnel($tunnel, 30, "resume $vmid");
+ PVE::Tunnel::write_tunnel($tunnel, 30, "resume $vmid");
};
if (my $err = $@) {
$self->log('err', $err);
@@ -1339,7 +1176,7 @@ sub phase3_cleanup {
# close tunnel on successful migration, on error phase2_cleanup closed it
if ($tunnel) {
- eval { finish_tunnel($self, $tunnel); };
+ eval { PVE::Tunnel::finish_tunnel($tunnel); };
if (my $err = $@) {
$self->log('err', $err);
$self->{errors} = 1;
diff --git a/test/MigrationTest/QemuMigrateMock.pm b/test/MigrationTest/QemuMigrateMock.pm
index 8e0b7d09..f2c02819 100644
--- a/test/MigrationTest/QemuMigrateMock.pm
+++ b/test/MigrationTest/QemuMigrateMock.pm
@@ -51,12 +51,26 @@ $MigrationTest::Shared::qemu_config_module->mock(
},
);
-my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate");
-$qemu_migrate_module->mock(
+my $tunnel_module = Test::MockModule->new("PVE::Tunnel");
+$tunnel_module->mock(
finish_tunnel => sub {
delete $expected_calls->{'finish_tunnel'};
return;
},
+ write_tunnel => sub {
+ my ($tunnel, $timeout, $command) = @_;
+
+ if ($command =~ m/^resume (\d+)$/) {
+ my $vmid = $1;
+ die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid;
+ return;
+ }
+ die "write_tunnel (mocked) - implement me: $command\n";
+ },
+);
+
+my $qemu_migrate_module = Test::MockModule->new("PVE::QemuMigrate");
+$qemu_migrate_module->mock(
fork_tunnel => sub {
die "fork_tunnel (mocked) - implement me\n"; # currently no call should lead here
},
@@ -73,16 +87,6 @@ $qemu_migrate_module->mock(
version => 1,
};
},
- write_tunnel => sub {
- my ($self, $tunnel, $timeout, $command) = @_;
-
- if ($command =~ m/^resume (\d+)$/) {
- my $vmid = $1;
- die "resuming wrong VM '$vmid'\n" if $vmid ne $test_vmid;
- return;
- }
- die "write_tunnel (mocked) - implement me: $command\n";
- },
log => sub {
my ($self, $level, $message) = @_;
$current_log .= "$level: $message\n";
--
2.30.2
next prev parent reply other threads:[~2022-02-09 13:08 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-02-09 13:07 [pve-devel] [PATCH-SERIES 0/21] remote migration Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 common 1/1] add 'map_id' helper for ID maps Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 container 1/3] fix #1532: add target-storage support to migration Fabian Grünbichler
2022-02-10 11:52 ` Fabian Ebner
2022-02-11 8:33 ` Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 container 2/3] config: add strict parser Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH PoC v5 container 3/3] migration: add remote migration Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 guest-common 1/3] migrate: add get_bwlimit helper Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 guest-common 2/3] add tunnel helper module Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 guest-common 3/3] add storage tunnel module Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 01/11] move map_storage to PVE::JSONSchema::map_id Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 02/11] schema: use pve-bridge-id Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 03/11] parse_config: optional strict mode Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 04/11] update_vm: allow simultaneous setting of boot-order and dev Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 05/11] nbd alloc helper: allow passing in explicit format Fabian Grünbichler
2022-02-09 13:07 ` Fabian Grünbichler [this message]
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 07/11] mtunnel: add API endpoints Fabian Grünbichler
2022-02-11 13:01 ` Fabian Ebner
[not found] ` <<0b8626f8-df25-05a6-3db3-698591688eab@proxmox.com>
2022-02-16 12:57 ` Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 08/11] migrate: refactor remote VM/tunnel start Fabian Grünbichler
2022-02-11 13:01 ` Fabian Ebner
[not found] ` <<ce49d9a8-03b6-01ed-ad01-5cc500bfba19@proxmox.com>
2022-02-16 12:58 ` Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 09/11] migrate: add remote migration handling Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 10/11] api: add remote migrate endpoint Fabian Grünbichler
2022-02-11 13:01 ` Fabian Ebner
[not found] ` <<e5069cdd-7a84-9664-2dea-1ac3e68e339c@proxmox.com>
2022-02-16 12:58 ` Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 qemu-server 11/11] qm: add remote-migrate command Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 storage 1/3] storage_migrate_snapshot: skip for btrfs without snapshots Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 storage 2/3] storage_migrate: pull out import/export_prepare Fabian Grünbichler
2022-02-09 13:07 ` [pve-devel] [PATCH v5 storage 3/3] add volume_import/export_start helpers Fabian Grünbichler
2022-02-09 17:56 ` [pve-devel] [PATCH-SERIES 0/21] remote migration Thomas Lamprecht
2022-02-11 10:38 ` [pve-devel] [PATCH qemu-server follow-up] schema: move 'pve-targetstorage' to pve-common Fabian Grünbichler
2022-02-11 10:38 ` [pve-devel] [PATCH common follow-up] schema: take over 'pve-targetstorage' option Fabian Grünbichler
2022-02-11 11:31 ` [pve-devel] [PATCH qemu-server follow-up] schema: move 'pve-targetstorage' to pve-common Fabian Ebner
2022-02-11 13:08 ` [pve-devel] [PATCH-SERIES 0/21] remote migration Fabian Ebner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220209130750.902245-14-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox