[PATCH v2 qemu-server] remote migration: allow insecure TCP data plane

[Bogdan Ionescu <bogdan@ionescu.at>]

Expose migration_type=insecure and migration_network for remote
migration.

The websocket tunnel remains used for control commands, while QEMU
migration state and NBD storage migration can use a direct TCP data
plane. This avoids websocket masking/TLS overhead on trusted private
migration networks.

Gate the feature with an insecure-remote mtunnel capability so source
nodes fail early when the target does not support it. Require Sys.Modify
on / when using the insecure data plane or explicitly selecting a
migration network.

This mode is only intended for fully trusted private networks, as guest
RAM and disk migration data may be transferred in clear text.

Signed-off-by: Bogdan Ionescu <bogdan@ionescu.at>
---
 src/PVE/API2/Qemu.pm   | 47 +++++++++++++++++++++++-----
 src/PVE/CLI/qm.pm      | 17 ++++++++++
 src/PVE/QemuMigrate.pm | 70 ++++++++++++++++++++++++++++++------------
 3 files changed, 107 insertions(+), 27 deletions(-)

diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm
index d762401b..43888ab5 100644
--- a/src/PVE/API2/Qemu.pm
+++ b/src/PVE/API2/Qemu.pm
@@ -5668,6 +5668,23 @@ __PACKAGE__->register_method({
                 minimum => '0',
                 default => 'migrate limit from datacenter or storage config',
             },
+            migration_type => {
+                type => 'string',
+                enum => ['secure', 'insecure'],
+                description =>
+                    "Migration traffic is encrypted using a websocket tunnel by default. "
+                    . "On secure, completely private networks this can be disabled to "
+                    . "increase performance. WARNING: with 'insecure', VM RAM and disk "
+                    . "migration data is transferred in clear text over the selected "
+                    . "migration network.",
+                optional => 1,
+            },
+            migration_network => {
+                type => 'string',
+                format => 'CIDR',
+                description => "CIDR of the trusted private network used for insecure remote migration.",
+                optional => 1,
+            },
         },
     },
     returns => {
@@ -5683,9 +5700,16 @@ __PACKAGE__->register_method({
         my $source_vmid = extract_param($param, 'vmid');
         my $target_endpoint = extract_param($param, 'target-endpoint');
         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
+        my $migration_type = extract_param($param, 'migration_type') // 'secure';
+        my $migration_network = extract_param($param, 'migration_network');
 
         my $delete = extract_param($param, 'delete') // 0;
 
+        # insecure remote migration can transfer VM RAM and disk data in clear text
+        if ($migration_type eq 'insecure' || defined($migration_network)) {
+            $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
+        }
+
         PVE::Cluster::check_cfs_quorum();
 
         # test if VM exists
@@ -5760,7 +5784,8 @@ __PACKAGE__->register_method({
             client => $api_client,
             vmid => $target_vmid,
         };
-        $param->{migration_type} = 'websocket';
+        $param->{migration_type} = $migration_type eq 'insecure' ? 'insecure' : 'websocket';
+        $param->{migration_network} = $migration_network if defined($migration_network);
         $param->{'with-local-disks'} = 1;
         $param->{delete} = $delete if $delete;
 
@@ -6716,6 +6741,7 @@ __PACKAGE__->register_method({
                     return {
                         api => $PVE::QemuMigrate::WS_TUNNEL_VERSION,
                         age => 0,
+                        caps => ['insecure-remote'],
                     };
                 },
                 'config' => sub {
@@ -6866,19 +6892,24 @@ __PACKAGE__->register_method({
                         $params->{migrate_opts},
                     );
 
-                    if ($info->{migrate}->{proto} ne 'unix') {
+                    if (
+                        $params->{migrate_opts}->{type} ne 'insecure'
+                        && $info->{migrate}->{proto} ne 'unix'
+                    ) {
                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
                         die "migration over non-UNIX sockets not possible\n";
                     }
 
-                    my $socket = $info->{migrate}->{addr};
-                    chown $state->{socket_uid}, -1, $socket;
-                    $state->{sockets}->{$socket} = 1;
-
-                    my $unix_sockets = $info->{migrate}->{unix_sockets};
-                    foreach my $socket (@$unix_sockets) {
+                    if ($info->{migrate}->{proto} eq 'unix') {
+                        my $socket = $info->{migrate}->{addr};
                         chown $state->{socket_uid}, -1, $socket;
                         $state->{sockets}->{$socket} = 1;
+
+                        my $unix_sockets = $info->{migrate}->{unix_sockets} // [];
+                        foreach my $socket (@$unix_sockets) {
+                            chown $state->{socket_uid}, -1, $socket;
+                            $state->{sockets}->{$socket} = 1;
+                        }
                     }
                     return $info;
                 },
diff --git a/src/PVE/CLI/qm.pm b/src/PVE/CLI/qm.pm
index bfa0d1d5..43ec442a 100755
--- a/src/PVE/CLI/qm.pm
+++ b/src/PVE/CLI/qm.pm
@@ -224,6 +224,23 @@ __PACKAGE__->register_method({
                 minimum => '0',
                 default => 'migrate limit from datacenter or storage config',
             },
+            migration_type => {
+                type => 'string',
+                enum => ['secure', 'insecure'],
+                description =>
+                    "Migration traffic is encrypted using a websocket tunnel by default. "
+                    . "On secure, completely private networks this can be disabled to "
+                    . "increase performance. WARNING: with 'insecure', VM RAM and disk "
+                    . "migration data is transferred in clear text over the selected "
+                    . "migration network.",
+                optional => 1,
+            },
+            migration_network => {
+                type => 'string',
+                format => 'CIDR',
+                description => "CIDR of the trusted private network used for insecure remote migration.",
+                optional => 1,
+            },
         },
     },
     returns => {
diff --git a/src/PVE/QemuMigrate.pm b/src/PVE/QemuMigrate.pm
index 8f38bf69..455024f0 100644
--- a/src/PVE/QemuMigrate.pm
+++ b/src/PVE/QemuMigrate.pm
@@ -46,6 +46,12 @@ use base qw(PVE::AbstractMigrate);
 # compared against remote end's minimum version
 our $WS_TUNNEL_VERSION = 2;
 
+sub remote_tunnel_has_cap {
+    my ($tunnel, $cap) = @_;
+
+    return grep { $_ eq $cap } @{ $tunnel->{caps} // [] };
+}
+
 sub fork_tunnel {
     my ($self, $ssh_forward_info) = @_;
 
@@ -351,6 +357,10 @@ sub prepare {
             if $WS_TUNNEL_VERSION < $min_version;
         die "Remote tunnel endpoint too old, upgrade required\n"
             if $WS_TUNNEL_VERSION > $tunnel->{version};
+        die "Remote tunnel endpoint does not support insecure remote migration, upgrade target or"
+            . " omit migration_type=insecure\n"
+            if $self->{opts}->{migration_type} eq 'insecure'
+            && !remote_tunnel_has_cap($tunnel, 'insecure-remote');
 
         print "websocket tunnel started\n";
         $self->{tunnel} = $tunnel;
@@ -1144,8 +1154,9 @@ sub phase2_start_local_cluster {
 sub phase2_start_remote_cluster {
     my ($self, $vmid, $params) = @_;
 
-    die "insecure migration to remote cluster not implemented\n"
-        if $params->{migrate_opts}->{type} ne 'websocket';
+    die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n"
+        if $params->{migrate_opts}->{type} ne 'websocket'
+        && $params->{migrate_opts}->{type} ne 'insecure';
 
     my $remote_vmid = $self->{opts}->{remote}->{vmid};
 
@@ -1159,8 +1170,13 @@ sub phase2_start_remote_cluster {
         $self->{stopnbd} = 1;
         $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr};
         my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri};
-        die "unexpected NBD uri for '$drive': $nbd_uri\n"
-            if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
+        if ($params->{migrate_opts}->{type} eq 'websocket') {
+            die "unexpected NBD uri for '$drive': $nbd_uri\n"
+                if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
+        } elsif ($params->{migrate_opts}->{type} eq 'insecure') {
+            die "unexpected NBD uri for '$drive': $nbd_uri\n"
+                if $nbd_uri !~ m!^nbd:(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):\d+:exportname=drive-[A-Za-z0-9_.-]+$!;
+        }
 
         $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri;
     }
@@ -1254,28 +1270,44 @@ sub phase2 {
         my $remote_vmid = $remote->{vmid};
         $params->{migrate_opts}->{remote_node} = $self->{node};
         ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params);
-        die "only UNIX sockets are supported for remote migration\n"
-            if $tunnel_info->{proto} ne 'unix';
-
-        # untaint
-        my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$|
-            or die "unexpected socket address '$tunnel_info->{addr}'\n";
-        my $local_socket = $remote_socket;
-        $local_socket =~ s/$remote_vmid/$vmid/g;
-        $tunnel_info->{addr} = $local_socket;
 
-        $self->log('info', "Setting up tunnel for '$local_socket'");
-        PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+        if ($params->{migrate_opts}->{type} eq 'websocket') {
+            die "only UNIX sockets are supported for remote migration\n"
+                if $tunnel_info->{proto} ne 'unix';
 
-        foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} }) {
             # untaint
-            ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$|
-                or die "unexpected socket address '$remote_socket'\n";
+            my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$|
+                or die "unexpected socket address '$tunnel_info->{addr}'\n";
             my $local_socket = $remote_socket;
             $local_socket =~ s/$remote_vmid/$vmid/g;
-            next if $self->{tunnel}->{forwarded}->{$local_socket};
+            $tunnel_info->{addr} = $local_socket;
+
             $self->log('info', "Setting up tunnel for '$local_socket'");
             PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+
+            foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} // [] }) {
+                # untaint
+                ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$|
+                    or die "unexpected socket address '$remote_socket'\n";
+                my $local_socket = $remote_socket;
+                $local_socket =~ s/$remote_vmid/$vmid/g;
+                next if $self->{tunnel}->{forwarded}->{$local_socket};
+                $self->log('info', "Setting up tunnel for '$local_socket'");
+                PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+            }
+        } elsif ($params->{migrate_opts}->{type} eq 'insecure') {
+            die "only TCP sockets are supported for insecure remote migration\n"
+                if $tunnel_info->{proto} ne 'tcp';
+
+            die "unexpected TCP migration address '$tunnel_info->{addr}'\n"
+                if $tunnel_info->{addr} !~ m/^(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\])$/;
+
+            die "unexpected TCP migration port '$tunnel_info->{port}'\n"
+                if $tunnel_info->{port} !~ /^\d+$/ || $tunnel_info->{port} <= 0 || $tunnel_info->{port} > 65535;
+
+            $self->log('info', "using direct TCP migration to $tunnel_info->{addr}:$tunnel_info->{port}");
+        } else {
+            die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n";
         }
     } else {
         ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
-- 
2.47.3