[PATCH v3 qemu-server] remote migration: allow insecure TCP data plane

[Bogdan Ionescu <bogdan@ionescu.at>]

Expose migration_type=insecure and migration_network for remote
migration.

The websocket tunnel remains used for control commands, while QEMU
migration state and NBD storage migration can use a direct TCP data
plane. This avoids websocket masking/TLS overhead on trusted private
migration networks.

Negotiate the opt-in data plane using a dedicated two-way
'capabilities' mtunnel command. The source sends the capabilities it
wants to use, the target returns the subset it supports and permits,
and both endpoints retain the negotiated set for subsequent handling.
Keep the version command limited to tunnel protocol compatibility.

Only negotiate capabilities when an opt-in feature is requested, so
regular remote migrations remain compatible with older targets. Fail
closed and clean up the target tunnel if capability negotiation is not
available or the requested capability is not accepted.

Require Sys.Modify on '/' on the source when selecting the insecure
data plane or an explicit migration network. On the target, only accept
the insecure-remote capability when the remote credentials also have
Sys.Modify on '/'.

This mode is only intended for fully trusted private networks, as guest
RAM and disk migration data may be transferred in clear text.

Changes since v2:
- replace one-way capabilities in the version reply with a dedicated
  two-way capabilities handler
- keep the negotiated capability set on both source and target
- make target acceptance of insecure-remote conditional on Sys.Modify
- clean up the remote tunnel when capability negotiation fails
- adapt the current mtunnel start schema for tcp/insecure migration
- preserve migration_network for the insecure target start path
- drop the pve-guest-common change; the existing JSON tunnel command
  path already carries the capabilities request and reply

Signed-off-by: Bogdan Ionescu <bogdan@ionescu.at>
---
 src/PVE/API2/Qemu.pm   |  90 ++++++++++++++++++++++++++----
 src/PVE/CLI/qm.pm      |  18 ++++++
 src/PVE/QemuMigrate.pm | 121 ++++++++++++++++++++++++++++++++++-------
 3 files changed, 197 insertions(+), 32 deletions(-)

diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm
index 71247ee..9eab183 100644
--- a/src/PVE/API2/Qemu.pm
+++ b/src/PVE/API2/Qemu.pm
@@ -5720,6 +5720,24 @@ __PACKAGE__->register_method({
                 minimum => '0',
                 default => 'migrate limit from datacenter or storage config',
             },
+            migration_type => {
+                type => 'string',
+                enum => ['secure', 'insecure'],
+                description =>
+                    "Migration traffic is encrypted using a websocket tunnel by default. "
+                    . "On secure, completely private networks this can be disabled to "
+                    . "increase performance. WARNING: with 'insecure', VM RAM and disk "
+                    . "migration data is transferred in clear text over the selected "
+                    . "migration network. The target credentials need Sys.Modify on '/'.",
+                optional => 1,
+            },
+            migration_network => {
+                type => 'string',
+                format => 'CIDR',
+                description =>
+                    "CIDR of the trusted private network used for insecure remote migration.",
+                optional => 1,
+            },
         },
     },
     returns => {
@@ -5735,9 +5753,16 @@ __PACKAGE__->register_method({
         my $source_vmid = extract_param($param, 'vmid');
         my $target_endpoint = extract_param($param, 'target-endpoint');
         my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid;
+        my $migration_type = extract_param($param, 'migration_type') // 'secure';
+        my $migration_network = extract_param($param, 'migration_network');
 
         my $delete = extract_param($param, 'delete') // 0;
 
+        # insecure remote migration can transfer VM RAM and disk data in clear text
+        if ($migration_type eq 'insecure' || defined($migration_network)) {
+            $rpcenv->check_full($authuser, "/", ['Sys.Modify']);
+        }
+
         PVE::Cluster::check_cfs_quorum();
 
         # test if VM exists
@@ -5837,7 +5862,8 @@ __PACKAGE__->register_method({
             client => $api_client,
             vmid => $target_vmid,
         };
-        $param->{migration_type} = 'websocket';
+        $param->{migration_type} = $migration_type eq 'insecure' ? 'insecure' : 'websocket';
+        $param->{migration_network} = $migration_network if defined($migration_network);
         $param->{'with-local-disks'} = 1;
         $param->{delete} = $delete if $delete;
 
@@ -6739,6 +6765,15 @@ __PACKAGE__->register_method({
             };
 
             my $cmd_desc = {
+                capabilities => {
+                    capabilities => {
+                        type => 'array',
+                        items => {
+                            type => 'string',
+                            pattern => '[a-z][a-z0-9-]*',
+                        },
+                    },
+                },
                 config => {
                     conf => {
                         type => 'string',
@@ -6768,7 +6803,7 @@ __PACKAGE__->register_method({
                         properties => {
                             statefile => {
                                 type => 'string',
-                                enum => ['unix'],
+                                enum => ['unix', 'tcp'],
                             },
                             forcemachine => get_standard_option('pve-qemu-machine'),
                             forcecpu => {
@@ -6793,7 +6828,7 @@ __PACKAGE__->register_method({
                             },
                             type => {
                                 type => 'string',
-                                enum => ['websocket'],
+                                enum => ['websocket', 'insecure'],
                             },
                             remote_node => get_standard_option('pve-node'),
                             network => {
@@ -6851,6 +6886,20 @@ __PACKAGE__->register_method({
                         age => 0,
                     };
                 },
+                'capabilities' => sub {
+                    my ($params) = @_;
+
+                    my $supported = {};
+                    $supported->{'insecure-remote'} = 1
+                        if $rpcenv->check($authuser, '/', ['Sys.Modify'], 1);
+
+                    my $requested = { map { $_ => 1 } $params->{capabilities}->@* };
+                    my @accepted = grep { $requested->{$_} } sort keys $supported->%*;
+
+                    $state->{capabilities} = { map { $_ => 1 } @accepted };
+
+                    return { capabilities => \@accepted };
+                },
                 'config' => sub {
                     my ($params) = @_;
 
@@ -6998,6 +7047,17 @@ __PACKAGE__->register_method({
                 'start' => sub {
                     my ($params) = @_;
 
+                    my $migration_type = $params->{migrate_opts}->{type};
+                    my $statefile = $params->{start_params}->{statefile};
+                    if ($migration_type eq 'insecure') {
+                        die "insecure remote migration capability was not negotiated\n"
+                            if !($state->{capabilities} // {})->{'insecure-remote'};
+                        die "insecure remote migration requires a TCP statefile\n"
+                            if $statefile ne 'tcp';
+                    } elsif ($statefile ne 'unix') {
+                        die "websocket remote migration requires a UNIX statefile\n";
+                    }
+
                     if (my $nbd = $params->{migrate_opts}->{nbd}) {
                         for my $ds (keys $nbd->%*) {
                             my $drive = $nbd->{$ds};
@@ -7029,7 +7089,7 @@ __PACKAGE__->register_method({
 
                     # not used, but in schema for compat reasons, could stop setting it..
                     delete $params->{migrate_opts}->{storagemap};
-                    delete $params->{migrate_opts}->{network};
+                    delete $params->{migrate_opts}->{network} if $migration_type eq 'websocket';
 
                     my $info = PVE::QemuServer::vm_start_nolock(
                         $state->{storecfg},
@@ -7039,19 +7099,25 @@ __PACKAGE__->register_method({
                         $params->{migrate_opts},
                     );
 
-                    if ($info->{migrate}->{proto} ne 'unix') {
+                    my $proto = $info->{migrate}->{proto};
+                    if (
+                        ($migration_type eq 'websocket' && $proto ne 'unix')
+                        || ($migration_type eq 'insecure' && $proto ne 'tcp')
+                    ) {
                         PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1);
-                        die "migration over non-UNIX sockets not possible\n";
+                        die "unexpected migration protocol '$proto' for '$migration_type' migration\n";
                     }
 
-                    my $socket = $info->{migrate}->{addr};
-                    chown $state->{socket_uid}, -1, $socket;
-                    $state->{sockets}->{$socket} = 1;
-
-                    my $unix_sockets = $info->{migrate}->{unix_sockets};
-                    foreach my $socket (@$unix_sockets) {
+                    if ($proto eq 'unix') {
+                        my $socket = $info->{migrate}->{addr};
                         chown $state->{socket_uid}, -1, $socket;
                         $state->{sockets}->{$socket} = 1;
+
+                        my $unix_sockets = $info->{migrate}->{unix_sockets} // [];
+                        foreach my $socket (@$unix_sockets) {
+                            chown $state->{socket_uid}, -1, $socket;
+                            $state->{sockets}->{$socket} = 1;
+                        }
                     }
                     return $info;
                 },
diff --git a/src/PVE/CLI/qm.pm b/src/PVE/CLI/qm.pm
index b903c1f..cefdf2d 100644
--- a/src/PVE/CLI/qm.pm
+++ b/src/PVE/CLI/qm.pm
@@ -224,6 +224,24 @@ __PACKAGE__->register_method({
                 minimum => '0',
                 default => 'migrate limit from datacenter or storage config',
             },
+            migration_type => {
+                type => 'string',
+                enum => ['secure', 'insecure'],
+                description =>
+                    "Migration traffic is encrypted using a websocket tunnel by default. "
+                    . "On secure, completely private networks this can be disabled to "
+                    . "increase performance. WARNING: with 'insecure', VM RAM and disk "
+                    . "migration data is transferred in clear text over the selected "
+                    . "migration network. The target credentials need Sys.Modify on '/'.",
+                optional => 1,
+            },
+            migration_network => {
+                type => 'string',
+                format => 'CIDR',
+                description =>
+                    "CIDR of the trusted private network used for insecure remote migration.",
+                optional => 1,
+            },
         },
     },
     returns => {
diff --git a/src/PVE/QemuMigrate.pm b/src/PVE/QemuMigrate.pm
index 8da6f15..c459102 100644
--- a/src/PVE/QemuMigrate.pm
+++ b/src/PVE/QemuMigrate.pm
@@ -46,6 +46,31 @@ use base qw(PVE::AbstractMigrate);
 # compared against remote end's minimum version
 our $WS_TUNNEL_VERSION = 2;
 
+sub negotiate_remote_tunnel_capabilities {
+    my ($tunnel, $wanted) = @_;
+
+    return {} if !@$wanted;
+
+    my $res = eval {
+        PVE::Tunnel::write_tunnel(
+            $tunnel, 10, 'capabilities', { capabilities => $wanted },
+        );
+    };
+    die "failed to negotiate remote migration capabilities - $@" if $@;
+    die "invalid capabilities response from remote tunnel endpoint\n"
+        if ref($res) ne 'HASH' || ref($res->{capabilities}) ne 'ARRAY';
+
+    my $wanted_caps = { map { $_ => 1 } @$wanted };
+    my $accepted = {};
+    for my $cap ($res->{capabilities}->@*) {
+        die "remote tunnel endpoint returned unrequested capability '$cap'\n"
+            if !$wanted_caps->{$cap};
+        $accepted->{$cap} = 1;
+    }
+
+    return $accepted;
+}
+
 sub fork_tunnel {
     my ($self, $ssh_forward_info) = @_;
 
@@ -352,6 +377,33 @@ sub prepare {
         die "Remote tunnel endpoint too old, upgrade required\n"
             if $WS_TUNNEL_VERSION > $tunnel->{version};
 
+        my @wanted_capabilities;
+        push @wanted_capabilities, 'insecure-remote'
+            if $self->{opts}->{migration_type} eq 'insecure';
+
+        my $capabilities = eval {
+            negotiate_remote_tunnel_capabilities($tunnel, \@wanted_capabilities);
+        };
+        my $capability_err = $@;
+
+        if (
+            $capability_err
+            || ($self->{opts}->{migration_type} eq 'insecure'
+                && !$capabilities->{'insecure-remote'})
+        ) {
+            eval { PVE::Tunnel::finish_tunnel($tunnel, 1); };
+            my $cleanup_err = $@;
+
+            my $err = "Remote tunnel endpoint did not accept insecure remote migration; "
+                . "upgrade target, grant Sys.Modify on '/' to the target credentials, or omit "
+                . "migration_type=insecure";
+            $err .= " - $capability_err" if $capability_err;
+            $err .= " (tunnel cleanup failed: $cleanup_err)" if $cleanup_err;
+            die "$err\n";
+        }
+
+        $tunnel->{capabilities} = $capabilities;
+
         print "websocket tunnel started\n";
         $self->{tunnel} = $tunnel;
     } else {
@@ -1150,8 +1202,9 @@ sub phase2_start_local_cluster {
 sub phase2_start_remote_cluster {
     my ($self, $vmid, $params) = @_;
 
-    die "insecure migration to remote cluster not implemented\n"
-        if $params->{migrate_opts}->{type} ne 'websocket';
+    die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n"
+        if $params->{migrate_opts}->{type} ne 'websocket'
+        && $params->{migrate_opts}->{type} ne 'insecure';
 
     my $remote_vmid = $self->{opts}->{remote}->{vmid};
 
@@ -1165,8 +1218,14 @@ sub phase2_start_remote_cluster {
         $self->{stopnbd} = 1;
         $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr};
         my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri};
-        die "unexpected NBD uri for '$drive': $nbd_uri\n"
-            if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
+        if ($params->{migrate_opts}->{type} eq 'websocket') {
+            die "unexpected NBD uri for '$drive': $nbd_uri\n"
+                if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!;
+        } elsif ($params->{migrate_opts}->{type} eq 'insecure') {
+            die "unexpected NBD uri for '$drive': $nbd_uri\n"
+                if $nbd_uri
+                !~ m!^nbd:(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):\d+:exportname=drive-[A-Za-z0-9_.-]+$!;
+        }
 
         $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri;
     }
@@ -1260,28 +1319,50 @@ sub phase2 {
         my $remote_vmid = $remote->{vmid};
         $params->{migrate_opts}->{remote_node} = $self->{node};
         ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params);
-        die "only UNIX sockets are supported for remote migration\n"
-            if $tunnel_info->{proto} ne 'unix';
-
-        # untaint
-        my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$|
-            or die "unexpected socket address '$tunnel_info->{addr}'\n";
-        my $local_socket = $remote_socket;
-        $local_socket =~ s/$remote_vmid/$vmid/g;
-        $tunnel_info->{addr} = $local_socket;
-
-        $self->log('info', "Setting up tunnel for '$local_socket'");
-        PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+        if ($params->{migrate_opts}->{type} eq 'websocket') {
+            die "only UNIX sockets are supported for remote migration\n"
+                if $tunnel_info->{proto} ne 'unix';
 
-        foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} }) {
             # untaint
-            ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$|
-                or die "unexpected socket address '$remote_socket'\n";
+            my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$|
+                or die "unexpected socket address '$tunnel_info->{addr}'\n";
             my $local_socket = $remote_socket;
             $local_socket =~ s/$remote_vmid/$vmid/g;
-            next if $self->{tunnel}->{forwarded}->{$local_socket};
+            $tunnel_info->{addr} = $local_socket;
+
             $self->log('info', "Setting up tunnel for '$local_socket'");
             PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+
+            foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} // [] }) {
+                # untaint
+                ($remote_socket) =
+                    $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$|
+                    or die "unexpected socket address '$remote_socket'\n";
+                my $local_socket = $remote_socket;
+                $local_socket =~ s/$remote_vmid/$vmid/g;
+                next if $self->{tunnel}->{forwarded}->{$local_socket};
+                $self->log('info', "Setting up tunnel for '$local_socket'");
+                PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket);
+            }
+        } elsif ($params->{migrate_opts}->{type} eq 'insecure') {
+            die "only TCP sockets are supported for insecure remote migration\n"
+                if $tunnel_info->{proto} ne 'tcp';
+
+            die "unexpected TCP migration address '$tunnel_info->{addr}'\n"
+                if $tunnel_info->{addr} !~ m/^(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\])$/;
+
+            die "unexpected TCP migration port '$tunnel_info->{port}'\n"
+                if !defined($tunnel_info->{port})
+                || $tunnel_info->{port} !~ /^\d+$/
+                || $tunnel_info->{port} <= 0
+                || $tunnel_info->{port} > 65535;
+
+            $self->log(
+                'info',
+                "using direct TCP migration to $tunnel_info->{addr}:$tunnel_info->{port}",
+            );
+        } else {
+            die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n";
         }
     } else {
         ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params);
base-commit: e6352be67f70042a7433a3a3c712b36d02f9f7cb
-- 
2.47.3