From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id E4A5C1FF14C for ; Fri, 15 May 2026 23:55:03 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 45AA4173AE; Fri, 15 May 2026 23:55:01 +0200 (CEST) From: Bogdan Ionescu To: f.gruenbichler@proxmox.com Subject: [PATCH v2 qemu-server] remote migration: allow insecure TCP data plane Date: Fri, 15 May 2026 23:54:49 +0200 Message-ID: <20260515215450.16564-1-bogdan@ionescu.at> X-Mailer: git-send-email 2.47.3 In-Reply-To: <1777552058.4o39hpnqt6.astroid@yuna.none> References: <1777552058.4o39hpnqt6.astroid@yuna.none> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_PASS -0.1 DMARC pass policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [qemumigrate.pm] Message-ID-Hash: 7JEYJBGELLR3A5HBUAMBEFOEKUYJQ3WO X-Message-ID-Hash: 7JEYJBGELLR3A5HBUAMBEFOEKUYJQ3WO X-MailFrom: bogdan@ionescu.at X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: pve-devel@lists.proxmox.com, Bogdan Ionescu X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Expose migration_type=insecure and migration_network for remote migration. The websocket tunnel remains used for control commands, while QEMU migration state and NBD storage migration can use a direct TCP data plane. This avoids websocket masking/TLS overhead on trusted private migration networks. Gate the feature with an insecure-remote mtunnel capability so source nodes fail early when the target does not support it. Require Sys.Modify on / when using the insecure data plane or explicitly selecting a migration network. This mode is only intended for fully trusted private networks, as guest RAM and disk migration data may be transferred in clear text. Signed-off-by: Bogdan Ionescu --- src/PVE/API2/Qemu.pm | 47 +++++++++++++++++++++++----- src/PVE/CLI/qm.pm | 17 ++++++++++ src/PVE/QemuMigrate.pm | 70 ++++++++++++++++++++++++++++++------------ 3 files changed, 107 insertions(+), 27 deletions(-) diff --git a/src/PVE/API2/Qemu.pm b/src/PVE/API2/Qemu.pm index d762401b..43888ab5 100644 --- a/src/PVE/API2/Qemu.pm +++ b/src/PVE/API2/Qemu.pm @@ -5668,6 +5668,23 @@ __PACKAGE__->register_method({ minimum => '0', default => 'migrate limit from datacenter or storage config', }, + migration_type => { + type => 'string', + enum => ['secure', 'insecure'], + description => + "Migration traffic is encrypted using a websocket tunnel by default. " + . "On secure, completely private networks this can be disabled to " + . "increase performance. WARNING: with 'insecure', VM RAM and disk " + . "migration data is transferred in clear text over the selected " + . "migration network.", + optional => 1, + }, + migration_network => { + type => 'string', + format => 'CIDR', + description => "CIDR of the trusted private network used for insecure remote migration.", + optional => 1, + }, }, }, returns => { @@ -5683,9 +5700,16 @@ __PACKAGE__->register_method({ my $source_vmid = extract_param($param, 'vmid'); my $target_endpoint = extract_param($param, 'target-endpoint'); my $target_vmid = extract_param($param, 'target-vmid') // $source_vmid; + my $migration_type = extract_param($param, 'migration_type') // 'secure'; + my $migration_network = extract_param($param, 'migration_network'); my $delete = extract_param($param, 'delete') // 0; + # insecure remote migration can transfer VM RAM and disk data in clear text + if ($migration_type eq 'insecure' || defined($migration_network)) { + $rpcenv->check_full($authuser, "/", ['Sys.Modify']); + } + PVE::Cluster::check_cfs_quorum(); # test if VM exists @@ -5760,7 +5784,8 @@ __PACKAGE__->register_method({ client => $api_client, vmid => $target_vmid, }; - $param->{migration_type} = 'websocket'; + $param->{migration_type} = $migration_type eq 'insecure' ? 'insecure' : 'websocket'; + $param->{migration_network} = $migration_network if defined($migration_network); $param->{'with-local-disks'} = 1; $param->{delete} = $delete if $delete; @@ -6716,6 +6741,7 @@ __PACKAGE__->register_method({ return { api => $PVE::QemuMigrate::WS_TUNNEL_VERSION, age => 0, + caps => ['insecure-remote'], }; }, 'config' => sub { @@ -6866,19 +6892,24 @@ __PACKAGE__->register_method({ $params->{migrate_opts}, ); - if ($info->{migrate}->{proto} ne 'unix') { + if ( + $params->{migrate_opts}->{type} ne 'insecure' + && $info->{migrate}->{proto} ne 'unix' + ) { PVE::QemuServer::vm_stop(undef, $state->{vmid}, 1, 1); die "migration over non-UNIX sockets not possible\n"; } - my $socket = $info->{migrate}->{addr}; - chown $state->{socket_uid}, -1, $socket; - $state->{sockets}->{$socket} = 1; - - my $unix_sockets = $info->{migrate}->{unix_sockets}; - foreach my $socket (@$unix_sockets) { + if ($info->{migrate}->{proto} eq 'unix') { + my $socket = $info->{migrate}->{addr}; chown $state->{socket_uid}, -1, $socket; $state->{sockets}->{$socket} = 1; + + my $unix_sockets = $info->{migrate}->{unix_sockets} // []; + foreach my $socket (@$unix_sockets) { + chown $state->{socket_uid}, -1, $socket; + $state->{sockets}->{$socket} = 1; + } } return $info; }, diff --git a/src/PVE/CLI/qm.pm b/src/PVE/CLI/qm.pm index bfa0d1d5..43ec442a 100755 --- a/src/PVE/CLI/qm.pm +++ b/src/PVE/CLI/qm.pm @@ -224,6 +224,23 @@ __PACKAGE__->register_method({ minimum => '0', default => 'migrate limit from datacenter or storage config', }, + migration_type => { + type => 'string', + enum => ['secure', 'insecure'], + description => + "Migration traffic is encrypted using a websocket tunnel by default. " + . "On secure, completely private networks this can be disabled to " + . "increase performance. WARNING: with 'insecure', VM RAM and disk " + . "migration data is transferred in clear text over the selected " + . "migration network.", + optional => 1, + }, + migration_network => { + type => 'string', + format => 'CIDR', + description => "CIDR of the trusted private network used for insecure remote migration.", + optional => 1, + }, }, }, returns => { diff --git a/src/PVE/QemuMigrate.pm b/src/PVE/QemuMigrate.pm index 8f38bf69..455024f0 100644 --- a/src/PVE/QemuMigrate.pm +++ b/src/PVE/QemuMigrate.pm @@ -46,6 +46,12 @@ use base qw(PVE::AbstractMigrate); # compared against remote end's minimum version our $WS_TUNNEL_VERSION = 2; +sub remote_tunnel_has_cap { + my ($tunnel, $cap) = @_; + + return grep { $_ eq $cap } @{ $tunnel->{caps} // [] }; +} + sub fork_tunnel { my ($self, $ssh_forward_info) = @_; @@ -351,6 +357,10 @@ sub prepare { if $WS_TUNNEL_VERSION < $min_version; die "Remote tunnel endpoint too old, upgrade required\n" if $WS_TUNNEL_VERSION > $tunnel->{version}; + die "Remote tunnel endpoint does not support insecure remote migration, upgrade target or" + . " omit migration_type=insecure\n" + if $self->{opts}->{migration_type} eq 'insecure' + && !remote_tunnel_has_cap($tunnel, 'insecure-remote'); print "websocket tunnel started\n"; $self->{tunnel} = $tunnel; @@ -1144,8 +1154,9 @@ sub phase2_start_local_cluster { sub phase2_start_remote_cluster { my ($self, $vmid, $params) = @_; - die "insecure migration to remote cluster not implemented\n" - if $params->{migrate_opts}->{type} ne 'websocket'; + die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n" + if $params->{migrate_opts}->{type} ne 'websocket' + && $params->{migrate_opts}->{type} ne 'insecure'; my $remote_vmid = $self->{opts}->{remote}->{vmid}; @@ -1159,8 +1170,13 @@ sub phase2_start_remote_cluster { $self->{stopnbd} = 1; $self->{target_drive}->{$drive}->{drivestr} = $res->{drives}->{$drive}->{drivestr}; my $nbd_uri = $res->{drives}->{$drive}->{nbd_uri}; - die "unexpected NBD uri for '$drive': $nbd_uri\n" - if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!; + if ($params->{migrate_opts}->{type} eq 'websocket') { + die "unexpected NBD uri for '$drive': $nbd_uri\n" + if $nbd_uri !~ s!/run/qemu-server/$remote_vmid\_!/run/qemu-server/$vmid\_!; + } elsif ($params->{migrate_opts}->{type} eq 'insecure') { + die "unexpected NBD uri for '$drive': $nbd_uri\n" + if $nbd_uri !~ m!^nbd:(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):\d+:exportname=drive-[A-Za-z0-9_.-]+$!; + } $self->{target_drive}->{$drive}->{nbd_uri} = $nbd_uri; } @@ -1254,28 +1270,44 @@ sub phase2 { my $remote_vmid = $remote->{vmid}; $params->{migrate_opts}->{remote_node} = $self->{node}; ($tunnel_info, $spice_port) = $self->phase2_start_remote_cluster($vmid, $params); - die "only UNIX sockets are supported for remote migration\n" - if $tunnel_info->{proto} ne 'unix'; - - # untaint - my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$| - or die "unexpected socket address '$tunnel_info->{addr}'\n"; - my $local_socket = $remote_socket; - $local_socket =~ s/$remote_vmid/$vmid/g; - $tunnel_info->{addr} = $local_socket; - $self->log('info', "Setting up tunnel for '$local_socket'"); - PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); + if ($params->{migrate_opts}->{type} eq 'websocket') { + die "only UNIX sockets are supported for remote migration\n" + if $tunnel_info->{proto} ne 'unix'; - foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} }) { # untaint - ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$| - or die "unexpected socket address '$remote_socket'\n"; + my ($remote_socket) = $tunnel_info->{addr} =~ m|^(/run/qemu-server/\d+\.migrate)$| + or die "unexpected socket address '$tunnel_info->{addr}'\n"; my $local_socket = $remote_socket; $local_socket =~ s/$remote_vmid/$vmid/g; - next if $self->{tunnel}->{forwarded}->{$local_socket}; + $tunnel_info->{addr} = $local_socket; + $self->log('info', "Setting up tunnel for '$local_socket'"); PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); + + foreach my $remote_socket (@{ $tunnel_info->{unix_sockets} // [] }) { + # untaint + ($remote_socket) = $remote_socket =~ m|^(/run/qemu-server/(?:(?!\.\./).)+\.migrate)$| + or die "unexpected socket address '$remote_socket'\n"; + my $local_socket = $remote_socket; + $local_socket =~ s/$remote_vmid/$vmid/g; + next if $self->{tunnel}->{forwarded}->{$local_socket}; + $self->log('info', "Setting up tunnel for '$local_socket'"); + PVE::Tunnel::forward_unix_socket($self->{tunnel}, $local_socket, $remote_socket); + } + } elsif ($params->{migrate_opts}->{type} eq 'insecure') { + die "only TCP sockets are supported for insecure remote migration\n" + if $tunnel_info->{proto} ne 'tcp'; + + die "unexpected TCP migration address '$tunnel_info->{addr}'\n" + if $tunnel_info->{addr} !~ m/^(?:localhost|[\d\.]+|\[[\d\.:a-fA-F]+\])$/; + + die "unexpected TCP migration port '$tunnel_info->{port}'\n" + if $tunnel_info->{port} !~ /^\d+$/ || $tunnel_info->{port} <= 0 || $tunnel_info->{port} > 65535; + + $self->log('info', "using direct TCP migration to $tunnel_info->{addr}:$tunnel_info->{port}"); + } else { + die "unsupported remote migration type '$params->{migrate_opts}->{type}'\n"; } } else { ($tunnel_info, $spice_port) = $self->phase2_start_local_cluster($vmid, $params); -- 2.47.3