From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id CC18A9221B for ; Mon, 3 Oct 2022 15:22:51 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9015131E40 for ; Mon, 3 Oct 2022 15:22:51 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Mon, 3 Oct 2022 15:22:47 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 2A56244713 for ; Mon, 3 Oct 2022 15:22:47 +0200 (CEST) Date: Mon, 03 Oct 2022 15:22:37 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox VE development discussion References: <20220928125059.1139296-1-f.gruenbichler@proxmox.com> <20220928125059.1139296-4-f.gruenbichler@proxmox.com> In-Reply-To: <20220928125059.1139296-4-f.gruenbichler@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1664803078.ydvho92erw.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.143 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: Re: [pve-devel] [PATCH FOLLOW-UP v6 container 1/3] migration: add remote migration X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 Oct 2022 13:22:51 -0000 same as in qemu-server, the following should be squashed into this=20 patch/commit: ----8<---- diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm index 4e21be4..3573b59 100644 --- a/src/PVE/API2/LXC.pm +++ b/src/PVE/API2/LXC.pm @@ -2870,7 +2870,7 @@ __PACKAGE__->register_method({ print "received command '$cmd'\n"; eval { if ($cmd_desc->{$cmd}) { - PVE::JSONSchema::validate($cmd_desc->{$cmd}, $parsed); + PVE::JSONSchema::validate($parsed, $cmd_desc->{$cmd}); } else { $parsed =3D {}; } ---->8---- On September 28, 2022 2:50 pm, Fabian Gr=C3=BCnbichler wrote: > modelled after the VM migration, but folded into a single commit since > the actual migration changes are a lot smaller here. >=20 > Signed-off-by: Fabian Gr=C3=BCnbichler > --- >=20 > Notes: > v6: > - check for Sys.Incoming in mtunnel API endpoint > - mark as experimental > - test_mp fix for non-snapshot calls > =20 > new in v5 - PoC to ensure helpers and abstractions are re-usable > =20 > requires bumped pve-storage to avoid tainted issue >=20 > src/PVE/API2/LXC.pm | 635 +++++++++++++++++++++++++++++++++++++++++ > src/PVE/LXC/Migrate.pm | 245 +++++++++++++--- > 2 files changed, 838 insertions(+), 42 deletions(-) >=20 > diff --git a/src/PVE/API2/LXC.pm b/src/PVE/API2/LXC.pm > index 589f96f..4e21be4 100644 > --- a/src/PVE/API2/LXC.pm > +++ b/src/PVE/API2/LXC.pm > @@ -3,6 +3,8 @@ package PVE::API2::LXC; > use strict; > use warnings; > =20 > +use Socket qw(SOCK_STREAM); > + > use PVE::SafeSyslog; > use PVE::Tools qw(extract_param run_command); > use PVE::Exception qw(raise raise_param_exc raise_perm_exc); > @@ -1089,6 +1091,174 @@ __PACKAGE__->register_method ({ > }}); > =20 > =20 > +__PACKAGE__->register_method({ > + name =3D> 'remote_migrate_vm', > + path =3D> '{vmid}/remote_migrate', > + method =3D> 'POST', > + protected =3D> 1, > + proxyto =3D> 'node', > + description =3D> "Migrate the container to another cluster. Creates = a new migration task. EXPERIMENTAL feature!", > + permissions =3D> { > + check =3D> ['perm', '/vms/{vmid}', [ 'VM.Migrate' ]], > + }, > + parameters =3D> { > + additionalProperties =3D> 0, > + properties =3D> { > + node =3D> get_standard_option('pve-node'), > + vmid =3D> get_standard_option('pve-vmid', { completion =3D> \&PVE::= LXC::complete_ctid }), > + 'target-vmid' =3D> get_standard_option('pve-vmid', { optional =3D> = 1 }), > + 'target-endpoint' =3D> get_standard_option('proxmox-remote', { > + description =3D> "Remote target endpoint", > + }), > + online =3D> { > + type =3D> 'boolean', > + description =3D> "Use online/live migration.", > + optional =3D> 1, > + }, > + restart =3D> { > + type =3D> 'boolean', > + description =3D> "Use restart migration", > + optional =3D> 1, > + }, > + timeout =3D> { > + type =3D> 'integer', > + description =3D> "Timeout in seconds for shutdown for restart migratio= n", > + optional =3D> 1, > + default =3D> 180, > + }, > + delete =3D> { > + type =3D> 'boolean', > + description =3D> "Delete the original CT and related data after succes= sful migration. By default the original CT is kept on the source cluster in= a stopped state.", > + optional =3D> 1, > + default =3D> 0, > + }, > + 'target-storage' =3D> get_standard_option('pve-targetstorage', { > + optional =3D> 0, > + }), > + 'target-bridge' =3D> { > + type =3D> 'string', > + description =3D> "Mapping from source to target bridges. Providing onl= y a single bridge ID maps all source bridges to that bridge. Providing the = special value '1' will map each source bridge to itself.", > + format =3D> 'bridge-pair-list', > + }, > + bwlimit =3D> { > + description =3D> "Override I/O bandwidth limit (in KiB/s).", > + optional =3D> 1, > + type =3D> 'number', > + minimum =3D> '0', > + default =3D> 'migrate limit from datacenter or storage config', > + }, > + }, > + }, > + returns =3D> { > + type =3D> 'string', > + description =3D> "the task ID.", > + }, > + code =3D> sub { > + my ($param) =3D @_; > + > + my $rpcenv =3D PVE::RPCEnvironment::get(); > + my $authuser =3D $rpcenv->get_user(); > + > + my $source_vmid =3D extract_param($param, 'vmid'); > + my $target_endpoint =3D extract_param($param, 'target-endpoint'); > + my $target_vmid =3D extract_param($param, 'target-vmid') // $source_vmi= d; > + > + my $delete =3D extract_param($param, 'delete') // 0; > + > + PVE::Cluster::check_cfs_quorum(); > + > + # test if CT exists > + my $conf =3D PVE::LXC::Config->load_config($source_vmid); > + PVE::LXC::Config->check_lock($conf); > + > + # try to detect errors early > + if (PVE::LXC::check_running($source_vmid)) { > + die "can't migrate running container without --online or --restart\= n" > + if !$param->{online} && !$param->{restart}; > + } > + > + raise_param_exc({ vmid =3D> "cannot migrate HA-managed CT to remote clu= ster" }) > + if PVE::HA::Config::vm_is_ha_managed($source_vmid); > + > + my $remote =3D PVE::JSONSchema::parse_property_string('proxmox-remote',= $target_endpoint); > + > + # TODO: move this as helper somewhere appropriate? > + my $conn_args =3D { > + protocol =3D> 'https', > + host =3D> $remote->{host}, > + port =3D> $remote->{port} // 8006, > + apitoken =3D> $remote->{apitoken}, > + }; > + > + my $fp; > + if ($fp =3D $remote->{fingerprint}) { > + $conn_args->{cached_fingerprints} =3D { uc($fp) =3D> 1 }; > + } > + > + print "Establishing API connection with remote at '$remote->{host}'\n"; > + > + my $api_client =3D PVE::APIClient::LWP->new(%$conn_args); > + > + if (!defined($fp)) { > + my $cert_info =3D $api_client->get("/nodes/localhost/certificates/i= nfo"); > + foreach my $cert (@$cert_info) { > + my $filename =3D $cert->{filename}; > + next if $filename ne 'pveproxy-ssl.pem' && $filename ne 'pve-ssl.pem'; > + $fp =3D $cert->{fingerprint} if !$fp || $filename eq 'pveproxy-ssl.pem= '; > + } > + $conn_args->{cached_fingerprints} =3D { uc($fp) =3D> 1 } > + if defined($fp); > + } > + > + my $storecfg =3D PVE::Storage::config(); > + my $target_storage =3D extract_param($param, 'target-storage'); > + my $storagemap =3D eval { PVE::JSONSchema::parse_idmap($target_storage,= 'pve-storage-id') }; > + raise_param_exc({ 'target-storage' =3D> "failed to parse storage map: $= @" }) > + if $@; > + > + my $target_bridge =3D extract_param($param, 'target-bridge'); > + my $bridgemap =3D eval { PVE::JSONSchema::parse_idmap($target_bridge, '= pve-bridge-id') }; > + raise_param_exc({ 'target-bridge' =3D> "failed to parse bridge map: $@"= }) > + if $@; > + > + die "remote migration requires explicit storage mapping!\n" > + if $storagemap->{identity}; > + > + $param->{storagemap} =3D $storagemap; > + $param->{bridgemap} =3D $bridgemap; > + $param->{remote} =3D { > + conn =3D> $conn_args, # re-use fingerprint for tunnel > + client =3D> $api_client, > + vmid =3D> $target_vmid, > + }; > + $param->{migration_type} =3D 'websocket'; > + $param->{delete} =3D $delete if $delete; > + > + my $cluster_status =3D $api_client->get("/cluster/status"); > + my $target_node; > + foreach my $entry (@$cluster_status) { > + next if $entry->{type} ne 'node'; > + if ($entry->{local}) { > + $target_node =3D $entry->{name}; > + last; > + } > + } > + > + die "couldn't determine endpoint's node name\n" > + if !defined($target_node); > + > + my $realcmd =3D sub { > + PVE::LXC::Migrate->migrate($target_node, $remote->{host}, $source_v= mid, $param); > + }; > + > + my $worker =3D sub { > + return PVE::GuestHelpers::guest_migration_lock($source_vmid, 10, $r= ealcmd); > + }; > + > + return $rpcenv->fork_worker('vzmigrate', $source_vmid, $authuser, $work= er); > + }}); > + > + > __PACKAGE__->register_method({ > name =3D> 'migrate_vm', > path =3D> '{vmid}/migrate', > @@ -2318,4 +2488,469 @@ __PACKAGE__->register_method({ > return PVE::GuestHelpers::config_with_pending_array($conf, $pending_del= ete_hash); > }}); > =20 > +__PACKAGE__->register_method({ > + name =3D> 'mtunnel', > + path =3D> '{vmid}/mtunnel', > + method =3D> 'POST', > + protected =3D> 1, > + description =3D> 'Migration tunnel endpoint - only for internal use = by CT migration.', > + permissions =3D> { > + check =3D> > + [ 'and', > + ['perm', '/vms/{vmid}', [ 'VM.Allocate' ]], > + ['perm', '/', [ 'Sys.Incoming' ]], > + ], > + description =3D> "You need 'VM.Allocate' permissions on '/vms/{vmid}' a= nd Sys.Incoming" . > + " on '/'. Further permission checks happen during the ac= tual migration.", > + }, > + parameters =3D> { > + additionalProperties =3D> 0, > + properties =3D> { > + node =3D> get_standard_option('pve-node'), > + vmid =3D> get_standard_option('pve-vmid'), > + storages =3D> { > + type =3D> 'string', > + format =3D> 'pve-storage-id-list', > + optional =3D> 1, > + description =3D> 'List of storages to check permission and availabilit= y. Will be checked again for all actually used storages during migration.', > + }, > + bridges =3D> { > + type =3D> 'string', > + format =3D> 'pve-bridge-id-list', > + optional =3D> 1, > + description =3D> 'List of network bridges to check availability. Will = be checked again for actually used bridges during migration.', > + }, > + }, > + }, > + returns =3D> { > + additionalProperties =3D> 0, > + properties =3D> { > + upid =3D> { type =3D> 'string' }, > + ticket =3D> { type =3D> 'string' }, > + socket =3D> { type =3D> 'string' }, > + }, > + }, > + code =3D> sub { > + my ($param) =3D @_; > + > + my $rpcenv =3D PVE::RPCEnvironment::get(); > + my $authuser =3D $rpcenv->get_user(); > + > + my $node =3D extract_param($param, 'node'); > + my $vmid =3D extract_param($param, 'vmid'); > + > + my $storages =3D extract_param($param, 'storages'); > + my $bridges =3D extract_param($param, 'bridges'); > + > + my $nodename =3D PVE::INotify::nodename(); > + > + raise_param_exc({ node =3D> "node needs to be 'localhost' or local host= name '$nodename'" }) > + if $node ne 'localhost' && $node ne $nodename; > + > + $node =3D $nodename; > + > + my $storecfg =3D PVE::Storage::config(); > + foreach my $storeid (PVE::Tools::split_list($storages)) { > + $check_storage_access_migrate->($rpcenv, $authuser, $storecfg, $sto= reid, $node); > + } > + > + foreach my $bridge (PVE::Tools::split_list($bridges)) { > + PVE::Network::read_bridge_mtu($bridge); > + } > + > + PVE::Cluster::check_cfs_quorum(); > + > + my $socket_addr =3D "/run/pve/ct-$vmid.mtunnel"; > + > + my $lock =3D 'create'; > + eval { PVE::LXC::Config->create_and_lock_config($vmid, 0, $lock); }; > + > + raise_param_exc({ vmid =3D> "unable to create empty CT config - $@"}) > + if $@; > + > + my $realcmd =3D sub { > + my $state =3D { > + storecfg =3D> PVE::Storage::config(), > + lock =3D> $lock, > + vmid =3D> $vmid, > + }; > + > + my $run_locked =3D sub { > + my ($code, $params) =3D @_; > + return PVE::LXC::Config->lock_config($state->{vmid}, sub { > + my $conf =3D PVE::LXC::Config->load_config($state->{vmid}); > + > + $state->{conf} =3D $conf; > + > + die "Encountered wrong lock - aborting mtunnel command handling.\n= " > + if $state->{lock} && !PVE::LXC::Config->has_lock($conf, $state->{lock= }); > + > + return $code->($params); > + }); > + }; > + > + my $cmd_desc =3D { > + config =3D> { > + conf =3D> { > + type =3D> 'string', > + description =3D> 'Full CT config, adapted for target cluster/node', > + }, > + 'firewall-config' =3D> { > + type =3D> 'string', > + description =3D> 'CT firewall config', > + optional =3D> 1, > + }, > + }, > + ticket =3D> { > + path =3D> { > + type =3D> 'string', > + description =3D> 'socket path for which the ticket should be valid. m= ust be known to current mtunnel instance.', > + }, > + }, > + quit =3D> { > + cleanup =3D> { > + type =3D> 'boolean', > + description =3D> 'remove CT config and volumes, aborting migration', > + default =3D> 0, > + }, > + }, > + 'disk-import' =3D> $PVE::StorageTunnel::cmd_schema->{'disk-import'}, > + 'query-disk-import' =3D> $PVE::StorageTunnel::cmd_schema->{'query-disk= -import'}, > + bwlimit =3D> $PVE::StorageTunnel::cmd_schema->{bwlimit}, > + }; > + > + my $cmd_handlers =3D { > + 'version' =3D> sub { > + # compared against other end's version > + # bump/reset for breaking changes > + # bump/bump for opt-in changes > + return { > + api =3D> $PVE::LXC::Migrate::WS_TUNNEL_VERSION, > + age =3D> 0, > + }; > + }, > + 'config' =3D> sub { > + my ($params) =3D @_; > + > + # parse and write out VM FW config if given > + if (my $fw_conf =3D $params->{'firewall-config'}) { > + my ($path, $fh) =3D PVE::Tools::tempfile_contents($fw_conf, 700); > + > + my $empty_conf =3D { > + rules =3D> [], > + options =3D> {}, > + aliases =3D> {}, > + ipset =3D> {} , > + ipset_comments =3D> {}, > + }; > + my $cluster_fw_conf =3D PVE::Firewall::load_clusterfw_conf(); > + > + # TODO: add flag for strict parsing? > + # TODO: add import sub that does all this given raw content? > + my $vmfw_conf =3D PVE::Firewall::generic_fw_config_parser($path, $clu= ster_fw_conf, $empty_conf, 'vm'); > + $vmfw_conf->{vmid} =3D $state->{vmid}; > + PVE::Firewall::save_vmfw_conf($state->{vmid}, $vmfw_conf); > + > + $state->{cleanup}->{fw} =3D 1; > + } > + > + my $conf_fn =3D "incoming/lxc/$state->{vmid}.conf"; > + my $new_conf =3D PVE::LXC::Config::parse_pct_config($conf_fn, $par= ams->{conf}, 1); > + delete $new_conf->{lock}; > + delete $new_conf->{digest}; > + > + my $unprivileged =3D delete $new_conf->{unprivileged}; > + my $arch =3D delete $new_conf->{arch}; > + > + # TODO handle properly? > + delete $new_conf->{snapshots}; > + delete $new_conf->{parent}; > + delete $new_conf->{pending}; > + delete $new_conf->{lxc}; > + > + PVE::LXC::Config->remove_lock($state->{vmid}, 'create'); > + > + eval { > + my $conf =3D { > + unprivileged =3D> $unprivileged, > + arch =3D> $arch, > + }; > + PVE::LXC::check_ct_modify_config_perm( > + $rpcenv, > + $authuser, > + $state->{vmid}, > + undef, > + $conf, > + $new_conf, > + undef, > + $unprivileged, > + ); > + my $errors =3D PVE::LXC::Config->update_pct_config( > + $state->{vmid}, > + $conf, > + 0, > + $new_conf, > + [], > + [], > + ); > + raise_param_exc($errors) if scalar(keys %$errors); > + PVE::LXC::Config->write_config($state->{vmid}, $conf); > + PVE::LXC::update_lxc_config($vmid, $conf); > + }; > + if (my $err =3D $@) { > + # revert to locked previous config > + my $conf =3D PVE::LXC::Config->load_config($state->{vmid}); > + $conf->{lock} =3D 'create'; > + PVE::LXC::Config->write_config($state->{vmid}, $conf); > + > + die $err; > + } > + > + my $conf =3D PVE::LXC::Config->load_config($state->{vmid}); > + $conf->{lock} =3D 'migrate'; > + PVE::LXC::Config->write_config($state->{vmid}, $conf); > + > + $state->{lock} =3D 'migrate'; > + > + return; > + }, > + 'bwlimit' =3D> sub { > + my ($params) =3D @_; > + return PVE::StorageTunnel::handle_bwlimit($params); > + }, > + 'disk-import' =3D> sub { > + my ($params) =3D @_; > + > + $check_storage_access_migrate->( > + $rpcenv, > + $authuser, > + $state->{storecfg}, > + $params->{storage}, > + $node > + ); > + > + $params->{unix} =3D "/run/pve/ct-$state->{vmid}.storage"; > + > + return PVE::StorageTunnel::handle_disk_import($state, $params); > + }, > + 'query-disk-import' =3D> sub { > + my ($params) =3D @_; > + > + return PVE::StorageTunnel::handle_query_disk_import($state, $param= s); > + }, > + 'unlock' =3D> sub { > + PVE::LXC::Config->remove_lock($state->{vmid}, $state->{lock}); > + delete $state->{lock}; > + return; > + }, > + 'start' =3D> sub { > + PVE::LXC::vm_start( > + $state->{vmid}, > + $state->{conf}, > + 0 > + ); > + > + return; > + }, > + 'stop' =3D> sub { > + PVE::LXC::vm_stop($state->{vmid}, 1, 10, 1); > + return; > + }, > + 'ticket' =3D> sub { > + my ($params) =3D @_; > + > + my $path =3D $params->{path}; > + > + die "Not allowed to generate ticket for unknown socket '$path'\n" > + if !defined($state->{sockets}->{$path}); > + > + return { ticket =3D> PVE::AccessControl::assemble_tunnel_ticket($a= uthuser, "/socket/$path") }; > + }, > + 'quit' =3D> sub { > + my ($params) =3D @_; > + > + if ($params->{cleanup}) { > + if ($state->{cleanup}->{fw}) { > + PVE::Firewall::remove_vmfw_conf($state->{vmid}); > + } > + > + for my $volid (keys $state->{cleanup}->{volumes}->%*) { > + print "freeing volume '$volid' as part of cleanup\n"; > + eval { PVE::Storage::vdisk_free($state->{storecfg}, $volid) }; > + warn $@ if $@; > + } > + > + PVE::LXC::destroy_lxc_container( > + $state->{storecfg}, > + $state->{vmid}, > + $state->{conf}, > + undef, > + 0, > + ); > + } > + > + print "switching to exit-mode, waiting for client to disconnect\n"= ; > + $state->{exit} =3D 1; > + return; > + }, > + }; > + > + $run_locked->(sub { > + my $socket_addr =3D "/run/pve/ct-$state->{vmid}.mtunnel"; > + unlink $socket_addr; > + > + $state->{socket} =3D IO::Socket::UNIX->new( > + Type =3D> SOCK_STREAM(), > + Local =3D> $socket_addr, > + Listen =3D> 1, > + ); > + > + $state->{socket_uid} =3D getpwnam('www-data') > + or die "Failed to resolve user 'www-data' to numeric UID\n"; > + chown $state->{socket_uid}, -1, $socket_addr; > + }); > + > + print "mtunnel started\n"; > + > + my $conn =3D eval { PVE::Tools::run_with_timeout(300, sub { $state-= >{socket}->accept() }) }; > + if ($@) { > + warn "Failed to accept tunnel connection - $@\n"; > + > + warn "Removing tunnel socket..\n"; > + unlink $state->{socket}; > + > + warn "Removing temporary VM config..\n"; > + $run_locked->(sub { > + PVE::LXC::destroy_config($state->{vmid}); > + }); > + > + die "Exiting mtunnel\n"; > + } > + > + $state->{conn} =3D $conn; > + > + my $reply_err =3D sub { > + my ($msg) =3D @_; > + > + my $reply =3D JSON::encode_json({ > + success =3D> JSON::false, > + msg =3D> $msg, > + }); > + $conn->print("$reply\n"); > + $conn->flush(); > + }; > + > + my $reply_ok =3D sub { > + my ($res) =3D @_; > + > + $res->{success} =3D JSON::true; > + my $reply =3D JSON::encode_json($res); > + $conn->print("$reply\n"); > + $conn->flush(); > + }; > + > + while (my $line =3D <$conn>) { > + chomp $line; > + > + # untaint, we validate below if needed > + ($line) =3D $line =3D~ /^(.*)$/; > + my $parsed =3D eval { JSON::decode_json($line) }; > + if ($@) { > + $reply_err->("failed to parse command - $@"); > + next; > + } > + > + my $cmd =3D delete $parsed->{cmd}; > + if (!defined($cmd)) { > + $reply_err->("'cmd' missing"); > + } elsif ($state->{exit}) { > + $reply_err->("tunnel is in exit-mode, processing '$cmd' cmd not po= ssible"); > + next; > + } elsif (my $handler =3D $cmd_handlers->{$cmd}) { > + print "received command '$cmd'\n"; > + eval { > + if ($cmd_desc->{$cmd}) { > + PVE::JSONSchema::validate($cmd_desc->{$cmd}, $parsed); > + } else { > + $parsed =3D {}; > + } > + my $res =3D $run_locked->($handler, $parsed); > + $reply_ok->($res); > + }; > + $reply_err->("failed to handle '$cmd' command - $@") > + if $@; > + } else { > + $reply_err->("unknown command '$cmd' given"); > + } > + } > + > + if ($state->{exit}) { > + print "mtunnel exited\n"; > + } else { > + die "mtunnel exited unexpectedly\n"; > + } > + }; > + > + my $ticket =3D PVE::AccessControl::assemble_tunnel_ticket($authuser, "/= socket/$socket_addr"); > + my $upid =3D $rpcenv->fork_worker('vzmtunnel', $vmid, $authuser, $realc= md); > + > + return { > + ticket =3D> $ticket, > + upid =3D> $upid, > + socket =3D> $socket_addr, > + }; > + }}); > + > +__PACKAGE__->register_method({ > + name =3D> 'mtunnelwebsocket', > + path =3D> '{vmid}/mtunnelwebsocket', > + method =3D> 'GET', > + permissions =3D> { > + description =3D> "You need to pass a ticket valid for the selected sock= et. Tickets can be created via the mtunnel API call, which will check permi= ssions accordingly.", > + user =3D> 'all', # check inside > + }, > + description =3D> 'Migration tunnel endpoint for websocket upgrade - = only for internal use by VM migration.', > + parameters =3D> { > + additionalProperties =3D> 0, > + properties =3D> { > + node =3D> get_standard_option('pve-node'), > + vmid =3D> get_standard_option('pve-vmid'), > + socket =3D> { > + type =3D> "string", > + description =3D> "unix socket to forward to", > + }, > + ticket =3D> { > + type =3D> "string", > + description =3D> "ticket return by initial 'mtunnel' API call, or retr= ieved via 'ticket' tunnel command", > + }, > + }, > + }, > + returns =3D> { > + type =3D> "object", > + properties =3D> { > + port =3D> { type =3D> 'string', optional =3D> 1 }, > + socket =3D> { type =3D> 'string', optional =3D> 1 }, > + }, > + }, > + code =3D> sub { > + my ($param) =3D @_; > + > + my $rpcenv =3D PVE::RPCEnvironment::get(); > + my $authuser =3D $rpcenv->get_user(); > + > + my $nodename =3D PVE::INotify::nodename(); > + my $node =3D extract_param($param, 'node'); > + > + raise_param_exc({ node =3D> "node needs to be 'localhost' or local host= name '$nodename'" }) > + if $node ne 'localhost' && $node ne $nodename; > + > + my $vmid =3D $param->{vmid}; > + # check VM exists > + PVE::LXC::Config->load_config($vmid); > + > + my $socket =3D $param->{socket}; > + PVE::AccessControl::verify_tunnel_ticket($param->{ticket}, $authuser, "= /socket/$socket"); > + > + return { socket =3D> $socket }; > + }}); > 1; > diff --git a/src/PVE/LXC/Migrate.pm b/src/PVE/LXC/Migrate.pm > index 2ef1cce..a0ab65e 100644 > --- a/src/PVE/LXC/Migrate.pm > +++ b/src/PVE/LXC/Migrate.pm > @@ -17,6 +17,9 @@ use PVE::Replication; > =20 > use base qw(PVE::AbstractMigrate); > =20 > +# compared against remote end's minimum version > +our $WS_TUNNEL_VERSION =3D 2; > + > sub lock_vm { > my ($self, $vmid, $code, @param) =3D @_; > =20 > @@ -28,6 +31,7 @@ sub prepare { > =20 > my $online =3D $self->{opts}->{online}; > my $restart=3D $self->{opts}->{restart}; > + my $remote =3D $self->{opts}->{remote}; > =20 > $self->{storecfg} =3D PVE::Storage::config(); > =20 > @@ -44,6 +48,7 @@ sub prepare { > } > $self->{was_running} =3D $running; > =20 > + my $storages =3D {}; > PVE::LXC::Config->foreach_volume_full($conf, { include_unused =3D> 1= }, sub { > my ($ms, $mountpoint) =3D @_; > =20 > @@ -70,7 +75,7 @@ sub prepare { > die "content type 'rootdir' is not available on storage '$storage'\n" > if !$scfg->{content}->{rootdir}; > =20 > - if ($scfg->{shared}) { > + if ($scfg->{shared} && !$remote) { > # PVE::Storage::activate_storage checks this for non-shared storage= s > my $plugin =3D PVE::Storage::Plugin->lookup($scfg->{type}); > warn "Used shared storage '$storage' is not online on source node!\= n" > @@ -83,18 +88,63 @@ sub prepare { > $targetsid =3D PVE::JSONSchema::map_id($self->{opts}->{storagemap},= $storage); > } > =20 > - my $target_scfg =3D PVE::Storage::storage_check_enabled($self->{storecf= g}, $targetsid, $self->{node}); > + if (!$remote) { > + my $target_scfg =3D PVE::Storage::storage_check_enabled($self->{sto= recfg}, $targetsid, $self->{node}); > + > + die "$volid: content type 'rootdir' is not available on storage '$t= argetsid'\n" > + if !$target_scfg->{content}->{rootdir}; > + } > =20 > - die "$volid: content type 'rootdir' is not available on storage '$targe= tsid'\n" > - if !$target_scfg->{content}->{rootdir}; > + $storages->{$targetsid} =3D 1; > }); > =20 > # todo: test if VM uses local resources > =20 > - # test ssh connection > - my $cmd =3D [ @{$self->{rem_ssh}}, '/bin/true' ]; > - eval { $self->cmd_quiet($cmd); }; > - die "Can't connect to destination address using public key\n" if $@; > + if ($remote) { > + # test & establish websocket connection > + my $bridges =3D map_bridges($conf, $self->{opts}->{bridgemap}, 1); > + > + my $remote =3D $self->{opts}->{remote}; > + my $conn =3D $remote->{conn}; > + > + my $log =3D sub { > + my ($level, $msg) =3D @_; > + $self->log($level, $msg); > + }; > + > + my $websocket_url =3D "https://$conn->{host}:$conn->{port}/api2/json/no= des/$self->{node}/lxc/$remote->{vmid}/mtunnelwebsocket"; > + my $url =3D "/nodes/$self->{node}/lxc/$remote->{vmid}/mtunnel"; > + > + my $tunnel_params =3D { > + url =3D> $websocket_url, > + }; > + > + my $storage_list =3D join(',', keys %$storages); > + my $bridge_list =3D join(',', keys %$bridges); > + > + my $req_params =3D { > + storages =3D> $storage_list, > + bridges =3D> $bridge_list, > + }; > + > + my $tunnel =3D PVE::Tunnel::fork_websocket_tunnel($conn, $url, $req_par= ams, $tunnel_params, $log); > + my $min_version =3D $tunnel->{version} - $tunnel->{age}; > + $self->log('info', "local WS tunnel version: $WS_TUNNEL_VERSION"); > + $self->log('info', "remote WS tunnel version: $tunnel->{version}"); > + $self->log('info', "minimum required WS tunnel version: $min_version"); > + die "Remote tunnel endpoint not compatible, upgrade required\n" > + if $WS_TUNNEL_VERSION < $min_version; > + die "Remote tunnel endpoint too old, upgrade required\n" > + if $WS_TUNNEL_VERSION > $tunnel->{version}; > + > + $self->log('info', "websocket tunnel started\n"); > + $self->{tunnel} =3D $tunnel; > + } else { > + # test ssh connection > + my $cmd =3D [ @{$self->{rem_ssh}}, '/bin/true' ]; > + eval { $self->cmd_quiet($cmd); }; > + die "Can't connect to destination address using public key\n" if $@; > + } > =20 > # in restart mode, we shutdown the container before migrating > if ($restart && $running) { > @@ -113,6 +163,8 @@ sub prepare { > sub phase1 { > my ($self, $vmid) =3D @_; > =20 > + my $remote =3D $self->{opts}->{remote}; > + > $self->log('info', "starting migration of CT $self->{vmid} to node '= $self->{node}' ($self->{nodeip})"); > =20 > my $conf =3D $self->{vmconf}; > @@ -147,7 +199,7 @@ sub phase1 { > =20 > my $targetsid =3D $sid; > =20 > - if ($scfg->{shared}) { > + if ($scfg->{shared} && !$remote) { > $self->log('info', "volume '$volid' is on shared storage '$sid'") > if !$snapname; > return; > @@ -155,7 +207,8 @@ sub phase1 { > $targetsid =3D PVE::JSONSchema::map_id($self->{opts}->{storagemap},= $sid); > } > =20 > - PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $sel= f->{node}); > + PVE::Storage::storage_check_enabled($self->{storecfg}, $targetsid, $sel= f->{node}) > + if !$remote; > =20 > my $bwlimit =3D $self->get_bwlimit($sid, $targetsid); > =20 > @@ -192,6 +245,9 @@ sub phase1 { > =20 > eval { > &$test_volid($volid, $snapname); > + > + die "remote migration with snapshots not supported yet\n" > + if $remote && $snapname; > }; > =20 > &$log_error($@, $volid) if $@; > @@ -201,7 +257,7 @@ sub phase1 { > my @sids =3D PVE::Storage::storage_ids($self->{storecfg}); > foreach my $storeid (@sids) { > my $scfg =3D PVE::Storage::storage_config($self->{storecfg}, $storeid); > - next if $scfg->{shared}; > + next if $scfg->{shared} && !$remote; > next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storei= d, undef, 1); > =20 > # get list from PVE::Storage (for unreferenced volumes) > @@ -211,10 +267,12 @@ sub phase1 { > =20 > # check if storage is available on target node > my $targetsid =3D PVE::JSONSchema::map_id($self->{opts}->{storagemap}, = $storeid); > - my $target_scfg =3D PVE::Storage::storage_check_enabled($self->{storecf= g}, $targetsid, $self->{node}); > + if (!$remote) { > + my $target_scfg =3D PVE::Storage::storage_check_enabled($self->{sto= recfg}, $targetsid, $self->{node}); > =20 > - die "content type 'rootdir' is not available on storage '$targetsid'\n" > - if !$target_scfg->{content}->{rootdir}; > + die "content type 'rootdir' is not available on storage '$targetsid= '\n" > + if !$target_scfg->{content}->{rootdir}; > + } > =20 > PVE::Storage::foreach_volid($dl, sub { > my ($volid, $sid, $volname) =3D @_; > @@ -240,12 +298,21 @@ sub phase1 { > my ($sid, $volname) =3D PVE::Storage::parse_volume_id($volid); > my $scfg =3D PVE::Storage::storage_config($self->{storecfg}, $sid)= ; > =20 > - my $migratable =3D ($scfg->{type} eq 'dir') || ($scfg->{type} eq 'z= fspool') > - || ($scfg->{type} eq 'lvmthin') || ($scfg->{type} eq 'lvm') > - || ($scfg->{type} eq 'btrfs'); > + # TODO move to storage plugin layer? > + my $migratable_storages =3D [ > + 'dir', > + 'zfspool', > + 'lvmthin', > + 'lvm', > + 'btrfs', > + ]; > + if ($remote) { > + push @$migratable_storages, 'cifs'; > + push @$migratable_storages, 'nfs'; > + } > =20 > die "storage type '$scfg->{type}' not supported\n" > - if !$migratable; > + if !grep { $_ eq $scfg->{type} } @$migratable_storages; > =20 > # image is a linked clone on local storage, se we can't migrate. > if (my $basename =3D (PVE::Storage::parse_volname($self->{storecfg}= , $volid))[3]) { > @@ -280,7 +347,10 @@ sub phase1 { > =20 > my $rep_cfg =3D PVE::ReplicationConfig->new(); > =20 > - if (my $jobcfg =3D $rep_cfg->find_local_replication_job($vmid, $self= ->{node})) { > + if ($remote) { > + die "cannot remote-migrate replicated VM\n" > + if $rep_cfg->check_for_existing_jobs($vmid, 1); > + } elsif (my $jobcfg =3D $rep_cfg->find_local_replication_job($vmid, = $self->{node})) { > die "can't live migrate VM with replicated volumes\n" if $self->{runnin= g}; > my $start_time =3D time(); > my $logfunc =3D sub { my ($msg) =3D @_; $self->log('info', $msg); }; > @@ -291,7 +361,6 @@ sub phase1 { > my $opts =3D $self->{opts}; > foreach my $volid (keys %$volhash) { > next if $rep_volumes->{$volid}; > - my ($sid, $volname) =3D PVE::Storage::parse_volume_id($volid); > push @{$self->{volumes}}, $volid; > =20 > # JSONSchema and get_bandwidth_limit use kbps - storage_migrate bps > @@ -301,22 +370,39 @@ sub phase1 { > my $targetsid =3D $volhash->{$volid}->{targetsid}; > =20 > my $new_volid =3D eval { > - my $storage_migrate_opts =3D { > - 'ratelimit_bps' =3D> $bwlimit, > - 'insecure' =3D> $opts->{migration_type} eq 'insecure', > - 'with_snapshots' =3D> $volhash->{$volid}->{snapshots}, > - 'allow_rename' =3D> 1, > - }; > - > - my $logfunc =3D sub { $self->log('info', $_[0]); }; > - return PVE::Storage::storage_migrate( > - $self->{storecfg}, > - $volid, > - $self->{ssh_info}, > - $targetsid, > - $storage_migrate_opts, > - $logfunc, > - ); > + if ($remote) { > + my $log =3D sub { > + my ($level, $msg) =3D @_; > + $self->log($level, $msg); > + }; > + > + return PVE::StorageTunnel::storage_migrate( > + $self->{tunnel}, > + $self->{storecfg}, > + $volid, > + $self->{vmid}, > + $remote->{vmid}, > + $volhash->{$volid}, > + $log, > + ); > + } else { > + my $storage_migrate_opts =3D { > + 'ratelimit_bps' =3D> $bwlimit, > + 'insecure' =3D> $opts->{migration_type} eq 'insecure', > + 'with_snapshots' =3D> $volhash->{$volid}->{snapshots}, > + 'allow_rename' =3D> 1, > + }; > + > + my $logfunc =3D sub { $self->log('info', $_[0]); }; > + return PVE::Storage::storage_migrate( > + $self->{storecfg}, > + $volid, > + $self->{ssh_info}, > + $targetsid, > + $storage_migrate_opts, > + $logfunc, > + ); > + } > }; > =20 > if (my $err =3D $@) { > @@ -346,13 +432,38 @@ sub phase1 { > my $vollist =3D PVE::LXC::Config->get_vm_volumes($conf); > PVE::Storage::deactivate_volumes($self->{storecfg}, $vollist); > =20 > - # transfer replication state before moving config > - $self->transfer_replication_state() if $rep_volumes; > - PVE::LXC::Config->update_volume_ids($conf, $self->{volume_map}); > - PVE::LXC::Config->write_config($vmid, $conf); > - PVE::LXC::Config->move_config_to_node($vmid, $self->{node}); > + if ($remote) { > + my $remote_conf =3D PVE::LXC::Config->load_config($vmid); > + PVE::LXC::Config->update_volume_ids($remote_conf, $self->{volume_map}); > + > + my $bridges =3D map_bridges($remote_conf, $self->{opts}->{bridgemap}); > + for my $target (keys $bridges->%*) { > + for my $nic (keys $bridges->{$target}->%*) { > + $self->log('info', "mapped: $nic from $bridges->{$target}->{$nic} to $= target"); > + } > + } > + my $conf_str =3D PVE::LXC::Config::write_pct_config("remote", $remote_c= onf); > + > + # TODO expose in PVE::Firewall? > + my $vm_fw_conf_path =3D "/etc/pve/firewall/$vmid.fw"; > + my $fw_conf_str; > + $fw_conf_str =3D PVE::Tools::file_get_contents($vm_fw_conf_path) > + if -e $vm_fw_conf_path; > + my $params =3D { > + conf =3D> $conf_str, > + 'firewall-config' =3D> $fw_conf_str, > + }; > + > + PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'config', $params); > + } else { > + # transfer replication state before moving config > + $self->transfer_replication_state() if $rep_volumes; > + PVE::LXC::Config->update_volume_ids($conf, $self->{volume_map}); > + PVE::LXC::Config->write_config($vmid, $conf); > + PVE::LXC::Config->move_config_to_node($vmid, $self->{node}); > + $self->switch_replication_job_target() if $rep_volumes; > + } > $self->{conf_migrated} =3D 1; > - $self->switch_replication_job_target() if $rep_volumes; > } > =20 > sub phase1_cleanup { > @@ -366,6 +477,12 @@ sub phase1_cleanup { > # fixme: try to remove ? > } > } > + > + if ($self->{opts}->{remote}) { > + # cleans up remote volumes > + PVE::Tunnel::finish_tunnel($self->{tunnel}, 1); > + delete $self->{tunnel}; > + } > } > =20 > sub phase3 { > @@ -373,6 +490,9 @@ sub phase3 { > =20 > my $volids =3D $self->{volumes}; > =20 > + # handled below in final_cleanup > + return if $self->{opts}->{remote}; > + > # destroy local copies > foreach my $volid (@$volids) { > eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; > @@ -401,6 +521,24 @@ sub final_cleanup { > my $skiplock =3D 1; > PVE::LXC::vm_start($vmid, $self->{vmconf}, $skiplock); > } > + } elsif ($self->{opts}->{remote}) { > + eval { PVE::Tunnel::write_tunnel($self->{tunnel}, 10, 'unlock') }; > + $self->log('err', "Failed to clear migrate lock - $@\n") if $@; > + > + if ($self->{opts}->{restart} && $self->{was_running}) { > + $self->log('info', "start container on target node"); > + PVE::Tunnel::write_tunnel($self->{tunnel}, 60, 'start'); > + } > + if ($self->{opts}->{delete}) { > + PVE::LXC::destroy_lxc_container( > + PVE::Storage::config(), > + $vmid, > + PVE::LXC::Config->load_config($vmid), > + undef, > + 0, > + ); > + } > + PVE::Tunnel::finish_tunnel($self->{tunnel}); > } else { > my $cmd =3D [ @{$self->{rem_ssh}}, 'pct', 'unlock', $vmid ]; > $self->cmd_logerr($cmd, errmsg =3D> "failed to clear migrate lock"); > @@ -413,7 +551,30 @@ sub final_cleanup { > $self->cmd($cmd); > } > } > +} > + > +sub map_bridges { > + my ($conf, $map, $scan_only) =3D @_; > + > + my $bridges =3D {}; > + > + foreach my $opt (keys %$conf) { > + next if $opt !~ m/^net\d+$/; > + > + next if !$conf->{$opt}; > + my $d =3D PVE::LXC::Config->parse_lxc_network($conf->{$opt}); > + next if !$d || !$d->{bridge}; > + > + my $target_bridge =3D PVE::JSONSchema::map_id($map, $d->{bridge}); > + $bridges->{$target_bridge}->{$opt} =3D $d->{bridge}; > + > + next if $scan_only; > + > + $d->{bridge} =3D $target_bridge; > + $conf->{$opt} =3D PVE::LXC::Config->print_lxc_network($d); > + } > =20 > + return $bridges; > } > =20 > 1; > --=20 > 2.30.2 >=20 >=20 >=20 > _______________________________________________ > pve-devel mailing list > pve-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel >=20