From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id A51546565C for ; Mon, 7 Mar 2022 13:18:01 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A1E7E27529 for ; Mon, 7 Mar 2022 13:18:01 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id D0D2527258 for ; Mon, 7 Mar 2022 13:17:51 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 9C16141E01 for ; Mon, 7 Mar 2022 13:17:51 +0100 (CET) From: Fabian Ebner To: pve-devel@lists.proxmox.com Date: Mon, 7 Mar 2022 13:17:41 +0100 Message-Id: <20220307121743.60206-14-f.ebner@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220307121743.60206-1-f.ebner@proxmox.com> References: <20220307121743.60206-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.125 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH v11 qemu-server 13/14] api: support VM disk import X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 07 Mar 2022 12:18:01 -0000 From: Dominic Jäger Extend qm importdisk functionality to the API. Co-authored-by: Fabian Grünbichler Co-authored-by: Dominic Jäger Signed-off-by: Fabian Ebner --- Changes from v10: * Switch to using clone_disk for PVE-managed volumes and check for VM.Clone in the permission check if there is an owner ID. * Require :0 syntax when using import-from. Allowing other values than 0 for the size would be confusing, because with import-from that size is never used (the size of the source image is). The check moved to check_drive_param, as that seemed to be the more fitting place. * Avoid making all foreach_volume iterators parse with the extended schema. Instead, provide a custom iterator for the places where it's actually required. * Mention that source volume should not be actively used in import-from description. * Add missing newline to error for size check for source image, and also die when size is zero. PVE/API2/Qemu.pm | 215 ++++++++++++++++++++++++++++++----- PVE/QemuServer/Drive.pm | 34 +++++- PVE/QemuServer/ImportDisk.pm | 2 +- 3 files changed, 216 insertions(+), 35 deletions(-) diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index f30c56f..c6d57e2 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -21,8 +21,9 @@ use PVE::ReplicationConfig; use PVE::GuestHelpers; use PVE::QemuConfig; use PVE::QemuServer; -use PVE::QemuServer::Drive; use PVE::QemuServer::CPUConfig; +use PVE::QemuServer::Drive; +use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::Machine; use PVE::QemuMigrate; @@ -63,28 +64,46 @@ my $resolve_cdrom_alias = sub { } }; +# Used in import-enabled API endpoints. Parses drives using the extended '_with_alloc' schema. +my $foreach_volume_with_alloc = sub { + my ($param, $func) = @_; + + for my $opt (sort keys $param->%*) { + next if !PVE::QemuServer::is_valid_drivename($opt); + + my $drive = PVE::QemuServer::Drive::parse_drive($opt, $param->{$opt}, 1); + next if !$drive; + + $func->($opt, $drive); + } +}; + +my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; + my $check_drive_param = sub { my ($param, $storecfg, $extra_checks) = @_; for my $opt (sort keys $param->%*) { next if !PVE::QemuServer::is_valid_drivename($opt); - my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); + my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + die "'import-from' requires special syntax - use :0,import-from=\n" + if $drive->{'import-from'} && ($drive->{file} !~ $NEW_DISK_RE || $3 != 0); + PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); $extra_checks->($drive) if $extra_checks; - $param->{$opt} = PVE::QemuServer::print_drive($drive); + $param->{$opt} = PVE::QemuServer::print_drive($drive, 1); } }; -my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; my $check_storage_access = sub { my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_; - PVE::QemuConfig->foreach_volume($settings, sub { + $foreach_volume_with_alloc->($settings, sub { my ($ds, $drive) = @_; my $isCDROM = PVE::QemuServer::drive_is_cdrom($drive); @@ -106,6 +125,20 @@ my $check_storage_access = sub { } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); } + + if (my $src_image = $drive->{'import-from'}) { + my $src_vmid; + my ($src_storeid) = PVE::Storage::parse_volume_id($src_image, 1); + if ($src_storeid) { # PVE-managed volume + $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_image))[2] + } + + if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk() + $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']); + } else { + PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image); + } + } }); $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace']) @@ -164,6 +197,87 @@ my $check_storage_access_migrate = sub { if !$scfg->{content}->{images}; }; +my $import_from_volid = sub { + my ($storecfg, $src_volid, $dest_info, $vollist) = @_; + + die "cannot import from cloudinit disk\n" + if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid }); + + my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid); + my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2]; + + my $src_vm_state = sub { + my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0; + + my $runs = 0; + if ($exists) { + eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); }; + die "owner VM $src_vmid not on local node\n" if $@; + $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0; + } + + return ($exists, $runs); + }; + + my ($src_vm_exists, $running) = $src_vm_state->(); + + die "cannot import from '$src_volid' - full clone feature is not supported\n" + if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running); + + my $clonefn = sub { + my ($src_vm_exists_now, $running_now) = $src_vm_state->(); + + die "owner VM $src_vmid changed state unexpectedly\n" + if $src_vm_exists_now != $src_vm_exists || $running_now != $running; + + my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {}; + + my $src_drive = { file => $src_volid }; + my $src_drivename; + PVE::QemuConfig->foreach_volume($src_conf, sub { + my ($ds, $drive) = @_; + + return if $src_drivename; + + if ($drive->{file} eq $src_volid) { + $src_drive = $drive; + $src_drivename = $ds; + } + }); + + my $source_info = { + vmid => $src_vmid, + running => $running_now, + drivename => $src_drivename, + drive => $src_drive, + snapname => undef, + }; + + return PVE::QemuServer::clone_disk( + $storecfg, + $source_info, + $dest_info, + 1, + $vollist, + undef, + undef, + $src_conf->{agent}, + PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]), + ); + }; + + my $cloned; + if ($running) { + $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn); + } elsif ($src_vmid) { + $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn); + } else { + $cloned = $clonefn->(); + } + + return $cloned->@{qw(file size)}; +}; + # Note: $pool is only needed when creating a VM, because pool permissions # are automatically inherited if VM already exists inside a pool. my $create_disks = sub { @@ -207,28 +321,73 @@ my $create_disks = sub { } elsif ($volid =~ $NEW_DISK_RE) { my ($storeid, $size) = ($2 || $default_storage, $3); die "no storage ID specified (and no default storage)\n" if !$storeid; - my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); - my $fmt = $disk->{format} || $defformat; - - $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb - - my $volid; - if ($ds eq 'efidisk0') { - my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); - ($volid, $size) = PVE::QemuServer::create_efidisk( - $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm); - } elsif ($ds eq 'tpmstate0') { - # swtpm can only use raw volumes, and uses a fixed size - $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb'); - $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size); + + if (my $source = delete $disk->{'import-from'}) { + my $dst_volid; + my ($src_storeid) = PVE::Storage::parse_volume_id($source, 1); + + if ($src_storeid) { # PVE-managed volume + die "could not get size of $source\n" + if !PVE::Storage::volume_size_info($storecfg, $source, 10); + + my $dest_info = { + vmid => $vmid, + conf => $conf, + drivename => $ds, + storage => $storeid, + format => $disk->{format}, + }; + + ($dst_volid, $size) = eval { + $import_from_volid->($storecfg, $source, $dest_info, $vollist); + }; + die "cannot import from '$source' - $@" if $@; + } else { + $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1); + $size = PVE::Storage::file_size_info($source); + die "could not get file size of $source\n" if !$size; + + (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import( + $source, + $vmid, + $storeid, + { + drive_name => $ds, + format => $disk->{format}, + 'skip-config-update' => 1, + }, + ); + push @$vollist, $dst_volid; + } + + $disk->{file} = $dst_volid; + $disk->{size} = $size; + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); } else { - $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); + my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); + my $fmt = $disk->{format} || $defformat; + + $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb + + my $volid; + if ($ds eq 'efidisk0') { + my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); + ($volid, $size) = PVE::QemuServer::create_efidisk( + $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm); + } elsif ($ds eq 'tpmstate0') { + # swtpm can only use raw volumes, and uses a fixed size + $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb'); + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size); + } else { + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); + } + push @$vollist, $volid; + $disk->{file} = $volid; + $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); } - push @$vollist, $volid; - $disk->{file} = $volid; - $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); - delete $disk->{format}; # no longer needed - $res->{$ds} = PVE::QemuServer::print_drive($disk); } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); @@ -242,7 +401,7 @@ my $create_disks = sub { } }; - eval { PVE::QemuConfig->foreach_volume($settings, $code); }; + eval { $foreach_volume_with_alloc->($settings, $code); }; # free allocated images on error if (my $err = $@) { @@ -1285,7 +1444,7 @@ my $update_vm_api = sub { my $check_drive_perms = sub { my ($opt, $val) = @_; - my $drive = PVE::QemuServer::parse_drive($opt, $val); + my $drive = PVE::QemuServer::parse_drive($opt, $val, 1); # FIXME: cloudinit: CDROM or Disk? if (PVE::QemuServer::drive_is_cdrom($drive)) { # CDROM $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.CDROM']); @@ -1391,7 +1550,7 @@ my $update_vm_api = sub { # default legacy boot order implies all cdroms anyway if (@bootorder) { # append new CD drives to bootorder to mark them bootable - my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); + my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); if (PVE::QemuServer::drive_is_cdrom($drive, 1) && !grep(/^$opt$/, @bootorder)) { push @bootorder, $opt; $conf->{pending}->{boot} = PVE::QemuServer::print_bootorder(\@bootorder); diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm index d5d4723..88f013a 100644 --- a/PVE/QemuServer/Drive.pm +++ b/PVE/QemuServer/Drive.pm @@ -409,6 +409,22 @@ my $alldrive_fmt = { %efitype_fmt, }; +my %import_from_fmt = ( + 'import-from' => { + type => 'string', + format => 'pve-volume-id-or-absolute-path', + format_description => 'source volume', + description => "Create a new disk, importing from this source. If the volume is not ". + "managed by Proxmox VE, it's up to you to ensure that it's not actively used by ". + "another process during the import!", + optional => 1, + }, +); +my $alldrive_fmt_with_alloc = { + %$alldrive_fmt, + %import_from_fmt, +}; + my $unused_fmt = { volume => { alias => 'file' }, file => { @@ -436,6 +452,8 @@ my $desc_with_alloc = sub { my $new_desc = dclone($desc); + $new_desc->{format}->{'import-from'} = $import_from_fmt{'import-from'}; + my $extra_note = ''; if ($type eq 'efidisk') { $extra_note = " Note that SIZE_IN_GiB is ignored here and that the default EFI vars are ". @@ -445,7 +463,8 @@ my $desc_with_alloc = sub { } $new_desc->{description} .= " Use the special syntax STORAGE_ID:SIZE_IN_GiB to allocate a new ". - "volume.${extra_note}"; + "volume.${extra_note} Use STORAGE_ID:0 and the 'import-from' parameter to import from an ". + "existing volume."; $with_alloc_desc_cache->{$type} = $new_desc; @@ -547,7 +566,7 @@ sub drive_is_read_only { # [,iothread=on][,serial=serial][,model=model] sub parse_drive { - my ($key, $data) = @_; + my ($key, $data, $with_alloc) = @_; my ($interface, $index); @@ -558,12 +577,14 @@ sub parse_drive { return; } - if (!defined($drivedesc_hash->{$key})) { + my $desc_hash = $with_alloc ? $drivedesc_hash_with_alloc : $drivedesc_hash; + + if (!defined($desc_hash->{$key})) { warn "invalid drive key: $key\n"; return; } - my $desc = $drivedesc_hash->{$key}->{format}; + my $desc = $desc_hash->{$key}->{format}; my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) }; return if !$res; $res->{interface} = $interface; @@ -623,9 +644,10 @@ sub parse_drive { } sub print_drive { - my ($drive) = @_; + my ($drive, $with_alloc) = @_; my $skip = [ 'index', 'interface' ]; - return PVE::JSONSchema::print_property_string($drive, $alldrive_fmt, $skip); + my $fmt = $with_alloc ? $alldrive_fmt_with_alloc : $alldrive_fmt; + return PVE::JSONSchema::print_property_string($drive, $fmt, $skip); } sub get_bootdisks { diff --git a/PVE/QemuServer/ImportDisk.pm b/PVE/QemuServer/ImportDisk.pm index 51ad52e..7557cac 100755 --- a/PVE/QemuServer/ImportDisk.pm +++ b/PVE/QemuServer/ImportDisk.pm @@ -71,7 +71,7 @@ sub do_import { PVE::Storage::activate_volumes($storecfg, [$dst_volid]); PVE::QemuServer::qemu_img_convert($src_path, $dst_volid, $src_size, undef, $zeroinit); PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]); - PVE::QemuConfig->lock_config($vmid, $create_drive); + PVE::QemuConfig->lock_config($vmid, $create_drive) if !$params->{'skip-config-update'}; }; if (my $err = $@) { eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) }; -- 2.30.2