From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D56626ADD6 for ; Thu, 17 Mar 2022 12:31:59 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 267BF2EBD for ; Thu, 17 Mar 2022 12:31:28 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 76E8D2CDA for ; Thu, 17 Mar 2022 12:31:16 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 3907F426FD for ; Thu, 17 Mar 2022 12:31:16 +0100 (CET) From: Fabian Ebner To: pve-devel@lists.proxmox.com Date: Thu, 17 Mar 2022 12:31:05 +0100 Message-Id: <20220317113107.60466-8-f.ebner@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220317113107.60466-1-f.ebner@proxmox.com> References: <20220317113107.60466-1-f.ebner@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.030 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [drive.pm, importdisk.pm, qemu.pm] Subject: [pve-devel] [PATCH v13 qemu-server 7/8] api: support VM disk import X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 17 Mar 2022 11:31:59 -0000 From: Dominic Jäger Extend qm importdisk functionality to the API. Co-authored-by: Fabian Grünbichler Co-authored-by: Dominic Jäger Signed-off-by: Fabian Ebner --- Changes from v12: * Switch to raise_param_exc rather than die in API helper. * Simplify check for PVE-managed volume. * Assert that volume type for import-source is 'images'. * Move existence/size check into $import_from_volid. * Add comment about new skip-config-update (and existing skiplock) options to do_import() * Clear up that passing an absolute path to import-from means that the user is responsible to ensure that the source is not used by something else. Previously it read "not managed by PVE", but there can be absolute paths pointing to PVE-managed volumes of course ;). We /could/ use PVE::Storage::path_to_volume_id to resolve such paths to volids, but that's a) automagic and b) only works with file-based storages and not things like zvol device paths. PVE/API2/Qemu.pm | 208 +++++++++++++++++++++++++++++++---- PVE/QemuServer/Drive.pm | 18 ++- PVE/QemuServer/ImportDisk.pm | 4 +- 3 files changed, 207 insertions(+), 23 deletions(-) diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index 1082d5e3..0e9f5c7e 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -21,8 +21,9 @@ use PVE::ReplicationConfig; use PVE::GuestHelpers; use PVE::QemuConfig; use PVE::QemuServer; -use PVE::QemuServer::Drive; use PVE::QemuServer::CPUConfig; +use PVE::QemuServer::Drive; +use PVE::QemuServer::ImportDisk; use PVE::QemuServer::Monitor qw(mon_cmd); use PVE::QemuServer::Machine; use PVE::QemuMigrate; @@ -88,6 +89,28 @@ my $check_drive_param = sub { my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}, 1); raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + if ($drive->{'import-from'}) { + if ($drive->{file} !~ $NEW_DISK_RE || $3 != 0) { + raise_param_exc({ + $opt => "'import-from' requires special syntax - ". + "use :0,import-from=", + }); + } + + if ($opt eq 'efidisk0') { + for my $required (qw(efitype pre-enrolled-keys)) { + if (!defined($drive->{$required})) { + raise_param_exc({ + $opt => "need to specify '$required' when using 'import-from'", + }); + } + } + } elsif ($opt eq 'tpmstate0') { + raise_param_exc({ $opt => "need to specify 'version' when using 'import-from'" }) + if !defined($drive->{version}); + } + } + PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); $extra_checks->($drive) if $extra_checks; @@ -121,6 +144,21 @@ my $check_storage_access = sub { } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); } + + if (my $src_image = $drive->{'import-from'}) { + my $src_vmid; + if (PVE::Storage::parse_volume_id($src_image, 1)) { # PVE-managed volume + (my $vtype, undef, $src_vmid) = PVE::Storage::parse_volname($storecfg, $src_image); + raise_param_exc({ $ds => "$src_image has wrong type '$vtype' - not an image" }) + if $vtype ne 'images'; + } + + if ($src_vmid) { # might be actively used by VM and will be copied via clone_disk() + $rpcenv->check($authuser, "/vms/${src_vmid}", ['VM.Clone']); + } else { + PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $src_image); + } + } }); $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace']) @@ -179,6 +217,91 @@ my $check_storage_access_migrate = sub { if !$scfg->{content}->{images}; }; +my $import_from_volid = sub { + my ($storecfg, $src_volid, $dest_info, $vollist) = @_; + + die "could not get size of $src_volid\n" + if !PVE::Storage::volume_size_info($storecfg, $src_volid, 10); + + die "cannot import from cloudinit disk\n" + if PVE::QemuServer::Drive::drive_is_cloudinit({ file => $src_volid }); + + my $src_vmid = (PVE::Storage::parse_volname($storecfg, $src_volid))[2]; + + my $src_vm_state = sub { + my $exists = $src_vmid && PVE::Cluster::get_vmlist()->{ids}->{$src_vmid} ? 1 : 0; + + my $runs = 0; + if ($exists) { + eval { PVE::QemuConfig::assert_config_exists_on_node($src_vmid); }; + die "owner VM $src_vmid not on local node\n" if $@; + $runs = PVE::QemuServer::Helpers::vm_running_locally($src_vmid) || 0; + } + + return ($exists, $runs); + }; + + my ($src_vm_exists, $running) = $src_vm_state->(); + + die "cannot import from '$src_volid' - full clone feature is not supported\n" + if !PVE::Storage::volume_has_feature($storecfg, 'copy', $src_volid, undef, $running); + + my $clonefn = sub { + my ($src_vm_exists_now, $running_now) = $src_vm_state->(); + + die "owner VM $src_vmid changed state unexpectedly\n" + if $src_vm_exists_now != $src_vm_exists || $running_now != $running; + + my $src_conf = $src_vm_exists_now ? PVE::QemuConfig->load_config($src_vmid) : {}; + + my $src_drive = { file => $src_volid }; + my $src_drivename; + PVE::QemuConfig->foreach_volume($src_conf, sub { + my ($ds, $drive) = @_; + + return if $src_drivename; + + if ($drive->{file} eq $src_volid) { + $src_drive = $drive; + $src_drivename = $ds; + } + }); + + my $source_info = { + vmid => $src_vmid, + running => $running_now, + drivename => $src_drivename, + drive => $src_drive, + snapname => undef, + }; + + my ($src_storeid) = PVE::Storage::parse_volume_id($src_volid); + + return PVE::QemuServer::clone_disk( + $storecfg, + $source_info, + $dest_info, + 1, + $vollist, + undef, + undef, + $src_conf->{agent}, + PVE::Storage::get_bandwidth_limit('clone', [$src_storeid, $dest_info->{storage}]), + ); + }; + + my $cloned; + if ($running) { + $cloned = PVE::QemuConfig->lock_config_full($src_vmid, 30, $clonefn); + } elsif ($src_vmid) { + $cloned = PVE::QemuConfig->lock_config_shared($src_vmid, 30, $clonefn); + } else { + $cloned = $clonefn->(); + } + + return $cloned->@{qw(file size)}; +}; + # Note: $pool is only needed when creating a VM, because pool permissions # are automatically inherited if VM already exists inside a pool. my $create_disks = sub { @@ -222,28 +345,71 @@ my $create_disks = sub { } elsif ($volid =~ $NEW_DISK_RE) { my ($storeid, $size) = ($2 || $default_storage, $3); die "no storage ID specified (and no default storage)\n" if !$storeid; - my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); - my $fmt = $disk->{format} || $defformat; - - $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb - - my $volid; - if ($ds eq 'efidisk0') { - my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); - ($volid, $size) = PVE::QemuServer::create_efidisk( - $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm); - } elsif ($ds eq 'tpmstate0') { - # swtpm can only use raw volumes, and uses a fixed size - $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb'); - $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size); + + if (my $source = delete $disk->{'import-from'}) { + my $dst_volid; + + if (PVE::Storage::parse_volume_id($source, 1)) { # PVE-managed volume + my $dest_info = { + vmid => $vmid, + drivename => $ds, + storage => $storeid, + format => $disk->{format}, + }; + + $dest_info->{efisize} = PVE::QemuServer::get_efivars_size($conf, $disk) + if $ds eq 'efidisk0'; + + ($dst_volid, $size) = eval { + $import_from_volid->($storecfg, $source, $dest_info, $vollist); + }; + die "cannot import from '$source' - $@" if $@; + } else { + $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1); + $size = PVE::Storage::file_size_info($source); + die "could not get file size of $source\n" if !$size; + + (undef, $dst_volid) = PVE::QemuServer::ImportDisk::do_import( + $source, + $vmid, + $storeid, + { + drive_name => $ds, + format => $disk->{format}, + 'skip-config-update' => 1, + }, + ); + push @$vollist, $dst_volid; + } + + $disk->{file} = $dst_volid; + $disk->{size} = $size; + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); } else { - $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); + my $defformat = PVE::Storage::storage_default_format($storecfg, $storeid); + my $fmt = $disk->{format} || $defformat; + + $size = PVE::Tools::convert_size($size, 'gb' => 'kb'); # vdisk_alloc uses kb + + my $volid; + if ($ds eq 'efidisk0') { + my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf); + ($volid, $size) = PVE::QemuServer::create_efidisk( + $storecfg, $storeid, $vmid, $fmt, $arch, $disk, $smm); + } elsif ($ds eq 'tpmstate0') { + # swtpm can only use raw volumes, and uses a fixed size + $size = PVE::Tools::convert_size(PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE, 'b' => 'kb'); + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, "raw", undef, $size); + } else { + $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $size); + } + push @$vollist, $volid; + $disk->{file} = $volid; + $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); + delete $disk->{format}; # no longer needed + $res->{$ds} = PVE::QemuServer::print_drive($disk); } - push @$vollist, $volid; - $disk->{file} = $volid; - $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); - delete $disk->{format}; # no longer needed - $res->{$ds} = PVE::QemuServer::print_drive($disk); } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); diff --git a/PVE/QemuServer/Drive.pm b/PVE/QemuServer/Drive.pm index cebf1730..1dc6171a 100644 --- a/PVE/QemuServer/Drive.pm +++ b/PVE/QemuServer/Drive.pm @@ -409,8 +409,21 @@ my $alldrive_fmt = { %efitype_fmt, }; +my %import_from_fmt = ( + 'import-from' => { + type => 'string', + format => 'pve-volume-id-or-absolute-path', + format_description => 'source volume', + description => "Create a new disk, importing from this source (volume ID or absolute ". + "path). When an absolute path is specified, it's up to you to ensure that the source ". + "is not actively used by another process during the import!", + optional => 1, + }, +); + my $alldrive_fmt_with_alloc = { %$alldrive_fmt, + %import_from_fmt, }; my $unused_fmt = { @@ -440,6 +453,8 @@ my $desc_with_alloc = sub { my $new_desc = dclone($desc); + $new_desc->{format}->{'import-from'} = $import_from_fmt{'import-from'}; + my $extra_note = ''; if ($type eq 'efidisk') { $extra_note = " Note that SIZE_IN_GiB is ignored here and that the default EFI vars are ". @@ -449,7 +464,8 @@ my $desc_with_alloc = sub { } $new_desc->{description} .= " Use the special syntax STORAGE_ID:SIZE_IN_GiB to allocate a new ". - "volume.${extra_note}"; + "volume.${extra_note} Use STORAGE_ID:0 and the 'import-from' parameter to import from an ". + "existing volume."; $with_alloc_desc_cache->{$type} = $new_desc; diff --git a/PVE/QemuServer/ImportDisk.pm b/PVE/QemuServer/ImportDisk.pm index 51ad52ea..3e0474b6 100755 --- a/PVE/QemuServer/ImportDisk.pm +++ b/PVE/QemuServer/ImportDisk.pm @@ -11,6 +11,8 @@ use PVE::Tools qw(run_command extract_param); # and creates by default a drive entry unused[n] pointing to the created volume # $params->{drive_name} may be used to specify ide0, scsi1, etc ... # $params->{format} may be used to specify qcow2, raw, etc ... +# $params->{skiplock} may be used to skip checking for a lock in the VM config +# $params->{'skip-config-update'} may be used to import the disk without updating the VM config sub do_import { my ($src_path, $vmid, $storage_id, $params) = @_; @@ -71,7 +73,7 @@ sub do_import { PVE::Storage::activate_volumes($storecfg, [$dst_volid]); PVE::QemuServer::qemu_img_convert($src_path, $dst_volid, $src_size, undef, $zeroinit); PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]); - PVE::QemuConfig->lock_config($vmid, $create_drive); + PVE::QemuConfig->lock_config($vmid, $create_drive) if !$params->{'skip-config-update'}; }; if (my $err = $@) { eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) }; -- 2.30.2