From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id D2FB87232D for ; Mon, 12 Apr 2021 12:08:32 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id D0A171CE01 for ; Mon, 12 Apr 2021 12:08:32 +0200 (CEST) Received: from dev.dominic.proxmox.com (unknown [94.136.29.99]) by firstgate.proxmox.com (Proxmox) with ESMTP id BD42E1CDED for ; Mon, 12 Apr 2021 12:08:29 +0200 (CEST) Received: by dev.dominic.proxmox.com (Postfix, from userid 0) id A5A522274E; Mon, 12 Apr 2021 12:08:29 +0200 (CEST) From: =?UTF-8?q?Dominic=20J=C3=A4ger?= To: pve-devel@lists.proxmox.com Date: Mon, 12 Apr 2021 12:08:24 +0200 Message-Id: <20210412100825.133698-1-d.jaeger@proxmox.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 2 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records RDNS_NONE 1.274 Delivered to internal network by a host with no rDNS SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record Subject: [pve-devel] [PATCH] Add API for VM import X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 12 Apr 2021 10:08:32 -0000 Extend qm importdisk/importovf functionality to the API. Co-authored-by: Fabian Grünbichler Signed-off-by: Dominic Jäger --- v8: - Fabian moved the import functions into the existing create_vm / update_vm_api - Dropped the separate API endpoints & import lock PVE/API2/Qemu.pm | 175 +++++++++++++++++++++++++++++++++++++++-- PVE/API2/Qemu/Makefile | 2 +- PVE/API2/Qemu/OVF.pm | 68 ++++++++++++++++ PVE/QemuServer.pm | 50 ++++++++++-- PVE/QemuServer/OVF.pm | 10 ++- 5 files changed, 289 insertions(+), 16 deletions(-) create mode 100644 PVE/API2/Qemu/OVF.pm diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index c56b609..5c31756 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -45,7 +45,6 @@ BEGIN { } } -use Data::Dumper; # fixme: remove use base qw(PVE::RESTHandler); @@ -61,7 +60,19 @@ my $resolve_cdrom_alias = sub { } }; +my $parse_import_sources = sub { + my $param = shift; + my $import = {}; + foreach my $pair (PVE::Tools::split_list($param)) { + my ($device, $diskimage) = split('=', $pair); + $import->{$device} = $diskimage; + } + + return $import; +}; + my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!; +my $IMPORT_DISK_RE = qr!^([^/:\s]+):-1$!; my $check_storage_access = sub { my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_; @@ -84,6 +95,12 @@ my $check_storage_access = sub { my $scfg = PVE::Storage::storage_config($storecfg, $storeid); raise_param_exc({ storage => "storage '$storeid' does not support vm images"}) if !$scfg->{content}->{images}; + } elsif ($volid =~ $IMPORT_DISK_RE) { + my $storeid = $1; + $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']); + my $scfg = PVE::Storage::storage_config($storecfg, $storeid); + raise_param_exc({ storage => "storage '$storeid' does not support vm images"}) + if !$scfg->{content}->{images}; } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); } @@ -91,6 +108,13 @@ my $check_storage_access = sub { $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace']) if defined($settings->{vmstatestorage}); + + if (defined($settings->{import_sources})) { + my $images = $parse_import_sources->($settings->{import_sources}); + foreach my $source_image (values %$images) { + PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $source_image); + } + } }; my $check_storage_access_clone = sub { @@ -133,10 +157,25 @@ my $check_storage_access_clone = sub { return $sharedvm; }; +# Raise exception if $format is not supported by $storeid +my $check_format_is_supported = sub { + my ($format, $storeid, $storecfg) = @_; + die "storage ID parameter must be passed to the sub" if !$storeid; + die "storage configuration must be passed to the sub" if !$storecfg; + + return if !$format; + + my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid); + my $supported = grep { $_ eq $format } @$valid_formats; + + die "format '$format' is not supported on storage $storeid" if !$supported; +}; + + # Note: $pool is only needed when creating a VM, because pool permissions # are automatically inherited if VM already exists inside a pool. my $create_disks = sub { - my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_; + my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage, $import) = @_; my $vollist = []; @@ -192,6 +231,69 @@ my $create_disks = sub { $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b'); delete $disk->{format}; # no longer needed $res->{$ds} = PVE::QemuServer::print_drive($disk); + } elsif ($volid =~ $IMPORT_DISK_RE) { + my $target_storage = $1; + + my $source = $import->{$ds}; + die "cannot import '$ds', no import source defined\n" if !$source; + $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1); + my $src_size = PVE::Storage::file_size_info($source); + die "Could not get file size of $source" if !defined($src_size); + + $check_format_is_supported->($disk->{format}, $storeid, $storecfg); + + my $dst_format = PVE::QemuServer::resolve_dst_disk_format( + $storecfg, + $storeid, + undef, + $disk->{format}, + ); + my $dst_volid = PVE::Storage::vdisk_alloc( + $storecfg, + $storeid, + $vmid, + $dst_format, + undef, + PVE::Tools::convert_size($src_size, 'b' => 'kb'), + ); + + print "Importing disk image '$source' as '$dst_volid'...\n"; + eval { + local $SIG{INT} = + local $SIG{TERM} = + local $SIG{QUIT} = + local $SIG{HUP} = + local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; }; + + my $zeroinit = PVE::Storage::volume_has_feature( + $storecfg, + 'sparseinit', + $dst_volid, + ); + PVE::Storage::activate_volumes($storecfg, [$dst_volid]); + PVE::QemuServer::qemu_img_convert( + $source, + $dst_volid, + $src_size, + undef, + $zeroinit, + ); + PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]); + + }; + if (my $err = $@) { + eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) }; + warn "Cleanup of $dst_volid failed: $@ \n" if $@; + + die "Importing disk '$source' failed: $err\n" if $err; + } + push @$vollist, $dst_volid; + $disk->{file} = $dst_volid; + if ($ds !~ /^unused\d+$/) { + $disk->{size} = $src_size; + delete $disk->{format}; # no longer needed + } + $res->{$ds} = PVE::QemuServer::print_drive($disk); } else { PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid); @@ -218,7 +320,7 @@ my $create_disks = sub { } }; - eval { PVE::QemuConfig->foreach_volume($settings, $code); }; + eval { PVE::QemuConfig->foreach_volume_full($settings, { include_unused => 1 }, $code); }; # free allocated images on error if (my $err = $@) { @@ -550,6 +652,13 @@ __PACKAGE__->register_method({ default => 0, description => "Start VM after it was created successfully.", }, + import_sources => { + description => "\\0 delimited mapping of devices to disk images to import." . + "For example, scsi0=/mnt/nfs/image1.vmdk", + type => 'string', + format => 'device-image-pair-alist', + optional => 1, + }, }), }, returns => { @@ -615,21 +724,34 @@ __PACKAGE__->register_method({ &$check_cpu_model_access($rpcenv, $authuser, $param); + my $import_devices = $parse_import_sources->($param->{import_sources}); + foreach my $opt (keys %$param) { if (PVE::QemuServer::is_valid_drivename($opt)) { my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + raise_param_exc({ $opt => "not marked for import, but import source defined" }) + if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt}; + raise_param_exc({ $opt => "marked for import, but no import source defined" }) + if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt}; PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); $param->{$opt} = PVE::QemuServer::print_drive($drive); } } + foreach my $opt (keys %$import_devices) { + raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" }) + if !defined($param->{$opt}); + + } PVE::QemuServer::add_random_macs($param); } else { my $keystr = join(' ', keys %$param); raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr; + raise_param_exc({ import_sources => "cannot import existing disk and restore backup." }) if $param->{import_sources}; + if ($archive eq '-') { die "pipe requires cli environment\n" if $rpcenv->{type} ne 'cli'; @@ -701,10 +823,22 @@ __PACKAGE__->register_method({ my $realcmd = sub { my $conf = $param; my $arch = PVE::QemuServer::get_vm_arch($conf); + my $import = $parse_import_sources->(extract_param($param, "import_sources")); my $vollist = []; eval { - $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage); + $vollist = &$create_disks( + $rpcenv, + $authuser, + $conf, + $arch, + $storecfg, + $vmid, + $pool, + $param, + $storage, + $import + ); if (!$conf->{boot}) { my $devs = PVE::QemuServer::get_default_bootdevices($conf); @@ -1181,11 +1315,17 @@ my $update_vm_api = sub { die "cannot add non-replicatable volume to a replicated VM\n"; }; + my $import_devices = $parse_import_sources->($param->{import_sources}); + foreach my $opt (keys %$param) { if (PVE::QemuServer::is_valid_drivename($opt)) { # cleanup drive path my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt}); raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive; + raise_param_exc({ $opt => "not marked for import, but import source defined" }) + if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt}; + raise_param_exc({ $opt => "marked for import, but no import source defined" }) + if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt}; PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive); $check_replication->($drive); $param->{$opt} = PVE::QemuServer::print_drive($drive); @@ -1203,12 +1343,20 @@ my $update_vm_api = sub { } } + foreach my $opt (keys %$import_devices) { + raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" }) + if !defined($param->{$opt}); + + } + &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]); &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]); &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param); + delete $param->{import_sources}; + my $updatefn = sub { my $conf = PVE::QemuConfig->load_config($vmid); @@ -1350,7 +1498,17 @@ my $update_vm_api = sub { PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt})) if defined($conf->{pending}->{$opt}); - &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}}); + &$create_disks( + $rpcenv, + $authuser, + $conf->{pending}, + $arch, $storecfg, + $vmid, + undef, + {$opt => $param->{$opt}}, + undef, + {$opt => $import_devices->{$opt}} + ); } elsif ($opt =~ m/^serial\d+/) { if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') { $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']); @@ -1494,6 +1652,13 @@ __PACKAGE__->register_method({ optional => 1, requires => 'delete', }, + import_sources => { + description => "\\0 delimited mapping of devices to disk images to import." . + "For example, scsi0=/mnt/nfs/image1.vmdk", + type => 'string', + format => 'device-image-pair-alist', + optional => 1, + }, digest => { type => 'string', description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile index 5d4abda..bdd4762 100644 --- a/PVE/API2/Qemu/Makefile +++ b/PVE/API2/Qemu/Makefile @@ -1,4 +1,4 @@ -SOURCES=Agent.pm CPU.pm Machine.pm +SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm .PHONY: install install: diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm new file mode 100644 index 0000000..bd6e90b --- /dev/null +++ b/PVE/API2/Qemu/OVF.pm @@ -0,0 +1,68 @@ +package PVE::API2::Qemu::OVF; + +use strict; +use warnings; + +use PVE::RESTHandler; +use PVE::JSONSchema qw(get_standard_option); +use PVE::QemuServer::OVF; + +use base qw(PVE::RESTHandler); + +__PACKAGE__->register_method ({ + name => 'index', + path => '', + method => 'GET', + proxyto => 'node', + description => "Read an .ovf manifest.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + manifest => { + description => ".ovf manifest", + type => 'string', + }, + }, + }, + returns => { + description => "VM config according to .ovf manifest and digest of manifest", + type => "object", + }, + returns => { + type => 'object', + additionalProperties => 1, + properties => PVE::QemuServer::json_ovf_properties({ + name => { + type => 'string', + optional => 1, + }, + cores => { + type => 'integer', + optional => 1, + }, + memory => { + type => 'integer', + optional => 1, + }, + }), + }, + code => sub { + my ($param) = @_; + + my $manifest = $param->{manifest}; + die "$manifest: non-existent or non-regular file\n" if (! -f $manifest); + + my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest, 0, 1); + my $result; + $result->{cores} = $parsed->{qm}->{cores}; + $result->{name} = $parsed->{qm}->{name}; + $result->{memory} = $parsed->{qm}->{memory}; + my $disks = $parsed->{disks}; + foreach my $disk (@$disks) { + $result->{$disk->{disk_address}} = $disk->{backing_file}; + } + return $result; + }}); + +1; \ No newline at end of file diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index fdb2ac9..ac2fe2e 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -987,19 +987,41 @@ PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_ sub verify_volume_id_or_qm_path { my ($volid, $noerr) = @_; - if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) { - return $volid; - } + return $volid eq 'none' || $volid eq 'cdrom' ? + $volid : + verify_volume_id_or_absolute_path($volid, $noerr); +} + +PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path); +sub verify_volume_id_or_absolute_path { + my ($volid, $noerr) = @_; + + return $volid if $volid =~ m|^/|; - # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') }; if ($@) { - return if $noerr; + return undef if $noerr; die $@; } return $volid; } +PVE::JSONSchema::register_format('device-image-pair', \&verify_device_image_pair); +sub verify_device_image_pair { + my ($pair, $noerr) = @_; + + my $error = sub { + return undef if $noerr; + die @_; + }; + + my ($device, $image) = split('=', $pair); + $error->("Invalid device '$device'") if !PVE::QemuServer::Drive::is_valid_drivename($device); + $error->("Invalid image '$image'") if !verify_volume_id_or_absolute_path($image); + + return $pair; +} + my $usb_fmt = { host => { default_key => 1, @@ -2075,6 +2097,22 @@ sub json_config_properties { return $prop; } +# Properties that we can read from an OVF file +sub json_ovf_properties { + my $prop = shift; + + foreach my $device ( PVE::QemuServer::Drive::valid_drive_names()) { + $prop->{$device} = { + type => 'string', + format => 'pve-volume-id-or-absolute-path', + description => "Disk image that gets imported to $device", + optional => 1, + }; + } + + return $prop; +} + # return copy of $confdesc_cloudinit to generate documentation sub cloudinit_config_properties { @@ -6914,7 +6952,7 @@ sub qemu_img_convert { $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname); $src_is_iscsi = ($src_path =~ m|^iscsi://|); $cachemode = 'none' if $src_scfg->{type} eq 'zfspool'; - } elsif (-f $src_volid) { + } elsif (-f $src_volid || -b $src_volid) { $src_path = $src_volid; if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { $src_format = $1; diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm index c76c199..48146e9 100644 --- a/PVE/QemuServer/OVF.pm +++ b/PVE/QemuServer/OVF.pm @@ -87,7 +87,7 @@ sub id_to_pve { # returns two references, $qm which holds qm.conf style key/values, and \@disks sub parse_ovf { - my ($ovf, $debug) = @_; + my ($ovf, $debug, $manifest_only) = @_; my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1); @@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id); die "error parsing $filepath, file seems not to exist at $backing_file_path\n"; } - my $virtual_size; - if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) { - die "error parsing $backing_file_path, size seems to be $virtual_size\n"; + my $virtual_size = undef; + if (!$manifest_only) { # Not possible if manifest is uploaded in web gui + if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) { + die "error parsing $backing_file_path: Could not get file size info: $@\n"; + } } $pve_disk = { -- 2.20.1