From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id BC8BB60201 for ; Fri, 5 Feb 2021 11:04:51 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id AD61D87F3 for ; Fri, 5 Feb 2021 11:04:51 +0100 (CET) Received: from dev.dominic.proxmox.com (212-186-127-178.static.upcbusiness.at [212.186.127.178]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6292287C4 for ; Fri, 5 Feb 2021 11:04:49 +0100 (CET) Received: by dev.dominic.proxmox.com (Postfix, from userid 0) id 2DF7C22705; Fri, 5 Feb 2021 11:04:49 +0100 (CET) From: =?UTF-8?q?Dominic=20J=C3=A4ger?= To: pve-devel@lists.proxmox.com Date: Fri, 5 Feb 2021 11:04:41 +0100 Message-Id: <20210205100442.28163-1-d.jaeger@proxmox.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 1 AWL -0.364 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods KHOP_HELO_FCRDNS 0.399 Relay HELO differs from its IP's reverse DNS NO_DNS_FOR_FROM 0.379 Envelope sender has no MX or A DNS records SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_NONE 0.001 SPF: sender does not publish an SPF Record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [qemu.pm, qemuserver.pm, ovf.pm] Subject: [pve-devel] [PATCH v4 qemu-server] Add API for disk & VM import X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Feb 2021 10:04:51 -0000 Extend qm importdisk/importovf functionality to the API. qm can be adapted to use this later. Signed-off-by: Dominic Jäger --- Biggest v3->v4 changes: * New code instead of bloating update_vm_api * Don't change anything in the existing schema, use new parameter "diskimages" * Because this can happen later: - Only root can use this - Don't touch qm (yet) PVE/API2/Qemu.pm | 375 +++++++++++++++++++++++++++++++++++++++++- PVE/QemuServer.pm | 16 +- PVE/QemuServer/OVF.pm | 10 +- 3 files changed, 394 insertions(+), 7 deletions(-) diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm index 3571f5e..1ed763b 100644 --- a/PVE/API2/Qemu.pm +++ b/PVE/API2/Qemu.pm @@ -45,7 +45,6 @@ BEGIN { } } -use Data::Dumper; # fixme: remove use base qw(PVE::RESTHandler); @@ -4325,4 +4324,378 @@ __PACKAGE__->register_method({ return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type}); }}); +# Raise exception if $format is not supported by $storageid +my $check_format_is_supported = sub { + my ($format, $storageid) = @_; + + return if !$format; + + my $store_conf = PVE::Storage::config(); + my (undef, $valid_formats) = PVE::Storage::storage_default_format($store_conf, $storageid); + my $supported = grep { $_ eq $format } @$valid_formats; + + if (!$supported) { + raise_param_exc({format => "$format is not supported on storage $storageid"}); + } +}; + +# paths are returned as is +# volids are returned as paths +# +# Also checks if $original actually exists +my $convert_to_path = sub { + my ($original) = @_; + my $volid_as_path = eval { # Nonempty iff $original_source is a volid + PVE::Storage::path(PVE::Storage::config(), $original); + }; + my $result = $volid_as_path || $original ; + if (!-e $result) { + die "Could not import because source '$original' does not exist!"; + } + return $result; +}; + +# vmid ... target VM ID +# source ... absolute path of the source image (volid must be converted before) +# storage ... target storage for the disk image +# format ... target format for the disk image (optional) +# +# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2) +my $import_disk_image = sub { + my ($param) = @_; + my $vmid = $param->{vmid}; + my $requested_format = $param->{format}; + my $storage = $param->{storage}; + my $source = $param->{source}; + + my $vm_conf = PVE::QemuConfig->load_config($vmid); + my $store_conf = PVE::Storage::config(); + if (!$source) { + die "It is necessary to pass the source parameter"; + } + if ($source !~ m!^/!) { + die "source must be an absolute path but is $source"; + } + if (!-e $source) { + die "Could not import because source $source does not exist!"; + } + if (!$storage) { + die "It is necessary to pass the storage parameter"; + } + + print "Importing disk image '$source'...\n"; + + my $src_size = PVE::Storage::file_size_info($source); + if (!defined($src_size)) { + die "Could not get file size of $source"; + } elsif (!$src_size) { + die "Size of file $source is 0"; + } elsif ($src_size==1) { + die "Cannot import a directory"; + } + + $check_format_is_supported->($requested_format, $storage); + + my $dst_format = PVE::QemuServer::resolve_dst_disk_format( + $store_conf, $storage, undef, $requested_format); + my $dst_volid = PVE::Storage::vdisk_alloc($store_conf, $storage, + $vmid, $dst_format, undef, $src_size / 1024); + + eval { + local $SIG{INT} = + local $SIG{TERM} = + local $SIG{QUIT} = + local $SIG{HUP} = + local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; }; + + my $zeroinit = PVE::Storage::volume_has_feature($store_conf, + 'sparseinit', $dst_volid); + + PVE::Storage::activate_volumes($store_conf, [$dst_volid]); + PVE::QemuServer::qemu_img_convert($source, $dst_volid, + $src_size, undef, $zeroinit); + PVE::Storage::deactivate_volumes($store_conf, [$dst_volid]); + + }; + if (my $err = $@) { + eval { PVE::Storage::vdisk_free($store_conf, $dst_volid) }; + warn "Cleanup of $dst_volid failed: $@ \n" if $@; + + die "Importing disk '$source' failed: $err\n" if $err; + } + + return $dst_volid; +}; + +__PACKAGE__->register_method ({ + name => 'importdisk', + path => '{vmid}/importdisk', + method => 'POST', + protected => 1, # for worker upid file + proxyto => 'node', + description => "Import an external disk image into a VM. The image format ". + "has to be supported by qemu-img.", + parameters => { + additionalProperties => 0, + properties => { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid', + {completion => \&PVE::QemuServer::complete_vmid}), + source => { + description => "Disk image to import. Can be a volid ". + "(local-lvm:vm-104-disk-0), an image on a PVE storage ". + "(local:104/toImport.raw) or (for root only) an absolute ". + "path on the server.", + type => 'string', + format => 'pve-volume-id-or-absolute-path', + }, + device => { + type => 'string', + description => "Bus/Device type of the new disk (e.g. 'ide0', ". + "'scsi2'). Will add the image as unused disk if omitted.", + enum => [PVE::QemuServer::Drive::valid_drive_names()], + optional => 1, + }, + device_options => { + type => 'string', + description => "Options to set for the new disk ". + "(e.g. 'discard=on,backup=0')", + optional => 1, + requires => 'device', + }, + storage => get_standard_option('pve-storage-id', { + description => "The storage to which the image will be imported to.", + completion => \&PVE::QemuServer::complete_storage, + }), + format => { + type => 'string', + description => 'Target format.', + enum => [ 'raw', 'qcow2', 'vmdk' ], + optional => 1, + }, + digest => get_standard_option('pve-config-digest'), + }, + }, + returns => { type => 'string'}, + code => sub { + my ($param) = @_; + my $vmid = extract_param($param, 'vmid'); + my $node = extract_param($param, 'node'); + my $original_source = extract_param($param, 'source'); + my $digest = extract_param($param, 'digest'); + my $device_options = extract_param($param, 'device_options'); + my $device = extract_param($param, 'device'); + my $storecfg = PVE::Storage::config(); + my $storeid = extract_param($param, 'storage'); + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + + my $format_explicit = extract_param($param, 'format'); + my $format_device_option; + if ($device_options) { + $device_options =~ m/format=([^,]*)/; + $format_device_option = $1; + if ($format_explicit && $format_device_option) { + raise_param_exc({format => "Disk format may be specified only once!"}); + } + } + my $format = $format_explicit || $format_device_option; + $check_format_is_supported->($format, $storeid); + + my $locked = sub { + my $conf = PVE::QemuConfig->load_config($vmid); + PVE::Tools::assert_if_modified($conf->{digest}, $digest); + + if ($device && $conf->{$device}) { + die "Could not import because device $device is already in ". + "use in VM $vmid. Choose a different device!"; + } + + my $imported_volid = $import_disk_image->({ + vmid => $vmid, + source => $convert_to_path->($original_source), + storage => $storeid, + format => $format, + }); + + my $volid = $imported_volid; + if ($device) { + # Attach with specified options + $volid .= ",${device_options}" if $device_options; + } else { + # Add as unused to config + $device = PVE::QemuConfig->add_unused_volume($conf, $imported_volid); + } + $update_vm_api->({ + node => $node, + vmid => $vmid, + $device => $volid, + }); + }; + my $worker = sub { + PVE::QemuConfig->lock_config_full($vmid, 1, $locked); + }; + return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker); + }}); + +__PACKAGE__->register_method({ + name => 'importvm', + path => '{vmid}/importvm', + method => 'POST', + description => "Import a VM from existing disk images.", + protected => 1, + proxyto => 'node', + parameters => { + additionalProperties => 0, + properties => PVE::QemuServer::json_config_properties( + { + node => get_standard_option('pve-node'), + vmid => get_standard_option('pve-vmid', { completion => \&PVE::Cluster::complete_next_vmid }), + diskimages => { + description => "Mapping of devices to disk images." . + "For example, scsi0:/mnt/nfs/image1.vmdk,scsi1:/mnt/nfs/image2", + type => 'string', + }, + start => { + optional => 1, + type => 'boolean', + default => 0, + description => "Start VM after it was imported successfully.", + }, + }), + }, + returns => { + type => 'string', + }, + code => sub { + my ($param) = @_; + my $node = extract_param($param, 'node'); + my $vmid = extract_param($param, 'vmid'); + + my $rpcenv = PVE::RPCEnvironment::get(); + my $authuser = $rpcenv->get_user(); + my $storecfg = PVE::Storage::config(); + + PVE::Cluster::check_cfs_quorum(); + + my $diskimages_string = extract_param($param, 'diskimages'); + my @diskimage_pairs = split(',', $diskimages_string); + + my $use_import = sub { + my ($opt) = @_; + return 0 if $opt eq 'efidisk0'; + return PVE::QemuServer::Drive::is_valid_drivename($opt); + }; + + my $msg = "There must be exactly as many devices specified as there " . + " are devices in the diskimage parameter.\n For example for " . + "--scsi0 local-lvm:0,discard=on --scsi1 local:0,cache=unsafe " . + "there must be --diskimages scsi0=/source/path,scsi1=/other/path"; + my $device_count = grep { $use_import->($_) } keys %$param; + + my $diskimages_count = @diskimage_pairs; + if ($device_count != $diskimages_count) { + raise_param_exc({diskimages => $msg}); + } + + my $diskimages = {}; + foreach ( @diskimage_pairs ) { + my ($device, $diskimage) = split('=', $_); + $diskimages->{$device} = $diskimage; + } + + my $worker = sub { + eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') }; + die "Unable to create config for VM import: $@" if $@; + + my @volids_of_imported = (); + eval { foreach my $opt (keys %$param) { + next if ($opt eq 'start'); + + my $updated_value; + if ($use_import->($opt)) { + # $opt is bus/device like ide0, scsi5 + + my $device = PVE::QemuServer::parse_drive($opt, $param->{$opt}); + raise_param_exc({ $opt => "Unable to parse drive options" }) + if !$device; + + my $source_path = $convert_to_path->($diskimages->{$opt}); + + $param->{$opt} =~ m/format=([^,]*)/; + my $format = $1; + + my $imported_volid = $import_disk_image->({ + vmid => $vmid, + source => $source_path, + device => $opt, + storage => (split ':', $device->{file})[0], + format => $format, + }); + push @volids_of_imported, $imported_volid; + + # $param->{opt} has all required options but also dummy + # import 0 instead of the image + # for example, local-lvm:0,discard=on,mbps_rd=100 + my $volid = $param->{$opt}; + # Replace 0 with allocated volid, for example + # local-lvm:vm-100-disk-2,discard=on,mbps_rd=100 + $volid =~ s/^.*?,/$imported_volid,/; + + $updated_value = $volid; + } else { + $updated_value = $param->{$opt}; + } + $update_vm_api->( + { + node => $node, + vmid => $vmid, + $opt => $updated_value, + skiplock => 1, + }, + 1, # avoid nested workers that only do a short operation + ); + }}; + + my $conf = PVE::QemuConfig->load_config($vmid); + my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf); + $update_vm_api->( + { + node => $node, + vmid => $vmid, + boot => PVE::QemuServer::print_bootorder($bootdevs), + skiplock => 1, + }, + 1, + ); + + my $err = $@; + if ($err) { + foreach my $volid (@volids_of_imported) { + eval { PVE::Storage::vdisk_free($storecfg, $volid) }; + warn $@ if $@; + } + + eval { + my $conffile = PVE::QemuConfig->config_file($vmid); + unlink($conffile) or die "Failed to remove config file: $!\n"; + }; + warn $@ if $@; + + die $err; + } + + eval { PVE::QemuConfig->remove_lock($vmid, 'import') }; + warn $@ if $@; + + if ($param->{start}) { + PVE::QemuServer::vm_start($storecfg, $vmid); + } + }; + + return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker); + }}); + + 1; diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm index 9c65d76..c02f5eb 100644 --- a/PVE/QemuServer.pm +++ b/PVE/QemuServer.pm @@ -300,7 +300,7 @@ my $confdesc = { optional => 1, type => 'string', description => "Lock/unlock the VM.", - enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)], + enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)], }, cpulimit => { optional => 1, @@ -998,6 +998,18 @@ sub verify_volume_id_or_qm_path { return $volid; } +PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path); +sub verify_volume_id_or_absolute_path { + my ($volid, $noerr) = @_; + + # Exactly these 2 are allowed in id_or_qm_path but should not be allowed here + if ($volid eq 'none' || $volid eq 'cdrom') { + return undef if $noerr; + die "Invalid format! Should be volume ID or absolute path."; + } + return verify_volume_id_or_qm_path($volid, $noerr); +} + my $usb_fmt = { host => { default_key => 1, @@ -6659,7 +6671,7 @@ sub qemu_img_convert { $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname); $src_is_iscsi = ($src_path =~ m|^iscsi://|); $cachemode = 'none' if $src_scfg->{type} eq 'zfspool'; - } elsif (-f $src_volid) { + } elsif (-f $src_volid || -b _) { # -b required to import from LVM images $src_path = $src_volid; if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) { $src_format = $1; diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm index c76c199..36b7fff 100644 --- a/PVE/QemuServer/OVF.pm +++ b/PVE/QemuServer/OVF.pm @@ -87,7 +87,7 @@ sub id_to_pve { # returns two references, $qm which holds qm.conf style key/values, and \@disks sub parse_ovf { - my ($ovf, $debug) = @_; + my ($ovf, $debug, $ignore_size) = @_; my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1); @@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id); die "error parsing $filepath, file seems not to exist at $backing_file_path\n"; } - my $virtual_size; - if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) { - die "error parsing $backing_file_path, size seems to be $virtual_size\n"; + my $virtual_size = 0; + if (!$ignore_size) { # Not possible if manifest is uploaded in web gui + if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) { + die "error parsing $backing_file_path: Could not get file size info: $@\n"; + } } $pve_disk = { -- 2.20.1