[pve-devel] [PATCH v7 qemu-server] Add API for import wizards

["Dominic Jäger" <d.jaeger@proxmox.com>]

Extend qm importdisk/importovf functionality to the API.

Signed-off-by: Dominic Jäger <d.jaeger@proxmox.com>

---
v6->v7: Feedback by Fabian G
- Introduce a regex for the import syntax <storeid>:0
- Use parameter list instead of hash for import helper
- More parsing, less string magic
- More VM config digest checking
- Create a schema format for diskimage source mapping
- Preliminarily remove some boot parameter handling
- Dare to really edit schema format subs for a cleaner solution
- Whitespace, variable names, ...

 PVE/API2/Qemu.pm       | 383 ++++++++++++++++++++++++++++++++++++++++-
 PVE/API2/Qemu/Makefile |   2 +-
 PVE/API2/Qemu/OVF.pm   |  68 ++++++++
 PVE/QemuServer.pm      |  52 +++++-
 PVE/QemuServer/OVF.pm  |  10 +-
 5 files changed, 502 insertions(+), 13 deletions(-)
 create mode 100644 PVE/API2/Qemu/OVF.pm

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index e95ab13..2f50f38 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -45,7 +45,6 @@ BEGIN {
     }
 }
 
-use Data::Dumper; # fixme: remove
 
 use base qw(PVE::RESTHandler);
 
@@ -62,6 +61,7 @@ my $resolve_cdrom_alias = sub {
 };
 
 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
+my $IMPORT_DISK_RE = qr!^([^/:\s]+):0$!;
 my $check_storage_access = sub {
    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 
@@ -4377,4 +4377,385 @@ __PACKAGE__->register_method({
 	return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
     }});
 
+# Raise exception if $format is not supported by $storeid
+my $check_format_is_supported = sub {
+    my ($format, $storeid, $storecfg) = @_;
+    die "storage ID parameter must be passed to the sub" if !$storeid;
+    die "storage configuration must be passed to the sub" if !$storecfg;
+
+    return if !$format;
+
+    my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+    my $supported = grep { $_ eq $format } @$valid_formats;
+
+    die "format '$format' is not supported on storage $storeid" if !$supported;
+};
+
+# storecfg ... PVE::Storage::config()
+# vmid ... target VM ID
+# vmconf ... target VM configuration
+# source ... source image (volid or absolute path)
+# target ... hash with
+#    storeid => storage ID
+#    format => disk format (optional)
+#    options => hash with device options (may or may not contain <storeid>:0)
+#    device => device where the disk is attached (for example, scsi3) (optional)
+#
+# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
+my $import_disk_image = sub {
+    my ($storecfg, $vmid, $vmconf, $source, $target) = @_;
+    my $requested_format = $target->{format};
+    my $storeid = $target->{storeid};
+
+    die "Source parameter is undefined!" if !defined $source;
+    $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
+
+    eval { PVE::Storage::storage_config($storecfg, $storeid) };
+    die "Error while importing disk image $source: $@\n" if $@;
+
+    my $src_size = PVE::Storage::file_size_info($source);
+    die "Could not get file size of $source" if !defined($src_size);
+
+    $check_format_is_supported->($requested_format, $storeid, $storecfg);
+
+    my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+	$storecfg,
+	$storeid,
+	undef,
+	$requested_format,
+    );
+    my $dst_volid = PVE::Storage::vdisk_alloc(
+	$storecfg,
+	$storeid,
+	$vmid,
+	$dst_format,
+	undef,
+	$src_size / 102,
+    );
+
+    print "Importing disk image '$source' as '$dst_volid'...\n";
+    eval {
+	local $SIG{INT} =
+	local $SIG{TERM} =
+	local $SIG{QUIT} =
+	local $SIG{HUP} =
+	local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+	my $zeroinit = PVE::Storage::volume_has_feature(
+	    $storecfg,
+	    'sparseinit',
+	    $dst_volid,
+	);
+	PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
+	PVE::QemuServer::qemu_img_convert(
+	    $source,
+	    $dst_volid,
+	    $src_size,
+	    undef,
+	    $zeroinit,
+	);
+	PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
+
+    };
+    if (my $err = $@) {
+	eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
+	warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+	die "Importing disk '$source' failed: $err\n" if $err;
+    }
+
+    $target->{options}->{file} = $dst_volid;
+    my $options_string = PVE::QemuServer::print_drive($target->{options});
+    $target->{device} = PVE::QemuConfig->add_unused_volume($vmconf, $dst_volid)
+	if !$target->{device};
+
+    $update_vm_api->(
+	{
+	    vmid => $vmid,
+	    $target->{device} => $options_string,
+	    skiplock => 1,
+	    digest => $vmconf->{digest},
+	},
+	1,
+    );
+
+    return $dst_volid;
+};
+
+__PACKAGE__->register_method ({
+    name => 'importdisk',
+    path => '{vmid}/importdisk',
+    method => 'POST',
+    proxyto => 'node',
+    protected => 1,
+    description => "Import an external disk image into a VM. The image format ".
+	"has to be supported by qemu-img.",
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    vmid => get_standard_option('pve-vmid',
+		{completion => \&PVE::QemuServer::complete_vmid}),
+	    source => {
+		description => "Disk image to import. Can be a volid ".
+		    "(local:99/imageToImport.raw) or an absolute path on the server.",
+		type => 'string',
+	    },
+	    device => {
+		type => 'string',
+		description => "Bus/Device type of the new disk (e.g. 'ide0', ".
+		    "'scsi2'). Will add the image as unused disk if omitted.",
+		enum => [PVE::QemuServer::Drive::valid_drive_names()],
+		optional => 1,
+	    },
+	    device_options => {
+		type => 'string',
+		description => "Options to set for the new disk (e.g. 'discard=on,backup=0')",
+		optional => 1,
+		requires => 'device',
+	    },
+	    storage => get_standard_option('pve-storage-id', {
+		description => "The storage to which the image will be imported to.",
+		completion => \&PVE::QemuServer::complete_storage,
+	    }),
+	    format => {
+		type => 'string',
+		description => 'Target format.',
+		enum => [ 'raw', 'qcow2', 'vmdk' ],
+		optional => 1,
+	    },
+	    digest => get_standard_option('pve-config-digest'),
+	},
+    },
+    returns => { type => 'string'},
+    code => sub {
+	my ($param) = @_;
+	my $vmid = extract_param($param, 'vmid');
+	my $node = extract_param($param, 'node');
+	my $source = extract_param($param, 'source');
+	my $digest = extract_param($param, 'digest');
+	my $device_options = extract_param($param, 'device_options');
+	my $device = extract_param($param, 'device');
+	my $storeid = extract_param($param, 'storage');
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $authuser = $rpcenv->get_user();
+	my $storecfg = PVE::Storage::config();
+	PVE::Storage::storage_config($storecfg, $storeid);
+
+
+	if ($device_options) {
+	    # $device_options may or may not contain <storeid>:0
+	    my $parsed = PVE::QemuServer::Drive::parse_drive($device, $device_options);
+	    if ($parsed) {
+		raise_param_exc({$device_options => "Invalid import syntax"})
+		    if !($parsed->{file} =~ $IMPORT_DISK_RE);
+	    } else {
+		my $fake = "$storeid:0,$device_options";
+		$parsed = PVE::QemuServer::Drive::parse_drive($device, $fake);
+	    }
+	    delete $parsed->{file};
+	    delete $parsed->{interface};
+	    delete $parsed->{index};
+	    $device_options = $parsed;
+	}
+
+	# Format can be set explicitly "--format vmdk"
+	# or as part of device options "--device_options discard=on,format=vmdk"
+	my $format = extract_param($param, 'format');
+	if ($device_options) {
+	    raise_param_exc({format => "Format already specified in device_options!"})
+		if $format && $device_options->{format};
+	    $format = $format || $device_options->{format}; # may be undefined
+	}
+	$check_format_is_supported->($format, $storeid, $storecfg);
+
+	# quick checks before fork + lock
+	my $conf = PVE::QemuConfig->load_config($vmid);
+	PVE::QemuConfig->check_lock($conf);
+	PVE::Tools::assert_if_modified($conf->{digest}, $digest);
+	if ($device && $conf->{$device}) {
+	    die "Could not import because device $device is already in ".
+		"use in VM $vmid. Choose a different device!";
+	}
+
+	my $worker = sub {
+	    PVE::QemuConfig->lock_config($vmid, sub {
+		$conf = PVE::QemuConfig->load_config($vmid);
+		PVE::QemuConfig->check_lock($conf);
+
+		PVE::Tools::assert_if_modified($conf->{digest}, $digest);
+		PVE::QemuConfig->set_lock($vmid, 'import');
+		$conf = PVE::QemuConfig->load_config($vmid);
+	    });
+
+	    my $target = {
+		node => $node,
+		storeid => $storeid,
+	    };
+	    $target->{format} = $format;
+	    $target->{device} = $device;
+	    $target->{options} = $device_options;
+	    eval { $import_disk_image->($storecfg, $vmid, $conf, $source, $target) };
+	    my $err = $@;
+	    eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
+	    warn $@ if $@;
+	    die $err if $err;
+	};
+	return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
+    }});
+
+__PACKAGE__->register_method({
+    name => 'importvm',
+    path => '{vmid}/importvm',
+    method => 'POST',
+    description => "Import a VM from existing disk images.",
+    protected => 1,
+    proxyto => 'node',
+    parameters => {
+	additionalProperties => 0,
+	properties => PVE::QemuServer::json_config_properties(
+	    {
+		node => get_standard_option('pve-node'),
+		vmid => get_standard_option('pve-vmid', { completion =>
+		    \&PVE::Cluster::complete_next_vmid }),
+		diskimage => {
+		    description => "\\0 delimited mapping of devices to disk images. For " .
+			"example, scsi0=/mnt/nfs/image1.vmdk",
+		    type => 'string',
+		    format => 'device-image-pair-alist',
+		},
+		start => {
+		    optional => 1,
+		    type => 'boolean',
+		    default => 0,
+		    description => "Start VM after it was imported successfully.",
+		},
+	    }),
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+	my $node = extract_param($param, 'node');
+	my $vmid = extract_param($param, 'vmid');
+	my $diskimages_string = extract_param($param, 'diskimage');
+	my $boot = extract_param($param, 'boot');
+	my $start = extract_param($param, 'start');
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $authuser = $rpcenv->get_user();
+	my $storecfg = PVE::Storage::config();
+
+	PVE::Cluster::check_cfs_quorum();
+
+	my $import_param = {};
+	foreach my $opt (keys %$param) {
+	    next if $opt eq 'efidisk0';
+	    raise_param_exc({bootdisk => "Deprecated: Use --boot order= instead"})
+		if $opt eq 'bootdisk';
+
+	    if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
+		my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
+		if ($drive->{file} =~ $IMPORT_DISK_RE) {
+		    $import_param->{$opt} = $drive;
+		    delete $param->{$opt};
+		}
+	    }
+	}
+
+	my $diskimages = {};
+	foreach my $pair (PVE::Tools::split_list($diskimages_string)) {
+	    my ($device, $diskimage) = split('=', $pair);
+	    $diskimages->{$device} = $diskimage;
+	    raise_param_exc({
+		$device => "Device '$device' not marked for import, " .
+		    "but import source '$diskimage' specified",
+	    }) if !defined($import_param->{$device});
+	    PVE::Storage::abs_filesystem_path($storecfg, $diskimage, 1);
+	}
+
+	foreach my $device (keys %$import_param) {
+	    raise_param_exc({
+		$device => "Device '$device' marked for import, but no source given\n",
+	    }) if !defined($diskimages->{$device});
+	}
+
+	eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
+	die "Unable to create config for VM import: $@" if $@;
+
+	my $worker = sub {
+	    my $reload_conf = sub {
+		my ($vmid) = @_;
+		my $conf = PVE::QemuConfig->load_config($vmid);
+		return $conf if PVE::QemuConfig->has_lock($conf, 'import');
+		die "import lock in VM $vmid config file missing!";
+	    };
+
+	    my $conf = $reload_conf->($vmid);
+	    $update_vm_api->(
+		{
+		    %$param,
+		    node => $node,
+		    vmid => $vmid,
+		    skiplock => 1,
+		    digest => $conf->{digest},
+		},
+		1
+	    );
+
+	    eval {
+		foreach my $device (keys %$import_param) {
+		    $conf = $reload_conf->($vmid);
+		    my $drive = $import_param->{$device};
+		    my $storeid = PVE::Storage::parse_volume_id($drive->{file});
+		    my $imported = $import_disk_image->(
+			$storecfg,
+			$vmid,
+			$conf,
+			$diskimages->{$device},
+			{
+			    storeid => $storeid,
+			    format => $drive->{format},
+			    options => $drive,
+			    device => $device,
+			},
+		    );
+		}
+	    };
+	    my $err = $@;
+	    if ($err) {
+		eval { PVE::QemuServer::destroy_vm($storecfg, $vmid, 1) };
+		warn "Could not destroy VM $vmid: $@" if "$@";
+
+		die "Import failed: $err";
+	    }
+
+	    $conf = $reload_conf->($vmid);
+	    if (!$boot) {
+		my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
+		$boot = PVE::QemuServer::print_bootorder($bootdevs);
+	    }
+	    $update_vm_api->(
+		{
+		    node => $node,
+		    vmid => $vmid,
+		    boot => $boot,
+		    skiplock => 1,
+		    digest => $conf->{digest},
+		},
+		1,
+	    );
+
+	    eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
+	    warn $@ if $@;
+
+	    PVE::QemuServer::vm_start($storecfg, $vmid) if $start;
+	};
+
+	return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
+    }});
+
+
 1;
diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile
index 5d4abda..bdd4762 100644
--- a/PVE/API2/Qemu/Makefile
+++ b/PVE/API2/Qemu/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Agent.pm CPU.pm Machine.pm
+SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm
 
 .PHONY: install
 install:
diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm
new file mode 100644
index 0000000..bd6e90b
--- /dev/null
+++ b/PVE/API2/Qemu/OVF.pm
@@ -0,0 +1,68 @@
+package PVE::API2::Qemu::OVF;
+
+use strict;
+use warnings;
+
+use PVE::RESTHandler;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::QemuServer::OVF;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+    name => 'index',
+    path => '',
+    method => 'GET',
+    proxyto => 'node',
+    description => "Read an .ovf manifest.",
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    manifest => {
+		description => ".ovf manifest",
+		type => 'string',
+	    },
+	},
+    },
+    returns => {
+	description => "VM config according to .ovf manifest and digest of manifest",
+	type => "object",
+    },
+    returns => {
+	type => 'object',
+	additionalProperties => 1,
+	properties => PVE::QemuServer::json_ovf_properties({
+	    name => {
+		type => 'string',
+		optional => 1,
+	    },
+	    cores => {
+		type => 'integer',
+		optional => 1,
+	    },
+	    memory => {
+		type => 'integer',
+		optional => 1,
+	    },
+	}),
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $manifest = $param->{manifest};
+	die "$manifest: non-existent or non-regular file\n" if (! -f $manifest);
+
+	my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest, 0, 1);
+	my $result;
+	$result->{cores} = $parsed->{qm}->{cores};
+	$result->{name} =  $parsed->{qm}->{name};
+	$result->{memory} = $parsed->{qm}->{memory};
+	my $disks = $parsed->{disks};
+	foreach my $disk (@$disks) {
+	    $result->{$disk->{disk_address}} = $disk->{backing_file};
+	}
+	return $result;
+    }});
+
+1;
\ No newline at end of file
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 1c0b5c2..131c0b6 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -300,7 +300,7 @@ my $confdesc = {
 	optional => 1,
 	type => 'string',
 	description => "Lock/unlock the VM.",
-	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
+	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)],
     },
     cpulimit => {
 	optional => 1,
@@ -985,19 +985,41 @@ PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_
 sub verify_volume_id_or_qm_path {
     my ($volid, $noerr) = @_;
 
-    if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
-	return $volid;
-    }
+    return $volid eq 'none' || $volid eq 'cdrom' ?
+	$volid :
+	verify_volume_id_or_absolute_path($volid, $noerr);
+}
+
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+    my ($volid, $noerr) = @_;
+
+    return $volid if $volid =~ m|^/|;
 
-    # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
     if ($@) {
-	return if $noerr;
+	return undef if $noerr;
 	die $@;
     }
     return $volid;
 }
 
+PVE::JSONSchema::register_format('device-image-pair', \&verify_device_image_pair);
+sub verify_device_image_pair {
+    my ($pair, $noerr) = @_;
+
+    my $error = sub {
+	return undef if $noerr;
+	die $@;
+    };
+
+    my ($device, $image) = split('=', $pair);
+    $error->("Invalid device '$device'") if !PVE::QemuServer::Drive::is_valid_drivename($device);
+    $error->("Invalid image '$image'") if !verify_volume_id_or_absolute_path($image);
+
+    return $pair;
+}
+
 my $usb_fmt = {
     host => {
 	default_key => 1,
@@ -2030,6 +2052,22 @@ sub json_config_properties {
     return $prop;
 }
 
+# Properties that we can read from an OVF file
+sub json_ovf_properties {
+    my $prop = shift;
+
+    foreach my $device ( PVE::QemuServer::Drive::valid_drive_names()) {
+	$prop->{$device} = {
+	    type => 'string',
+	    format => 'pve-volume-id-or-absolute-path',
+	    description => "Disk image that gets imported to $device",
+	    optional => 1,
+	};
+    }
+
+    return $prop;
+}
+
 # return copy of $confdesc_cloudinit to generate documentation
 sub cloudinit_config_properties {
 
@@ -6748,7 +6786,7 @@ sub qemu_img_convert {
 	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
 	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
 	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
-    } elsif (-f $src_volid) {
+    } elsif (-f $src_volid || -b $src_volid) {
 	$src_path = $src_volid;
 	if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
 	    $src_format = $1;
diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
index c76c199..48146e9 100644
--- a/PVE/QemuServer/OVF.pm
+++ b/PVE/QemuServer/OVF.pm
@@ -87,7 +87,7 @@ sub id_to_pve {
 
 # returns two references, $qm which holds qm.conf style key/values, and \@disks
 sub parse_ovf {
-    my ($ovf, $debug) = @_;
+    my ($ovf, $debug, $manifest_only) = @_;
 
     my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
 
@@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
 	    die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
 	}
 
-	my $virtual_size;
-	if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
-	    die "error parsing $backing_file_path, size seems to be $virtual_size\n";
+	my $virtual_size = undef;
+	if (!$manifest_only) { # Not possible if manifest is uploaded in web gui
+	    if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
+		die "error parsing $backing_file_path: Could not get file size info: $@\n";
+	    }
 	}
 
 	$pve_disk = {
-- 
2.20.1