[pve-devel] [PATCH] Add API for VM import

* [pve-devel] [PATCH] Add API for VM import
@ 2021-04-12 10:08 Dominic Jäger
  2021-04-12 10:08 ` [pve-devel] [PATCH] Add GUI to import disk & VM Dominic Jäger
  2021-04-22 20:06 ` [pve-devel] [PATCH] Add API for VM import Thomas Lamprecht
  0 siblings, 2 replies; 4+ messages in thread
From: Dominic Jäger @ 2021-04-12 10:08 UTC (permalink / raw)
  To: pve-devel

Extend qm importdisk/importovf functionality to the API.

Co-authored-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
Signed-off-by: Dominic Jäger <d.jaeger@proxmox.com>
---
v8:
- Fabian moved the import functions into the existing create_vm / update_vm_api
- Dropped the separate API endpoints & import lock

 PVE/API2/Qemu.pm       | 175 +++++++++++++++++++++++++++++++++++++++--
 PVE/API2/Qemu/Makefile |   2 +-
 PVE/API2/Qemu/OVF.pm   |  68 ++++++++++++++++
 PVE/QemuServer.pm      |  50 ++++++++++--
 PVE/QemuServer/OVF.pm  |  10 ++-
 5 files changed, 289 insertions(+), 16 deletions(-)
 create mode 100644 PVE/API2/Qemu/OVF.pm

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index c56b609..5c31756 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -45,7 +45,6 @@ BEGIN {
     }
 }
 
-use Data::Dumper; # fixme: remove
 
 use base qw(PVE::RESTHandler);
 
@@ -61,7 +60,19 @@ my $resolve_cdrom_alias = sub {
     }
 };
 
+my $parse_import_sources = sub {
+    my $param = shift;
+    my $import = {};
+    foreach my $pair (PVE::Tools::split_list($param)) {
+	my ($device, $diskimage) = split('=', $pair);
+	$import->{$device} = $diskimage;
+    }
+
+    return $import;
+};
+
 my $NEW_DISK_RE = qr!^(([^/:\s]+):)?(\d+(\.\d+)?)$!;
+my $IMPORT_DISK_RE = qr!^([^/:\s]+):-1$!;
 my $check_storage_access = sub {
    my ($rpcenv, $authuser, $storecfg, $vmid, $settings, $default_storage) = @_;
 
@@ -84,6 +95,12 @@ my $check_storage_access = sub {
 	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
 	    raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
 		if !$scfg->{content}->{images};
+	} elsif ($volid =~ $IMPORT_DISK_RE) {
+	    my $storeid = $1;
+	    $rpcenv->check($authuser, "/storage/$storeid", ['Datastore.AllocateSpace']);
+	    my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+	    raise_param_exc({ storage => "storage '$storeid' does not support vm images"})
+		if !$scfg->{content}->{images};
 	} else {
 	    PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
 	}
@@ -91,6 +108,13 @@ my $check_storage_access = sub {
 
    $rpcenv->check($authuser, "/storage/$settings->{vmstatestorage}", ['Datastore.AllocateSpace'])
        if defined($settings->{vmstatestorage});
+
+    if (defined($settings->{import_sources})) {
+	my $images = $parse_import_sources->($settings->{import_sources});
+	foreach my $source_image (values %$images) {
+	    PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $source_image);
+	}
+    }
 };
 
 my $check_storage_access_clone = sub {
@@ -133,10 +157,25 @@ my $check_storage_access_clone = sub {
    return $sharedvm;
 };
 
+# Raise exception if $format is not supported by $storeid
+my $check_format_is_supported = sub {
+    my ($format, $storeid, $storecfg) = @_;
+    die "storage ID parameter must be passed to the sub" if !$storeid;
+    die "storage configuration must be passed to the sub" if !$storecfg;
+
+    return if !$format;
+
+    my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+    my $supported = grep { $_ eq $format } @$valid_formats;
+
+    die "format '$format' is not supported on storage $storeid" if !$supported;
+};
+
+
 # Note: $pool is only needed when creating a VM, because pool permissions
 # are automatically inherited if VM already exists inside a pool.
 my $create_disks = sub {
-    my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage) = @_;
+    my ($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $settings, $default_storage, $import) = @_;
 
     my $vollist = [];
 
@@ -192,6 +231,69 @@ my $create_disks = sub {
 	    $disk->{size} = PVE::Tools::convert_size($size, 'kb' => 'b');
 	    delete $disk->{format}; # no longer needed
 	    $res->{$ds} = PVE::QemuServer::print_drive($disk);
+	} elsif ($volid =~ $IMPORT_DISK_RE) {
+	    my $target_storage = $1;
+
+	    my $source = $import->{$ds};
+	    die "cannot import '$ds', no import source defined\n" if !$source;
+	    $source = PVE::Storage::abs_filesystem_path($storecfg, $source, 1);
+	    my $src_size = PVE::Storage::file_size_info($source);
+	    die "Could not get file size of $source" if !defined($src_size);
+
+	    $check_format_is_supported->($disk->{format}, $storeid, $storecfg);
+
+	    my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+		$storecfg,
+		$storeid,
+		undef,
+		$disk->{format},
+	    );
+	    my $dst_volid = PVE::Storage::vdisk_alloc(
+		$storecfg,
+		$storeid,
+		$vmid,
+		$dst_format,
+		undef,
+		PVE::Tools::convert_size($src_size, 'b' => 'kb'),
+	    );
+
+	    print "Importing disk image '$source' as '$dst_volid'...\n";
+	    eval {
+		local $SIG{INT} =
+		local $SIG{TERM} =
+		local $SIG{QUIT} =
+		local $SIG{HUP} =
+		local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+		my $zeroinit = PVE::Storage::volume_has_feature(
+		    $storecfg,
+		    'sparseinit',
+		    $dst_volid,
+		);
+		PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
+		PVE::QemuServer::qemu_img_convert(
+		    $source,
+		    $dst_volid,
+		    $src_size,
+		    undef,
+		    $zeroinit,
+		);
+		PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
+
+	    };
+	    if (my $err = $@) {
+		eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
+		warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+		die "Importing disk '$source' failed: $err\n" if $err;
+	    }
+	    push @$vollist, $dst_volid;
+	    $disk->{file} = $dst_volid;
+	    if ($ds !~ /^unused\d+$/) {
+		$disk->{size} = $src_size;
+		delete $disk->{format}; # no longer needed
+	    }
+	    $res->{$ds} = PVE::QemuServer::print_drive($disk);
 	} else {
 
 	    PVE::Storage::check_volume_access($rpcenv, $authuser, $storecfg, $vmid, $volid);
@@ -218,7 +320,7 @@ my $create_disks = sub {
 	}
     };
 
-    eval { PVE::QemuConfig->foreach_volume($settings, $code); };
+    eval { PVE::QemuConfig->foreach_volume_full($settings, { include_unused => 1 }, $code); };
 
     # free allocated images on error
     if (my $err = $@) {
@@ -550,6 +652,13 @@ __PACKAGE__->register_method({
 		    default => 0,
 		    description => "Start VM after it was created successfully.",
 		},
+		import_sources => {
+		    description => "\\0 delimited mapping of devices to disk images to import." .
+			"For example, scsi0=/mnt/nfs/image1.vmdk",
+		    type => 'string',
+		    format => 'device-image-pair-alist',
+		    optional => 1,
+		},
 	    }),
     },
     returns => {
@@ -615,21 +724,34 @@ __PACKAGE__->register_method({
 
 	    &$check_cpu_model_access($rpcenv, $authuser, $param);
 
+	    my $import_devices = $parse_import_sources->($param->{import_sources});
+
 	    foreach my $opt (keys %$param) {
 		if (PVE::QemuServer::is_valid_drivename($opt)) {
 		    my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
 		    raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+		    raise_param_exc({ $opt => "not marked for import, but import source defined" })
+			if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt};
+		    raise_param_exc({ $opt => "marked for import, but no import source defined" })
+			if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt};
 
 		    PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 		    $param->{$opt} = PVE::QemuServer::print_drive($drive);
 		}
 	    }
+	    foreach my $opt (keys %$import_devices) {
+		raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" })
+		    if !defined($param->{$opt});
+
+	    }
 
 	    PVE::QemuServer::add_random_macs($param);
 	} else {
 	    my $keystr = join(' ', keys %$param);
 	    raise_param_exc({ archive => "option conflicts with other options ($keystr)"}) if $keystr;
 
+	    raise_param_exc({ import_sources => "cannot import existing disk and restore backup." }) if $param->{import_sources};
+
 	    if ($archive eq '-') {
 		die "pipe requires cli environment\n"
 		    if $rpcenv->{type} ne 'cli';
@@ -701,10 +823,22 @@ __PACKAGE__->register_method({
 	    my $realcmd = sub {
 		my $conf = $param;
 		my $arch = PVE::QemuServer::get_vm_arch($conf);
+		my $import = $parse_import_sources->(extract_param($param, "import_sources"));
 
 		my $vollist = [];
 		eval {
-		    $vollist = &$create_disks($rpcenv, $authuser, $conf, $arch, $storecfg, $vmid, $pool, $param, $storage);
+		    $vollist = &$create_disks(
+			$rpcenv,
+			$authuser,
+			$conf,
+			$arch,
+			$storecfg,
+			$vmid,
+			$pool,
+			$param,
+			$storage,
+			$import
+		    );
 
 		    if (!$conf->{boot}) {
 			my $devs = PVE::QemuServer::get_default_bootdevices($conf);
@@ -1181,11 +1315,17 @@ my $update_vm_api  = sub {
 	die "cannot add non-replicatable volume to a replicated VM\n";
     };
 
+    my $import_devices = $parse_import_sources->($param->{import_sources});
+
     foreach my $opt (keys %$param) {
 	if (PVE::QemuServer::is_valid_drivename($opt)) {
 	    # cleanup drive path
 	    my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
 	    raise_param_exc({ $opt => "unable to parse drive options" }) if !$drive;
+	    raise_param_exc({ $opt => "not marked for import, but import source defined" })
+		if $drive->{file} !~ $IMPORT_DISK_RE && $import_devices->{$opt};
+	    raise_param_exc({ $opt => "marked for import, but no import source defined" })
+		if $drive->{file} =~ $IMPORT_DISK_RE && !$import_devices->{$opt};
 	    PVE::QemuServer::cleanup_drive_path($opt, $storecfg, $drive);
 	    $check_replication->($drive);
 	    $param->{$opt} = PVE::QemuServer::print_drive($drive);
@@ -1203,12 +1343,20 @@ my $update_vm_api  = sub {
 	}
     }
 
+    foreach my $opt (keys %$import_devices) {
+	raise_param_exc({ import_sources => "$opt not marked for import, but import source defined" })
+	    if !defined($param->{$opt});
+
+    }
+
     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [@delete]);
 
     &$check_vm_modify_config_perm($rpcenv, $authuser, $vmid, undef, [keys %$param]);
 
     &$check_storage_access($rpcenv, $authuser, $storecfg, $vmid, $param);
 
+    delete $param->{import_sources};
+
     my $updatefn =  sub {
 
 	my $conf = PVE::QemuConfig->load_config($vmid);
@@ -1350,7 +1498,17 @@ my $update_vm_api  = sub {
 		    PVE::QemuServer::vmconfig_register_unused_drive($storecfg, $vmid, $conf, PVE::QemuServer::parse_drive($opt, $conf->{pending}->{$opt}))
 			if defined($conf->{pending}->{$opt});
 
-		    &$create_disks($rpcenv, $authuser, $conf->{pending}, $arch, $storecfg, $vmid, undef, {$opt => $param->{$opt}});
+		    &$create_disks(
+			$rpcenv,
+			$authuser,
+			$conf->{pending},
+			$arch, $storecfg,
+			$vmid,
+			undef,
+			{$opt => $param->{$opt}},
+			undef,
+			{$opt => $import_devices->{$opt}}
+		    );
 		} elsif ($opt =~ m/^serial\d+/) {
 		    if ((!defined($conf->{$opt}) || $conf->{$opt} eq 'socket') && $param->{$opt} eq 'socket') {
 			$rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.HWType']);
@@ -1494,6 +1652,13 @@ __PACKAGE__->register_method({
 		    optional => 1,
 		    requires => 'delete',
 		},
+		import_sources => {
+		    description => "\\0 delimited mapping of devices to disk images to import." .
+			"For example, scsi0=/mnt/nfs/image1.vmdk",
+		    type => 'string',
+		    format => 'device-image-pair-alist',
+		    optional => 1,
+		},
 		digest => {
 		    type => 'string',
 		    description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.',
diff --git a/PVE/API2/Qemu/Makefile b/PVE/API2/Qemu/Makefile
index 5d4abda..bdd4762 100644
--- a/PVE/API2/Qemu/Makefile
+++ b/PVE/API2/Qemu/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Agent.pm CPU.pm Machine.pm
+SOURCES=Agent.pm CPU.pm Machine.pm OVF.pm
 
 .PHONY: install
 install:
diff --git a/PVE/API2/Qemu/OVF.pm b/PVE/API2/Qemu/OVF.pm
new file mode 100644
index 0000000..bd6e90b
--- /dev/null
+++ b/PVE/API2/Qemu/OVF.pm
@@ -0,0 +1,68 @@
+package PVE::API2::Qemu::OVF;
+
+use strict;
+use warnings;
+
+use PVE::RESTHandler;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::QemuServer::OVF;
+
+use base qw(PVE::RESTHandler);
+
+__PACKAGE__->register_method ({
+    name => 'index',
+    path => '',
+    method => 'GET',
+    proxyto => 'node',
+    description => "Read an .ovf manifest.",
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    manifest => {
+		description => ".ovf manifest",
+		type => 'string',
+	    },
+	},
+    },
+    returns => {
+	description => "VM config according to .ovf manifest and digest of manifest",
+	type => "object",
+    },
+    returns => {
+	type => 'object',
+	additionalProperties => 1,
+	properties => PVE::QemuServer::json_ovf_properties({
+	    name => {
+		type => 'string',
+		optional => 1,
+	    },
+	    cores => {
+		type => 'integer',
+		optional => 1,
+	    },
+	    memory => {
+		type => 'integer',
+		optional => 1,
+	    },
+	}),
+    },
+    code => sub {
+	my ($param) = @_;
+
+	my $manifest = $param->{manifest};
+	die "$manifest: non-existent or non-regular file\n" if (! -f $manifest);
+
+	my $parsed = PVE::QemuServer::OVF::parse_ovf($manifest, 0, 1);
+	my $result;
+	$result->{cores} = $parsed->{qm}->{cores};
+	$result->{name} =  $parsed->{qm}->{name};
+	$result->{memory} = $parsed->{qm}->{memory};
+	my $disks = $parsed->{disks};
+	foreach my $disk (@$disks) {
+	    $result->{$disk->{disk_address}} = $disk->{backing_file};
+	}
+	return $result;
+    }});
+
+1;
\ No newline at end of file
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index fdb2ac9..ac2fe2e 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -987,19 +987,41 @@ PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_
 sub verify_volume_id_or_qm_path {
     my ($volid, $noerr) = @_;
 
-    if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
-	return $volid;
-    }
+    return $volid eq 'none' || $volid eq 'cdrom' ?
+	$volid :
+	verify_volume_id_or_absolute_path($volid, $noerr);
+}
+
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+    my ($volid, $noerr) = @_;
+
+    return $volid if $volid =~ m|^/|;
 
-    # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
     if ($@) {
-	return if $noerr;
+	return undef if $noerr;
 	die $@;
     }
     return $volid;
 }
 
+PVE::JSONSchema::register_format('device-image-pair', \&verify_device_image_pair);
+sub verify_device_image_pair {
+    my ($pair, $noerr) = @_;
+
+    my $error = sub {
+	return undef if $noerr;
+	die @_;
+    };
+
+    my ($device, $image) = split('=', $pair);
+    $error->("Invalid device '$device'") if !PVE::QemuServer::Drive::is_valid_drivename($device);
+    $error->("Invalid image '$image'") if !verify_volume_id_or_absolute_path($image);
+
+    return $pair;
+}
+
 my $usb_fmt = {
     host => {
 	default_key => 1,
@@ -2075,6 +2097,22 @@ sub json_config_properties {
     return $prop;
 }
 
+# Properties that we can read from an OVF file
+sub json_ovf_properties {
+    my $prop = shift;
+
+    foreach my $device ( PVE::QemuServer::Drive::valid_drive_names()) {
+	$prop->{$device} = {
+	    type => 'string',
+	    format => 'pve-volume-id-or-absolute-path',
+	    description => "Disk image that gets imported to $device",
+	    optional => 1,
+	};
+    }
+
+    return $prop;
+}
+
 # return copy of $confdesc_cloudinit to generate documentation
 sub cloudinit_config_properties {
 
@@ -6914,7 +6952,7 @@ sub qemu_img_convert {
 	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
 	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
 	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
-    } elsif (-f $src_volid) {
+    } elsif (-f $src_volid || -b $src_volid) {
 	$src_path = $src_volid;
 	if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
 	    $src_format = $1;
diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
index c76c199..48146e9 100644
--- a/PVE/QemuServer/OVF.pm
+++ b/PVE/QemuServer/OVF.pm
@@ -87,7 +87,7 @@ sub id_to_pve {
 
 # returns two references, $qm which holds qm.conf style key/values, and \@disks
 sub parse_ovf {
-    my ($ovf, $debug) = @_;
+    my ($ovf, $debug, $manifest_only) = @_;
 
     my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
 
@@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
 	    die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
 	}
 
-	my $virtual_size;
-	if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
-	    die "error parsing $backing_file_path, size seems to be $virtual_size\n";
+	my $virtual_size = undef;
+	if (!$manifest_only) { # Not possible if manifest is uploaded in web gui
+	    if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
+		die "error parsing $backing_file_path: Could not get file size info: $@\n";
+	    }
 	}
 
 	$pve_disk = {
-- 
2.20.1




^ permalink raw reply	[flat|nested] 4+ messages in thread