all lists on lists.proxmox.com
 help / color / mirror / Atom feed
* [pve-devel] [PATCH v5 qemu-server] Add API for import wizards
@ 2021-03-05 11:11 Dominic Jäger
  2021-03-05 11:11 ` [pve-devel] [PATCH v5 storage] Optionally allow blockdev in abs_filesystem_path Dominic Jäger
  2021-03-05 11:11 ` [pve-devel] [PATCH v5 manager] gui: Add import for disk & VM Dominic Jäger
  0 siblings, 2 replies; 3+ messages in thread
From: Dominic Jäger @ 2021-03-05 11:11 UTC (permalink / raw)
  To: pve-devel

Extend qm importdisk/importovf functionality to the API.

Signed-off-by: Dominic Jäger <d.jaeger@proxmox.com>
---
v4->v5: Feedback by Fabian Grünbichler
Use more existing helpers, parse more, change signature of import helper
function, more detailed errors esp. for API, ...

Remaining todo: Some error cases still require handling & double check if I've
missed something

 PVE/API2/Qemu.pm      | 435 +++++++++++++++++++++++++++++++++++++++++-
 PVE/QemuServer.pm     |  16 +-
 PVE/QemuServer/OVF.pm |  10 +-
 3 files changed, 454 insertions(+), 7 deletions(-)

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index feb9ea8..2efbca9 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -45,7 +45,6 @@ BEGIN {
     }
 }
 
-use Data::Dumper; # fixme: remove
 
 use base qw(PVE::RESTHandler);
 
@@ -4330,4 +4329,438 @@ __PACKAGE__->register_method({
 	return PVE::QemuServer::Cloudinit::dump_cloudinit_config($conf, $param->{vmid}, $param->{type});
     }});
 
+# Raise exception if $format is not supported by $storeid
+my $check_format_is_supported = sub {
+    my ($format, $storeid, $storecfg) = @_;
+    die "You have to provide storage ID" if !$storeid;
+    die "You have to provide the storage configurration" if !$storecfg;
+
+    return if !$format;
+
+    my (undef, $valid_formats) = PVE::Storage::storage_default_format($storecfg, $storeid);
+    my $supported = grep { $_ eq $format } @$valid_formats;
+
+    die "$format is not supported on storage $storeid" if !$supported;
+};
+
+# storecfg ... PVE::Storage::config()
+# vmid ... target VM ID
+# vmconf ... target VM configuration
+# source ... source image (volid or absolute path)
+# target ... hash with
+#    storeid => storage ID
+#    format => disk format (optional)
+#    options => string with device options (may or may not contain <storeid>:0)
+#    device => device where the disk is attached (for example, scsi3) (optional)
+#
+# returns ... volid of the allocated disk image (e.g. local-lvm:vm-100-disk-2)
+my $import_disk_image = sub {
+    my ($param) = @_;
+    my $storecfg = $param->{storecfg};
+    my $vmid = $param->{vmid};
+    my $vmconf = $param->{vmconf};
+    my $target = $param->{target};
+    my $requested_format = $target->{format};
+    my $storeid = $target->{storeid};
+
+    die "Source parameter is undefined!" if !defined $param->{source};
+    my $source = PVE::Storage::abs_filesystem_path($storecfg, $param->{source}, 1);
+
+    eval { PVE::Storage::storage_config($storecfg, $storeid) };
+    die "Error while importing disk image $source: $@\n" if $@;
+
+    my $src_size = PVE::Storage::file_size_info($source);
+    # Previous abs_filesystem_path performs additional checks
+    die "Could not get file size of $source" if !defined($src_size);
+
+    $check_format_is_supported->($requested_format, $storeid, $storecfg);
+
+    my $dst_format = PVE::QemuServer::resolve_dst_disk_format(
+	$storecfg, $storeid, undef, $requested_format);
+    my $dst_volid = PVE::Storage::vdisk_alloc($storecfg, $storeid,
+	$vmid, $dst_format, undef, $src_size / 1024);
+
+    print "Importing disk image '$source'...\n";
+    eval {
+	local $SIG{INT} =
+	local $SIG{TERM} =
+	local $SIG{QUIT} =
+	local $SIG{HUP} =
+	local $SIG{PIPE} = sub { die "Interrupted by signal $!\n"; };
+
+	my $zeroinit = PVE::Storage::volume_has_feature($storecfg,
+	    'sparseinit', $dst_volid);
+
+	PVE::Storage::activate_volumes($storecfg, [$dst_volid]);
+	PVE::QemuServer::qemu_img_convert($source, $dst_volid,
+	$src_size, undef, $zeroinit);
+	PVE::Storage::deactivate_volumes($storecfg, [$dst_volid]);
+
+    };
+    if (my $err = $@) {
+	eval { PVE::Storage::vdisk_free($storecfg, $dst_volid) };
+	warn "Cleanup of $dst_volid failed: $@ \n" if $@;
+
+	die "Importing disk '$source' failed: $err\n" if $err;
+    }
+
+    my $drive = $dst_volid;
+    if ($target->{device}) {
+	# Attach to target device with options if they are specified
+	if (defined $target->{options}) {
+	    # Options string with or without storeid is allowed
+	    # => Avoid potential duplicate storeid for update
+	    $target->{options} =~ s/$storeid:0,//;
+	    $drive .= ",$target->{options}" ;
+	}
+    } else {
+	$target->{device} = PVE::QemuConfig->add_unused_volume($vmconf, $dst_volid);
+    }
+    print "Imported '$source' to $dst_volid\n";
+    $update_vm_api->(
+	{
+	    node => $target->{node},
+	    vmid => $vmid,
+	    $target->{device} => $drive,
+	    skiplock => 1,
+	},
+	1,
+    );
+
+    return $dst_volid;
+};
+
+__PACKAGE__->register_method ({
+    name => 'importdisk',
+    path => '{vmid}/importdisk',
+    method => 'POST',
+    proxyto => 'node',
+    protected => 1,
+    description => "Import an external disk image into a VM. The image format ".
+	"has to be supported by qemu-img.",
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    vmid => get_standard_option('pve-vmid',
+		{completion => \&PVE::QemuServer::complete_vmid}),
+	    source => {
+		description => "Disk image to import. Can be a volid ".
+		    "(local:99/imageToImport.raw) or an absolute path on the server.",
+		type => 'string',
+		format => 'pve-volume-id-or-absolute-path',
+	    },
+	    device => {
+		type => 'string',
+		description => "Bus/Device type of the new disk (e.g. 'ide0', ".
+		    "'scsi2'). Will add the image as unused disk if omitted.",
+		enum => [PVE::QemuServer::Drive::valid_drive_names()],
+		optional => 1,
+	    },
+	    device_options => {
+		type => 'string',
+		description => "Options to set for the new disk (e.g. 'discard=on,backup=0')",
+		optional => 1,
+		requires => 'device',
+	    },
+	    storage => get_standard_option('pve-storage-id', {
+		description => "The storage to which the image will be imported to.",
+		completion => \&PVE::QemuServer::complete_storage,
+	    }),
+	    format => {
+		type => 'string',
+		description => 'Target format.',
+		enum => [ 'raw', 'qcow2', 'vmdk' ],
+		optional => 1,
+	    },
+	    digest => get_standard_option('pve-config-digest'),
+	},
+    },
+    returns => { type => 'string'},
+    code => sub {
+	my ($param) = @_;
+	my $vmid = extract_param($param, 'vmid');
+	my $node = extract_param($param, 'node');
+	my $source = extract_param($param, 'source');
+	my $digest_param = extract_param($param, 'digest');
+	my $device_options = extract_param($param, 'device_options');
+	my $device = extract_param($param, 'device');
+	my $storeid = extract_param($param, 'storage');
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $authuser = $rpcenv->get_user();
+	my $storecfg = PVE::Storage::config();
+	PVE::Storage::storage_config($storecfg, $storeid); # check for errors
+
+	# Format can be set explicitly "--format vmdk"
+	# or as part of device options "--device_options discard=on,format=vmdk"
+	# or not at all, but not both together
+	my $device_options_format;
+	if ($device_options) {
+	    # parse device_options string according to disk schema for
+	    # validation and to make usage easier
+
+	    # any existing storage ID is OK to get a valid (fake) string for parse_drive
+	    my $valid_string = "$storeid:0,$device_options";
+
+	    # member "file" is fake
+	    my $drive_full = PVE::QemuServer::Drive::parse_drive($device, $valid_string);
+	    $device_options_format = $drive_full->{format};
+	}
+
+	my $format = extract_param($param, 'format'); # may be undefined
+	if ($device_options) {
+	    if ($format && $device_options_format) {
+		raise_param_exc({format => "Format already specified in device_options!"});
+	    } else {
+		$format = $format || $device_options_format; # may still be undefined
+	    }
+	}
+
+	$check_format_is_supported->($format, $storeid, $storecfg);
+
+	# provide a useful error (in the API response) before forking
+	my $no_lock_conf = PVE::QemuConfig->load_config($vmid);
+	PVE::QemuConfig->check_lock($no_lock_conf);
+	PVE::Tools::assert_if_modified($no_lock_conf->{digest}, $digest_param);
+	if ($device && $no_lock_conf->{$device}) {
+	    die "Could not import because device $device is already in ".
+		"use in VM $vmid. Choose a different device!";
+	}
+
+	my $worker = sub {
+	    my $conf;
+	    PVE::QemuConfig->lock_config($vmid, sub {
+		$conf = PVE::QemuConfig->load_config($vmid);
+		PVE::QemuConfig->check_lock($conf);
+
+		# Our device-in-use check may be invalid if the new conf is different
+		PVE::Tools::assert_if_modified($conf->{digest}, $no_lock_conf->{digest});
+
+		PVE::QemuConfig->set_lock($vmid, 'import');
+	    });
+
+	   my $target = {
+		node => $node,
+		storeid => $storeid,
+	    };
+	    # Avoid keys with undef values
+	    $target->{format} = $format if defined $format;
+	    $target->{device} = $device if defined $device;
+	    $target->{options} = $device_options if defined $device_options;
+	    $import_disk_image->({
+		storecfg => $storecfg,
+		vmid => $vmid,
+		vmconf => $conf,
+		source => $source,
+		target => $target,
+	    });
+
+	    PVE::QemuConfig->remove_lock($vmid, 'import');
+	};
+	return $rpcenv->fork_worker('importdisk', $vmid, $authuser, $worker);
+    }});
+
+__PACKAGE__->register_method({
+    name => 'importvm',
+    path => '{vmid}/importvm',
+    method => 'POST',
+    description => "Import a VM from existing disk images.",
+    protected => 1,
+    proxyto => 'node',
+    parameters => {
+	additionalProperties => 0,
+	properties => PVE::QemuServer::json_config_properties(
+	    {
+		node => get_standard_option('pve-node'),
+		vmid => get_standard_option('pve-vmid', { completion =>
+		    \&PVE::Cluster::complete_next_vmid }),
+		diskimages => {
+		    description => "Mapping of devices to disk images. For " .
+			"example, scsi0=/mnt/nfs/image1.vmdk,scsi1=/mnt/nfs/image2",
+		    type => 'string',
+		},
+		start => {
+		    optional => 1,
+		    type => 'boolean',
+		    default => 0,
+		    description => "Start VM after it was imported successfully.",
+		},
+	    }),
+    },
+    returns => {
+	type => 'string',
+    },
+    code => sub {
+	my ($param) = @_;
+	my $node = extract_param($param, 'node');
+	my $vmid = extract_param($param, 'vmid');
+	my $diskimages_string = extract_param($param, 'diskimages');
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+	my $authuser = $rpcenv->get_user();
+	my $storecfg = PVE::Storage::config();
+
+	PVE::Cluster::check_cfs_quorum();
+
+	# Return true iff $opt is to be imported, that means it is a device 
+	# like scsi2 and the special import syntax/zero is specified for it,
+	# for example local-lvm:0 but not local-lvm:5
+	my $is_import = sub {
+	    my ($opt) = @_;
+	    return 0 if $opt eq 'efidisk0';
+	    if (PVE::QemuServer::Drive::is_valid_drivename($opt)) {
+		my $drive = PVE::QemuServer::parse_drive($opt, $param->{$opt});
+		return $drive->{file} =~ m/0$/;
+	    }
+	    return 0;
+	};
+
+	# bus/device like ide0, scsi5 where the imported disk images get attached
+	my $target_devices = [];
+	# List of VM parameters like memory, cpu type, but also disks that are newly created
+	my $vm_params = [];
+	foreach my $opt (keys %$param) {
+	    next if ($opt eq 'start'); # does not belong in config
+	    my $list = $is_import->($opt) ? $target_devices : $vm_params;
+	    push @$list, $opt;
+	}
+
+	my $diskimages = {};
+	foreach (split ',', $diskimages_string) {
+	    my ($device, $diskimage) = split('=', $_);
+	    $diskimages->{$device} = $diskimage;
+	    if (defined $param->{$device}) {
+		my $drive = PVE::QemuServer::parse_drive($device, $param->{$device});
+		$drive->{file} =~ m/(\d)$/;
+		if ($1 != 0) {
+		    raise_param_exc({
+			$device => "Each entry of --diskimages must have a ".
+			    "corresponding device with special import syntax " .
+			    "(e.g. --scsi3 local-lvm:0). $device from " .
+			    "--diskimages has a matching device but the size " .
+			    "of that is $1 instead of 0!",
+		    });
+		}
+	    } else {
+		# It is possible to also create new empty disk images during
+		# import by adding something like scsi2=local:10, therefore
+		# vice-versa check is not required
+		raise_param_exc({
+		    diskimages => "There must be a matching device for each " .
+			"--diskimages entry that should be imported, but " .
+			"there is no matching device for $device!\n" .
+			" For example, for --diskimages scsi0=/source/path,scsi1=/other/path " .
+			"there must be --scsi0 local-lvm:0,discard=on --scsi1 local:0,cache=unsafe",
+		    });
+	    }
+	    # Dies if $diskimage cannot be found
+	    PVE::Storage::abs_filesystem_path($storecfg, $diskimage, 1);
+	}
+	foreach my $device (@$target_devices) {
+	    my $drive = PVE::QemuServer::parse_drive($device, $param->{$device});
+	    if ($drive->{file} =~ m/0$/) {
+		if (!defined $diskimages->{$device}) {
+		    raise_param_exc({
+			$device => "Each device with the special import " .
+			    "syntax (the 0) must have a corresponding in " .
+			    "--diskimages that specifies the source of the " .
+			    "import, but there is no such entry for $device!",
+		    });
+		}
+	    }
+	}
+
+	eval { PVE::QemuConfig->create_and_lock_config($vmid, 0, 'import') };
+	die "Unable to create config for VM import: $@" if $@;
+
+	my $worker = sub {
+	    my $get_conf = sub {
+		my ($vmid) = @_;
+		PVE::QemuConfig->lock_config($vmid, sub {
+		    my $conf = PVE::QemuConfig->load_config($vmid);
+		    if (PVE::QemuConfig->has_lock($conf, 'import')) {
+			return $conf;
+		    } else {
+			die "import lock in VM $vmid config file missing!";
+		    }
+		});
+	    };
+
+	    $get_conf->($vmid); # quick check for lock
+
+	    my $short_update = {
+		node => $node,
+		vmid => $vmid,
+		skiplock => 1,
+	    };
+	    foreach ( @$vm_params ) {
+		$short_update->{$_} = $param->{$_};
+	    }
+	    $update_vm_api->($short_update, 1); # writes directly in config file
+
+	    my $conf = $get_conf->($vmid);
+
+	    # When all short updates were succesfull, then the long imports
+	    my @imported_successfully = ();
+	    eval { foreach my $device (@$target_devices) {
+		my $param_parsed = PVE::QemuServer::parse_drive($device, $param->{$device});
+		die "Parsing $param->{$device} failed" if !$param_parsed;
+
+		my $imported = $import_disk_image->({
+		    storecfg => $storecfg,
+		    vmid => $vmid,
+		    vmconf => $conf,
+		    source => $diskimages->{$device},
+		    target => {
+			storeid => (split ':', $param_parsed->{file})[0],
+			format => $param_parsed->{format},
+			options => $param->{$device},
+			device => $device,
+		    },
+		});
+		push @imported_successfully, $imported;
+	    }};
+	    my $err = $@;
+	    if ($err) {
+		foreach my $volid (@imported_successfully) {
+		    eval { PVE::Storage::vdisk_free($storecfg, $volid) };
+		    warn $@ if $@;
+		}
+
+		eval {
+		    my $conffile = PVE::QemuConfig->config_file($vmid);
+		    unlink($conffile) or die "Failed to remove config file: $!";
+		};
+		warn $@ if $@;
+
+		die "Import aborted: $err";
+	    } else {
+		$conf = $get_conf->($vmid); # import_disk_image changed config file directly
+		if (!$conf->{boot}) {
+		    my $bootdevs = PVE::QemuServer::get_default_bootdevices($conf);
+		    $update_vm_api->(
+			{
+			    node => $node,
+			    vmid => $vmid,
+			    boot => PVE::QemuServer::print_bootorder($bootdevs),
+			    skiplock => 1,
+			},
+			1,
+		    );
+		}
+
+		eval { PVE::QemuConfig->remove_lock($vmid, 'import') };
+		warn $@ if $@;
+
+		if ($param->{start}) {
+		    PVE::QemuServer::vm_start($storecfg, $vmid);
+		}
+	    }
+	};
+
+	return $rpcenv->fork_worker('importvm', $vmid, $authuser, $worker);
+    }});
+
+
 1;
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 4a433a5..3262a0c 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -300,7 +300,7 @@ my $confdesc = {
 	optional => 1,
 	type => 'string',
 	description => "Lock/unlock the VM.",
-	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
+	enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended import)],
     },
     cpulimit => {
 	optional => 1,
@@ -998,6 +998,18 @@ sub verify_volume_id_or_qm_path {
     return $volid;
 }
 
+PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
+sub verify_volume_id_or_absolute_path {
+    my ($volid, $noerr) = @_;
+
+    # Exactly these 2 are allowed in id_or_qm_path but should not be allowed here
+    if ($volid eq 'none' || $volid eq 'cdrom') {
+	return undef if $noerr;
+	die "Invalid format! Should be volume ID or absolute path.";
+    }
+    return verify_volume_id_or_qm_path($volid, $noerr);
+}
+
 my $usb_fmt = {
     host => {
 	default_key => 1,
@@ -6700,7 +6712,7 @@ sub qemu_img_convert {
 	$src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
 	$src_is_iscsi = ($src_path =~ m|^iscsi://|);
 	$cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
-    } elsif (-f $src_volid) {
+    } elsif (-f $src_volid || -b _) { # -b required to import from LVM images
 	$src_path = $src_volid;
 	if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
 	    $src_format = $1;
diff --git a/PVE/QemuServer/OVF.pm b/PVE/QemuServer/OVF.pm
index c76c199..36b7fff 100644
--- a/PVE/QemuServer/OVF.pm
+++ b/PVE/QemuServer/OVF.pm
@@ -87,7 +87,7 @@ sub id_to_pve {
 
 # returns two references, $qm which holds qm.conf style key/values, and \@disks
 sub parse_ovf {
-    my ($ovf, $debug) = @_;
+    my ($ovf, $debug, $ignore_size) = @_;
 
     my $dom = XML::LibXML->load_xml(location => $ovf, no_blanks => 1);
 
@@ -220,9 +220,11 @@ ovf:Item[rasd:InstanceID='%s']/rasd:ResourceType", $controller_id);
 	    die "error parsing $filepath, file seems not to exist at $backing_file_path\n";
 	}
 
-	my $virtual_size;
-	if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
-	    die "error parsing $backing_file_path, size seems to be $virtual_size\n";
+	my $virtual_size = 0;
+	if (!$ignore_size) { # Not possible if manifest is uploaded in web gui
+	    if ( !($virtual_size = PVE::Storage::file_size_info($backing_file_path)) ) {
+		die "error parsing $backing_file_path: Could not get file size info: $@\n";
+	    }
 	}
 
 	$pve_disk = {
-- 
2.20.1




^ permalink raw reply	[flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-03-05 11:18 UTC | newest]

Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-05 11:11 [pve-devel] [PATCH v5 qemu-server] Add API for import wizards Dominic Jäger
2021-03-05 11:11 ` [pve-devel] [PATCH v5 storage] Optionally allow blockdev in abs_filesystem_path Dominic Jäger
2021-03-05 11:11 ` [pve-devel] [PATCH v5 manager] gui: Add import for disk & VM Dominic Jäger

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.
Service provided by Proxmox Server Solutions GmbH | Privacy | Legal