[pve-devel] Extend cloud-init API with MTU and userdata

[pve-devel] Extend cloud-init API with MTU and userdata

[Marius Schellenberger]

To fully automate virtual machine creation and configuration via the
Proxmox API, I added configuration support for MTU and userdata to the
cloud-init options.

These options can now be configured using the API or Web-UI and are
stored in the qemu-server/*.conf configuration file, removing the need
for an extra userdata snippets file volume.

[pve-devel] [PATCH qemu-server] api: cloud-init support for mtu and userdata

[Marius Schellenberger]

Extended the PVE API to configure cloud-init userdata and network
interface MTU.

Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
---
 PVE/API2/Qemu.pm            |  1 +
 PVE/QemuServer.pm           | 18 +++++++++++++++++-
 PVE/QemuServer/Cloudinit.pm | 31 +++++++++++++++++++++++++++++--
 3 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index b33359d..02507de 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -324,6 +324,7 @@ my $cloudinitoptions = {
     cipassword => 1,
     citype => 1,
     ciuser => 1,
+    ciuserdata => 1,
     nameserver => 1,
     searchdomain => 1,
     sshkeys => 1,
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 6872e06..ef4342d 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -727,6 +727,12 @@ my $confdesc_cloudinit = {
 	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 	format => 'pve-qm-cicustom',
     },
+    ciuserdata => {
+        optional => 1,
+        type => 'string',
+        format => 'urlencoded',
+        description => 'cloud-init: Specify custom user-data as urlencoded base64 string to replace the automatically generated one at start. When set, the following options have no effect: `ciuser`, `cipassword`, `sshkeys`',
+    },
     searchdomain => {
 	optional => 1,
 	type => 'string',
@@ -932,6 +938,12 @@ my $ipconfig_fmt = {
 	optional => 1,
 	requires => 'ip6',
     },
+    mtu => {
+	type => 'string',
+	format_description => 'MTU',
+	description => 'MTU value for interface.',
+	optional => 1,
+    },
 };
 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 my $ipconfigdesc = {
@@ -1701,7 +1713,7 @@ sub parse_net {
     return $res;
 }
 
-# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
+# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip,mtu=mtu
 sub parse_ipconfig {
     my ($data) = @_;
 
@@ -1711,6 +1723,10 @@ sub parse_ipconfig {
 	return undef;
     }
 
+    if ($res->{mtu} && !$res->{ip} && !$res->{ip6}) {
+	warn 'mtu specified without specifying an IP or IPv6 address';
+	return undef;
+    }
     if ($res->{gw} && !$res->{ip}) {
 	warn 'gateway specified without specifying an IP address';
 	return undef;
diff --git a/PVE/QemuServer/Cloudinit.pm b/PVE/QemuServer/Cloudinit.pm
index 439de99..a47e0ce 100644
--- a/PVE/QemuServer/Cloudinit.pm
+++ b/PVE/QemuServer/Cloudinit.pm
@@ -6,6 +6,7 @@ use warnings;
 use File::Path;
 use Digest::SHA;
 use URI::Escape;
+use MIME::Base64 qw(decode_base64);
 
 use PVE::Tools qw(run_command file_set_contents);
 use PVE::Storage;
@@ -111,6 +112,11 @@ sub get_dns_conf {
 sub cloudinit_userdata {
     my ($conf, $vmid) = @_;
 
+    my $userdata = $conf->{ciuserdata};
+    if (defined($userdata) && $userdata ne "") {
+        return decode_base64(uri_unescape($userdata));
+    }
+
     my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid);
 
     my $content = "#cloud-config\n";
@@ -176,7 +182,9 @@ sub configdrive2_network {
 	(my $id = $iface) =~ s/^net//;
 	next if !$conf->{"ipconfig$id"};
 	my $net = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($net);
 	$id = "eth$id";
+	my $mtu = $net->{mtu};
 
 	$content .="auto $id\n";
 	if ($net->{ip}) {
@@ -189,6 +197,9 @@ sub configdrive2_network {
 		$content .= "        netmask $mask\n";
 		$content .= "        gateway $net->{gw}\n" if $net->{gw};
 	    }
+	    if (defined($mtu) && $mtu ne "") {
+		$content .= "        mtu $mtu\n";
+	    }
 	}
 	if ($net->{ip6}) {
 	    if ($net->{ip6} =~ /^(auto|dhcp)$/) {
@@ -200,6 +211,9 @@ sub configdrive2_network {
 		$content .= "        netmask $mask\n";
 		$content .= "        gateway $net->{gw6}\n" if $net->{gw6};
 	    }
+	    if (defined($mtu) && $mtu ne "") {
+		$content .= "        mtu $mtu\n";
+	    }
 	}
     }
 
@@ -261,6 +275,7 @@ sub nocloud_network_v2 {
 
 	my $net = PVE::QemuServer::parse_net($conf->{$iface});
 	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($ipconfig);
 
 	my $mac = $net->{macaddr}
 	    or die "network interface '$iface' has no mac address\n";
@@ -295,6 +310,10 @@ sub nocloud_network_v2 {
 	if (defined(my $gw = $ipconfig->{gw6})) {
 	    $content .= "${i}gateway6: '$gw'\n";
 	}
+	my $mtu = $ipconfig->{mtu};
+	if (defined($mtu) && $mtu ne "") {
+	    $content .= "${i}mtu: $mtu\n";
+	}
 
 	next if $dns_done;
 	$dns_done = 1;
@@ -332,14 +351,22 @@ sub nocloud_network {
 
 	my $net = PVE::QemuServer::parse_net($conf->{$iface});
 	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($ipconfig);
 
 	my $mac = lc($net->{macaddr})
 	    or die "network interface '$iface' has no mac address\n";
 
+	my $mtu = $ipconfig->{mtu};
+
 	$content .= "${i}- type: physical\n"
 	          . "${i}  name: eth$id\n"
-	          . "${i}  mac_address: '$mac'\n"
-	          . "${i}  subnets:\n";
+	          . "${i}  mac_address: '$mac'\n";
+
+	if (defined($mtu) && $mtu ne "") {
+	    $content .= "${i}  mtu: $mtu\n";
+	}
+
+	$content .= "${i}  subnets:\n";
 	$i .= '  ';
 	if (defined(my $ip = $ipconfig->{ip})) {
 	    if ($ip eq 'dhcp') {
-- 
2.27.0

[pve-devel] [PATCH manager] ui: added cloud-init mtu and userdata options

[Marius Schellenberger]

Added options to configure the cloud-init MTU and userdata fields
via the Web-UI.

Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
---
 www/manager6/Makefile             |  1 +
 www/manager6/Parser.js            |  6 ++
 www/manager6/Utils.js             |  8 +++
 www/manager6/qemu/CloudInit.js    | 10 ++++
 www/manager6/qemu/IPConfigEdit.js |  6 ++
 www/manager6/qemu/UserDataEdit.js | 94 +++++++++++++++++++++++++++++++
 6 files changed, 125 insertions(+)
 create mode 100644 www/manager6/qemu/UserDataEdit.js

diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index ff452184..4453c637 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -152,6 +152,7 @@ JSSRC= 				                 	\
 	qemu/Config.js					\
 	qemu/CreateWizard.js				\
 	qemu/USBEdit.js					\
+	qemu/UserDataEdit.js				\
 	qemu/PCIEdit.js					\
 	qemu/SerialEdit.js				\
 	qemu/AgentIPView.js				\
diff --git a/www/manager6/Parser.js b/www/manager6/Parser.js
index b793a28e..580ed2ec 100644
--- a/www/manager6/Parser.js
+++ b/www/manager6/Parser.js
@@ -275,6 +275,8 @@ Ext.define('PVE.Parser', { statics: {
 		res.ip6 = match_res[1];
 	    } else if ((match_res = p.match(/^gw6=(\S+)$/)) !== null) {
 		res.gw6 = match_res[1];
+	    } else if ((match_res = p.match(/^mtu=(\S+)$/)) !== null) {
+		res.mtu = match_res[1];
 	    } else {
 		errors = true;
 		return false; // break
@@ -307,6 +309,10 @@ Ext.define('PVE.Parser', { statics: {
 	    str += c + "gw6=" + cfg.gw6;
 	    c = ",";
 	}
+	if (cfg.mtu) {
+	    str += c + "mtu=" + cfg.mtu;
+	    c = ",";
+	}
 	return str;
     },
 
diff --git a/www/manager6/Utils.js b/www/manager6/Utils.js
index 1dae292e..bd6aae4e 100644
--- a/www/manager6/Utils.js
+++ b/www/manager6/Utils.js
@@ -1260,6 +1260,14 @@ Ext.define('PVE.Utils', { utilities: {
 	reader.readAsText(file);
     },
 
+    loadUserDataFromFile: function(file, callback) {
+	var reader = new FileReader();
+	reader.onload = function(evt) {
+	    callback(evt.target.result);
+	};
+	reader.readAsText(file);
+    },
+
     diskControllerMaxIDs: {
 	ide: 4,
 	sata: 6,
diff --git a/www/manager6/qemu/CloudInit.js b/www/manager6/qemu/CloudInit.js
index a588f09b..ac960672 100644
--- a/www/manager6/qemu/CloudInit.js
+++ b/www/manager6/qemu/CloudInit.js
@@ -253,6 +253,16 @@ Ext.define('PVE.qemu.CloudInit', {
 		never_delete: true,
 		defaultValue: gettext('use host settings')
 	    },
+	    ciuserdata: {
+		header: 'User-Data',
+		iconCls: 'fa fa-code',
+		editor: caps.vms['VM.Config.Network'] ? 'PVE.qemu.UserDataEdit' : undefined,
+		never_delete: true,
+		defaultValue: '',
+		renderer: function(value) {
+		    return value || Proxmox.Utils.noneText;
+		}
+	    },
 	    sshkeys: {
 		header: gettext('SSH public key'),
 		iconCls: 'fa fa-key',
diff --git a/www/manager6/qemu/IPConfigEdit.js b/www/manager6/qemu/IPConfigEdit.js
index 934a86be..f1e2fc92 100644
--- a/www/manager6/qemu/IPConfigEdit.js
+++ b/www/manager6/qemu/IPConfigEdit.js
@@ -66,6 +66,12 @@ Ext.define('PVE.qemu.IPConfigPanel', {
 		fieldLabel: gettext('Network Device'),
 		value: me.netid
 	    },
+	    {
+		xtype: 'textfield',
+		name: 'mtu',
+		value: me.ipconfig.mtu,
+		fieldLabel: 'MTU'
+	    },
 	    {
 		layout: {
 		    type: 'hbox',
diff --git a/www/manager6/qemu/UserDataEdit.js b/www/manager6/qemu/UserDataEdit.js
new file mode 100644
index 00000000..465f5b8f
--- /dev/null
+++ b/www/manager6/qemu/UserDataEdit.js
@@ -0,0 +1,94 @@
+Ext.define('PVE.qemu.UserDataInputPanel', {
+    extend: 'Proxmox.panel.InputPanel',
+    xtype: 'pveQemuUserDataInputPanel',
+
+    insideWizard: false,
+
+    onGetValues: function(values) {
+	var me = this;
+	if (!values.ciuserdata.length) {
+	    values = {};
+	    values['delete'] = 'ciuserdata';
+	    return values;
+	} else {
+	    var fileheader = "#cloud-config"
+	    var data = values.ciuserdata.split('\n');
+	    if (data[0] != fileheader) {
+	        values.ciuserdata = fileheader + '\n' + values.ciuserdata;
+	    }
+	    values.ciuserdata = encodeURIComponent(btoa(values.ciuserdata));
+	}
+	return values;
+    },
+
+    items: [
+	{
+	    xtype: 'textarea',
+	    itemId: 'ciuserdata',
+	    name: 'ciuserdata',
+	    height: 250
+	},
+	{
+	    xtype: 'filebutton',
+	    itemId: 'filebutton',
+	    name: 'file',
+	    text: gettext('Load User-Data File'),
+	    fieldLabel: 'test',
+	    listeners: {
+		change: function(btn, e, value) {
+		    var me = this.up('inputpanel');
+		    e = e.event;
+		    Ext.Array.each(e.target.files, function(file) {
+			PVE.Utils.loadUserDataFromFile(file, function(res) {
+			    var dataField = me.down('#ciuserdata');
+			    var old = dataField.getValue();
+			    dataField.setValue(old + '\n' + res);
+			});
+		    });
+		    btn.reset();
+		}
+	    }
+	}
+    ],
+
+    initComponent: function() {
+	var me = this;
+
+	me.callParent();
+	if (!window.FileReader) {
+	    me.down('#filebutton').setVisible(false);
+	}
+
+    }
+});
+
+Ext.define('PVE.qemu.UserDataEdit', {
+    extend: 'Proxmox.window.Edit',
+
+    width: 800,
+
+    initComponent : function() {
+	var me = this;
+
+	var ipanel = Ext.create('PVE.qemu.UserDataInputPanel');
+
+	Ext.apply(me, {
+	    subject: 'User-Data',
+	    items: [ ipanel ]
+	});
+
+	me.callParent();
+
+	if (!me.create) {
+	    me.load({
+		success: function(response, options) {
+		    var data = response.result.data;
+		    if (data.ciuserdata) {
+			data.ciuserdata = atob(decodeURIComponent(data.ciuserdata));
+			ipanel.setValues(data);
+		    }
+		}
+	    });
+	}
+    }
+});
-- 
2.27.0

Re: [pve-devel] Extend cloud-init API with MTU and userdata

[Mira Limbeck]

Hi,

Thank you for the patch series.

I'm currently taking a look and will provide comments on the individual 
patches.


On 7/2/20 6:08 PM, Marius Schellenberger wrote:
>
> To fully automate virtual machine creation and configuration via the
> Proxmox API, I added configuration support for MTU and userdata to the
> cloud-init options.
>
> These options can now be configured using the API or Web-UI and are
> stored in the qemu-server/*.conf configuration file, removing the need
> for an extra userdata snippets file volume.
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>
>

Re: [pve-devel] [PATCH qemu-server] api: cloud-init support for mtu and userdata

[Mira Limbeck]

The MTU part of the patch looks good and works fine here. It would be 
great if you could send that as a separate patch when you send a v2 of 
the patch series.


Any reason why snippets [0] are not enough for custom userdata?


Also we don't want any further cloudinit options in the global config 
scope. Instead we use cicustom for new cloudinit options. There you 
should be able to extend the user part to support custom userdata as a 
urlencoded base64 string instead of only as a volume id (snippet). This 
can be done by registering a new format 
(PVE::JSONSchema::register_format) for it with a validator function that 
accepts both a pve-volume-id as well as the userdata string and then 
using this format in cicustom_fmt.


[0] https://pve.proxmox.com/pve-docs/pve-admin-guide.html#qm_cloud_init 
(10.8.3 Custom Cloud-Init Configuration)

On 7/2/20 6:08 PM, Marius Schellenberger wrote:
> Extended the PVE API to configure cloud-init userdata and network
> interface MTU.
>
> Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
> ---
>   PVE/API2/Qemu.pm            |  1 +
>   PVE/QemuServer.pm           | 18 +++++++++++++++++-
>   PVE/QemuServer/Cloudinit.pm | 31 +++++++++++++++++++++++++++++--
>   3 files changed, 47 insertions(+), 3 deletions(-)
>
> diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
> index b33359d..02507de 100644
> --- a/PVE/API2/Qemu.pm
> +++ b/PVE/API2/Qemu.pm
> @@ -324,6 +324,7 @@ my $cloudinitoptions = {
>       cipassword => 1,
>       citype => 1,
>       ciuser => 1,
> +    ciuserdata => 1,
>       nameserver => 1,
>       searchdomain => 1,
>       sshkeys => 1,
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 6872e06..ef4342d 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -727,6 +727,12 @@ my $confdesc_cloudinit = {
>   	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
>   	format => 'pve-qm-cicustom',
>       },
> +    ciuserdata => {
> +        optional => 1,
> +        type => 'string',
> +        format => 'urlencoded',
> +        description => 'cloud-init: Specify custom user-data as urlencoded base64 string to replace the automatically generated one at start. When set, the following options have no effect: `ciuser`, `cipassword`, `sshkeys`',
> +    },
>       searchdomain => {
>   	optional => 1,
>   	type => 'string',
> @@ -932,6 +938,12 @@ my $ipconfig_fmt = {
>   	optional => 1,
>   	requires => 'ip6',
>       },
> +    mtu => {
> +	type => 'string',
> +	format_description => 'MTU',
> +	description => 'MTU value for interface.',
> +	optional => 1,
> +    },
>   };
>   PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
>   my $ipconfigdesc = {
> @@ -1701,7 +1713,7 @@ sub parse_net {
>       return $res;
>   }
>   
> -# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
> +# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip,mtu=mtu
>   sub parse_ipconfig {
>       my ($data) = @_;
>   
> @@ -1711,6 +1723,10 @@ sub parse_ipconfig {
>   	return undef;
>       }
>   
> +    if ($res->{mtu} && !$res->{ip} && !$res->{ip6}) {
> +	warn 'mtu specified without specifying an IP or IPv6 address';
> +	return undef;
> +    }
>       if ($res->{gw} && !$res->{ip}) {
>   	warn 'gateway specified without specifying an IP address';
>   	return undef;
> diff --git a/PVE/QemuServer/Cloudinit.pm b/PVE/QemuServer/Cloudinit.pm
> index 439de99..a47e0ce 100644
> --- a/PVE/QemuServer/Cloudinit.pm
> +++ b/PVE/QemuServer/Cloudinit.pm
> @@ -6,6 +6,7 @@ use warnings;
>   use File::Path;
>   use Digest::SHA;
>   use URI::Escape;
> +use MIME::Base64 qw(decode_base64);
>   
>   use PVE::Tools qw(run_command file_set_contents);
>   use PVE::Storage;
> @@ -111,6 +112,11 @@ sub get_dns_conf {
>   sub cloudinit_userdata {
>       my ($conf, $vmid) = @_;
>   
> +    my $userdata = $conf->{ciuserdata};
> +    if (defined($userdata) && $userdata ne "") {
> +        return decode_base64(uri_unescape($userdata));
> +    }
> +
>       my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid);
>   
>       my $content = "#cloud-config\n";
> @@ -176,7 +182,9 @@ sub configdrive2_network {
>   	(my $id = $iface) =~ s/^net//;
>   	next if !$conf->{"ipconfig$id"};
>   	my $net = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($net);
>   	$id = "eth$id";
> +	my $mtu = $net->{mtu};
>   
>   	$content .="auto $id\n";
>   	if ($net->{ip}) {
> @@ -189,6 +197,9 @@ sub configdrive2_network {
>   		$content .= "        netmask $mask\n";
>   		$content .= "        gateway $net->{gw}\n" if $net->{gw};
>   	    }
> +	    if (defined($mtu) && $mtu ne "") {
> +		$content .= "        mtu $mtu\n";
> +	    }
>   	}
>   	if ($net->{ip6}) {
>   	    if ($net->{ip6} =~ /^(auto|dhcp)$/) {
> @@ -200,6 +211,9 @@ sub configdrive2_network {
>   		$content .= "        netmask $mask\n";
>   		$content .= "        gateway $net->{gw6}\n" if $net->{gw6};
>   	    }
> +	    if (defined($mtu) && $mtu ne "") {
> +		$content .= "        mtu $mtu\n";
> +	    }
>   	}
>       }
>   
> @@ -261,6 +275,7 @@ sub nocloud_network_v2 {
>   
>   	my $net = PVE::QemuServer::parse_net($conf->{$iface});
>   	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($ipconfig);
>   
>   	my $mac = $net->{macaddr}
>   	    or die "network interface '$iface' has no mac address\n";
> @@ -295,6 +310,10 @@ sub nocloud_network_v2 {
>   	if (defined(my $gw = $ipconfig->{gw6})) {
>   	    $content .= "${i}gateway6: '$gw'\n";
>   	}
> +	my $mtu = $ipconfig->{mtu};
> +	if (defined($mtu) && $mtu ne "") {
> +	    $content .= "${i}mtu: $mtu\n";
> +	}
>   
>   	next if $dns_done;
>   	$dns_done = 1;
> @@ -332,14 +351,22 @@ sub nocloud_network {
>   
>   	my $net = PVE::QemuServer::parse_net($conf->{$iface});
>   	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($ipconfig);
>   
>   	my $mac = lc($net->{macaddr})
>   	    or die "network interface '$iface' has no mac address\n";
>   
> +	my $mtu = $ipconfig->{mtu};
> +
>   	$content .= "${i}- type: physical\n"
>   	          . "${i}  name: eth$id\n"
> -	          . "${i}  mac_address: '$mac'\n"
> -	          . "${i}  subnets:\n";
> +	          . "${i}  mac_address: '$mac'\n";
> +
> +	if (defined($mtu) && $mtu ne "") {
> +	    $content .= "${i}  mtu: $mtu\n";
> +	}
> +
> +	$content .= "${i}  subnets:\n";
>   	$i .= '  ';
>   	if (defined(my $ip = $ipconfig->{ip})) {
>   	    if ($ip eq 'dhcp') {