[pve-devel] Extend API for cloud-init MTU and userdata

[pve-devel] Extend API for cloud-init MTU and userdata

[Marius Schellenberger]

To fully automate virtual machine creation and configuration via the
Proxmox API, I added configuration support for MTU and userdata to the
cloud-init options.

[pve-devel] [PATCH] api: cloud-init support for mtu and userdata

[Marius Schellenberger]

Extended the PVE API to configure cloud-init userdata and network
interface MTU.

Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
---
 PVE/API2/Qemu.pm            |  1 +
 PVE/QemuServer.pm           | 18 +++++++++++++++++-
 PVE/QemuServer/Cloudinit.pm | 31 +++++++++++++++++++++++++++++--
 3 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
index 8da616a..5d149a5 100644
--- a/PVE/API2/Qemu.pm
+++ b/PVE/API2/Qemu.pm
@@ -324,6 +324,7 @@ my $cloudinitoptions = {
     cipassword => 1,
     citype => 1,
     ciuser => 1,
+    ciuserdata => 1,
     nameserver => 1,
     searchdomain => 1,
     sshkeys => 1,
diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index a9c0dac..974a070 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -727,6 +727,12 @@ my $confdesc_cloudinit = {
 	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 	format => 'pve-qm-cicustom',
     },
+    ciuserdata => {
+        optional => 1,
+        type => 'string',
+        format => 'urlencoded',
+        description => 'cloud-init: Specify custom user-data as urlencoded base64 string to replace the automatically generated one at start. When set, the following options have no effect: `ciuser`, `cipassword`, `sshkeys`',
+    },
     searchdomain => {
 	optional => 1,
 	type => 'string',
@@ -932,6 +938,12 @@ my $ipconfig_fmt = {
 	optional => 1,
 	requires => 'ip6',
     },
+    mtu => {
+	type => 'string',
+	format_description => 'MTU',
+	description => 'MTU value for interface.',
+	optional => 1,
+    },
 };
 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 my $ipconfigdesc = {
@@ -1701,7 +1713,7 @@ sub parse_net {
     return $res;
 }
 
-# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
+# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip,mtu=mtu
 sub parse_ipconfig {
     my ($data) = @_;
 
@@ -1711,6 +1723,10 @@ sub parse_ipconfig {
 	return undef;
     }
 
+    if ($res->{mtu} && !$res->{ip} && !$res->{ip6}) {
+	warn 'mtu specified without specifying an IP or IPv6 address';
+	return undef;
+    }
     if ($res->{gw} && !$res->{ip}) {
 	warn 'gateway specified without specifying an IP address';
 	return undef;
diff --git a/PVE/QemuServer/Cloudinit.pm b/PVE/QemuServer/Cloudinit.pm
index 439de99..a47e0ce 100644
--- a/PVE/QemuServer/Cloudinit.pm
+++ b/PVE/QemuServer/Cloudinit.pm
@@ -6,6 +6,7 @@ use warnings;
 use File::Path;
 use Digest::SHA;
 use URI::Escape;
+use MIME::Base64 qw(decode_base64);
 
 use PVE::Tools qw(run_command file_set_contents);
 use PVE::Storage;
@@ -111,6 +112,11 @@ sub get_dns_conf {
 sub cloudinit_userdata {
     my ($conf, $vmid) = @_;
 
+    my $userdata = $conf->{ciuserdata};
+    if (defined($userdata) && $userdata ne "") {
+        return decode_base64(uri_unescape($userdata));
+    }
+
     my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid);
 
     my $content = "#cloud-config\n";
@@ -176,7 +182,9 @@ sub configdrive2_network {
 	(my $id = $iface) =~ s/^net//;
 	next if !$conf->{"ipconfig$id"};
 	my $net = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($net);
 	$id = "eth$id";
+	my $mtu = $net->{mtu};
 
 	$content .="auto $id\n";
 	if ($net->{ip}) {
@@ -189,6 +197,9 @@ sub configdrive2_network {
 		$content .= "        netmask $mask\n";
 		$content .= "        gateway $net->{gw}\n" if $net->{gw};
 	    }
+	    if (defined($mtu) && $mtu ne "") {
+		$content .= "        mtu $mtu\n";
+	    }
 	}
 	if ($net->{ip6}) {
 	    if ($net->{ip6} =~ /^(auto|dhcp)$/) {
@@ -200,6 +211,9 @@ sub configdrive2_network {
 		$content .= "        netmask $mask\n";
 		$content .= "        gateway $net->{gw6}\n" if $net->{gw6};
 	    }
+	    if (defined($mtu) && $mtu ne "") {
+		$content .= "        mtu $mtu\n";
+	    }
 	}
     }
 
@@ -261,6 +275,7 @@ sub nocloud_network_v2 {
 
 	my $net = PVE::QemuServer::parse_net($conf->{$iface});
 	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($ipconfig);
 
 	my $mac = $net->{macaddr}
 	    or die "network interface '$iface' has no mac address\n";
@@ -295,6 +310,10 @@ sub nocloud_network_v2 {
 	if (defined(my $gw = $ipconfig->{gw6})) {
 	    $content .= "${i}gateway6: '$gw'\n";
 	}
+	my $mtu = $ipconfig->{mtu};
+	if (defined($mtu) && $mtu ne "") {
+	    $content .= "${i}mtu: $mtu\n";
+	}
 
 	next if $dns_done;
 	$dns_done = 1;
@@ -332,14 +351,22 @@ sub nocloud_network {
 
 	my $net = PVE::QemuServer::parse_net($conf->{$iface});
 	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
+	next if !defined($ipconfig);
 
 	my $mac = lc($net->{macaddr})
 	    or die "network interface '$iface' has no mac address\n";
 
+	my $mtu = $ipconfig->{mtu};
+
 	$content .= "${i}- type: physical\n"
 	          . "${i}  name: eth$id\n"
-	          . "${i}  mac_address: '$mac'\n"
-	          . "${i}  subnets:\n";
+	          . "${i}  mac_address: '$mac'\n";
+
+	if (defined($mtu) && $mtu ne "") {
+	    $content .= "${i}  mtu: $mtu\n";
+	}
+
+	$content .= "${i}  subnets:\n";
 	$i .= '  ';
 	if (defined(my $ip = $ipconfig->{ip})) {
 	    if ($ip eq 'dhcp') {
-- 
2.27.0

[pve-devel] [PATCH] ui: added cloud-init mtu and userdata options

[Marius Schellenberger]

Added options to configure the cloud-init MTU and userdata fields
via the Web-UI.

Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
---
 www/manager6/Makefile             |  1 +
 www/manager6/Parser.js            |  6 ++
 www/manager6/Utils.js             |  8 +++
 www/manager6/qemu/CloudInit.js    | 10 ++++
 www/manager6/qemu/IPConfigEdit.js |  6 ++
 www/manager6/qemu/UserDataEdit.js | 94 +++++++++++++++++++++++++++++++
 6 files changed, 125 insertions(+)
 create mode 100644 www/manager6/qemu/UserDataEdit.js

diff --git a/www/manager6/Makefile b/www/manager6/Makefile
index 4288acdd..096454fd 100644
--- a/www/manager6/Makefile
+++ b/www/manager6/Makefile
@@ -201,6 +201,7 @@ JSSRC= 							\
 	qemu/Smbios1Edit.js				\
 	qemu/SystemEdit.js				\
 	qemu/USBEdit.js					\
+	qemu/UserDataEdit.js				\
 	sdn/Browser.js					\
 	sdn/ControllerView.js				\
 	sdn/Status.js					\
diff --git a/www/manager6/Parser.js b/www/manager6/Parser.js
index b793a28e..580ed2ec 100644
--- a/www/manager6/Parser.js
+++ b/www/manager6/Parser.js
@@ -275,6 +275,8 @@ Ext.define('PVE.Parser', { statics: {
 		res.ip6 = match_res[1];
 	    } else if ((match_res = p.match(/^gw6=(\S+)$/)) !== null) {
 		res.gw6 = match_res[1];
+	    } else if ((match_res = p.match(/^mtu=(\S+)$/)) !== null) {
+		res.mtu = match_res[1];
 	    } else {
 		errors = true;
 		return false; // break
@@ -307,6 +309,10 @@ Ext.define('PVE.Parser', { statics: {
 	    str += c + "gw6=" + cfg.gw6;
 	    c = ",";
 	}
+	if (cfg.mtu) {
+	    str += c + "mtu=" + cfg.mtu;
+	    c = ",";
+	}
 	return str;
     },
 
diff --git a/www/manager6/Utils.js b/www/manager6/Utils.js
index bf9ceda9..7b3ffec3 100644
--- a/www/manager6/Utils.js
+++ b/www/manager6/Utils.js
@@ -1351,6 +1351,14 @@ Ext.define('PVE.Utils', { utilities: {
 	reader.readAsText(file);
     },
 
+    loadUserDataFromFile: function(file, callback) {
+	var reader = new FileReader();
+	reader.onload = function(evt) {
+	    callback(evt.target.result);
+	};
+	reader.readAsText(file);
+    },
+
     diskControllerMaxIDs: {
 	ide: 4,
 	sata: 6,
diff --git a/www/manager6/qemu/CloudInit.js b/www/manager6/qemu/CloudInit.js
index a588f09b..ac960672 100644
--- a/www/manager6/qemu/CloudInit.js
+++ b/www/manager6/qemu/CloudInit.js
@@ -253,6 +253,16 @@ Ext.define('PVE.qemu.CloudInit', {
 		never_delete: true,
 		defaultValue: gettext('use host settings')
 	    },
+	    ciuserdata: {
+		header: 'User-Data',
+		iconCls: 'fa fa-code',
+		editor: caps.vms['VM.Config.Network'] ? 'PVE.qemu.UserDataEdit' : undefined,
+		never_delete: true,
+		defaultValue: '',
+		renderer: function(value) {
+		    return value || Proxmox.Utils.noneText;
+		}
+	    },
 	    sshkeys: {
 		header: gettext('SSH public key'),
 		iconCls: 'fa fa-key',
diff --git a/www/manager6/qemu/IPConfigEdit.js b/www/manager6/qemu/IPConfigEdit.js
index 934a86be..f1e2fc92 100644
--- a/www/manager6/qemu/IPConfigEdit.js
+++ b/www/manager6/qemu/IPConfigEdit.js
@@ -66,6 +66,12 @@ Ext.define('PVE.qemu.IPConfigPanel', {
 		fieldLabel: gettext('Network Device'),
 		value: me.netid
 	    },
+	    {
+		xtype: 'textfield',
+		name: 'mtu',
+		value: me.ipconfig.mtu,
+		fieldLabel: 'MTU'
+	    },
 	    {
 		layout: {
 		    type: 'hbox',
diff --git a/www/manager6/qemu/UserDataEdit.js b/www/manager6/qemu/UserDataEdit.js
new file mode 100644
index 00000000..465f5b8f
--- /dev/null
+++ b/www/manager6/qemu/UserDataEdit.js
@@ -0,0 +1,94 @@
+Ext.define('PVE.qemu.UserDataInputPanel', {
+    extend: 'Proxmox.panel.InputPanel',
+    xtype: 'pveQemuUserDataInputPanel',
+
+    insideWizard: false,
+
+    onGetValues: function(values) {
+	var me = this;
+	if (!values.ciuserdata.length) {
+	    values = {};
+	    values['delete'] = 'ciuserdata';
+	    return values;
+	} else {
+	    var fileheader = "#cloud-config"
+	    var data = values.ciuserdata.split('\n');
+	    if (data[0] != fileheader) {
+	        values.ciuserdata = fileheader + '\n' + values.ciuserdata;
+	    }
+	    values.ciuserdata = encodeURIComponent(btoa(values.ciuserdata));
+	}
+	return values;
+    },
+
+    items: [
+	{
+	    xtype: 'textarea',
+	    itemId: 'ciuserdata',
+	    name: 'ciuserdata',
+	    height: 250
+	},
+	{
+	    xtype: 'filebutton',
+	    itemId: 'filebutton',
+	    name: 'file',
+	    text: gettext('Load User-Data File'),
+	    fieldLabel: 'test',
+	    listeners: {
+		change: function(btn, e, value) {
+		    var me = this.up('inputpanel');
+		    e = e.event;
+		    Ext.Array.each(e.target.files, function(file) {
+			PVE.Utils.loadUserDataFromFile(file, function(res) {
+			    var dataField = me.down('#ciuserdata');
+			    var old = dataField.getValue();
+			    dataField.setValue(old + '\n' + res);
+			});
+		    });
+		    btn.reset();
+		}
+	    }
+	}
+    ],
+
+    initComponent: function() {
+	var me = this;
+
+	me.callParent();
+	if (!window.FileReader) {
+	    me.down('#filebutton').setVisible(false);
+	}
+
+    }
+});
+
+Ext.define('PVE.qemu.UserDataEdit', {
+    extend: 'Proxmox.window.Edit',
+
+    width: 800,
+
+    initComponent : function() {
+	var me = this;
+
+	var ipanel = Ext.create('PVE.qemu.UserDataInputPanel');
+
+	Ext.apply(me, {
+	    subject: 'User-Data',
+	    items: [ ipanel ]
+	});
+
+	me.callParent();
+
+	if (!me.create) {
+	    me.load({
+		success: function(response, options) {
+		    var data = response.result.data;
+		    if (data.ciuserdata) {
+			data.ciuserdata = atob(decodeURIComponent(data.ciuserdata));
+			ipanel.setValues(data);
+		    }
+		}
+	    });
+	}
+    }
+});
-- 
2.27.0

Re: [pve-devel] [PATCH] api: cloud-init support for mtu and userdata

[Mira Limbeck]

Thank you for the patch.

Seems to be the exactly the same as the one you sent some time ago: 
https://lists.proxmox.com/pipermail/pve-devel/2020-July/044241.html

Was this intentional?


On 8/11/20 9:02 PM, Marius Schellenberger wrote:
> Extended the PVE API to configure cloud-init userdata and network
> interface MTU.
>
> Signed-off-by: Marius Schellenberger <proxmox@giftfish.de>
> ---
>   PVE/API2/Qemu.pm            |  1 +
>   PVE/QemuServer.pm           | 18 +++++++++++++++++-
>   PVE/QemuServer/Cloudinit.pm | 31 +++++++++++++++++++++++++++++--
>   3 files changed, 47 insertions(+), 3 deletions(-)
>
> diff --git a/PVE/API2/Qemu.pm b/PVE/API2/Qemu.pm
> index 8da616a..5d149a5 100644
> --- a/PVE/API2/Qemu.pm
> +++ b/PVE/API2/Qemu.pm
> @@ -324,6 +324,7 @@ my $cloudinitoptions = {
>       cipassword => 1,
>       citype => 1,
>       ciuser => 1,
> +    ciuserdata => 1,
>       nameserver => 1,
>       searchdomain => 1,
>       sshkeys => 1,
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index a9c0dac..974a070 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -727,6 +727,12 @@ my $confdesc_cloudinit = {
>   	description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
>   	format => 'pve-qm-cicustom',
>       },
> +    ciuserdata => {
> +        optional => 1,
> +        type => 'string',
> +        format => 'urlencoded',
> +        description => 'cloud-init: Specify custom user-data as urlencoded base64 string to replace the automatically generated one at start. When set, the following options have no effect: `ciuser`, `cipassword`, `sshkeys`',
> +    },
>       searchdomain => {
>   	optional => 1,
>   	type => 'string',
> @@ -932,6 +938,12 @@ my $ipconfig_fmt = {
>   	optional => 1,
>   	requires => 'ip6',
>       },
> +    mtu => {
> +	type => 'string',
> +	format_description => 'MTU',
> +	description => 'MTU value for interface.',
> +	optional => 1,
> +    },
>   };
>   PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
>   my $ipconfigdesc = {
> @@ -1701,7 +1713,7 @@ sub parse_net {
>       return $res;
>   }
>   
> -# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
> +# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip,mtu=mtu
>   sub parse_ipconfig {
>       my ($data) = @_;
>   
> @@ -1711,6 +1723,10 @@ sub parse_ipconfig {
>   	return undef;
>       }
>   
> +    if ($res->{mtu} && !$res->{ip} && !$res->{ip6}) {
> +	warn 'mtu specified without specifying an IP or IPv6 address';
> +	return undef;
> +    }
>       if ($res->{gw} && !$res->{ip}) {
>   	warn 'gateway specified without specifying an IP address';
>   	return undef;
> diff --git a/PVE/QemuServer/Cloudinit.pm b/PVE/QemuServer/Cloudinit.pm
> index 439de99..a47e0ce 100644
> --- a/PVE/QemuServer/Cloudinit.pm
> +++ b/PVE/QemuServer/Cloudinit.pm
> @@ -6,6 +6,7 @@ use warnings;
>   use File::Path;
>   use Digest::SHA;
>   use URI::Escape;
> +use MIME::Base64 qw(decode_base64);
>   
>   use PVE::Tools qw(run_command file_set_contents);
>   use PVE::Storage;
> @@ -111,6 +112,11 @@ sub get_dns_conf {
>   sub cloudinit_userdata {
>       my ($conf, $vmid) = @_;
>   
> +    my $userdata = $conf->{ciuserdata};
> +    if (defined($userdata) && $userdata ne "") {
> +        return decode_base64(uri_unescape($userdata));
> +    }
> +
>       my ($hostname, $fqdn) = get_hostname_fqdn($conf, $vmid);
>   
>       my $content = "#cloud-config\n";
> @@ -176,7 +182,9 @@ sub configdrive2_network {
>   	(my $id = $iface) =~ s/^net//;
>   	next if !$conf->{"ipconfig$id"};
>   	my $net = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($net);
>   	$id = "eth$id";
> +	my $mtu = $net->{mtu};
>   
>   	$content .="auto $id\n";
>   	if ($net->{ip}) {
> @@ -189,6 +197,9 @@ sub configdrive2_network {
>   		$content .= "        netmask $mask\n";
>   		$content .= "        gateway $net->{gw}\n" if $net->{gw};
>   	    }
> +	    if (defined($mtu) && $mtu ne "") {
> +		$content .= "        mtu $mtu\n";
> +	    }
>   	}
>   	if ($net->{ip6}) {
>   	    if ($net->{ip6} =~ /^(auto|dhcp)$/) {
> @@ -200,6 +211,9 @@ sub configdrive2_network {
>   		$content .= "        netmask $mask\n";
>   		$content .= "        gateway $net->{gw6}\n" if $net->{gw6};
>   	    }
> +	    if (defined($mtu) && $mtu ne "") {
> +		$content .= "        mtu $mtu\n";
> +	    }
>   	}
>       }
>   
> @@ -261,6 +275,7 @@ sub nocloud_network_v2 {
>   
>   	my $net = PVE::QemuServer::parse_net($conf->{$iface});
>   	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($ipconfig);
>   
>   	my $mac = $net->{macaddr}
>   	    or die "network interface '$iface' has no mac address\n";
> @@ -295,6 +310,10 @@ sub nocloud_network_v2 {
>   	if (defined(my $gw = $ipconfig->{gw6})) {
>   	    $content .= "${i}gateway6: '$gw'\n";
>   	}
> +	my $mtu = $ipconfig->{mtu};
> +	if (defined($mtu) && $mtu ne "") {
> +	    $content .= "${i}mtu: $mtu\n";
> +	}
>   
>   	next if $dns_done;
>   	$dns_done = 1;
> @@ -332,14 +351,22 @@ sub nocloud_network {
>   
>   	my $net = PVE::QemuServer::parse_net($conf->{$iface});
>   	my $ipconfig = PVE::QemuServer::parse_ipconfig($conf->{"ipconfig$id"});
> +	next if !defined($ipconfig);
>   
>   	my $mac = lc($net->{macaddr})
>   	    or die "network interface '$iface' has no mac address\n";
>   
> +	my $mtu = $ipconfig->{mtu};
> +
>   	$content .= "${i}- type: physical\n"
>   	          . "${i}  name: eth$id\n"
> -	          . "${i}  mac_address: '$mac'\n"
> -	          . "${i}  subnets:\n";
> +	          . "${i}  mac_address: '$mac'\n";
> +
> +	if (defined($mtu) && $mtu ne "") {
> +	    $content .= "${i}  mtu: $mtu\n";
> +	}
> +
> +	$content .= "${i}  subnets:\n";
>   	$i .= '  ';
>   	if (defined(my $ip = $ipconfig->{ip})) {
>   	    if ($ip eq 'dhcp') {