[pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 7577 bytes --]

From: Maurice Klein <klein@aetherus.de>
To: pve-devel@lists.proxmox.com
Subject: [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>
Date: Fri,  9 Jan 2026 13:10:49 +0100
Message-ID: <20260109121049.70740-2-klein@aetherus.de>

qemu-server: add routed tap and helper scripts
---
 src/PVE/QemuServer.pm         |  9 +++++-
 src/PVE/QemuServer/Network.pm | 19 +++++++++++
 src/usr/pve-tap               | 59 +++++++++++++++++++++++++++++++++++
 src/usr/pve-tap-hotplug       |  3 ++
 src/usr/pve-tapdown           | 16 ++++++++++
 5 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100755 src/usr/pve-tap
 create mode 100755 src/usr/pve-tap-hotplug
 create mode 100755 src/usr/pve-tapdown

diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 69991843..2c0b784e 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -1443,8 +1443,15 @@ sub print_netdev_full {
 
     my $netdev = "";
     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
+    if ($net->{taprouted}) {
+            $script = $hotplug ? "pve-tap" : "pve-tap-hotplug";
+    }
+
 
-    if ($net->{bridge}) {
+    if ($net->{taprouted}) {
+		    $netdev= "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
+            . ",downscript=/usr/libexec/qemu-server/pve-tapdown$vhostparam";
+    } elsif ($net->{bridge}) {
         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
             . ",downscript=/usr/libexec/qemu-server/pve-bridgedown$vhostparam";
     } else {
diff --git a/src/PVE/QemuServer/Network.pm b/src/PVE/QemuServer/Network.pm
index eb8222e8..c11f002c 100644
--- a/src/PVE/QemuServer/Network.pm
+++ b/src/PVE/QemuServer/Network.pm
@@ -116,6 +116,25 @@ my $net_fmt = {
             "Force MTU of network device (VirtIO only). Setting to '1' or empty will use the bridge MTU",
         optional => 1,
     },
+    taprouted => {
+        type => 'boolean',
+        description => "routed network, just make tap interface and execute routing script",
+        optional => 1,
+    },
+    hostip => {
+        type => 'string',
+        format => 'ipv4',
+        format_description => 'IPv4Format',
+        description => 'IPv4 address for the host.',
+        optional => 1,
+    },
+    guestip => {
+        type => 'string',
+        format => 'ipv4',
+        format_description => 'GuestIPv4',
+        description => 'IPv4 address for the guest.',
+        optional => 1,
+    },
 };
 
 our $netdesc = {
diff --git a/src/usr/pve-tap b/src/usr/pve-tap
new file mode 100755
index 00000000..10623c17
--- /dev/null
+++ b/src/usr/pve-tap
@@ -0,0 +1,59 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use PVE::Tools qw(run_command);
+use PVE::Firewall;
+
+use PVE::QemuServer::Network;
+
+my $iface = shift;
+
+my $hotplug = 0;
+if ($iface eq '--hotplug') {
+    $hotplug = 1;
+    $iface = shift;
+}
+
+die "no interface specified\n" if !$iface;
+
+die "got strange interface name '$iface'\n"
+    if $iface !~ m/^tap(\d+)i(\d+)$/;
+
+my $vmid = $1;
+my $netid = "net$2";
+
+my $migratedfrom = $hotplug ? undef : $ENV{PVE_MIGRATED_FROM};
+
+my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
+
+my $netconf = $conf->{$netid};
+
+$netconf = $conf->{pending}->{$netid} if !$migratedfrom && defined($conf->{pending}->{$netid});
+
+die "unable to get network config '$netid'\n"
+    if !defined($netconf);
+
+my $net = PVE::QemuServer::Network::parse_net($netconf);
+die "unable to parse network config '$netid'\n" if !$net;
+
+
+# Bring up the tap interface
+run_command(['ip', 'link', 'set', $iface, 'up']);
+#set host ip if specified
+if (defined($net->{hostip})) {
+    run_command(['ip', 'addr', 'add', $net->{hostip}, 'dev', $iface]);
+}
+
+#set route to guest if specified
+if (defined($net->{guestip})) {
+run_command(['ip', 'route', 'add', $net->{guestip}, 'dev', $iface]);
+}
+
+
+
+
+
+
+exit 0;
diff --git a/src/usr/pve-tap-hotplug b/src/usr/pve-tap-hotplug
new file mode 100755
index 00000000..6fcdcd2a
--- /dev/null
+++ b/src/usr/pve-tap-hotplug
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /usr/libexec/qemu-server/pve-tap --hotplug "$@"
diff --git a/src/usr/pve-tapdown b/src/usr/pve-tapdown
new file mode 100755
index 00000000..e867b640
--- /dev/null
+++ b/src/usr/pve-tapdown
@@ -0,0 +1,16 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use PVE::Network;
+
+my $iface = shift;
+
+die "no interface specified\n" if !$iface;
+
+die "got strange interface name '$iface'\n"
+    if $iface !~ m/^tap(\d+)i(\d+)$/;
+
+PVE::Network::tap_unplug($iface);
+
+exit 0;
-- 
2.39.5 (Apple Git-154)



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 8526 bytes --]

From: Maurice Klein <klein@aetherus.de>
To: pve-devel@lists.proxmox.com
Subject: Re: [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>
Date: Mon, 19 Jan 2026 09:37:05 +0100
Message-ID: <ec4caf97-a205-4586-b371-89caafc27b06@aetherus.de>

Hi,

just a gentle ping on this series.
Happy to rework or adjust anything if I missed something or did 
something the wrong way.

Thanks,
Maurice


Am 09.01.26 um 13:10 schrieb Maurice Klein:
> qemu-server: add routed tap and helper scripts
> ---
> src/PVE/QemuServer.pm | 9 +++++-
> src/PVE/QemuServer/Network.pm | 19 +++++++++++
> src/usr/pve-tap | 59 +++++++++++++++++++++++++++++++++++
> src/usr/pve-tap-hotplug | 3 ++
> src/usr/pve-tapdown | 16 ++++++++++
> 5 files changed, 105 insertions(+), 1 deletion(-)
> create mode 100755 src/usr/pve-tap
> create mode 100755 src/usr/pve-tap-hotplug
> create mode 100755 src/usr/pve-tapdown
>
> diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
> index 69991843..2c0b784e 100644
> --- a/src/PVE/QemuServer.pm
> +++ b/src/PVE/QemuServer.pm
> @@ -1443,8 +1443,15 @@ sub print_netdev_full {
> my $netdev = "";
> my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
> + if ($net->{taprouted}) {
> + $script = $hotplug ? "pve-tap" : "pve-tap-hotplug";
> + }
> +
> - if ($net->{bridge}) {
> + if ($net->{taprouted}) {
> + $netdev= 
> "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
> + . ",downscript=/usr/libexec/qemu-server/pve-tapdown$vhostparam";
> + } elsif ($net->{bridge}) {
> $netdev = 
> "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
> . ",downscript=/usr/libexec/qemu-server/pve-bridgedown$vhostparam";
> } else {
> diff --git a/src/PVE/QemuServer/Network.pm b/src/PVE/QemuServer/Network.pm
> index eb8222e8..c11f002c 100644
> --- a/src/PVE/QemuServer/Network.pm
> +++ b/src/PVE/QemuServer/Network.pm
> @@ -116,6 +116,25 @@ my $net_fmt = {
> "Force MTU of network device (VirtIO only). Setting to '1' or empty 
> will use the bridge MTU",
> optional => 1,
> },
> + taprouted => {
> + type => 'boolean',
> + description => "routed network, just make tap interface and execute 
> routing script",
> + optional => 1,
> + },
> + hostip => {
> + type => 'string',
> + format => 'ipv4',
> + format_description => 'IPv4Format',
> + description => 'IPv4 address for the host.',
> + optional => 1,
> + },
> + guestip => {
> + type => 'string',
> + format => 'ipv4',
> + format_description => 'GuestIPv4',
> + description => 'IPv4 address for the guest.',
> + optional => 1,
> + },
> };
> our $netdesc = {
> diff --git a/src/usr/pve-tap b/src/usr/pve-tap
> new file mode 100755
> index 00000000..10623c17
> --- /dev/null
> +++ b/src/usr/pve-tap
> @@ -0,0 +1,59 @@
> +#!/usr/bin/perl
> +
> +use strict;
> +use warnings;
> +
> +use PVE::Tools qw(run_command);
> +use PVE::Firewall;
> +
> +use PVE::QemuServer::Network;
> +
> +my $iface = shift;
> +
> +my $hotplug = 0;
> +if ($iface eq '--hotplug') {
> + $hotplug = 1;
> + $iface = shift;
> +}
> +
> +die "no interface specified\n" if !$iface;
> +
> +die "got strange interface name '$iface'\n"
> + if $iface !~ m/^tap(\d+)i(\d+)$/;
> +
> +my $vmid = $1;
> +my $netid = "net$2";
> +
> +my $migratedfrom = $hotplug ? undef : $ENV{PVE_MIGRATED_FROM};
> +
> +my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
> +
> +my $netconf = $conf->{$netid};
> +
> +$netconf = $conf->{pending}->{$netid} if !$migratedfrom && 
> defined($conf->{pending}->{$netid});
> +
> +die "unable to get network config '$netid'\n"
> + if !defined($netconf);
> +
> +my $net = PVE::QemuServer::Network::parse_net($netconf);
> +die "unable to parse network config '$netid'\n" if !$net;
> +
> +
> +# Bring up the tap interface
> +run_command(['ip', 'link', 'set', $iface, 'up']);
> +#set host ip if specified
> +if (defined($net->{hostip})) {
> + run_command(['ip', 'addr', 'add', $net->{hostip}, 'dev', $iface]);
> +}
> +
> +#set route to guest if specified
> +if (defined($net->{guestip})) {
> +run_command(['ip', 'route', 'add', $net->{guestip}, 'dev', $iface]);
> +}
> +
> +
> +
> +
> +
> +
> +exit 0;
> diff --git a/src/usr/pve-tap-hotplug b/src/usr/pve-tap-hotplug
> new file mode 100755
> index 00000000..6fcdcd2a
> --- /dev/null
> +++ b/src/usr/pve-tap-hotplug
> @@ -0,0 +1,3 @@
> +#!/bin/sh
> +
> +exec /usr/libexec/qemu-server/pve-tap --hotplug "$@"
> diff --git a/src/usr/pve-tapdown b/src/usr/pve-tapdown
> new file mode 100755
> index 00000000..e867b640
> --- /dev/null
> +++ b/src/usr/pve-tapdown
> @@ -0,0 +1,16 @@
> +#!/usr/bin/perl
> +
> +use strict;
> +use warnings;
> +use PVE::Network;
> +
> +my $iface = shift;
> +
> +die "no interface specified\n" if !$iface;
> +
> +die "got strange interface name '$iface'\n"
> + if $iface !~ m/^tap(\d+)i(\d+)$/;
> +
> +PVE::Network::tap_unplug($iface);
> +
> +exit 0;



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Stefan Hanreich]

Hi!

Thanks for your contribution, did you already check out our guidelines
[1] [2] and send a signed CLA? Without it, we cannot accept any
contributions.


I've looked at the proposal, but I wanted to take some time to think
more about the general concept. It seems like you want to build
something similar to current Kubernetes networking solutions that
utilize BGP but without the whole EVPN / VXLAN stuff?

Maybe it'd make more sense to discuss about how we could improve the
EVPN zone or SDN in general to make such setups easier - potentially a
new zone that is something of an inbetween of the simple zone and EVPN
zone could make sense. In any case, I think removing the bridge and
implementing it this way is the wrong way to go about this.

[1] https://www.proxmox.com/en/about/open-source/developers
[2] https://pve.proxmox.com/wiki/Developer_Documentation


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 6320 bytes --]

From: Maurice Klein <klein@aetherus.de>
To: pve-devel@lists.proxmox.com
Subject: Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>
Date: Wed, 21 Jan 2026 20:04:48 +0100
Message-ID: <d18928a0-6ab0-4e90-ad3a-0674bbdedb72@aetherus.de>

Hi,

Thanks for getting back to me!
I did sign a CLA and it is on record since 12.01.

I agree it would work nicely as a SDN plugin and I was also considering
that approach.
The problem I saw with that is that SDN relies on there being a bridge
for every zone and making it work without one seems to be a huge refactor.
Do you think the bridge should not be removed at all, even for a pure l3
routed setup?
It could also work with one bridge per guest but that would, in my
opinion, bring unnecessary overhead.

The motivation on my side comes from setups where L2 between guests is
not required at all, and where using routing protocols (OSPF, IS-IS, 
BGP) to the hosts
simplifies redundancy and failure handling significantly.

I'd love to get a conversation going on how something could be
implemented, and what would be the best way to go about it.



Mit freundlichen Grüßen,

Maurice Klein

Aetherus

TEL: 0212 7846460
Mail:Klein@aetherus.de


Am 19.01.26 um 15:35 schrieb Stefan Hanreich:
> Hi!
>
> Thanks for your contribution, did you already check out our guidelines
> [1] [2] and send a signed CLA? Without it, we cannot accept any
> contributions.
>
>
> I've looked at the proposal, but I wanted to take some time to think
> more about the general concept. It seems like you want to build
> something similar to current Kubernetes networking solutions that
> utilize BGP but without the whole EVPN / VXLAN stuff?
>
> Maybe it'd make more sense to discuss about how we could improve the
> EVPN zone or SDN in general to make such setups easier - potentially a
> new zone that is something of an inbetween of the simple zone and EVPN
> zone could make sense. In any case, I think removing the bridge and
> implementing it this way is the wrong way to go about this.
>
> [1]https://www.proxmox.com/en/about/open-source/developers
> [2]https://pve.proxmox.com/wiki/Developer_Documentation
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Stefan Hanreich]

On 1/21/26 8:04 PM, Maurice Klein wrote:

Some thoughts below from my side, but I'm still unsure on what would be
the best approach for this.

> I agree it would work nicely as a SDN plugin and I was also considering
> that approach.
> The problem I saw with that is that SDN relies on there being a bridge
> for every zone and making it work without one seems to be a huge refactor.
> Do you think the bridge should not be removed at all, even for a pure l3
> routed setup?

That would be one reason, but there are others. The gateway IP only
needs to be configured once on the bridge / vnet itself then, whereas it
needs to be specified explicitly for every guest with your approach.
You'd most likely also need to generate a MAC address that is the same
for the GW on all PVE hosts, so VM mobility works properly. With tap
interfaces that is even more complicated as you'd need to handle setting
the MAC for each tap interface. It's cleaner and simpler that way imo,
since you can just set up the gateway once and be done.

A simple zone with port isolation is already quite similar to what
you're trying to achieve imo. It denies L2 connectivity between guests
via the isolated flag [1] on bridge members and the PVE node acts as a
router for the zone. I think that could be used as a starting point and
then build upon it. Simple zones have IPAM support, so we could utilize
that for managing the guest IPs. It would probably also make sense to
manage neighbor / fdb table entries statically for this kind of setup.

[1] https://man7.org/linux/man-pages/man8/bridge.8.html


_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 7343 bytes --]

From: Maurice Klein <klein@aetherus.de>
To: Stefan Hanreich <s.hanreich@proxmox.com>, pve-devel@lists.proxmox.com
Subject: Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>
Date: Tue, 27 Jan 2026 11:37:00 +0100
Message-ID: <321bd4ff-f147-4329-9788-50061d569fa6@aetherus.de>

Hi,

I didn't even think about mac address, since arp does recover quickly 
but it would be a unwanted interruption.

I like the idea with using a bridge with l2 isolation.
I'd like to implement the zone then, with the use of a l2 isolated bridge.

I'd propose the first step would be to get the zone working, to get the 
mechanism working to add host routes.
I would use one VRF then per zone.

A proposed name would be "Routed".
Do you have better Ideas?

A roadmap I'd have in my head then would look the following way:

- implement zone "Routed"
    ensuring that all routing between guests and host works and that 
default routes get put in the vrf as well

- implement possibility to export host routes of routed zones via BGP

- implement possibility to add static routes per Routed zone, like 
different default routes or others

- implement dynamic routing updates into routed zones vrf tables

Where I'm still not sure is how to get routing in a cluster running 
between the same zone.
It could be implemented via IBGP Sessions between the hosts, but i don't 
know if that is the preffered way and it needs to be clear to the user 
which path will be taken and how it works.


Let me know if that is a way you agree with and if it is i would like to 
get started implementing the first step.


Mit freundlichen Grüßen,

Maurice Klein

Aetherus

TEL: 0212 7846460
Mail: Klein@aetherus.de


Am 27.01.26 um 11:02 schrieb Stefan Hanreich:
> On 1/21/26 8:04 PM, Maurice Klein wrote:
>
> Some thoughts below from my side, but I'm still unsure on what would be
> the best approach for this.
>
>> I agree it would work nicely as a SDN plugin and I was also considering
>> that approach.
>> The problem I saw with that is that SDN relies on there being a bridge
>> for every zone and making it work without one seems to be a huge refactor.
>> Do you think the bridge should not be removed at all, even for a pure l3
>> routed setup?
> That would be one reason, but there are others. The gateway IP only
> needs to be configured once on the bridge / vnet itself then, whereas it
> needs to be specified explicitly for every guest with your approach.
> You'd most likely also need to generate a MAC address that is the same
> for the GW on all PVE hosts, so VM mobility works properly. With tap
> interfaces that is even more complicated as you'd need to handle setting
> the MAC for each tap interface. It's cleaner and simpler that way imo,
> since you can just set up the gateway once and be done.
>
> A simple zone with port isolation is already quite similar to what
> you're trying to achieve imo. It denies L2 connectivity between guests
> via the isolated flag [1] on bridge members and the PVE node acts as a
> router for the zone. I think that could be used as a starting point and
> then build upon it. Simple zones have IPAM support, so we could utilize
> that for managing the guest IPs. It would probably also make sense to
> manage neighbor / fdb table entries statically for this kind of setup.
>
> [1] https://man7.org/linux/man-pages/man8/bridge.8.html
>



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Stefan Hanreich]

On 1/27/26 11:36 AM, Maurice Klein wrote:
> Hi,
> 
> I didn't even think about mac address, since arp does recover quickly
> but it would be a unwanted interruption.
> 
> I like the idea with using a bridge with l2 isolation.
> I'd like to implement the zone then, with the use of a l2 isolated bridge.
> 
> I'd propose the first step would be to get the zone working, to get the
> mechanism working to add host routes.
> I would use one VRF then per zone.

For a PoC we could just utilize the default routing table I think? VRF
support for SDN entities is something that is on the mid-term roadmap,
so if we want VRF support we'd need to be careful not to put any
barriers in that make implementing that feature harder. We'd also have
to make sure that VRFs stay unique across all entities that are using
them (currently only EVPN zone). If we generate the names analogous to
the EVPN zones we should be fine since all zones regardless of type
share the same ID space.

With multiple VRFs, you'd ideally want to have the option to announce
the routes per-VRF. Utilizing transit VLANs for zones would make sense
then and also being able to map each VNet in a zone to a VLAN. That'd
require creating VLAN subinterfaces inside a VRF as well via our stack
and adding them to a VNet (might be done automatically) / VRF. At that
point we're building essentially what is VRF-lite though, so that's a
bit more involved.

> A proposed name would be "Routed".
> Do you have better Ideas?
> 
> A roadmap I'd have in my head then would look the following way:
> 
> - implement zone "Routed"
>    ensuring that all routing between guests and host works and that
> default routes get put in the vrf as well

I'd not necessarily put in the default route unconditionally, either as
a config option, or - if we skip VRF support for now - we'd get this for
free via implementing the VRF feature afterwards.

> - implement possibility to export host routes of routed zones via BGP

see below

> - implement possibility to add static routes per Routed zone, like
> different default routes or others

we would get that via VRF support as well

> - implement dynamic routing updates into routed zones vrf tables
> 
> Where I'm still not sure is how to get routing in a cluster running
> between the same zone.
> It could be implemented via IBGP Sessions between the hosts, but i don't
> know if that is the preffered way and it needs to be clear to the user
> which path will be taken and how it works.

One option would be to utilize e.g. the fabrics where we plan to add
initial support for route redistribution soon. This should get
integrated with a future VRF feature as well. That would decouple the
zone from the redistribution / announcing itself and give users
flexibility in choosing their routing protocol. Users can create
fabrics, then select which VRFs to announce via them (and apply filters
via route-maps).

Re: [pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein]

So i was looking into implementation today.

Unfortunately I found some points where I'm kinda stuck with the current 
way SDN is designed.

I'm hoping you have some ideas or advise how to continue.

Basicly the vnet and subnet part I see as in issue.
Since in this kind of setup there is no defined subsets required the 
current configuration doesn't fully make sense.
I guess you could still have a subnet configuration and configure all 
the host addresses inside that subnet, but it's not really necessary .

Every VM route would be a /32 route and also the configured address on 
that bridge (gateway field) would be a /32.
When the tap interface of a vm gets plugged a route needs to be created.
Routes per VM get created with the comand ip route add 192.168.1.5/32 
dev routedbridge.
The /32 gateway address needs to be configured on the bridge as well.

There needs to be some way to configure the guests IPs as well, but in 
ipam there is currently no way to set a ip for a vm, it's only ip mac 
bindings.

A potential security flaw is also devices on that bridge can steal a 
configured ip by just replying to arp.
That could be mitigated by disabling bridge learning and creating static 
atp entires as well for those configured IPs.

I don't know yet what's the best way to configure guest ip addresses.
My initial idea was to have that done on a per interface vm level, that 
is incompatible with sdn though.
I don't know how it could be integrated in IPAM though, especially 
considering that pve IPAM isn't the only one available to be used.




Mit freundlichen Grüßen,

Maurice Klein

Aetherus

TEL: 0212 7846460
Mail: Klein@aetherus.de


Am 29.01.26 um 13:20 schrieb Stefan Hanreich:
> On 1/27/26 11:36 AM, Maurice Klein wrote:
>> Hi,
>>
>> I didn't even think about mac address, since arp does recover quickly
>> but it would be a unwanted interruption.
>>
>> I like the idea with using a bridge with l2 isolation.
>> I'd like to implement the zone then, with the use of a l2 isolated bridge.
>>
>> I'd propose the first step would be to get the zone working, to get the
>> mechanism working to add host routes.
>> I would use one VRF then per zone.
> For a PoC we could just utilize the default routing table I think? VRF
> support for SDN entities is something that is on the mid-term roadmap,
> so if we want VRF support we'd need to be careful not to put any
> barriers in that make implementing that feature harder. We'd also have
> to make sure that VRFs stay unique across all entities that are using
> them (currently only EVPN zone). If we generate the names analogous to
> the EVPN zones we should be fine since all zones regardless of type
> share the same ID space.
>
> With multiple VRFs, you'd ideally want to have the option to announce
> the routes per-VRF. Utilizing transit VLANs for zones would make sense
> then and also being able to map each VNet in a zone to a VLAN. That'd
> require creating VLAN subinterfaces inside a VRF as well via our stack
> and adding them to a VNet (might be done automatically) / VRF. At that
> point we're building essentially what is VRF-lite though, so that's a
> bit more involved.
>
>> A proposed name would be "Routed".
>> Do you have better Ideas?
>>
>> A roadmap I'd have in my head then would look the following way:
>>
>> - implement zone "Routed"
>>     ensuring that all routing between guests and host works and that
>> default routes get put in the vrf as well
> I'd not necessarily put in the default route unconditionally, either as
> a config option, or - if we skip VRF support for now - we'd get this for
> free via implementing the VRF feature afterwards.
>
>> - implement possibility to export host routes of routed zones via BGP
> see below
>
>> - implement possibility to add static routes per Routed zone, like
>> different default routes or others
> we would get that via VRF support as well
>
>> - implement dynamic routing updates into routed zones vrf tables
>>
>> Where I'm still not sure is how to get routing in a cluster running
>> between the same zone.
>> It could be implemented via IBGP Sessions between the hosts, but i don't
>> know if that is the preffered way and it needs to be clear to the user
>> which path will be taken and how it works.
> One option would be to utilize e.g. the fabrics where we plan to add
> initial support for route redistribution soon. This should get
> integrated with a future VRF feature as well. That would decouple the
> zone from the redistribution / announcing itself and give users
> flexibility in choosing their routing protocol. Users can create
> fabrics, then select which VRFs to announce via them (and apply filters
> via route-maps).
>

[pve-devel] [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>

[Maurice Klein via pve-devel]

[-- Attachment #1: Type: message/rfc822, Size: 8366 bytes --]

From: Maurice Klein <klein@aetherus.de>
To: pve-devel@lists.proxmox.com
Subject: [PATCH container 1/1] Signed-off-by: Maurice Klein <klein@aetherus.de>
Date: Fri,  9 Jan 2026 13:45:14 +0100
Message-ID: <20260109124514.72991-2-klein@aetherus.de>

qemu-server: add routed tap and helper scripts
---
 src/PVE/QemuServer.pm         |  9 +++++-
 src/PVE/QemuServer/Network.pm | 19 +++++++++++
 src/usr/pve-tap               | 59 +++++++++++++++++++++++++++++++++++
 src/usr/pve-tap-hotplug       |  3 ++
 src/usr/pve-tapdown           | 16 ++++++++++
 5 files changed, 105 insertions(+), 1 deletion(-)
 create mode 100755 src/usr/pve-tap
 create mode 100755 src/usr/pve-tap-hotplug
 create mode 100755 src/usr/pve-tapdown

diff --git a/src/PVE/QemuServer.pm b/src/PVE/QemuServer.pm
index 69991843..2c0b784e 100644
--- a/src/PVE/QemuServer.pm
+++ b/src/PVE/QemuServer.pm
@@ -1443,8 +1443,15 @@ sub print_netdev_full {
 
     my $netdev = "";
     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
+    if ($net->{taprouted}) {
+            $script = $hotplug ? "pve-tap" : "pve-tap-hotplug";
+    }
+
 
-    if ($net->{bridge}) {
+    if ($net->{taprouted}) {
+		    $netdev= "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
+            . ",downscript=/usr/libexec/qemu-server/pve-tapdown$vhostparam";
+    } elsif ($net->{bridge}) {
         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/usr/libexec/qemu-server/$script"
             . ",downscript=/usr/libexec/qemu-server/pve-bridgedown$vhostparam";
     } else {
diff --git a/src/PVE/QemuServer/Network.pm b/src/PVE/QemuServer/Network.pm
index eb8222e8..c11f002c 100644
--- a/src/PVE/QemuServer/Network.pm
+++ b/src/PVE/QemuServer/Network.pm
@@ -116,6 +116,25 @@ my $net_fmt = {
             "Force MTU of network device (VirtIO only). Setting to '1' or empty will use the bridge MTU",
         optional => 1,
     },
+    taprouted => {
+        type => 'boolean',
+        description => "routed network, just make tap interface and execute routing script",
+        optional => 1,
+    },
+    hostip => {
+        type => 'string',
+        format => 'ipv4',
+        format_description => 'IPv4Format',
+        description => 'IPv4 address for the host.',
+        optional => 1,
+    },
+    guestip => {
+        type => 'string',
+        format => 'ipv4',
+        format_description => 'GuestIPv4',
+        description => 'IPv4 address for the guest.',
+        optional => 1,
+    },
 };
 
 our $netdesc = {
diff --git a/src/usr/pve-tap b/src/usr/pve-tap
new file mode 100755
index 00000000..10623c17
--- /dev/null
+++ b/src/usr/pve-tap
@@ -0,0 +1,59 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+
+use PVE::Tools qw(run_command);
+use PVE::Firewall;
+
+use PVE::QemuServer::Network;
+
+my $iface = shift;
+
+my $hotplug = 0;
+if ($iface eq '--hotplug') {
+    $hotplug = 1;
+    $iface = shift;
+}
+
+die "no interface specified\n" if !$iface;
+
+die "got strange interface name '$iface'\n"
+    if $iface !~ m/^tap(\d+)i(\d+)$/;
+
+my $vmid = $1;
+my $netid = "net$2";
+
+my $migratedfrom = $hotplug ? undef : $ENV{PVE_MIGRATED_FROM};
+
+my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
+
+my $netconf = $conf->{$netid};
+
+$netconf = $conf->{pending}->{$netid} if !$migratedfrom && defined($conf->{pending}->{$netid});
+
+die "unable to get network config '$netid'\n"
+    if !defined($netconf);
+
+my $net = PVE::QemuServer::Network::parse_net($netconf);
+die "unable to parse network config '$netid'\n" if !$net;
+
+
+# Bring up the tap interface
+run_command(['ip', 'link', 'set', $iface, 'up']);
+#set host ip if specified
+if (defined($net->{hostip})) {
+    run_command(['ip', 'addr', 'add', $net->{hostip}, 'dev', $iface]);
+}
+
+#set route to guest if specified
+if (defined($net->{guestip})) {
+run_command(['ip', 'route', 'add', $net->{guestip}, 'dev', $iface]);
+}
+
+
+
+
+
+
+exit 0;
diff --git a/src/usr/pve-tap-hotplug b/src/usr/pve-tap-hotplug
new file mode 100755
index 00000000..6fcdcd2a
--- /dev/null
+++ b/src/usr/pve-tap-hotplug
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec /usr/libexec/qemu-server/pve-tap --hotplug "$@"
diff --git a/src/usr/pve-tapdown b/src/usr/pve-tapdown
new file mode 100755
index 00000000..e867b640
--- /dev/null
+++ b/src/usr/pve-tapdown
@@ -0,0 +1,16 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use PVE::Network;
+
+my $iface = shift;
+
+die "no interface specified\n" if !$iface;
+
+die "got strange interface name '$iface'\n"
+    if $iface !~ m/^tap(\d+)i(\d+)$/;
+
+PVE::Network::tap_unplug($iface);
+
+exit 0;
-- 
2.39.5 (Apple Git-154)



[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
pve-devel mailing list
pve-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel