From: Alexandre Derumier <aderumier@odiso.com>
To: pve-devel@lists.proxmox.com
Subject: [pve-devel] [PATCH V2 pve-network 1/4] controllers: improve bgp-evpn
Date: Wed, 25 Nov 2020 10:01:36 +0100 [thread overview]
Message-ID: <20201125090139.3553272-2-aderumier@odiso.com> (raw)
In-Reply-To: <20201125090139.3553272-1-aderumier@odiso.com>
- add new bgp plugin
- add ebgp support
- add loopback support
- move gateway-nodes option to zone as 'exitnodes'
- move external-peers to bgp plugin
---
PVE/API2/Network/SDN/Controllers.pm | 1 +
PVE/Network/SDN/Controllers.pm | 12 +-
PVE/Network/SDN/Controllers/BgpPlugin.pm | 153 ++++++++++++++++++++
PVE/Network/SDN/Controllers/EvpnPlugin.pm | 122 ++++++++++------
PVE/Network/SDN/Controllers/FaucetPlugin.pm | 4 +-
PVE/Network/SDN/Controllers/Makefile | 2 +-
PVE/Network/SDN/Controllers/Plugin.pm | 9 +-
PVE/Network/SDN/Zones/EvpnPlugin.pm | 11 +-
PVE/Network/SDN/Zones/Plugin.pm | 9 +-
9 files changed, 269 insertions(+), 54 deletions(-)
create mode 100644 PVE/Network/SDN/Controllers/BgpPlugin.pm
diff --git a/PVE/API2/Network/SDN/Controllers.pm b/PVE/API2/Network/SDN/Controllers.pm
index 75beb6b..e761b6c 100644
--- a/PVE/API2/Network/SDN/Controllers.pm
+++ b/PVE/API2/Network/SDN/Controllers.pm
@@ -11,6 +11,7 @@ use PVE::Network::SDN::Zones;
use PVE::Network::SDN::Controllers;
use PVE::Network::SDN::Controllers::Plugin;
use PVE::Network::SDN::Controllers::EvpnPlugin;
+use PVE::Network::SDN::Controllers::BgpPlugin;
use PVE::Network::SDN::Controllers::FaucetPlugin;
use Storable qw(dclone);
diff --git a/PVE/Network/SDN/Controllers.pm b/PVE/Network/SDN/Controllers.pm
index f652d7f..9937755 100644
--- a/PVE/Network/SDN/Controllers.pm
+++ b/PVE/Network/SDN/Controllers.pm
@@ -13,9 +13,11 @@ use PVE::Network::SDN::Vnets;
use PVE::Network::SDN::Zones;
use PVE::Network::SDN::Controllers::EvpnPlugin;
+use PVE::Network::SDN::Controllers::BgpPlugin;
use PVE::Network::SDN::Controllers::FaucetPlugin;
use PVE::Network::SDN::Controllers::Plugin;
PVE::Network::SDN::Controllers::EvpnPlugin->register();
+PVE::Network::SDN::Controllers::BgpPlugin->register();
PVE::Network::SDN::Controllers::FaucetPlugin->register();
PVE::Network::SDN::Controllers::Plugin->init();
@@ -95,24 +97,24 @@ sub generate_controller_config {
#generate configuration
my $config = {};
- foreach my $id (keys %{$controller_cfg->{ids}}) {
+ foreach my $id (sort keys %{$controller_cfg->{ids}}) {
my $plugin_config = $controller_cfg->{ids}->{$id};
my $plugin = PVE::Network::SDN::Controllers::Plugin->lookup($plugin_config->{type});
- $plugin->generate_controller_config($plugin_config, $plugin_config, $id, $uplinks, $config);
+ $plugin->generate_controller_config($plugin_config, $controller_cfg, $id, $uplinks, $config);
}
- foreach my $id (keys %{$zone_cfg->{ids}}) {
+ foreach my $id (sort keys %{$zone_cfg->{ids}}) {
my $plugin_config = $zone_cfg->{ids}->{$id};
my $controllerid = $plugin_config->{controller};
next if !$controllerid;
my $controller = $controller_cfg->{ids}->{$controllerid};
if ($controller) {
my $controller_plugin = PVE::Network::SDN::Controllers::Plugin->lookup($controller->{type});
- $controller_plugin->generate_controller_zone_config($plugin_config, $controller, $id, $uplinks, $config);
+ $controller_plugin->generate_controller_zone_config($plugin_config, $controller, $controller_cfg, $id, $uplinks, $config);
}
}
- foreach my $id (keys %{$vnet_cfg->{ids}}) {
+ foreach my $id (sort keys %{$vnet_cfg->{ids}}) {
my $plugin_config = $vnet_cfg->{ids}->{$id};
my $zoneid = $plugin_config->{zone};
next if !$zoneid;
diff --git a/PVE/Network/SDN/Controllers/BgpPlugin.pm b/PVE/Network/SDN/Controllers/BgpPlugin.pm
new file mode 100644
index 0000000..ccc06a8
--- /dev/null
+++ b/PVE/Network/SDN/Controllers/BgpPlugin.pm
@@ -0,0 +1,153 @@
+package PVE::Network::SDN::Controllers::BgpPlugin;
+
+use strict;
+use warnings;
+
+use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Tools qw(run_command file_set_contents file_get_contents);
+
+use PVE::Network::SDN::Controllers::Plugin;
+use PVE::Network::SDN::Zones::Plugin;
+use Net::IP;
+
+use base('PVE::Network::SDN::Controllers::Plugin');
+
+sub type {
+ return 'bgp';
+}
+
+sub properties {
+ return {
+ ebgp => {
+ type => 'boolean',
+ optional => 1,
+ description => "Enable ebgp. (remote-as external)",
+ },
+ loopback => {
+ description => "source loopback interface.",
+ type => 'string'
+ },
+ node => get_standard_option('pve-node'),
+ };
+}
+
+sub options {
+ return {
+ 'node' => { optional => 0 },
+ 'asn' => { optional => 0 },
+ 'peers' => { optional => 0 },
+ 'ebgp' => { optional => 1 },
+ 'loopback' => { optional => 1 },
+ };
+}
+
+# Plugin implementation
+sub generate_controller_config {
+ my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+
+ my @peers;
+ @peers = PVE::Tools::split_list($plugin_config->{'peers'}) if $plugin_config->{'peers'};
+
+ my $asn = $plugin_config->{asn};
+ my $ebgp = $plugin_config->{ebgp};
+ my $loopback = $plugin_config->{loopback};
+ my $local_node = PVE::INotify::nodename();
+
+
+ return if !$asn;
+ return if $local_node ne $plugin_config->{node};
+
+ my $bgp = $config->{frr}->{router}->{"bgp $asn"} //= {};
+
+ my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
+
+ my $remoteas = $ebgp ? "external" : $asn;
+
+ #global options
+ my @controller_config = (
+ "bgp router-id $ifaceip",
+ "no bgp default ipv4-unicast",
+ "coalesce-time 1000",
+ "bgp network import-check"
+ );
+
+ push(@{$bgp->{""}}, @controller_config) if keys %{$bgp} == 0;
+
+ @controller_config = ();
+ if($ebgp) {
+ push @controller_config, "no bgp ebgp-requires-policy";
+ push @controller_config, "bgp disable-ebgp-connected-route-check" if $loopback;
+ }
+
+ #BGP neighbors
+ if(@peers) {
+ push @controller_config, "neighbor BGP peer-group";
+ push @controller_config, "neighbor BGP remote-as $remoteas";
+ push @controller_config, "neighbor BGP bfd";
+ }
+
+ # BGP peers
+ foreach my $address (@peers) {
+ push @controller_config, "neighbor $address peer-group BGP";
+ }
+ push(@{$bgp->{""}}, @controller_config);
+
+ # address-family unicast
+ if (@peers) {
+ my $ipversion = Net::IP::ip_is_ipv6($ifaceip) ? "ipv6" : "ipv4";
+ my $mask = Net::IP::ip_is_ipv6($ifaceip) ? "/128" : "32";
+
+ push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "network $ifaceip/$mask") if $loopback;
+ push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "neighbor BGP activate");
+ push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "neighbor BGP soft-reconfiguration inbound");
+ }
+
+ return $config;
+}
+
+sub generate_controller_zone_config {
+ my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
+
+}
+
+sub on_delete_hook {
+ my ($class, $controllerid, $zone_cfg) = @_;
+
+ # verify that zone is associated to this controller
+ foreach my $id (keys %{$zone_cfg->{ids}}) {
+ my $zone = $zone_cfg->{ids}->{$id};
+ die "controller $controllerid is used by $id"
+ if (defined($zone->{controller}) && $zone->{controller} eq $controllerid);
+ }
+}
+
+sub on_update_hook {
+ my ($class, $controllerid, $controller_cfg) = @_;
+
+ # we can only have 1 bgp controller by node
+ my $local_node = PVE::INotify::nodename();
+ my $controllernb = 0;
+ foreach my $id (keys %{$controller_cfg->{ids}}) {
+ next if $id eq $controllerid;
+ my $controller = $controller_cfg->{ids}->{$id};
+ next if $controller->{type} ne "bgp";
+ next if $controller->{node} ne $local_node;
+ $controllernb++;
+ die "only 1 bgp controller can be defined" if $controllernb > 1;
+ }
+}
+
+sub write_controller_config {
+ my ($class, $plugin_config, $config) = @_;
+ return;
+}
+
+sub reload_controller {
+ my ($class) = @_;
+ return;
+}
+
+1;
+
+
diff --git a/PVE/Network/SDN/Controllers/EvpnPlugin.pm b/PVE/Network/SDN/Controllers/EvpnPlugin.pm
index d82de2a..e59c142 100644
--- a/PVE/Network/SDN/Controllers/EvpnPlugin.pm
+++ b/PVE/Network/SDN/Controllers/EvpnPlugin.pm
@@ -9,6 +9,7 @@ use PVE::Tools qw(run_command file_set_contents file_get_contents);
use PVE::Network::SDN::Controllers::Plugin;
use PVE::Network::SDN::Zones::Plugin;
+use Net::IP;
use base('PVE::Network::SDN::Controllers::Plugin');
@@ -26,11 +27,6 @@ sub properties {
description => "peers address list.",
type => 'string', format => 'ip-list'
},
- 'gateway-nodes' => get_standard_option('pve-node-list'),
- 'gateway-external-peers' => {
- description => "upstream bgp peers address list.",
- type => 'string', format => 'ip-list'
- },
};
}
@@ -38,80 +34,97 @@ sub options {
return {
'asn' => { optional => 0 },
'peers' => { optional => 0 },
- 'gateway-nodes' => { optional => 1 },
- 'gateway-external-peers' => { optional => 1 },
};
}
# Plugin implementation
sub generate_controller_config {
- my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+ my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
my @peers;
@peers = PVE::Tools::split_list($plugin_config->{'peers'}) if $plugin_config->{'peers'};
+ my $local_node = PVE::INotify::nodename();
+
my $asn = $plugin_config->{asn};
- my $gatewaynodes = $plugin_config->{'gateway-nodes'};
- my @gatewaypeers;
- @gatewaypeers = PVE::Tools::split_list($plugin_config->{'gateway-external-peers'}) if $plugin_config->{'gateway-external-peers'};
+ my $ebgp = undef;
+ my $loopback = undef;
+ my $autortas = undef;
+ my $bgprouter = find_bgp_controller($local_node, $controller_cfg);
+ if($bgprouter) {
+ $ebgp = 1 if $plugin_config->{'asn'} ne $bgprouter->{asn};
+ $loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+ $asn = $bgprouter->{asn} if $bgprouter->{asn};
+ $autortas = $plugin_config->{'asn'} if $ebgp;
+ }
return if !$asn;
my $bgp = $config->{frr}->{router}->{"bgp $asn"} //= {};
- my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers);
+ my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
- my $is_gateway = undef;
- my $local_node = PVE::INotify::nodename();
-
- foreach my $gatewaynode (PVE::Tools::split_list($gatewaynodes)) {
- $is_gateway = 1 if $gatewaynode eq $local_node;
- }
+ my $remoteas = $ebgp ? "external" : $asn;
+ #global options
my @controller_config = (
"bgp router-id $ifaceip",
"no bgp default ipv4-unicast",
"coalesce-time 1000",
);
+ push(@{$bgp->{""}}, @controller_config) if keys %{$bgp} == 0;
+
+ @controller_config = ();
+
+ #VTEP neighbors
+ push @controller_config, "neighbor VTEP peer-group";
+ push @controller_config, "neighbor VTEP remote-as $remoteas";
+ push @controller_config, "neighbor VTEP bfd";
+
+ if($ebgp && $loopback) {
+ push @controller_config, "neighbor VTEP ebgp-multihop 10";
+ push @controller_config, "neighbor VTEP update-source $loopback";
+ }
+
+ # VTEP peers
foreach my $address (@peers) {
next if $address eq $ifaceip;
- push @controller_config, "neighbor $address remote-as $asn";
+ push @controller_config, "neighbor $address peer-group VTEP";
}
- if ($is_gateway) {
- foreach my $address (@gatewaypeers) {
- push @controller_config, "neighbor $address remote-as external";
- }
- }
push(@{$bgp->{""}}, @controller_config);
+ # address-family l2vpn
@controller_config = ();
- foreach my $address (@peers) {
- next if $address eq $ifaceip;
- push @controller_config, "neighbor $address activate";
- }
+ push @controller_config, "neighbor VTEP activate";
push @controller_config, "advertise-all-vni";
+ push @controller_config, "autort as $autortas" if $autortas;
push(@{$bgp->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
- if ($is_gateway) {
- # import /32 routes of evpn network from vrf1 to default vrf (for packet return)
- @controller_config = map { "neighbor $_ activate" } @gatewaypeers;
-
- push(@{$bgp->{"address-family"}->{"ipv4 unicast"}}, @controller_config);
- push(@{$bgp->{"address-family"}->{"ipv6 unicast"}}, @controller_config);
- }
-
return $config;
}
sub generate_controller_zone_config {
- my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+ my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
+
+ my $local_node = PVE::INotify::nodename();
my $vrf = "vrf_$id";
my $vrfvxlan = $plugin_config->{'vrf-vxlan'};
+ my $exitnodes = $plugin_config->{'exitnodes'};
+
my $asn = $controller->{asn};
- my $gatewaynodes = $controller->{'gateway-nodes'};
+ my $ebgp = undef;
+ my $loopback = undef;
+ my $autortas = undef;
+ my $bgprouter = find_bgp_controller($local_node, $controller_cfg);
+ if($bgprouter) {
+ $ebgp = 1 if $controller->{'asn'} ne $bgprouter->{asn};
+ $loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+ $asn = $bgprouter->{asn} if $bgprouter->{asn};
+ $autortas = $controller->{'asn'} if $ebgp;
+ }
return if !$vrf || !$vrfvxlan || !$asn;
@@ -120,11 +133,18 @@ sub generate_controller_zone_config {
push @controller_config, "vni $vrfvxlan";
push(@{$config->{frr}->{vrf}->{"$vrf"}}, @controller_config);
- push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{""}}, "!");
+ #main vrf router
+ @controller_config = ();
+ push @controller_config, "no bgp ebgp-requires-policy" if $ebgp;
+# push @controller_config, "!";
+ push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{""}}, @controller_config);
- my $local_node = PVE::INotify::nodename();
+ if ($autortas) {
+ push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target import $autortas:$vrfvxlan");
+ push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target export $autortas:$vrfvxlan");
+ }
- my $is_gateway = grep { $_ eq $local_node } PVE::Tools::split_list($gatewaynodes);
+ my $is_gateway = grep { $_ eq $local_node } PVE::Tools::split_list($exitnodes);
if ($is_gateway) {
@controller_config = ();
@@ -165,13 +185,31 @@ sub on_update_hook {
# we can only have 1 evpn controller / 1 asn by server
+ my $controllernb = 0;
foreach my $id (keys %{$controller_cfg->{ids}}) {
next if $id eq $controllerid;
my $controller = $controller_cfg->{ids}->{$id};
- die "only 1 evpn controller can be defined" if $controller->{type} eq "evpn";
+ next if $controller->{type} ne "evpn";
+ $controllernb++;
+ die "only 1 global evpn controller can be defined" if $controllernb > 1;
+ }
+}
+
+sub find_bgp_controller {
+ my ($nodename, $controller_cfg) = @_;
+
+ my $controller = undef;
+ foreach my $id (keys %{$controller_cfg->{ids}}) {
+ $controller = $controller_cfg->{ids}->{$id};
+ next if $controller->{type} ne 'bgp';
+ next if $controller->{node} ne $nodename;
+ last;
}
+
+ return $controller;
}
+
sub sort_frr_config {
my $order = {};
$order->{''} = 0;
diff --git a/PVE/Network/SDN/Controllers/FaucetPlugin.pm b/PVE/Network/SDN/Controllers/FaucetPlugin.pm
index dcac6eb..5742187 100644
--- a/PVE/Network/SDN/Controllers/FaucetPlugin.pm
+++ b/PVE/Network/SDN/Controllers/FaucetPlugin.pm
@@ -22,12 +22,12 @@ sub properties {
# Plugin implementation
sub generate_controller_config {
- my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+ my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
}
sub generate_controller_zone_config {
- my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+ my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
my $dpid = $plugin_config->{'dp-id'};
my $dphex = printf("%x",$dpid);
diff --git a/PVE/Network/SDN/Controllers/Makefile b/PVE/Network/SDN/Controllers/Makefile
index 3324125..11686a3 100644
--- a/PVE/Network/SDN/Controllers/Makefile
+++ b/PVE/Network/SDN/Controllers/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm FaucetPlugin.pm EvpnPlugin.pm
+SOURCES=Plugin.pm FaucetPlugin.pm EvpnPlugin.pm BgpPlugin.pm
PERL5DIR=${DESTDIR}/usr/share/perl5
diff --git a/PVE/Network/SDN/Controllers/Plugin.pm b/PVE/Network/SDN/Controllers/Plugin.pm
index 06cd576..0c92b17 100644
--- a/PVE/Network/SDN/Controllers/Plugin.pm
+++ b/PVE/Network/SDN/Controllers/Plugin.pm
@@ -70,7 +70,14 @@ sub generate_sdn_config {
}
sub generate_controller_config {
- my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+ my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
+
+ die "please implement inside plugin";
+}
+
+
+sub generate_controller_zone_config {
+ my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
die "please implement inside plugin";
}
diff --git a/PVE/Network/SDN/Zones/EvpnPlugin.pm b/PVE/Network/SDN/Zones/EvpnPlugin.pm
index 5338a1b..d50ddb9 100644
--- a/PVE/Network/SDN/Zones/EvpnPlugin.pm
+++ b/PVE/Network/SDN/Zones/EvpnPlugin.pm
@@ -4,6 +4,7 @@ use strict;
use warnings;
use PVE::Network::SDN::Zones::VxlanPlugin;
use PVE::Exception qw(raise raise_param_exc);
+use PVE::JSONSchema qw(get_standard_option);
use PVE::Tools qw($IPV4RE);
use PVE::INotify;
use PVE::Cluster;
@@ -27,6 +28,7 @@ sub properties {
type => 'string',
description => "Frr router name",
},
+ 'exitnodes' => get_standard_option('pve-node-list'),
};
}
@@ -35,7 +37,8 @@ sub options {
return {
nodes => { optional => 1},
'vrf-vxlan' => { optional => 0 },
- 'controller' => { optional => 0 },
+ controller => { optional => 0 },
+ exitnodes => { optional => 1 },
mtu => { optional => 1 },
dns => { optional => 1 },
reversedns => { optional => 1 },
@@ -59,10 +62,14 @@ sub generate_sdn_config {
my $local_node = PVE::INotify::nodename();
die "missing vxlan tag" if !$tag;
+ die "missing controller" if !$controller;
warn "vlan-aware vnet can't be enabled with evpn plugin" if $vnet->{vlanaware};
my @peers = PVE::Tools::split_list($controller->{'peers'});
- my ($ifaceip, $iface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers);
+# my $bgprouter = PVE::Network::SDN::Controllers::EvpnController::find_bgp_controller($local_node, $controller_cfg);
+# my $loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+ my $loopback = undef;
+ my ($ifaceip, $iface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
my $mtu = 1450;
$mtu = $interfaces_config->{$iface}->{mtu} - 50 if $interfaces_config->{$iface}->{mtu};
diff --git a/PVE/Network/SDN/Zones/Plugin.pm b/PVE/Network/SDN/Zones/Plugin.pm
index 6fc13eb..aa795a3 100644
--- a/PVE/Network/SDN/Zones/Plugin.pm
+++ b/PVE/Network/SDN/Zones/Plugin.pm
@@ -274,10 +274,17 @@ sub get_local_route_ip {
sub find_local_ip_interface_peers {
- my ($peers) = @_;
+ my ($peers, $iface) = @_;
my $network_config = PVE::INotify::read_file('interfaces');
my $ifaces = $network_config->{ifaces};
+
+ #if iface is defined, return ip if exist (if not,try to find it on other ifaces)
+ if ($iface) {
+ my $ip = $ifaces->{$iface}->{address};
+ return ($ip,$iface) if $ip;
+ }
+
#is a local ip member of peers list ?
foreach my $address (@{$peers}) {
while (my $interface = each %$ifaces) {
--
2.20.1
next prev parent reply other threads:[~2020-11-25 9:02 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-11-25 9:01 [pve-devel] [PATCH V2 pve-network 0/4] add ebgp-evpn support Alexandre Derumier
2020-11-25 9:01 ` Alexandre Derumier [this message]
2020-11-25 9:01 ` [pve-devel] [PATCH V2 pve-network 2/4] zones: evpn : add support for loopback Alexandre Derumier
2020-11-25 9:01 ` [pve-devel] [PATCH V2 pve-network 3/4] update test documentation Alexandre Derumier
2020-11-25 9:01 ` [pve-devel] [PATCH V2 pve-network 4/4] sdn: fix : pending parser Alexandre Derumier
2020-11-25 14:00 ` [pve-devel] applied-series: [PATCH V2 pve-network 0/4] add ebgp-evpn support Thomas Lamprecht
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20201125090139.3553272-2-aderumier@odiso.com \
--to=aderumier@odiso.com \
--cc=pve-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.