* [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin @ 2025-05-26 12:19 Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-network v5 2/2] ipam: add test cases for nautobot plugin Hannes Duerr ` (3 more replies) 0 siblings, 4 replies; 6+ messages in thread From: Hannes Duerr @ 2025-05-26 12:19 UTC (permalink / raw) To: pve-devel Nautobot is a network source of truth [0] and as such this plugin offers the possibility to automatically enter IP addresses and subnets (prefixes in Nautobot jargon) into Nautobot as soon as they are created in Proxmox VE. Limitations: * The IPAM plugin currently does not recognize whether VMs/CTs are active/online and initially sets the status to Active but when the VMs/CTs are switched off the status is not deactivated. * Nautobot does not support ranges that can be searched for free ip addresses, so we map as many pool prefixes that we search until we have searched the entire range for free ip addresses * Nautobot has the possibility to map DNS names to IP addresses. However, since we have no standardized way to set the DNS names in Proxmox VE (for containers the container name in Proxmox VE is set as the host name in the container, but this is not possible for VMs) we refrain from setting a DNS name at the moment. * Nautobot does not have a field in IP address objects for a MAC address or to mark them as gateway, so we write this in the comment. [0] https://networktocode.com/nautobot/ Co-authored-by: lou lecrivain <lou.lecrivain@wdz.de> Signed-off-by: Hannes Duerr <h.duerr@proxmox.com> --- Notes: Changes from v4 -> v5: * instead of skipping already created prefixes, search them Changes from v3 -> v4: * Merge the check for empty prefixes/subnets into this commit * Create nautobot_api_request to unify API calls * add update_subnet function * fix add_range_next_freeip * fix and unify error handling * Fix perl style src/PVE/API2/Network/SDN/Ipams.pm | 1 + src/PVE/Network/SDN/Ipams.pm | 3 + src/PVE/Network/SDN/Ipams/Makefile | 2 +- src/PVE/Network/SDN/Ipams/NautobotPlugin.pm | 510 ++++++++++++++++++++ 4 files changed, 515 insertions(+), 1 deletion(-) create mode 100644 src/PVE/Network/SDN/Ipams/NautobotPlugin.pm diff --git a/src/PVE/API2/Network/SDN/Ipams.pm b/src/PVE/API2/Network/SDN/Ipams.pm index 27ead02..8074512 100644 --- a/src/PVE/API2/Network/SDN/Ipams.pm +++ b/src/PVE/API2/Network/SDN/Ipams.pm @@ -12,6 +12,7 @@ use PVE::Network::SDN::Ipams::Plugin; use PVE::Network::SDN::Ipams::PVEPlugin; use PVE::Network::SDN::Ipams::PhpIpamPlugin; use PVE::Network::SDN::Ipams::NetboxPlugin; +use PVE::Network::SDN::Ipams::NautobotPlugin; use PVE::Network::SDN::Dhcp; use PVE::Network::SDN::Vnets; use PVE::Network::SDN::Zones; diff --git a/src/PVE/Network/SDN/Ipams.pm b/src/PVE/Network/SDN/Ipams.pm index c689b8f..2ecb75e 100644 --- a/src/PVE/Network/SDN/Ipams.pm +++ b/src/PVE/Network/SDN/Ipams.pm @@ -12,11 +12,14 @@ use PVE::Network; use PVE::Network::SDN::Ipams::PVEPlugin; use PVE::Network::SDN::Ipams::NetboxPlugin; +use PVE::Network::SDN::Ipams::NautobotPlugin; use PVE::Network::SDN::Ipams::PhpIpamPlugin; use PVE::Network::SDN::Ipams::Plugin; + PVE::Network::SDN::Ipams::PVEPlugin->register(); PVE::Network::SDN::Ipams::NetboxPlugin->register(); +PVE::Network::SDN::Ipams::NautobotPlugin->register(); PVE::Network::SDN::Ipams::PhpIpamPlugin->register(); PVE::Network::SDN::Ipams::Plugin->init(); diff --git a/src/PVE/Network/SDN/Ipams/Makefile b/src/PVE/Network/SDN/Ipams/Makefile index 4e7d65f..75e5b9a 100644 --- a/src/PVE/Network/SDN/Ipams/Makefile +++ b/src/PVE/Network/SDN/Ipams/Makefile @@ -1,4 +1,4 @@ -SOURCES=Plugin.pm PhpIpamPlugin.pm NetboxPlugin.pm PVEPlugin.pm +SOURCES=Plugin.pm PhpIpamPlugin.pm NetboxPlugin.pm PVEPlugin.pm NautobotPlugin.pm PERL5DIR=${DESTDIR}/usr/share/perl5 diff --git a/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm new file mode 100644 index 0000000..a6eb7ff --- /dev/null +++ b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm @@ -0,0 +1,510 @@ +package PVE::Network::SDN::Ipams::NautobotPlugin; + +use strict; +use warnings; +use PVE::INotify; +use PVE::Cluster; +use PVE::Tools; +use NetAddr::IP; +use Net::Subnet qw(subnet_matcher); + +use base('PVE::Network::SDN::Ipams::Plugin'); + +sub type { + return 'nautobot'; +} + +sub properties { + return { + namespace => { + type => 'string', + }, + }; +} + +sub options { + return { + url => { optional => 0 }, + token => { optional => 0 }, + namespace => { optional => 0 }, + fingerprint => { optional => 1 }, + }; +} + +sub default_ip_status { + return 'Active'; +} + +sub nautobot_api_request { + my ($config, $method, $path, $params) = @_; + + return PVE::Network::SDN::api_request( + $method, + "$config->{url}${path}", + [ + 'Content-Type' => 'application/json; charset=UTF-8', + 'Authorization' => "token $config->{token}", + 'Accept' => "application/json", + ], + $params, + $config->{fingerprint}, + ); +} + +sub add_subnet { + my ($class, $config, undef, $subnet, $noerr) = @_; + + my $cidr = $subnet->{cidr}; + my $namespace = $config->{namespace}; + + my $internalid = get_prefix_id($config, $cidr, $noerr); + if ($internalid) { + return if $noerr; + die "could not add the subnet $subnet because it already exists in nautobot\n"; + } + + my $params = { + prefix => $cidr, + namespace => $namespace, + status => default_ip_status(), + }; + + eval { nautobot_api_request($config, "POST", "/ipam/prefixes/", $params); }; + if ($@) { + return if $noerr; + die "error adding the subnet $subnet to nautobot $@\n"; + } +} + +sub update_subnet { + my ($class, $plugin_config, $subnetid, $subnet, $old_subnet, $noerr) = @_; + # dhcp ranges are not supported in nautobot so we don't have to update them +} + +sub del_subnet { + my ($class, $config, $subnetid, $subnet, $noerr) = @_; + + my $cidr = $subnet->{cidr}; + + my $internalid = get_prefix_id($config, $cidr, $noerr); + if (!$internalid) { + warn("could not find delete the subnet $cidr because it does not exist in nautobot\n"); + return; + } + + if (!subnet_is_deletable($config, $subnetid, $subnet, $internalid, $noerr)) { + return if $noerr; + die "could not delete the subnet $cidr, it still contains ip addresses!\n"; + } + + # delete associated gateway IP addresses + $class->empty_subnet($config, $subnetid, $subnet, $internalid, $noerr); + + eval { nautobot_api_request($config, "DELETE", "/ipam/prefixes/$internalid/"); }; + if ($@) { + return if $noerr; + die "error deleting subnet from nautobot: $@\n"; + } + return 1; +} + +sub add_ip { + my ($class, $config, undef, $subnet, $ip, $hostname, $mac, undef, $is_gateway, $noerr) = @_; + + my $mask = $subnet->{mask}; + my $namespace = $config->{namespace}; + + my $description = undef; + if ($is_gateway) { + $description = 'gateway'; + } elsif ($mac) { + $description = "mac:$mac"; + } + + my $params = { + address => "$ip/$mask", + type => "dhcp", + description => $description, + namespace => $namespace, + status => default_ip_status(), + }; + + eval { nautobot_api_request($config, "POST", "/ipam/ip-addresses/", $params); }; + + if ($@) { + if ($is_gateway) { + die "error add subnet ip to ipam: ip $ip already exist: $@" + if !is_ip_gateway($config, $ip, $noerr); + } elsif (!$noerr) { + die "error add subnet ip to ipam: ip already exist: $@"; + } + } +} + +sub add_next_freeip { + my ($class, $config, undef, $subnet, $hostname, $mac, undef, $noerr) = @_; + + my $cidr = $subnet->{cidr}; + my $namespace = $config->{namespace}; + + my $internalid = get_prefix_id($config, $cidr, $noerr); + if (!defined($internalid)) { + return if $noerr; + die "could not find prefix $cidr in nautobot\n"; + } + + my $description = undef; + $description = "mac:$mac" if $mac; + + my $params = { + type => "dhcp", + description => $description, + namespace => $namespace, + status => default_ip_status(), + }; + + my $response = eval { + return nautobot_api_request( + $config, "POST", "/ipam/prefixes/$internalid/available-ips/", $params, + ); + }; + if ($@ || !$response) { + return if $noerr; + die "could not allocate ip in subnet $cidr: $@\n"; + } + + my $ip = NetAddr::IP->new($response->{address}); + + return $ip->addr; +} + +sub find_ip_in_prefix { + my ($config, $prefix_id, $limit, $start_range, $end_range) = @_; + + # Fetch available IPs from the temporary pool and find a matching IP + my $result = eval { + return nautobot_api_request( + $config, + "GET", + "/ipam/prefixes/$prefix_id/available-ips/?limit=$limit", + ); + }; + + # search list for IPs in actual range + if (!$@ && defined($result)) { + foreach my $entry (@$result) { + my $ip = NetAddr::IP->new($entry->{address}); + # comparison is only possible because they are in the same subnet + if ($start_range <= $ip && $ip <= $end_range) { + return $ip->addr; + } + } + } + return; +} + +sub add_range_next_freeip { + my ($class, $config, $subnet, $range, $data, $noerr) = @_; + + my $cidr = NetAddr::IP->new($subnet->{cidr}); + my $namespace = $config->{namespace}; + + # Nautobot does not support IP ranges, only prefixes. + # Therefore we divide the range into smaller pool prefixes, + # each containing 256 addresses, and search them for available IPs + my $prefix_size = $cidr->version == 4 ? 24 : 120; + my $increment = 256; + my $found_ip = undef; + + my $start_range = NetAddr::IP->new($range->{'start-address'}, $prefix_size); + my $end_range = NetAddr::IP->new($range->{'end-address'}, $prefix_size); + my $matcher = subnet_matcher($end_range->cidr); + my $current_ip = $start_range; + + while (1) { + my $current_cidr = $current_ip->addr . "/$prefix_size"; + + my $params = { + prefix => $current_cidr, + namespace => $namespace, + status => default_ip_status(), + type => "pool", + }; + + my $prefix_id = get_prefix_id($config, $current_cidr, $noerr); + if ($prefix_id) { + # search the existing prefix for valid ip + $found_ip = + find_ip_in_prefix($config, $prefix_id, $increment, $start_range, $end_range); + } else { + # create temporary pool prefix + my $temp_prefix = + eval { return nautobot_api_request($config, "POST", "/ipam/prefixes/", $params); }; + + my $temp_prefix_id = $temp_prefix->{id}; + + # search temporarly created prefix + $found_ip = + find_ip_in_prefix($config, $temp_prefix_id, $increment, $start_range, $end_range); + + # Delete temporary prefix pool + eval { nautobot_api_request($config, "DELETE", "/ipam/prefixes/$temp_prefix_id/"); }; + } + + last if $found_ip; + + # we searched the last pool prefix + last if $matcher->($current_ip->addr); + + $current_ip = $current_ip->plus($increment); + } + + if (!$found_ip) { + return if $noerr; + die "could not allocate ip in the range " + . $start_range->addr . " - " + . $end_range->addr + . ": $@\n"; + } + + $class->add_ip( + $config, + undef, + $subnet, + $found_ip, + $data->{hostname}, + $data->{mac}, + undef, + 0, + $noerr, + ); + + return $found_ip; +} + +sub update_ip { + my ($class, $config, $subnetid, $subnet, $ip, $hostname, $mac, undef, $is_gateway, $noerr) = @_; + + my $mask = $subnet->{mask}; + my $namespace = $config->{namespace}; + + my $description = undef; + if ($is_gateway) { + $description = 'gateway'; + } elsif ($mac) { + $description = "mac:$mac"; + } + + my $params = { + address => "$ip/$mask", + type => "dhcp", + description => $description, + namespace => $namespace, + status => default_ip_status(), + }; + + my $ip_id = get_ip_id($config, $ip, $noerr); + if (!defined($ip_id)) { + return if $noerr; + die "could not find the ip $ip in nautobot\n"; + } + + eval { nautobot_api_request($config, "PATCH", "/ipam/ip-addresses/$ip_id/", $params); }; + if ($@) { + return if $noerr; + die "error updating ip $ip: $@"; + } +} + +sub del_ip { + my ($class, $config, undef, undef, $ip, $noerr) = @_; + + return if !$ip; + + my $ip_id = get_ip_id($config, $ip, $noerr); + if (!defined($ip_id)) { + warn("could not find the ip $ip in nautobot\n"); + return; + } + + eval { nautobot_api_request($config, "DELETE", "/ipam/ip-addresses/$ip_id/"); }; + if ($@) { + return if $noerr; + die "error deleting ip $ip : $@\n"; + } + + return 1; +} + +sub empty_subnet { + my ($class, $config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; + + my $namespace = $config->{namespace}; + + my $response = eval { + return nautobot_api_request( + $config, + "GET", + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", + ); + }; + if ($@) { + return if $noerr; + die "could not find the subnet $subnet in nautobot: $@\n"; + } + + for my $ip (@{ $response->{results} }) { + del_ip($class, $config, undef, undef, $ip->{host}, $noerr); + } + + return 1; +} + +sub subnet_is_deletable { + my ($config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; + + my $namespace = $config->{namespace}; + + my $response = eval { + return nautobot_api_request( + $config, + "GET", + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", + ); + }; + if ($@) { + return if $noerr; + die "error querying prefix $subnet: $@\n"; + } + my $n_ips = scalar $response->{results}->@*; + + # least costly check operation 1st + return 1 if ($n_ips == 0); + + for my $ip (values $response->{results}->@*) { + if (!is_ip_gateway($config, $ip->{host}, $noerr)) { + # some remaining IP is not a gateway so we can't delete the subnet + return 0; + } + } + #all remaining IPs are gateways + return 1; +} + +sub verify_api { + my ($class, $config) = @_; + + my $namespace = $config->{namespace}; + + # check if the namespace and the status "Active" exist + eval { + get_namespace_id($config, $namespace) // die "namespace $namespace does not exist"; + get_status_id($config, default_ip_status()) + // die "the status " . default_ip_status() . " does not exist"; + }; + if ($@) { + die "could not use nautobot api: $@\n"; + } +} + +sub get_ips_from_mac { + my ($class, $config, $mac, $zone) = @_; + + my $ip4 = undef; + my $ip6 = undef; + + my $data = eval { nautobot_api_request($config, "GET", "/ipam/ip-addresses/?q=$mac"); }; + if ($@) { + die "could not query ip address entry for mac $mac: $@"; + } + + for my $ip (@{ $data->{results} }) { + if ($ip->{ip_version} == 4 && !$ip4) { + ($ip4, undef) = split(/\//, $ip->{address}); + } + + if ($ip->{ip_version} == 6 && !$ip6) { + ($ip6, undef) = split(/\//, $ip->{address}); + } + } + + return ($ip4, $ip6); +} + +sub on_update_hook { + my ($class, $config) = @_; + + PVE::Network::SDN::Ipams::NautobotPlugin::verify_api($class, $config); +} + +sub get_ip_id { + my ($config, $ip, $noerr) = @_; + + my $result = + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; + if ($@) { + return if $noerr; + die "error while querying for ip $ip id: $@\n"; + } + + my $data = @{ $result->{results} }[0]; + return $data->{id}; +} + +sub get_prefix_id { + my ($config, $cidr, $noerr) = @_; + + my $result = + eval { return nautobot_api_request($config, "GET", "/ipam/prefixes/?prefix=$cidr"); }; + if ($@) { + return if $noerr; + die "error while querying for cidr $cidr prefix id: $@\n"; + } + + my $data = @{ $result->{results} }[0]; + return $data->{id}; +} + +sub get_namespace_id { + my ($config, $namespace, $noerr) = @_; + + my $result = + eval { return nautobot_api_request($config, "GET", "/ipam/namespaces/?name=$namespace"); }; + if ($@) { + return if $noerr; + die "error while querying for namespace $namespace id: $@\n"; + } + + my $data = @{ $result->{results} }[0]; + return $data->{id}; +} + +sub get_status_id { + my ($config, $status, $noerr) = @_; + + my $result = + eval { return nautobot_api_request($config, "GET", "/extras/statuses/?name=$status"); }; + if ($@) { + return if $noerr; + die "error while querying for status $status id: $@\n"; + } + + my $data = @{ $result->{results} }[0]; + return $data->{id}; +} + +sub is_ip_gateway { + my ($config, $ip, $noerr) = @_; + + my $result = + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; + if ($@) { + return if $noerr; + die "error while checking if $ip is a gateway: $@\n"; + } + + my $data = @{ $result->{results} }[0]; + return $data->{description} eq 'gateway'; +} + +1; -- 2.39.5 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [PATCH pve-network v5 2/2] ipam: add test cases for nautobot plugin 2025-05-26 12:19 [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Hannes Duerr @ 2025-05-26 12:19 ` Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-docs v5 1/1] add documentation for nautobot ipam plugin Hannes Duerr ` (2 subsequent siblings) 3 siblings, 0 replies; 6+ messages in thread From: Hannes Duerr @ 2025-05-26 12:19 UTC (permalink / raw) To: pve-devel Co-authored-by: Lou Lecrivain <lou.lecrivain@wdz.de> Signed-off-by: Hannes Duerr <h.duerr@proxmox.com> --- src/test/ipams/nautobot/expected.add_ip | 11 +++++++++ .../ipams/nautobot/expected.add_ip_notgateway | 11 +++++++++ .../ipams/nautobot/expected.add_next_freeip | 11 +++++++++ src/test/ipams/nautobot/expected.add_subnet | 11 +++++++++ src/test/ipams/nautobot/expected.del_ip | 11 +++++++++ src/test/ipams/nautobot/expected.update_ip | 11 +++++++++ src/test/ipams/nautobot/ipam_config | 24 +++++++++++++++++++ src/test/ipams/nautobot/sdn_config | 20 ++++++++++++++++ src/test/ipams/netbox/ipam_config | 8 ++++++- src/test/ipams/phpipam/ipam_config | 8 ++++++- 10 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 src/test/ipams/nautobot/expected.add_ip create mode 100644 src/test/ipams/nautobot/expected.add_ip_notgateway create mode 100644 src/test/ipams/nautobot/expected.add_next_freeip create mode 100644 src/test/ipams/nautobot/expected.add_subnet create mode 100644 src/test/ipams/nautobot/expected.del_ip create mode 100644 src/test/ipams/nautobot/expected.update_ip create mode 100644 src/test/ipams/nautobot/ipam_config create mode 100644 src/test/ipams/nautobot/sdn_config diff --git a/src/test/ipams/nautobot/expected.add_ip b/src/test/ipams/nautobot/expected.add_ip new file mode 100644 index 0000000..bec1f3a --- /dev/null +++ b/src/test/ipams/nautobot/expected.add_ip @@ -0,0 +1,11 @@ +bless( { + '_content' => '{"address":"10.0.0.1/24","description":"gateway","namespace":"TestNamespace","status":"Active","type":"dhcp"}', + '_headers' => bless( { + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers' ), + '_max_body_size' => undef, + '_method' => 'POST', + '_uri' => bless( do{\(my $o = 'http://localhost:8080/api/ipam/ip-addresses/')}, 'URI::http' ) + }, 'HTTP::Request' ); diff --git a/src/test/ipams/nautobot/expected.add_ip_notgateway b/src/test/ipams/nautobot/expected.add_ip_notgateway new file mode 100644 index 0000000..bae92e6 --- /dev/null +++ b/src/test/ipams/nautobot/expected.add_ip_notgateway @@ -0,0 +1,11 @@ +bless( { + '_content' => '{"address":"10.0.0.1/24","description":"mac:da:65:8f:18:9b:6f","namespace":"TestNamespace","status":"Active","type":"dhcp"}', + '_headers' => bless( { + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers' ), + '_max_body_size' => undef, + '_method' => 'POST', + '_uri' => bless( do{\(my $o = 'http://localhost:8080/api/ipam/ip-addresses/')}, 'URI::http' ) + }, 'HTTP::Request' ); diff --git a/src/test/ipams/nautobot/expected.add_next_freeip b/src/test/ipams/nautobot/expected.add_next_freeip new file mode 100644 index 0000000..0181719 --- /dev/null +++ b/src/test/ipams/nautobot/expected.add_next_freeip @@ -0,0 +1,11 @@ +bless( { + '_content' => '{"description":"mac:da:65:8f:18:9b:6f","namespace":"TestNamespace","status":"Active","type":"dhcp"}', + '_headers' => bless( { + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers' ), + '_max_body_size' => undef, + '_method' => 'POST', + '_uri' => bless( do{\(my $o = 'http://localhost:8080/api/ipam/prefixes/1/available-ips/')}, 'URI::http' ) + }, 'HTTP::Request' ); diff --git a/src/test/ipams/nautobot/expected.add_subnet b/src/test/ipams/nautobot/expected.add_subnet new file mode 100644 index 0000000..78f911d --- /dev/null +++ b/src/test/ipams/nautobot/expected.add_subnet @@ -0,0 +1,11 @@ +bless({ + '_content' => '{"namespace":"TestNamespace","prefix":"10.0.0.0/24","status":"Active"}', + '_headers' => bless({ + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers'), + '_max_body_size' => undef, + '_method' => 'POST', + '_uri' => bless(do{\(my $o = 'http://localhost:8080/api/ipam/prefixes/')}, 'URI::http'), +}, 'HTTP::Request'); diff --git a/src/test/ipams/nautobot/expected.del_ip b/src/test/ipams/nautobot/expected.del_ip new file mode 100644 index 0000000..eaa133e --- /dev/null +++ b/src/test/ipams/nautobot/expected.del_ip @@ -0,0 +1,11 @@ +bless( { + '_content' => '', + '_headers' => bless( { + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers' ), + '_max_body_size' => undef, + '_method' => 'DELETE', + '_uri' => bless( do{\(my $o = 'http://localhost:8080/api/ipam/ip-addresses/1/')}, 'URI::http' ) + }, 'HTTP::Request' ); diff --git a/src/test/ipams/nautobot/expected.update_ip b/src/test/ipams/nautobot/expected.update_ip new file mode 100644 index 0000000..be6df47 --- /dev/null +++ b/src/test/ipams/nautobot/expected.update_ip @@ -0,0 +1,11 @@ +bless( { + '_content' => '{"address":"10.0.0.1/24","description":"gateway","namespace":"TestNamespace","status":"Active","type":"dhcp"}', + '_headers' => bless( { + 'accept' => 'application/json', + 'authorization' => 'token FAKETESTTOKEN', + 'content-type' => 'application/json; charset=UTF-8', + }, 'HTTP::Headers' ), + '_max_body_size' => undef, + '_method' => 'PATCH', + '_uri' => bless( do{\(my $o = 'http://localhost:8080/api/ipam/ip-addresses/1/')}, 'URI::http' ) + }, 'HTTP::Request' ); diff --git a/src/test/ipams/nautobot/ipam_config b/src/test/ipams/nautobot/ipam_config new file mode 100644 index 0000000..014d6b1 --- /dev/null +++ b/src/test/ipams/nautobot/ipam_config @@ -0,0 +1,24 @@ +{ + 'ids' => { + 'phpipam' => { + 'url' => 'https://localhost/api/apiadmin', + 'type' => 'phpipam', + 'section' => 1, + 'token' => 'JPHkPSLB4O_XL-GQz4qtEFmNpx-99Htw' + }, + 'pve' => { + 'type' => 'pve' + }, + 'netbox' => { + 'token' => '0123456789abcdef0123456789abcdef01234567', + 'type' => 'netbox', + 'url' => 'http://localhost:8000/api' + }, + 'nautobot' => { + 'url' => 'http://localhost:8080/api', + 'type' => 'nautobot', + 'token' => 'FAKETESTTOKEN', + 'namespace' => 'TestNamespace' + } + }, +} diff --git a/src/test/ipams/nautobot/sdn_config b/src/test/ipams/nautobot/sdn_config new file mode 100644 index 0000000..784cd95 --- /dev/null +++ b/src/test/ipams/nautobot/sdn_config @@ -0,0 +1,20 @@ +{ + version => 1, + vnets => { + ids => { + myvnet => { type => "vnet", zone => "myzone" }, + }, + }, + + zones => { + ids => { myzone => { ipam => "nautobot" } }, + }, + + subnets => { + ids => { 'myzone-10.0.0.0-24' => { + 'type' => 'subnet', + 'vnet' => 'myvnet', + } + } + } +} diff --git a/src/test/ipams/netbox/ipam_config b/src/test/ipams/netbox/ipam_config index a33be30..6a6fe44 100644 --- a/src/test/ipams/netbox/ipam_config +++ b/src/test/ipams/netbox/ipam_config @@ -13,6 +13,12 @@ 'token' => '0123456789abcdef0123456789abcdef01234567', 'type' => 'netbox', 'url' => 'http://localhost:8000/api' - } + }, + 'nautobot' => { + 'url' => 'http://localhost:8080/api', + 'type' => 'nautobot', + 'token' => 'FAKETESTTOKEN', + 'namespace' => 'TestNamespace' + } }, } diff --git a/src/test/ipams/phpipam/ipam_config b/src/test/ipams/phpipam/ipam_config index a33be30..014d6b1 100644 --- a/src/test/ipams/phpipam/ipam_config +++ b/src/test/ipams/phpipam/ipam_config @@ -13,6 +13,12 @@ 'token' => '0123456789abcdef0123456789abcdef01234567', 'type' => 'netbox', 'url' => 'http://localhost:8000/api' - } + }, + 'nautobot' => { + 'url' => 'http://localhost:8080/api', + 'type' => 'nautobot', + 'token' => 'FAKETESTTOKEN', + 'namespace' => 'TestNamespace' + } }, } -- 2.39.5 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [PATCH pve-docs v5 1/1] add documentation for nautobot ipam plugin 2025-05-26 12:19 [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-network v5 2/2] ipam: add test cases for nautobot plugin Hannes Duerr @ 2025-05-26 12:19 ` Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-manager v5 1/1] ipam: add UI dialog " Hannes Duerr 2025-05-28 9:32 ` [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Stefan Hanreich 3 siblings, 0 replies; 6+ messages in thread From: Hannes Duerr @ 2025-05-26 12:19 UTC (permalink / raw) To: pve-devel; +Cc: Jonatan Crystall Co-authored-by: Jonatan Crystall <jonatan.crystall@gwdg.de> Co-authored-by: Lou Lecrivain <lou.lecrivain@wdz.de> Signed-off-by: Hannes Duerr <h.duerr@proxmox.com> --- pvesdn.adoc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pvesdn.adoc b/pvesdn.adoc index 5e58cc3..2967ec2 100644 --- a/pvesdn.adoc +++ b/pvesdn.adoc @@ -563,6 +563,25 @@ URL:: The NetBox REST API endpoint: `http://yournetbox.domain.com/api` Token:: An API access token +[[pvesdn_ipam_plugin_nautobot]] +Nautobot IPAM Plugin +~~~~~~~~~~~~~~~~~~ + +link:https://github.com/nautobot/nautobot[Nautobot] is an open-source IP +Address Management (IPAM) and datacenter infrastructure management (DCIM) tool. + +To integrate Nautobot with {pve} SDN, create an API token in Nautobot as described +here: +https://docs.nautobot.com/projects/core/en/stable/user-guide/platform-functionality/users/token/ + +The Nautobot configuration properties are: + +URL:: The Nautobot REST API endpoint: `http://yournautobot.domain.com/api` + +Token:: An API access token + +Namespace:: Nautobot IP Namespace + [[pvesdn_ipam_plugin_phpipam]] phpIPAM Plugin -- 2.39.5 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
* [pve-devel] [PATCH pve-manager v5 1/1] ipam: add UI dialog for nautobot ipam plugin 2025-05-26 12:19 [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-network v5 2/2] ipam: add test cases for nautobot plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-docs v5 1/1] add documentation for nautobot ipam plugin Hannes Duerr @ 2025-05-26 12:19 ` Hannes Duerr 2025-05-28 9:32 ` [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Stefan Hanreich 3 siblings, 0 replies; 6+ messages in thread From: Hannes Duerr @ 2025-05-26 12:19 UTC (permalink / raw) To: pve-devel; +Cc: Jonatan Crystall Co-authored-by: Jonatan Crystall <jonatan.crystall@gwdg.de> Co-authored-by: Lou Lecrivain <lou.lecrivain@wdz.de> Signed-off-by: Hannes Duerr <h.duerr@proxmox.com> --- www/manager6/Makefile | 1 + www/manager6/Utils.js | 5 +++ www/manager6/sdn/ipams/NautobotEdit.js | 62 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 www/manager6/sdn/ipams/NautobotEdit.js diff --git a/www/manager6/Makefile b/www/manager6/Makefile index fdf0e816..e873e6c5 100644 --- a/www/manager6/Makefile +++ b/www/manager6/Makefile @@ -296,6 +296,7 @@ JSSRC= \ sdn/IpamView.js \ sdn/ipams/Base.js \ sdn/ipams/NetboxEdit.js \ + sdn/ipams/NautobotEdit.js \ sdn/ipams/PVEIpamEdit.js \ sdn/ipams/PhpIpamEdit.js \ sdn/DnsView.js \ diff --git a/www/manager6/Utils.js b/www/manager6/Utils.js index 1f6778cd..a95c21ce 100644 --- a/www/manager6/Utils.js +++ b/www/manager6/Utils.js @@ -942,6 +942,11 @@ Ext.define('PVE.Utils', { ipanel: 'PhpIpamInputPanel', faIcon: 'th', }, + nautobot: { + name: 'Nautobot', + ipanel: 'NautobotInputPanel', + faIcon: 'th', + }, }, sdndnsSchema: { diff --git a/www/manager6/sdn/ipams/NautobotEdit.js b/www/manager6/sdn/ipams/NautobotEdit.js new file mode 100644 index 00000000..46162d98 --- /dev/null +++ b/www/manager6/sdn/ipams/NautobotEdit.js @@ -0,0 +1,62 @@ +Ext.define('PVE.sdn.ipams.NautobotInputPanel', { + extend: 'PVE.panel.SDNIpamBase', + + onlineHelp: 'pvesdn_ipam_plugin_nautobot', + + onGetValues: function(values) { + var me = this; + + if (me.isCreate) { + values.type = me.type; + } else { + delete values.ipam; + } + + return values; + }, + + initComponent: function() { + var me = this; + + me.column1 = [ + { + xtype: me.isCreate ? 'textfield' : 'displayfield', + name: 'ipam', + maxLength: 10, + value: me.zone || '', + fieldLabel: 'ID', + allowBlank: false, + }, + { + xtype: 'textfield', + name: 'token', + fieldLabel: gettext('Token'), + allowBlank: false, + }, + ]; + me.column2 = [ + { + xtype: 'textfield', + name: 'url', + fieldLabel: gettext('URL'), + allowBlank: false, + }, + { + xtype: 'textfield', + name: 'namespace', + fieldLabel: gettext('Namespace'), + allowBlank: false, + }, + ]; + me.columnB = [ + { + xtype: 'pmxFingerprintField', + name: 'fingerprint', + value: me.isCreate ? null : undefined, + deleteEmpty: !me.isCreate, + }, + ]; + + me.callParent(); + }, +}); -- 2.39.5 _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin 2025-05-26 12:19 [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Hannes Duerr ` (2 preceding siblings ...) 2025-05-26 12:19 ` [pve-devel] [PATCH pve-manager v5 1/1] ipam: add UI dialog " Hannes Duerr @ 2025-05-28 9:32 ` Stefan Hanreich 2025-06-03 12:25 ` Hannes Duerr 3 siblings, 1 reply; 6+ messages in thread From: Stefan Hanreich @ 2025-05-28 9:32 UTC (permalink / raw) To: Proxmox VE development discussion, Hannes Duerr Tested the following things and they worked: * Create a new simple zone with Nautobot IPAM * Create a VNet with a Subnet + GW -> IP allocated * Create a VM on that VNet -> IP allocated * Create a CT on that VNet -> IP allocated * Change network device to another bridge -> IP deleted * Change back to IPAM VNet -> IP allocated * Live-Migrate VM * Check if VM / CT get IP via DHCP * try to delete subnet with entries -> fails * delete network devices and try again -> succeeds * update gateway -> changes in IPAM * add DHCP range and test C/R/U/D network device again I added a second IPAM with a different namespace, and this is where some problems started to occur. The PVE Web UI threw an error when trying to create the same subnet, but in a different namespace: create sdn subnet object failed: could not add the subnet HASH(0x58bc2d033e10) because it already exists in nautobot (500) (also note that here we need to print the id field of the hash in the error message) Some additional errors occurred when I manually created the subnet in the SDN configuration, because it seems like many functions that fetch IDs don't seem to consider the namespace of the objects they're fetching (e.g. get_ip_id / get_prefix_id / ...). This means that IP addresses get created in the wrong prefix, because the plugin just takes the first prefix found, regardless of their namespace. Might make issues for users that use a separated namespace for PVE, but have the same prefix configured in other namespaces. Since most methods use the ID returned by the helper functions, this should be easily fixed by adjusting those helpers. some additional comments inline On 5/26/25 14:19, Hannes Duerr wrote: > Nautobot is a network source of truth [0] and as such this plugin offers > the possibility to automatically enter IP addresses and subnets > (prefixes in Nautobot jargon) into Nautobot as soon as they are created > in Proxmox VE. > > Limitations: > * The IPAM plugin currently does not recognize whether VMs/CTs are > active/online and initially sets the status to Active but when the > VMs/CTs are switched off the status is not deactivated. > * Nautobot does not support ranges that can be searched for free ip > addresses, so we map as many pool prefixes that we search until we > have searched the entire range for free ip addresses > * Nautobot has the possibility to map DNS names to IP addresses. > However, since we have no standardized way to set the DNS names in > Proxmox VE (for containers the container name in Proxmox VE is set as > the host name in the container, but this is not possible for VMs) we > refrain from setting a DNS name at the moment. > * Nautobot does not have a field in IP address objects for a MAC address > or to mark them as gateway, so we write this in the comment. > > [0] https://networktocode.com/nautobot/ > > Co-authored-by: lou lecrivain <lou.lecrivain@wdz.de> > Signed-off-by: Hannes Duerr <h.duerr@proxmox.com> > --- > > Notes: > Changes from v4 -> v5: > * instead of skipping already created prefixes, search them > > Changes from v3 -> v4: > * Merge the check for empty prefixes/subnets into this commit > * Create nautobot_api_request to unify API calls > * add update_subnet function > * fix add_range_next_freeip > * fix and unify error handling > * Fix perl style > > src/PVE/API2/Network/SDN/Ipams.pm | 1 + > src/PVE/Network/SDN/Ipams.pm | 3 + > src/PVE/Network/SDN/Ipams/Makefile | 2 +- > src/PVE/Network/SDN/Ipams/NautobotPlugin.pm | 510 ++++++++++++++++++++ > 4 files changed, 515 insertions(+), 1 deletion(-) > create mode 100644 src/PVE/Network/SDN/Ipams/NautobotPlugin.pm > > diff --git a/src/PVE/API2/Network/SDN/Ipams.pm b/src/PVE/API2/Network/SDN/Ipams.pm > index 27ead02..8074512 100644 > --- a/src/PVE/API2/Network/SDN/Ipams.pm > +++ b/src/PVE/API2/Network/SDN/Ipams.pm > @@ -12,6 +12,7 @@ use PVE::Network::SDN::Ipams::Plugin; > use PVE::Network::SDN::Ipams::PVEPlugin; > use PVE::Network::SDN::Ipams::PhpIpamPlugin; > use PVE::Network::SDN::Ipams::NetboxPlugin; > +use PVE::Network::SDN::Ipams::NautobotPlugin; > use PVE::Network::SDN::Dhcp; > use PVE::Network::SDN::Vnets; > use PVE::Network::SDN::Zones; > diff --git a/src/PVE/Network/SDN/Ipams.pm b/src/PVE/Network/SDN/Ipams.pm > index c689b8f..2ecb75e 100644 > --- a/src/PVE/Network/SDN/Ipams.pm > +++ b/src/PVE/Network/SDN/Ipams.pm > @@ -12,11 +12,14 @@ use PVE::Network; > > use PVE::Network::SDN::Ipams::PVEPlugin; > use PVE::Network::SDN::Ipams::NetboxPlugin; > +use PVE::Network::SDN::Ipams::NautobotPlugin; > use PVE::Network::SDN::Ipams::PhpIpamPlugin; > use PVE::Network::SDN::Ipams::Plugin; > > + > PVE::Network::SDN::Ipams::PVEPlugin->register(); > PVE::Network::SDN::Ipams::NetboxPlugin->register(); > +PVE::Network::SDN::Ipams::NautobotPlugin->register(); > PVE::Network::SDN::Ipams::PhpIpamPlugin->register(); > PVE::Network::SDN::Ipams::Plugin->init(); > > diff --git a/src/PVE/Network/SDN/Ipams/Makefile b/src/PVE/Network/SDN/Ipams/Makefile > index 4e7d65f..75e5b9a 100644 > --- a/src/PVE/Network/SDN/Ipams/Makefile > +++ b/src/PVE/Network/SDN/Ipams/Makefile > @@ -1,4 +1,4 @@ > -SOURCES=Plugin.pm PhpIpamPlugin.pm NetboxPlugin.pm PVEPlugin.pm > +SOURCES=Plugin.pm PhpIpamPlugin.pm NetboxPlugin.pm PVEPlugin.pm NautobotPlugin.pm > > > PERL5DIR=${DESTDIR}/usr/share/perl5 > diff --git a/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm > new file mode 100644 > index 0000000..a6eb7ff > --- /dev/null > +++ b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm > @@ -0,0 +1,510 @@ > +package PVE::Network::SDN::Ipams::NautobotPlugin; > + > +use strict; > +use warnings; > +use PVE::INotify; > +use PVE::Cluster; > +use PVE::Tools; > +use NetAddr::IP; > +use Net::Subnet qw(subnet_matcher); > + > +use base('PVE::Network::SDN::Ipams::Plugin'); > + > +sub type { > + return 'nautobot'; > +} > + > +sub properties { > + return { > + namespace => { > + type => 'string', > + }, > + }; > +} missing description here. afaict we should also add descriptions for url / token? since they're not defined in the base plugin. > +sub options { > + return { > + url => { optional => 0 }, > + token => { optional => 0 }, > + namespace => { optional => 0 }, > + fingerprint => { optional => 1 }, > + }; > +} > + > +sub default_ip_status { > + return 'Active'; > +} could maybe just be a constant my $DEFAULT_IP_STATUS; ? > + > +sub nautobot_api_request { > + my ($config, $method, $path, $params) = @_; > + > + return PVE::Network::SDN::api_request( > + $method, > + "$config->{url}${path}", > + [ > + 'Content-Type' => 'application/json; charset=UTF-8', > + 'Authorization' => "token $config->{token}", > + 'Accept' => "application/json", > + ], > + $params, > + $config->{fingerprint}, > + ); > +} > + > +sub add_subnet { > + my ($class, $config, undef, $subnet, $noerr) = @_; > + > + my $cidr = $subnet->{cidr}; > + my $namespace = $config->{namespace}; > + > + my $internalid = get_prefix_id($config, $cidr, $noerr); > + if ($internalid) { > + return if $noerr; > + die "could not add the subnet $subnet because it already exists in nautobot\n"; should use the CIDR field of the subnet, since $subnet is a hash > + } > + > + my $params = { > + prefix => $cidr, > + namespace => $namespace, > + status => default_ip_status(), > + }; > + > + eval { nautobot_api_request($config, "POST", "/ipam/prefixes/", $params); }; > + if ($@) { > + return if $noerr; > + die "error adding the subnet $subnet to nautobot $@\n"; > + } > +} > + > +sub update_subnet { > + my ($class, $plugin_config, $subnetid, $subnet, $old_subnet, $noerr) = @_; > + # dhcp ranges are not supported in nautobot so we don't have to update them > +} > + > +sub del_subnet { > + my ($class, $config, $subnetid, $subnet, $noerr) = @_; > + > + my $cidr = $subnet->{cidr}; > + > + my $internalid = get_prefix_id($config, $cidr, $noerr); > + if (!$internalid) { > + warn("could not find delete the subnet $cidr because it does not exist in nautobot\n"); > + return; > + } > + > + if (!subnet_is_deletable($config, $subnetid, $subnet, $internalid, $noerr)) { > + return if $noerr; > + die "could not delete the subnet $cidr, it still contains ip addresses!\n"; > + } > + > + # delete associated gateway IP addresses > + $class->empty_subnet($config, $subnetid, $subnet, $internalid, $noerr); > + > + eval { nautobot_api_request($config, "DELETE", "/ipam/prefixes/$internalid/"); }; > + if ($@) { > + return if $noerr; > + die "error deleting subnet from nautobot: $@\n"; > + } > + return 1; > +} > + > +sub add_ip { > + my ($class, $config, undef, $subnet, $ip, $hostname, $mac, undef, $is_gateway, $noerr) = @_; > + > + my $mask = $subnet->{mask}; > + my $namespace = $config->{namespace}; > + > + my $description = undef; > + if ($is_gateway) { > + $description = 'gateway'; > + } elsif ($mac) { > + $description = "mac:$mac"; > + } > + > + my $params = { > + address => "$ip/$mask", > + type => "dhcp", > + description => $description, > + namespace => $namespace, > + status => default_ip_status(), > + }; > + > + eval { nautobot_api_request($config, "POST", "/ipam/ip-addresses/", $params); }; > + > + if ($@) { > + if ($is_gateway) { > + die "error add subnet ip to ipam: ip $ip already exist: $@" > + if !is_ip_gateway($config, $ip, $noerr); > + } elsif (!$noerr) { > + die "error add subnet ip to ipam: ip already exist: $@"; > + } > + } > +} > + > +sub add_next_freeip { > + my ($class, $config, undef, $subnet, $hostname, $mac, undef, $noerr) = @_; > + > + my $cidr = $subnet->{cidr}; > + my $namespace = $config->{namespace}; > + > + my $internalid = get_prefix_id($config, $cidr, $noerr); > + if (!defined($internalid)) { > + return if $noerr; > + die "could not find prefix $cidr in nautobot\n"; > + } > + > + my $description = undef; > + $description = "mac:$mac" if $mac; > + > + my $params = { > + type => "dhcp", > + description => $description, > + namespace => $namespace, > + status => default_ip_status(), > + }; > + > + my $response = eval { > + return nautobot_api_request( > + $config, "POST", "/ipam/prefixes/$internalid/available-ips/", $params, > + ); > + }; > + if ($@ || !$response) { > + return if $noerr; > + die "could not allocate ip in subnet $cidr: $@\n"; > + } > + > + my $ip = NetAddr::IP->new($response->{address}); > + > + return $ip->addr; > +} > + > +sub find_ip_in_prefix { > + my ($config, $prefix_id, $limit, $start_range, $end_range) = @_; > + > + # Fetch available IPs from the temporary pool and find a matching IP > + my $result = eval { > + return nautobot_api_request( > + $config, > + "GET", > + "/ipam/prefixes/$prefix_id/available-ips/?limit=$limit", > + ); > + }; > + > + # search list for IPs in actual range > + if (!$@ && defined($result)) { > + foreach my $entry (@$result) { > + my $ip = NetAddr::IP->new($entry->{address}); > + # comparison is only possible because they are in the same subnet > + if ($start_range <= $ip && $ip <= $end_range) { > + return $ip->addr; > + } > + } > + } > + return; > +} > + > +sub add_range_next_freeip { > + my ($class, $config, $subnet, $range, $data, $noerr) = @_; > + > + my $cidr = NetAddr::IP->new($subnet->{cidr}); > + my $namespace = $config->{namespace}; > + > + # Nautobot does not support IP ranges, only prefixes. > + # Therefore we divide the range into smaller pool prefixes, > + # each containing 256 addresses, and search them for available IPs > + my $prefix_size = $cidr->version == 4 ? 24 : 120; maybe we could default to the prefix size of the subnet if it is smaller than 24 / 120? Not so likely for IPv6, but for IPv4 it'd make sense? > + my $increment = 256; > + my $found_ip = undef; > + > + my $start_range = NetAddr::IP->new($range->{'start-address'}, $prefix_size); > + my $end_range = NetAddr::IP->new($range->{'end-address'}, $prefix_size); > + my $matcher = subnet_matcher($end_range->cidr); > + my $current_ip = $start_range; > + > + while (1) { > + my $current_cidr = $current_ip->addr . "/$prefix_size"; > + > + my $params = { > + prefix => $current_cidr, > + namespace => $namespace, > + status => default_ip_status(), > + type => "pool", > + }; > + > + my $prefix_id = get_prefix_id($config, $current_cidr, $noerr); > + if ($prefix_id) { > + # search the existing prefix for valid ip > + $found_ip = > + find_ip_in_prefix($config, $prefix_id, $increment, $start_range, $end_range); > + } else { > + # create temporary pool prefix > + my $temp_prefix = > + eval { return nautobot_api_request($config, "POST", "/ipam/prefixes/", $params); }; should probably do some error handling? > + > + my $temp_prefix_id = $temp_prefix->{id}; > + > + # search temporarly created prefix > + $found_ip = > + find_ip_in_prefix($config, $temp_prefix_id, $increment, $start_range, $end_range); > + > + # Delete temporary prefix pool > + eval { nautobot_api_request($config, "DELETE", "/ipam/prefixes/$temp_prefix_id/"); }; here as well > + } > + > + last if $found_ip; > + > + # we searched the last pool prefix > + last if $matcher->($current_ip->addr); > + > + $current_ip = $current_ip->plus($increment); > + } > + > + if (!$found_ip) { > + return if $noerr; > + die "could not allocate ip in the range " > + . $start_range->addr . " - " > + . $end_range->addr > + . ": $@\n"; > + } > + > + $class->add_ip( > + $config, > + undef, > + $subnet, > + $found_ip, > + $data->{hostname}, > + $data->{mac}, > + undef, > + 0, > + $noerr, > + ); > + > + return $found_ip; > +} we might really wanna upstream a little patch that adds at least offset support, or defines a start / end range. Seems like this shouldn't be too hard with the current implementation in nautobot. That would simplify this method by a lot. We could also just deny using DHCP ranges with the Nautobot plugin for now... see https://github.com/nautobot/nautobot/blob/develop/nautobot/ipam/api/views.py#L309 > +sub update_ip { > + my ($class, $config, $subnetid, $subnet, $ip, $hostname, $mac, undef, $is_gateway, $noerr) = @_; > + > + my $mask = $subnet->{mask}; > + my $namespace = $config->{namespace}; > + > + my $description = undef; > + if ($is_gateway) { > + $description = 'gateway'; > + } elsif ($mac) { > + $description = "mac:$mac"; > + } > + > + my $params = { > + address => "$ip/$mask", > + type => "dhcp", > + description => $description, > + namespace => $namespace, > + status => default_ip_status(), > + }; > + > + my $ip_id = get_ip_id($config, $ip, $noerr); > + if (!defined($ip_id)) { > + return if $noerr; > + die "could not find the ip $ip in nautobot\n"; > + } > + > + eval { nautobot_api_request($config, "PATCH", "/ipam/ip-addresses/$ip_id/", $params); }; > + if ($@) { > + return if $noerr; > + die "error updating ip $ip: $@"; > + } > +} > + > +sub del_ip { > + my ($class, $config, undef, undef, $ip, $noerr) = @_; > + > + return if !$ip; > + > + my $ip_id = get_ip_id($config, $ip, $noerr); > + if (!defined($ip_id)) { > + warn("could not find the ip $ip in nautobot\n"); > + return; > + } > + > + eval { nautobot_api_request($config, "DELETE", "/ipam/ip-addresses/$ip_id/"); }; > + if ($@) { > + return if $noerr; > + die "error deleting ip $ip : $@\n"; > + } > + > + return 1; > +} > + > +sub empty_subnet { sounds more like a check if the subnet is empty, maybe delete_ips_from_subnet or something is better? > + my ($class, $config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; > + > + my $namespace = $config->{namespace}; > + > + my $response = eval { > + return nautobot_api_request( > + $config, > + "GET", > + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", > + ); > + }; > + if ($@) { > + return if $noerr; > + die "could not find the subnet $subnet in nautobot: $@\n"; > + } > + > + for my $ip (@{ $response->{results} }) { > + del_ip($class, $config, undef, undef, $ip->{host}, $noerr); > + } > + > + return 1; > +} > + > +sub subnet_is_deletable { > + my ($config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; > + > + my $namespace = $config->{namespace}; > + > + my $response = eval { > + return nautobot_api_request( > + $config, > + "GET", > + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", > + ); > + }; > + if ($@) { > + return if $noerr; > + die "error querying prefix $subnet: $@\n"; > + } > + my $n_ips = scalar $response->{results}->@*; > + > + # least costly check operation 1st > + return 1 if ($n_ips == 0); > + > + for my $ip (values $response->{results}->@*) { > + if (!is_ip_gateway($config, $ip->{host}, $noerr)) { Maybe we could simply check this by comparing with the subnet configuration instead of querying nautobot for every IP? > + # some remaining IP is not a gateway so we can't delete the subnet > + return 0; > + } > + } > + #all remaining IPs are gateways > + return 1; > +} > + > +sub verify_api { > + my ($class, $config) = @_; > + > + my $namespace = $config->{namespace}; > + > + # check if the namespace and the status "Active" exist > + eval { > + get_namespace_id($config, $namespace) // die "namespace $namespace does not exist"; > + get_status_id($config, default_ip_status()) > + // die "the status " . default_ip_status() . " does not exist"; > + }; > + if ($@) { > + die "could not use nautobot api: $@\n"; > + } > +} > + > +sub get_ips_from_mac { > + my ($class, $config, $mac, $zone) = @_; > + > + my $ip4 = undef; > + my $ip6 = undef; > + > + my $data = eval { nautobot_api_request($config, "GET", "/ipam/ip-addresses/?q=$mac"); }; > + if ($@) { > + die "could not query ip address entry for mac $mac: $@"; > + } > + > + for my $ip (@{ $data->{results} }) { > + if ($ip->{ip_version} == 4 && !$ip4) { > + ($ip4, undef) = split(/\//, $ip->{address}); > + } > + > + if ($ip->{ip_version} == 6 && !$ip6) { > + ($ip6, undef) = split(/\//, $ip->{address}); > + } > + } > + > + return ($ip4, $ip6); > +} > + > +sub on_update_hook { > + my ($class, $config) = @_; > + > + PVE::Network::SDN::Ipams::NautobotPlugin::verify_api($class, $config); > +} > + > +sub get_ip_id { filter for namespace below > + my ($config, $ip, $noerr) = @_; > + > + my $result = > + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; > + if ($@) { > + return if $noerr; > + die "error while querying for ip $ip id: $@\n"; > + } > + > + my $data = @{ $result->{results} }[0]; > + return $data->{id}; > +} > + > +sub get_prefix_id { filter for namespace below > + my ($config, $cidr, $noerr) = @_; > + > + my $result = > + eval { return nautobot_api_request($config, "GET", "/ipam/prefixes/?prefix=$cidr"); }; > + if ($@) { > + return if $noerr; > + die "error while querying for cidr $cidr prefix id: $@\n"; > + } > + > + my $data = @{ $result->{results} }[0]; > + return $data->{id}; > +} > + > +sub get_namespace_id { > + my ($config, $namespace, $noerr) = @_; > + > + my $result = > + eval { return nautobot_api_request($config, "GET", "/ipam/namespaces/?name=$namespace"); }; > + if ($@) { > + return if $noerr; > + die "error while querying for namespace $namespace id: $@\n"; > + } > + > + my $data = @{ $result->{results} }[0]; > + return $data->{id}; > +} > + > +sub get_status_id { > + my ($config, $status, $noerr) = @_; > + > + my $result = > + eval { return nautobot_api_request($config, "GET", "/extras/statuses/?name=$status"); }; > + if ($@) { > + return if $noerr; > + die "error while querying for status $status id: $@\n"; > + } > + > + my $data = @{ $result->{results} }[0]; > + return $data->{id}; > +} > + > +sub is_ip_gateway { filter for namespace below > + my ($config, $ip, $noerr) = @_; > + > + my $result = > + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; > + if ($@) { > + return if $noerr; > + die "error while checking if $ip is a gateway: $@\n"; > + } > + > + my $data = @{ $result->{results} }[0]; > + return $data->{description} eq 'gateway'; > +} > + > +1; _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin 2025-05-28 9:32 ` [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Stefan Hanreich @ 2025-06-03 12:25 ` Hannes Duerr 0 siblings, 0 replies; 6+ messages in thread From: Hannes Duerr @ 2025-06-03 12:25 UTC (permalink / raw) To: Stefan Hanreich, Proxmox VE development discussion On 5/28/25 11:32, Stefan Hanreich wrote: > Tested the following things and they worked: > > * Create a new simple zone with Nautobot IPAM > * Create a VNet with a Subnet + GW -> IP allocated > * Create a VM on that VNet -> IP allocated > * Create a CT on that VNet -> IP allocated > * Change network device to another bridge -> IP deleted > * Change back to IPAM VNet -> IP allocated > * Live-Migrate VM > * Check if VM / CT get IP via DHCP > * try to delete subnet with entries -> fails > * delete network devices and try again -> succeeds > * update gateway -> changes in IPAM > * add DHCP range and test C/R/U/D network device again > > > I added a second IPAM with a different namespace, and this is where some > problems started to occur. The PVE Web UI threw an error when trying to > create the same subnet, but in a different namespace: > > create sdn subnet object failed: could not add the subnet > HASH(0x58bc2d033e10) because it already exists in nautobot (500) > > (also note that here we need to print the id field of the hash in the > error message) > > > Some additional errors occurred when I manually created the subnet in > the SDN configuration, because it seems like many functions that fetch > IDs don't seem to consider the namespace of the objects they're fetching > (e.g. get_ip_id / get_prefix_id / ...). This means that IP addresses get > created in the wrong prefix, because the plugin just takes the first > prefix found, regardless of their namespace. Might make issues for users > that use a separated namespace for PVE, but have the same prefix > configured in other namespaces. Since most methods use the ID returned > by the helper functions, this should be easily fixed by adjusting those > helpers. > > some additional comments inline Thanks for testing and reviewing. I answered to some of your comments inline. I will try to upstream the offset parameter in nautobot and publish a new version with all the changes afterwards. > > > On 5/26/25 14:19, Hannes Duerr wrote: >> PERL5DIR=${DESTDIR}/usr/share/perl5 >> diff --git a/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm >> new file mode 100644 >> index 0000000..a6eb7ff >> --- /dev/null >> +++ b/src/PVE/Network/SDN/Ipams/NautobotPlugin.pm >> @@ -0,0 +1,510 @@ >> +package PVE::Network::SDN::Ipams::NautobotPlugin; >> + >> +use strict; >> +use warnings; >> +use PVE::INotify; >> +use PVE::Cluster; >> +use PVE::Tools; >> +use NetAddr::IP; >> +use Net::Subnet qw(subnet_matcher); >> + >> +use base('PVE::Network::SDN::Ipams::Plugin'); >> + >> +sub type { >> + return 'nautobot'; >> +} >> + >> +sub properties { >> + return { >> + namespace => { >> + type => 'string', >> + }, >> + }; >> +} > missing description here. afaict we should also add descriptions for url > / token? since they're not defined in the base plugin. Good point yes they're missing, will add them. Will also send patches for the other plugins as they're missing there as well. >> +sub options { >> + return { >> + url => { optional => 0 }, >> + token => { optional => 0 }, >> + namespace => { optional => 0 }, >> + fingerprint => { optional => 1 }, >> + }; >> +} >> + >> +sub default_ip_status { >> + return 'Active'; >> +} > could maybe just be a constant my $DEFAULT_IP_STATUS; ? I switched to default_ip_status () {} as this gives the perl interpreter the option to inline the funtion [0]. Another option would be using 'use contant DEFAULT_IP_STATUS ="Active"' which is comparable to using the function AFAIU. [0] https://www.oreilly.com/library/view/programming-perl-4th/9781449321451/ch07s04s01.html >> + >> +sub nautobot_api_request { >> + my ($config, $method, $path, $params) = @_; >> + >> + return PVE::Network::SDN::api_request( >> + $method, >> + "$config->{url}${path}", >> + [ >> + 'Content-Type' => 'application/json; charset=UTF-8', >> + 'Authorization' => "token $config->{token}", >> + 'Accept' => "application/json", >> + ], >> + $params, >> + $config->{fingerprint}, >> + ); >> +} >> + >> +sub add_subnet { >> + my ($class, $config, undef, $subnet, $noerr) = @_; >> + >> + my $cidr = $subnet->{cidr}; >> + my $namespace = $config->{namespace}; >> + >> + my $internalid = get_prefix_id($config, $cidr, $noerr); >> + if ($internalid) { >> + return if $noerr; >> + die "could not add the subnet $subnet because it already exists in nautobot\n"; > should use the CIDR field of the subnet, since $subnet is a hash agree > + > +sub add_range_next_freeip { > + my ($class, $config, $subnet, $range, $data, $noerr) = @_; > + > + my $cidr = NetAddr::IP->new($subnet->{cidr}); > + my $namespace = $config->{namespace}; > + > + # Nautobot does not support IP ranges, only prefixes. > + # Therefore we divide the range into smaller pool prefixes, > + # each containing 256 addresses, and search them for available IPs > + my $prefix_size = $cidr->version == 4 ? 24 : 120; > maybe we could default to the prefix size of the subnet if it is smaller > than 24 / 120? Not so likely for IPv6, but for IPv4 it'd make sense? > >> + my $increment = 256; >> + my $found_ip = undef; >> + >> + my $start_range = NetAddr::IP->new($range->{'start-address'}, $prefix_size); >> + my $end_range = NetAddr::IP->new($range->{'end-address'}, $prefix_size); >> + my $matcher = subnet_matcher($end_range->cidr); >> + my $current_ip = $start_range; >> + >> + while (1) { >> + my $current_cidr = $current_ip->addr . "/$prefix_size"; >> + >> + my $params = { >> + prefix => $current_cidr, >> + namespace => $namespace, >> + status => default_ip_status(), >> + type => "pool", >> + }; >> + >> + my $prefix_id = get_prefix_id($config, $current_cidr, $noerr); >> + if ($prefix_id) { >> + # search the existing prefix for valid ip >> + $found_ip = >> + find_ip_in_prefix($config, $prefix_id, $increment, $start_range, $end_range); >> + } else { >> + # create temporary pool prefix >> + my $temp_prefix = >> + eval { return nautobot_api_request($config, "POST", "/ipam/prefixes/", $params); }; > should probably do some error handling? > >> + >> + my $temp_prefix_id = $temp_prefix->{id}; >> + >> + # search temporarly created prefix >> + $found_ip = >> + find_ip_in_prefix($config, $temp_prefix_id, $increment, $start_range, $end_range); >> + >> + # Delete temporary prefix pool >> + eval { nautobot_api_request($config, "DELETE", "/ipam/prefixes/$temp_prefix_id/"); }; > here as well > >> + } >> + >> + last if $found_ip; >> + >> + # we searched the last pool prefix >> + last if $matcher->($current_ip->addr); >> + >> + $current_ip = $current_ip->plus($increment); >> + } >> + >> + if (!$found_ip) { >> + return if $noerr; >> + die "could not allocate ip in the range " >> + . $start_range->addr . " - " >> + . $end_range->addr >> + . ": $@\n"; >> + } >> + >> + $class->add_ip( >> + $config, >> + undef, >> + $subnet, >> + $found_ip, >> + $data->{hostname}, >> + $data->{mac}, >> + undef, >> + 0, >> + $noerr, >> + ); >> + >> + return $found_ip; >> +} > we might really wanna upstream a little patch that adds at least offset > support, or defines a start / end range. Seems like this shouldn't be > too hard with the current implementation in nautobot. That would > simplify this method by a lot. We could also just deny using DHCP ranges > with the Nautobot plugin for now... yes i thought it would be nicer if we didn't have to depend on a special nautobot version, but i agree with you that the current implementation is definitely more complex and therefore more error-prone than simply having an offset parameter. I will see if I can get the offset parameter implemented in nautobot upstream. > see > https://github.com/nautobot/nautobot/blob/develop/nautobot/ipam/api/views.py#L309 > >> +sub update_ip { >> + my ($class, $config, $subnetid, $subnet, $ip, $hostname, $mac, undef, $is_gateway, $noerr) = @_; >> + >> + my $mask = $subnet->{mask}; >> + my $namespace = $config->{namespace}; >> + >> + my $description = undef; >> + if ($is_gateway) { >> + $description = 'gateway'; >> + } elsif ($mac) { >> + $description = "mac:$mac"; >> + } >> + >> + my $params = { >> + address => "$ip/$mask", >> + type => "dhcp", >> + description => $description, >> + namespace => $namespace, >> + status => default_ip_status(), >> + }; >> + >> + my $ip_id = get_ip_id($config, $ip, $noerr); >> + if (!defined($ip_id)) { >> + return if $noerr; >> + die "could not find the ip $ip in nautobot\n"; >> + } >> + >> + eval { nautobot_api_request($config, "PATCH", "/ipam/ip-addresses/$ip_id/", $params); }; >> + if ($@) { >> + return if $noerr; >> + die "error updating ip $ip: $@"; >> + } >> +} >> + >> +sub del_ip { >> + my ($class, $config, undef, undef, $ip, $noerr) = @_; >> + >> + return if !$ip; >> + >> + my $ip_id = get_ip_id($config, $ip, $noerr); >> + if (!defined($ip_id)) { >> + warn("could not find the ip $ip in nautobot\n"); >> + return; >> + } >> + >> + eval { nautobot_api_request($config, "DELETE", "/ipam/ip-addresses/$ip_id/"); }; >> + if ($@) { >> + return if $noerr; >> + die "error deleting ip $ip : $@\n"; >> + } >> + >> + return 1; >> +} >> + >> +sub empty_subnet { > sounds more like a check if the subnet is empty, maybe > delete_ips_from_subnet or something is better? Agree > >> + my ($class, $config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; >> + >> + my $namespace = $config->{namespace}; >> + >> + my $response = eval { >> + return nautobot_api_request( >> + $config, >> + "GET", >> + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", >> + ); >> + }; >> + if ($@) { >> + return if $noerr; >> + die "could not find the subnet $subnet in nautobot: $@\n"; >> + } >> + >> + for my $ip (@{ $response->{results} }) { >> + del_ip($class, $config, undef, undef, $ip->{host}, $noerr); >> + } >> + >> + return 1; >> +} >> + >> +sub subnet_is_deletable { >> + my ($config, $subnetid, $subnet, $subnetuuid, $noerr) = @_; >> + >> + my $namespace = $config->{namespace}; >> + >> + my $response = eval { >> + return nautobot_api_request( >> + $config, >> + "GET", >> + "/ipam/ip-addresses/?namespace=$namespace&parent=$subnetuuid", >> + ); >> + }; >> + if ($@) { >> + return if $noerr; >> + die "error querying prefix $subnet: $@\n"; >> + } >> + my $n_ips = scalar $response->{results}->@*; >> + >> + # least costly check operation 1st >> + return 1 if ($n_ips == 0); >> + >> + for my $ip (values $response->{results}->@*) { >> + if (!is_ip_gateway($config, $ip->{host}, $noerr)) { > Maybe we could simply check this by comparing with the subnet > configuration instead of querying nautobot for every IP? yes good point. > >> + # some remaining IP is not a gateway so we can't delete the subnet >> + return 0; >> + } >> + } >> + #all remaining IPs are gateways >> + return 1; >> +} >> + >> +sub verify_api { >> + my ($class, $config) = @_; >> + >> + my $namespace = $config->{namespace}; >> + >> + # check if the namespace and the status "Active" exist >> + eval { >> + get_namespace_id($config, $namespace) // die "namespace $namespace does not exist"; >> + get_status_id($config, default_ip_status()) >> + // die "the status " . default_ip_status() . " does not exist"; >> + }; >> + if ($@) { >> + die "could not use nautobot api: $@\n"; >> + } >> +} >> + >> +sub get_ips_from_mac { >> + my ($class, $config, $mac, $zone) = @_; >> + >> + my $ip4 = undef; >> + my $ip6 = undef; >> + >> + my $data = eval { nautobot_api_request($config, "GET", "/ipam/ip-addresses/?q=$mac"); }; >> + if ($@) { >> + die "could not query ip address entry for mac $mac: $@"; >> + } >> + >> + for my $ip (@{ $data->{results} }) { >> + if ($ip->{ip_version} == 4 && !$ip4) { >> + ($ip4, undef) = split(/\//, $ip->{address}); >> + } >> + >> + if ($ip->{ip_version} == 6 && !$ip6) { >> + ($ip6, undef) = split(/\//, $ip->{address}); >> + } >> + } >> + >> + return ($ip4, $ip6); >> +} >> + >> +sub on_update_hook { >> + my ($class, $config) = @_; >> + >> + PVE::Network::SDN::Ipams::NautobotPlugin::verify_api($class, $config); >> +} >> + >> +sub get_ip_id { > filter for namespace below good catch, added it to the other spots you mentioned and also get_ips_from_mac as it was missing there as well. >> + my ($config, $ip, $noerr) = @_; >> + >> + my $result = >> + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; >> + if ($@) { >> + return if $noerr; >> + die "error while querying for ip $ip id: $@\n"; >> + } >> + >> + my $data = @{ $result->{results} }[0]; >> + return $data->{id}; >> +} >> + >> +sub get_prefix_id { > filter for namespace below > >> + my ($config, $cidr, $noerr) = @_; >> + >> + my $result = >> + eval { return nautobot_api_request($config, "GET", "/ipam/prefixes/?prefix=$cidr"); }; >> + if ($@) { >> + return if $noerr; >> + die "error while querying for cidr $cidr prefix id: $@\n"; >> + } >> + >> + my $data = @{ $result->{results} }[0]; >> + return $data->{id}; >> +} >> + >> +sub get_namespace_id { >> + my ($config, $namespace, $noerr) = @_; >> + >> + my $result = >> + eval { return nautobot_api_request($config, "GET", "/ipam/namespaces/?name=$namespace"); }; >> + if ($@) { >> + return if $noerr; >> + die "error while querying for namespace $namespace id: $@\n"; >> + } >> + >> + my $data = @{ $result->{results} }[0]; >> + return $data->{id}; >> +} >> + >> +sub get_status_id { >> + my ($config, $status, $noerr) = @_; >> + >> + my $result = >> + eval { return nautobot_api_request($config, "GET", "/extras/statuses/?name=$status"); }; >> + if ($@) { >> + return if $noerr; >> + die "error while querying for status $status id: $@\n"; >> + } >> + >> + my $data = @{ $result->{results} }[0]; >> + return $data->{id}; >> +} >> + >> +sub is_ip_gateway { > filter for namespace below > >> + my ($config, $ip, $noerr) = @_; >> + >> + my $result = >> + eval { return nautobot_api_request($config, "GET", "/ipam/ip-addresses/?address=$ip"); }; >> + if ($@) { >> + return if $noerr; >> + die "error while checking if $ip is a gateway: $@\n"; >> + } >> + >> + my $data = @{ $result->{results} }[0]; >> + return $data->{description} eq 'gateway'; >> +} >> + >> +1; _______________________________________________ pve-devel mailing list pve-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pve-devel ^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2025-06-03 12:25 UTC | newest] Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2025-05-26 12:19 [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-network v5 2/2] ipam: add test cases for nautobot plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-docs v5 1/1] add documentation for nautobot ipam plugin Hannes Duerr 2025-05-26 12:19 ` [pve-devel] [PATCH pve-manager v5 1/1] ipam: add UI dialog " Hannes Duerr 2025-05-28 9:32 ` [pve-devel] [PATCH pve-network v5 1/2] ipam: add Nautobot plugin Stefan Hanreich 2025-06-03 12:25 ` Hannes Duerr
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inboxService provided by Proxmox Server Solutions GmbH | Privacy | Legal