[PATCH pve-network v6 08/13] sdn: ipam: keep the MAC cache per vnet

[Hannes Laimer <h.laimer@proxmox.com>]

The same MAC can sit on several vnets, but the cache of its addresses
was keyed by the MAC alone. A mapping created or edited on one vnet
changed the cached answer on every other vnet as well. A guest with
that MAC on another vnet then got a reservation for an address outside
its own vnet.

The cache now holds the addresses per vnet. A zone's dnsmasq
reservations hold a line for each vnet a MAC sits on. A push of a
MAC's reservation looks the MAC up and checks the other lines against
the cache alone. A push at a resume asks the IPAM before it takes
the reservation file's lock. Then it does not wait long for the lock
and writes its line from the cache, or from the answer when the cache
has nothing. The built-in IPAM matches a MAC in any spelling, since a
push looks the MAC up in lower case.

The map under the MAC alone stays, since nodes not yet upgraded read
and write only that one. Every address written to the cache goes there
as well, one address per family and MAC as before. An entry there
answers for a vnet without an entry of its own, but only with an
address in that vnet's subnets. So a guest started before the upgrade
keeps its reservation.

A release or an edit on a node not yet upgraded shows only in that
map. A vnet's cached address is therefore used only while that map
still names it. Otherwise the lookup asks the IPAM again. A line the
cache cannot vouch for gets the IPAM's answer at any push on the
zone, and a push at a resume asks for its own line only.

A node not yet upgraded may also have released a MAC's address and
given it to another guest. So a release now leaves an IPAM record held
by another MAC in place, with its DNS records, and fails when it
cannot check the holder. Deleting such a mapping through the API
clears the MAC's cache entry and fails.

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/PVE/API2/Network/SDN/Ips.pm            |  13 +-
 src/PVE/Network/SDN/Dhcp.pm                |   5 +-
 src/PVE/Network/SDN/Dhcp/Dnsmasq.pm        | 176 ++++---
 src/PVE/Network/SDN/Dhcp/Plugin.pm         |   8 +-
 src/PVE/Network/SDN/Ipams.pm               | 196 +++++---
 src/PVE/Network/SDN/Ipams/NetboxPlugin.pm  |  15 +-
 src/PVE/Network/SDN/Ipams/PVEPlugin.pm     |  13 +-
 src/PVE/Network/SDN/Ipams/PhpIpamPlugin.pm |  20 +-
 src/PVE/Network/SDN/Ipams/Plugin.pm        |   4 +-
 src/PVE/Network/SDN/Subnets.pm             |  14 +-
 src/PVE/Network/SDN/Vnets.pm               |  10 +-
 src/test/run_test_ipams.pl                 |  49 ++
 src/test/run_test_vnets_blackbox.pl        | 518 +++++++++++++++++++--
 13 files changed, 865 insertions(+), 176 deletions(-)

diff --git a/src/PVE/API2/Network/SDN/Ips.pm b/src/PVE/API2/Network/SDN/Ips.pm
index 8a6106ba29a7..c4816a6cd4ca 100644
--- a/src/PVE/API2/Network/SDN/Ips.pm
+++ b/src/PVE/API2/Network/SDN/Ips.pm
@@ -124,10 +124,13 @@ __PACKAGE__->register_method({
             $vnet,
             $mac,
             sub {
-                eval { PVE::Network::SDN::Vnets::del_ip($vnet, $ip, '', $mac); };
+                my $released = eval { PVE::Network::SDN::Vnets::del_ip($vnet, $ip, '', $mac); };
                 die "$@\n" if $@;
 
                 $push_mapping->($vnet, $mac);
+
+                # the cache entry is cleared, the IPAM record belongs to another MAC
+                die "IP '$ip' is not mapped to $mac\n" if defined($released) && !$released;
             },
         );
 
@@ -245,12 +248,16 @@ __PACKAGE__->register_method({
                     PVE::Network::SDN::Vnets::get_ips_from_mac($vnet, $mac);
                 my $old_ip = (Net::IP::ip_get_version($ip) == 4) ? $old_ip4 : $old_ip6;
 
-                PVE::Network::SDN::Vnets::del_ip($vnet, $old_ip, '', $mac);
+                # a MAC without an address in that family has nothing to release.
+                # A release answers 0 when another MAC holds the address
+                my $released = defined($old_ip)
+                    && (PVE::Network::SDN::Vnets::del_ip($vnet, $old_ip, '', $mac) // 1);
 
                 eval { PVE::Network::SDN::Vnets::add_ip($vnet, $ip, '', $mac, $vmid); };
                 my $error = $@;
 
-                if ($error) {
+                # an address that was not released is not added back
+                if ($error && $released) {
                     PVE::Network::SDN::Vnets::add_ip($vnet, $old_ip, '', $mac, $vmid);
                 }
 
diff --git a/src/PVE/Network/SDN/Dhcp.pm b/src/PVE/Network/SDN/Dhcp.pm
index 11bb6df8c23a..a986408ecbfa 100644
--- a/src/PVE/Network/SDN/Dhcp.pm
+++ b/src/PVE/Network/SDN/Dhcp.pm
@@ -22,7 +22,7 @@ sub plugin_types {
 }
 
 sub update_mapping {
-    my ($vnetid, $mac) = @_;
+    my ($vnetid, $mac, $resume) = @_;
 
     # a record without a MAC has no guest NIC behind it, so no backend serves it
     return if !$mac;
@@ -39,7 +39,8 @@ sub update_mapping {
     # a zone confined to other nodes has no vnet bridge on this node
     return if defined($zone->{nodes}) && !$zone->{nodes}->{ PVE::INotify::nodename() };
 
-    PVE::Network::SDN::Dhcp::Plugin->lookup($zone->{dhcp})->update_ip_mapping($zoneid, $mac);
+    PVE::Network::SDN::Dhcp::Plugin->lookup($zone->{dhcp})
+        ->update_ip_mapping($zoneid, $vnetid, $mac, $resume);
 }
 
 sub guest_node_by_mac {
diff --git a/src/PVE/Network/SDN/Dhcp/Dnsmasq.pm b/src/PVE/Network/SDN/Dhcp/Dnsmasq.pm
index 5864ead87d29..e99bfa75e506 100644
--- a/src/PVE/Network/SDN/Dhcp/Dnsmasq.pm
+++ b/src/PVE/Network/SDN/Dhcp/Dnsmasq.pm
@@ -12,8 +12,8 @@ use File::Copy;
 use Net::DBus;
 
 use PVE::Cluster;
-use PVE::RESTEnvironment qw(log_warn);
 use PVE::Network::SDN::Ipams;
+use PVE::RESTEnvironment qw(log_warn);
 
 my $DNSMASQ_CONFIG_ROOT = '/etc/dnsmasq.d';
 my $DNSMASQ_DEFAULT_ROOT = '/etc/default';
@@ -54,36 +54,80 @@ sub update_lease {
 }
 
 sub update_ip_mapping {
-    my ($class, $dhcpid, $mac) = @_;
+    my ($class, $dhcpid, $vnetid, $mac, $resume) = @_;
     $mac = lc($mac);
 
     my $ethers_file = ethers_file($dhcpid);
     my $ethers_tmp_file = "$ethers_file.tmp";
 
-    my ($ip4, $ip6);
-    my $reservation = undef;
+    my $ip4;
     my $changed = undef;
     my @resume_answer;
 
-    my $rewriteFn = sub {
-        # the lines are decided from the cache, which this process may hold an
-        # old read of, and the local lock does not refresh it
+    my $rewrite_fn = sub {
+        # the lines are decided from the cache and the applied config, which
+        # the local lock does not refresh
         PVE::Cluster::cfs_update();
 
-        # read under the ethers lock. A snapshot from before it would sweep a
-        # line another push wrote meanwhile, and an older push would put its
-        # addresses back
-
-        my $macs = PVE::Network::SDN::Ipams::read_macdb()->{macs};
-
-        my $entry = $macs->{$mac};
-        ($ip4, $ip6) = ($entry->{ip4}, $entry->{ip6}) if $entry;
-        if ($ip4 || $ip6) {
-            $reservation = $mac;
-            $reservation .= ",$ip4" if $ip4;
-            $reservation .= ",[$ip6]" if $ip6;
+        # the file holds a line per vnet a MAC sits on, and a line belongs to
+        # the vnet of the applied subnet that holds its address
+        my $vnets = PVE::Network::SDN::Zones::get_vnets($dhcpid, 1) // {};
+        my $subnets = PVE::Network::SDN::Vnets::get_subnets(undef, 1) // {};
+        $subnets = {
+            map { $_ => $subnets->{$_} } grep { $vnets->{ $subnets->{$_}->{vnet} } }
+                keys %$subnets
+        };
+        my $vnet_of = sub {
+            my ($ip) = @_;
+            my (undef, $subnet) =
+                eval { PVE::Network::SDN::Subnets::find_ip_subnet($ip, $subnets) };
+            return $subnet ? $subnet->{vnet} : undef;
+        };
+        my %subnets_of;
+        $subnets_of{ $subnets->{$_}->{vnet} }->{$_} = $subnets->{$_} for keys %$subnets;
+
+        # the changed MAC is looked up first, under the ethers lock so that two
+        # edits of one MAC cannot land out of order, and its lines come from
+        # that answer. The other lines are checked against one read of the
+        # cache, taken after the lookup's fill, and only a line the cache
+        # cannot vouch for asks the IPAM as well. A push at a resume looked
+        # the MAC up before the lock and takes its line from the cache, which
+        # an edit that landed in between wrote as well
+        my $cache;
+        my $line_of = sub {
+            my ($line_mac, $line_ip4, $line_ip6) = @_;
+            return (undef, undef) if !$line_ip4 && !$line_ip6;
+            my $line = $line_mac;
+            $line .= ",$line_ip4" if $line_ip4;
+            $line .= ",[$line_ip6]" if $line_ip6;
             # an external IPAM spells addresses its own way, while the file holds one spelling
-            $reservation = lc($reservation);
+            return (lc($line), $line_ip4);
+        };
+        my $reservation = sub {
+            my ($vnet, $line_mac, $from_cache) = @_;
+            my @ips =
+                $from_cache
+                ? PVE::Network::SDN::Ipams::cached_ips_from_mac(
+                    $cache,
+                    $vnet,
+                    $line_mac,
+                    $subnets_of{$vnet} // {},
+                )
+                : PVE::Network::SDN::Vnets::get_ips_from_mac($vnet, $line_mac, 1);
+            return $line_of->($line_mac, @ips);
+        };
+        my $own;
+        if ($resume) {
+            # the lookup's answer is taken when the cache has no line to vouch
+            # for. A delete since the lookup empties the cache too, the next
+            # sweep drops that line
+            $cache = PVE::Network::SDN::Ipams::read_cache();
+            ($own, $ip4) = $reservation->($vnetid, $mac, 1)
+                if PVE::Network::SDN::Ipams::cache_witnessed($cache, $vnetid, $mac);
+            ($own, $ip4) = $line_of->($mac, @resume_answer) if !defined($own);
+        } else {
+            ($own, $ip4) = $reservation->($vnetid, $mac);
+            $cache = PVE::Network::SDN::Ipams::read_cache();
         }
 
         file_set_contents($ethers_file, '') if !-e $ethers_file;
@@ -91,62 +135,56 @@ sub update_ip_mapping {
         open(my $out, '>', $ethers_tmp_file)
             or die "Could not open file '$ethers_tmp_file' $!\n";
 
-        my $kept = undef;
-
+        my %written;
+        my $ipam_failed;
         while (my $line = <$in>) {
             chomp($line);
             next if $line !~ m/\S/;
-            my ($parsed_mac, $parsed_ip1, $parsed_ip2) = split(/,/, $line);
-            # a line from before the file was written in lower case keeps its writer's spelling
+            my ($parsed_mac, $parsed_ip) = split(/,/, $line);
             $parsed_mac = lc($parsed_mac);
-
-            # the MAC's own reservation is rewritten. An unchanged one stays in place
-            if ($parsed_mac eq $mac) {
-                if (defined($reservation) && !$kept && lc($line) eq $reservation) {
-                    $kept = 1;
-                    print $out "$line\n";
+            $parsed_ip = $1 if defined($parsed_ip) && $parsed_ip =~ m/\[(\S+)\]/;
+            my $vnet = defined($parsed_ip) ? $vnet_of->($parsed_ip) : undef;
+            my $key = defined($vnet) ? "$parsed_mac,$vnet" : undef;
+
+            # a line that no longer matches is replaced, a dropped one would
+            # leave the guest unknown to dnsmasq until its next start. A line
+            # whose cache entry is not vouched for gets the IPAM's answer, so
+            # the file is right after any push
+            my ($want) = eval {
+                if (!$key || $written{$key}) {
+                    ();
+                } elsif ($key eq "$mac,$vnetid") {
+                    ($own);
+                } elsif ($resume || $ipam_failed) {
+                    # a push at a resume asks for its own line only, and a
+                    # push whose lookup failed asks no further, so an IPAM
+                    # that does not answer costs one request. The other lines
+                    # are kept as they are
+                    (lc($line));
+                } elsif (PVE::Network::SDN::Ipams::cache_witnessed($cache, $vnet, $parsed_mac)) {
+                    $reservation->($vnet, $parsed_mac, 1);
                 } else {
-                    $changed = 1;
+                    $reservation->($vnet, $parsed_mac);
                 }
-                next;
-            }
-
-            my ($parsed_ip4, $parsed_ip6);
-            if ($parsed_ip2) {
-                $parsed_ip4 = $parsed_ip1;
-                $parsed_ip6 = $parsed_ip2;
-            } elsif (defined($parsed_ip1) && Net::IP::ip_is_ipv4($parsed_ip1)) {
-                $parsed_ip4 = $parsed_ip1;
-            } elsif (defined($parsed_ip1)) {
-                $parsed_ip6 = $parsed_ip1;
-            }
-            $parsed_ip6 = $1 if $parsed_ip6 && $parsed_ip6 =~ m/\[(\S+)\]/;
-            # a line from before addresses were lower-cased may spell one in upper case
-            $parsed_ip6 = lc($parsed_ip6) if $parsed_ip6;
-
-            # a line that no longer matches the cache is replaced by the cache's
-            # current one, a dropped one would leave that guest unknown to
-            # dnsmasq until its next start
-            my $current = $macs->{$parsed_mac};
-            if (
-                !$current
-                || ($parsed_ip4 && $current->{ip4} && $current->{ip4} ne $parsed_ip4)
-                || ($parsed_ip6 && $current->{ip6} && lc($current->{ip6}) ne $parsed_ip6)
-            ) {
+            };
+            if ($@) {
+                warn "could not read the reservation of $parsed_mac: $@";
+                $ipam_failed = 1;
+                print $out "$line\n";
+            } elsif (!defined($want)) {
                 $changed = 1;
-                next if !$current;
-                my $replacement = $parsed_mac;
-                $replacement .= ",$current->{ip4}" if $current->{ip4};
-                $replacement .= ",[$current->{ip6}]" if $current->{ip6};
-                print $out lc($replacement) . "\n";
                 next;
+            } elsif (lc($line) ne $want) {
+                $changed = 1;
+                print $out "$want\n";
+            } else {
+                print $out "$line\n";
             }
-
-            print $out "$line\n";
+            $written{$key}++;
         }
 
-        if (defined($reservation) && !$kept) {
-            print $out "$reservation\n";
+        if (defined($own) && !$written{"$mac,$vnetid"}) {
+            print $out "$own\n";
             $changed = 1;
         }
 
@@ -158,7 +196,13 @@ sub update_ip_mapping {
 
     # the rewrite replaces the reservation file, so a lock on the file itself
     # would not outlive the rename
-    PVE::Tools::lock_file("$ethers_file.lock", 10, $rewriteFn);
+    if ($resume) {
+        # a push at a resume holds up the guest, so it asks the IPAM before
+        # the lock and does not wait long for it
+        @resume_answer = eval { PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac, 1) };
+        die "Unable to update $mac in the dnsmasq configuration: $@" if $@;
+    }
+    PVE::Tools::lock_file("$ethers_file.lock", $resume ? 1 : 10, $rewrite_fn);
     die "Unable to update $mac in the dnsmasq configuration: $@" if $@;
 
     systemctl_service('reload', "dnsmasq\@$dhcpid") if $changed;
diff --git a/src/PVE/Network/SDN/Dhcp/Plugin.pm b/src/PVE/Network/SDN/Dhcp/Plugin.pm
index 472ff1534743..a3e5102b601a 100644
--- a/src/PVE/Network/SDN/Dhcp/Plugin.pm
+++ b/src/PVE/Network/SDN/Dhcp/Plugin.pm
@@ -22,10 +22,12 @@ sub private {
     return $defaultData;
 }
 
-# the MAC's records changed, so make the backend serve the current ones. With no
-# record left its reservation goes. Dies if the backend could not be updated
+# the MAC's records on the vnet changed, so make the backend serve the current
+# ones. With no record left its reservation on the vnet goes. Dies if the
+# backend could not be updated. For a resume the push does not wait long for
+# a busy backend
 sub update_ip_mapping {
-    my ($class, $dhcpid, $mac) = @_;
+    my ($class, $dhcpid, $vnetid, $mac, $resume) = @_;
     die 'implement in sub class';
 }
 
diff --git a/src/PVE/Network/SDN/Ipams.pm b/src/PVE/Network/SDN/Ipams.pm
index 596f43951b23..8e7bd537f4ec 100644
--- a/src/PVE/Network/SDN/Ipams.pm
+++ b/src/PVE/Network/SDN/Ipams.pm
@@ -35,40 +35,10 @@ sub json_writer {
     return encode_json($data);
 }
 
-# Writers spell a MAC their own way, the generated ones in upper case and the
-# API in either, and nodes not yet upgraded look an entry up under the spelling
-# their caller hands them, so the file keeps its keys as written. Readers get a
-# view with every MAC in lower case. Two spellings that disagree on an address
-# are left out of it, since one of them is stale and nothing here says which,
-# and so is an entry without an address. The next lookup then asks the IPAM
-my sub fold_macdb {
-    my ($db) = @_;
-
-    my $macs = {};
-    my %clash;
-    for my $key (sort keys %{ $db->{macs} // {} }) {
-        my $entry = $db->{macs}->{$key};
-        for my $family (grep { defined($entry->{$_}) } qw(ip4 ip6)) {
-            my $seen = $macs->{ lc($key) }->{$family};
-            $clash{ lc($key) } = 1 if defined($seen) && lc($seen) ne lc($entry->{$family});
-            $macs->{ lc($key) }->{$family} //= $entry->{$family};
-        }
-    }
-    delete $macs->{$_} for keys %clash;
-    return { %$db, macs => $macs };
-}
-
-# the file as written, which the writers change and write back
-sub read_macdb_file {
-    my () = @_;
-
-    return cfs_read_file($macdb_filename);
-}
-
 sub read_macdb {
     my () = @_;
 
-    return fold_macdb(read_macdb_file());
+    return cfs_read_file($macdb_filename);
 }
 
 sub write_macdb {
@@ -77,9 +47,42 @@ sub write_macdb {
     cfs_write_file($macdb_filename, $data);
 }
 
-# a write goes to every spelling the file holds for the MAC, so each reader
-# finds it under its own, and a new MAC is written as the caller spells it
-my sub set_cache_entry {
+# the map under the MAC alone is the one nodes not yet upgraded read and
+# write, under the MAC as their caller spelled it, so one read indexes it by
+# the MAC in lower case for every lookup on that read. Spellings that
+# disagree on an address are both left out, since one of them is stale
+my sub old_index {
+    my ($db) = @_;
+
+    my (%index, %clash);
+    my $macs = $db->{macs} // {};
+    for my $key (sort keys %$macs) {
+        my $mac = lc($key);
+        next if $clash{$mac};
+        my $entry = $index{$mac} //= {};
+        for my $family (qw(ip4 ip6)) {
+            my $ip = $macs->{$key}->{$family} // next;
+            if (defined($entry->{$family}) && lc($entry->{$family}) ne lc($ip)) {
+                $clash{$mac} = 1;
+                last;
+            }
+            $entry->{$family} = $ip;
+        }
+    }
+    delete $index{$_} for keys %clash;
+    return \%index;
+}
+
+sub read_cache {
+    my $db = read_macdb();
+    return { db => $db, old => old_index($db) };
+}
+
+# a write lands in the map under the MAC alone as well, under the keys it
+# already has, with one address per MAC as a node not yet upgraded reads
+# and writes it. A new MAC is written as the caller spells it, nodes not yet
+# upgraded look it up like that
+my sub set_old_cache_entry {
     my ($db, $mac, $family, $ip) = @_;
 
     my $macs = $db->{macs} //= {};
@@ -87,15 +90,78 @@ my sub set_cache_entry {
     $macs->{$_}->{$family} = $ip for (@keys ? @keys : ($mac));
 }
 
+my sub in_subnets {
+    my ($ip, $subnets) = @_;
+    return !!eval { PVE::Network::SDN::Subnets::find_ip_subnet($ip, $subnets) };
+}
+
+# the addresses the per-vnet entries hold for the MAC. A write puts its
+# address into the map under the MAC alone as well, so a value found in both
+# is some vnet's and not an entry from before
+my sub claimed_addresses {
+    my ($db, $mac) = @_;
+
+    my %claimed;
+    for my $vnet_entries (values %{ $db->{vnets} // {} }) {
+        my $entry = $vnet_entries->{ lc($mac) } // next;
+        $claimed{ lc($_) } = 1 for grep { defined($_) } @{$entry}{qw(ip4 ip6)};
+    }
+    return \%claimed;
+}
+
+# the vnet's entry answers, and an entry from before the cache was kept per
+# vnet fills the families it lacks with addresses in the vnet's subnets
+my sub cache_entry {
+    my ($cache, $mac, $vnetid, $subnets) = @_;
+
+    my $entry = (($cache->{db}->{vnets} // {})->{$vnetid} // {})->{ lc($mac) } // {};
+    my $old = $cache->{old}->{ lc($mac) } // {};
+    my $claimed = claimed_addresses($cache->{db}, $mac);
+    my $cached = {};
+    for my $family (qw(ip4 ip6)) {
+        my $ip = $entry->{$family};
+        $ip = $old->{$family}
+            if !defined($ip)
+            && defined($old->{$family})
+            && !$claimed->{ lc($old->{$family}) }
+            && in_subnets($old->{$family}, $subnets);
+        $cached->{$family} = $ip if defined($ip);
+    }
+    return %$cached ? $cached : undef;
+}
+
+# a node not yet upgraded releases or moves an address in the map under the
+# MAC alone, so a vnet's address holds only while that map still names it. The
+# same address on another vnet of the MAC leaves that map unable to say which
+# vnet's it names, so neither is trusted
+sub cache_witnessed {
+    my ($cache, $vnetid, $mac) = @_;
+
+    my $vnets = $cache->{db}->{vnets} // {};
+    my $entry = ($vnets->{$vnetid} // {})->{ lc($mac) } // {};
+    my $old = $cache->{old}->{ lc($mac) } // {};
+    for my $family (grep { defined($entry->{$_}) } qw(ip4 ip6)) {
+        my $ip = lc($entry->{$family});
+        return 0 if lc($old->{$family} // '') ne $ip;
+        for my $other (grep { $_ ne $vnetid } keys %$vnets) {
+            my $other_entry = $vnets->{$other}->{ lc($mac) } // next;
+            return 0 if lc($other_entry->{$family} // '') eq $ip;
+        }
+    }
+    return 1;
+}
+
 sub add_cache_mac_ip {
-    my ($mac, $ip) = @_;
+    my ($vnetid, $mac, $ip) = @_;
+    my $family = Net::IP::ip_is_ipv4($ip) ? 'ip4' : 'ip6';
 
     cfs_lock_file(
         $macdb_filename,
         undef,
         sub {
-            my $db = read_macdb_file();
-            set_cache_entry($db, $mac, Net::IP::ip_is_ipv4($ip) ? 'ip4' : 'ip6', $ip);
+            my $db = read_macdb();
+            $db->{vnets}->{$vnetid}->{ lc($mac) }->{$family} = $ip;
+            set_old_cache_entry($db, $mac, $family, $ip);
             write_macdb($db);
         },
     );
@@ -103,19 +169,29 @@ sub add_cache_mac_ip {
 }
 
 sub del_cache_mac_ip {
-    my ($mac, $ip) = @_;
+    my ($vnetid, $mac, $ip) = @_;
+    $mac = lc($mac);
+    my $family = Net::IP::ip_is_ipv4($ip) ? 'ip4' : 'ip6';
 
     cfs_lock_file(
         $macdb_filename,
         undef,
         sub {
-            my $db = read_macdb_file();
-            my $family = Net::IP::ip_is_ipv4($ip) ? 'ip4' : 'ip6';
+            my $db = read_macdb();
+            my $vnet_entries = ($db->{vnets} // {})->{$vnetid};
+            if (my $entry = $vnet_entries ? $vnet_entries->{$mac} : undef) {
+                delete $entry->{$family};
+                delete $vnet_entries->{$mac} if !%$entry;
+                delete $db->{vnets}->{$vnetid} if !%$vnet_entries;
+                delete $db->{vnets} if !%{ $db->{vnets} };
+            }
+            # the map under the MAC alone must not hand a released address back
             my $macs = $db->{macs} // {};
-            for my $key (grep { lc($_) eq lc($mac) } keys %$macs) {
-                delete $macs->{$key}->{$family};
-                delete $macs->{$key}
-                    if !defined($macs->{$key}->{ip4}) && !defined($macs->{$key}->{ip6});
+            for my $key (grep { lc($_) eq $mac } keys %$macs) {
+                my $old = $macs->{$key};
+                next if lc($old->{$family} // '') ne lc($ip);
+                delete $old->{$family};
+                delete $macs->{$key} if !defined($old->{ip4}) && !defined($old->{ip6});
             }
             write_macdb($db);
         },
@@ -168,16 +244,22 @@ sub complete_sdn_vnet {
     return $cmdname eq 'add' ? [] : [PVE::Network::SDN::Vnets::sdn_ipams_ids($cfg)];
 }
 
+# the addresses one read of the cache holds for a MAC on a vnet
+sub cached_ips_from_mac {
+    my ($cache, $vnetid, $mac, $subnets) = @_;
+
+    my $cached = cache_entry($cache, $mac, $vnetid, $subnets);
+    return $cached ? ($cached->{ip4}, $cached->{ip6}) : ();
+}
+
 sub get_ips_from_mac {
-    my ($mac, $zoneid, $zone, $subnets) = @_;
+    my ($vnetid, $mac, $zoneid, $zone, $subnets) = @_;
 
     PVE::Cluster::cfs_update(); # the lock refreshes, so the snapshot must too
     my $version = PVE::Cluster::cfs_file_version($macdb_filename);
-    my $cached = read_macdb()->{macs}->{ lc($mac) };
-    # an entry without an address was left behind by a lookup that missed, and
-    # says nothing about its MAC
-    return ($cached->{ip4}, $cached->{ip6})
-        if $cached && (defined($cached->{ip4}) || defined($cached->{ip6}));
+    my $cache = read_cache();
+    my $cached = cache_entry($cache, $mac, $vnetid, $subnets);
+    return ($cached->{ip4}, $cached->{ip6}) if $cached && cache_witnessed($cache, $vnetid, $mac);
 
     my $plugin_config = get_plugin_config($zone);
     my $plugin = PVE::Network::SDN::Ipams::Plugin->lookup($plugin_config->{type});
@@ -195,9 +277,17 @@ sub get_ips_from_mac {
             return
                 if (PVE::Cluster::cfs_file_version($macdb_filename) // '') ne ($version // '');
 
-            my $db = read_macdb_file();
-            set_cache_entry($db, $mac, 'ip4', $ip4) if defined($ip4);
-            set_cache_entry($db, $mac, 'ip6', $ip6) if defined($ip6);
+            my $db = read_macdb();
+            my $entry = $db->{vnets}->{$vnetid}->{ lc($mac) } //= {};
+            for my $family (qw(ip4 ip6)) {
+                my $ip = $family eq 'ip4' ? $ip4 : $ip6;
+                if (!defined($ip)) {
+                    delete $entry->{$family};
+                    next;
+                }
+                $entry->{$family} = $ip;
+                set_old_cache_entry($db, $mac, $family, $ip);
+            }
             write_macdb($db);
         },
     );
diff --git a/src/PVE/Network/SDN/Ipams/NetboxPlugin.pm b/src/PVE/Network/SDN/Ipams/NetboxPlugin.pm
index 9e1c9537e6ec..84ce4382e0e7 100644
--- a/src/PVE/Network/SDN/Ipams/NetboxPlugin.pm
+++ b/src/PVE/Network/SDN/Ipams/NetboxPlugin.pm
@@ -360,7 +360,7 @@ sub add_range_next_freeip {
 }
 
 sub del_ip {
-    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr) = @_;
+    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr, $mac) = @_;
 
     return if !$ip;
 
@@ -369,6 +369,19 @@ sub del_ip {
         warn "could not find id for ip $ip";
         return;
     }
+    if ($mac) {
+        my $record =
+            eval { netbox_api_request($plugin_config, "GET", "/ipam/ip-addresses/$ip_id/") };
+        if ($@) {
+            return 0 if $noerr;
+            die "could not check who holds IP $ip: $@";
+        }
+        my ($owner) = ($record->{description} // '') =~ m/^mac:(\S+)$/;
+        if ($owner && lc($owner) ne lc($mac)) {
+            warn "not releasing IP '$ip' for $mac, it is held by $owner\n";
+            return 0;
+        }
+    }
 
     eval { netbox_api_request($plugin_config, "DELETE", "/ipam/ip-addresses/$ip_id/"); };
     if ($@) {
diff --git a/src/PVE/Network/SDN/Ipams/PVEPlugin.pm b/src/PVE/Network/SDN/Ipams/PVEPlugin.pm
index 43d0a3daaf9a..86e594a4c604 100644
--- a/src/PVE/Network/SDN/Ipams/PVEPlugin.pm
+++ b/src/PVE/Network/SDN/Ipams/PVEPlugin.pm
@@ -249,11 +249,12 @@ sub add_range_next_freeip {
 }
 
 sub del_ip {
-    my ($class, $plugin_config, $subnetid, $subnet, $ip) = @_;
+    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr, $mac) = @_;
 
     my $cidr = $subnet->{cidr};
     my $zone = $subnet->{zone};
 
+    my $released = 1;
     cfs_lock_file(
         $ipamdb_file,
         undef,
@@ -271,11 +272,18 @@ sub del_ip {
                 warn "IP '$ip' does not exist in IPAM DB\n";
                 return;
             }
+            my $owner = $dbsubnet->{ips}->{$ip}->{mac};
+            if ($mac && $owner && lc($owner) ne lc($mac)) {
+                warn "not releasing IP '$ip' for $mac, it is held by $owner\n";
+                $released = 0;
+                return;
+            }
             delete $dbsubnet->{ips}->{$ip};
             write_db($db);
         },
     );
     die "$@" if $@;
+    return $released;
 }
 
 sub get_ips_from_mac {
@@ -297,7 +305,8 @@ sub get_ips_from_mac {
         my $ips = $dbzone->{subnets}->{$subnet}->{ips} // next;
         for my $ip (keys %$ips) {
             my $ipobject = $ips->{$ip};
-            if ($ipobject->{mac} && $ipobject->{mac} eq $mac) {
+            # a record keeps the MAC as its writer spelled it
+            if ($ipobject->{mac} && lc($ipobject->{mac}) eq lc($mac)) {
                 if (Net::IP::ip_is_ipv4($ip)) {
                     $ip4 = $ip;
                 } else {
diff --git a/src/PVE/Network/SDN/Ipams/PhpIpamPlugin.pm b/src/PVE/Network/SDN/Ipams/PhpIpamPlugin.pm
index 12d470e802c4..e969c3dcdf4f 100644
--- a/src/PVE/Network/SDN/Ipams/PhpIpamPlugin.pm
+++ b/src/PVE/Network/SDN/Ipams/PhpIpamPlugin.pm
@@ -239,7 +239,7 @@ sub add_range_next_freeip {
 }
 
 sub del_ip {
-    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr) = @_;
+    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr, $mac) = @_;
 
     return if !$ip;
 
@@ -250,6 +250,22 @@ sub del_ip {
 
     my $ip_id = get_ip_id($url, $ip, $headers);
     return if !$ip_id;
+    if ($mac) {
+        my $record = eval {
+            PVE::Network::SDN::api_request(
+                "GET", "$url/addresses/$ip_id/", $headers, undef, $fingerprint,
+            );
+        };
+        if ($@) {
+            return 0 if $noerr;
+            die "could not check who holds IP $ip: $@";
+        }
+        my $owner = ($record->{data} // {})->{mac};
+        if ($owner && lc($owner) ne lc($mac)) {
+            warn "not releasing IP '$ip' for $mac, it is held by $owner\n";
+            return 0;
+        }
+    }
 
     eval {
         PVE::Network::SDN::api_request(
@@ -259,6 +275,8 @@ sub del_ip {
     if ($@) {
         die "error delete ip $ip: $@" if !$noerr;
     }
+
+    return 1;
 }
 
 sub get_ips_from_mac {
diff --git a/src/PVE/Network/SDN/Ipams/Plugin.pm b/src/PVE/Network/SDN/Ipams/Plugin.pm
index 14f8fe9e67fe..6e23d5434d4b 100644
--- a/src/PVE/Network/SDN/Ipams/Plugin.pm
+++ b/src/PVE/Network/SDN/Ipams/Plugin.pm
@@ -132,8 +132,10 @@ sub add_range_next_freeip {
     die "please implement inside plugin";
 }
 
+# a MAC names the guest the address is released for, a record holding the
+# address for another MAC is then left alone and 0 returned
 sub del_ip {
-    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr) = @_;
+    my ($class, $plugin_config, $subnetid, $subnet, $ip, $noerr, $mac) = @_;
 
     die "please implement inside plugin";
 }
diff --git a/src/PVE/Network/SDN/Subnets.pm b/src/PVE/Network/SDN/Subnets.pm
index f8be98201476..a7fa7056e556 100644
--- a/src/PVE/Network/SDN/Subnets.pm
+++ b/src/PVE/Network/SDN/Subnets.pm
@@ -266,7 +266,7 @@ sub add_next_free_ip {
 
         die $@ if $@;
 
-        eval { PVE::Network::SDN::Ipams::add_cache_mac_ip($mac, $ip); };
+        eval { PVE::Network::SDN::Ipams::add_cache_mac_ip($subnet->{vnet}, $mac, $ip); };
         warn $@ if $@;
     }
 
@@ -325,7 +325,7 @@ sub add_ip {
         };
         die $@ if $@;
 
-        eval { PVE::Network::SDN::Ipams::add_cache_mac_ip($mac, $ip) if $mac; };
+        eval { PVE::Network::SDN::Ipams::add_cache_mac_ip($subnet->{vnet}, $mac, $ip) if $mac; };
         warn $@ if $@;
     }
 
@@ -413,18 +413,22 @@ sub del_ip {
         verify_dns_zone($reversednszone, $reversedns);
     }
 
+    my $released;
     if ($ipamid) {
         my $ipam_cfg = PVE::Network::SDN::Ipams::config();
         my $plugin_config = $ipam_cfg->{ids}->{$ipamid};
         my $plugin = PVE::Network::SDN::Ipams::Plugin->lookup($plugin_config->{type});
-        $plugin->del_ip($plugin_config, $subnetid, $subnet, $ip);
+        $released = $plugin->del_ip($plugin_config, $subnetid, $subnet, $ip, undef, $mac);
 
         if ($mac) {
-            eval { PVE::Network::SDN::Ipams::del_cache_mac_ip($mac, $ip) };
+            eval { PVE::Network::SDN::Ipams::del_cache_mac_ip($subnet->{vnet}, $mac, $ip) };
             warn $@ if $@;
         }
     }
 
+    # a record another MAC holds keeps its DNS entries as well
+    return 0 if defined($released) && !$released;
+
     eval {
         if (!$skipdns) {
             del_dns_record($dnszone, $dns, $hostname, $ip);
@@ -434,6 +438,8 @@ sub del_ip {
     if ($@) {
         warn $@;
     }
+
+    return $released;
 }
 
 =head3 api_get_subnet($cfg, $id)
diff --git a/src/PVE/Network/SDN/Vnets.pm b/src/PVE/Network/SDN/Vnets.pm
index 395aa32b6a84..58db0f803f8b 100644
--- a/src/PVE/Network/SDN/Vnets.pm
+++ b/src/PVE/Network/SDN/Vnets.pm
@@ -194,18 +194,18 @@ sub del_ip {
 }
 
 sub get_ips_from_mac {
-    my ($vnetid, $mac) = @_;
+    my ($vnetid, $mac, $running) = @_;
 
-    my $vnet = PVE::Network::SDN::Vnets::get_vnet($vnetid);
+    my $vnet = PVE::Network::SDN::Vnets::get_vnet($vnetid, $running);
     return if !$vnet;
 
     my $zoneid = $vnet->{zone};
-    my $zone = PVE::Network::SDN::Zones::get_zone($zoneid);
+    my $zone = PVE::Network::SDN::Zones::get_zone($zoneid, $running);
 
     return if !$zone->{ipam} || !$zone->{dhcp};
 
-    my $subnets = PVE::Network::SDN::Vnets::get_subnets($vnetid);
-    return PVE::Network::SDN::Ipams::get_ips_from_mac($mac, $zoneid, $zone, $subnets);
+    my $subnets = PVE::Network::SDN::Vnets::get_subnets($vnetid, $running);
+    return PVE::Network::SDN::Ipams::get_ips_from_mac($vnetid, $mac, $zoneid, $zone, $subnets);
 }
 
 sub del_ips_from_mac {
diff --git a/src/test/run_test_ipams.pl b/src/test/run_test_ipams.pl
index f9c44fe1a3b9..f07b4bb4b486 100755
--- a/src/test/run_test_ipams.pl
+++ b/src/test/run_test_ipams.pl
@@ -219,5 +219,54 @@ foreach my $path (@plugins) {
 
 }
 
+# an address released for a MAC stays when the external IPAM records it
+# for another MAC
+{
+    my $subnet = { zone => 'z', cidr => '10.0.0.0/24' };
+    my $owner;
+    my $check_fails = 0;
+    my @deletes;
+    my $sdn_module = Test::MockModule->new("PVE::Network::SDN");
+    $sdn_module->mock(
+        api_request => sub {
+            my ($method, $url) = @_;
+            die "Invalid response from server: 503 Service Unavailable\n"
+                if $check_fails && $method eq 'GET';
+            push @deletes, $url if $method eq 'DELETE';
+            return {
+                description => "mac:$owner",
+                data => { mac => $owner },
+            };
+        },
+    );
+    my $netbox = Test::MockModule->new('PVE::Network::SDN::Ipams::NetboxPlugin');
+    $netbox->mock(subnet_prefix => sub { return (1, 2); }, get_ip_id => sub { return 3; });
+    my $phpipam = Test::MockModule->new('PVE::Network::SDN::Ipams::PhpIpamPlugin');
+    $phpipam->mock(subnet_id => sub { return 7; }, get_ip_id => sub { return 3; });
+
+    for my $plugin (
+        'PVE::Network::SDN::Ipams::NetboxPlugin',
+        'PVE::Network::SDN::Ipams::PhpIpamPlugin',
+    ) {
+        my $config = { url => 'u', token => 't' };
+        @deletes = ();
+        $owner = 'da:65:8f:18:9b:70';
+        $plugin->del_ip($config, 'z-10.0.0.0-24', $subnet, '10.0.0.50', 1, 'DA:65:8F:18:9B:6F');
+        is(scalar(@deletes), 0, "$plugin keeps the record of another MAC");
+        $owner = 'da:65:8f:18:9b:6f';
+        $plugin->del_ip($config, 'z-10.0.0.0-24', $subnet, '10.0.0.50', 1, 'DA:65:8F:18:9B:6F');
+        is(scalar(@deletes), 1, "$plugin releases the MAC's own record");
+
+        # a failed check of the holder releases nothing either
+        @deletes = ();
+        $check_fails = 1;
+        my $released =
+            $plugin->del_ip($config, 'z-10.0.0.0-24', $subnet, '10.0.0.50', 1, 'DA:65:8F:18:9B:6F');
+        $check_fails = 0;
+        is($released, 0, "$plugin releases nothing when the check of the holder fails");
+        is(scalar(@deletes), 0, "$plugin sent no delete for it");
+    }
+}
+
 done_testing();
 
diff --git a/src/test/run_test_vnets_blackbox.pl b/src/test/run_test_vnets_blackbox.pl
index 752e5b0d3c70..b00be65bae48 100755
--- a/src/test/run_test_vnets_blackbox.pl
+++ b/src/test/run_test_vnets_blackbox.pl
@@ -4,8 +4,11 @@ use strict;
 use warnings;
 
 use lib qw(..);
+use Fcntl qw(LOCK_EX);
 use File::Slurp;
+use IO::File;
 use JSON;
+use POSIX qw(_exit);
 use List::Util qw(first all);
 use NetAddr::IP qw(:lower);
 
@@ -195,7 +198,7 @@ $mocked_sdn_ipams->mock(
         my ($cfg) = @_;
         $test_state->{ipam_config} = $cfg;
     },
-    read_macdb_file => sub {
+    read_macdb => sub {
         return PVE::Network::SDN::Ipams::json_reader(undef, encode_json($test_state->{macdb}));
     },
     write_macdb => sub {
@@ -518,74 +521,118 @@ sub test_ipam_cache_lookup {
     # an entry holding no address, left behind by an earlier miss, does not
     # hide a record created since
     create_ip({ zone => $zoneid, vnet => $vnetid, mac => $mac, ip => "10.0.0.50" });
-    $test_state->{macdb}->{macs}->{$mac} = { ip4 => undef, ip6 => undef };
+    $test_state->{macdb} = { macs => { $mac => { ip4 => undef, ip6 => undef } } };
     my @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
     is($ips[0], "10.0.0.50", "$test_name: an address-less entry counts as a miss");
-    is(
-        $test_state->{macdb}->{macs}->{$mac}->{ip4},
-        "10.0.0.50",
-        "$test_name: the answer is cached",
+    is_deeply(
+        $test_state->{macdb},
+        {
+            macs => { $mac => { ip4 => "10.0.0.50", ip6 => undef } },
+            vnets => { $vnetid => { $mac => { ip4 => "10.0.0.50" } } },
+        },
+        "$test_name: the answer is cached in both maps",
     );
 
     # a write to the cache between the miss and the fill, which the file's
     # version shows, keeps the IPAM's answer out of the cache
-    delete $test_state->{macdb}->{macs}->{$mac};
+    $test_state->{macdb} = {};
     my $version = 0;
     $mocked_pve_cluster_obj->mock(cfs_file_version => sub { return $version++ });
     @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
     $mocked_pve_cluster_obj->unmock('cfs_file_version');
     is($ips[0], "10.0.0.50", "$test_name: the lookup still answers");
-    ok(!exists $test_state->{macdb}->{macs}->{$mac},
-        "$test_name: but leaves a changed cache alone");
+    is_deeply($test_state->{macdb}, {}, "$test_name: but leaves a changed cache alone");
 
-    # an entry written under the config's spelling, as a node not yet upgraded
-    # writes it, is read and released like any other and keeps its key
+    # a MAC new to the map under the MAC alone is written as the caller
+    # spells it, a node not yet upgraded looks it up so
+    $test_state->{macdb} = {};
+    PVE::Network::SDN::Ipams::add_cache_mac_ip($vnetid, uc($mac), '10.0.0.160');
+    is_deeply(
+        [keys %{ $test_state->{macdb}->{macs} }],
+        [uc($mac)],
+        "$test_name: a new key carries the caller's spelling",
+    );
+    is(
+        $test_state->{macdb}->{vnets}->{$vnetid}->{$mac}->{ip4},
+        '10.0.0.160',
+        "$test_name: and the vnet's entry is keyed in lower case",
+    );
+    $test_state->{macdb} = {};
+
+    # an entry written under the config's spelling is read and released like
+    # any other
     my $other = "da:65:8f:18:9b:70";
     $test_state->{macdb}->{macs} = {
         uc($mac) => { ip4 => '10.0.0.150' },
         uc($other) => { ip4 => '10.0.0.170' },
     };
     my ($ip4) = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
-    is($ip4, '10.0.0.150', "$test_name: an entry is found however it is spelled");
-    PVE::Network::SDN::Ipams::del_cache_mac_ip($mac, '10.0.0.150');
+    is($ip4, '10.0.0.150', "$test_name: an old entry is found however it is spelled");
+    PVE::Network::SDN::Ipams::del_cache_mac_ip($vnetid, $mac, '10.0.0.150');
     is_deeply(
         $test_state->{macdb}->{macs},
         { uc($other) => { ip4 => '10.0.0.170' } },
         "$test_name: and released, the other entry left as written",
     );
 
-    # a write in another spelling goes to the spelling the file holds
+    # a write by this code updates an old entry under the spelling a node not
+    # yet upgraded wrote it in, so that node still finds it
     $test_state->{macdb}->{macs} = { uc($mac) => { ip4 => '10.0.0.150' } };
-    PVE::Network::SDN::Ipams::add_cache_mac_ip($mac, 'fd00::150');
+    PVE::Network::SDN::Ipams::add_cache_mac_ip($vnetid, uc($mac), 'fd00::150');
     is_deeply(
         $test_state->{macdb}->{macs},
         { uc($mac) => { ip4 => '10.0.0.150', ip6 => 'fd00::150' } },
-        "$test_name: a write keeps the key as written",
+        "$test_name: a write updates the old entry under its spelling",
+    );
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac)],
+        ['10.0.0.150', 'fd00::150'],
+        "$test_name: holding both addresses",
     );
 
-    # a lookup past an address-less entry asks the IPAM and fills the entry
-    # under its spelling
-    $test_state->{macdb}->{macs} = { uc($mac) => { ip4 => undef, ip6 => undef } };
+    # a vnet's entry and an old one that disagree are not trusted either, a
+    # node not yet upgraded may have moved the address since, and a family
+    # the old entry lacks was released by such a node
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.0.0.151' } },
+        vnets => { $vnetid => { $mac => { ip4 => '10.0.0.150', ip6 => 'fd00::150' } } },
+    };
     @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
-    is($ips[0], '10.0.0.50', "$test_name: a lookup past an address-less entry asks the IPAM");
+    is(
+        $ips[0],
+        '10.0.0.50',
+        "$test_name: an old entry disagreeing with the vnet's counts as a miss",
+    );
     is_deeply(
-        $test_state->{macdb}->{macs},
-        { uc($mac) => { ip4 => '10.0.0.50', ip6 => undef } },
-        "$test_name: and fills the entry under its spelling",
+        $test_state->{macdb},
+        {
+            macs => { $mac => { ip4 => '10.0.0.50' } },
+            vnets => { $vnetid => { $mac => { ip4 => '10.0.0.50' } } },
+        },
+        "$test_name: the answer replaces the vnet's entry and the old one,"
+            . " the family without a record is gone",
     );
 
     # two spellings that disagree are not trusted, the lookup asks the IPAM
-    $test_state->{macdb}->{macs} = {
-        uc($mac) => { ip4 => '10.0.0.150' },
-        $mac => { ip4 => '10.0.0.151' },
+    $test_state->{macdb} = {
+        macs => {
+            uc($mac) => { ip4 => '10.0.0.150' },
+            $mac => { ip4 => '10.0.0.151' },
+        },
     };
     @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
     is($ips[0], '10.0.0.50', "$test_name: a clash of spellings counts as a miss");
-    is_deeply(
-        $test_state->{macdb}->{macs},
-        { uc($mac) => { ip4 => '10.0.0.50' }, $mac => { ip4 => '10.0.0.50' } },
-        "$test_name: and its write brings both spellings in line",
-    );
+
+    # an old entry's address outside the vnet's subnets is another vnet's
+    $test_state->{macdb} = { macs => { uc($mac) => { ip4 => '10.9.0.150' } } };
+    @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, $mac);
+    is($ips[0], '10.0.0.50', "$test_name: an old address of another vnet counts as a miss");
+
+    # the record keeps the MAC as its writer spelled it, a lookup in another
+    # spelling finds it
+    $test_state->{macdb} = {};
+    @ips = PVE::Network::SDN::Vnets::get_ips_from_mac($vnetid, uc($mac));
+    is($ips[0], '10.0.0.50', "$test_name: a lookup in another spelling finds the record");
 
     # a mapping whose record is gone is still deleted, and its cache entry
     # with it
@@ -1169,21 +1216,40 @@ sub test_dnsmasq_stale_reservation_replaced {
     my $stale = "da:65:8f:18:9b:01";
     my $gone = "da:65:8f:18:9b:02";
     my $added = "da:65:8f:18:9b:03";
+    create_dhcp_zone("TESTZONE", "testvnet", "10.0.1.0/24");
     $test_state->{macdb} = {
-        macs => {
-            $stale => { ip4 => '10.0.1.11' },
-            $added => { ip4 => '10.0.1.13' },
+        macs => { $stale => { ip4 => '10.0.1.11' }, $added => { ip4 => '10.0.1.13' } },
+        vnets => {
+            testvnet => {
+                $stale => { ip4 => '10.0.1.11' },
+                $added => { ip4 => '10.0.1.13' },
+            },
         },
     };
     PVE::Tools::file_set_contents($TMP_ETHERS_FILE, "$stale,10.0.1.10\n$gone,10.0.1.12\n");
 
-    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping('TESTZONE', $added);
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping('TESTZONE', 'testvnet', $added);
 
     is(
         PVE::Tools::file_get_contents($TMP_ETHERS_FILE),
         "$stale,10.0.1.11\n$added,10.0.1.13\n",
         "$test_name: the stale line is replaced, the unknown one dropped",
     );
+
+    # the changed MAC's own line follows the IPAM's answer, not the cache,
+    # when its entry is no longer witnessed
+    create_ip({ zone => "TESTZONE", vnet => "testvnet", mac => $added, ip => "10.0.1.20" });
+    $test_state->{macdb} = {
+        macs => { $added => { ip4 => '10.0.1.22' } },
+        vnets => { testvnet => { $added => { ip4 => '10.0.1.21' } } },
+    };
+    PVE::Tools::file_set_contents($TMP_ETHERS_FILE, "$added,10.0.1.21\n");
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping('TESTZONE', 'testvnet', $added);
+    is(
+        PVE::Tools::file_get_contents($TMP_ETHERS_FILE),
+        "$added,10.0.1.20\n",
+        "$test_name: the changed MAC's stale line is replaced by the IPAM's answer",
+    );
 }
 
 run_test(\&test_dnsmasq_stale_reservation_replaced);
@@ -1405,8 +1471,390 @@ sub test_ipam_per_vnet {
             "$test_name: vnet$n answers with its own address",
         );
     }
+
+    # a mapping edit on one vnet leaves the other's answer alone
+    PVE::API2::Network::SDN::Ips->ipupdate(
+        { zone => $zoneid, vnet => 'vnet1', mac => $mac, ip => '10.1.0.51' });
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet2', $mac)],
+        ['10.2.0.50', undef],
+        "$test_name: an edit on the first vnet leaves the second's answer alone",
+    );
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet1', $mac)],
+        ['10.1.0.51', undef],
+        "$test_name: and changes the first's",
+    );
+
+    # an entry from before the cache was kept per vnet answers for a vnet
+    # without an entry of its own, until a write for the MAC on another vnet
+    # replaces it in the map under the MAC alone, which holds one address.
+    # The vnet then asks the IPAM
+    $test_state->{macdb} = { macs => { $mac => { ip4 => '10.2.0.60' } } };
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet2', $mac)],
+        ['10.2.0.60', undef],
+        "$test_name: an old entry answers for a vnet without an entry of its own",
+    );
+    create_ip({ zone => $zoneid, vnet => 'vnet1', mac => $mac, ip => '10.1.0.52' });
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet2', $mac)],
+        ['10.2.0.50', undef],
+        "$test_name: once a write for the MAC elsewhere replaced it, the IPAM answers",
+    );
+
+    # a vnet's address holds only while the map under the MAC alone names it,
+    # since that map is where a node not yet upgraded releases or moves an
+    # address. With the MAC on two vnets the map witnesses one of them
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.2.0.50' } },
+        vnets => {
+            vnet1 => { $mac => { ip4 => '10.1.0.50' } },
+            vnet2 => { $mac => { ip4 => '10.2.0.50' } },
+        },
+    };
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet2', $mac)],
+        ['10.2.0.50', undef],
+        "$test_name: the vnet whose address the old map names answers from the cache",
+    );
+    like(
+        (PVE::Network::SDN::Vnets::get_ips_from_mac('vnet1', $mac))[0],
+        qr/^10\.1\.0\.5[12]$/,
+        "$test_name: the other one asks the IPAM",
+    );
+    like(
+        $test_state->{macdb}->{macs}->{$mac}->{ip4},
+        qr/^10\.1\.0\.5[12]$/,
+        "$test_name: and its answer takes the old map",
+    );
+    delete $test_state->{macdb}->{macs};
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet2', $mac)],
+        ['10.2.0.50', undef],
+        "$test_name: without the family in the old map the IPAM is asked",
+    );
+
+    # the address of the MAC on a vnet of another zone with the same range
+    # sits in the old map as well, and is that vnet's, so a vnet without an
+    # entry asks the IPAM rather than taking it, and one with an entry the
+    # old map does not name asks as well
+    create_zone({ type => "simple", dhcp => "dnsmasq", ipam => "pve", zone => "ZONEB" });
+    create_dhcp_vnet("ZONEB", "vnet3", "10.1.0.0/24");
+    create_ip({ zone => "ZONEB", vnet => "vnet3", mac => $mac, ip => "10.1.0.70" });
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.1.0.52' } },
+        vnets => { vnet1 => { $mac => { ip4 => '10.1.0.52' } } },
+    };
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet3', $mac)],
+        ['10.1.0.70', undef],
+        "$test_name: another zone's address in the old map is not taken"
+            . " for a vnet without an entry",
+    );
+    $test_state->{macdb}->{macs}->{$mac}->{ip4} = '10.1.0.52';
+    $test_state->{macdb}->{vnets}->{vnet3}->{$mac} = { ip4 => '10.1.0.71' };
+    is_deeply(
+        [PVE::Network::SDN::Vnets::get_ips_from_mac('vnet3', $mac)],
+        ['10.1.0.70', undef],
+        "$test_name: and an entry the old map does not name is checked with the IPAM",
+    );
+
+    # a release on an upgraded node clears the old map's slot, the MAC's
+    # other vnet asks the IPAM at its next start and takes it
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.2.0.50' } },
+        vnets => {
+            vnet1 => { $mac => { ip4 => '10.1.0.52' } },
+            vnet2 => { $mac => { ip4 => '10.2.0.50' } },
+        },
+    };
+    PVE::Network::SDN::Ipams::del_cache_mac_ip('vnet2', $mac, '10.2.0.50');
+    is_deeply(
+        $test_state->{macdb},
+        { macs => {}, vnets => { vnet1 => { $mac => { ip4 => '10.1.0.52' } } } },
+        "$test_name: a release clears the old map's slot",
+    );
 }
 
 run_test(\&test_ipam_per_vnet);
 
+sub test_ipam_edit_stale_entry {
+    my $test_name = (split(/::/, (caller(0))[3]))[-1];
+    my $zoneid = "TESTZONE";
+    my $vnetid = "testvnet";
+    my $mac = "da:65:8f:18:9b:6f";
+    my $other = "da:65:8f:18:9b:70";
+
+    create_dhcp_zone($zoneid, $vnetid, "10.0.0.0/24");
+
+    # a node not yet upgraded released the MAC's address and handed it to
+    # another guest, while the vnet's cache entry still holds it. The entry
+    # is not trusted, so the edit releases nothing and records the new address
+    create_ip({ zone => $zoneid, vnet => $vnetid, mac => $other, ip => '10.0.0.50' });
+    $test_state->{macdb}->{vnets}->{$vnetid}->{$mac} = { ip4 => '10.0.0.50' };
+
+    PVE::API2::Network::SDN::Ips->ipupdate(
+        { zone => $zoneid, vnet => $vnetid, mac => $mac, ip => '10.0.0.60' });
+    is(
+        get_ip4(get_ips_from_mac($other)),
+        '10.0.0.50',
+        "$test_name: an edit leaves the other guest's record alone",
+    );
+    is(get_ip4(get_ips_from_mac($mac)), '10.0.0.60', "$test_name: and records the new address");
+    is(
+        $test_state->{macdb}->{vnets}->{$vnetid}->{$mac}->{ip4},
+        '10.0.0.60',
+        "$test_name: which the cache holds now",
+    );
+
+    # an edit that fails keeps its own error, the address the other guest
+    # holds was not released and is not added back
+    my $third = "da:65:8f:18:9b:71";
+    create_ip({ zone => $zoneid, vnet => $vnetid, mac => $third, ip => '10.0.0.70' });
+    $test_state->{macdb}->{vnets}->{$vnetid}->{$mac} = { ip4 => '10.0.0.50' };
+    $test_state->{macdb}->{macs}->{$mac} = { ip4 => '10.0.0.50' };
+    eval {
+        PVE::API2::Network::SDN::Ips->ipupdate(
+            { zone => $zoneid, vnet => $vnetid, mac => $mac, ip => '10.0.0.70' });
+    };
+    like($@, qr/10\.0\.0\.70/, "$test_name: a failed edit names the address it could not add");
+    is(
+        get_ip4(get_ips_from_mac($other)),
+        '10.0.0.50',
+        "$test_name: and leaves the other guest's record alone",
+    );
+
+    # a deletion of a mapping the IPAM does not hold is refused
+    $test_state->{macdb}->{vnets}->{$vnetid}->{$mac} = { ip4 => '10.0.0.50' };
+    eval {
+        PVE::API2::Network::SDN::Ips->ipdelete(
+            { zone => $zoneid, vnet => $vnetid, mac => $mac, ip => '10.0.0.50' });
+    };
+    like($@, qr/not mapped to/, "$test_name: a deletion for another guest's address is refused");
+}
+
+run_test(\&test_ipam_edit_stale_entry);
+
+sub test_dnsmasq_reservation_per_vnet {
+    my $test_name = (split(/::/, (caller(0))[3]))[-1];
+    my $zoneid = "TESTZONE";
+    my $mac = "da:65:8f:18:9b:6f";
+
+    # the same MAC on two vnets of one zone has a reservation on each, and a
+    # start on one vnet changes that vnet's line only
+    create_zone({ type => "simple", dhcp => "dnsmasq", ipam => "pve", zone => $zoneid });
+    for my $n (1, 2) {
+        create_dhcp_vnet($zoneid, "vnet$n", "10.$n.0.0/24");
+        create_ip({ zone => $zoneid, vnet => "vnet$n", mac => $mac, ip => "10.$n.0.50" });
+    }
+
+    my $reservations = sub {
+        return [sort grep { $_ } split(/\n/, PVE::Tools::file_get_contents($TMP_ETHERS_FILE))];
+    };
+
+    nic_start("vnet$_", $mac, "999", "testhostname") for (1, 2, 1);
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.50", "$mac,10.2.0.50"],
+        "$test_name: each vnet keeps its reservation",
+    );
+
+    PVE::API2::Network::SDN::Ips->ipupdate(
+        { zone => $zoneid, vnet => 'vnet1', mac => $mac, ip => '10.1.0.51' });
+    nic_start('vnet1', $mac, "999", "testhostname");
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51", "$mac,10.2.0.50"],
+        "$test_name: an edit replaces the reservation of its vnet only",
+    );
+
+    # the other lines are checked against the cache alone, so a line neither
+    # map holds is dropped without the IPAM being asked
+    delete $test_state->{macdb}->{vnets}->{vnet2};
+    delete $test_state->{macdb}->{macs};
+    nic_start('vnet1', $mac, "999", "testhostname");
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51"],
+        "$test_name: a line the cache does not hold is dropped",
+    );
+
+    # the changed MAC's own line is answered by the IPAM when the cache does
+    # not hold it
+    delete $test_state->{macdb}->{vnets}->{vnet1};
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac);
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51"],
+        "$test_name: the changed MAC's line is kept without a cache entry",
+    );
+
+    # a push at a resume asks the IPAM like any other when the cache cannot
+    # vouch for the MAC
+    $test_state->{macdb} = {};
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1);
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51"],
+        "$test_name: a push at a resume asks the IPAM for a MAC the cache does not hold",
+    );
+    is(
+        $test_state->{macdb}->{vnets}->{vnet1}->{$mac}->{ip4},
+        '10.1.0.51',
+        "$test_name: and fills the cache",
+    );
+
+    # a push at a resume waits for a push holding the file, but not long,
+    # so the guest is not kept paused for a push waiting on the IPAM
+    $mocked_pve_tools->unmock('lock_file');
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.1.0.52' } },
+        vnets => { vnet1 => { $mac => { ip4 => '10.1.0.52' } } },
+    };
+    # the holder says when it has the lock and leaves a file behind when it
+    # lets go, so the tests measure no time
+    my $let_go = "$TMP_ETHERS_FILE.let-go";
+    my $hold_lock = sub {
+        my ($seconds) = @_;
+        unlink($let_go);
+        pipe(my $has_lock, my $holder_end) or die "pipe failed: $!";
+        my $pid = fork() // die "fork failed: $!";
+        if (!$pid) {
+            my $lock = IO::File->new(">>$TMP_ETHERS_FILE.lock");
+            flock($lock, LOCK_EX);
+            close($holder_end);
+            select(undef, undef, undef, $seconds);
+            IO::File->new(">$let_go");
+            _exit(0);
+        }
+        close($holder_end);
+        readline($has_lock);
+        return $pid;
+    };
+    my $pid = $hold_lock->(0.3);
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1);
+    ok(-e $let_go, "$test_name: a push at a resume waits for a push holding the file");
+    waitpid($pid, 0);
+    is_deeply(
+        $reservations->(), ["$mac,10.1.0.52"], "$test_name: and writes the line",
+    );
+
+    $test_state->{macdb}->{vnets}->{vnet1}->{$mac}->{ip4} = '10.1.0.53';
+    $test_state->{macdb}->{macs}->{$mac}->{ip4} = '10.1.0.53';
+    $pid = $hold_lock->(2.5);
+    eval { PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1) };
+    my $err = $@;
+    ok(!-e $let_go, "$test_name: a push at a resume does not wait out a long hold");
+    waitpid($pid, 0);
+    like($err, qr/Unable to update/, "$test_name: and fails");
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.52"],
+        "$test_name: and left the line to the next push",
+    );
+    $mocked_pve_tools->mock(lock_file => $mocked_cfs_lock_file);
+
+    # the lookup's fill is skipped when the cache file changed meanwhile, the
+    # line then comes from the lookup's answer, the cache cannot vouch for it
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.1.0.61' } },
+        vnets => { vnet1 => { $mac => { ip4 => '10.1.0.60' } } },
+    };
+    PVE::Tools::file_set_contents($TMP_ETHERS_FILE, "$mac,10.1.0.60\n");
+    my $version = 0;
+    $mocked_pve_cluster_obj->mock(cfs_file_version => sub { return $version++ });
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1);
+    $mocked_pve_cluster_obj->unmock('cfs_file_version');
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51"],
+        "$test_name: a push at a resume writes the IPAM's answer when its fill was skipped",
+    );
+    is(
+        $test_state->{macdb}->{vnets}->{vnet1}->{$mac}->{ip4},
+        '10.1.0.60',
+        "$test_name: with the cache left as it was",
+    );
+
+    # the same with no entry for the MAC in either map
+    $test_state->{macdb} = {};
+    PVE::Tools::file_set_contents($TMP_ETHERS_FILE, "$mac,10.1.0.60\n");
+    $version = 0;
+    $mocked_pve_cluster_obj->mock(cfs_file_version => sub { return $version++ });
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1);
+    $mocked_pve_cluster_obj->unmock('cfs_file_version');
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.51"],
+        "$test_name: a push at a resume writes the answer when the cache has nothing",
+    );
+    is_deeply($test_state->{macdb}, {}, "$test_name: and leaves the cache empty");
+
+    # a lookup that fails in the sweep ends the asking for that push, so an
+    # IPAM that does not answer costs one request, the lines are kept
+    my ($other1, $other2) = ('da:65:8f:18:9b:72', 'da:65:8f:18:9b:73');
+    $test_state->{macdb} = {
+        macs => {
+            $mac => { ip4 => '10.1.0.53' },
+            $other1 => { ip4 => '10.1.0.71' },
+            $other2 => { ip4 => '10.2.0.71' },
+        },
+        vnets => {
+            vnet1 => { $mac => { ip4 => '10.1.0.53' }, $other1 => { ip4 => '10.1.0.70' } },
+            vnet2 => { $other2 => { ip4 => '10.2.0.70' } },
+        },
+    };
+    PVE::Tools::file_set_contents(
+        $TMP_ETHERS_FILE,
+        "$mac,10.1.0.53\n$other1,10.1.0.70\n$other2,10.2.0.70\n",
+    );
+    my $asked = 0;
+    my $mocked_pve_plugin = Test::MockModule->new('PVE::Network::SDN::Ipams::PVEPlugin');
+    $mocked_pve_plugin->mock(get_ips_from_mac => sub { $asked++; die "IPAM down\n"; });
+    my @warned;
+    {
+        local $SIG{__WARN__} = sub { push @warned, @_; };
+        PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac);
+    }
+    $mocked_pve_plugin->unmock('get_ips_from_mac');
+    is($asked, 1, "$test_name: a failed lookup in the sweep ends the asking");
+    is(scalar(@warned), 1, "$test_name: with one warning");
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.53", "$other1,10.1.0.70", "$other2,10.2.0.70"],
+        "$test_name: and the lines are kept",
+    );
+
+    # a line the cache cannot vouch for gets the IPAM's answer at any push,
+    # a push at a resume writes its own line only
+    $test_state->{macdb} = {
+        macs => { $mac => { ip4 => '10.1.0.52' } },
+        vnets => {
+            vnet1 => { $mac => { ip4 => '10.1.0.52' } },
+            vnet2 => { $mac => { ip4 => '10.2.0.60' } },
+        },
+    };
+    PVE::Tools::file_set_contents($TMP_ETHERS_FILE, "$mac,10.1.0.52\n$mac,10.2.0.61\n");
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac, 1);
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.52", "$mac,10.2.0.61"],
+        "$test_name: a push at a resume leaves the other lines alone",
+    );
+    PVE::Network::SDN::Dhcp::Dnsmasq->update_ip_mapping($zoneid, 'vnet1', $mac);
+    is_deeply(
+        $reservations->(),
+        ["$mac,10.1.0.52", "$mac,10.2.0.50"],
+        "$test_name: another push gives a line the cache cannot vouch for the IPAM's answer",
+    );
+    is(
+        $test_state->{macdb}->{vnets}->{vnet2}->{$mac}->{ip4},
+        '10.2.0.50',
+        "$test_name: and fills its cache entry",
+    );
+}
+
+run_test(\&test_dnsmasq_reservation_per_vnet);
+
 done_testing();
-- 
2.47.3