From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id 80CDC1FF0B6 for ; Thu, 12 Nov 2026 13:38:06 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 58FCF21689; Thu, 12 Nov 2026 13:38:03 +0100 (CET) Date: Thu, 12 Nov 2026 13:37:55 +0100 From: Arthur Bied-Charreton To: Stefan Hanreich Subject: Re: [RFC pve-network 1/1] sdn: apply changes on all nodes in parallel Message-ID: References: <20260709140625.275618-1-s.hanreich@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20260709140625.275618-1-s.hanreich@proxmox.com> X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1794487076716 X-SPAM-LEVEL: Spam detection results: 0 AWL -1.907 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_DBL_SPAM 5 Contains a spam URL listed in the Spamhaus DBL blocklist [sdn.pm] Message-ID-Hash: NIQIZPCJFP46UFVDIXYJYKF6PCOZ7P65 X-Message-ID-Hash: NIQIZPCJFP46UFVDIXYJYKF6PCOZ7P65 X-MailFrom: a.bied-charreton@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: pve-devel@lists.proxmox.com X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox VE development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On Thu, Jul 09, 2026 at 04:06:24PM +0200, Stefan Hanreich wrote: > Utilize the API client to spawn tasks on each node and then monitor > the progress of the tasks via the API as well. This allows for > starting the reload tasks in parallel. It is particularly useful for > large setups, where applying the SDN configuration can take a > considerable amount of time. > > Signed-off-by: Stefan Hanreich few comments inline, but works as advertised > --- > > Notes: > Sending this as an RFC, as this requires careful testing imo. > > An example where this change could backfire has been encountered > recently. A user applied the SDN configuration, which broke the > cluster / management network of the node that was carrying out the > reload network task. Since it was executed sequentially, only one node > was affected. In this scenario, with this patch applied, the whole > cluster would have been down. > > We could potentially provide an option that limits the amount of nodes > where the reload task runs in parallel and expose that in the UI? +1, with a default of 1 we would also get the opt-in behavior Alexandre mentioned [0] [0] https://lore.proxmox.com/all/081589622ba199dd6576bb504165b50c8e263e81.camel@groupe-cyllene.com/ > > Also, currently the task could never fail and errors were shown in the > respective reload tasks executed on the nodes, I've kept that behavior > the same, but introduced warnings if there are any issues with API > calls. > > src/PVE/API2/Network/SDN.pm | 109 ++++++++++++++++++++++++++---------- > 1 file changed, 78 insertions(+), 31 deletions(-) > > diff --git a/src/PVE/API2/Network/SDN.pm b/src/PVE/API2/Network/SDN.pm > index e3c8d9dd..dd2cf05a 100644 > --- a/src/PVE/API2/Network/SDN.pm > +++ b/src/PVE/API2/Network/SDN.pm > @@ -10,6 +10,7 @@ use PVE::Cluster qw(cfs_lock_file cfs_read_file cfs_write_file); > use PVE::Exception qw(raise_param_exc); > use PVE::JSONSchema qw(get_standard_option); > use PVE::RESTHandler; > +use PVE::RESTEnvironment qw(log_warn); > use PVE::RPCEnvironment; > use PVE::SafeSyslog; > use PVE::Tools qw(run_command extract_param); > @@ -107,30 +108,6 @@ __PACKAGE__->register_method({ > }, > }); > > -my $create_reload_network_worker = sub { > - my ($nodename, $regenerate_frr) = @_; > - > - my @command = ('pvesh', 'set', "/nodes/$nodename/network"); > - push(@command, '--regenerate-frr', $regenerate_frr); > - > - # FIXME: how to proxy to final node ? > - my $upid; > - print "$nodename: reloading network config\n"; > - run_command( > - \@command, > - outfunc => sub { > - my $line = shift; > - if ($line =~ /["']?(UPID:[^\s"']+)["']?$/) { > - $upid = $1; > - } > - }, > - ); > - #my $upid = PVE::API2::Network->reload_network_config({ node => $nodename }); > - my $res = PVE::Tools::upid_decode($upid); > - > - return $res->{pid}; > -}; > - > __PACKAGE__->register_method({ > name => 'lock', > protected => 1, > @@ -282,6 +259,41 @@ __PACKAGE__->register_method({ > }, > }); > > +sub create_api_client { > + my ($request_timeout) = @_; > + > + my $rpcenv = PVE::RPCEnvironment::get(); > + my $authuser = $rpcenv->get_user(); > + my $credentials = $rpcenv->get_credentials(); > + > + my $api_token = $credentials->{api_token}; > + my $ticket = $credentials->{ticket}; > + my $csrf_token = $credentials->{token}; > + > + my $node = PVE::INotify::nodename(); > + my $fingerprint = PVE::Cluster::get_node_fingerprint($node); > + > + my $conn_args = { > + protocol => 'https', > + host => 'localhost', # always call the api locally, let pveproxy handle the proxying > + port => 8006, > + username => $authuser, > + ticket => $ticket, > + apitoken => $api_token, > + timeout => $request_timeout // 25, # default slightly shorter than the proxy->daemon timeout > + cached_fingerprints => { > + $fingerprint => 1, > + }, > + }; > + > + my $api_client = PVE::APIClient::LWP->new($conn_args->%*); > + if (defined($csrf_token)) { > + $api_client->update_csrftoken($csrf_token); > + } > + > + return $api_client; > +} nit: afaict this is the third instance of such a function (pve-manager/PVE/API2/Cluster/Ceph.pm, pve-manager/PVE/API2/Cluster/BulkAction/Guest.pm also have such a function), we might wanna think about putting this somewhere where it could be reusable? > + > __PACKAGE__->register_method({ > name => 'reload', > protected => 1, > @@ -291,6 +303,7 @@ __PACKAGE__->register_method({ > permissions => { > check => ['perm', '/sdn', ['SDN.Allocate']], > }, > + expose_credentials => 1, > parameters => { > additionalProperties => 0, > properties => { > @@ -336,22 +349,56 @@ __PACKAGE__->register_method({ > my $regenerate_frr = ($previous_config_has_frr || $new_config_has_frr) ? 1 : 0; > > my $code = sub { > - $rpcenv->{type} = 'priv'; # to start tasks in background > PVE::Cluster::check_cfs_quorum(); > + > + my $api_client = create_api_client(); > my $nodelist = PVE::Cluster::get_nodelist(); > + > + my $tasks = {}; > + > for my $node (@$nodelist) { > - my $pid = eval { $create_reload_network_worker->($node, $regenerate_frr) }; > - warn $@ if $@; > + print "$node: reloading network config\n"; > + > + my $upid = eval { > + $api_client->put( > + "/nodes/$node/network", > + { > + 'regenerate-frr' => $regenerate_frr, > + }, > + ); > + }; > + > + if ($@) { > + log_warn("$node: could not reload network configuration: $@\n"); > + next; > + } > + > + $tasks->{$upid} = $node; > } > > - # FIXME: use libpve-apiclient (like in cluster join) to create > - # tasks and moitor the tasks. > + print "waiting for reload tasks to finish\n"; > > - return; > + while ($tasks->%*) { > + for my $upid (keys $tasks->%*) { > + my $node = $tasks->{$upid}; > + my $task = eval { $api_client->get("/nodes/$node/tasks/$upid/status") }; > + > + if ($@) { > + log_warn("could not get status of reload task: $@\n"); > + delete $tasks->{$upid}; > + next; > + } +1 for some kind of retry logic here, FWIW make_get_request already does something like this in pve-manager/PVE/API2/Cluster/BulkAction/Guest.pm/ > + > + next if $task->{status} eq 'running'; > + print "$node: reload task finished\n"; > + delete $tasks->{$upid}; > + } > + > + sleep(1); > + } > }; > > return $rpcenv->fork_worker('reloadnetworkall', undef, $authuser, $code); > - > }, > }); > > -- > 2.47.3 > > > > >