From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 87B1F1FF14C for ; Fri, 29 May 2026 15:30:36 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id DBCFBBD8D; Fri, 29 May 2026 15:30:31 +0200 (CEST) From: Dominik Csapak To: pdm-devel@lists.proxmox.com Subject: [PATCH datacenter-manager 3/4] server: connection: multi-client: use back-off state from remote cache Date: Fri, 29 May 2026 15:30:19 +0200 Message-ID: <20260529133026.3149896-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260529133026.3149896-1-d.csapak@proxmox.com> References: <20260529133026.3149896-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.049 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [connection.rs] Message-ID-Hash: JSYLAI7OMMIYQDN3KP3Z7SEJ3IXADKQF X-Message-ID-Hash: JSYLAI7OMMIYQDN3KP3Z7SEJ3IXADKQF X-MailFrom: d.csapak@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Datacenter Manager development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: This greatly reduces the wait time for api calls if a specific remote is offline for a prolonged period of time. Instead of waiting for a timeout on every api call to a remote which is known to be offline, return the last error immediately. Signed-off-by: Dominik Csapak --- server/src/connection.rs | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/server/src/connection.rs b/server/src/connection.rs index 28e4799f..7122713f 100644 --- a/server/src/connection.rs +++ b/server/src/connection.rs @@ -20,6 +20,7 @@ use serde::Serialize; use proxmox_acme_api::CertificateInfo; use proxmox_client::{Client, HttpApiClient, HttpApiResponse, HttpApiResponseStream, TlsOptions}; +use proxmox_time::epoch_i64; use pdm_api_types::remotes::{NodeUrl, Remote, RemoteType, TlsProbeOutcome}; use pve_api_types::client::PveClientImpl; @@ -718,6 +719,17 @@ macro_rules! try_request { ($self:expr, $method:expr, $path_and_query:expr, $params:expr, $how:ident) => { let params = $params.map(serde_json::to_value); Box::pin(async move { + // first check if the remote is reachable + { + let cache = crate::remote_cache::RemoteMappingCache::get(); + let (back_off_time, error) = + cache.remote_remaining_backoff_time(&$self.remote, epoch_i64()); + let error = error.unwrap_or("unknown error".to_string()); + if back_off_time > 0 { + return Err(proxmox_client::Error::Connect(error.into())); + } + } + let params = params .transpose() .map_err(|err| proxmox_client::Error::Anyhow(err.into()))?; @@ -748,13 +760,15 @@ macro_rules! try_request { last_err = Some(err); } Ok(result) => { - if !reachable { - log::error!("marking {hostname:?} as reachable again!"); - if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write() - { + if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write() { + if !reachable { cache.mark_host_reachable(&$self.remote, &hostname, true); - let _ = cache.save(); } + cache.mark_remote_reachable( + &$self.remote, + crate::remote_cache::RemoteState::Reachable, + ); + let _ = cache.save(); } return result; } @@ -764,6 +778,18 @@ macro_rules! try_request { } } + if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write() { + let error = last_err + .as_ref() + .map(|err| format!("remote not reachable: {err:?}")) + .unwrap_or("unknown error".to_string()); + cache.mark_remote_reachable( + &$self.remote, + crate::remote_cache::RemoteState::Unreachable(error), + ); + let _ = cache.save(); + } + if let Some(err) = last_err { let path = $path_and_query; log::error!("client error on request {path}, giving up - {err:?}"); -- 2.47.3