[PATCH datacenter-manager 3/4] server: connection: multi-client: use back-off state from remote cache

[Dominik Csapak <d.csapak@proxmox.com>]

This greatly reduces the wait time for api calls if a specific remote is
offline for a prolonged period of time. Instead of waiting for a timeout
on every api call to a remote which is known to be offline, return the
last error immediately.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
 server/src/connection.rs | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/server/src/connection.rs b/server/src/connection.rs
index 28e4799f..7122713f 100644
--- a/server/src/connection.rs
+++ b/server/src/connection.rs
@@ -20,6 +20,7 @@ use serde::Serialize;
 
 use proxmox_acme_api::CertificateInfo;
 use proxmox_client::{Client, HttpApiClient, HttpApiResponse, HttpApiResponseStream, TlsOptions};
+use proxmox_time::epoch_i64;
 
 use pdm_api_types::remotes::{NodeUrl, Remote, RemoteType, TlsProbeOutcome};
 use pve_api_types::client::PveClientImpl;
@@ -718,6 +719,17 @@ macro_rules! try_request {
     ($self:expr, $method:expr, $path_and_query:expr, $params:expr, $how:ident) => {
         let params = $params.map(serde_json::to_value);
         Box::pin(async move {
+            // first check if the remote is reachable
+            {
+                let cache = crate::remote_cache::RemoteMappingCache::get();
+                let (back_off_time, error) =
+                    cache.remote_remaining_backoff_time(&$self.remote, epoch_i64());
+                let error = error.unwrap_or("unknown error".to_string());
+                if back_off_time > 0 {
+                    return Err(proxmox_client::Error::Connect(error.into()));
+                }
+            }
+
             let params = params
                 .transpose()
                 .map_err(|err| proxmox_client::Error::Anyhow(err.into()))?;
@@ -748,13 +760,15 @@ macro_rules! try_request {
                         last_err = Some(err);
                     }
                     Ok(result) => {
-                        if !reachable {
-                            log::error!("marking {hostname:?} as reachable again!");
-                            if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write()
-                            {
+                        if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write() {
+                            if !reachable {
                                 cache.mark_host_reachable(&$self.remote, &hostname, true);
-                                let _ = cache.save();
                             }
+                            cache.mark_remote_reachable(
+                                &$self.remote,
+                                crate::remote_cache::RemoteState::Reachable,
+                            );
+                            let _ = cache.save();
                         }
                         return result;
                     }
@@ -764,6 +778,18 @@ macro_rules! try_request {
                 }
             }
 
+            if let Ok(mut cache) = crate::remote_cache::RemoteMappingCache::write() {
+                let error = last_err
+                    .as_ref()
+                    .map(|err| format!("remote not reachable: {err:?}"))
+                    .unwrap_or("unknown error".to_string());
+                cache.mark_remote_reachable(
+                    &$self.remote,
+                    crate::remote_cache::RemoteState::Unreachable(error),
+                );
+                let _ = cache.save();
+            }
+
             if let Some(err) = last_err {
                 let path = $path_and_query;
                 log::error!("client error on request {path}, giving up - {err:?}");
-- 
2.47.3