[pdm-devel] [PATCH datacenter-manager v2 3/3] api/ui: pve: get cluster resources from cache if remote is unreachable

[Dominik Csapak <d.csapak@proxmox.com>]

so that the last cached data can be shown instead of an error.
This now means that we'll always use the full api call here and filter
on the PDM side to get a fresh version in the cache.

To show a notice in the ui that it was loaded from the cache, we need to
add a marker property in the result, and need to extract that in the
pdm-client so the gui has access to that.

Also fix the CLI client to properly show the data.

Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
---
changes from v1:
* update cli client too
* use the new ApiCache mechanism for updated values
* use proper api type for conveying the cache result
* use the returned error in the ui if it exists

 cli/client/src/pve.rs             |  4 +-
 lib/pdm-api-types/src/resource.rs | 24 ++++++++++++
 lib/pdm-client/src/lib.rs         |  4 +-
 server/src/api/pve/mod.rs         | 63 +++++++++++++++++++++----------
 server/src/api/resources.rs       | 15 +++++++-
 ui/src/pve/mod.rs                 | 27 ++++++++++---
 6 files changed, 106 insertions(+), 31 deletions(-)

diff --git a/cli/client/src/pve.rs b/cli/client/src/pve.rs
index 1b124036..520119e0 100644
--- a/cli/client/src/pve.rs
+++ b/cli/client/src/pve.rs
@@ -230,14 +230,14 @@ async fn cluster_resources(
 ) -> Result<(), Error> {
     const CLUSTER_LIST_SCHEMA: Schema = ArraySchema::new(
         "cluster resources",
-        &pve_api_types::ClusterResource::API_SCHEMA,
+        &pdm_api_types::resource::PveResource::API_SCHEMA,
     )
     .schema();
 
     let data = client()?.pve_cluster_resources(&remote, kind).await?;
 
     format_and_print_result_full(
-        &mut serde_json::to_value(data)?,
+        &mut serde_json::to_value(data.list)?,
         &ReturnType {
             optional: false,
             schema: &CLUSTER_LIST_SCHEMA,
diff --git a/lib/pdm-api-types/src/resource.rs b/lib/pdm-api-types/src/resource.rs
index b2192505..0118eb05 100644
--- a/lib/pdm-api-types/src/resource.rs
+++ b/lib/pdm-api-types/src/resource.rs
@@ -642,3 +642,27 @@ pub struct TopEntities {
     /// The top entries for Node Memory
     pub node_memory: Vec<TopEntity>,
 }
+
+#[api(
+    properties: {
+        list: {
+            type: Array,
+            items: {
+                type: PveResource,
+            },
+        },
+    },
+)]
+#[derive(Serialize, Deserialize, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// List of [`PveResource`] with additional information about the cache
+pub struct PveResourceList {
+    /// The list of resources
+    pub list: Vec<PveResource>,
+    /// If the data came from the cache or not
+    pub cache: bool,
+    /// The timestamp of updating or the cache (if it was retrieved from there)
+    pub cached_timestamp: i64,
+    /// Any error from updating
+    pub error: Option<String>,
+}
diff --git a/lib/pdm-client/src/lib.rs b/lib/pdm-client/src/lib.rs
index 2f36fab1..ddf41436 100644
--- a/lib/pdm-client/src/lib.rs
+++ b/lib/pdm-client/src/lib.rs
@@ -4,7 +4,7 @@ use std::collections::HashMap;
 use std::time::Duration;
 
 use pdm_api_types::remotes::{RemoteType, TlsProbeOutcome};
-use pdm_api_types::resource::{PveResource, RemoteResources, ResourceType, TopEntities};
+use pdm_api_types::resource::{PveResourceList, RemoteResources, ResourceType, TopEntities};
 use pdm_api_types::rrddata::{
     LxcDataPoint, NodeDataPoint, PbsDatastoreDataPoint, PbsNodeDataPoint, PveStorageDataPoint,
     QemuDataPoint,
@@ -408,7 +408,7 @@ impl<T: HttpApiClient> PdmClient<T> {
         &self,
         remote: &str,
         kind: Option<pve_api_types::ClusterResourceKind>,
-    ) -> Result<Vec<PveResource>, Error> {
+    ) -> Result<PveResourceList, Error> {
         let query = ApiPathBuilder::new(format!("/api2/extjs/pve/remotes/{remote}/resources"))
             .maybe_arg("kind", &kind)
             .build();
diff --git a/server/src/api/pve/mod.rs b/server/src/api/pve/mod.rs
index 0de82323..4051100a 100644
--- a/server/src/api/pve/mod.rs
+++ b/server/src/api/pve/mod.rs
@@ -16,19 +16,19 @@ use proxmox_sortable_macro::sortable;
 use pdm_api_types::remotes::{
     NodeUrl, Remote, RemoteListEntry, RemoteType, TlsProbeOutcome, REMOTE_ID_SCHEMA,
 };
-use pdm_api_types::resource::PveResource;
+use pdm_api_types::resource::{PveResource, PveResourceList, Resource};
 use pdm_api_types::{
     Authid, RemoteUpid, HOST_OPTIONAL_PORT_FORMAT, PRIV_RESOURCE_AUDIT, PRIV_RESOURCE_DELETE,
     PRIV_SYS_MODIFY,
 };
 
 use pve_api_types::ClusterNodeStatus;
+use pve_api_types::ClusterResourceKind;
 use pve_api_types::ListRealm;
 use pve_api_types::PveUpid;
-use pve_api_types::{ClusterResourceKind, ClusterResourceType};
-
-use super::resources::{map_pve_lxc, map_pve_node, map_pve_qemu, map_pve_storage};
 
+use crate::api::resources::get_updated_resources_or_cache;
+use crate::api_cache::CacheUpdateResult;
 use crate::connection::PveClient;
 use crate::connection::{self, probe_tls_connection};
 use crate::remote_tasks;
@@ -175,9 +175,7 @@ pub async fn list_nodes(
         },
     },
     returns: {
-        type: Array,
-        description: "List all the resources in a PVE cluster.",
-        items: { type: PveResource },
+        type: PveResourceList,
     },
     access: {
         permission: &Permission::Privilege(&["resource", "{remote}"], PRIV_RESOURCE_AUDIT, false),
@@ -191,7 +189,7 @@ pub async fn cluster_resources(
     remote: String,
     kind: Option<ClusterResourceKind>,
     rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Vec<PveResource>, Error> {
+) -> Result<PveResourceList, Error> {
     let (remotes, _) = pdm_config::remotes::config()?;
     let user_info = CachedUserInfo::new()?;
     let auth_id: Authid = rpcenv
@@ -202,13 +200,37 @@ pub async fn cluster_resources(
         http_bail!(UNAUTHORIZED, "user has no access to resource list");
     }
 
-    let cluster_resources = connect_to_remote(&remotes, &remote)?
-        .cluster_resources(kind)
-        .await?
+    let remote = get_remote(&remotes, &remote)?;
+
+    let CacheUpdateResult {
+        data,
+        cache,
+        cached_timestamp,
+        error,
+    } = get_updated_resources_or_cache(remote).await?;
+
+    let cluster_resources = data
         .into_iter()
-        .filter_map(|r| map_pve_resource(&remote, r));
+        .filter(|res| match kind {
+            Some(kind) => matches!(
+                (kind, res),
+                (ClusterResourceKind::Vm, Resource::PveQemu(_))
+                    | (ClusterResourceKind::Vm, Resource::PveLxc(_))
+                    | (ClusterResourceKind::Storage, Resource::PveStorage(_))
+                    | (ClusterResourceKind::Node, Resource::PveNode(_))
+                    | (ClusterResourceKind::Sdn, Resource::PveSdn(_))
+            ),
+            None => true,
+        })
+        .filter_map(map_pve_resource)
+        .collect();
 
-    Ok(cluster_resources.collect())
+    Ok(PveResourceList {
+        list: cluster_resources,
+        cache,
+        cached_timestamp,
+        error: error.map(|err| err.to_string()),
+    })
 }
 
 #[api(
@@ -245,13 +267,14 @@ pub async fn cluster_status(
     Ok(status)
 }
 
-fn map_pve_resource(remote: &str, resource: pve_api_types::ClusterResource) -> Option<PveResource> {
-    match resource.ty {
-        ClusterResourceType::Node => map_pve_node(remote, resource).map(PveResource::Node),
-        ClusterResourceType::Lxc => map_pve_lxc(remote, resource).map(PveResource::Lxc),
-        ClusterResourceType::Qemu => map_pve_qemu(remote, resource).map(PveResource::Qemu),
-        ClusterResourceType::Storage => map_pve_storage(remote, resource).map(PveResource::Storage),
-        _ => None,
+fn map_pve_resource(resource: Resource) -> Option<PveResource> {
+    match resource {
+        Resource::PveStorage(storage) => Some(PveResource::Storage(storage)),
+        Resource::PveQemu(qemu) => Some(PveResource::Qemu(qemu)),
+        Resource::PveLxc(lxc) => Some(PveResource::Lxc(lxc)),
+        Resource::PveNode(node) => Some(PveResource::Node(node)),
+        Resource::PveSdn(sdn) => Some(PveResource::Sdn(sdn)),
+        Resource::PbsNode(_) | Resource::PbsDatastore(_) => None,
     }
 }
 
diff --git a/server/src/api/resources.rs b/server/src/api/resources.rs
index 0df9c781..f81dfedc 100644
--- a/server/src/api/resources.rs
+++ b/server/src/api/resources.rs
@@ -28,7 +28,7 @@ use proxmox_sortable_macro::sortable;
 use proxmox_subscription::SubscriptionStatus;
 use pve_api_types::{ClusterResource, ClusterResourceType};
 
-use crate::api_cache::ApiCache;
+use crate::api_cache::{ApiCache, CacheUpdateResult};
 use crate::connection;
 use crate::metric_collection::top_entities;
 
@@ -647,6 +647,19 @@ async fn get_resources_for_remote(remote: &Remote, max_age: u64) -> Result<Vec<R
     Ok(data)
 }
 
+/// Tries to update the resources and returns data from the cache if it was not
+/// possible to reach the remote.
+pub async fn get_updated_resources_or_cache(
+    remote: &Remote,
+) -> Result<CacheUpdateResult<Vec<Resource>>, Error> {
+    let remote_name = &remote.id;
+    CACHE
+        .update_or_get_cache(remote_name, |_| async move {
+            fetch_remote_resource(remote).await
+        })
+        .await
+}
+
 /// Read cached resource data from the cache
 pub fn get_cached_resources(remote: &str, max_age: u64) -> Option<Vec<Resource>> {
     let (data, _) = CACHE.get_value(remote, max_age as i64)?;
diff --git a/ui/src/pve/mod.rs b/ui/src/pve/mod.rs
index 058424a9..8ea85da7 100644
--- a/ui/src/pve/mod.rs
+++ b/ui/src/pve/mod.rs
@@ -11,7 +11,7 @@ use yew::{
 use pwt::{
     css::AlignItems,
     state::NavigationContainer,
-    widget::{Button, Container, Fa},
+    widget::{error_message, Button, Container, Fa},
 };
 use pwt::{
     css::FlexFit,
@@ -20,7 +20,7 @@ use pwt::{
     widget::{Column, Panel, Row},
 };
 
-use pdm_api_types::resource::{PveResource, ResourceType};
+use pdm_api_types::resource::{PveResource, PveResourceList, ResourceType};
 
 pub mod lxc;
 pub mod node;
@@ -122,13 +122,15 @@ impl GuestInfo {
 
 pub enum Msg {
     SelectedView(tree::PveTreeNode),
-    ResourcesList(Result<Vec<PveResource>, Error>),
+    ResourcesList(Result<PveResourceList, Error>),
 }
 
 pub struct PveRemoteComp {
     view: tree::PveTreeNode,
     resources: Rc<Vec<PveResource>>,
     last_error: Option<String>,
+    from_cache: bool,
+    update_error: Option<String>,
 }
 
 impl LoadableComponent for PveRemoteComp {
@@ -142,6 +144,8 @@ impl LoadableComponent for PveRemoteComp {
             view: PveTreeNode::Root,
             resources: Rc::new(Vec::new()),
             last_error: None,
+            from_cache: false,
+            update_error: None,
         }
     }
 
@@ -151,9 +155,11 @@ impl LoadableComponent for PveRemoteComp {
                 self.view = node;
             }
             Msg::ResourcesList(res) => match res {
-                Ok(res) => {
+                Ok(resource_list) => {
                     self.last_error = None;
-                    self.resources = Rc::new(res);
+                    self.from_cache = resource_list.cache;
+                    self.resources = Rc::new(resource_list.list);
+                    self.update_error = resource_list.error;
                 }
                 Err(err) => {
                     self.last_error = Some(err.to_string());
@@ -245,7 +251,16 @@ impl LoadableComponent for PveRemoteComp {
                                     let link = link.clone();
                                     move |_| link.send_reload()
                                 },
-                            )),
+                            ))
+                            .with_optional_child(self.from_cache.then_some({
+                                let mut error_text =
+                                    tr!("Could not reach remote, data shown from cache.");
+                                if let Some(err) = self.update_error.as_ref() {
+                                    error_text.push(' ');
+                                    error_text.push_str(&tr!("Got error: {0}", err));
+                                }
+                                error_message(&error_text)
+                            })),
                     ),
             )
             .with_child(
-- 
2.47.3



_______________________________________________
pdm-devel mailing list
pdm-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel