From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 87BD51FF179 for ; Wed, 15 Oct 2025 14:47:37 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 05B971B260; Wed, 15 Oct 2025 14:47:57 +0200 (CEST) From: Lukas Wagner To: pdm-devel@lists.proxmox.com Date: Wed, 15 Oct 2025 14:47:02 +0200 Message-ID: <20251015124711.312943-4-l.wagner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20251015124711.312943-1-l.wagner@proxmox.com> References: <20251015124711.312943-1-l.wagner@proxmox.com> MIME-Version: 1.0 X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1760532436943 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.027 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pdm-devel] [PATCH proxmox-datacenter-manager 03/12] remote updates: add cache for remote update availability X-BeenThere: pdm-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Datacenter Manager development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Datacenter Manager development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pdm-devel-bounces@lists.proxmox.com Sender: "pdm-devel" The cache will be filled by either the `refresh_summary_cache` function, or when the list of updates for a single node is requested. The cache contains only the summary (number of updates) and not the full list of packages. We can always choose to add the latter later if we need it. Signed-off-by: Lukas Wagner --- server/src/remote_updates.rs | 192 +++++++++++++++++++++++++++++++++-- 1 file changed, 186 insertions(+), 6 deletions(-) diff --git a/server/src/remote_updates.rs b/server/src/remote_updates.rs index f833062c..ab5845bd 100644 --- a/server/src/remote_updates.rs +++ b/server/src/remote_updates.rs @@ -1,20 +1,53 @@ +use std::fs::File; +use std::io::ErrorKind; + use anyhow::{bail, Error}; -use pdm_api_types::RemoteUpid; +use serde::{Deserialize, Serialize}; use proxmox_apt_api_types::APTUpdateInfo; +use pdm_api_types::remote_updates::{ + NodeUpdateStatus, NodeUpdateSummary, NodeUpdateSummaryWrapper, RemoteUpdateStatus, + RemoteUpdateSummary, UpdateSummary, +}; use pdm_api_types::remotes::{Remote, RemoteType}; +use pdm_api_types::RemoteUpid; +use pdm_buildcfg::PDM_CACHE_DIR_M; use crate::api::pve::new_remote_upid; use crate::connection; +use crate::parallel_fetcher::{NodeResults, ParallelFetcher}; + +pub const UPDATE_CACHE: &str = concat!(PDM_CACHE_DIR_M!(), "/remote-updates.json"); + +#[derive(Clone, Default, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +struct NodeUpdateInfo { + updates: Vec, + last_refresh: i64, +} + +impl From for NodeUpdateSummary { + fn from(value: NodeUpdateInfo) -> Self { + Self { + number_of_updates: value.updates.len() as u32, + last_refresh: value.last_refresh, + status: NodeUpdateStatus::Success, + status_message: None, + } + } +} /// Return a list of available updates for a given remote node. pub async fn list_available_updates( remote: Remote, node: &str, ) -> Result, Error> { - let updates = fetch_available_updates(remote, node.to_string()).await?; - Ok(updates) + let updates = fetch_available_updates((), remote.clone(), node.to_string()).await?; + + update_cached_summary_for_node(remote, node.into(), updates.clone().into()).await?; + + Ok(updates.updates) } /// Trigger `apt update` on a remote node. @@ -52,10 +85,154 @@ pub async fn get_changelog(remote: Remote, node: &str, package: String) -> Resul } } -async fn fetch_available_updates( +/// Get update summary for all managed remotes. +pub fn get_available_updates_summary() -> Result { + let (config, _digest) = pdm_config::remotes::config()?; + + let cache_content = get_cached_summary_or_default()?; + + let mut summary = UpdateSummary::default(); + + for (remote_name, remote) in &config { + match cache_content.remotes.get(remote_name) { + Some(remote_summary) => { + summary + .remotes + .insert(remote_name.into(), remote_summary.clone()); + } + None => { + summary.remotes.insert( + remote_name.into(), + RemoteUpdateSummary { + nodes: NodeUpdateSummaryWrapper::default(), + remote_type: remote.ty, + status: RemoteUpdateStatus::Unknown, + }, + ); + } + } + } + + Ok(summary) +} + +fn get_cached_summary_or_default() -> Result { + match File::open(UPDATE_CACHE) { + Ok(file) => { + let content = match serde_json::from_reader(file) { + Ok(cache_content) => cache_content, + Err(err) => { + log::error!("failed to deserialize remote update cache: {err:#}"); + Default::default() + } + }; + + Ok(content) + } + Err(err) if err.kind() == ErrorKind::NotFound => Ok(Default::default()), + Err(err) => Err(err.into()), + } +} + +async fn update_cached_summary_for_node( remote: Remote, node: String, -) -> Result, Error> { + node_data: NodeUpdateSummary, +) -> Result<(), Error> { + let mut file = File::open(UPDATE_CACHE)?; + let mut cache_content: UpdateSummary = serde_json::from_reader(&mut file)?; + let remote_entry = + cache_content + .remotes + .entry(remote.id) + .or_insert_with(|| RemoteUpdateSummary { + nodes: Default::default(), + remote_type: remote.ty, + status: RemoteUpdateStatus::Success, + }); + + remote_entry.nodes.insert(node, node_data); + + let options = proxmox_product_config::default_create_options(); + proxmox_sys::fs::replace_file( + UPDATE_CACHE, + &serde_json::to_vec(&cache_content)?, + options, + true, + )?; + + Ok(()) +} + +/// Refresh the remote update cache. +pub async fn refresh_update_summary_cache(remotes: Vec) -> Result<(), Error> { + let fetcher = ParallelFetcher::new(()); + + let fetch_results = fetcher + .do_for_all_remote_nodes(remotes.clone().into_iter(), fetch_available_updates) + .await; + + let mut content = get_cached_summary_or_default()?; + + for (remote_name, result) in fetch_results.remote_results { + let entry = content + .remotes + .entry(remote_name.clone()) + .or_insert_with(|| { + // unwrap: remote name came from the same config, should be safe. + // TODO: Include type in ParallelFetcher results - should be much more efficient. + let remote_type = remotes.iter().find(|r| r.id == remote_name).unwrap().ty; + + RemoteUpdateSummary { + nodes: Default::default(), + remote_type, + status: RemoteUpdateStatus::Success, + } + }); + + match result { + Ok(remote_result) => { + for (node_name, node_result) in remote_result.node_results { + match node_result { + Ok(NodeResults { data, .. }) => { + entry.nodes.insert(node_name, data.into()); + } + Err(err) => { + // Could not fetch updates from node + entry.nodes.insert( + node_name.clone(), + NodeUpdateSummary { + number_of_updates: 0, + last_refresh: 0, + status: NodeUpdateStatus::Error, + status_message: Some(format!("{err:#}")), + }, + ); + log::error!( + "could not fetch available updates from node '{node_name}': {err}" + ); + } + } + } + } + Err(err) => { + entry.status = RemoteUpdateStatus::Error; + log::error!("could not fetch available updates from remote '{remote_name}': {err}"); + } + } + } + + let options = proxmox_product_config::default_create_options(); + proxmox_sys::fs::replace_file(UPDATE_CACHE, &serde_json::to_vec(&content)?, options, true)?; + + Ok(()) +} + +async fn fetch_available_updates( + _context: (), + remote: Remote, + node: String, +) -> Result { match remote.ty { RemoteType::Pve => { let client = connection::make_pve_client(&remote)?; @@ -67,7 +244,10 @@ async fn fetch_available_updates( .map(map_pve_update_info) .collect(); - Ok(updates) + Ok(NodeUpdateInfo { + last_refresh: proxmox_time::epoch_i64(), + updates, + }) } RemoteType::Pbs => bail!("PBS is not supported yet"), } -- 2.47.3 _______________________________________________ pdm-devel mailing list pdm-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel