From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 16B741FF183 for ; Wed, 8 Oct 2025 15:55:57 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 41ED0A1D7; Wed, 8 Oct 2025 15:56:02 +0200 (CEST) From: Dominik Csapak To: pdm-devel@lists.proxmox.com Date: Wed, 8 Oct 2025 15:54:04 +0200 Message-ID: <20251008135558.3586369-2-d.csapak@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20251008135558.3586369-1-d.csapak@proxmox.com> References: <20251008135558.3586369-1-d.csapak@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.028 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pdm-devel] [RFC PATCH datacenter-manager 1/2] server: add new streaming 'content' api call for pbs X-BeenThere: pdm-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Datacenter Manager development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Datacenter Manager development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pdm-devel-bounces@lists.proxmox.com Sender: "pdm-devel" that makes use of 'application/json-seq' mime type to stream data from pbs to pdm, removing the need to buffer the data on pdm in the api call. The pbs-client code is more or less a copy of the current snapshot call list, but if needed this method can be refactored. This requires the new 'content' api call for pbs [0] 0: https://lore.proxmox.com/pbs-devel/20251008134344.3512958-1-d.csapak@proxmox.com/ Signed-off-by: Dominik Csapak --- server/src/api/pbs/mod.rs | 40 +++++++++++++++++++++++++++++- server/src/pbs_client.rs | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/server/src/api/pbs/mod.rs b/server/src/api/pbs/mod.rs index dc31f620..6e53ce6f 100644 --- a/server/src/api/pbs/mod.rs +++ b/server/src/api/pbs/mod.rs @@ -1,7 +1,7 @@ use anyhow::{format_err, Error}; use futures::StreamExt; -use proxmox_router::{list_subdirs_api_method, Permission, Router, SubdirMap}; +use proxmox_router::{list_subdirs_api_method, Permission, Record, Router, SubdirMap}; use proxmox_schema::api; use proxmox_schema::property_string::PropertyString; use proxmox_sortable_macro::sortable; @@ -63,6 +63,7 @@ const DATASTORE_ITEM_SUBDIRS: SubdirMap = &sorted!([ "snapshots", &Router::new().get(&API_METHOD_LIST_SNAPSHOTS_2) ), + ("content", &Router::new().get(&API_METHOD_LIST_CONTENT)), ]); #[api( @@ -175,6 +176,43 @@ async fn list_snapshots_2( .into()) } +#[api( + stream: true, + input: { + properties: { + remote: { schema: REMOTE_ID_SCHEMA }, + datastore: { schema: pbs_api_types::DATASTORE_SCHEMA }, + ns: { + schema: pbs_api_types::BACKUP_NAMESPACE_SCHEMA, + optional: true, + }, + "max-depth": { + schema: pbs_api_types::NS_MAX_DEPTH_SCHEMA, + optional: true, + } + }, + }, + returns: pbs_api_types::ADMIN_DATASTORE_LIST_CONTENT_RETURN_TYPE, + access: { + permission: &Permission::Privilege(&["resource", "{remote}", "datastore", "{datastore}"], PRIV_RESOURCE_AUDIT, false), + }, +)] +/// List the PBS remote's datastores. +async fn list_content( + remote: String, + datastore: String, + ns: Option, + max_depth: Option, +) -> Result { + let (remotes, _) = pdm_config::remotes::config()?; + let remote = get_remote(&remotes, &remote)?; + let snapshots = connection::make_pbs_client(remote)? + .list_content(&datastore, ns.as_deref(), max_depth) + .await? + .map(Record::from_result); + Ok(snapshots.into()) +} + #[api( input: { properties: { diff --git a/server/src/pbs_client.rs b/server/src/pbs_client.rs index d8278c8a..76041090 100644 --- a/server/src/pbs_client.rs +++ b/server/src/pbs_client.rs @@ -190,6 +190,57 @@ impl PbsClient { } } + /// List a datastore's content. + pub async fn list_content( + &self, + datastore: &str, + namespace: Option<&str>, + max_depth: Option, + ) -> Result, anyhow::Error> { + let path = ApiPathBuilder::new(format!("/api2/json/admin/datastore/{datastore}/content")) + .maybe_arg("ns", &namespace) + .maybe_arg("max-depth", &max_depth) + .build(); + let response = self + .0 + .streaming_request(http::Method::GET, &path, None::<()>) + .await?; + + let body = response + .body + .ok_or_else(|| Error::Other("missing response body"))?; + + if response.status == 200 { + if response + .content_type + .is_some_and(|c| c.starts_with("application/json-seq")) + { + Ok(JsonRecords::from_body(body)) + } else { + let response: JsonData<_> = serde_json::from_slice( + &body + .collect() + .await + .map_err(|err| { + Error::Anyhow(Box::new(err).context("failed to retrieve response body")) + })? + .to_bytes(), + )?; + Ok(JsonRecords::from_vec(response.data)) + } + } else { + let data = body + .collect() + .await + .map_err(|err| { + Error::Anyhow(Box::new(err).context("failed to retrieve response body")) + })? + .to_bytes(); + let error = String::from_utf8_lossy(&data).into_owned(); + Err(anyhow::Error::msg(error)) + } + } + /// create an API-Token on the PBS remote and give the token admin ACL on everything. pub async fn create_admin_token( &self, -- 2.47.3 _______________________________________________ pdm-devel mailing list pdm-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel