From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id C4FC77A044 for ; Thu, 28 Oct 2021 15:01:50 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id C301A20DCF for ; Thu, 28 Oct 2021 15:01:50 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id B259D20DC3 for ; Thu, 28 Oct 2021 15:01:49 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8B57045FDF for ; Thu, 28 Oct 2021 15:01:49 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Thu, 28 Oct 2021 15:00:55 +0200 Message-Id: <20211028130058.1308810-10-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20211028130058.1308810-1-f.gruenbichler@proxmox.com> References: <20211028130058.1308810-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.329 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox-backup-manager.rs, sync.rs, remote.rs] Subject: [pbs-devel] [PATCH v3 proxmox-backup 08/11] remote: add backup group scanning X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Oct 2021 13:01:50 -0000 Signed-off-by: Fabian Grünbichler --- src/api2/config/remote.rs | 73 ++++++++++++++++- src/bin/proxmox-backup-manager.rs | 105 ++++++++++++++++++++++--- src/bin/proxmox_backup_manager/sync.rs | 2 + 3 files changed, 166 insertions(+), 14 deletions(-) diff --git a/src/api2/config/remote.rs b/src/api2/config/remote.rs index 4dffe6bb..8077b610 100644 --- a/src/api2/config/remote.rs +++ b/src/api2/config/remote.rs @@ -1,4 +1,7 @@ use anyhow::{bail, format_err, Error}; +use proxmox::sortable; +use proxmox_router::SubdirMap; +use proxmox_router::list_subdirs_api_method; use serde_json::Value; use ::serde::{Deserialize, Serialize}; @@ -8,8 +11,8 @@ use proxmox_schema::api; use pbs_client::{HttpClient, HttpClientOptions}; use pbs_api_types::{ REMOTE_ID_SCHEMA, REMOTE_PASSWORD_SCHEMA, Remote, RemoteConfig, RemoteConfigUpdater, - Authid, PROXMOX_CONFIG_DIGEST_SCHEMA, DataStoreListItem, SyncJobConfig, - PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY, + Authid, PROXMOX_CONFIG_DIGEST_SCHEMA, DATASTORE_SCHEMA, GroupListItem, + DataStoreListItem, SyncJobConfig, PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY, }; use pbs_config::sync; @@ -340,8 +343,72 @@ pub async fn scan_remote_datastores(name: String) -> Result Result, Error> { + let (remote_config, _digest) = pbs_config::remote::config()?; + let remote: Remote = remote_config.lookup("remote", &name)?; + + let map_remote_err = |api_err| { + http_err!(INTERNAL_SERVER_ERROR, + "failed to scan remote '{}' - {}", + &name, + api_err) + }; + + let client = remote_client(&remote) + .await + .map_err(map_remote_err)?; + let api_res = client + .get(&format!("api2/json/admin/datastore/{}/groups", store), None) + .await + .map_err(map_remote_err)?; + let parse_res = match api_res.get("data") { + Some(data) => serde_json::from_value::>(data.to_owned()), + None => bail!("remote {} did not return any group list data", &name), + }; + + match parse_res { + Ok(parsed) => Ok(parsed), + Err(_) => bail!("Failed to parse remote scan api result."), + } +} + +#[sortable] +const DATASTORE_SCAN_SUBDIRS: SubdirMap = &[ + ( + "groups", + &Router::new() + .get(&API_METHOD_SCAN_REMOTE_GROUPS) + ), +]; + +const DATASTORE_SCAN_ROUTER: Router = Router::new() + .get(&list_subdirs_api_method!(DATASTORE_SCAN_SUBDIRS)) + .subdirs(DATASTORE_SCAN_SUBDIRS); + const SCAN_ROUTER: Router = Router::new() - .get(&API_METHOD_SCAN_REMOTE_DATASTORES); + .get(&API_METHOD_SCAN_REMOTE_DATASTORES) + .match_all("store", &DATASTORE_SCAN_ROUTER); const ITEM_ROUTER: Router = Router::new() .get(&API_METHOD_READ_REMOTE) diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs index 9e52a474..637c04ad 100644 --- a/src/bin/proxmox-backup-manager.rs +++ b/src/bin/proxmox-backup-manager.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::io::{self, Write}; -use anyhow::{format_err, Error}; +use anyhow::Error; use serde_json::{json, Value}; use proxmox::tools::fs::CreateOptions; @@ -9,10 +9,11 @@ use proxmox_router::{cli::*, RpcEnvironment}; use proxmox_schema::api; use pbs_client::{display_task_log, view_task_result}; +use pbs_config::sync; use pbs_tools::percent_encoding::percent_encode_component; use pbs_tools::json::required_string_param; use pbs_api_types::{ - GroupFilter, + GroupFilter, SyncJobConfig, DATASTORE_SCHEMA, GROUP_FILTER_LIST_SCHEMA, IGNORE_VERIFIED_BACKUPS_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA, UPID_SCHEMA, VERIFICATION_OUTDATED_AFTER_SCHEMA, }; @@ -398,6 +399,7 @@ async fn run() -> Result<(), Error> { .completion_cb("local-store", pbs_config::datastore::complete_datastore_name) .completion_cb("remote", pbs_config::remote::complete_remote_name) .completion_cb("remote-store", complete_remote_datastore_name) + .completion_cb("groups", complete_remote_datastore_group_filter) ) .insert( "verify", @@ -440,24 +442,105 @@ fn main() -> Result<(), Error> { pbs_runtime::main(run()) } +fn get_sync_job(id: &String) -> Result { + let (config, _digest) = sync::config()?; + + config.lookup("sync", id) +} + +fn get_remote(param: &HashMap) -> Option { + param + .get("remote") + .map(|r| r.to_owned()) + .or_else(|| { + if let Some(id) = param.get("id") { + if let Ok(job) = get_sync_job(id) { + return Some(job.remote.clone()); + } + } + None + }) +} + +fn get_remote_store(param: &HashMap) -> Option<(String, String)> { + let mut job: Option = None; + + let remote = param + .get("remote") + .map(|r| r.to_owned()) + .or_else(|| { + if let Some(id) = param.get("id") { + job = get_sync_job(id).ok(); + if let Some(ref job) = job { + return Some(job.remote.clone()); + } + } + None + }); + + if let Some(remote) = remote { + let store = param + .get("remote-store") + .map(|r| r.to_owned()) + .or_else(|| job.map(|job| job.remote_store.clone())); + + if let Some(store) = store { + return Some((remote, store)) + } + } + + None +} + // shell completion helper pub fn complete_remote_datastore_name(_arg: &str, param: &HashMap) -> Vec { let mut list = Vec::new(); - let _ = proxmox_lang::try_block!({ - let remote = param.get("remote").ok_or_else(|| format_err!("no remote"))?; + if let Some(remote) = get_remote(param) { + if let Ok(data) = pbs_runtime::block_on(async move { + crate::api2::config::remote::scan_remote_datastores(remote).await + }) { - let data = pbs_runtime::block_on(async move { - crate::api2::config::remote::scan_remote_datastores(remote.clone()).await - })?; + for item in data { + list.push(item.store); + } + } + } - for item in data { - list.push(item.store); + list +} + +// shell completion helper +pub fn complete_remote_datastore_group(_arg: &str, param: &HashMap) -> Vec { + + let mut list = Vec::new(); + + if let Some((remote, remote_store)) = get_remote_store(param) { + if let Ok(data) = pbs_runtime::block_on(async move { + crate::api2::config::remote::scan_remote_groups(remote.clone(), remote_store.clone()).await + }) { + + for item in data { + list.push(format!("{}/{}", item.backup_type, item.backup_id)); + } } + } + + list +} + +// shell completion helper +pub fn complete_remote_datastore_group_filter(_arg: &str, param: &HashMap) -> Vec { + + let mut list = Vec::new(); + + list.push("regex:".to_string()); + list.push("type:ct".to_string()); + list.push("type:host".to_string()); + list.push("type:vm".to_string()); - Ok(()) - }).map_err(|_err: Error| { /* ignore */ }); + list.extend(complete_remote_datastore_group(_arg, param).iter().map(|group| format!("group:{}", group))); list } diff --git a/src/bin/proxmox_backup_manager/sync.rs b/src/bin/proxmox_backup_manager/sync.rs index dfd8688d..8bf490ea 100644 --- a/src/bin/proxmox_backup_manager/sync.rs +++ b/src/bin/proxmox_backup_manager/sync.rs @@ -89,6 +89,7 @@ pub fn sync_job_commands() -> CommandLineInterface { .completion_cb("store", pbs_config::datastore::complete_datastore_name) .completion_cb("remote", pbs_config::remote::complete_remote_name) .completion_cb("remote-store", crate::complete_remote_datastore_name) + .completion_cb("groups", crate::complete_remote_datastore_group_filter) ) .insert("update", CliCommand::new(&api2::config::sync::API_METHOD_UPDATE_SYNC_JOB) @@ -97,6 +98,7 @@ pub fn sync_job_commands() -> CommandLineInterface { .completion_cb("schedule", pbs_config::datastore::complete_calendar_event) .completion_cb("store", pbs_config::datastore::complete_datastore_name) .completion_cb("remote-store", crate::complete_remote_datastore_name) + .completion_cb("groups", crate::complete_remote_datastore_group_filter) ) .insert("remove", CliCommand::new(&api2::config::sync::API_METHOD_DELETE_SYNC_JOB) -- 2.30.2