From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 91D4265CB8 for ; Thu, 5 Nov 2020 08:42:43 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 7A6D714ED4 for ; Thu, 5 Nov 2020 08:42:13 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id B7FF614EC7 for ; Thu, 5 Nov 2020 08:42:09 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 81AA8459FB for ; Thu, 5 Nov 2020 08:42:09 +0100 (CET) Date: Thu, 05 Nov 2020 08:42:02 +0100 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion , Thomas Lamprecht References: <20201104131026.4017010-1-f.gruenbichler@proxmox.com> <20201104131026.4017010-3-f.gruenbichler@proxmox.com> <81720327-15a1-c04f-cec4-f993cc21a7f5@proxmox.com> In-Reply-To: <81720327-15a1-c04f-cec4-f993cc21a7f5@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.15.0 (https://github.com/astroidmail/astroid) Message-Id: <1604561929.s6xxo0fncs.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.025 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox-backup-manager.rs, remote.rs, pull.rs] Subject: Re: [pbs-devel] [PATCH proxmox-backup 3/4] api: refactor remote client and add remote scan X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 05 Nov 2020 07:42:43 -0000 On November 4, 2020 5:57 pm, Thomas Lamprecht wrote: > On 04.11.20 14:10, Fabian Gr=C3=BCnbichler wrote: >> to allow on-demand scanning of remote datastores accessible for the >> configured remote user. >>=20 >> Signed-off-by: Fabian Gr=C3=BCnbichler >> --- >>=20 >> Notes: >> not 100% sure about PRIV_REMOTE_AUDIT vs PRIV_REMOTE_READ.. the latt= er is required to use a datastore for syncing/pull purposes >=20 >=20 > you are not syncing here, so why should the permissions required for > that matter, when getting a general list of datastores of a remote? because the only thing that a remote datastore can currently be used for=20 is syncing ;) but I am fine with AUDIT as well, I just wanted to mention=20 it. > If, that would be an extra filter param to set. >=20 > I setup a remote with a token, got -> > GET /api2/json/config/remote/tuxis/scan: 401 Unauthorized: [client [::fff= f:192.168.16.38]:47544] authentication failed - invalid user name in user i= d I think (as we discussed directly) this was an artifact of version=20 mismatch? >=20 >>=20 >> src/api2/config/remote.rs | 66 ++++++++++++++++++++++++++++++- >> src/api2/pull.rs | 12 +----- >> src/bin/proxmox-backup-manager.rs | 26 +++--------- >> 3 files changed, 71 insertions(+), 33 deletions(-) >>=20 >> diff --git a/src/api2/config/remote.rs b/src/api2/config/remote.rs >> index ffbba1d2..b415f63d 100644 >> --- a/src/api2/config/remote.rs >> +++ b/src/api2/config/remote.rs >> @@ -1,4 +1,4 @@ >> -use anyhow::{bail, Error}; >> +use anyhow::{bail, format_err, Error}; >> use serde_json::Value; >> use ::serde::{Deserialize, Serialize}; >> =20 >> @@ -6,6 +6,7 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironmen= t, Permission}; >> use proxmox::tools::fs::open_file_locked; >> =20 >> use crate::api2::types::*; >> +use crate::client::{HttpClient, HttpClientOptions}; >> use crate::config::cached_user_info::CachedUserInfo; >> use crate::config::remote; >> use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY}; >> @@ -301,10 +302,71 @@ pub fn delete_remote(name: String, digest: Option<= String>) -> Result<(), Error> >> Ok(()) >> } >> =20 >> +/// Helper to get client for remote.cfg entry >> +pub async fn remote_client(remote: remote::Remote) -> Result { >> + let options =3D HttpClientOptions::new() >> + .password(Some(remote.password.clone())) >> + .fingerprint(remote.fingerprint.clone()); >> + >> + let client =3D HttpClient::new( >> + &remote.host, >> + remote.port.unwrap_or(8007), >> + &remote.userid, >=20 > sure about userid, shouldn't this be authid or is that the same here? > At least would explain the error I get.. the field in the config is called userid, it contains an Authid=20 (renaming would require postinst fixup, but if you want I can send a=20 patch for switching it over). >=20 >> + options)?; >> + let _auth_info =3D client.login() // make sure we can auth >> + .await >> + .map_err(|err| format_err!("remote connection to '{}' failed - = {}", remote.host, err))?; >> + >> + Ok(client) >> +} >> + >> + >> +#[api( >> + input: { >> + properties: { >> + name: { >> + schema: REMOTE_ID_SCHEMA, >> + }, >> + }, >> + }, >> + access: { >> + permission: &Permission::Privilege(&["remote", "{name}"], PRIV_= REMOTE_AUDIT, false), >> + }, >> + returns: { >> + description: "List the accessible datastores.", >> + type: Array, >> + items: { >> + description: "Datastore name and description.", >> + type: DataStoreListItem, >> + }, >> + }, >> +)] >> +/// List datastores of a remote.cfg entry >> +pub async fn scan_remote_datastores(name: String) -> Result, Error> { >> + let (remote_config, _digest) =3D remote::config()?; >> + let remote: remote::Remote =3D remote_config.lookup("remote", &name= )?; >> + >> + let client =3D remote_client(remote).await?; >> + let api_res =3D client.get("api2/json/admin/datastore", None).await= ?; >> + let parse_res =3D match api_res.get("data") { >> + Some(data) =3D> serde_json::from_value::= >(data.to_owned()), >> + None =3D> bail!("remote {} did not return any datastore list da= ta", &name), >> + }; >> + >> + match parse_res { >> + Ok(parsed) =3D> Ok(parsed), >> + Err(_) =3D> bail!("Failed to parse remote scan api result."), >> + } >> +} >> + >> +const SCAN_ROUTER: Router =3D Router::new() >> + .get(&API_METHOD_SCAN_REMOTE_DATASTORES); >> + >> const ITEM_ROUTER: Router =3D Router::new() >> .get(&API_METHOD_READ_REMOTE) >> .put(&API_METHOD_UPDATE_REMOTE) >> - .delete(&API_METHOD_DELETE_REMOTE); >> + .delete(&API_METHOD_DELETE_REMOTE) >> + .subdirs(&[("scan", &SCAN_ROUTER)]); >> =20 >> pub const ROUTER: Router =3D Router::new() >> .get(&API_METHOD_LIST_REMOTES) >> diff --git a/src/api2/pull.rs b/src/api2/pull.rs >> index d9e9d31d..87015c72 100644 >> --- a/src/api2/pull.rs >> +++ b/src/api2/pull.rs >> @@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Pe= rmission}; >> =20 >> use crate::server::{WorkerTask, jobstate::Job}; >> use crate::backup::DataStore; >> -use crate::client::{HttpClient, HttpClientOptions, BackupRepository, pu= ll::pull_store}; >> +use crate::client::{HttpClient, BackupRepository, pull::pull_store}; >> use crate::api2::types::*; >> use crate::config::{ >> remote, >> @@ -50,17 +50,9 @@ pub async fn get_pull_parameters( >> let (remote_config, _digest) =3D remote::config()?; >> let remote: remote::Remote =3D remote_config.lookup("remote", remot= e)?; >> =20 >> - let options =3D HttpClientOptions::new() >> - .password(Some(remote.password.clone())) >> - .fingerprint(remote.fingerprint.clone()); >> - >> let src_repo =3D BackupRepository::new(Some(remote.userid.clone()),= Some(remote.host.clone()), remote.port, remote_store.to_string()); >> =20 >> - let client =3D HttpClient::new(&src_repo.host(), src_repo.port(), &= src_repo.auth_id(), options)?; >> - let _auth_info =3D client.login() // make sure we can auth >> - .await >> - .map_err(|err| format_err!("remote connection to '{}' failed - = {}", remote.host, err))?; >> - >> + let client =3D crate::api2::config::remote::remote_client(remote).a= wait?; >> =20 >> Ok((client, src_repo, tgt_store)) >> } >> diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-= manager.rs >> index 7499446b..e52c2f76 100644 >> --- a/src/bin/proxmox-backup-manager.rs >> +++ b/src/bin/proxmox-backup-manager.rs >> @@ -413,29 +413,13 @@ pub fn complete_remote_datastore_name(_arg: &str, = param: &HashMap> =20 >> let _ =3D proxmox::try_block!({ >> let remote =3D param.get("remote").ok_or_else(|| format_err!("n= o remote"))?; >> - let (remote_config, _digest) =3D config::remote::config()?; >> =20 >> - let remote: config::remote::Remote =3D remote_config.lookup("re= mote", &remote)?; >> + let data =3D crate::tools::runtime::block_on(async move { >> + crate::api2::config::remote::scan_remote_datastores(remote.= clone()).await >> + })?; >> =20 >> - let options =3D HttpClientOptions::new() >> - .password(Some(remote.password.clone())) >> - .fingerprint(remote.fingerprint.clone()); >> - >> - let client =3D HttpClient::new( >> - &remote.host, >> - remote.port.unwrap_or(8007), >> - &remote.userid, >> - options, >> - )?; >> - >> - let result =3D crate::tools::runtime::block_on(client.get("api2= /json/admin/datastore", None))?; >> - >> - if let Some(data) =3D result["data"].as_array() { >> - for item in data { >> - if let Some(store) =3D item["store"].as_str() { >> - list.push(store.to_owned()); >> - } >> - } >> + for item in data { >> + list.push(item.store); >> } >> =20 >> Ok(()) >>=20 >=20 >=20 >=20 =