From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 7817365AD3 for ; Wed, 4 Nov 2020 17:57:56 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6EB8CF8AA for ; Wed, 4 Nov 2020 17:57:56 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 142CAF89E for ; Wed, 4 Nov 2020 17:57:52 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id D9D8546027 for ; Wed, 4 Nov 2020 17:57:51 +0100 (CET) To: Proxmox Backup Server development discussion , =?UTF-8?Q?Fabian_Gr=c3=bcnbichler?= References: <20201104131026.4017010-1-f.gruenbichler@proxmox.com> <20201104131026.4017010-3-f.gruenbichler@proxmox.com> From: Thomas Lamprecht Message-ID: <81720327-15a1-c04f-cec4-f993cc21a7f5@proxmox.com> Date: Wed, 4 Nov 2020 17:57:49 +0100 User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:83.0) Gecko/20100101 Thunderbird/83.0 MIME-Version: 1.0 In-Reply-To: <20201104131026.4017010-3-f.gruenbichler@proxmox.com> Content-Type: text/plain; charset=UTF-8 Content-Language: en-US Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL -0.118 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment NICE_REPLY_A -0.001 Looks like a legit reply (A) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [pull.rs, proxmox-backup-manager.rs, remote.rs] Subject: Re: [pbs-devel] [PATCH proxmox-backup 3/4] api: refactor remote client and add remote scan X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Nov 2020 16:57:56 -0000 On 04.11.20 14:10, Fabian Gr=C3=BCnbichler wrote: > to allow on-demand scanning of remote datastores accessible for the > configured remote user. >=20 > Signed-off-by: Fabian Gr=C3=BCnbichler > --- >=20 > Notes: > not 100% sure about PRIV_REMOTE_AUDIT vs PRIV_REMOTE_READ.. the lat= ter is required to use a datastore for syncing/pull purposes you are not syncing here, so why should the permissions required for that matter, when getting a general list of datastores of a remote? If, that would be an extra filter param to set. I setup a remote with a token, got -> GET /api2/json/config/remote/tuxis/scan: 401 Unauthorized: [client [::fff= f:192.168.16.38]:47544] authentication failed - invalid user name in user= id >=20 > src/api2/config/remote.rs | 66 ++++++++++++++++++++++++++++++-= > src/api2/pull.rs | 12 +----- > src/bin/proxmox-backup-manager.rs | 26 +++--------- > 3 files changed, 71 insertions(+), 33 deletions(-) >=20 > diff --git a/src/api2/config/remote.rs b/src/api2/config/remote.rs > index ffbba1d2..b415f63d 100644 > --- a/src/api2/config/remote.rs > +++ b/src/api2/config/remote.rs > @@ -1,4 +1,4 @@ > -use anyhow::{bail, Error}; > +use anyhow::{bail, format_err, Error}; > use serde_json::Value; > use ::serde::{Deserialize, Serialize}; > =20 > @@ -6,6 +6,7 @@ use proxmox::api::{api, ApiMethod, Router, RpcEnvironme= nt, Permission}; > use proxmox::tools::fs::open_file_locked; > =20 > use crate::api2::types::*; > +use crate::client::{HttpClient, HttpClientOptions}; > use crate::config::cached_user_info::CachedUserInfo; > use crate::config::remote; > use crate::config::acl::{PRIV_REMOTE_AUDIT, PRIV_REMOTE_MODIFY}; > @@ -301,10 +302,71 @@ pub fn delete_remote(name: String, digest: Option= ) -> Result<(), Error> > Ok(()) > } > =20 > +/// Helper to get client for remote.cfg entry > +pub async fn remote_client(remote: remote::Remote) -> Result { > + let options =3D HttpClientOptions::new() > + .password(Some(remote.password.clone())) > + .fingerprint(remote.fingerprint.clone()); > + > + let client =3D HttpClient::new( > + &remote.host, > + remote.port.unwrap_or(8007), > + &remote.userid, sure about userid, shouldn't this be authid or is that the same here? At least would explain the error I get.. > + options)?; > + let _auth_info =3D client.login() // make sure we can auth > + .await > + .map_err(|err| format_err!("remote connection to '{}' failed -= {}", remote.host, err))?; > + > + Ok(client) > +} > + > + > +#[api( > + input: { > + properties: { > + name: { > + schema: REMOTE_ID_SCHEMA, > + }, > + }, > + }, > + access: { > + permission: &Permission::Privilege(&["remote", "{name}"], PRIV= _REMOTE_AUDIT, false), > + }, > + returns: { > + description: "List the accessible datastores.", > + type: Array, > + items: { > + description: "Datastore name and description.", > + type: DataStoreListItem, > + }, > + }, > +)] > +/// List datastores of a remote.cfg entry > +pub async fn scan_remote_datastores(name: String) -> Result, Error> { > + let (remote_config, _digest) =3D remote::config()?; > + let remote: remote::Remote =3D remote_config.lookup("remote", &nam= e)?; > + > + let client =3D remote_client(remote).await?; > + let api_res =3D client.get("api2/json/admin/datastore", None).awai= t?; > + let parse_res =3D match api_res.get("data") { > + Some(data) =3D> serde_json::from_value::>(data.to_owned()), > + None =3D> bail!("remote {} did not return any datastore list d= ata", &name), > + }; > + > + match parse_res { > + Ok(parsed) =3D> Ok(parsed), > + Err(_) =3D> bail!("Failed to parse remote scan api result."), > + } > +} > + > +const SCAN_ROUTER: Router =3D Router::new() > + .get(&API_METHOD_SCAN_REMOTE_DATASTORES); > + > const ITEM_ROUTER: Router =3D Router::new() > .get(&API_METHOD_READ_REMOTE) > .put(&API_METHOD_UPDATE_REMOTE) > - .delete(&API_METHOD_DELETE_REMOTE); > + .delete(&API_METHOD_DELETE_REMOTE) > + .subdirs(&[("scan", &SCAN_ROUTER)]); > =20 > pub const ROUTER: Router =3D Router::new() > .get(&API_METHOD_LIST_REMOTES) > diff --git a/src/api2/pull.rs b/src/api2/pull.rs > index d9e9d31d..87015c72 100644 > --- a/src/api2/pull.rs > +++ b/src/api2/pull.rs > @@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, P= ermission}; > =20 > use crate::server::{WorkerTask, jobstate::Job}; > use crate::backup::DataStore; > -use crate::client::{HttpClient, HttpClientOptions, BackupRepository, p= ull::pull_store}; > +use crate::client::{HttpClient, BackupRepository, pull::pull_store}; > use crate::api2::types::*; > use crate::config::{ > remote, > @@ -50,17 +50,9 @@ pub async fn get_pull_parameters( > let (remote_config, _digest) =3D remote::config()?; > let remote: remote::Remote =3D remote_config.lookup("remote", remo= te)?; > =20 > - let options =3D HttpClientOptions::new() > - .password(Some(remote.password.clone())) > - .fingerprint(remote.fingerprint.clone()); > - > let src_repo =3D BackupRepository::new(Some(remote.userid.clone())= , Some(remote.host.clone()), remote.port, remote_store.to_string()); > =20 > - let client =3D HttpClient::new(&src_repo.host(), src_repo.port(), = &src_repo.auth_id(), options)?; > - let _auth_info =3D client.login() // make sure we can auth > - .await > - .map_err(|err| format_err!("remote connection to '{}' failed -= {}", remote.host, err))?; > - > + let client =3D crate::api2::config::remote::remote_client(remote).= await?; > =20 > Ok((client, src_repo, tgt_store)) > } > diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup= -manager.rs > index 7499446b..e52c2f76 100644 > --- a/src/bin/proxmox-backup-manager.rs > +++ b/src/bin/proxmox-backup-manager.rs > @@ -413,29 +413,13 @@ pub fn complete_remote_datastore_name(_arg: &str,= param: &HashMap =20 > let _ =3D proxmox::try_block!({ > let remote =3D param.get("remote").ok_or_else(|| format_err!("= no remote"))?; > - let (remote_config, _digest) =3D config::remote::config()?; > =20 > - let remote: config::remote::Remote =3D remote_config.lookup("r= emote", &remote)?; > + let data =3D crate::tools::runtime::block_on(async move { > + crate::api2::config::remote::scan_remote_datastores(remote= =2Eclone()).await > + })?; > =20 > - let options =3D HttpClientOptions::new() > - .password(Some(remote.password.clone())) > - .fingerprint(remote.fingerprint.clone()); > - > - let client =3D HttpClient::new( > - &remote.host, > - remote.port.unwrap_or(8007), > - &remote.userid, > - options, > - )?; > - > - let result =3D crate::tools::runtime::block_on(client.get("api= 2/json/admin/datastore", None))?; > - > - if let Some(data) =3D result["data"].as_array() { > - for item in data { > - if let Some(store) =3D item["store"].as_str() { > - list.push(store.to_owned()); > - } > - } > + for item in data { > + list.push(item.store); > } > =20 > Ok(()) >=20