From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id C15D661692 for ; Wed, 21 Oct 2020 11:41:31 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id BF04D16875 for ; Wed, 21 Oct 2020 11:41:31 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id F167716838 for ; Wed, 21 Oct 2020 11:41:28 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id B14E345EAF for ; Wed, 21 Oct 2020 11:41:28 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Wed, 21 Oct 2020 11:41:11 +0200 Message-Id: <20201021094116.32501-3-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201021094116.32501-1-s.reiter@proxmox.com> References: <20201021094116.32501-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.032 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [tools.rs, http.rs] Subject: [pbs-devel] [PATCH proxmox-backup 2/7] add tools::http for generic HTTP GET and move HttpsConnector there X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 21 Oct 2020 09:41:31 -0000 ...to avoid having the tools:: module depend on api2. The get_string function is based directly on hyper and thus relatively simple, not supporting redirects for example. Signed-off-by: Stefan Reiter --- src/client/http_client.rs | 75 ++-------------------------- src/tools.rs | 1 + src/tools/http.rs | 100 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 72 deletions(-) create mode 100644 src/tools/http.rs diff --git a/src/client/http_client.rs b/src/client/http_client.rs index e92d4b18..b57630f8 100644 --- a/src/client/http_client.rs +++ b/src/client/http_client.rs @@ -1,8 +1,6 @@ use std::io::Write; -use std::task::{Context, Poll}; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; -use std::os::unix::io::AsRawFd; use anyhow::{bail, format_err, Error}; use futures::*; @@ -19,22 +17,16 @@ use xdg::BaseDirectories; use proxmox::{ api::error::HttpError, sys::linux::tty, - tools::{ - fs::{file_get_json, replace_file, CreateOptions}, - } + tools::fs::{file_get_json, replace_file, CreateOptions}, }; use super::pipe_to_stream::PipeToSendStream; use crate::api2::types::Userid; -use crate::tools::async_io::EitherStream; use crate::tools::{ self, BroadcastFuture, DEFAULT_ENCODE_SET, - socket::{ - set_tcp_keepalive, - PROXMOX_BACKUP_TCP_KEEPALIVE_TIME, - }, + http::HttpsConnector, }; #[derive(Clone)] @@ -301,7 +293,7 @@ impl HttpClient { ssl_connector_builder.set_verify(openssl::ssl::SslVerifyMode::NONE); } - let mut httpc = hyper::client::HttpConnector::new(); + let mut httpc = HttpConnector::new(); httpc.set_nodelay(true); // important for h2 download performance! httpc.enforce_http(false); // we want https... @@ -929,64 +921,3 @@ impl H2Client { } } } - -#[derive(Clone)] -pub struct HttpsConnector { - http: HttpConnector, - ssl_connector: std::sync::Arc, -} - -impl HttpsConnector { - pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self { - http.enforce_http(false); - - Self { - http, - ssl_connector: std::sync::Arc::new(ssl_connector), - } - } -} - -type MaybeTlsStream = EitherStream< - tokio::net::TcpStream, - tokio_openssl::SslStream, ->; - -impl hyper::service::Service for HttpsConnector { - type Response = MaybeTlsStream; - type Error = Error; - type Future = std::pin::Pin> + Send + 'static - >>; - - fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { - // This connector is always ready, but others might not be. - Poll::Ready(Ok(())) - } - - fn call(&mut self, dst: Uri) -> Self::Future { - let mut this = self.clone(); - async move { - let is_https = dst - .scheme() - .ok_or_else(|| format_err!("missing URL scheme"))? - == "https"; - let host = dst - .host() - .ok_or_else(|| format_err!("missing hostname in destination url?"))? - .to_string(); - - let config = this.ssl_connector.configure(); - let conn = this.http.call(dst).await?; - - let _ = set_tcp_keepalive(conn.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); - - if is_https { - let conn = tokio_openssl::connect(config?, &host, conn).await?; - Ok(MaybeTlsStream::Right(conn)) - } else { - Ok(MaybeTlsStream::Left(conn)) - } - }.boxed() - } -} diff --git a/src/tools.rs b/src/tools.rs index 22d6c344..0e4a65c7 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -37,6 +37,7 @@ pub mod loopdev; pub mod fuse_loop; pub mod socket; pub mod zip; +pub mod http; mod parallel_handler; pub use parallel_handler::*; diff --git a/src/tools/http.rs b/src/tools/http.rs new file mode 100644 index 00000000..fe432806 --- /dev/null +++ b/src/tools/http.rs @@ -0,0 +1,100 @@ +use anyhow::{Error, format_err, bail}; +use lazy_static::lazy_static; +use std::task::{Context, Poll}; +use std::os::unix::io::AsRawFd; + +use hyper::{Uri, Body}; +use hyper::client::{Client, HttpConnector}; +use openssl::ssl::{SslConnector, SslMethod}; +use futures::*; + +use crate::tools::{ + async_io::EitherStream, + socket::{ + set_tcp_keepalive, + PROXMOX_BACKUP_TCP_KEEPALIVE_TIME, + }, +}; + +lazy_static! { + static ref HTTP_CLIENT: Client = { + let connector = SslConnector::builder(SslMethod::tls()).unwrap().build(); + let httpc = HttpConnector::new(); + let https = HttpsConnector::with_connector(httpc, connector); + Client::builder().build(https) + }; +} + +pub async fn get_string>(uri: U) -> Result { + let res = HTTP_CLIENT.get(uri.as_ref().parse()?).await?; + + let status = res.status(); + if !status.is_success() { + bail!("Got bad status '{}' from server", status) + } + + let buf = hyper::body::to_bytes(res).await?; + String::from_utf8(buf.to_vec()) + .map_err(|err| format_err!("Error converting HTTP result data: {}", err)) +} + +#[derive(Clone)] +pub struct HttpsConnector { + http: HttpConnector, + ssl_connector: std::sync::Arc, +} + +impl HttpsConnector { + pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self { + http.enforce_http(false); + + Self { + http, + ssl_connector: std::sync::Arc::new(ssl_connector), + } + } +} + +type MaybeTlsStream = EitherStream< + tokio::net::TcpStream, + tokio_openssl::SslStream, +>; + +impl hyper::service::Service for HttpsConnector { + type Response = MaybeTlsStream; + type Error = Error; + type Future = std::pin::Pin> + Send + 'static + >>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + // This connector is always ready, but others might not be. + Poll::Ready(Ok(())) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + let mut this = self.clone(); + async move { + let is_https = dst + .scheme() + .ok_or_else(|| format_err!("missing URL scheme"))? + == "https"; + let host = dst + .host() + .ok_or_else(|| format_err!("missing hostname in destination url?"))? + .to_string(); + + let config = this.ssl_connector.configure(); + let conn = this.http.call(dst).await?; + + let _ = set_tcp_keepalive(conn.as_raw_fd(), PROXMOX_BACKUP_TCP_KEEPALIVE_TIME); + + if is_https { + let conn = tokio_openssl::connect(config?, &host, conn).await?; + Ok(MaybeTlsStream::Right(conn)) + } else { + Ok(MaybeTlsStream::Left(conn)) + } + }.boxed() + } +} -- 2.20.1