From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 3F33166A49 for ; Tue, 28 Jul 2020 10:59:56 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 31AD222505 for ; Tue, 28 Jul 2020 10:59:26 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 4E5CF224F6 for ; Tue, 28 Jul 2020 10:59:24 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 177DB4031B for ; Tue, 28 Jul 2020 10:59:24 +0200 (CEST) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Tue, 28 Jul 2020 10:58:44 +0200 Message-Id: <20200728085846.28816-3-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20200728085846.28816-1-s.reiter@proxmox.com> References: <20200728085846.28816-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.051 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [http.rs, tools.rs] Subject: [pbs-devel] [RFC proxmox-backup 2/4] add tools::http for generic HTTP GET and move HttpsConnector there X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 28 Jul 2020 08:59:56 -0000 ...to avoid having the tools:: module depend on api2. The get_string function is based directly on hyper and thus relatively simple, not supporting redirects for example. Signed-off-by: Stefan Reiter --- src/client/http_client.rs | 64 +--------------------------- src/tools.rs | 1 + src/tools/http.rs | 90 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 src/tools/http.rs diff --git a/src/client/http_client.rs b/src/client/http_client.rs index a930ea56..ee0cd623 100644 --- a/src/client/http_client.rs +++ b/src/client/http_client.rs @@ -1,5 +1,4 @@ use std::io::Write; -use std::task::{Context, Poll}; use std::sync::{Arc, Mutex}; use chrono::Utc; @@ -24,8 +23,7 @@ use proxmox::{ }; use super::pipe_to_stream::PipeToSendStream; -use crate::tools::async_io::EitherStream; -use crate::tools::{self, BroadcastFuture, DEFAULT_ENCODE_SET}; +use crate::tools::{self, BroadcastFuture, DEFAULT_ENCODE_SET, http::HttpsConnector}; #[derive(Clone)] pub struct AuthInfo { @@ -286,7 +284,7 @@ impl HttpClient { ssl_connector_builder.set_verify(openssl::ssl::SslVerifyMode::NONE); } - let mut httpc = hyper::client::HttpConnector::new(); + let mut httpc = HttpConnector::new(); httpc.set_nodelay(true); // important for h2 download performance! httpc.set_recv_buffer_size(Some(1024*1024)); //important for h2 download performance! httpc.enforce_http(false); // we want https... @@ -861,61 +859,3 @@ impl H2Client { } } } - -#[derive(Clone)] -pub struct HttpsConnector { - http: HttpConnector, - ssl_connector: std::sync::Arc, -} - -impl HttpsConnector { - pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self { - http.enforce_http(false); - - Self { - http, - ssl_connector: std::sync::Arc::new(ssl_connector), - } - } -} - -type MaybeTlsStream = EitherStream< - tokio::net::TcpStream, - tokio_openssl::SslStream, ->; - -impl hyper::service::Service for HttpsConnector { - type Response = MaybeTlsStream; - type Error = Error; - type Future = std::pin::Pin> + Send + 'static - >>; - - fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { - // This connector is always ready, but others might not be. - Poll::Ready(Ok(())) - } - - fn call(&mut self, dst: Uri) -> Self::Future { - let mut this = self.clone(); - async move { - let is_https = dst - .scheme() - .ok_or_else(|| format_err!("missing URL scheme"))? - == "https"; - let host = dst - .host() - .ok_or_else(|| format_err!("missing hostname in destination url?"))? - .to_string(); - - let config = this.ssl_connector.configure(); - let conn = this.http.call(dst).await?; - if is_https { - let conn = tokio_openssl::connect(config?, &host, conn).await?; - Ok(MaybeTlsStream::Right(conn)) - } else { - Ok(MaybeTlsStream::Left(conn)) - } - }.boxed() - } -} diff --git a/src/tools.rs b/src/tools.rs index 44db796d..c99f5120 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -35,6 +35,7 @@ pub mod timer; pub mod statistics; pub mod systemd; pub mod nom; +pub mod http; mod wrapped_reader_stream; pub use wrapped_reader_stream::*; diff --git a/src/tools/http.rs b/src/tools/http.rs new file mode 100644 index 00000000..b4b1424c --- /dev/null +++ b/src/tools/http.rs @@ -0,0 +1,90 @@ +use anyhow::{Error, format_err, bail}; +use lazy_static::lazy_static; +use std::task::{Context, Poll}; + +use hyper::{Uri, Body}; +use hyper::client::{Client, HttpConnector}; +use openssl::ssl::{SslConnector, SslMethod}; +use futures::*; + +use crate::tools::async_io::EitherStream; + +lazy_static! { + static ref HTTP_CLIENT: Client = { + let connector = SslConnector::builder(SslMethod::tls()).unwrap().build(); + let httpc = HttpConnector::new(); + let https = HttpsConnector::with_connector(httpc, connector); + Client::builder().build(https) + }; +} + +pub async fn get_string>(uri: U) -> Result { + let res = HTTP_CLIENT.get(uri.as_ref().parse()?).await?; + + let status = res.status(); + if !status.is_success() { + bail!("Got bad status '{}' from server", status) + } + + let buf = hyper::body::to_bytes(res).await?; + String::from_utf8(buf.to_vec()) + .map_err(|err| format_err!("Error converting HTTP result data: {}", err)) +} + +#[derive(Clone)] +pub struct HttpsConnector { + http: HttpConnector, + ssl_connector: std::sync::Arc, +} + +impl HttpsConnector { + pub fn with_connector(mut http: HttpConnector, ssl_connector: SslConnector) -> Self { + http.enforce_http(false); + + Self { + http, + ssl_connector: std::sync::Arc::new(ssl_connector), + } + } +} + +type MaybeTlsStream = EitherStream< + tokio::net::TcpStream, + tokio_openssl::SslStream, +>; + +impl hyper::service::Service for HttpsConnector { + type Response = MaybeTlsStream; + type Error = Error; + type Future = std::pin::Pin> + Send + 'static + >>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + // This connector is always ready, but others might not be. + Poll::Ready(Ok(())) + } + + fn call(&mut self, dst: Uri) -> Self::Future { + let mut this = self.clone(); + async move { + let is_https = dst + .scheme() + .ok_or_else(|| format_err!("missing URL scheme"))? + == "https"; + let host = dst + .host() + .ok_or_else(|| format_err!("missing hostname in destination url?"))? + .to_string(); + + let config = this.ssl_connector.configure(); + let conn = this.http.call(dst).await?; + if is_https { + let conn = tokio_openssl::connect(config?, &host, conn).await?; + Ok(MaybeTlsStream::Right(conn)) + } else { + Ok(MaybeTlsStream::Left(conn)) + } + }.boxed() + } +} -- 2.20.1