From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 406001FF2B0 for ; Fri, 5 Jul 2024 15:04:49 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E75CF3288; Fri, 5 Jul 2024 15:05:07 +0200 (CEST) From: Maximiliano Sandoval To: pbs-devel@lists.proxmox.com Date: Fri, 5 Jul 2024 15:04:32 +0200 Message-Id: <20240705130432.360361-6-m.sandoval@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240705130432.360361-1-m.sandoval@proxmox.com> References: <20240705130432.360361-1-m.sandoval@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.129 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox v5 5/5] http: teach the Client how to decode deflate content X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Cc: Lukas Wagner Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" The Backup Server can compress the content using deflate so we teach the client how to decode it. If a request is sent with the `Accept-Encoding` [2] header set to `deflate`, and the response's `Content-Encoding` [1] header is equal to `deflate` we wrap the Body stream with a stream that can decode `zlib` on the run. Note that from the `Accept-Encoding` docs [2], the `deflate` encoding is actually `zlib`. This can be also tested against http://eu.httpbin.org/#/Response_formats/get_deflate by adding the following test: ```rust #[tokio::test] async fn test_client() { let client = Client::new(); let headers = HashMap::from([( hyper::header::ACCEPT_ENCODING.to_string(), "deflate".to_string(), )]); let response = client .get_string("https://eu.httpbin.org/deflate", Some(&headers)) .await; assert!(response.is_ok()); } ``` at `proxmox-http/src/client/simple.rs` and running ``` cargo test --features=client,client-trait ``` [1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding [2] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding Suggested-by: Lukas Wagner Signed-off-by: Maximiliano Sandoval --- proxmox-http/Cargo.toml | 7 ++++ proxmox-http/src/client/simple.rs | 65 ++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/proxmox-http/Cargo.toml b/proxmox-http/Cargo.toml index 9ece24eb..4455ba85 100644 --- a/proxmox-http/Cargo.toml +++ b/proxmox-http/Cargo.toml @@ -26,6 +26,11 @@ proxmox-async = { workspace = true, optional = true } proxmox-sys = { workspace = true, optional = true } proxmox-io = { workspace = true, optional = true } proxmox-lang = { workspace = true, optional = true } +proxmox-compression = { workspace = true, optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = [ "macros" ] } +flate2 = { workspace = true } [features] default = [] @@ -42,12 +47,14 @@ client = [ "dep:futures", "dep:hyper", "dep:openssl", + "dep:proxmox-compression", "dep:tokio", "dep:tokio-openssl", "http-helpers", "hyper?/client", "hyper?/http1", "hyper?/http2", + "hyper?/stream", "hyper?/tcp", "rate-limited-stream", "tokio?/io-util", diff --git a/proxmox-http/src/client/simple.rs b/proxmox-http/src/client/simple.rs index e9910802..062889ac 100644 --- a/proxmox-http/src/client/simple.rs +++ b/proxmox-http/src/client/simple.rs @@ -78,7 +78,8 @@ impl Client { self.add_proxy_headers(&mut request)?; - self.client.request(request).map_err(Error::from).await + let encoded_response = self.client.request(request).map_err(Error::from).await?; + decode_response(encoded_response).await } pub async fn post( @@ -245,3 +246,65 @@ impl crate::HttpClient for Client { }) } } + +/// Wraps the `Body` stream in a DeflateDecoder stream if the `Content-Encoding` +/// header of the response is `deflate`, otherwise returns the original +/// response. +async fn decode_response(mut res: Response) -> Result, Error> { + let Some(content_encoding) = res.headers_mut().remove(&hyper::header::CONTENT_ENCODING) else { + return Ok(res); + }; + + let encodings = content_encoding.to_str()?; + if encodings == "deflate" { + let (parts, body) = res.into_parts(); + let decoder = proxmox_compression::DeflateDecoder::builder(body) + .zlib(true) + .build(); + let decoded_body = Body::wrap_stream(decoder); + Ok(Response::from_parts(parts, decoded_body)) + } else { + bail!("Unknown encoding format: {encodings}"); + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::io::Write; + + const BODY: &str = r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do +eiusmod tempor incididunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut +enim aeque doleamus animo, cum corpore dolemus, fieri tamen permagna accessio potest, +si aliquod aeternum et infinitum impendere."#; + + #[tokio::test] + async fn test_parse_response_deflate() { + let encoded = encode_deflate(BODY.as_bytes()).unwrap(); + let encoded_body = Body::from(encoded); + let encoded_response = Response::builder() + .header(hyper::header::CONTENT_ENCODING, "deflate") + .body(encoded_body) + .unwrap(); + + let decoded_response = decode_response(encoded_response).await.unwrap(); + + assert_eq!( + Client::response_body_string(decoded_response) + .await + .unwrap(), + BODY + ); + } + + fn encode_deflate(bytes: &[u8]) -> Result, std::io::Error> { + use flate2::write::ZlibEncoder; + use flate2::Compression; + + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(bytes).unwrap(); + + e.finish() + } +} -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel