From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9])
by lore.proxmox.com (Postfix) with ESMTPS id 406001FF2B0
for <inbox@lore.proxmox.com>; Fri, 5 Jul 2024 15:04:49 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id E75CF3288;
Fri, 5 Jul 2024 15:05:07 +0200 (CEST)
From: Maximiliano Sandoval <m.sandoval@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Fri, 5 Jul 2024 15:04:32 +0200
Message-Id: <20240705130432.360361-6-m.sandoval@proxmox.com>
X-Mailer: git-send-email 2.39.2
In-Reply-To: <20240705130432.360361-1-m.sandoval@proxmox.com>
References: <20240705130432.360361-1-m.sandoval@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.129 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
Subject: [pbs-devel] [PATCH proxmox v5 5/5] http: teach the Client how to
decode deflate content
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Cc: Lukas Wagner <l.wagner@proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
The Backup Server can compress the content using deflate so we teach the
client how to decode it.
If a request is sent with the `Accept-Encoding` [2] header set to
`deflate`, and the response's `Content-Encoding` [1] header is equal to
`deflate` we wrap the Body stream with a stream that can decode `zlib`
on the run.
Note that from the `Accept-Encoding` docs [2], the `deflate` encoding is
actually `zlib`.
This can be also tested against
http://eu.httpbin.org/#/Response_formats/get_deflate by adding the
following test:
```rust
#[tokio::test]
async fn test_client() {
let client = Client::new();
let headers = HashMap::from([(
hyper::header::ACCEPT_ENCODING.to_string(),
"deflate".to_string(),
)]);
let response = client
.get_string("https://eu.httpbin.org/deflate", Some(&headers))
.await;
assert!(response.is_ok());
}
```
at `proxmox-http/src/client/simple.rs` and running
```
cargo test --features=client,client-trait
```
[1] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding
[2] https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
Suggested-by: Lukas Wagner <l.wagner@proxmox.com>
Signed-off-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---
proxmox-http/Cargo.toml | 7 ++++
proxmox-http/src/client/simple.rs | 65 ++++++++++++++++++++++++++++++-
2 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/proxmox-http/Cargo.toml b/proxmox-http/Cargo.toml
index 9ece24eb..4455ba85 100644
--- a/proxmox-http/Cargo.toml
+++ b/proxmox-http/Cargo.toml
@@ -26,6 +26,11 @@ proxmox-async = { workspace = true, optional = true }
proxmox-sys = { workspace = true, optional = true }
proxmox-io = { workspace = true, optional = true }
proxmox-lang = { workspace = true, optional = true }
+proxmox-compression = { workspace = true, optional = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = [ "macros" ] }
+flate2 = { workspace = true }
[features]
default = []
@@ -42,12 +47,14 @@ client = [
"dep:futures",
"dep:hyper",
"dep:openssl",
+ "dep:proxmox-compression",
"dep:tokio",
"dep:tokio-openssl",
"http-helpers",
"hyper?/client",
"hyper?/http1",
"hyper?/http2",
+ "hyper?/stream",
"hyper?/tcp",
"rate-limited-stream",
"tokio?/io-util",
diff --git a/proxmox-http/src/client/simple.rs b/proxmox-http/src/client/simple.rs
index e9910802..062889ac 100644
--- a/proxmox-http/src/client/simple.rs
+++ b/proxmox-http/src/client/simple.rs
@@ -78,7 +78,8 @@ impl Client {
self.add_proxy_headers(&mut request)?;
- self.client.request(request).map_err(Error::from).await
+ let encoded_response = self.client.request(request).map_err(Error::from).await?;
+ decode_response(encoded_response).await
}
pub async fn post(
@@ -245,3 +246,65 @@ impl crate::HttpClient<String, String> for Client {
})
}
}
+
+/// Wraps the `Body` stream in a DeflateDecoder stream if the `Content-Encoding`
+/// header of the response is `deflate`, otherwise returns the original
+/// response.
+async fn decode_response(mut res: Response<Body>) -> Result<Response<Body>, Error> {
+ let Some(content_encoding) = res.headers_mut().remove(&hyper::header::CONTENT_ENCODING) else {
+ return Ok(res);
+ };
+
+ let encodings = content_encoding.to_str()?;
+ if encodings == "deflate" {
+ let (parts, body) = res.into_parts();
+ let decoder = proxmox_compression::DeflateDecoder::builder(body)
+ .zlib(true)
+ .build();
+ let decoded_body = Body::wrap_stream(decoder);
+ Ok(Response::from_parts(parts, decoded_body))
+ } else {
+ bail!("Unknown encoding format: {encodings}");
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ use std::io::Write;
+
+ const BODY: &str = r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
+eiusmod tempor incididunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut
+enim aeque doleamus animo, cum corpore dolemus, fieri tamen permagna accessio potest,
+si aliquod aeternum et infinitum impendere."#;
+
+ #[tokio::test]
+ async fn test_parse_response_deflate() {
+ let encoded = encode_deflate(BODY.as_bytes()).unwrap();
+ let encoded_body = Body::from(encoded);
+ let encoded_response = Response::builder()
+ .header(hyper::header::CONTENT_ENCODING, "deflate")
+ .body(encoded_body)
+ .unwrap();
+
+ let decoded_response = decode_response(encoded_response).await.unwrap();
+
+ assert_eq!(
+ Client::response_body_string(decoded_response)
+ .await
+ .unwrap(),
+ BODY
+ );
+ }
+
+ fn encode_deflate(bytes: &[u8]) -> Result<Vec<u8>, std::io::Error> {
+ use flate2::write::ZlibEncoder;
+ use flate2::Compression;
+
+ let mut e = ZlibEncoder::new(Vec::new(), Compression::default());
+ e.write_all(bytes).unwrap();
+
+ e.finish()
+ }
+}
--
2.39.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel