From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from ronja.mits.lan
by ronja.mits.lan with LMTP
id cD3CK54sfGZlaQAAxxbTJA
(envelope-from <pbs-devel-bounces@lists.proxmox.com>); Wed, 26 Jun 2024 16:58:38 +0200
Received: from proxmox-new.maurer-it.com (unknown [192.168.2.33])
by ronja.mits.lan (Postfix) with ESMTPS id 9E898F6463A;
Wed, 26 Jun 2024 16:58:38 +0200 (CEST)
Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1])
by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 8473948740;
Wed, 26 Jun 2024 16:58:38 +0200 (CEST)
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
(using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits)
key-exchange X25519 server-signature RSA-PSS (4096 bits))
(No client certificate requested)
by proxmox-new.maurer-it.com (Proxmox) with ESMTPS;
Wed, 26 Jun 2024 16:58:37 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 2FDC15421;
Wed, 26 Jun 2024 16:58:37 +0200 (CEST)
From: Maximiliano Sandoval <m.sandoval@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Wed, 26 Jun 2024 16:58:00 +0200
Message-Id: <20240626145801.751326-5-m.sandoval@proxmox.com>
X-Mailer: git-send-email 2.39.2
In-Reply-To: <20240626145801.751326-1-m.sandoval@proxmox.com>
References: <20240626145801.751326-1-m.sandoval@proxmox.com>
MIME-Version: 1.0
Subject: [pbs-devel] [PATCH promox v4 4/5] compression: deflate: add test
module
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
X-SPAM-LEVEL: Spam detection results: 0
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
KAM_LAZY_DOMAIN_SECURITY 1 Sending domain does not have any anti-forgery methods
MAILING_LIST_MULTI -2 Multiple indicators imply a widely-seen list manager
RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_NONE 0.001 SPF: sender does not publish an SPF Record
We test the deflate encoder against the deflate decoder using (or not)
zlib and with different small buffer sizes. We also test compression and
decompression against the flate2 crate.
Signed-off-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---
proxmox-compression/Cargo.toml | 3 +-
proxmox-compression/src/deflate/mod.rs | 160 +++++++++++++++++++++++++
2 files changed, 161 insertions(+), 2 deletions(-)
diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml
index b41c796f..b217e67f 100644
--- a/proxmox-compression/Cargo.toml
+++ b/proxmox-compression/Cargo.toml
@@ -27,5 +27,4 @@ proxmox-io = { workspace = true, features = [ "tokio" ] }
proxmox-lang.workspace = true
[dev-dependencies]
-tokio = { workspace = true, features = [ "macros" ] }
-
+tokio = { workspace = true, features = [ "macros", "rt-multi-thread" ] }
diff --git a/proxmox-compression/src/deflate/mod.rs b/proxmox-compression/src/deflate/mod.rs
index 6867176c..94faabb3 100644
--- a/proxmox-compression/src/deflate/mod.rs
+++ b/proxmox-compression/src/deflate/mod.rs
@@ -5,3 +5,163 @@ pub use compression::{DeflateEncoder, Level};
pub use decompression::DeflateDecoder;
const BUFFER_SIZE: usize = 8192;
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ use std::io::Write;
+
+ use flate2::Compression;
+ use futures::StreamExt;
+
+ const BUFFER_SIZE: usize = 25;
+ const BODY: &str = r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
+eiusmod tempor incididunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut
+enim aeque doleamus animo, cum corpore dolemus, fieri tamen permagna accessio potest,
+si aliquod aeternum et infinitum impendere."#;
+
+ fn chunker(content: &[u8]) -> Vec<Result<Vec<u8>, std::io::Error>> {
+ vec![
+ Ok(content[..10].to_vec()),
+ Ok(content[10..20].to_vec()),
+ Ok(content[20..30].to_vec()),
+ Ok(content[30..40].to_vec()),
+ Ok(content[40..].to_vec()),
+ ]
+ }
+
+ #[tokio::test]
+ async fn test_encoder_against_decoder() {
+ // We use mixed sizes for the buffers, on the next test we invert the
+ // sizes.
+ let stream = futures::stream::iter(chunker(BODY.as_bytes()));
+ let encoder = DeflateEncoder::builder(stream)
+ .buffer_size(BUFFER_SIZE * 2)
+ .build();
+ let mut decoder = DeflateDecoder::builder(encoder)
+ .buffer_size(BUFFER_SIZE)
+ .build();
+
+ let mut buf = Vec::with_capacity(BODY.len());
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res).unwrap();
+ }
+
+ assert_eq!(buf, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_encoder_against_decoder() {
+ let stream = futures::stream::iter(chunker(BODY.as_bytes()));
+ let encoder = DeflateEncoder::builder(stream)
+ .zlib(true)
+ .buffer_size(BUFFER_SIZE)
+ .build();
+ let mut decoder = DeflateDecoder::builder(encoder)
+ .zlib(true)
+ .buffer_size(BUFFER_SIZE * 2)
+ .build();
+
+ let mut buf = Vec::with_capacity(BODY.len());
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res).unwrap();
+ }
+
+ assert_eq!(buf, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_deflate_decompression_against_flate2() {
+ let encoded = flate2_encode(BODY.as_bytes(), false).unwrap();
+ let decoded = decode(&encoded, false, 7).await.unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_decompression_against_flate2() {
+ let encoded = flate2_encode(BODY.as_bytes(), true).unwrap();
+ let decoded = decode(&encoded, true, 4).await.unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_deflate_compression_against_flate2() {
+ let encoded = encode(BODY.as_bytes(), false, 5).await.unwrap();
+ let decoded = flate2_decode(&encoded, false).unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_compression_against_flate2() {
+ let encoded = encode(BODY.as_bytes(), true, 3).await.unwrap();
+ let decoded = flate2_decode(&encoded, true).unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ fn flate2_encode(bytes: &[u8], is_zlib: bool) -> Result<Vec<u8>, std::io::Error> {
+ if is_zlib {
+ let mut e = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ } else {
+ let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ }
+ }
+
+ fn flate2_decode(bytes: &[u8], is_zlib: bool) -> Result<Vec<u8>, std::io::Error> {
+ if is_zlib {
+ let mut e = flate2::write::ZlibDecoder::new(Vec::new());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ } else {
+ let mut e = flate2::write::DeflateDecoder::new(Vec::new());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ }
+ }
+
+ async fn decode(
+ content: &[u8],
+ is_zlib: bool,
+ buffer_size: usize,
+ ) -> Result<Vec<u8>, std::io::Error> {
+ let stream = futures::stream::iter(chunker(content));
+ let mut decoder = DeflateDecoder::builder(stream)
+ .zlib(is_zlib)
+ .buffer_size(buffer_size)
+ .build();
+ let mut buf = Vec::new();
+
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res)?;
+ }
+
+ Ok(buf)
+ }
+
+ async fn encode(
+ content: &[u8],
+ is_zlib: bool,
+ buffer_size: usize,
+ ) -> Result<Vec<u8>, std::io::Error> {
+ let stream = futures::stream::iter(chunker(content));
+ let mut encoder = DeflateEncoder::builder(stream)
+ .zlib(is_zlib)
+ .buffer_size(buffer_size)
+ .build();
+ let mut buf = Vec::with_capacity(BODY.len());
+
+ while let Some(Ok(res)) = encoder.next().await {
+ buf.write_all(&res)?;
+ }
+
+ Ok(buf)
+ }
+}
--
2.39.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel