From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id AA1AE1FF2B0 for ; Fri, 5 Jul 2024 15:04:21 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 93DFE3117; Fri, 5 Jul 2024 15:04:37 +0200 (CEST) From: Maximiliano Sandoval To: pbs-devel@lists.proxmox.com Date: Fri, 5 Jul 2024 15:04:31 +0200 Message-Id: <20240705130432.360361-5-m.sandoval@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240705130432.360361-1-m.sandoval@proxmox.com> References: <20240705130432.360361-1-m.sandoval@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.129 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [mod.rs] Subject: [pbs-devel] [PATCH proxmox v5 4/5] compression: deflate: add test module X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" We test the deflate encoder against the deflate decoder using (or not) zlib and with different small buffer sizes. We also test compression and decompression against the flate2 crate. Signed-off-by: Maximiliano Sandoval --- proxmox-compression/Cargo.toml | 3 +- proxmox-compression/src/deflate/mod.rs | 160 +++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 2 deletions(-) diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml index b41c796f..b217e67f 100644 --- a/proxmox-compression/Cargo.toml +++ b/proxmox-compression/Cargo.toml @@ -27,5 +27,4 @@ proxmox-io = { workspace = true, features = [ "tokio" ] } proxmox-lang.workspace = true [dev-dependencies] -tokio = { workspace = true, features = [ "macros" ] } - +tokio = { workspace = true, features = [ "macros", "rt-multi-thread" ] } diff --git a/proxmox-compression/src/deflate/mod.rs b/proxmox-compression/src/deflate/mod.rs index 6867176c..94faabb3 100644 --- a/proxmox-compression/src/deflate/mod.rs +++ b/proxmox-compression/src/deflate/mod.rs @@ -5,3 +5,163 @@ pub use compression::{DeflateEncoder, Level}; pub use decompression::DeflateDecoder; const BUFFER_SIZE: usize = 8192; + +#[cfg(test)] +mod test { + use super::*; + + use std::io::Write; + + use flate2::Compression; + use futures::StreamExt; + + const BUFFER_SIZE: usize = 25; + const BODY: &str = r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do +eiusmod tempor incididunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut +enim aeque doleamus animo, cum corpore dolemus, fieri tamen permagna accessio potest, +si aliquod aeternum et infinitum impendere."#; + + fn chunker(content: &[u8]) -> Vec, std::io::Error>> { + vec![ + Ok(content[..10].to_vec()), + Ok(content[10..20].to_vec()), + Ok(content[20..30].to_vec()), + Ok(content[30..40].to_vec()), + Ok(content[40..].to_vec()), + ] + } + + #[tokio::test] + async fn test_encoder_against_decoder() { + // We use mixed sizes for the buffers, on the next test we invert the + // sizes. + let stream = futures::stream::iter(chunker(BODY.as_bytes())); + let encoder = DeflateEncoder::builder(stream) + .buffer_size(BUFFER_SIZE * 2) + .build(); + let mut decoder = DeflateDecoder::builder(encoder) + .buffer_size(BUFFER_SIZE) + .build(); + + let mut buf = Vec::with_capacity(BODY.len()); + while let Some(Ok(res)) = decoder.next().await { + buf.write_all(&res).unwrap(); + } + + assert_eq!(buf, BODY.as_bytes()); + } + + #[tokio::test] + async fn test_zlib_encoder_against_decoder() { + let stream = futures::stream::iter(chunker(BODY.as_bytes())); + let encoder = DeflateEncoder::builder(stream) + .zlib(true) + .buffer_size(BUFFER_SIZE) + .build(); + let mut decoder = DeflateDecoder::builder(encoder) + .zlib(true) + .buffer_size(BUFFER_SIZE * 2) + .build(); + + let mut buf = Vec::with_capacity(BODY.len()); + while let Some(Ok(res)) = decoder.next().await { + buf.write_all(&res).unwrap(); + } + + assert_eq!(buf, BODY.as_bytes()); + } + + #[tokio::test] + async fn test_deflate_decompression_against_flate2() { + let encoded = flate2_encode(BODY.as_bytes(), false).unwrap(); + let decoded = decode(&encoded, false, 7).await.unwrap(); + + assert_eq!(decoded, BODY.as_bytes()); + } + + #[tokio::test] + async fn test_zlib_decompression_against_flate2() { + let encoded = flate2_encode(BODY.as_bytes(), true).unwrap(); + let decoded = decode(&encoded, true, 4).await.unwrap(); + + assert_eq!(decoded, BODY.as_bytes()); + } + + #[tokio::test] + async fn test_deflate_compression_against_flate2() { + let encoded = encode(BODY.as_bytes(), false, 5).await.unwrap(); + let decoded = flate2_decode(&encoded, false).unwrap(); + + assert_eq!(decoded, BODY.as_bytes()); + } + + #[tokio::test] + async fn test_zlib_compression_against_flate2() { + let encoded = encode(BODY.as_bytes(), true, 3).await.unwrap(); + let decoded = flate2_decode(&encoded, true).unwrap(); + + assert_eq!(decoded, BODY.as_bytes()); + } + + fn flate2_encode(bytes: &[u8], is_zlib: bool) -> Result, std::io::Error> { + if is_zlib { + let mut e = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(bytes).unwrap(); + e.finish() + } else { + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default()); + e.write_all(bytes).unwrap(); + e.finish() + } + } + + fn flate2_decode(bytes: &[u8], is_zlib: bool) -> Result, std::io::Error> { + if is_zlib { + let mut e = flate2::write::ZlibDecoder::new(Vec::new()); + e.write_all(bytes).unwrap(); + e.finish() + } else { + let mut e = flate2::write::DeflateDecoder::new(Vec::new()); + e.write_all(bytes).unwrap(); + e.finish() + } + } + + async fn decode( + content: &[u8], + is_zlib: bool, + buffer_size: usize, + ) -> Result, std::io::Error> { + let stream = futures::stream::iter(chunker(content)); + let mut decoder = DeflateDecoder::builder(stream) + .zlib(is_zlib) + .buffer_size(buffer_size) + .build(); + let mut buf = Vec::new(); + + while let Some(Ok(res)) = decoder.next().await { + buf.write_all(&res)?; + } + + Ok(buf) + } + + async fn encode( + content: &[u8], + is_zlib: bool, + buffer_size: usize, + ) -> Result, std::io::Error> { + let stream = futures::stream::iter(chunker(content)); + let mut encoder = DeflateEncoder::builder(stream) + .zlib(is_zlib) + .buffer_size(buffer_size) + .build(); + let mut buf = Vec::with_capacity(BODY.len()); + + while let Some(Ok(res)) = encoder.next().await { + buf.write_all(&res)?; + } + + Ok(buf) + } +} -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel