From mboxrd@z Thu Jan 1 00:00:00 1970
Return-Path: <pbs-devel-bounces@lists.proxmox.com>
Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68])
by lore.proxmox.com (Postfix) with ESMTPS id AA1AE1FF2B0
for <inbox@lore.proxmox.com>; Fri, 5 Jul 2024 15:04:21 +0200 (CEST)
Received: from firstgate.proxmox.com (localhost [127.0.0.1])
by firstgate.proxmox.com (Proxmox) with ESMTP id 93DFE3117;
Fri, 5 Jul 2024 15:04:37 +0200 (CEST)
From: Maximiliano Sandoval <m.sandoval@proxmox.com>
To: pbs-devel@lists.proxmox.com
Date: Fri, 5 Jul 2024 15:04:31 +0200
Message-Id: <20240705130432.360361-5-m.sandoval@proxmox.com>
X-Mailer: git-send-email 2.39.2
In-Reply-To: <20240705130432.360361-1-m.sandoval@proxmox.com>
References: <20240705130432.360361-1-m.sandoval@proxmox.com>
MIME-Version: 1.0
X-SPAM-LEVEL: Spam detection results: 0
AWL 0.129 Adjusted score from AWL reputation of From: address
BAYES_00 -1.9 Bayes spam probability is 0 to 1%
DMARC_MISSING 0.1 Missing DMARC policy
KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment
SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record
SPF_PASS -0.001 SPF: sender matches SPF record
URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See
http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more
information. [mod.rs]
Subject: [pbs-devel] [PATCH proxmox v5 4/5] compression: deflate: add test
module
X-BeenThere: pbs-devel@lists.proxmox.com
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: Proxmox Backup Server development discussion
<pbs-devel.lists.proxmox.com>
List-Unsubscribe: <https://lists.proxmox.com/cgi-bin/mailman/options/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=unsubscribe>
List-Archive: <http://lists.proxmox.com/pipermail/pbs-devel/>
List-Post: <mailto:pbs-devel@lists.proxmox.com>
List-Help: <mailto:pbs-devel-request@lists.proxmox.com?subject=help>
List-Subscribe: <https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel>,
<mailto:pbs-devel-request@lists.proxmox.com?subject=subscribe>
Reply-To: Proxmox Backup Server development discussion
<pbs-devel@lists.proxmox.com>
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit
Errors-To: pbs-devel-bounces@lists.proxmox.com
Sender: "pbs-devel" <pbs-devel-bounces@lists.proxmox.com>
We test the deflate encoder against the deflate decoder using (or not)
zlib and with different small buffer sizes. We also test compression and
decompression against the flate2 crate.
Signed-off-by: Maximiliano Sandoval <m.sandoval@proxmox.com>
---
proxmox-compression/Cargo.toml | 3 +-
proxmox-compression/src/deflate/mod.rs | 160 +++++++++++++++++++++++++
2 files changed, 161 insertions(+), 2 deletions(-)
diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml
index b41c796f..b217e67f 100644
--- a/proxmox-compression/Cargo.toml
+++ b/proxmox-compression/Cargo.toml
@@ -27,5 +27,4 @@ proxmox-io = { workspace = true, features = [ "tokio" ] }
proxmox-lang.workspace = true
[dev-dependencies]
-tokio = { workspace = true, features = [ "macros" ] }
-
+tokio = { workspace = true, features = [ "macros", "rt-multi-thread" ] }
diff --git a/proxmox-compression/src/deflate/mod.rs b/proxmox-compression/src/deflate/mod.rs
index 6867176c..94faabb3 100644
--- a/proxmox-compression/src/deflate/mod.rs
+++ b/proxmox-compression/src/deflate/mod.rs
@@ -5,3 +5,163 @@ pub use compression::{DeflateEncoder, Level};
pub use decompression::DeflateDecoder;
const BUFFER_SIZE: usize = 8192;
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ use std::io::Write;
+
+ use flate2::Compression;
+ use futures::StreamExt;
+
+ const BUFFER_SIZE: usize = 25;
+ const BODY: &str = r#"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do
+eiusmod tempor incididunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut
+enim aeque doleamus animo, cum corpore dolemus, fieri tamen permagna accessio potest,
+si aliquod aeternum et infinitum impendere."#;
+
+ fn chunker(content: &[u8]) -> Vec<Result<Vec<u8>, std::io::Error>> {
+ vec![
+ Ok(content[..10].to_vec()),
+ Ok(content[10..20].to_vec()),
+ Ok(content[20..30].to_vec()),
+ Ok(content[30..40].to_vec()),
+ Ok(content[40..].to_vec()),
+ ]
+ }
+
+ #[tokio::test]
+ async fn test_encoder_against_decoder() {
+ // We use mixed sizes for the buffers, on the next test we invert the
+ // sizes.
+ let stream = futures::stream::iter(chunker(BODY.as_bytes()));
+ let encoder = DeflateEncoder::builder(stream)
+ .buffer_size(BUFFER_SIZE * 2)
+ .build();
+ let mut decoder = DeflateDecoder::builder(encoder)
+ .buffer_size(BUFFER_SIZE)
+ .build();
+
+ let mut buf = Vec::with_capacity(BODY.len());
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res).unwrap();
+ }
+
+ assert_eq!(buf, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_encoder_against_decoder() {
+ let stream = futures::stream::iter(chunker(BODY.as_bytes()));
+ let encoder = DeflateEncoder::builder(stream)
+ .zlib(true)
+ .buffer_size(BUFFER_SIZE)
+ .build();
+ let mut decoder = DeflateDecoder::builder(encoder)
+ .zlib(true)
+ .buffer_size(BUFFER_SIZE * 2)
+ .build();
+
+ let mut buf = Vec::with_capacity(BODY.len());
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res).unwrap();
+ }
+
+ assert_eq!(buf, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_deflate_decompression_against_flate2() {
+ let encoded = flate2_encode(BODY.as_bytes(), false).unwrap();
+ let decoded = decode(&encoded, false, 7).await.unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_decompression_against_flate2() {
+ let encoded = flate2_encode(BODY.as_bytes(), true).unwrap();
+ let decoded = decode(&encoded, true, 4).await.unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_deflate_compression_against_flate2() {
+ let encoded = encode(BODY.as_bytes(), false, 5).await.unwrap();
+ let decoded = flate2_decode(&encoded, false).unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ #[tokio::test]
+ async fn test_zlib_compression_against_flate2() {
+ let encoded = encode(BODY.as_bytes(), true, 3).await.unwrap();
+ let decoded = flate2_decode(&encoded, true).unwrap();
+
+ assert_eq!(decoded, BODY.as_bytes());
+ }
+
+ fn flate2_encode(bytes: &[u8], is_zlib: bool) -> Result<Vec<u8>, std::io::Error> {
+ if is_zlib {
+ let mut e = flate2::write::ZlibEncoder::new(Vec::new(), Compression::default());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ } else {
+ let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::default());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ }
+ }
+
+ fn flate2_decode(bytes: &[u8], is_zlib: bool) -> Result<Vec<u8>, std::io::Error> {
+ if is_zlib {
+ let mut e = flate2::write::ZlibDecoder::new(Vec::new());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ } else {
+ let mut e = flate2::write::DeflateDecoder::new(Vec::new());
+ e.write_all(bytes).unwrap();
+ e.finish()
+ }
+ }
+
+ async fn decode(
+ content: &[u8],
+ is_zlib: bool,
+ buffer_size: usize,
+ ) -> Result<Vec<u8>, std::io::Error> {
+ let stream = futures::stream::iter(chunker(content));
+ let mut decoder = DeflateDecoder::builder(stream)
+ .zlib(is_zlib)
+ .buffer_size(buffer_size)
+ .build();
+ let mut buf = Vec::new();
+
+ while let Some(Ok(res)) = decoder.next().await {
+ buf.write_all(&res)?;
+ }
+
+ Ok(buf)
+ }
+
+ async fn encode(
+ content: &[u8],
+ is_zlib: bool,
+ buffer_size: usize,
+ ) -> Result<Vec<u8>, std::io::Error> {
+ let stream = futures::stream::iter(chunker(content));
+ let mut encoder = DeflateEncoder::builder(stream)
+ .zlib(is_zlib)
+ .buffer_size(buffer_size)
+ .build();
+ let mut buf = Vec::with_capacity(BODY.len());
+
+ while let Some(Ok(res)) = encoder.next().await {
+ buf.write_all(&res)?;
+ }
+
+ Ok(buf)
+ }
+}
--
2.39.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel