From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 348B31FF389 for ; Wed, 5 Jun 2024 13:01:32 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id DB8D6324AF; Wed, 5 Jun 2024 13:02:00 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 5 Jun 2024 12:54:04 +0200 Message-Id: <20240605105416.278748-47-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240605105416.278748-1-c.ebner@proxmox.com> References: <20240605105416.278748-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.028 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [ctx.total, chunker.rs] Subject: [pbs-devel] [PATCH v9 proxmox-backup 46/58] chunker: tests: add regression tests for payload chunker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Test chunking of a payload stream with suggested chunk boundaries. Signed-off-by: Christian Ebner --- changes since version 8: - no changes pbs-datastore/src/chunker.rs | 94 ++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/pbs-datastore/src/chunker.rs b/pbs-datastore/src/chunker.rs index d0543bca0..ecdbca296 100644 --- a/pbs-datastore/src/chunker.rs +++ b/pbs-datastore/src/chunker.rs @@ -382,3 +382,97 @@ fn test_chunker1() { panic!("got different chunks"); } } + +#[test] +fn test_suggested_boundary() { + let mut buffer = Vec::new(); + + for i in 0..(256 * 1024) { + for j in 0..4 { + let byte = ((i >> (j << 3)) & 0xff) as u8; + buffer.push(byte); + } + } + let (tx, rx) = std::sync::mpsc::channel(); + let mut chunker = PayloadChunker::new(64 * 1024, rx); + + // Suggest chunk boundary within regular chunk + tx.send(32 * 1024).unwrap(); + // Suggest chunk boundary within regular chunk, resulting chunk being 0 + tx.send(32 * 1024).unwrap(); + // Suggest chunk boundary in the past, must be ignored + tx.send(0).unwrap(); + // Suggest chunk boundary aligned with regular boundary + tx.send(405521).unwrap(); + + let mut pos = 0; + let mut last = 0; + + let mut chunks1: Vec<(usize, usize)> = vec![]; + let mut chunks2: Vec<(usize, usize)> = vec![]; + let mut ctx = Context::default(); + + // test1: feed single bytes with suggeset boundary + while pos < buffer.len() { + ctx.total += 1; + let k = chunker.scan(&buffer[pos..pos + 1], &ctx); + pos += 1; + if k != 0 { + let prev = last; + last = pos; + ctx.base += pos as u64; + ctx.total = 0; + chunks1.push((prev, pos - prev)); + } + } + chunks1.push((last, buffer.len() - last)); + + let mut pos = 0; + let mut ctx = Context::default(); + ctx.total = buffer.len() as u64; + chunker.reset(); + // Suggest chunk boundary within regular chunk + tx.send(32 * 1024).unwrap(); + // Suggest chunk boundary within regular chunk, + // resulting chunk being to small and therefore ignored + tx.send(32 * 1024).unwrap(); + // Suggest chunk boundary in the past, must be ignored + tx.send(0).unwrap(); + // Suggest chunk boundary aligned with regular boundary + tx.send(405521).unwrap(); + + while pos < buffer.len() { + let k = chunker.scan(&buffer[pos..], &ctx); + if k != 0 { + chunks2.push((pos, k)); + pos += k; + ctx.base += pos as u64; + ctx.total = (buffer.len() - pos) as u64; + } else { + break; + } + } + + chunks2.push((pos, buffer.len() - pos)); + + if chunks1 != chunks2 { + let mut size1 = 0; + for (_offset, len) in &chunks1 { + size1 += len; + } + println!("Chunks1: {size1}\n{chunks1:?}\n"); + + let mut size2 = 0; + for (_offset, len) in &chunks2 { + size2 += len; + } + println!("Chunks2: {size2}\n{chunks2:?}\n"); + + panic!("got different chunks"); + } + + let expected_sizes = [32768, 110609, 229376, 32768, 262144, 262144, 118767]; + for ((_, chunk_size), expected) in chunks1.iter().zip(expected_sizes.iter()) { + assert_eq!(chunk_size, expected); + } +} -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel