From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 30CCE1FF17A for ; Mon, 5 Aug 2024 11:24:43 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 56D3B1ED6; Mon, 5 Aug 2024 11:24:49 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Mon, 5 Aug 2024 11:24:12 +0200 Message-Id: <20240805092414.1178930-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240805092414.1178930-1-d.csapak@proxmox.com> References: <20240805092414.1178930-1-d.csapak@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.014 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_VALIDITY_CERTIFIED_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_RPBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. RCVD_IN_VALIDITY_SAFE_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to Validity was blocked. See https://knowledge.validity.com/hc/en-us/articles/20961730681243 for more information. SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v3 3/5] datastore: data blob: add helper and test for checking zstd_safe error code X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" We want to check the error code of zstd not to be 'Destination buffer to small', but there is no practical api at the moment. So we introduce a helper that uses the internal logic of zstd to determine the error. Since this is not guaranteed to be a stable api, add a test for that so we catch that error early on build. This should be fine, as long as this zstd behavior only changes with e.g. major debian upgrades. (Which means a new version) Signed-off-by: Dominik Csapak --- new in v3 note that this probably throws a warning until the next patch due to the unused function. it would also be possible to merge this patch into the next if that's wanted Cargo.toml | 1 + pbs-datastore/Cargo.toml | 1 + pbs-datastore/src/data_blob.rs | 28 +++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2b51ec82..275e3c95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,6 +158,7 @@ url = "2.1" walkdir = "2" xdg = "2.2" zstd = { version = "0.12", features = [ "bindgen" ] } +zstd-safe = "6.0" [dependencies] anyhow.workspace = true diff --git a/pbs-datastore/Cargo.toml b/pbs-datastore/Cargo.toml index d8997e1d..494c231b 100644 --- a/pbs-datastore/Cargo.toml +++ b/pbs-datastore/Cargo.toml @@ -23,6 +23,7 @@ tokio = { workspace = true, features = [] } tracing.workspace = true walkdir.workspace = true zstd.workspace = true +zstd-safe.workspace = true pathpatterns.workspace = true pxar.workspace = true diff --git a/pbs-datastore/src/data_blob.rs b/pbs-datastore/src/data_blob.rs index a3a41c5e..adf5a932 100644 --- a/pbs-datastore/src/data_blob.rs +++ b/pbs-datastore/src/data_blob.rs @@ -12,6 +12,17 @@ use super::file_formats::*; const MAX_BLOB_SIZE: usize = 128 * 1024 * 1024; +// tests if the error code was 'Destination buffer is too small' +// by subtracting the error code from 0 (with underflow) +// see https://github.com/facebook/zstd/blob/dev/lib/common/error_private.h +// we test for this below so we catch errors if the interface changes +fn zstd_error_is_target_too_small(err: usize) -> bool { + let (real_code, _) = 0usize.overflowing_sub(err); + // ZSTD_error_dstSize_tooSmall = 70, + // see https://github.com/facebook/zstd/blob/dev/lib/zstd_errors.h + real_code == 70 +} + /// Encoded data chunk with digest and positional information pub struct ChunkInfo { pub chunk: DataBlob, @@ -567,7 +578,7 @@ impl<'a, 'b> DataChunkBuilder<'a, 'b> { mod test { use pbs_tools::crypt_config::CryptConfig; - use super::DataChunkBuilder; + use super::{zstd_error_is_target_too_small, DataChunkBuilder}; const TEST_DATA_LEN: usize = 50; @@ -640,4 +651,19 @@ mod test { .expect("cannot decode encrypted, compressed chunk"); assert_eq!(data, data_decoded); } + + #[test] + fn zstd_error_code_test() { + // test for the error code internal logic of zstd so we catch interface changes here + let data = &build_test_data(); + let mut target = Vec::new(); + match zstd_safe::compress(&mut target, data, 1) { + Ok(_) => panic!("unexpected success with zero-sized buffer"), + Err(err) => { + if !zstd_error_is_target_too_small(err) { + panic!("unexpected error code"); + } + } + } + } } -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel