From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 691FE1FF2CA for ; Tue, 23 Jul 2024 12:10:07 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 6C648753; Tue, 23 Jul 2024 12:10:41 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Tue, 23 Jul 2024 12:10:37 +0200 Message-Id: <20240723101037.1596714-2-d.csapak@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240723101037.1596714-1-d.csapak@proxmox.com> References: <20240723101037.1596714-1-d.csapak@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.184 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_2 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_4 0.1 random spam to be learned in bayes PROLO_LEO1 0.1 Meta Catches all Leo drug variations so far SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [lib.rs] Subject: [pbs-devel] [PATCH proxmox-backup 2/2] remove data blob writer X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" this is leftover code that is not currently used outside of tests. Should we need it again, we can just revert this commit. Signed-off-by: Dominik Csapak --- pbs-datastore/src/data_blob_writer.rs | 212 -------------------------- pbs-datastore/src/lib.rs | 2 - tests/blob_writer.rs | 105 ------------- 3 files changed, 319 deletions(-) delete mode 100644 pbs-datastore/src/data_blob_writer.rs delete mode 100644 tests/blob_writer.rs diff --git a/pbs-datastore/src/data_blob_writer.rs b/pbs-datastore/src/data_blob_writer.rs deleted file mode 100644 index 30d9645f..00000000 --- a/pbs-datastore/src/data_blob_writer.rs +++ /dev/null @@ -1,212 +0,0 @@ -use std::io::{Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::Error; - -use proxmox_io::WriteExt; - -use pbs_tools::crypt_config::CryptConfig; - -use crate::checksum_writer::ChecksumWriter; -use crate::crypt_writer::CryptWriter; -use crate::file_formats::{self, DataBlobHeader, EncryptedDataBlobHeader}; - -enum BlobWriterState<'writer, W: Write> { - Uncompressed { - csum_writer: ChecksumWriter, - }, - Compressed { - compr: zstd::stream::write::Encoder<'writer, ChecksumWriter>, - }, - Encrypted { - crypt_writer: CryptWriter>, - }, - EncryptedCompressed { - compr: zstd::stream::write::Encoder<'writer, CryptWriter>>, - }, -} - -/// Data blob writer -pub struct DataBlobWriter<'writer, W: Write> { - state: BlobWriterState<'writer, W>, -} - -impl DataBlobWriter<'_, W> { - pub fn new_uncompressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - Ok(Self { - state: BlobWriterState::Uncompressed { csum_writer }, - }) - } - - pub fn new_compressed(mut writer: W) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: [0; 4], - }; - unsafe { - writer.write_le_value(head)?; - } - let csum_writer = ChecksumWriter::new(writer, None); - let compr = zstd::stream::write::Encoder::new(csum_writer, 1)?; - Ok(Self { - state: BlobWriterState::Compressed { compr }, - }) - } - - pub fn new_encrypted(mut writer: W, config: Arc) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - Ok(Self { - state: BlobWriterState::Encrypted { crypt_writer }, - }) - } - - pub fn new_encrypted_compressed( - mut writer: W, - config: Arc, - ) -> Result { - writer.seek(SeekFrom::Start(0))?; - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: [0; 4], - }, - iv: [0u8; 16], - tag: [0u8; 16], - }; - unsafe { - writer.write_le_value(head)?; - } - - let csum_writer = ChecksumWriter::new(writer, None); - let crypt_writer = CryptWriter::new(csum_writer, config)?; - let compr = zstd::stream::write::Encoder::new(crypt_writer, 1)?; - Ok(Self { - state: BlobWriterState::EncryptedCompressed { compr }, - }) - } - - pub fn finish(self) -> Result { - match self.state { - BlobWriterState::Uncompressed { csum_writer } => { - // write CRC - let (mut writer, crc, _) = csum_writer.finish()?; - let head = DataBlobHeader { - magic: file_formats::UNCOMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Compressed { compr } => { - let csum_writer = compr.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = DataBlobHeader { - magic: file_formats::COMPRESSED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }; - - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - - Ok(writer) - } - BlobWriterState::Encrypted { crypt_writer } => { - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCRYPTED_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - BlobWriterState::EncryptedCompressed { compr } => { - let crypt_writer = compr.finish()?; - let (csum_writer, iv, tag) = crypt_writer.finish()?; - let (mut writer, crc, _) = csum_writer.finish()?; - - let head = EncryptedDataBlobHeader { - head: DataBlobHeader { - magic: file_formats::ENCR_COMPR_BLOB_MAGIC_1_0, - crc: crc.to_le_bytes(), - }, - iv, - tag, - }; - writer.seek(SeekFrom::Start(0))?; - unsafe { - writer.write_le_value(head)?; - } - Ok(writer) - } - } - } -} - -impl Write for DataBlobWriter<'_, W> { - fn write(&mut self, buf: &[u8]) -> Result { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.write(buf), - BlobWriterState::Compressed { ref mut compr } => compr.write(buf), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.write(buf), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.write(buf), - } - } - - fn flush(&mut self) -> Result<(), std::io::Error> { - match self.state { - BlobWriterState::Uncompressed { - ref mut csum_writer, - } => csum_writer.flush(), - BlobWriterState::Compressed { ref mut compr } => compr.flush(), - BlobWriterState::Encrypted { - ref mut crypt_writer, - } => crypt_writer.flush(), - BlobWriterState::EncryptedCompressed { ref mut compr } => compr.flush(), - } - } -} diff --git a/pbs-datastore/src/lib.rs b/pbs-datastore/src/lib.rs index 3e4aa34c..202b0955 100644 --- a/pbs-datastore/src/lib.rs +++ b/pbs-datastore/src/lib.rs @@ -179,7 +179,6 @@ pub mod crypt_reader; pub mod crypt_writer; pub mod data_blob; pub mod data_blob_reader; -pub mod data_blob_writer; pub mod file_formats; pub mod index; pub mod manifest; @@ -201,7 +200,6 @@ pub use crypt_reader::CryptReader; pub use crypt_writer::CryptWriter; pub use data_blob::DataBlob; pub use data_blob_reader::DataBlobReader; -pub use data_blob_writer::DataBlobWriter; pub use manifest::BackupManifest; pub use store_progress::StoreProgress; diff --git a/tests/blob_writer.rs b/tests/blob_writer.rs deleted file mode 100644 index 23a3283d..00000000 --- a/tests/blob_writer.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::io::Cursor; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::sync::Arc; - -use anyhow::{bail, Error}; -use lazy_static::lazy_static; - -use pbs_datastore::{DataBlob, DataBlobReader, DataBlobWriter}; -use pbs_tools::crypt_config::CryptConfig; - -lazy_static! { - static ref TEST_DATA: Vec = { - let mut data = Vec::new(); - - for i in 0..100_000 { - data.push((i % 255) as u8); - } - - data - }; - static ref CRYPT_CONFIG: Arc = { - let key = [1u8; 32]; - Arc::new(CryptConfig::new(key).unwrap()) - }; - static ref TEST_DIGEST_PLAIN: [u8; 32] = [ - 83, 154, 96, 195, 167, 204, 38, 142, 204, 224, 130, 201, 24, 71, 2, 188, 130, 155, 177, 6, - 162, 100, 61, 238, 38, 219, 63, 240, 191, 132, 87, 238 - ]; - static ref TEST_DIGEST_ENC: [u8; 32] = [ - 50, 162, 191, 93, 255, 132, 9, 14, 127, 23, 92, 39, 246, 102, 245, 204, 130, 104, 4, 106, - 182, 239, 218, 14, 80, 17, 150, 188, 239, 253, 198, 117 - ]; -} - -fn verify_test_blob(mut cursor: Cursor>, digest: &[u8; 32]) -> Result<(), Error> { - // run read tests with different buffer sizes - for size in [1, 3, 64 * 1024].iter() { - println!("Starting DataBlobReader test (size = {})", size); - - cursor.seek(SeekFrom::Start(0))?; - let mut reader = DataBlobReader::new(&mut cursor, Some(CRYPT_CONFIG.clone()))?; - let mut buffer = Vec::::new(); - // read the whole file - //reader.read_to_end(&mut buffer)?; - let mut buf = vec![0u8; *size]; - loop { - let count = reader.read(&mut buf)?; - if count == 0 { - break; - } - buffer.extend(&buf[..count]); - } - - reader.finish()?; - if buffer != *TEST_DATA { - bail!("blob data is wrong (read buffer size {})", size); - } - } - - let raw_data = cursor.into_inner(); - - let blob = DataBlob::load_from_reader(&mut &raw_data[..])?; - - let data = blob.decode(Some(&CRYPT_CONFIG), Some(digest))?; - if data != *TEST_DATA { - bail!("blob data is wrong (decode)"); - } - Ok(()) -} - -#[test] -fn test_uncompressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_uncompressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_compressed(tmp)?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_PLAIN) -} - -#[test] -fn test_encrypted_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -} - -#[test] -fn test_encrypted_compressed_blob_writer() -> Result<(), Error> { - let tmp = Cursor::new(Vec::::new()); - let mut blob_writer = DataBlobWriter::new_encrypted_compressed(tmp, CRYPT_CONFIG.clone())?; - blob_writer.write_all(&TEST_DATA)?; - - verify_test_blob(blob_writer.finish()?, &TEST_DIGEST_ENC) -} -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel