From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 85891BC02D for ; Thu, 28 Mar 2024 13:37:42 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8FB919E15 for ; Thu, 28 Mar 2024 13:37:39 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 28 Mar 2024 13:37:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 1D8AD4293D for ; Thu, 28 Mar 2024 13:37:38 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 28 Mar 2024 13:36:21 +0100 Message-Id: <20240328123707.336951-13-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240328123707.336951-1-c.ebner@proxmox.com> References: <20240328123707.336951-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.031 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v3 pxar 12/58] format: add payload stream start marker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 28 Mar 2024 12:37:42 -0000 Mark the beginning of the payload stream with a magic number. Allows for version and file type detection. Signed-off-by: Christian Ebner --- changes since version 2: - not present in previous version examples/mk-format-hashes.rs | 5 +++++ src/decoder/mod.rs | 17 +++++++++++++++-- src/encoder/mod.rs | 18 +++++++++++------- src/format/mod.rs | 2 ++ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index de73df0..35cff99 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -56,6 +56,11 @@ const CONSTANTS: &[(&str, &str, &str)] = &[ "PXAR_GOODBYE_TAIL_MARKER", "__PROXMOX_FORMAT_PXAR_GOODBYE_TAIL_MARKER__", ), + ( + "The start marker used in the separate payload stream", + "PXAR_PAYLOAD_START_MARKER", + "__PROXMOX_FORMAT_PXAR_PAYLOAD_START_MARKER__", + ), ( "The end marker used in the separate payload stream", "PXAR_PAYLOAD_TAIL_MARKER", diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 8cc4877..00d9abf 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -213,8 +213,21 @@ impl DecoderImpl { input: I, path: PathBuf, eof_after_entry: bool, - payload_input: Option, + mut payload_input: Option, ) -> io::Result { + let payload_consumed = if let Some(payload_input) = payload_input.as_mut() { + let header: Header = seq_read_entry(payload_input).await?; + if header.htype != format::PXAR_PAYLOAD_START_MARKER { + io_bail!( + "unexpected header in payload input: expected {:#x?} , got {header:#x?}", + format::PXAR_PAYLOAD_START_MARKER, + ); + } + header.full_size() + } else { + 0 + }; + let this = DecoderImpl { input, current_header: unsafe { mem::zeroed() }, @@ -227,7 +240,7 @@ impl DecoderImpl { state: State::Begin, with_goodbye_tables: false, payload_input, - payload_consumed: 0, + payload_consumed, eof_after_entry, }; diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 24dcbc9..88c0ed5 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -313,15 +313,23 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { pub async fn new( output: EncoderOutput<'a, T>, metadata: &Metadata, - payload_output: Option, + mut payload_output: Option, ) -> io::Result> { if !metadata.is_dir() { io_bail!("directory metadata must contain the directory mode flag"); } + + let mut state = EncoderState::default(); + if let Some(payload_output) = payload_output.as_mut() { + let header = format::Header::with_content_size(format::PXAR_PAYLOAD_START_MARKER, 0); + header.check_header_size()?; + seq_write_struct(payload_output, header, &mut state.payload_write_position).await?; + } + let mut this = Self { output, - payload_output: EncoderOutput::Owned(None), - state: vec![EncoderState::default()], + payload_output: EncoderOutput::Owned(payload_output), + state: vec![state], finished: false, file_copy_buffer: Arc::new(Mutex::new(unsafe { crate::util::vec_new_uninitialized(1024 * 1024) @@ -332,10 +340,6 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { let state = this.state_mut()?; state.files_offset = state.position(); - if let Some(payload_output) = payload_output { - this.payload_output = EncoderOutput::Owned(Some(payload_output)); - } - Ok(this) } diff --git a/src/format/mod.rs b/src/format/mod.rs index 10192e7..a672d19 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -106,6 +106,8 @@ pub const PXAR_PAYLOAD_REF: u64 = 0x419d3d6bc4ba977e; pub const PXAR_GOODBYE: u64 = 0x2fec4fa642d5731d; /// The end marker used in the GOODBYE object pub const PXAR_GOODBYE_TAIL_MARKER: u64 = 0xef5eed5b753e1555; +/// The start marker used in the separate payload stream +pub const PXAR_PAYLOAD_START_MARKER: u64 = 0x834c68c2194a4ed2; /// The end marker used in the separate payload stream pub const PXAR_PAYLOAD_TAIL_MARKER: u64 = 0x6c72b78b984c81b5; -- 2.39.2