From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 1F54A1FF389 for ; Tue, 7 May 2024 17:53:18 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 9D02F11C30; Tue, 7 May 2024 17:53:17 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 7 May 2024 17:51:54 +0200 Message-Id: <20240507155244.793819-13-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240507155244.793819-1-c.ebner@proxmox.com> References: <20240507155244.793819-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.028 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v5 pxar 12/62] format: add payload stream start marker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Mark the beginning of the payload stream with a magic number. Allows for version and file type detection. Signed-off-by: Christian Ebner --- changes since version 4: - no changes examples/mk-format-hashes.rs | 5 +++++ src/accessor/mod.rs | 1 + src/decoder/mod.rs | 28 +++++++++++++++++++--------- src/encoder/mod.rs | 18 +++++++++++------- src/format/mod.rs | 2 ++ 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index de73df0..35cff99 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -56,6 +56,11 @@ const CONSTANTS: &[(&str, &str, &str)] = &[ "PXAR_GOODBYE_TAIL_MARKER", "__PROXMOX_FORMAT_PXAR_GOODBYE_TAIL_MARKER__", ), + ( + "The start marker used in the separate payload stream", + "PXAR_PAYLOAD_START_MARKER", + "__PROXMOX_FORMAT_PXAR_PAYLOAD_START_MARKER__", + ), ( "The end marker used in the separate payload stream", "PXAR_PAYLOAD_TAIL_MARKER", diff --git a/src/accessor/mod.rs b/src/accessor/mod.rs index fadd3d2..9048c94 100644 --- a/src/accessor/mod.rs +++ b/src/accessor/mod.rs @@ -239,6 +239,7 @@ async fn get_decoder( path, true, payload_input.map(|(input, range)| SeqReadAtAdapter::new(input, range)), + 0, ) .await } diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 7c5cc12..3bca835 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -206,8 +206,21 @@ pub(crate) enum ItemResult { } impl DecoderImpl { - pub async fn new(input: I, payload_input: Option) -> io::Result { - Self::new_full(input, "/".into(), false, payload_input).await + pub async fn new(input: I, mut payload_input: Option) -> io::Result { + let payload_consumed = if let Some(payload_input) = payload_input.as_mut() { + let header: Header = seq_read_entry(payload_input).await?; + if header.htype != format::PXAR_PAYLOAD_START_MARKER { + io_bail!( + "unexpected header in payload input: expected {:#x?} , got {header:#x?}", + format::PXAR_PAYLOAD_START_MARKER, + ); + } + header.full_size() + } else { + 0 + }; + + Self::new_full(input, "/".into(), false, payload_input, payload_consumed).await } pub(crate) fn input(&self) -> &I { @@ -219,8 +232,9 @@ impl DecoderImpl { path: PathBuf, eof_after_entry: bool, payload_input: Option, + payload_consumed: u64, ) -> io::Result { - let this = DecoderImpl { + Ok(DecoderImpl { input, current_header: unsafe { mem::zeroed() }, entry: Entry { @@ -232,13 +246,9 @@ impl DecoderImpl { state: State::Begin, with_goodbye_tables: false, payload_input, - payload_consumed: 0, + payload_consumed, eof_after_entry, - }; - - // this.read_next_entry().await?; - - Ok(this) + }) } /// Get the next file entry, recursing into directories. diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index cf16449..ea82eef 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -345,15 +345,23 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { pub async fn new( output: EncoderOutput<'a, T>, metadata: &Metadata, - payload_output: Option, + mut payload_output: Option, ) -> io::Result> { if !metadata.is_dir() { io_bail!("directory metadata must contain the directory mode flag"); } + + let mut state = EncoderState::default(); + if let Some(payload_output) = payload_output.as_mut() { + let header = format::Header::with_content_size(format::PXAR_PAYLOAD_START_MARKER, 0); + header.check_header_size()?; + seq_write_struct(payload_output, header, &mut state.payload_write_position).await?; + } + let mut this = Self { output, - payload_output: EncoderOutput::Owned(None), - state: vec![EncoderState::default()], + payload_output: EncoderOutput::Owned(payload_output), + state: vec![state], finished: false, file_copy_buffer: Arc::new(Mutex::new(unsafe { crate::util::vec_new_uninitialized(1024 * 1024) @@ -364,10 +372,6 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { let state = this.state_mut()?; state.files_offset = state.position(); - if let Some(payload_output) = payload_output { - this.payload_output = EncoderOutput::Owned(Some(payload_output)); - } - Ok(this) } diff --git a/src/format/mod.rs b/src/format/mod.rs index e451b0f..6519bfc 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -106,6 +106,8 @@ pub const PXAR_PAYLOAD_REF: u64 = 0x419d3d6bc4ba977e; pub const PXAR_GOODBYE: u64 = 0x2fec4fa642d5731d; /// The end marker used in the GOODBYE object pub const PXAR_GOODBYE_TAIL_MARKER: u64 = 0xef5eed5b753e1555; +/// The start marker used in the separate payload stream +pub const PXAR_PAYLOAD_START_MARKER: u64 = 0x834c68c2194a4ed2; /// The end marker used in the separate payload stream pub const PXAR_PAYLOAD_TAIL_MARKER: u64 = 0x6c72b78b984c81b5; -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel