From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 80A371FF38E for ; Tue, 28 May 2024 11:43:41 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 01179125DC; Tue, 28 May 2024 11:44:03 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Tue, 28 May 2024 11:42:07 +0200 Message-Id: <20240528094303.309806-14-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240528094303.309806-1-c.ebner@proxmox.com> References: <20240528094303.309806-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.029 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v8 pxar 13/69] format: add payload stream start marker X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Mark the beginning of the payload stream with a magic number. Allows for version and file type detection. Signed-off-by: Christian Ebner --- changes since version 7: - no changes changes since version 6: - no changes examples/mk-format-hashes.rs | 5 +++++ src/accessor/mod.rs | 2 +- src/decoder/mod.rs | 28 +++++++++++++++++++--------- src/encoder/mod.rs | 12 ++++++++++-- src/format/mod.rs | 2 ++ 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index de73df0..35cff99 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -56,6 +56,11 @@ const CONSTANTS: &[(&str, &str, &str)] = &[ "PXAR_GOODBYE_TAIL_MARKER", "__PROXMOX_FORMAT_PXAR_GOODBYE_TAIL_MARKER__", ), + ( + "The start marker used in the separate payload stream", + "PXAR_PAYLOAD_START_MARKER", + "__PROXMOX_FORMAT_PXAR_PAYLOAD_START_MARKER__", + ), ( "The end marker used in the separate payload stream", "PXAR_PAYLOAD_TAIL_MARKER", diff --git a/src/accessor/mod.rs b/src/accessor/mod.rs index 51e846a..c3a5e14 100644 --- a/src/accessor/mod.rs +++ b/src/accessor/mod.rs @@ -238,7 +238,7 @@ async fn get_decoder( |input| SeqReadAtAdapter::new(input, entry_range.clone()), |(payload_input, range)| SeqReadAtAdapter::new(payload_input, range), ); - DecoderImpl::new_full(input, path, true).await + DecoderImpl::new_full(input, path, true, 0).await } // NOTE: This performs the Decoder::read_next_item() behavior! Keep in mind when changing! diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index 0fc3698..19b1b5c 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -205,8 +205,21 @@ pub(crate) enum ItemResult { } impl DecoderImpl { - pub async fn new(input: PxarVariant) -> io::Result { - Self::new_full(input, "/".into(), false).await + pub async fn new(mut input: PxarVariant) -> io::Result { + let payload_consumed = if let Some(payload_input) = input.payload_mut() { + let header: Header = seq_read_entry(payload_input).await?; + if header.htype != format::PXAR_PAYLOAD_START_MARKER { + io_bail!( + "unexpected header in payload input: expected {:#x?} , got {header:#x?}", + format::PXAR_PAYLOAD_START_MARKER, + ); + } + header.full_size() + } else { + 0 + }; + + Self::new_full(input, "/".into(), false, payload_consumed).await } pub(crate) fn input(&self) -> &I { @@ -217,8 +230,9 @@ impl DecoderImpl { input: PxarVariant, path: PathBuf, eof_after_entry: bool, + payload_consumed: u64, ) -> io::Result { - let this = DecoderImpl { + Ok(DecoderImpl { input, current_header: unsafe { mem::zeroed() }, entry: Entry { @@ -229,13 +243,9 @@ impl DecoderImpl { path_lengths: Vec::new(), state: State::Begin, with_goodbye_tables: false, - payload_consumed: 0, + payload_consumed, eof_after_entry, - }; - - // this.read_next_entry().await?; - - Ok(this) + }) } /// Get the next file entry, recursing into directories. diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index 54e147d..b579e18 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -342,15 +342,23 @@ impl<'a, T: SeqWrite + 'a> Drop for EncoderImpl<'a, T> { impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { pub async fn new( - output: PxarVariant, T>, + mut output: PxarVariant, T>, metadata: &Metadata, ) -> io::Result> { if !metadata.is_dir() { io_bail!("directory metadata must contain the directory mode flag"); } + + let mut state = EncoderState::default(); + if let Some(payload_output) = output.payload_mut() { + let header = format::Header::with_content_size(format::PXAR_PAYLOAD_START_MARKER, 0); + header.check_header_size()?; + seq_write_struct(payload_output, header, &mut state.payload_write_position).await?; + } + let mut this = Self { output, - state: vec![EncoderState::default()], + state: vec![state], finished: false, file_copy_buffer: Arc::new(Mutex::new(unsafe { crate::util::vec_new_uninitialized(1024 * 1024) diff --git a/src/format/mod.rs b/src/format/mod.rs index e451b0f..6519bfc 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -106,6 +106,8 @@ pub const PXAR_PAYLOAD_REF: u64 = 0x419d3d6bc4ba977e; pub const PXAR_GOODBYE: u64 = 0x2fec4fa642d5731d; /// The end marker used in the GOODBYE object pub const PXAR_GOODBYE_TAIL_MARKER: u64 = 0xef5eed5b753e1555; +/// The start marker used in the separate payload stream +pub const PXAR_PAYLOAD_START_MARKER: u64 = 0x834c68c2194a4ed2; /// The end marker used in the separate payload stream pub const PXAR_PAYLOAD_TAIL_MARKER: u64 = 0x6c72b78b984c81b5; -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel