From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id A595B90C43 for ; Thu, 25 Jan 2024 14:26:58 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 81E901997D for ; Thu, 25 Jan 2024 14:26:28 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Thu, 25 Jan 2024 14:26:26 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 7FB5B492C0 for ; Thu, 25 Jan 2024 14:26:26 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Thu, 25 Jan 2024 14:25:47 +0100 Message-Id: <20240125132608.1172472-9-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240125132608.1172472-1-c.ebner@proxmox.com> References: <20240125132608.1172472-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.052 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v6 pxar 8/29] fix #3174: add pxar format version entry X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 25 Jan 2024 13:26:58 -0000 Adds an additional entry type at the start of each pxar archive signaling the encoding format version. If not present, the default version 1 is assumed. This allows to early on detect the pxar encoding version, allowing tools to switch mode or bail on non compatible encoder/decoder functionality. Signed-off-by: Christian Ebner --- Changes since v5: - pxar archives in version 2 are prefixed by an entry kind version, allowing encoders/decoders to early on detect the used format version. examples/mk-format-hashes.rs | 5 +++++ src/decoder/mod.rs | 32 +++++++++++++++++++++++++++++++- src/encoder/aio.rs | 23 ++++++++++++++++------- src/encoder/mod.rs | 32 ++++++++++++++++++++++++++++++++ src/encoder/sync.rs | 7 ++++--- src/format/mod.rs | 11 ++++++++++- src/lib.rs | 3 +++ 7 files changed, 101 insertions(+), 12 deletions(-) diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs index 7fb938d..8a43d81 100644 --- a/examples/mk-format-hashes.rs +++ b/examples/mk-format-hashes.rs @@ -1,6 +1,11 @@ use pxar::format::hash_filename; const CONSTANTS: &[(&str, &str, &str)] = &[ + ( + "Pxar format version entry, fallback to version 1 if not present", + "PXAR_FORMAT_VERSION", + "__PROXMOX_FORMAT_VERSION__", + ), ( "Beginning of an entry (current version).", "PXAR_ENTRY", diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs index a0f322e..ec2f079 100644 --- a/src/decoder/mod.rs +++ b/src/decoder/mod.rs @@ -160,6 +160,7 @@ pub(crate) struct DecoderImpl { /// The random access code uses decoders for sub-ranges which may not end in a `PAYLOAD` for /// entries like FIFOs or sockets, so there we explicitly allow an item to terminate with EOF. eof_after_entry: bool, + version: format::FormatVersion, } enum State { @@ -220,6 +221,7 @@ impl DecoderImpl { state: State::Begin, with_goodbye_tables: false, eof_after_entry, + version: format::FormatVersion::default(), }; // this.read_next_entry().await?; @@ -236,7 +238,16 @@ impl DecoderImpl { loop { match self.state { State::Eof => return Ok(None), - State::Begin => return self.read_next_entry().await.map(Some), + State::Begin => { + let entry = self.read_next_entry().await.map(Some); + if let Ok(Some(ref entry)) = entry { + if let EntryKind::Version(version) = entry.kind() { + self.version = version.clone(); + return self.read_next_entry().await.map(Some); + } + } + return entry; + } State::Default => { // we completely finished an entry, so now we're going "up" in the directory // hierarchy and parse the next PXAR_FILENAME or the PXAR_GOODBYE: @@ -277,6 +288,9 @@ impl DecoderImpl { match self.current_header.htype { format::PXAR_FILENAME => return self.handle_file_entry().await, format::PXAR_APPENDIX_REF => { + if self.version == format::FormatVersion::Version1 { + io_bail!("appendix reference not supported in pxar format version 1"); + } self.state = State::Default; return self.handle_appendix_ref_entry().await; } @@ -296,6 +310,9 @@ impl DecoderImpl { } } format::PXAR_APPENDIX => { + if self.version == format::FormatVersion::Version1 { + io_bail!("appendix not supported in pxar format version 1"); + } self.state = State::Default; return Ok(Some(self.entry.take())); } @@ -397,6 +414,11 @@ impl DecoderImpl { self.entry.metadata = Metadata::default(); self.entry.kind = EntryKind::Hardlink(self.read_hardlink().await?); + Ok(Some(self.entry.take())) + } else if header.htype == format::PXAR_FORMAT_VERSION { + self.current_header = header; + self.entry.kind = EntryKind::Version(self.read_format_version().await?); + Ok(Some(self.entry.take())) } else if header.htype == format::PXAR_ENTRY || header.htype == format::PXAR_ENTRY_V1 { if header.htype == format::PXAR_ENTRY { @@ -696,6 +718,14 @@ impl DecoderImpl { async fn read_quota_project_id(&mut self) -> io::Result { self.read_simple_entry("quota project id").await } + + async fn read_format_version(&mut self) -> io::Result { + match seq_read_entry(&mut self.input).await? { + 1u64 => Ok(format::FormatVersion::Version1), + 2u64 => Ok(format::FormatVersion::Version2), + _ => io_bail!("unexpected pxar format version"), + } + } } /// Reader for file contents inside a pxar archive. diff --git a/src/encoder/aio.rs b/src/encoder/aio.rs index 5a833c5..8f586b6 100644 --- a/src/encoder/aio.rs +++ b/src/encoder/aio.rs @@ -24,8 +24,9 @@ impl<'a, T: tokio::io::AsyncWrite + 'a> Encoder<'a, TokioWriter> { pub async fn from_tokio( output: T, metadata: &Metadata, + version: format::FormatVersion, ) -> io::Result>> { - Encoder::new(TokioWriter::new(output), metadata).await + Encoder::new(TokioWriter::new(output), metadata, version).await } } @@ -46,9 +47,13 @@ impl<'a> Encoder<'a, TokioWriter> { impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { /// Create an asynchronous encoder for an output implementing our internal write interface. - pub async fn new(output: T, metadata: &Metadata) -> io::Result> { + pub async fn new( + output: T, + metadata: &Metadata, + version: format::FormatVersion, + ) -> io::Result> { Ok(Self { - inner: encoder::EncoderImpl::new(output.into(), metadata).await?, + inner: encoder::EncoderImpl::new(output.into(), metadata, version).await?, }) } @@ -299,7 +304,7 @@ mod test { use std::task::{Context, Poll}; use super::Encoder; - use crate::Metadata; + use crate::{format, Metadata}; struct DummyOutput; @@ -321,9 +326,13 @@ mod test { /// Assert that `Encoder` is `Send` fn send_test() { let test = async { - let mut encoder = Encoder::new(DummyOutput, &Metadata::dir_builder(0o700).build()) - .await - .unwrap(); + let mut encoder = Encoder::new( + DummyOutput, + &Metadata::dir_builder(0o700).build(), + format::FormatVersion::default(), + ) + .await + .unwrap(); { let mut dir = encoder .create_directory("baba", &Metadata::dir_builder(0o700).build()) diff --git a/src/encoder/mod.rs b/src/encoder/mod.rs index c33b2c3..7e18dc8 100644 --- a/src/encoder/mod.rs +++ b/src/encoder/mod.rs @@ -247,6 +247,7 @@ pub async fn encoded_size(filename: &std::ffi::CStr, metadata: &Metadata) -> io: file_copy_buffer: Arc::new(Mutex::new(unsafe { crate::util::vec_new_uninitialized(1024 * 1024) })), + version: format::FormatVersion::default(), }; this.start_file_do(Some(metadata), filename.to_bytes()) @@ -356,6 +357,8 @@ pub(crate) struct EncoderImpl<'a, T: SeqWrite + 'a> { /// Since only the "current" entry can be actively writing files, we share the file copy /// buffer. file_copy_buffer: Arc>>, + /// Pxar format version to encode + version: format::FormatVersion, } impl<'a, T: SeqWrite + 'a> Drop for EncoderImpl<'a, T> { @@ -377,6 +380,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { pub async fn new( output: EncoderOutput<'a, T>, metadata: &Metadata, + version: format::FormatVersion, ) -> io::Result> { if !metadata.is_dir() { io_bail!("directory metadata must contain the directory mode flag"); @@ -389,8 +393,10 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { file_copy_buffer: Arc::new(Mutex::new(unsafe { crate::util::vec_new_uninitialized(1024 * 1024) })), + version, }; + this.encode_format_version().await?; this.encode_metadata(metadata).await?; this.state.files_offset = this.position(); @@ -509,6 +515,9 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { appendix_ref_offset: AppendixRefOffset, file_size: u64, ) -> io::Result<()> { + if self.version == format::FormatVersion::Version1 { + io_bail!("unable to add appendix reference for pxar format version 1"); + } self.check()?; let offset = self.position(); @@ -544,6 +553,9 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { &mut self, full_size: AppendixRefOffset, ) -> io::Result { + if self.version == format::FormatVersion::Version1 { + io_bail!("unable to add appendix for pxar format version 1"); + } self.check()?; let data = &full_size.raw().to_le_bytes().to_vec(); @@ -740,6 +752,7 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { parent: Some(&mut self.state), finished: false, file_copy_buffer, + version: self.version.clone(), }) } @@ -755,6 +768,25 @@ impl<'a, T: SeqWrite + 'a> EncoderImpl<'a, T> { Ok(()) } + async fn encode_format_version(&mut self) -> io::Result<()> { + if self.state.write_position != 0 { + io_bail!("pxar format version must be encoded at the beginning of an archive"); + } + + let version_bytes = match self.version { + format::FormatVersion::Version1 => return Ok(()), + format::FormatVersion::Version2 => 2u64.to_le_bytes(), + }; + + seq_write_pxar_entry( + self.output.as_mut(), + format::PXAR_FORMAT_VERSION, + &version_bytes, + &mut self.state.write_position, + ) + .await + } + async fn encode_metadata(&mut self, metadata: &Metadata) -> io::Result<()> { seq_write_pxar_struct_entry( self.output.as_mut(), diff --git a/src/encoder/sync.rs b/src/encoder/sync.rs index 5ede554..8516662 100644 --- a/src/encoder/sync.rs +++ b/src/encoder/sync.rs @@ -28,7 +28,7 @@ impl<'a, T: io::Write + 'a> Encoder<'a, StandardWriter> { /// Encode a `pxar` archive into a regular `std::io::Write` output. #[inline] pub fn from_std(output: T, metadata: &Metadata) -> io::Result>> { - Encoder::new(StandardWriter::new(output), metadata) + Encoder::new(StandardWriter::new(output), metadata, format::FormatVersion::default()) } } @@ -41,6 +41,7 @@ impl<'a> Encoder<'a, StandardWriter> { Encoder::new( StandardWriter::new(std::fs::File::create(path.as_ref())?), metadata, + format::FormatVersion::default(), ) } } @@ -50,9 +51,9 @@ impl<'a, T: SeqWrite + 'a> Encoder<'a, T> { /// /// Note that the `output`'s `SeqWrite` implementation must always return `Poll::Ready` and is /// not allowed to use the `Waker`, as this will cause a `panic!`. - pub fn new(output: T, metadata: &Metadata) -> io::Result { + pub fn new(output: T, metadata: &Metadata, version: format::FormatVersion) -> io::Result { Ok(Self { - inner: poll_result_once(encoder::EncoderImpl::new(output.into(), metadata))?, + inner: poll_result_once(encoder::EncoderImpl::new(output.into(), metadata, version))?, }) } diff --git a/src/format/mod.rs b/src/format/mod.rs index 8016ab1..5154a0a 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -44,7 +44,7 @@ //! * final goodbye table //! * `APPENDIX_TAIL` -- marks the end of an archive containing a APPENDIX section -use std::cmp::Ordering; +use std::cmp::{Ordering, PartialEq}; use std::ffi::{CStr, OsStr}; use std::fmt; use std::fmt::Display; @@ -88,6 +88,8 @@ pub mod mode { } // Generated by `cargo run --example mk-format-hashes` +/// Pxar format version entry, fallback to version 1 if not present +pub const PXAR_FORMAT_VERSION: u64 = 0x730f6c75df16a40d; /// Beginning of an entry (current version). pub const PXAR_ENTRY: u64 = 0xd5956474e588acef; /// Previous version of the entry struct @@ -561,6 +563,13 @@ pub struct Filename { pub name: Vec, } +#[derive(Clone, Debug, Default, PartialEq)] +pub enum FormatVersion { + #[default] + Version1, + Version2, +} + #[derive(Clone, Debug)] pub struct Symlink { pub data: Vec, diff --git a/src/lib.rs b/src/lib.rs index 035f995..a08055e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -342,6 +342,9 @@ impl Acl { /// Identifies whether the entry is a file, symlink, directory, etc. #[derive(Clone, Debug)] pub enum EntryKind { + /// Pxar file format version + Version(format::FormatVersion), + /// Symbolic links. Symlink(format::Symlink), -- 2.39.2