* [pbs-devel] [PATCH stable-2 pxar 1/2] format/decoder/accessor: backport pxar entry type `Version`
2024-05-29 15:57 [pbs-devel] [PATCH stable-2 pxar proxmox-backup 0/2] backport pxar format Christian Ebner
@ 2024-05-29 15:57 ` Christian Ebner
2024-05-29 15:57 ` [pbs-devel] [PATCH stable-2 proxmox-backup 2/2] client: pxar: bail on incompatible format versions Christian Ebner
2024-06-05 15:42 ` [pbs-devel] [PATCH stable-2 pxar proxmox-backup 0/2] backport pxar format Christian Ebner
2 siblings, 0 replies; 4+ messages in thread
From: Christian Ebner @ 2024-05-29 15:57 UTC (permalink / raw)
To: pbs-devel
Backports the pxar format entry type `Version` and the associated
decoder methods. The format version entry is expected once as the
first entry of the pxar archive, marked with a `PXAR_FORMAT_VERSION`
header followed by the encoded version number for archives with
format version 2 or higher.
If not present, the default format version 1 is assumed as encoding
format for the archive.
The entry allows to early detect and bail if an incompatible archive
version is encountered.
The format version entry is not backwards compatible to pxar format
version 1.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
Note:
This patch is intended to be applied on a dedicated branch to be forked
from current master commit 675ecff32fbeff0973eaea016c4b8f3877015adb
examples/mk-format-hashes.rs | 5 +++++
src/accessor/mod.rs | 12 ++++++++++++
src/decoder/mod.rs | 29 ++++++++++++++++++++++++++++-
src/format/mod.rs | 21 +++++++++++++++++++++
src/lib.rs | 3 +++
tests/simple/fs.rs | 1 +
6 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/examples/mk-format-hashes.rs b/examples/mk-format-hashes.rs
index 6e00654..afd0924 100644
--- a/examples/mk-format-hashes.rs
+++ b/examples/mk-format-hashes.rs
@@ -1,6 +1,11 @@
use pxar::format::hash_filename;
const CONSTANTS: &[(&str, &str, &str)] = &[
+ (
+ "Pxar format version entry, fallback to version 1 if not present",
+ "PXAR_FORMAT_VERSION",
+ "__PROXMOX_FORMAT_VERSION__",
+ ),
(
"Beginning of an entry (current version).",
"PXAR_ENTRY",
diff --git a/src/accessor/mod.rs b/src/accessor/mod.rs
index 6a2de73..bf18773 100644
--- a/src/accessor/mod.rs
+++ b/src/accessor/mod.rs
@@ -293,6 +293,12 @@ impl<T: Clone + ReadAt> AccessorImpl<T> {
.next()
.await
.ok_or_else(|| io_format_err!("unexpected EOF while decoding file entry"))??;
+
+ if let EntryKind::Version(version) = entry.kind() {
+ // client is incompatible with any format version entry (version 1 is never encoded)
+ io_bail!("got format version {version}, not compatible with this client.");
+ }
+
Ok(FileEntryImpl {
input: self.input.clone(),
entry,
@@ -516,6 +522,12 @@ impl<T: Clone + ReadAt> DirectoryImpl<T> {
.next()
.await
.ok_or_else(|| io_format_err!("unexpected EOF while decoding directory entry"))??;
+
+ if let EntryKind::Version(version) = entry.kind() {
+ // client is incompatible with any format version entry (version 1 is never encoded)
+ io_bail!("got format version {version}, not compatible with this client.");
+ }
+
Ok((entry, decoder))
}
diff --git a/src/decoder/mod.rs b/src/decoder/mod.rs
index d1fb911..4d0957c 100644
--- a/src/decoder/mod.rs
+++ b/src/decoder/mod.rs
@@ -162,6 +162,7 @@ pub(crate) struct DecoderImpl<T> {
eof_after_entry: bool,
}
+#[derive(Clone, PartialEq)]
enum State {
Begin,
Default,
@@ -236,7 +237,16 @@ impl<I: SeqRead> DecoderImpl<I> {
loop {
match self.state {
State::Eof => return Ok(None),
- State::Begin => return self.read_next_entry().await.map(Some),
+ State::Begin => {
+ let entry = self.read_next_entry().await.map(Some);
+ if let Ok(Some(ref entry)) = entry {
+ if let EntryKind::Version(version) = entry.kind() {
+ // client is incompatible with any format version entry (version 1 is never encoded)
+ io_bail!("got format version {version}, not compatible with this client.");
+ }
+ }
+ return entry;
+ }
State::Default => {
// we completely finished an entry, so now we're going "up" in the directory
// hierarchy and parse the next PXAR_FILENAME or the PXAR_GOODBYE:
@@ -354,6 +364,7 @@ impl<I: SeqRead> DecoderImpl<I> {
}
async fn read_next_entry_or_eof(&mut self) -> io::Result<Option<Entry>> {
+ let previous_state = self.state.clone();
self.state = State::Default;
self.entry.clear_data();
@@ -373,6 +384,14 @@ impl<I: SeqRead> DecoderImpl<I> {
self.entry.metadata = Metadata::default();
self.entry.kind = EntryKind::Hardlink(self.read_hardlink().await?);
+ Ok(Some(self.entry.take()))
+ } else if header.htype == format::PXAR_FORMAT_VERSION {
+ if previous_state != State::Begin {
+ io_bail!("Got format version entry at unexpected position");
+ }
+ self.current_header = header;
+ self.entry.kind = EntryKind::Version(self.read_format_version().await?);
+
Ok(Some(self.entry.take()))
} else if header.htype == format::PXAR_ENTRY || header.htype == format::PXAR_ENTRY_V1 {
if header.htype == format::PXAR_ENTRY {
@@ -661,6 +680,14 @@ impl<I: SeqRead> DecoderImpl<I> {
async fn read_quota_project_id(&mut self) -> io::Result<format::QuotaProjectId> {
self.read_simple_entry("quota project id").await
}
+
+ async fn read_format_version(&mut self) -> io::Result<format::FormatVersion> {
+ match seq_read_entry(&mut self.input).await? {
+ 1u64 => Ok(format::FormatVersion::Version1),
+ 2u64 => Ok(format::FormatVersion::Version2),
+ version => io_bail!("unexpected pxar format version {version}"),
+ }
+ }
}
/// Reader for file contents inside a pxar archive.
diff --git a/src/format/mod.rs b/src/format/mod.rs
index bfea9f6..980bad6 100644
--- a/src/format/mod.rs
+++ b/src/format/mod.rs
@@ -6,6 +6,7 @@
//! item data.
//!
//! An archive contains items in the following order:
+//! * `FORMAT_VERSION` -- (optional for v1), version of encoding format
//! * `ENTRY` -- containing general stat() data and related bits
//! * `XATTR` -- one extended attribute
//! * ... -- more of these when there are multiple defined
@@ -79,6 +80,8 @@ pub mod mode {
}
// Generated by `cargo run --example mk-format-hashes`
+/// Pxar format version entry, fallback to version 1 if not present
+pub const PXAR_FORMAT_VERSION: u64 = 0x730f6c75df16a40d;
/// Beginning of an entry (current version).
pub const PXAR_ENTRY: u64 = 0xd5956474e588acef;
/// Previous version of the entry struct
@@ -177,6 +180,7 @@ impl Header {
impl Display for Header {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let readable = match self.htype {
+ PXAR_FORMAT_VERSION => "FORMAT_VERSION",
PXAR_FILENAME => "FILENAME",
PXAR_SYMLINK => "SYMLINK",
PXAR_HARDLINK => "HARDLINK",
@@ -540,6 +544,23 @@ impl From<&std::fs::Metadata> for Stat {
}
}
+#[derive(Clone, Debug, Default, PartialEq)]
+pub enum FormatVersion {
+ #[default]
+ Version1,
+ Version2,
+}
+
+impl Display for FormatVersion {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let version = match self {
+ FormatVersion::Version1 => "1",
+ FormatVersion::Version2 => "2",
+ };
+ write!(f, "{version}")
+ }
+}
+
#[derive(Clone, Debug)]
pub struct Filename {
pub name: Vec<u8>,
diff --git a/src/lib.rs b/src/lib.rs
index 210c4b1..b63d43c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -342,6 +342,9 @@ impl Acl {
/// Identifies whether the entry is a file, symlink, directory, etc.
#[derive(Clone, Debug)]
pub enum EntryKind {
+ /// Pxar file format version
+ Version(format::FormatVersion),
+
/// Symbolic links.
Symlink(format::Symlink),
diff --git a/tests/simple/fs.rs b/tests/simple/fs.rs
index 9a89c4d..fd13e65 100644
--- a/tests/simple/fs.rs
+++ b/tests/simple/fs.rs
@@ -229,6 +229,7 @@ impl Entry {
})?))
};
match item.kind() {
+ PxarEntryKind::Version(_) => continue,
PxarEntryKind::GoodbyeTable => break,
PxarEntryKind::File { size, .. } => {
let mut data = Vec::new();
--
2.30.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
^ permalink raw reply [flat|nested] 4+ messages in thread* [pbs-devel] [PATCH stable-2 proxmox-backup 2/2] client: pxar: bail on incompatible format versions
2024-05-29 15:57 [pbs-devel] [PATCH stable-2 pxar proxmox-backup 0/2] backport pxar format Christian Ebner
2024-05-29 15:57 ` [pbs-devel] [PATCH stable-2 pxar 1/2] format/decoder/accessor: backport pxar entry type `Version` Christian Ebner
@ 2024-05-29 15:57 ` Christian Ebner
2024-06-05 15:42 ` [pbs-devel] [PATCH stable-2 pxar proxmox-backup 0/2] backport pxar format Christian Ebner
2 siblings, 0 replies; 4+ messages in thread
From: Christian Ebner @ 2024-05-29 15:57 UTC (permalink / raw)
To: pbs-devel
Implementing the change detection mode with reusable file payloads
required a breaking change in the pxar file format. Unfortunately,
there initial format does not encode a version for clients to check
compatibility to. Therefore, a pxar format version entry is
introduced in version 2 of the file format.
Add a compatibility check based on that entry also for the client,
making it possible to bail early when an incompatible archive is
encountered.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-client/src/catalog_shell.rs | 2 +-
| 3 +++
pbs-client/src/pxar/tools.rs | 7 ++++---
3 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs
index 98af5699..8bb2f3e0 100644
--- a/pbs-client/src/catalog_shell.rs
+++ b/pbs-client/src/catalog_shell.rs
@@ -767,7 +767,7 @@ impl Shell {
let file = Self::walk_pxar_archive(&self.accessor, &mut stack).await?;
std::io::stdout()
- .write_all(crate::pxar::format_multi_line_entry(file.entry()).as_bytes())?;
+ .write_all(crate::pxar::format_multi_line_entry(file.entry())?.as_bytes())?;
Ok(())
}
--git a/pbs-client/src/pxar/extract.rs b/pbs-client/src/pxar/extract.rs
index f6c1991f..51b2c92c 100644
--- a/pbs-client/src/pxar/extract.rs
+++ b/pbs-client/src/pxar/extract.rs
@@ -119,6 +119,9 @@ where
None => current_match,
};
match (did_match, entry.kind()) {
+ (_, EntryKind::Version(version)) => {
+ bail!("got format version {version}, incompatible with this client");
+ }
(_, EntryKind::Directory) => {
callback(entry.path());
diff --git a/pbs-client/src/pxar/tools.rs b/pbs-client/src/pxar/tools.rs
index 844a0f73..f28289bd 100644
--- a/pbs-client/src/pxar/tools.rs
+++ b/pbs-client/src/pxar/tools.rs
@@ -146,12 +146,13 @@ pub fn format_single_line_entry(entry: &Entry) -> String {
)
}
-pub fn format_multi_line_entry(entry: &Entry) -> String {
+pub fn format_multi_line_entry(entry: &Entry) -> Result<String, Error> {
let mode_string = mode_string(entry);
let meta = entry.metadata();
let (size, link, type_name) = match entry.kind() {
+ EntryKind::Version(version) => bail!("got incompatible format version {version}"),
EntryKind::File { size, .. } => (format!("{}", *size), String::new(), "file"),
EntryKind::Symlink(link) => (
"0".to_string(),
@@ -185,7 +186,7 @@ pub fn format_multi_line_entry(entry: &Entry) -> String {
Err(_) => std::borrow::Cow::Owned(format!("{:?}", entry.path())),
};
- format!(
+ Ok(format!(
" File: {}{}\n \
Size: {:<13} Type: {}\n\
Access: ({:o}/{}) Uid: {:<5} Gid: {:<5}\n\
@@ -199,5 +200,5 @@ pub fn format_multi_line_entry(entry: &Entry) -> String {
meta.stat.uid,
meta.stat.gid,
format_mtime(&meta.stat.mtime),
- )
+ ))
}
--
2.30.2
_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
^ permalink raw reply [flat|nested] 4+ messages in thread