From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) by lore.proxmox.com (Postfix) with ESMTPS id 29C4F1FF2D9 for ; Mon, 22 Jul 2024 12:30:52 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E027D30795; Mon, 22 Jul 2024 12:31:24 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 22 Jul 2024 12:30:34 +0200 Message-Id: <20240722103034.343303-8-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240722103034.343303-1-c.ebner@proxmox.com> References: <20240722103034.343303-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.021 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH v2 proxmox-backup 7/7] client: catalog shell: fallback to accessor for navigation X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Make the catalog optional and use the pxar accessor for navigation if the catalog is not provided. This allows to use the metadata archive for navigraion, as for split pxar archives no dedicated catalog is encoded. Signed-off-by: Christian Ebner --- changes since version 1: - not present in previous version pbs-client/src/catalog_shell.rs | 303 ++++++++++++++++++++++----- proxmox-backup-client/src/catalog.rs | 24 ++- 2 files changed, 268 insertions(+), 59 deletions(-) diff --git a/pbs-client/src/catalog_shell.rs b/pbs-client/src/catalog_shell.rs index 349bb7cbc..eb734ad74 100644 --- a/pbs-client/src/catalog_shell.rs +++ b/pbs-client/src/catalog_shell.rs @@ -21,7 +21,7 @@ use pxar::accessor::ReadAt; use pxar::{EntryKind, Metadata}; use pbs_datastore::catalog::{self, DirEntryAttribute}; -use proxmox_async::runtime::block_in_place; +use proxmox_async::runtime::{block_in_place, block_on}; use crate::pxar::Flags; @@ -312,8 +312,9 @@ pub struct Shell { /// Interactive prompt. prompt: String, - /// Catalog reader instance to navigate - catalog: CatalogReader, + /// Optional catalog reader instance to navigate, if not present the Accessor is used for + /// navigation + catalog: Option, /// List of selected paths for restore selected: HashMap, @@ -347,7 +348,7 @@ impl PathStackEntry { impl Shell { /// Create a new shell for the given catalog and pxar archive. pub async fn new( - mut catalog: CatalogReader, + mut catalog: Option, archive_name: &str, archive: Accessor, ) -> Result { @@ -355,11 +356,31 @@ impl Shell { let mut rl = rustyline::Editor::::new(); rl.set_helper(Some(cli_helper)); - let catalog_root = catalog.root()?; - let archive_root = catalog - .lookup(&catalog_root, archive_name.as_bytes())? - .ok_or_else(|| format_err!("archive not found in catalog"))?; - let position = vec![PathStackEntry::new(archive_root)]; + let mut position = Vec::new(); + if let Some(catalog) = catalog.as_mut() { + let catalog_root = catalog.root()?; + let archive_root = catalog + .lookup(&catalog_root, archive_name.as_bytes())? + .ok_or_else(|| format_err!("archive not found in catalog"))?; + position.push(PathStackEntry::new(archive_root)); + } else { + let root = archive.open_root().await?; + let root_entry = root.lookup_self().await?; + if let EntryKind::Directory = root_entry.kind() { + let entry_attr = DirEntryAttribute::Directory { + start: root_entry.entry_range_info().entry_range.start, + }; + position.push(PathStackEntry { + catalog: catalog::DirEntry { + name: archive_name.into(), + attr: entry_attr, + }, + pxar: Some(root_entry), + }); + } else { + bail!("unexpected root entry type"); + } + } let mut this = Self { rl, @@ -450,7 +471,7 @@ impl Shell { async fn resolve_symlink( stack: &mut Vec, - catalog: &mut CatalogReader, + catalog: &mut Option, accessor: &Accessor, follow_symlinks: &mut Option, ) -> Result<(), Error> { @@ -468,7 +489,7 @@ impl Shell { }; let new_stack = - Self::lookup(stack, &mut *catalog, accessor, Some(path), follow_symlinks).await?; + Self::lookup(stack, catalog, accessor, Some(path), follow_symlinks).await?; *stack = new_stack; @@ -484,7 +505,7 @@ impl Shell { /// out. async fn step( stack: &mut Vec, - catalog: &mut CatalogReader, + catalog: &mut Option, accessor: &Accessor, component: std::path::Component<'_>, follow_symlinks: &mut Option, @@ -503,9 +524,37 @@ impl Shell { if stack.last().unwrap().catalog.is_symlink() { Self::resolve_symlink(stack, catalog, accessor, follow_symlinks).await?; } - match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? { - Some(dir) => stack.push(PathStackEntry::new(dir)), - None => bail!("no such file or directory: {:?}", entry), + if let Some(catalog) = catalog { + match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? { + Some(dir) => stack.push(PathStackEntry::new(dir)), + None => bail!("no such file or directory: {entry:?}"), + } + } else { + let parent_dir = if let Some(parent) = stack.last().as_ref() { + if let Some(parent) = parent.pxar.as_ref() { + parent.enter_directory().await? + } else { + bail!("missing pxar entry on stack"); + } + } else { + bail!("missing parent entry on stack"); + }; + match parent_dir.lookup(entry).await? { + Some(entry) => { + let entry_attr = match crate::tools::map_to_dir_entry_attr(&entry)? { + Some(attr) => attr, + None => bail!("entry without attributes"), + }; + stack.push(PathStackEntry { + catalog: catalog::DirEntry { + name: entry.entry().file_name().as_bytes().into(), + attr: entry_attr, + }, + pxar: Some(entry), + }) + } + None => bail!("no such file or directory: {entry:?}"), + } } } } @@ -515,7 +564,7 @@ impl Shell { fn step_nofollow( stack: &mut Vec, - catalog: &mut CatalogReader, + catalog: &mut Option, component: std::path::Component<'_>, ) -> Result<(), Error> { use std::path::Component; @@ -531,11 +580,33 @@ impl Shell { Component::Normal(entry) => { if stack.last().unwrap().catalog.is_symlink() { bail!("target is a symlink"); - } else { + } else if let Some(catalog) = catalog.as_mut() { match catalog.lookup(&stack.last().unwrap().catalog, entry.as_bytes())? { Some(dir) => stack.push(PathStackEntry::new(dir)), None => bail!("no such file or directory: {:?}", entry), } + } else { + let parent_dir = if let Some(parent) = stack.last().as_ref() { + block_on(parent.pxar.as_ref().unwrap().enter_directory())? + } else { + bail!("missing pxar entry on stack"); + }; + match block_on(parent_dir.lookup(entry))? { + Some(entry) => { + let entry_attr = match crate::tools::map_to_dir_entry_attr(&entry)? { + Some(attr) => attr, + None => bail!("entry without attributes"), + }; + stack.push(PathStackEntry { + catalog: catalog::DirEntry { + name: entry.entry().file_name().as_bytes().into(), + attr: entry_attr, + }, + pxar: Some(entry), + }) + } + None => bail!("no such file or directory: {entry:?}"), + } } } } @@ -545,7 +616,7 @@ impl Shell { /// The pxar accessor is required to resolve symbolic links async fn walk_catalog( stack: &mut Vec, - catalog: &mut CatalogReader, + catalog: &mut Option, accessor: &Accessor, path: &Path, follow_symlinks: &mut Option, @@ -559,7 +630,7 @@ impl Shell { /// Non-async version cannot follow symlinks. fn walk_catalog_nofollow( stack: &mut Vec, - catalog: &mut CatalogReader, + catalog: &mut Option, path: &Path, ) -> Result<(), Error> { for c in path.components() { @@ -612,12 +683,34 @@ impl Shell { tmp_stack = self.position.clone(); } Self::walk_catalog_nofollow(&mut tmp_stack, &mut self.catalog, &path)?; - (&tmp_stack.last().unwrap().catalog, base, part) + (&tmp_stack.last().unwrap(), base, part) } - None => (&self.position.last().unwrap().catalog, "", input), + None => (&self.position.last().unwrap(), "", input), }; - let entries = self.catalog.read_dir(parent)?; + let entries = if let Some(catalog) = self.catalog.as_mut() { + catalog.read_dir(&parent.catalog)? + } else { + let dir = if let Some(entry) = parent.pxar.as_ref() { + block_on(entry.enter_directory())? + } else { + bail!("missing pxar entry on stack"); + }; + let mut out = Vec::new(); + let entries = block_on(crate::tools::pxar_metadata_read_dir(dir))?; + for entry in entries { + let mut name = base.to_string(); + let file_name = entry.file_name().as_bytes(); + if file_name.starts_with(part.as_bytes()) { + name.push_str(std::str::from_utf8(file_name)?); + if entry.is_dir() { + name.push('/'); + } + out.push(name); + } + } + return Ok(out); + }; let mut out = Vec::new(); for entry in entries { @@ -637,7 +730,7 @@ impl Shell { // Break async recursion here: lookup -> walk_catalog -> step -> lookup fn lookup<'future, 's, 'c, 'a, 'p, 'y>( stack: &'s [PathStackEntry], - catalog: &'c mut CatalogReader, + catalog: &'c mut Option, accessor: &'a Accessor, path: Option<&'p Path>, follow_symlinks: &'y mut Option, @@ -678,7 +771,23 @@ impl Shell { let last = stack.last().unwrap(); if last.catalog.is_directory() { - let items = self.catalog.read_dir(&stack.last().unwrap().catalog)?; + let items = if let Some(catalog) = self.catalog.as_mut() { + catalog.read_dir(&stack.last().unwrap().catalog)? + } else { + let dir = if let Some(entry) = last.pxar.as_ref() { + entry.enter_directory().await? + } else { + bail!("missing pxar entry on stack"); + }; + + let mut out = std::io::stdout(); + let items = crate::tools::pxar_metadata_read_dir(dir).await?; + for item in items { + out.write_all(&item.file_name().as_bytes())?; + out.write_all(b"\n")?; + } + return Ok(()); + }; let mut out = std::io::stdout(); // FIXME: columnize for item in items { @@ -820,17 +929,37 @@ impl Shell { async fn list_matching_files(&mut self) -> Result<(), Error> { let matches = self.build_match_list(); - self.catalog.find( - &self.position[0].catalog, - &mut Vec::new(), - &matches, - &mut |path: &[u8]| -> Result<(), Error> { - let mut out = std::io::stdout(); - out.write_all(path)?; - out.write_all(b"\n")?; - Ok(()) - }, - )?; + if let Some(catalog) = self.catalog.as_mut() { + catalog.find( + &self.position[0].catalog, + &mut Vec::new(), + &matches, + &mut |path: &[u8]| -> Result<(), Error> { + let mut out = std::io::stdout(); + out.write_all(path)?; + out.write_all(b"\n")?; + Ok(()) + }, + )?; + } else { + let parent_dir = self.position[0] + .pxar + .as_ref() + .unwrap() + .enter_directory() + .await?; + crate::tools::pxar_metadata_catalog_find( + parent_dir, + &matches, + &|path: &[u8]| -> Result<(), Error> { + let mut out = std::io::stdout(); + out.write_all(path)?; + out.write_all(b"\n")?; + Ok(()) + }, + ) + .await?; + } Ok(()) } @@ -841,18 +970,38 @@ impl Shell { MatchEntry::parse_pattern(pattern, PatternFlag::PATH_NAME, MatchType::Include)?; let mut found_some = false; - self.catalog.find( - &self.position[0].catalog, - &mut Vec::new(), - &[&pattern_entry], - &mut |path: &[u8]| -> Result<(), Error> { - found_some = true; - let mut out = std::io::stdout(); - out.write_all(path)?; - out.write_all(b"\n")?; - Ok(()) - }, - )?; + if let Some(catalog) = self.catalog.as_mut() { + catalog.find( + &self.position[0].catalog, + &mut Vec::new(), + &[&pattern_entry], + &mut |path: &[u8]| -> Result<(), Error> { + found_some = true; + let mut out = std::io::stdout(); + out.write_all(path)?; + out.write_all(b"\n")?; + Ok(()) + }, + )?; + } else { + let parent_dir = self.position[0] + .pxar + .as_ref() + .unwrap() + .enter_directory() + .await?; + crate::tools::pxar_metadata_catalog_find( + parent_dir, + &[&pattern_entry], + &|path: &[u8]| -> Result<(), Error> { + let mut out = std::io::stdout(); + out.write_all(path)?; + out.write_all(b"\n")?; + Ok(()) + }, + ) + .await?; + } if found_some && select { self.selected.insert(pattern_os, pattern_entry); @@ -960,22 +1109,44 @@ struct ExtractorState<'a> { extractor: crate::pxar::extract::Extractor, - catalog: &'a mut CatalogReader, + catalog: &'a mut Option, match_list: &'a [MatchEntry], accessor: &'a Accessor, } impl<'a> ExtractorState<'a> { pub fn new( - catalog: &'a mut CatalogReader, + catalog: &'a mut Option, dir_stack: Vec, extractor: crate::pxar::extract::Extractor, match_list: &'a [MatchEntry], accessor: &'a Accessor, ) -> Result { - let read_dir = catalog - .read_dir(&dir_stack.last().unwrap().catalog)? - .into_iter(); + let read_dir = if let Some(catalog) = catalog.as_mut() { + catalog + .read_dir(&dir_stack.last().unwrap().catalog)? + .into_iter() + } else { + let dir = if let Some(entry) = dir_stack.last().unwrap().pxar.as_ref() { + block_on(entry.enter_directory())? + } else { + bail!("missing pxar entry on stack"); + }; + let entries = block_on(crate::tools::pxar_metadata_read_dir(dir))?; + entries + .into_iter() + .map(|entry| { + let entry_attr = crate::tools::map_to_dir_entry_attr(&entry) + .unwrap() + .unwrap(); + catalog::DirEntry { + name: entry.entry().file_name().as_bytes().into(), + attr: entry_attr, + } + }) + .collect::>() + .into_iter() + }; Ok(Self { path: Vec::new(), path_len: 0, @@ -1053,11 +1224,31 @@ impl<'a> ExtractorState<'a> { entry: catalog::DirEntry, match_result: Option, ) -> Result<(), Error> { + let entry_iter = if let Some(catalog) = self.catalog.as_mut() { + catalog.read_dir(&entry)?.into_iter() + } else { + self.dir_stack.push(PathStackEntry::new(entry.clone())); + let dir = Shell::walk_pxar_archive(self.accessor, &mut self.dir_stack).await?; + self.dir_stack.pop(); + let dir = dir.enter_directory().await?; + let entries = block_on(crate::tools::pxar_metadata_read_dir(dir))?; + entries + .into_iter() + .map(|entry| { + let entry_attr = crate::tools::map_to_dir_entry_attr(&entry) + .unwrap() + .unwrap(); + catalog::DirEntry { + name: entry.entry().file_name().as_bytes().into(), + attr: entry_attr, + } + }) + .collect::>() + .into_iter() + }; // enter a new directory: - self.read_dir_stack.push(mem::replace( - &mut self.read_dir, - self.catalog.read_dir(&entry)?.into_iter(), - )); + self.read_dir_stack + .push(mem::replace(&mut self.read_dir, entry_iter)); self.matches_stack.push(self.matches); self.dir_stack.push(PathStackEntry::new(entry)); self.path_len_stack.push(self.path_len); diff --git a/proxmox-backup-client/src/catalog.rs b/proxmox-backup-client/src/catalog.rs index 8cb1eb414..0a61db1a0 100644 --- a/proxmox-backup-client/src/catalog.rs +++ b/proxmox-backup-client/src/catalog.rs @@ -228,11 +228,29 @@ async fn catalog_shell(param: Value) -> Result<(), Error> { ) .await?; - let mut tmpfile = pbs_client::tools::create_tmp_file()?; - let (manifest, _) = client.download_manifest().await?; manifest.check_fingerprint(crypt_config.as_ref().map(Arc::as_ref))?; + if let Err(_err) = manifest.lookup_file_info(CATALOG_NAME) { + // No catalog, fallback to pxar archive accessor if present + let accessor = helper::get_pxar_fuse_accessor( + &server_archive_name, + client.clone(), + &manifest, + crypt_config.clone(), + ) + .await?; + + let state = Shell::new(None, &server_archive_name, accessor).await?; + log::info!("Starting interactive shell"); + state.shell().await?; + record_repository(&repo); + + return Ok(()); + } + + let mut tmpfile = pbs_client::tools::create_tmp_file()?; + let decoder = helper::get_pxar_fuse_accessor( &server_archive_name, client.clone(), @@ -266,7 +284,7 @@ async fn catalog_shell(param: Value) -> Result<(), Error> { catalogfile.seek(SeekFrom::Start(0))?; let catalog_reader = CatalogReader::new(catalogfile); - let state = Shell::new(catalog_reader, &server_archive_name, decoder).await?; + let state = Shell::new(Some(catalog_reader), &server_archive_name, decoder).await?; log::info!("Starting interactive shell"); state.shell().await?; -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel