From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id EC2B91FF3A2 for ; Wed, 5 Jun 2024 12:54:49 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8F66731CA6; Wed, 5 Jun 2024 12:55:03 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 5 Jun 2024 12:54:10 +0200 Message-Id: <20240605105416.278748-53-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240605105416.278748-1-c.ebner@proxmox.com> References: <20240605105416.278748-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.123 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment POISEN_SPAM_PILL 0.1 Meta: its spam POISEN_SPAM_PILL_1 0.1 random spam to be learned in bayes POISEN_SPAM_PILL_3 0.1 random spam to be learned in bayes SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v9 proxmox-backup 52/58] api: datastore: add endpoint to lookup entries via pxar archive X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Add an api endpoint `pxar-lookup` to access the contents of a pxar archive via a server side pxar accessor, providing the same response as currently the `catalog` lookup. The intention is to fully replace the catalog for split pxar archives, accessing all the entries via the metadata archive instead of the catalog. Signed-off-by: Christian Ebner --- changes since version 8: - not present in previous version src/api2/admin/datastore.rs | 149 +++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 2 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 34a9105dd..abc4a4fba 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -30,7 +30,8 @@ use proxmox_sys::{task_log, task_warn}; use proxmox_time::CalendarEvent; use pxar::accessor::aio::Accessor; -use pxar::EntryKind; +use pxar::format::SignedDuration; +use pxar::{mode, EntryKind}; use pbs_api_types::{ print_ns_and_snapshot, print_store_and_ns, Authid, BackupContent, BackupNamespace, BackupType, @@ -47,7 +48,7 @@ use pbs_client::pxar::{create_tar, create_zip}; use pbs_config::CachedUserInfo; use pbs_datastore::backup_info::BackupInfo; use pbs_datastore::cached_chunk_reader::CachedChunkReader; -use pbs_datastore::catalog::{ArchiveEntry, CatalogReader}; +use pbs_datastore::catalog::{ArchiveEntry, CatalogReader, DirEntryAttribute}; use pbs_datastore::data_blob::DataBlob; use pbs_datastore::data_blob_reader::DataBlobReader; use pbs_datastore::dynamic_index::{BufferedDynamicReader, DynamicIndexReader, LocalDynamicReadAt}; @@ -1720,6 +1721,149 @@ pub async fn catalog( .await? } +#[api( + input: { + properties: { + store: { schema: DATASTORE_SCHEMA }, + "archive-name": { + type: String, + description: "Name of the archive to lookup given filepath (base64 encoded)", + }, + ns: { + type: BackupNamespace, + optional: true, + }, + backup_dir: { + type: pbs_api_types::BackupDir, + flatten: true, + }, + "filepath": { + description: "Base64 encoded path.", + type: String, + } + }, + }, + access: { + description: "Requires on /datastore/{store}[/{namespace}] either DATASTORE_READ for any or \ + DATASTORE_BACKUP and being the owner of the group", + permission: &Permission::Anybody, + }, +)] +/// Get the entries of the given path of the pxar (metadata) archive +pub async fn pxar_lookup( + store: String, + archive_name: String, + ns: Option, + backup_dir: pbs_api_types::BackupDir, + filepath: String, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let archive_name = base64::decode(archive_name) + .map_err(|err| format_err!("base64 decode of archive-name failed - {err}"))?; + + let archive_name = std::str::from_utf8(&archive_name)?; + + let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; + + let ns = ns.unwrap_or_default(); + + let datastore = check_privs_and_load_store( + &store, + &ns, + &auth_id, + PRIV_DATASTORE_READ, + PRIV_DATASTORE_BACKUP, + Some(Operation::Read), + &backup_dir.group, + )?; + + let backup_dir = datastore.backup_dir(ns, backup_dir)?; + + let file_path = if filepath != "root" && filepath != "/" { + base64::decode(filepath) + .map_err(|err| format_err!("base64 decode of filepath failed - {err}"))? + } else { + vec![b'/'] + }; + + let (manifest, files) = read_backup_index(&backup_dir)?; + for file in files { + if file.filename == archive_name && file.crypt_mode == Some(CryptMode::Encrypt) { + bail!("cannot decode '{archive_name}' - is encrypted"); + } + } + + let (archive_name, payload_archive_name) = + pbs_client::tools::get_pxar_archive_names(archive_name, &manifest)?; + let (reader, archive_size) = + get_local_pxar_reader(datastore.clone(), &manifest, &backup_dir, &archive_name)?; + + let reader = if let Some(payload_archive_name) = payload_archive_name { + let payload_input = + get_local_pxar_reader(datastore, &manifest, &backup_dir, &payload_archive_name)?; + pxar::PxarVariant::Split(reader, payload_input) + } else { + pxar::PxarVariant::Unified(reader) + }; + let accessor = Accessor::new(reader, archive_size).await?; + + let root = accessor.open_root().await?; + let path = OsStr::from_bytes(&file_path).to_os_string(); + let dir_entry = root + .lookup(&path) + .await + .map_err(|err| format_err!("lookup failed - {err}"))? + .ok_or_else(|| format_err!("lookup failed - error opening '{path:?}'"))?; + + let mut entries = Vec::new(); + if let EntryKind::Directory = dir_entry.kind() { + let dir_entry = dir_entry + .enter_directory() + .map_err(|err| format_err!("failed to enter directory - {err}")) + .await?; + + let mut entries_iter = dir_entry.read_dir(); + while let Some(entry) = entries_iter.next().await { + let entry = entry?.decode_entry().await?; + + let entry_attr = match entry.kind() { + EntryKind::Version(_) | EntryKind::Prelude(_) | EntryKind::GoodbyeTable => continue, + EntryKind::Directory => DirEntryAttribute::Directory { + start: entry.entry_range_info().entry_range.start, + }, + EntryKind::File { size, .. } => { + let mtime = match entry.metadata().mtime_as_duration() { + SignedDuration::Positive(val) => i64::try_from(val.as_secs())?, + SignedDuration::Negative(val) => -1 * i64::try_from(val.as_secs())?, + }; + DirEntryAttribute::File { size: *size, mtime } + } + EntryKind::Device(_) => match entry.metadata().file_type() { + mode::IFBLK => DirEntryAttribute::BlockDevice, + mode::IFCHR => DirEntryAttribute::CharDevice, + _ => bail!("encountered unknown device type"), + }, + EntryKind::Symlink(_) => DirEntryAttribute::Symlink, + EntryKind::Hardlink(_) => DirEntryAttribute::Hardlink, + EntryKind::Fifo => DirEntryAttribute::Fifo, + EntryKind::Socket => DirEntryAttribute::Socket, + }; + + entries.push(ArchiveEntry::new( + entry.path().as_os_str().as_bytes(), + Some(&entry_attr), + )); + } + } else { + bail!(format!( + "expected directory entry, got entry kind '{:?}'", + dir_entry.kind() + )); + } + + Ok(entries) +} + #[sortable] pub const API_METHOD_PXAR_FILE_DOWNLOAD: ApiMethod = ApiMethod::new( &ApiHandler::AsyncHttp(&pxar_file_download), @@ -2414,6 +2558,7 @@ const DATASTORE_INFO_SUBDIRS: SubdirMap = &[ "pxar-file-download", &Router::new().download(&API_METHOD_PXAR_FILE_DOWNLOAD), ), + ("pxar-lookup", &Router::new().get(&API_METHOD_PXAR_LOOKUP)), ("rrd", &Router::new().get(&API_METHOD_GET_RRD_STATS)), ( "snapshots", -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel