From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 1B5B71FF399 for ; Fri, 7 Jun 2024 12:23:31 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 875FE1CCC1; Fri, 7 Jun 2024 12:24:02 +0200 (CEST) Date: Fri, 07 Jun 2024 12:23:26 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20240607094313.178742-1-c.ebner@proxmox.com> <20240607094313.178742-5-c.ebner@proxmox.com> In-Reply-To: <20240607094313.178742-5-c.ebner@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.16.0 (https://github.com/astroidmail/astroid) Message-Id: <1717755543.csxq1estz9.astroid@yuna.none> X-SPAM-LEVEL: Spam detection results: 0 AWL 0.058 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com, datastore.rs] Subject: Re: [pbs-devel] [PATCH proxmox-backup 4/8] api: datastore: conditional lookup for catalog endpoint X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" On June 7, 2024 11:43 am, Christian Ebner wrote: > Add an optional `archive-name` parameter, indicating the metadata > archive to be used for directory content lookups instead of the > catalog. If provided, instead of the catalog reader, a pxar Accessor > instance is created to perform the lookup. > > This is in preparation for dropping catalog encoding for snapshots > with split pxar archive encoding. > > Signed-off-by: Christian Ebner > --- > src/api2/admin/datastore.rs | 73 ++++++++++++++++++++++++++++--------- > 1 file changed, 55 insertions(+), 18 deletions(-) > > diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs > index 117dab080..e25a78bca 100644 > --- a/src/api2/admin/datastore.rs > +++ b/src/api2/admin/datastore.rs > @@ -1659,7 +1659,12 @@ fn decode_path(path: &str) -> Result, Error> { > "filepath": { > description: "Base64 encoded path.", > type: String, > - } > + }, > + "archive-name": { > + type: String, > + description: "Name of the archive to lookup given filepath (base64 encoded)", > + optional: true, > + }, why is this base64 encoded? the archive name is a safe ID.. > }, > }, > access: { > @@ -1674,8 +1679,18 @@ pub async fn catalog( > ns: Option, > backup_dir: pbs_api_types::BackupDir, > filepath: String, > + archive_name: Option, > rpcenv: &mut dyn RpcEnvironment, > ) -> Result, Error> { > + let file_name = if let Some(ref archive_name) = archive_name { > + String::from_utf8( > + base64::decode(archive_name) > + .map_err(|err| format_err!("base64 decode of 'archive-name' failed - {err}"))?, > + )? > + } else { > + CATALOG_NAME.to_string() > + }; without b64, this would just be an unwrap_or_else ;) > + > let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; > > let ns = ns.unwrap_or_default(); > @@ -1692,8 +1707,6 @@ pub async fn catalog( > > let backup_dir = datastore.backup_dir(ns, backup_dir)?; > > - let file_name = CATALOG_NAME; > - > let (manifest, files) = read_backup_index(&backup_dir)?; > for file in files { > if file.filename == file_name && file.crypt_mode == Some(CryptMode::Encrypt) { > @@ -1701,26 +1714,50 @@ pub async fn catalog( > } > } > > - tokio::task::spawn_blocking(move || { > - let mut path = datastore.base_path(); > - path.push(backup_dir.relative_path()); > - path.push(file_name); > + if archive_name.is_none() { > + tokio::task::spawn_blocking(move || { > + let mut path = datastore.base_path(); > + path.push(backup_dir.relative_path()); > + path.push(&file_name); > > - let index = DynamicIndexReader::open(&path) > - .map_err(|err| format_err!("unable to read dynamic index '{:?}' - {}", &path, err))?; > + let index = DynamicIndexReader::open(&path) > + .map_err(|err| format_err!("unable to read dynamic index '{path:?}' - {err}"))?; > > - let (csum, size) = index.compute_csum(); > - manifest.verify_file(file_name, &csum, size)?; > + let (csum, size) = index.compute_csum(); > + manifest.verify_file(&file_name, &csum, size)?; > > - let chunk_reader = LocalChunkReader::new(datastore, None, CryptMode::None); > - let reader = BufferedDynamicReader::new(index, chunk_reader); > + let chunk_reader = LocalChunkReader::new(datastore, None, CryptMode::None); > + let reader = BufferedDynamicReader::new(index, chunk_reader); > > - let mut catalog_reader = CatalogReader::new(reader); > + let mut catalog_reader = CatalogReader::new(reader); > > - let path = decode_path(&filepath)?; > - catalog_reader.list_dir_contents(&path) > - }) > - .await? > + let path = decode_path(&filepath)?; > + catalog_reader.list_dir_contents(&path) > + }) > + .await? > + } else { > + let (archive_name, payload_archive_name) = > + pbs_client::tools::get_pxar_archive_names(&file_name, &manifest)?; > + let (reader, archive_size) = > + get_local_pxar_reader(datastore.clone(), &manifest, &backup_dir, &archive_name)?; > + > + let reader = if let Some(payload_archive_name) = payload_archive_name { > + let payload_input = > + get_local_pxar_reader(datastore, &manifest, &backup_dir, &payload_archive_name)?; > + pxar::PxarVariant::Split(reader, payload_input) > + } else { > + pxar::PxarVariant::Unified(reader) > + }; > + let accessor = Accessor::new(reader, archive_size).await?; > + > + let file_path = decode_path(&filepath)?; > + pbs_client::tools::pxar_metadata_catalog_lookup( > + accessor, > + OsStr::from_bytes(&file_path), > + None, > + ) > + .await > + } > } > > #[sortable] > -- > 2.39.2 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel > > > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel