From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id C79026A576 for ; Tue, 16 Feb 2021 18:08:02 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id A76C718E1F for ; Tue, 16 Feb 2021 18:07:32 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5FEC318DBE for ; Tue, 16 Feb 2021 18:07:29 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 25D7D461E1 for ; Tue, 16 Feb 2021 18:07:29 +0100 (CET) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Tue, 16 Feb 2021 18:06:52 +0100 Message-Id: <20210216170710.31767-5-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210216170710.31767-1-s.reiter@proxmox.com> References: <20210216170710.31767-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.029 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [catalog.rs, datastore.rs, helpers.rs, mod.rs] Subject: [pbs-devel] [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 16 Feb 2021 17:08:02 -0000 From: Dominik Csapak we will reuse that later in the client, so we need it somewhere we can use from there Signed-off-by: Dominik Csapak [add strongly typed ArchiveEntry and put api code into helpers.rs] Signed-off-by: Stefan Reiter --- src/api2/admin/datastore.rs | 53 ++++++------------------------------- src/api2/helpers.rs | 31 ++++++++++++++++++++++ src/api2/types/mod.rs | 43 ++++++++++++++++++++++++++++++ src/backup/catalog.rs | 26 ++++++++++++++++++ 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs index 6f02e460..ab88d172 100644 --- a/src/api2/admin/datastore.rs +++ b/src/api2/admin/datastore.rs @@ -27,6 +27,7 @@ use pxar::EntryKind; use crate::api2::types::*; use crate::api2::node::rrd::create_value_from_rrd; +use crate::api2::helpers; use crate::backup::*; use crate::config::datastore; use crate::config::cached_user_info::CachedUserInfo; @@ -1294,7 +1295,7 @@ pub fn catalog( backup_time: i64, filepath: String, rpcenv: &mut dyn RpcEnvironment, -) -> Result { +) -> Result, Error> { let datastore = DataStore::lookup_datastore(&store)?; let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?; @@ -1326,52 +1327,14 @@ pub fn catalog( let reader = BufferedDynamicReader::new(index, chunk_reader); let mut catalog_reader = CatalogReader::new(reader); - let mut current = catalog_reader.root()?; - let mut components = vec![]; + let path = if filepath != "root" { + base64::decode(filepath)? + } else { + vec![b'/'] + }; - if filepath != "root" { - components = base64::decode(filepath)?; - if !components.is_empty() && components[0] == b'/' { - components.remove(0); - } - for component in components.split(|c| *c == b'/') { - if let Some(entry) = catalog_reader.lookup(¤t, component)? { - current = entry; - } else { - bail!("path {:?} not found in catalog", &String::from_utf8_lossy(&components)); - } - } - } - - let mut res = Vec::new(); - - for direntry in catalog_reader.read_dir(¤t)? { - let mut components = components.clone(); - components.push(b'/'); - components.extend(&direntry.name); - let path = base64::encode(components); - let text = String::from_utf8_lossy(&direntry.name); - let mut entry = json!({ - "filepath": path, - "text": text, - "type": CatalogEntryType::from(&direntry.attr).to_string(), - "leaf": true, - }); - match direntry.attr { - DirEntryAttribute::Directory { start: _ } => { - entry["leaf"] = false.into(); - }, - DirEntryAttribute::File { size, mtime } => { - entry["size"] = size.into(); - entry["mtime"] = mtime.into(); - }, - _ => {}, - } - res.push(entry); - } - - Ok(res.into()) + helpers::list_dir_content(&mut catalog_reader, &path) } fn recurse_files<'a, T, W>( diff --git a/src/api2/helpers.rs b/src/api2/helpers.rs index 2a822654..41391b77 100644 --- a/src/api2/helpers.rs +++ b/src/api2/helpers.rs @@ -1,3 +1,4 @@ +use std::io::{Read, Seek}; use std::path::PathBuf; use anyhow::Error; @@ -6,6 +7,9 @@ use hyper::{Body, Response, StatusCode, header}; use proxmox::http_bail; +use crate::api2::types::ArchiveEntry; +use crate::backup::{CatalogReader, DirEntryAttribute}; + pub async fn create_download_response(path: PathBuf) -> Result, Error> { let file = match tokio::fs::File::open(path.clone()).await { Ok(file) => file, @@ -27,3 +31,30 @@ pub async fn create_download_response(path: PathBuf) -> Result, E .body(body) .unwrap()) } + +/// Returns the list of content of the given path +pub fn list_dir_content( + reader: &mut CatalogReader, + path: &[u8], +) -> Result, Error> { + let dir = reader.lookup_recursive(path)?; + let mut res = vec![]; + let mut path = path.to_vec(); + if !path.is_empty() && path[0] == b'/' { + path.remove(0); + } + + for direntry in reader.read_dir(&dir)? { + let mut components = path.clone(); + components.push(b'/'); + components.extend(&direntry.name); + let mut entry = ArchiveEntry::new(&components, &direntry.attr); + if let DirEntryAttribute::File { size, mtime } = direntry.attr { + entry.size = size.into(); + entry.mtime = mtime.into(); + } + res.push(entry); + } + + Ok(res) +} diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs index d9394586..4c663335 100644 --- a/src/api2/types/mod.rs +++ b/src/api2/types/mod.rs @@ -12,6 +12,8 @@ use crate::{ CryptMode, Fingerprint, BACKUP_ID_REGEX, + DirEntryAttribute, + CatalogEntryType, }, server::UPID, config::acl::Role, @@ -1303,6 +1305,47 @@ pub struct DatastoreNotify { pub sync: Option, } +/// An entry in a hierarchy of files for restore and listing. +#[api()] +#[derive(Serialize, Deserialize)] +pub struct ArchiveEntry { + /// Base64-encoded full path to the file, including the filename + pub filepath: String, + /// Displayable filename text for UIs + pub text: String, + /// File or directory type of this entry + #[serde(rename = "type")] + pub entry_type: String, + /// Is this entry a leaf node, or does it have children (i.e. a directory)? + pub leaf: bool, + /// The file size, if entry_type is 'f' (file) + #[serde(skip_serializing_if="Option::is_none")] + pub size: Option, + /// The file "last modified" time stamp, if entry_type is 'f' (file) + #[serde(skip_serializing_if="Option::is_none")] + pub mtime: Option, +} + +impl ArchiveEntry { + pub fn new(filepath: &[u8], entry_type: &DirEntryAttribute) -> Self { + Self { + filepath: base64::encode(filepath), + text: String::from_utf8_lossy(filepath.split(|x| *x == b'/').last().unwrap()) + .to_string(), + entry_type: CatalogEntryType::from(entry_type).to_string(), + leaf: matches!(entry_type, DirEntryAttribute::Directory { .. }), + size: match entry_type { + DirEntryAttribute::File { size, .. } => Some(*size), + _ => None + }, + mtime: match entry_type { + DirEntryAttribute::File { mtime, .. } => Some(*mtime), + _ => None + }, + } + } +} + pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new( "Datastore notification setting") .format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA)) diff --git a/src/backup/catalog.rs b/src/backup/catalog.rs index 224e6bf7..a307f9d8 100644 --- a/src/backup/catalog.rs +++ b/src/backup/catalog.rs @@ -468,6 +468,32 @@ impl CatalogReader { Ok(entry_list) } + /// Lookup a DirEntry from an absolute path + pub fn lookup_recursive( + &mut self, + path: &[u8], + ) -> Result { + let mut current = self.root()?; + if path == b"/" { + return Ok(current); + } + + let components = if !path.is_empty() && path[0] == b'/' { + &path[1..] + } else { + path + }.split(|c| *c == b'/'); + + for comp in components { + if let Some(entry) = self.lookup(¤t, comp)? { + current = entry; + } else { + bail!("path {:?} not found in catalog", String::from_utf8_lossy(&path)); + } + } + Ok(current) + } + /// Lockup a DirEntry inside a parent directory pub fn lookup( &mut self, -- 2.20.1