From: Stefan Reiter <s.reiter@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader
Date: Tue, 16 Feb 2021 18:06:52 +0100 [thread overview]
Message-ID: <20210216170710.31767-5-s.reiter@proxmox.com> (raw)
In-Reply-To: <20210216170710.31767-1-s.reiter@proxmox.com>
From: Dominik Csapak <d.csapak@proxmox.com>
we will reuse that later in the client, so we need it somewhere
we can use from there
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
[add strongly typed ArchiveEntry and put api code into helpers.rs]
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
src/api2/admin/datastore.rs | 53 ++++++-------------------------------
src/api2/helpers.rs | 31 ++++++++++++++++++++++
src/api2/types/mod.rs | 43 ++++++++++++++++++++++++++++++
src/backup/catalog.rs | 26 ++++++++++++++++++
4 files changed, 108 insertions(+), 45 deletions(-)
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 6f02e460..ab88d172 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -27,6 +27,7 @@ use pxar::EntryKind;
use crate::api2::types::*;
use crate::api2::node::rrd::create_value_from_rrd;
+use crate::api2::helpers;
use crate::backup::*;
use crate::config::datastore;
use crate::config::cached_user_info::CachedUserInfo;
@@ -1294,7 +1295,7 @@ pub fn catalog(
backup_time: i64,
filepath: String,
rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<ArchiveEntry>, Error> {
let datastore = DataStore::lookup_datastore(&store)?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1326,52 +1327,14 @@ pub fn catalog(
let reader = BufferedDynamicReader::new(index, chunk_reader);
let mut catalog_reader = CatalogReader::new(reader);
- let mut current = catalog_reader.root()?;
- let mut components = vec![];
+ let path = if filepath != "root" {
+ base64::decode(filepath)?
+ } else {
+ vec![b'/']
+ };
- if filepath != "root" {
- components = base64::decode(filepath)?;
- if !components.is_empty() && components[0] == b'/' {
- components.remove(0);
- }
- for component in components.split(|c| *c == b'/') {
- if let Some(entry) = catalog_reader.lookup(¤t, component)? {
- current = entry;
- } else {
- bail!("path {:?} not found in catalog", &String::from_utf8_lossy(&components));
- }
- }
- }
-
- let mut res = Vec::new();
-
- for direntry in catalog_reader.read_dir(¤t)? {
- let mut components = components.clone();
- components.push(b'/');
- components.extend(&direntry.name);
- let path = base64::encode(components);
- let text = String::from_utf8_lossy(&direntry.name);
- let mut entry = json!({
- "filepath": path,
- "text": text,
- "type": CatalogEntryType::from(&direntry.attr).to_string(),
- "leaf": true,
- });
- match direntry.attr {
- DirEntryAttribute::Directory { start: _ } => {
- entry["leaf"] = false.into();
- },
- DirEntryAttribute::File { size, mtime } => {
- entry["size"] = size.into();
- entry["mtime"] = mtime.into();
- },
- _ => {},
- }
- res.push(entry);
- }
-
- Ok(res.into())
+ helpers::list_dir_content(&mut catalog_reader, &path)
}
fn recurse_files<'a, T, W>(
diff --git a/src/api2/helpers.rs b/src/api2/helpers.rs
index 2a822654..41391b77 100644
--- a/src/api2/helpers.rs
+++ b/src/api2/helpers.rs
@@ -1,3 +1,4 @@
+use std::io::{Read, Seek};
use std::path::PathBuf;
use anyhow::Error;
@@ -6,6 +7,9 @@ use hyper::{Body, Response, StatusCode, header};
use proxmox::http_bail;
+use crate::api2::types::ArchiveEntry;
+use crate::backup::{CatalogReader, DirEntryAttribute};
+
pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, Error> {
let file = match tokio::fs::File::open(path.clone()).await {
Ok(file) => file,
@@ -27,3 +31,30 @@ pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, E
.body(body)
.unwrap())
}
+
+/// Returns the list of content of the given path
+pub fn list_dir_content<R: Read + Seek>(
+ reader: &mut CatalogReader<R>,
+ path: &[u8],
+) -> Result<Vec<ArchiveEntry>, Error> {
+ let dir = reader.lookup_recursive(path)?;
+ let mut res = vec![];
+ let mut path = path.to_vec();
+ if !path.is_empty() && path[0] == b'/' {
+ path.remove(0);
+ }
+
+ for direntry in reader.read_dir(&dir)? {
+ let mut components = path.clone();
+ components.push(b'/');
+ components.extend(&direntry.name);
+ let mut entry = ArchiveEntry::new(&components, &direntry.attr);
+ if let DirEntryAttribute::File { size, mtime } = direntry.attr {
+ entry.size = size.into();
+ entry.mtime = mtime.into();
+ }
+ res.push(entry);
+ }
+
+ Ok(res)
+}
diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs
index d9394586..4c663335 100644
--- a/src/api2/types/mod.rs
+++ b/src/api2/types/mod.rs
@@ -12,6 +12,8 @@ use crate::{
CryptMode,
Fingerprint,
BACKUP_ID_REGEX,
+ DirEntryAttribute,
+ CatalogEntryType,
},
server::UPID,
config::acl::Role,
@@ -1303,6 +1305,47 @@ pub struct DatastoreNotify {
pub sync: Option<Notify>,
}
+/// An entry in a hierarchy of files for restore and listing.
+#[api()]
+#[derive(Serialize, Deserialize)]
+pub struct ArchiveEntry {
+ /// Base64-encoded full path to the file, including the filename
+ pub filepath: String,
+ /// Displayable filename text for UIs
+ pub text: String,
+ /// File or directory type of this entry
+ #[serde(rename = "type")]
+ pub entry_type: String,
+ /// Is this entry a leaf node, or does it have children (i.e. a directory)?
+ pub leaf: bool,
+ /// The file size, if entry_type is 'f' (file)
+ #[serde(skip_serializing_if="Option::is_none")]
+ pub size: Option<u64>,
+ /// The file "last modified" time stamp, if entry_type is 'f' (file)
+ #[serde(skip_serializing_if="Option::is_none")]
+ pub mtime: Option<i64>,
+}
+
+impl ArchiveEntry {
+ pub fn new(filepath: &[u8], entry_type: &DirEntryAttribute) -> Self {
+ Self {
+ filepath: base64::encode(filepath),
+ text: String::from_utf8_lossy(filepath.split(|x| *x == b'/').last().unwrap())
+ .to_string(),
+ entry_type: CatalogEntryType::from(entry_type).to_string(),
+ leaf: matches!(entry_type, DirEntryAttribute::Directory { .. }),
+ size: match entry_type {
+ DirEntryAttribute::File { size, .. } => Some(*size),
+ _ => None
+ },
+ mtime: match entry_type {
+ DirEntryAttribute::File { mtime, .. } => Some(*mtime),
+ _ => None
+ },
+ }
+ }
+}
+
pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new(
"Datastore notification setting")
.format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA))
diff --git a/src/backup/catalog.rs b/src/backup/catalog.rs
index 224e6bf7..a307f9d8 100644
--- a/src/backup/catalog.rs
+++ b/src/backup/catalog.rs
@@ -468,6 +468,32 @@ impl <R: Read + Seek> CatalogReader<R> {
Ok(entry_list)
}
+ /// Lookup a DirEntry from an absolute path
+ pub fn lookup_recursive(
+ &mut self,
+ path: &[u8],
+ ) -> Result<DirEntry, Error> {
+ let mut current = self.root()?;
+ if path == b"/" {
+ return Ok(current);
+ }
+
+ let components = if !path.is_empty() && path[0] == b'/' {
+ &path[1..]
+ } else {
+ path
+ }.split(|c| *c == b'/');
+
+ for comp in components {
+ if let Some(entry) = self.lookup(¤t, comp)? {
+ current = entry;
+ } else {
+ bail!("path {:?} not found in catalog", String::from_utf8_lossy(&path));
+ }
+ }
+ Ok(current)
+ }
+
/// Lockup a DirEntry inside a parent directory
pub fn lookup(
&mut self,
--
2.20.1
next prev parent reply other threads:[~2021-02-16 17:08 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-16 17:06 [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 01/22] decoder/aio: add contents() and content_size() calls Stefan Reiter
2021-02-17 7:56 ` Wolfgang Bumiller
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 02/22] decoder: add peek() Stefan Reiter
2021-02-17 8:20 ` Wolfgang Bumiller
2021-02-17 8:38 ` Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-restore-vm-data 03/22] initial commit Stefan Reiter
2021-03-15 18:35 ` [pbs-devel] applied: " Thomas Lamprecht
2021-03-16 15:33 ` Stefan Reiter
2021-02-16 17:06 ` Stefan Reiter [this message]
2021-02-17 7:50 ` [pbs-devel] applied: [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 05/22] api2/admin/datastore: accept "/" as path for root Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 06/22] api2/admin/datastore: refactor create_zip into pxar/extract Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 07/22] pxar/extract: add extract_sub_dir Stefan Reiter
2021-02-17 7:51 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 08/22] pxar/extract: add sequential variants to create_zip, extract_sub_dir Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 09/22] client: extract common functions to proxmox_client_tools module Stefan Reiter
2021-02-17 6:49 ` Dietmar Maurer
2021-02-17 7:58 ` Stefan Reiter
2021-02-17 8:50 ` Dietmar Maurer
2021-02-17 9:47 ` Stefan Reiter
2021-02-17 10:12 ` Dietmar Maurer
2021-02-17 9:13 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 10/22] proxmox_client_tools: extract 'key' from client module Stefan Reiter
2021-02-17 9:11 ` Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 11/22] file-restore: add binary and basic commands Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 12/22] file-restore: allow specifying output-format Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 13/22] rest: implement tower service for UnixStream Stefan Reiter
2021-02-17 6:52 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 14/22] client: add VsockClient to connect to virtio-vsock VMs Stefan Reiter
2021-02-17 7:24 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 15/22] file-restore-daemon: add binary with virtio-vsock API server Stefan Reiter
2021-02-17 10:17 ` Dietmar Maurer
2021-02-17 10:25 ` Dietmar Maurer
2021-02-17 10:30 ` Stefan Reiter
2021-02-17 11:13 ` Dietmar Maurer
2021-02-17 11:26 ` Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 16/22] file-restore-daemon: add watchdog module Stefan Reiter
2021-02-17 10:52 ` Wolfgang Bumiller
2021-02-17 11:14 ` Stefan Reiter
2021-02-17 11:29 ` Wolfgang Bumiller
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 17/22] file-restore-daemon: add disk module Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 18/22] file-restore: add basic VM/block device support Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 19/22] file-restore: improve logging of VM with logrotate Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 20/22] debian/client: add postinst hook to rebuild file-restore initramfs Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 21/22] file-restore(-daemon): implement list API Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 22/22] file-restore: add 'extract' command for VM file restore Stefan Reiter
2021-02-16 17:11 ` [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210216170710.31767-5-s.reiter@proxmox.com \
--to=s.reiter@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox