From: Stefan Reiter <s.reiter@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader
Date: Tue, 16 Feb 2021 18:06:52 +0100 [thread overview]
Message-ID: <20210216170710.31767-5-s.reiter@proxmox.com> (raw)
In-Reply-To: <20210216170710.31767-1-s.reiter@proxmox.com>
From: Dominik Csapak <d.csapak@proxmox.com>
we will reuse that later in the client, so we need it somewhere
we can use from there
Signed-off-by: Dominik Csapak <d.csapak@proxmox.com>
[add strongly typed ArchiveEntry and put api code into helpers.rs]
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
src/api2/admin/datastore.rs | 53 ++++++-------------------------------
src/api2/helpers.rs | 31 ++++++++++++++++++++++
src/api2/types/mod.rs | 43 ++++++++++++++++++++++++++++++
src/backup/catalog.rs | 26 ++++++++++++++++++
4 files changed, 108 insertions(+), 45 deletions(-)
diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
index 6f02e460..ab88d172 100644
--- a/src/api2/admin/datastore.rs
+++ b/src/api2/admin/datastore.rs
@@ -27,6 +27,7 @@ use pxar::EntryKind;
use crate::api2::types::*;
use crate::api2::node::rrd::create_value_from_rrd;
+use crate::api2::helpers;
use crate::backup::*;
use crate::config::datastore;
use crate::config::cached_user_info::CachedUserInfo;
@@ -1294,7 +1295,7 @@ pub fn catalog(
backup_time: i64,
filepath: String,
rpcenv: &mut dyn RpcEnvironment,
-) -> Result<Value, Error> {
+) -> Result<Vec<ArchiveEntry>, Error> {
let datastore = DataStore::lookup_datastore(&store)?;
let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
@@ -1326,52 +1327,14 @@ pub fn catalog(
let reader = BufferedDynamicReader::new(index, chunk_reader);
let mut catalog_reader = CatalogReader::new(reader);
- let mut current = catalog_reader.root()?;
- let mut components = vec![];
+ let path = if filepath != "root" {
+ base64::decode(filepath)?
+ } else {
+ vec![b'/']
+ };
- if filepath != "root" {
- components = base64::decode(filepath)?;
- if !components.is_empty() && components[0] == b'/' {
- components.remove(0);
- }
- for component in components.split(|c| *c == b'/') {
- if let Some(entry) = catalog_reader.lookup(¤t, component)? {
- current = entry;
- } else {
- bail!("path {:?} not found in catalog", &String::from_utf8_lossy(&components));
- }
- }
- }
-
- let mut res = Vec::new();
-
- for direntry in catalog_reader.read_dir(¤t)? {
- let mut components = components.clone();
- components.push(b'/');
- components.extend(&direntry.name);
- let path = base64::encode(components);
- let text = String::from_utf8_lossy(&direntry.name);
- let mut entry = json!({
- "filepath": path,
- "text": text,
- "type": CatalogEntryType::from(&direntry.attr).to_string(),
- "leaf": true,
- });
- match direntry.attr {
- DirEntryAttribute::Directory { start: _ } => {
- entry["leaf"] = false.into();
- },
- DirEntryAttribute::File { size, mtime } => {
- entry["size"] = size.into();
- entry["mtime"] = mtime.into();
- },
- _ => {},
- }
- res.push(entry);
- }
-
- Ok(res.into())
+ helpers::list_dir_content(&mut catalog_reader, &path)
}
fn recurse_files<'a, T, W>(
diff --git a/src/api2/helpers.rs b/src/api2/helpers.rs
index 2a822654..41391b77 100644
--- a/src/api2/helpers.rs
+++ b/src/api2/helpers.rs
@@ -1,3 +1,4 @@
+use std::io::{Read, Seek};
use std::path::PathBuf;
use anyhow::Error;
@@ -6,6 +7,9 @@ use hyper::{Body, Response, StatusCode, header};
use proxmox::http_bail;
+use crate::api2::types::ArchiveEntry;
+use crate::backup::{CatalogReader, DirEntryAttribute};
+
pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, Error> {
let file = match tokio::fs::File::open(path.clone()).await {
Ok(file) => file,
@@ -27,3 +31,30 @@ pub async fn create_download_response(path: PathBuf) -> Result<Response<Body>, E
.body(body)
.unwrap())
}
+
+/// Returns the list of content of the given path
+pub fn list_dir_content<R: Read + Seek>(
+ reader: &mut CatalogReader<R>,
+ path: &[u8],
+) -> Result<Vec<ArchiveEntry>, Error> {
+ let dir = reader.lookup_recursive(path)?;
+ let mut res = vec![];
+ let mut path = path.to_vec();
+ if !path.is_empty() && path[0] == b'/' {
+ path.remove(0);
+ }
+
+ for direntry in reader.read_dir(&dir)? {
+ let mut components = path.clone();
+ components.push(b'/');
+ components.extend(&direntry.name);
+ let mut entry = ArchiveEntry::new(&components, &direntry.attr);
+ if let DirEntryAttribute::File { size, mtime } = direntry.attr {
+ entry.size = size.into();
+ entry.mtime = mtime.into();
+ }
+ res.push(entry);
+ }
+
+ Ok(res)
+}
diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs
index d9394586..4c663335 100644
--- a/src/api2/types/mod.rs
+++ b/src/api2/types/mod.rs
@@ -12,6 +12,8 @@ use crate::{
CryptMode,
Fingerprint,
BACKUP_ID_REGEX,
+ DirEntryAttribute,
+ CatalogEntryType,
},
server::UPID,
config::acl::Role,
@@ -1303,6 +1305,47 @@ pub struct DatastoreNotify {
pub sync: Option<Notify>,
}
+/// An entry in a hierarchy of files for restore and listing.
+#[api()]
+#[derive(Serialize, Deserialize)]
+pub struct ArchiveEntry {
+ /// Base64-encoded full path to the file, including the filename
+ pub filepath: String,
+ /// Displayable filename text for UIs
+ pub text: String,
+ /// File or directory type of this entry
+ #[serde(rename = "type")]
+ pub entry_type: String,
+ /// Is this entry a leaf node, or does it have children (i.e. a directory)?
+ pub leaf: bool,
+ /// The file size, if entry_type is 'f' (file)
+ #[serde(skip_serializing_if="Option::is_none")]
+ pub size: Option<u64>,
+ /// The file "last modified" time stamp, if entry_type is 'f' (file)
+ #[serde(skip_serializing_if="Option::is_none")]
+ pub mtime: Option<i64>,
+}
+
+impl ArchiveEntry {
+ pub fn new(filepath: &[u8], entry_type: &DirEntryAttribute) -> Self {
+ Self {
+ filepath: base64::encode(filepath),
+ text: String::from_utf8_lossy(filepath.split(|x| *x == b'/').last().unwrap())
+ .to_string(),
+ entry_type: CatalogEntryType::from(entry_type).to_string(),
+ leaf: matches!(entry_type, DirEntryAttribute::Directory { .. }),
+ size: match entry_type {
+ DirEntryAttribute::File { size, .. } => Some(*size),
+ _ => None
+ },
+ mtime: match entry_type {
+ DirEntryAttribute::File { mtime, .. } => Some(*mtime),
+ _ => None
+ },
+ }
+ }
+}
+
pub const DATASTORE_NOTIFY_STRING_SCHEMA: Schema = StringSchema::new(
"Datastore notification setting")
.format(&ApiStringFormat::PropertyString(&DatastoreNotify::API_SCHEMA))
diff --git a/src/backup/catalog.rs b/src/backup/catalog.rs
index 224e6bf7..a307f9d8 100644
--- a/src/backup/catalog.rs
+++ b/src/backup/catalog.rs
@@ -468,6 +468,32 @@ impl <R: Read + Seek> CatalogReader<R> {
Ok(entry_list)
}
+ /// Lookup a DirEntry from an absolute path
+ pub fn lookup_recursive(
+ &mut self,
+ path: &[u8],
+ ) -> Result<DirEntry, Error> {
+ let mut current = self.root()?;
+ if path == b"/" {
+ return Ok(current);
+ }
+
+ let components = if !path.is_empty() && path[0] == b'/' {
+ &path[1..]
+ } else {
+ path
+ }.split(|c| *c == b'/');
+
+ for comp in components {
+ if let Some(entry) = self.lookup(¤t, comp)? {
+ current = entry;
+ } else {
+ bail!("path {:?} not found in catalog", String::from_utf8_lossy(&path));
+ }
+ }
+ Ok(current)
+ }
+
/// Lockup a DirEntry inside a parent directory
pub fn lookup(
&mut self,
--
2.20.1
next prev parent reply other threads:[~2021-02-16 17:08 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-16 17:06 [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 01/22] decoder/aio: add contents() and content_size() calls Stefan Reiter
2021-02-17 7:56 ` Wolfgang Bumiller
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 02/22] decoder: add peek() Stefan Reiter
2021-02-17 8:20 ` Wolfgang Bumiller
2021-02-17 8:38 ` Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-restore-vm-data 03/22] initial commit Stefan Reiter
2021-03-15 18:35 ` [pbs-devel] applied: " Thomas Lamprecht
2021-03-16 15:33 ` Stefan Reiter
2021-02-16 17:06 ` Stefan Reiter [this message]
2021-02-17 7:50 ` [pbs-devel] applied: [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 05/22] api2/admin/datastore: accept "/" as path for root Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 06/22] api2/admin/datastore: refactor create_zip into pxar/extract Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 07/22] pxar/extract: add extract_sub_dir Stefan Reiter
2021-02-17 7:51 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 08/22] pxar/extract: add sequential variants to create_zip, extract_sub_dir Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 09/22] client: extract common functions to proxmox_client_tools module Stefan Reiter
2021-02-17 6:49 ` Dietmar Maurer
2021-02-17 7:58 ` Stefan Reiter
2021-02-17 8:50 ` Dietmar Maurer
2021-02-17 9:47 ` Stefan Reiter
2021-02-17 10:12 ` Dietmar Maurer
2021-02-17 9:13 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 10/22] proxmox_client_tools: extract 'key' from client module Stefan Reiter
2021-02-17 9:11 ` Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 11/22] file-restore: add binary and basic commands Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 12/22] file-restore: allow specifying output-format Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 13/22] rest: implement tower service for UnixStream Stefan Reiter
2021-02-17 6:52 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 14/22] client: add VsockClient to connect to virtio-vsock VMs Stefan Reiter
2021-02-17 7:24 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 15/22] file-restore-daemon: add binary with virtio-vsock API server Stefan Reiter
2021-02-17 10:17 ` Dietmar Maurer
2021-02-17 10:25 ` Dietmar Maurer
2021-02-17 10:30 ` Stefan Reiter
2021-02-17 11:13 ` Dietmar Maurer
2021-02-17 11:26 ` Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 16/22] file-restore-daemon: add watchdog module Stefan Reiter
2021-02-17 10:52 ` Wolfgang Bumiller
2021-02-17 11:14 ` Stefan Reiter
2021-02-17 11:29 ` Wolfgang Bumiller
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 17/22] file-restore-daemon: add disk module Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 18/22] file-restore: add basic VM/block device support Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 19/22] file-restore: improve logging of VM with logrotate Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 20/22] debian/client: add postinst hook to rebuild file-restore initramfs Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 21/22] file-restore(-daemon): implement list API Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 22/22] file-restore: add 'extract' command for VM file restore Stefan Reiter
2021-02-16 17:11 ` [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210216170710.31767-5-s.reiter@proxmox.com \
--to=s.reiter@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.