[pbs-devel] [PATCH v3 proxmox-backup 17/20] file-restore(-daemon): implement list API

[Stefan Reiter <s.reiter@proxmox.com>]

Allows listing files and directories on a block device snapshot.
Hierarchy displayed is:

/archive.img.fidx/bucket/component/<path>
e.g.
/drive-scsi0.img.fidx/part/2/etc/passwd
(corresponding to /etc/passwd on the second partition of drive-scsi0)

Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
 src/bin/proxmox-file-restore.rs               |  19 +++
 src/bin/proxmox_file_restore/block_driver.rs  |  19 +++
 .../proxmox_file_restore/block_driver_qemu.rs |  21 +++
 src/bin/proxmox_restore_daemon/api.rs         | 131 +++++++++++++++++-
 4 files changed, 187 insertions(+), 3 deletions(-)

diff --git a/src/bin/proxmox-file-restore.rs b/src/bin/proxmox-file-restore.rs
index 0c2050f2..d45c12af 100644
--- a/src/bin/proxmox-file-restore.rs
+++ b/src/bin/proxmox-file-restore.rs
@@ -41,6 +41,7 @@ use proxmox_file_restore::*;
 enum ExtractPath {
     ListArchives,
     Pxar(String, Vec<u8>),
+    VM(String, Vec<u8>),
 }
 
 fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
@@ -67,6 +68,8 @@ fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
 
     if file.ends_with(".pxar.didx") {
         Ok(ExtractPath::Pxar(file, path))
+    } else if file.ends_with(".img.fidx") {
+        Ok(ExtractPath::VM(file, path))
     } else {
         bail!("'{}' is not supported for file-restore", file);
     }
@@ -105,6 +108,10 @@ fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
                type: CryptMode,
                optional: true,
            },
+           "driver": {
+               type: BlockDriverType,
+               optional: true,
+           },
            "output-format": {
                schema: OUTPUT_FORMAT,
                optional: true,
@@ -194,6 +201,18 @@ async fn list(
 
             helpers::list_dir_content(&mut catalog_reader, &fullpath)
         }
+        ExtractPath::VM(file, path) => {
+            let details = SnapRestoreDetails {
+                manifest,
+                repo,
+                snapshot,
+            };
+            let driver: Option<BlockDriverType> = match param.get("driver") {
+                Some(drv) => Some(serde_json::from_value(drv.clone())?),
+                None => None,
+            };
+            data_list(driver, details, file, path).await
+        }
     }?;
 
     let options = default_table_format_options()
diff --git a/src/bin/proxmox_file_restore/block_driver.rs b/src/bin/proxmox_file_restore/block_driver.rs
index 9c6fc5ac..63872f04 100644
--- a/src/bin/proxmox_file_restore/block_driver.rs
+++ b/src/bin/proxmox_file_restore/block_driver.rs
@@ -9,6 +9,7 @@ use std::hash::BuildHasher;
 use std::pin::Pin;
 
 use proxmox_backup::backup::{BackupDir, BackupManifest};
+use proxmox_backup::api2::types::ArchiveEntry;
 use proxmox_backup::client::BackupRepository;
 
 use proxmox::api::{api, cli::*};
@@ -32,6 +33,14 @@ pub type Async<R> = Pin<Box<dyn Future<Output = R> + Send>>;
 
 /// An abstract implementation for retrieving data out of a block file backup
 pub trait BlockRestoreDriver {
+    /// List ArchiveEntrys for the given image file and path
+    fn data_list(
+        &self,
+        details: SnapRestoreDetails,
+        img_file: String,
+        path: Vec<u8>,
+    ) -> Async<Result<Vec<ArchiveEntry>, Error>>;
+
     /// Return status of all running/mapped images, result value is (id, extra data), where id must
     /// match with the ones returned from list()
     fn status(&self) -> Async<Result<Vec<DriverStatus>, Error>>;
@@ -60,6 +69,16 @@ impl BlockDriverType {
 const DEFAULT_DRIVER: BlockDriverType = BlockDriverType::Qemu;
 const ALL_DRIVERS: &[BlockDriverType] = &[BlockDriverType::Qemu];
 
+pub async fn data_list(
+    driver: Option<BlockDriverType>,
+    details: SnapRestoreDetails,
+    img_file: String,
+    path: Vec<u8>,
+) -> Result<Vec<ArchiveEntry>, Error> {
+    let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve();
+    driver.data_list(details, img_file, path).await
+}
+
 #[api(
    input: {
        properties: {
diff --git a/src/bin/proxmox_file_restore/block_driver_qemu.rs b/src/bin/proxmox_file_restore/block_driver_qemu.rs
index 5fda5d6f..1a96ef10 100644
--- a/src/bin/proxmox_file_restore/block_driver_qemu.rs
+++ b/src/bin/proxmox_file_restore/block_driver_qemu.rs
@@ -9,6 +9,7 @@ use std::fs::{File, OpenOptions};
 use std::io::{prelude::*, SeekFrom};
 
 use proxmox::tools::fs::lock_file;
+use proxmox_backup::api2::types::ArchiveEntry;
 use proxmox_backup::backup::BackupDir;
 use proxmox_backup::buildcfg;
 use proxmox_backup::client::*;
@@ -215,6 +216,26 @@ async fn start_vm(cid: i32, details: &SnapRestoreDetails) -> Result<VMState, Err
 }
 
 impl BlockRestoreDriver for QemuBlockDriver {
+    fn data_list(
+        &self,
+        details: SnapRestoreDetails,
+        img_file: String,
+        mut path: Vec<u8>,
+    ) -> Async<Result<Vec<ArchiveEntry>, Error>> {
+        async move {
+            let client = ensure_running(&details).await?;
+            if !path.is_empty() && path[0] != b'/' {
+                path.insert(0, b'/');
+            }
+            let path = base64::encode(img_file.bytes().chain(path).collect::<Vec<u8>>());
+            let mut result = client
+                .get("api2/json/list", Some(json!({ "path": path })))
+                .await?;
+            serde_json::from_value(result["data"].take()).map_err(|err| err.into())
+        }
+        .boxed()
+    }
+
     fn status(&self) -> Async<Result<Vec<DriverStatus>, Error>> {
         async move {
             let mut state_map = VMStateMap::load()?;
diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs
index 4c78a0e8..2f990f36 100644
--- a/src/bin/proxmox_restore_daemon/api.rs
+++ b/src/bin/proxmox_restore_daemon/api.rs
@@ -1,19 +1,24 @@
 ///! File-restore API running inside the restore VM
-use anyhow::Error;
-use serde_json::Value;
+use anyhow::{bail, Error};
+use std::ffi::OsStr;
 use std::fs;
+use std::os::unix::ffi::OsStrExt;
+use std::path::{Path, PathBuf};
 
 use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment, SubdirMap};
 use proxmox::list_subdirs_api_method;
 
 use proxmox_backup::api2::types::*;
+use proxmox_backup::backup::DirEntryAttribute;
+use proxmox_backup::tools::fs::read_subdir;
 
-use super::{watchdog_remaining, watchdog_ping};
+use super::{disk::ResolveResult, watchdog_remaining, watchdog_ping};
 
 // NOTE: All API endpoints must have Permission::Superuser, as the configs for authentication do
 // not exist within the restore VM. Safety is guaranteed by checking a ticket via a custom ApiAuth.
 
 const SUBDIRS: SubdirMap = &[
+    ("list", &Router::new().get(&API_METHOD_LIST)),
     ("status", &Router::new().get(&API_METHOD_STATUS)),
     ("stop", &Router::new().get(&API_METHOD_STOP)),
 ];
@@ -72,3 +77,123 @@ fn stop() {
     println!("'reboot' syscall failed: {}", err);
     std::process::exit(1);
 }
+
+fn get_dir_entry(path: &Path) -> Result<DirEntryAttribute, Error> {
+    use nix::sys::stat;
+
+    let stat = stat::stat(path)?;
+    Ok(match stat.st_mode & libc::S_IFMT {
+        libc::S_IFREG => DirEntryAttribute::File {
+            size: stat.st_size as u64,
+            mtime: stat.st_mtime,
+        },
+        libc::S_IFDIR => DirEntryAttribute::Directory { start: 0 },
+        _ => bail!("unsupported file type: {}", stat.st_mode),
+    })
+}
+
+#[api(
+    input: {
+        properties: {
+            "path": {
+                type: String,
+                description: "base64-encoded path to list files and directories under",
+            },
+        },
+    },
+    access: {
+        description: "Permissions are handled outside restore VM.",
+        permission: &Permission::Superuser,
+    },
+)]
+/// List file details for given file or a list of files and directories under the given path if it
+/// points to a directory.
+fn list(
+    path: String,
+    _info: &ApiMethod,
+    _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<ArchiveEntry>, Error> {
+    watchdog_ping();
+
+    let mut res = Vec::new();
+
+    let param_path = base64::decode(path)?;
+    let mut path = param_path.clone();
+    if let Some(b'/') = path.last() {
+        path.pop();
+    }
+    let path_str = OsStr::from_bytes(&path[..]);
+    let param_path_buf = Path::new(path_str);
+
+    let mut disk_state = crate::DISK_STATE.lock().unwrap();
+    let query_result = disk_state.resolve(&param_path_buf)?;
+
+    match query_result {
+        ResolveResult::Path(vm_path) => {
+            let root_entry = get_dir_entry(&vm_path)?;
+            match root_entry {
+                DirEntryAttribute::File { .. } => {
+                    // list on file, return details
+                    res.push(ArchiveEntry::new(&param_path, &root_entry));
+                }
+                DirEntryAttribute::Directory { .. } => {
+                    // list on directory, return all contained files/dirs
+                    for f in read_subdir(libc::AT_FDCWD, &vm_path)? {
+                        if let Ok(f) = f {
+                            let name = f.file_name().to_bytes();
+                            let path = &Path::new(OsStr::from_bytes(name));
+                            if path.components().count() == 1 {
+                                // ignore '.' and '..'
+                                match path.components().next().unwrap() {
+                                    std::path::Component::CurDir
+                                    | std::path::Component::ParentDir => continue,
+                                    _ => {}
+                                }
+                            }
+
+                            let mut full_vm_path = PathBuf::new();
+                            full_vm_path.push(&vm_path);
+                            full_vm_path.push(path);
+                            let mut full_path = PathBuf::new();
+                            full_path.push(param_path_buf);
+                            full_path.push(path);
+
+                            let entry = get_dir_entry(&full_vm_path);
+                            if let Ok(entry) = entry {
+                                res.push(ArchiveEntry::new(
+                                    full_path.as_os_str().as_bytes(),
+                                    &entry,
+                                ));
+                            }
+                        }
+                    }
+                }
+                _ => unreachable!(),
+            }
+        }
+        ResolveResult::BucketTypes(types) => {
+            for t in types {
+                let mut t_path = path.clone();
+                t_path.push(b'/');
+                t_path.extend(t.as_bytes());
+                res.push(ArchiveEntry::new(
+                    &t_path[..],
+                    &DirEntryAttribute::Directory { start: 0 },
+                ));
+            }
+        }
+        ResolveResult::BucketComponents(comps) => {
+            for c in comps {
+                let mut c_path = path.clone();
+                c_path.push(b'/');
+                c_path.extend(c.as_bytes());
+                res.push(ArchiveEntry::new(
+                    &c_path[..],
+                    &DirEntryAttribute::Directory { start: 0 },
+                ));
+            }
+        }
+    }
+
+    Ok(res)
+}
-- 
2.20.1