From: Stefan Reiter <s.reiter@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 21/22] file-restore(-daemon): implement list API
Date: Tue, 16 Feb 2021 18:07:09 +0100 [thread overview]
Message-ID: <20210216170710.31767-22-s.reiter@proxmox.com> (raw)
In-Reply-To: <20210216170710.31767-1-s.reiter@proxmox.com>
Allows listing files and directories on a block device snapshot.
Hierarchy displayed is:
/archive.img.fidx/bucket/component/<path>
e.g.
/drive-scsi0.img.fidx/part/2/etc/passwd
(corresponding to /etc/passwd on the second partition of drive-scsi0)
Signed-off-by: Stefan Reiter <s.reiter@proxmox.com>
---
src/bin/proxmox-file-restore.rs | 19 +++
src/bin/proxmox_file_restore/block_driver.rs | 19 +++
.../proxmox_file_restore/block_driver_qemu.rs | 21 +++
src/bin/proxmox_restore_daemon/api.rs | 133 +++++++++++++++++-
4 files changed, 188 insertions(+), 4 deletions(-)
diff --git a/src/bin/proxmox-file-restore.rs b/src/bin/proxmox-file-restore.rs
index 767cc057..232931d9 100644
--- a/src/bin/proxmox-file-restore.rs
+++ b/src/bin/proxmox-file-restore.rs
@@ -38,6 +38,7 @@ use proxmox_file_restore::*;
enum ExtractPath {
ListArchives,
Pxar(String, Vec<u8>),
+ VM(String, Vec<u8>),
}
fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
@@ -64,6 +65,8 @@ fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
if file.ends_with(".pxar.didx") {
Ok(ExtractPath::Pxar(file, path))
+ } else if file.ends_with(".img.fidx") {
+ Ok(ExtractPath::VM(file, path))
} else {
bail!("'{}' is not supported for file-restore", file);
}
@@ -102,6 +105,10 @@ fn parse_path(path: String, base64: bool) -> Result<ExtractPath, Error> {
type: CryptMode,
optional: true,
},
+ "driver": {
+ type: BlockDriverType,
+ optional: true,
+ },
"output-format": {
schema: OUTPUT_FORMAT,
optional: true,
@@ -190,6 +197,18 @@ async fn list(param: Value) -> Result<Value, Error> {
helpers::list_dir_content(&mut catalog_reader, &fullpath)
}
+ ExtractPath::VM(file, path) => {
+ let details = SnapRestoreDetails {
+ manifest,
+ repo,
+ snapshot,
+ };
+ let driver: Option<BlockDriverType> = match param.get("driver") {
+ Some(drv) => Some(serde_json::from_value(drv.clone())?),
+ None => None,
+ };
+ data_list(driver, details, file, path).await
+ }
}?;
let options = default_table_format_options()
diff --git a/src/bin/proxmox_file_restore/block_driver.rs b/src/bin/proxmox_file_restore/block_driver.rs
index f2d5b00e..5ed35f25 100644
--- a/src/bin/proxmox_file_restore/block_driver.rs
+++ b/src/bin/proxmox_file_restore/block_driver.rs
@@ -8,6 +8,7 @@ use std::future::Future;
use std::hash::BuildHasher;
use std::pin::Pin;
+use proxmox_backup::api2::types::ArchiveEntry;
use proxmox_backup::backup::{backup_user, BackupDir, BackupManifest};
use proxmox_backup::buildcfg;
use proxmox_backup::client::BackupRepository;
@@ -28,6 +29,14 @@ pub type Async<R> = Pin<Box<dyn Future<Output = R> + Send>>;
/// An abstract implementation for retrieving data out of a block file backup
pub trait BlockRestoreDriver {
+ /// List ArchiveEntrys for the given image file and path
+ fn data_list(
+ &self,
+ details: SnapRestoreDetails,
+ img_file: String,
+ path: Vec<u8>,
+ ) -> Async<Result<Vec<ArchiveEntry>, Error>>;
+
/// Return status of all running/mapped images, result value is (id, extra data), where id must
/// match with the ones returned from list()
fn status(&self) -> Async<Result<Vec<(String, Value)>, Error>>;
@@ -56,6 +65,16 @@ impl BlockDriverType {
const DEFAULT_DRIVER: BlockDriverType = BlockDriverType::Qemu;
const ALL_DRIVERS: &[BlockDriverType] = &[BlockDriverType::Qemu];
+pub async fn data_list(
+ driver: Option<BlockDriverType>,
+ details: SnapRestoreDetails,
+ img_file: String,
+ path: Vec<u8>,
+) -> Result<Vec<ArchiveEntry>, Error> {
+ let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve();
+ driver.data_list(details, img_file, path).await
+}
+
#[api(
input: {
properties: {
diff --git a/src/bin/proxmox_file_restore/block_driver_qemu.rs b/src/bin/proxmox_file_restore/block_driver_qemu.rs
index d406d523..3277af5d 100644
--- a/src/bin/proxmox_file_restore/block_driver_qemu.rs
+++ b/src/bin/proxmox_file_restore/block_driver_qemu.rs
@@ -13,6 +13,7 @@ use std::time::Duration;
use tokio::time;
use proxmox::tools::fs::{file_read_string, lock_file, make_tmp_file, CreateOptions};
+use proxmox_backup::api2::types::ArchiveEntry;
use proxmox_backup::backup::BackupDir;
use proxmox_backup::buildcfg;
use proxmox_backup::client::*;
@@ -348,6 +349,26 @@ async fn start_vm(
}
impl BlockRestoreDriver for QemuBlockDriver {
+ fn data_list(
+ &self,
+ details: SnapRestoreDetails,
+ img_file: String,
+ mut path: Vec<u8>,
+ ) -> Async<Result<Vec<ArchiveEntry>, Error>> {
+ async move {
+ let client = ensure_running(&details).await?;
+ if !path.is_empty() && path[0] != b'/' {
+ path.insert(0, b'/');
+ }
+ let path = base64::encode(img_file.bytes().chain(path).collect::<Vec<u8>>());
+ let mut result = client
+ .get("api2/json/list", Some(json!({ "path": path })))
+ .await?;
+ serde_json::from_value(result["data"].take()).map_err(|err| err.into())
+ }
+ .boxed()
+ }
+
fn status(&self) -> Async<Result<Vec<(String, Value)>, Error>> {
async move {
let mut state_map = VMStateMap::load()?;
diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs
index 8eb727df..125b5bfb 100644
--- a/src/bin/proxmox_restore_daemon/api.rs
+++ b/src/bin/proxmox_restore_daemon/api.rs
@@ -1,20 +1,27 @@
///! File-restore API running inside the restore VM
-use anyhow::Error;
-use serde_json::Value;
+use anyhow::{bail, Error};
+use std::ffi::OsStr;
use std::fs;
+use std::os::unix::ffi::OsStrExt;
+use std::path::{Path, PathBuf};
use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment, SubdirMap};
use proxmox::list_subdirs_api_method;
use proxmox_backup::api2::types::*;
+use proxmox_backup::backup::DirEntryAttribute;
+use proxmox_backup::tools::fs::read_subdir;
-use super::{watchdog_remaining, watchdog_undo_ping};
+use super::{disk::ResolveResult, watchdog_remaining, watchdog_undo_ping};
// NOTE: All API endpoints must have Permission::World, as the configs for authentication do not
// exist within the restore VM. Safety is guaranteed since we use a low port, so only root on the
// host can contact us - and there the proxmox-backup-client validates permissions already.
-const SUBDIRS: SubdirMap = &[("status", &Router::new().get(&API_METHOD_STATUS))];
+const SUBDIRS: SubdirMap = &[
+ ("list", &Router::new().get(&API_METHOD_LIST)),
+ ("status", &Router::new().get(&API_METHOD_STATUS)),
+];
pub const ROUTER: Router = Router::new()
.get(&list_subdirs_api_method!(SUBDIRS))
@@ -55,3 +62,121 @@ fn status(keep_timeout: bool) -> Result<RestoreDaemonStatus, Error> {
timeout: watchdog_remaining(false),
})
}
+
+fn get_dir_entry(path: &Path) -> Result<DirEntryAttribute, Error> {
+ use nix::sys::stat;
+
+ let stat = stat::stat(path)?;
+ Ok(match stat.st_mode & libc::S_IFMT {
+ libc::S_IFREG => DirEntryAttribute::File {
+ size: stat.st_size as u64,
+ mtime: stat.st_mtime,
+ },
+ libc::S_IFDIR => DirEntryAttribute::Directory { start: 0 },
+ _ => bail!("unsupported file type: {}", stat.st_mode),
+ })
+}
+
+#[api(
+ input: {
+ properties: {
+ "path": {
+ type: String,
+ description: "base64-encoded path to list files and directories under",
+ },
+ },
+ },
+ access: {
+ description: "Permissions are handled outside restore VM.",
+ permission: &Permission::World,
+ },
+)]
+/// List file details for given file or a list of files and directories under the given path if it
+/// points to a directory.
+fn list(
+ path: String,
+ _info: &ApiMethod,
+ _rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<ArchiveEntry>, Error> {
+ let mut res = Vec::new();
+
+ let param_path = base64::decode(path)?;
+ let mut path = param_path.clone();
+ if let Some(b'/') = path.last() {
+ path.pop();
+ }
+ let path_str = OsStr::from_bytes(&path[..]);
+ let param_path_buf = Path::new(path_str);
+
+ let mut disk_state = crate::DISK_STATE.lock().unwrap();
+ let query_result = disk_state.resolve(¶m_path_buf)?;
+
+ match query_result {
+ ResolveResult::Path(vm_path) => {
+ let root_entry = get_dir_entry(&vm_path)?;
+ match root_entry {
+ DirEntryAttribute::File { .. } => {
+ // list on file, return details
+ res.push(ArchiveEntry::new(¶m_path, &root_entry));
+ }
+ DirEntryAttribute::Directory { .. } => {
+ // list on directory, return all contained files/dirs
+ for f in read_subdir(libc::AT_FDCWD, &vm_path)? {
+ if let Ok(f) = f {
+ let name = f.file_name().to_bytes();
+ let path = &Path::new(OsStr::from_bytes(name));
+ if path.components().count() == 1 {
+ // ignore '.' and '..'
+ match path.components().next().unwrap() {
+ std::path::Component::CurDir
+ | std::path::Component::ParentDir => continue,
+ _ => {}
+ }
+ }
+
+ let mut full_vm_path = PathBuf::new();
+ full_vm_path.push(&vm_path);
+ full_vm_path.push(path);
+ let mut full_path = PathBuf::new();
+ full_path.push(param_path_buf);
+ full_path.push(path);
+
+ let entry = get_dir_entry(&full_vm_path);
+ if let Ok(entry) = entry {
+ res.push(ArchiveEntry::new(
+ full_path.as_os_str().as_bytes(),
+ &entry,
+ ));
+ }
+ }
+ }
+ }
+ _ => unreachable!(),
+ }
+ }
+ ResolveResult::BucketTypes(types) => {
+ for t in types {
+ let mut t_path = path.clone();
+ t_path.push(b'/');
+ t_path.extend(t.as_bytes());
+ res.push(ArchiveEntry::new(
+ &t_path[..],
+ &DirEntryAttribute::Directory { start: 0 },
+ ));
+ }
+ }
+ ResolveResult::BucketComponents(comps) => {
+ for c in comps {
+ let mut c_path = path.clone();
+ c_path.push(b'/');
+ c_path.extend(c.as_bytes());
+ res.push(ArchiveEntry::new(
+ &c_path[..],
+ &DirEntryAttribute::Directory { start: 0 },
+ ));
+ }
+ }
+ }
+
+ Ok(res)
+}
--
2.20.1
next prev parent reply other threads:[~2021-02-16 17:08 UTC|newest]
Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-02-16 17:06 [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 01/22] decoder/aio: add contents() and content_size() calls Stefan Reiter
2021-02-17 7:56 ` Wolfgang Bumiller
2021-02-16 17:06 ` [pbs-devel] [PATCH pxar 02/22] decoder: add peek() Stefan Reiter
2021-02-17 8:20 ` Wolfgang Bumiller
2021-02-17 8:38 ` Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-restore-vm-data 03/22] initial commit Stefan Reiter
2021-03-15 18:35 ` [pbs-devel] applied: " Thomas Lamprecht
2021-03-16 15:33 ` Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 04/22] api2/admin/datastore: refactor list_dir_content in catalog_reader Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 05/22] api2/admin/datastore: accept "/" as path for root Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 06/22] api2/admin/datastore: refactor create_zip into pxar/extract Stefan Reiter
2021-02-17 7:50 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 07/22] pxar/extract: add extract_sub_dir Stefan Reiter
2021-02-17 7:51 ` [pbs-devel] applied: " Thomas Lamprecht
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 08/22] pxar/extract: add sequential variants to create_zip, extract_sub_dir Stefan Reiter
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 09/22] client: extract common functions to proxmox_client_tools module Stefan Reiter
2021-02-17 6:49 ` Dietmar Maurer
2021-02-17 7:58 ` Stefan Reiter
2021-02-17 8:50 ` Dietmar Maurer
2021-02-17 9:47 ` Stefan Reiter
2021-02-17 10:12 ` Dietmar Maurer
2021-02-17 9:13 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 10/22] proxmox_client_tools: extract 'key' from client module Stefan Reiter
2021-02-17 9:11 ` Dietmar Maurer
2021-02-16 17:06 ` [pbs-devel] [PATCH proxmox-backup 11/22] file-restore: add binary and basic commands Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 12/22] file-restore: allow specifying output-format Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 13/22] rest: implement tower service for UnixStream Stefan Reiter
2021-02-17 6:52 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 14/22] client: add VsockClient to connect to virtio-vsock VMs Stefan Reiter
2021-02-17 7:24 ` [pbs-devel] applied: " Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 15/22] file-restore-daemon: add binary with virtio-vsock API server Stefan Reiter
2021-02-17 10:17 ` Dietmar Maurer
2021-02-17 10:25 ` Dietmar Maurer
2021-02-17 10:30 ` Stefan Reiter
2021-02-17 11:13 ` Dietmar Maurer
2021-02-17 11:26 ` Dietmar Maurer
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 16/22] file-restore-daemon: add watchdog module Stefan Reiter
2021-02-17 10:52 ` Wolfgang Bumiller
2021-02-17 11:14 ` Stefan Reiter
2021-02-17 11:29 ` Wolfgang Bumiller
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 17/22] file-restore-daemon: add disk module Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 18/22] file-restore: add basic VM/block device support Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 19/22] file-restore: improve logging of VM with logrotate Stefan Reiter
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 20/22] debian/client: add postinst hook to rebuild file-restore initramfs Stefan Reiter
2021-02-16 17:07 ` Stefan Reiter [this message]
2021-02-16 17:07 ` [pbs-devel] [PATCH proxmox-backup 22/22] file-restore: add 'extract' command for VM file restore Stefan Reiter
2021-02-16 17:11 ` [pbs-devel] [PATCH 00/22] Single file restore for VM images Stefan Reiter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210216170710.31767-22-s.reiter@proxmox.com \
--to=s.reiter@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox