From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 5FCEE6A627 for ; Tue, 16 Feb 2021 18:08:16 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 4F00F1910B for ; Tue, 16 Feb 2021 18:07:46 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id C0C1E18E79 for ; Tue, 16 Feb 2021 18:07:33 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 89AAC461C0 for ; Tue, 16 Feb 2021 18:07:33 +0100 (CET) From: Stefan Reiter To: pbs-devel@lists.proxmox.com Date: Tue, 16 Feb 2021 18:07:09 +0100 Message-Id: <20210216170710.31767-22-s.reiter@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210216170710.31767-1-s.reiter@proxmox.com> References: <20210216170710.31767-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.028 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [api.rs, proxmox-file-restore.rs] Subject: [pbs-devel] [PATCH proxmox-backup 21/22] file-restore(-daemon): implement list API X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 16 Feb 2021 17:08:16 -0000 Allows listing files and directories on a block device snapshot. Hierarchy displayed is: /archive.img.fidx/bucket/component/ e.g. /drive-scsi0.img.fidx/part/2/etc/passwd (corresponding to /etc/passwd on the second partition of drive-scsi0) Signed-off-by: Stefan Reiter --- src/bin/proxmox-file-restore.rs | 19 +++ src/bin/proxmox_file_restore/block_driver.rs | 19 +++ .../proxmox_file_restore/block_driver_qemu.rs | 21 +++ src/bin/proxmox_restore_daemon/api.rs | 133 +++++++++++++++++- 4 files changed, 188 insertions(+), 4 deletions(-) diff --git a/src/bin/proxmox-file-restore.rs b/src/bin/proxmox-file-restore.rs index 767cc057..232931d9 100644 --- a/src/bin/proxmox-file-restore.rs +++ b/src/bin/proxmox-file-restore.rs @@ -38,6 +38,7 @@ use proxmox_file_restore::*; enum ExtractPath { ListArchives, Pxar(String, Vec), + VM(String, Vec), } fn parse_path(path: String, base64: bool) -> Result { @@ -64,6 +65,8 @@ fn parse_path(path: String, base64: bool) -> Result { if file.ends_with(".pxar.didx") { Ok(ExtractPath::Pxar(file, path)) + } else if file.ends_with(".img.fidx") { + Ok(ExtractPath::VM(file, path)) } else { bail!("'{}' is not supported for file-restore", file); } @@ -102,6 +105,10 @@ fn parse_path(path: String, base64: bool) -> Result { type: CryptMode, optional: true, }, + "driver": { + type: BlockDriverType, + optional: true, + }, "output-format": { schema: OUTPUT_FORMAT, optional: true, @@ -190,6 +197,18 @@ async fn list(param: Value) -> Result { helpers::list_dir_content(&mut catalog_reader, &fullpath) } + ExtractPath::VM(file, path) => { + let details = SnapRestoreDetails { + manifest, + repo, + snapshot, + }; + let driver: Option = match param.get("driver") { + Some(drv) => Some(serde_json::from_value(drv.clone())?), + None => None, + }; + data_list(driver, details, file, path).await + } }?; let options = default_table_format_options() diff --git a/src/bin/proxmox_file_restore/block_driver.rs b/src/bin/proxmox_file_restore/block_driver.rs index f2d5b00e..5ed35f25 100644 --- a/src/bin/proxmox_file_restore/block_driver.rs +++ b/src/bin/proxmox_file_restore/block_driver.rs @@ -8,6 +8,7 @@ use std::future::Future; use std::hash::BuildHasher; use std::pin::Pin; +use proxmox_backup::api2::types::ArchiveEntry; use proxmox_backup::backup::{backup_user, BackupDir, BackupManifest}; use proxmox_backup::buildcfg; use proxmox_backup::client::BackupRepository; @@ -28,6 +29,14 @@ pub type Async = Pin + Send>>; /// An abstract implementation for retrieving data out of a block file backup pub trait BlockRestoreDriver { + /// List ArchiveEntrys for the given image file and path + fn data_list( + &self, + details: SnapRestoreDetails, + img_file: String, + path: Vec, + ) -> Async, Error>>; + /// Return status of all running/mapped images, result value is (id, extra data), where id must /// match with the ones returned from list() fn status(&self) -> Async, Error>>; @@ -56,6 +65,16 @@ impl BlockDriverType { const DEFAULT_DRIVER: BlockDriverType = BlockDriverType::Qemu; const ALL_DRIVERS: &[BlockDriverType] = &[BlockDriverType::Qemu]; +pub async fn data_list( + driver: Option, + details: SnapRestoreDetails, + img_file: String, + path: Vec, +) -> Result, Error> { + let driver = driver.unwrap_or(DEFAULT_DRIVER).resolve(); + driver.data_list(details, img_file, path).await +} + #[api( input: { properties: { diff --git a/src/bin/proxmox_file_restore/block_driver_qemu.rs b/src/bin/proxmox_file_restore/block_driver_qemu.rs index d406d523..3277af5d 100644 --- a/src/bin/proxmox_file_restore/block_driver_qemu.rs +++ b/src/bin/proxmox_file_restore/block_driver_qemu.rs @@ -13,6 +13,7 @@ use std::time::Duration; use tokio::time; use proxmox::tools::fs::{file_read_string, lock_file, make_tmp_file, CreateOptions}; +use proxmox_backup::api2::types::ArchiveEntry; use proxmox_backup::backup::BackupDir; use proxmox_backup::buildcfg; use proxmox_backup::client::*; @@ -348,6 +349,26 @@ async fn start_vm( } impl BlockRestoreDriver for QemuBlockDriver { + fn data_list( + &self, + details: SnapRestoreDetails, + img_file: String, + mut path: Vec, + ) -> Async, Error>> { + async move { + let client = ensure_running(&details).await?; + if !path.is_empty() && path[0] != b'/' { + path.insert(0, b'/'); + } + let path = base64::encode(img_file.bytes().chain(path).collect::>()); + let mut result = client + .get("api2/json/list", Some(json!({ "path": path }))) + .await?; + serde_json::from_value(result["data"].take()).map_err(|err| err.into()) + } + .boxed() + } + fn status(&self) -> Async, Error>> { async move { let mut state_map = VMStateMap::load()?; diff --git a/src/bin/proxmox_restore_daemon/api.rs b/src/bin/proxmox_restore_daemon/api.rs index 8eb727df..125b5bfb 100644 --- a/src/bin/proxmox_restore_daemon/api.rs +++ b/src/bin/proxmox_restore_daemon/api.rs @@ -1,20 +1,27 @@ ///! File-restore API running inside the restore VM -use anyhow::Error; -use serde_json::Value; +use anyhow::{bail, Error}; +use std::ffi::OsStr; use std::fs; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; use proxmox::api::{api, ApiMethod, Permission, Router, RpcEnvironment, SubdirMap}; use proxmox::list_subdirs_api_method; use proxmox_backup::api2::types::*; +use proxmox_backup::backup::DirEntryAttribute; +use proxmox_backup::tools::fs::read_subdir; -use super::{watchdog_remaining, watchdog_undo_ping}; +use super::{disk::ResolveResult, watchdog_remaining, watchdog_undo_ping}; // NOTE: All API endpoints must have Permission::World, as the configs for authentication do not // exist within the restore VM. Safety is guaranteed since we use a low port, so only root on the // host can contact us - and there the proxmox-backup-client validates permissions already. -const SUBDIRS: SubdirMap = &[("status", &Router::new().get(&API_METHOD_STATUS))]; +const SUBDIRS: SubdirMap = &[ + ("list", &Router::new().get(&API_METHOD_LIST)), + ("status", &Router::new().get(&API_METHOD_STATUS)), +]; pub const ROUTER: Router = Router::new() .get(&list_subdirs_api_method!(SUBDIRS)) @@ -55,3 +62,121 @@ fn status(keep_timeout: bool) -> Result { timeout: watchdog_remaining(false), }) } + +fn get_dir_entry(path: &Path) -> Result { + use nix::sys::stat; + + let stat = stat::stat(path)?; + Ok(match stat.st_mode & libc::S_IFMT { + libc::S_IFREG => DirEntryAttribute::File { + size: stat.st_size as u64, + mtime: stat.st_mtime, + }, + libc::S_IFDIR => DirEntryAttribute::Directory { start: 0 }, + _ => bail!("unsupported file type: {}", stat.st_mode), + }) +} + +#[api( + input: { + properties: { + "path": { + type: String, + description: "base64-encoded path to list files and directories under", + }, + }, + }, + access: { + description: "Permissions are handled outside restore VM.", + permission: &Permission::World, + }, +)] +/// List file details for given file or a list of files and directories under the given path if it +/// points to a directory. +fn list( + path: String, + _info: &ApiMethod, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let mut res = Vec::new(); + + let param_path = base64::decode(path)?; + let mut path = param_path.clone(); + if let Some(b'/') = path.last() { + path.pop(); + } + let path_str = OsStr::from_bytes(&path[..]); + let param_path_buf = Path::new(path_str); + + let mut disk_state = crate::DISK_STATE.lock().unwrap(); + let query_result = disk_state.resolve(¶m_path_buf)?; + + match query_result { + ResolveResult::Path(vm_path) => { + let root_entry = get_dir_entry(&vm_path)?; + match root_entry { + DirEntryAttribute::File { .. } => { + // list on file, return details + res.push(ArchiveEntry::new(¶m_path, &root_entry)); + } + DirEntryAttribute::Directory { .. } => { + // list on directory, return all contained files/dirs + for f in read_subdir(libc::AT_FDCWD, &vm_path)? { + if let Ok(f) = f { + let name = f.file_name().to_bytes(); + let path = &Path::new(OsStr::from_bytes(name)); + if path.components().count() == 1 { + // ignore '.' and '..' + match path.components().next().unwrap() { + std::path::Component::CurDir + | std::path::Component::ParentDir => continue, + _ => {} + } + } + + let mut full_vm_path = PathBuf::new(); + full_vm_path.push(&vm_path); + full_vm_path.push(path); + let mut full_path = PathBuf::new(); + full_path.push(param_path_buf); + full_path.push(path); + + let entry = get_dir_entry(&full_vm_path); + if let Ok(entry) = entry { + res.push(ArchiveEntry::new( + full_path.as_os_str().as_bytes(), + &entry, + )); + } + } + } + } + _ => unreachable!(), + } + } + ResolveResult::BucketTypes(types) => { + for t in types { + let mut t_path = path.clone(); + t_path.push(b'/'); + t_path.extend(t.as_bytes()); + res.push(ArchiveEntry::new( + &t_path[..], + &DirEntryAttribute::Directory { start: 0 }, + )); + } + } + ResolveResult::BucketComponents(comps) => { + for c in comps { + let mut c_path = path.clone(); + c_path.push(b'/'); + c_path.extend(c.as_bytes()); + res.push(ArchiveEntry::new( + &c_path[..], + &DirEntryAttribute::Directory { start: 0 }, + )); + } + } + } + + Ok(res) +} -- 2.20.1