From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id ECCE773F93; Tue, 31 May 2022 13:18:01 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id E8E608068; Tue, 31 May 2022 13:17:31 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 7D5CD802E; Tue, 31 May 2022 13:17:27 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4F8DA4398A; Tue, 31 May 2022 13:17:27 +0200 (CEST) From: Dominik Csapak To: pve-devel@lists.proxmox.com, pbs-devel@lists.proxmox.com Date: Tue, 31 May 2022 13:17:21 +0200 Message-Id: <20220531111726.2972022-3-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220531111726.2972022-1-d.csapak@proxmox.com> References: <20220531111726.2972022-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.113 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pve-devel] [PATCH proxmox 2/2] proxmox-compression: add 'tar_directory' X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 31 May 2022 11:18:02 -0000 similar to 'zip_directory', this is intended to tar a local directory, e.g. when we're in a restore vm. Signed-off-by: Dominik Csapak --- proxmox-compression/Cargo.toml | 1 + proxmox-compression/src/tar.rs | 116 +++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/proxmox-compression/Cargo.toml b/proxmox-compression/Cargo.toml index c5ad3fd..0618465 100644 --- a/proxmox-compression/Cargo.toml +++ b/proxmox-compression/Cargo.toml @@ -15,6 +15,7 @@ crc32fast = "1" endian_trait = { version = "0.6" } flate2 = "1.0" futures = "0.3" +libc = "0.2" tokio = { version = "1.6", features = [ "fs", "io-util"] } walkdir = "2" tar = "0.4" diff --git a/proxmox-compression/src/tar.rs b/proxmox-compression/src/tar.rs index 7489e43..5aa4167 100644 --- a/proxmox-compression/src/tar.rs +++ b/proxmox-compression/src/tar.rs @@ -1,9 +1,12 @@ //! tar helper +use std::collections::HashMap; use std::io; use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::str; +use anyhow::Error; + use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tar::{EntryType, Header}; @@ -162,3 +165,116 @@ where } Ok(()) } + +pub async fn tar_directory(target: W, source: &Path) -> Result<(), Error> +where + W: AsyncWrite + Unpin + Send, +{ + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + use walkdir::WalkDir; + + let base_path = source.parent().unwrap_or_else(|| Path::new("/")); + let mut encoder = Builder::new(target); + let mut hardlinks: HashMap> = HashMap::new(); // dev -> inode -> first path + + for entry in WalkDir::new(&source).into_iter() { + match entry { + Ok(entry) => { + let entry_path = entry.path().to_owned(); + let encoder = &mut encoder; + let hardlinks = &mut hardlinks; + + if let Err(err) = async move { + let entry_path_no_base = entry.path().strip_prefix(base_path)?; + let metadata = entry.metadata()?; + let mut header = Header::new_gnu(); + header.set_mode(metadata.mode()); + header.set_mtime(metadata.mtime() as u64); + header.set_uid(metadata.uid() as u64); + header.set_gid(metadata.gid() as u64); + header.set_size(0); + let dev = metadata.dev(); + + let file_type = entry.file_type(); + + if file_type.is_file() { + if metadata.nlink() > 1 { + let ino = metadata.ino(); + if let Some(map) = hardlinks.get_mut(&dev) { + if let Some(target) = map.get(&ino) { + header.set_entry_type(tar::EntryType::Link); + encoder + .add_link(&mut header, entry_path_no_base, target) + .await?; + return Ok(()); + } else { + map.insert(ino, entry_path_no_base.to_path_buf()); + } + } else { + let mut map = HashMap::new(); + map.insert(ino, entry_path_no_base.to_path_buf()); + hardlinks.insert(dev, map); + } + } + let file = tokio::fs::File::open(entry.path()).await?; + header.set_size(metadata.size()); + header.set_cksum(); + encoder + .add_entry(&mut header, entry_path_no_base, file) + .await?; + } else if file_type.is_dir() { + header.set_entry_type(EntryType::Directory); + header.set_cksum(); + encoder + .add_entry(&mut header, entry_path_no_base, tokio::io::empty()) + .await?; + } else if file_type.is_symlink() { + let target = std::fs::read_link(entry.path())?; + header.set_entry_type(EntryType::Symlink); + encoder + .add_link(&mut header, entry_path_no_base, target) + .await?; + } else if file_type.is_block_device() { + header.set_entry_type(EntryType::Block); + header.set_device_major(unsafe { libc::major(dev) })?; + header.set_device_minor(unsafe { libc::minor(dev) })?; + header.set_cksum(); + encoder + .add_entry(&mut header, entry_path_no_base, tokio::io::empty()) + .await?; + } else if file_type.is_char_device() { + header.set_entry_type(EntryType::Char); + header.set_device_major(unsafe { libc::major(dev) })?; + header.set_device_minor(unsafe { libc::minor(dev) })?; + header.set_cksum(); + encoder + .add_entry(&mut header, entry_path_no_base, tokio::io::empty()) + .await?; + } else if file_type.is_fifo() { + header.set_entry_type(EntryType::Fifo); + header.set_device_major(0)?; + header.set_device_minor(0)?; + header.set_cksum(); + encoder + .add_entry(&mut header, entry_path_no_base, tokio::io::empty()) + .await?; + } + // ignore other file_types + Ok::<_, Error>(()) + } + .await + { + eprintln!( + "zip: error encoding file or directory '{}': {}", + entry_path.display(), + err + ); + } + } + Err(err) => { + eprintln!("zip: error reading directory entry: {}", err); + } + } + } + Ok(()) +} -- 2.30.2