From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 0C0D16AEE7 for ; Mon, 25 Jan 2021 14:43:38 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0A25CAEC1 for ; Mon, 25 Jan 2021 14:43:38 +0100 (CET) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 24059AEB7 for ; Mon, 25 Jan 2021 14:43:37 +0100 (CET) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id E48E9460E1 for ; Mon, 25 Jan 2021 14:43:36 +0100 (CET) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Mon, 25 Jan 2021 14:42:54 +0100 Message-Id: <20210125134302.3394328-10-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210125134302.3394328-1-f.gruenbichler@proxmox.com> References: <20210125134302.3394328-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.026 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 09/15] pxar: factor out PxarCreateOptions X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 25 Jan 2021 13:43:38 -0000 containing the CLI parameters that are mostly passed-through from the client to our pxar archive creation wrapper in pxar::create Signed-off-by: Fabian Grünbichler --- src/bin/proxmox-backup-client.rs | 27 ++++++++++++--------------- src/bin/pxar.rs | 22 ++++++++++++++-------- src/client/pxar_backup_stream.rs | 28 ++++++---------------------- src/pxar/create.rs | 28 ++++++++++++++++++++++------ src/pxar/mod.rs | 2 +- tests/catar.rs | 10 ++++++---- 6 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index af3b8464..c3d6a0ed 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -283,23 +283,15 @@ async fn backup_directory>( dir_path: P, archive_name: &str, chunk_size: Option, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - exclude_pattern: Vec, - entries_max: usize, + pxar_create_options: proxmox_backup::pxar::PxarCreateOptions, upload_options: UploadOptions, ) -> Result { let pxar_stream = PxarBackupStream::open( dir_path.as_ref(), - device_set, - verbose, - skip_lost_and_found, catalog, - exclude_pattern, - entries_max, + pxar_create_options, )?; let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size); @@ -1039,6 +1031,15 @@ async fn create_backup( println!("Upload directory '{}' to '{}' as {}", filename, repo, target); catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?; + + let pxar_options = proxmox_backup::pxar::PxarCreateOptions { + device_set: devices.clone(), + patterns: pattern_list.clone(), + entries_max: entries_max as usize, + skip_lost_and_found, + verbose, + }; + let upload_options = UploadOptions { previous_manifest: previous_manifest.clone(), compress: true, @@ -1051,12 +1052,8 @@ async fn create_backup( &filename, &target, chunk_size_opt, - devices.clone(), - verbose, - skip_lost_and_found, catalog.clone(), - pattern_list.clone(), - entries_max as usize, + pxar_options, upload_options, ).await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs index 3d92dbe0..85606c1d 100644 --- a/src/bin/pxar.rs +++ b/src/bin/pxar.rs @@ -311,16 +311,16 @@ fn create_archive( exclude: Option>, entries_max: isize, ) -> Result<(), Error> { - let pattern_list = { + let patterns = { let input = exclude.unwrap_or_else(Vec::new); - let mut pattern_list = Vec::with_capacity(input.len()); + let mut patterns = Vec::with_capacity(input.len()); for entry in input { - pattern_list.push( + patterns.push( MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude) .map_err(|err| format_err!("error in exclude pattern: {}", err))?, ); } - pattern_list + patterns }; let device_set = if all_file_systems { @@ -329,6 +329,15 @@ fn create_archive( Some(HashSet::new()) }; + let options = proxmox_backup::pxar::PxarCreateOptions { + entries_max: entries_max as usize, + device_set, + patterns, + verbose, + skip_lost_and_found: false, + }; + + let source = PathBuf::from(source); let dir = nix::dir::Dir::open( @@ -368,18 +377,15 @@ fn create_archive( proxmox_backup::pxar::create_archive( dir, writer, - pattern_list, feature_flags, - device_set, - false, |path| { if verbose { println!("{:?}", path); } Ok(()) }, - entries_max as usize, None, + options, )?; Ok(()) diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs index aa3355fe..5fb28fd5 100644 --- a/src/client/pxar_backup_stream.rs +++ b/src/client/pxar_backup_stream.rs @@ -1,4 +1,3 @@ -use std::collections::HashSet; use std::io::Write; //use std::os::unix::io::FromRawFd; use std::path::Path; @@ -13,8 +12,6 @@ use nix::dir::Dir; use nix::fcntl::OFlag; use nix::sys::stat::Mode; -use pathpatterns::MatchEntry; - use crate::backup::CatalogWriter; /// Stream implementation to encode and upload .pxar archives. @@ -38,12 +35,8 @@ impl Drop for PxarBackupStream { impl PxarBackupStream { pub fn new( dir: Dir, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - patterns: Vec, - entries_max: usize, + options: crate::pxar::PxarCreateOptions, ) -> Result { let (tx, rx) = std::sync::mpsc::sync_channel(10); @@ -61,22 +54,21 @@ impl PxarBackupStream { crate::tools::StdChannelWriter::new(tx), ); + let verbose = options.verbose; + let writer = pxar::encoder::sync::StandardWriter::new(writer); if let Err(err) = crate::pxar::create_archive( dir, writer, - patterns, crate::pxar::Flags::DEFAULT, - device_set, - skip_lost_and_found, |path| { if verbose { println!("{:?}", path); } Ok(()) }, - entries_max, Some(&mut *catalog_guard), + options, ) { let mut error = error.lock().unwrap(); *error = Some(err.to_string()); @@ -93,23 +85,15 @@ impl PxarBackupStream { pub fn open( dirname: &Path, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - patterns: Vec, - entries_max: usize, + options: crate::pxar::PxarCreateOptions, ) -> Result { let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?; Self::new( dir, - device_set, - verbose, - skip_lost_and_found, catalog, - patterns, - entries_max, + options, ) } } diff --git a/src/pxar/create.rs b/src/pxar/create.rs index a21511c3..229e3f59 100644 --- a/src/pxar/create.rs +++ b/src/pxar/create.rs @@ -27,6 +27,22 @@ use crate::pxar::Flags; use crate::pxar::tools::assert_single_path_component; use crate::tools::{acl, fs, xattr, Fd}; +/// Pxar options for creating a pxar archive/stream +#[derive(Default, Clone)] +pub struct PxarCreateOptions { + /// Device/mountpoint st_dev numbers that should be included. None for no limitation. + pub device_set: Option>, + /// Exclusion patterns + pub patterns: Vec, + /// Maximum number of entries to hold in memory + pub entries_max: usize, + /// Skip lost+found directory + pub skip_lost_and_found: bool, + /// Verbose output + pub verbose: bool, +} + + fn detect_fs_type(fd: RawFd) -> Result { let mut fs_stat = std::mem::MaybeUninit::uninit(); let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) }; @@ -136,13 +152,10 @@ type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::Seq pub fn create_archive( source_dir: Dir, mut writer: T, - mut patterns: Vec, feature_flags: Flags, - mut device_set: Option>, - skip_lost_and_found: bool, mut callback: F, - entry_limit: usize, catalog: Option<&mut dyn BackupCatalogWriter>, + options: PxarCreateOptions, ) -> Result<(), Error> where T: pxar::encoder::SeqWrite, @@ -164,6 +177,7 @@ where ) .map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?; + let mut device_set = options.device_set.clone(); if let Some(ref mut set) = device_set { set.insert(stat.st_dev); } @@ -171,7 +185,9 @@ where let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite; let mut encoder = Encoder::new(writer, &metadata)?; - if skip_lost_and_found { + let mut patterns = options.patterns.clone(); + + if options.skip_lost_and_found { patterns.push(MatchEntry::parse_pattern( "lost+found", PatternFlag::PATH_NAME, @@ -188,7 +204,7 @@ where catalog, path: PathBuf::new(), entry_counter: 0, - entry_limit, + entry_limit: options.entries_max, current_st_dev: stat.st_dev, device_set, hardlinks: HashMap::new(), diff --git a/src/pxar/mod.rs b/src/pxar/mod.rs index 82998cf8..c10bb6fb 100644 --- a/src/pxar/mod.rs +++ b/src/pxar/mod.rs @@ -58,7 +58,7 @@ pub(crate) mod tools; mod flags; pub use flags::Flags; -pub use create::create_archive; +pub use create::{create_archive, PxarCreateOptions}; pub use extract::{extract_archive, ErrorHandler}; /// The format requires to build sorted directory lookup tables in diff --git a/tests/catar.rs b/tests/catar.rs index a6acf9af..2d9dea71 100644 --- a/tests/catar.rs +++ b/tests/catar.rs @@ -25,16 +25,18 @@ fn run_test(dir_name: &str) -> Result<(), Error> { dir_name, nix::fcntl::OFlag::O_NOFOLLOW, nix::sys::stat::Mode::empty())?; + let options = PxarCreateOptions { + entries_max: ENCODER_MAX_ENTRIES, + ..PxarCreateOptions::default() + }; + create_archive( dir, writer, - Vec::new(), Flags::DEFAULT, - None, - false, |_| Ok(()), - ENCODER_MAX_ENTRIES, None, + options, )?; Command::new("cmp") -- 2.20.1