From: "Fabian Grünbichler" <f.gruenbichler@proxmox.com>
To: pbs-devel@lists.proxmox.com
Subject: [pbs-devel] [PATCH proxmox-backup 09/15] pxar: factor out PxarCreateOptions
Date: Mon, 25 Jan 2021 14:42:54 +0100 [thread overview]
Message-ID: <20210125134302.3394328-10-f.gruenbichler@proxmox.com> (raw)
In-Reply-To: <20210125134302.3394328-1-f.gruenbichler@proxmox.com>
containing the CLI parameters that are mostly passed-through from the
client to our pxar archive creation wrapper in pxar::create
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/bin/proxmox-backup-client.rs | 27 ++++++++++++---------------
src/bin/pxar.rs | 22 ++++++++++++++--------
src/client/pxar_backup_stream.rs | 28 ++++++----------------------
src/pxar/create.rs | 28 ++++++++++++++++++++++------
src/pxar/mod.rs | 2 +-
tests/catar.rs | 10 ++++++----
6 files changed, 61 insertions(+), 56 deletions(-)
diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs
index af3b8464..c3d6a0ed 100644
--- a/src/bin/proxmox-backup-client.rs
+++ b/src/bin/proxmox-backup-client.rs
@@ -283,23 +283,15 @@ async fn backup_directory<P: AsRef<Path>>(
dir_path: P,
archive_name: &str,
chunk_size: Option<usize>,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<crate::tools::StdChannelWriter>>>,
- exclude_pattern: Vec<MatchEntry>,
- entries_max: usize,
+ pxar_create_options: proxmox_backup::pxar::PxarCreateOptions,
upload_options: UploadOptions,
) -> Result<BackupStats, Error> {
let pxar_stream = PxarBackupStream::open(
dir_path.as_ref(),
- device_set,
- verbose,
- skip_lost_and_found,
catalog,
- exclude_pattern,
- entries_max,
+ pxar_create_options,
)?;
let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
@@ -1039,6 +1031,15 @@ async fn create_backup(
println!("Upload directory '{}' to '{}' as {}", filename, repo, target);
catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?;
+
+ let pxar_options = proxmox_backup::pxar::PxarCreateOptions {
+ device_set: devices.clone(),
+ patterns: pattern_list.clone(),
+ entries_max: entries_max as usize,
+ skip_lost_and_found,
+ verbose,
+ };
+
let upload_options = UploadOptions {
previous_manifest: previous_manifest.clone(),
compress: true,
@@ -1051,12 +1052,8 @@ async fn create_backup(
&filename,
&target,
chunk_size_opt,
- devices.clone(),
- verbose,
- skip_lost_and_found,
catalog.clone(),
- pattern_list.clone(),
- entries_max as usize,
+ pxar_options,
upload_options,
).await?;
manifest.add_file(target, stats.size, stats.csum, crypt_mode)?;
diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs
index 3d92dbe0..85606c1d 100644
--- a/src/bin/pxar.rs
+++ b/src/bin/pxar.rs
@@ -311,16 +311,16 @@ fn create_archive(
exclude: Option<Vec<String>>,
entries_max: isize,
) -> Result<(), Error> {
- let pattern_list = {
+ let patterns = {
let input = exclude.unwrap_or_else(Vec::new);
- let mut pattern_list = Vec::with_capacity(input.len());
+ let mut patterns = Vec::with_capacity(input.len());
for entry in input {
- pattern_list.push(
+ patterns.push(
MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude)
.map_err(|err| format_err!("error in exclude pattern: {}", err))?,
);
}
- pattern_list
+ patterns
};
let device_set = if all_file_systems {
@@ -329,6 +329,15 @@ fn create_archive(
Some(HashSet::new())
};
+ let options = proxmox_backup::pxar::PxarCreateOptions {
+ entries_max: entries_max as usize,
+ device_set,
+ patterns,
+ verbose,
+ skip_lost_and_found: false,
+ };
+
+
let source = PathBuf::from(source);
let dir = nix::dir::Dir::open(
@@ -368,18 +377,15 @@ fn create_archive(
proxmox_backup::pxar::create_archive(
dir,
writer,
- pattern_list,
feature_flags,
- device_set,
- false,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
- entries_max as usize,
None,
+ options,
)?;
Ok(())
diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs
index aa3355fe..5fb28fd5 100644
--- a/src/client/pxar_backup_stream.rs
+++ b/src/client/pxar_backup_stream.rs
@@ -1,4 +1,3 @@
-use std::collections::HashSet;
use std::io::Write;
//use std::os::unix::io::FromRawFd;
use std::path::Path;
@@ -13,8 +12,6 @@ use nix::dir::Dir;
use nix::fcntl::OFlag;
use nix::sys::stat::Mode;
-use pathpatterns::MatchEntry;
-
use crate::backup::CatalogWriter;
/// Stream implementation to encode and upload .pxar archives.
@@ -38,12 +35,8 @@ impl Drop for PxarBackupStream {
impl PxarBackupStream {
pub fn new<W: Write + Send + 'static>(
dir: Dir,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
- patterns: Vec<MatchEntry>,
- entries_max: usize,
+ options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let (tx, rx) = std::sync::mpsc::sync_channel(10);
@@ -61,22 +54,21 @@ impl PxarBackupStream {
crate::tools::StdChannelWriter::new(tx),
);
+ let verbose = options.verbose;
+
let writer = pxar::encoder::sync::StandardWriter::new(writer);
if let Err(err) = crate::pxar::create_archive(
dir,
writer,
- patterns,
crate::pxar::Flags::DEFAULT,
- device_set,
- skip_lost_and_found,
|path| {
if verbose {
println!("{:?}", path);
}
Ok(())
},
- entries_max,
Some(&mut *catalog_guard),
+ options,
) {
let mut error = error.lock().unwrap();
*error = Some(err.to_string());
@@ -93,23 +85,15 @@ impl PxarBackupStream {
pub fn open<W: Write + Send + 'static>(
dirname: &Path,
- device_set: Option<HashSet<u64>>,
- verbose: bool,
- skip_lost_and_found: bool,
catalog: Arc<Mutex<CatalogWriter<W>>>,
- patterns: Vec<MatchEntry>,
- entries_max: usize,
+ options: crate::pxar::PxarCreateOptions,
) -> Result<Self, Error> {
let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
Self::new(
dir,
- device_set,
- verbose,
- skip_lost_and_found,
catalog,
- patterns,
- entries_max,
+ options,
)
}
}
diff --git a/src/pxar/create.rs b/src/pxar/create.rs
index a21511c3..229e3f59 100644
--- a/src/pxar/create.rs
+++ b/src/pxar/create.rs
@@ -27,6 +27,22 @@ use crate::pxar::Flags;
use crate::pxar::tools::assert_single_path_component;
use crate::tools::{acl, fs, xattr, Fd};
+/// Pxar options for creating a pxar archive/stream
+#[derive(Default, Clone)]
+pub struct PxarCreateOptions {
+ /// Device/mountpoint st_dev numbers that should be included. None for no limitation.
+ pub device_set: Option<HashSet<u64>>,
+ /// Exclusion patterns
+ pub patterns: Vec<MatchEntry>,
+ /// Maximum number of entries to hold in memory
+ pub entries_max: usize,
+ /// Skip lost+found directory
+ pub skip_lost_and_found: bool,
+ /// Verbose output
+ pub verbose: bool,
+}
+
+
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
let mut fs_stat = std::mem::MaybeUninit::uninit();
let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
@@ -136,13 +152,10 @@ type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::Seq
pub fn create_archive<T, F>(
source_dir: Dir,
mut writer: T,
- mut patterns: Vec<MatchEntry>,
feature_flags: Flags,
- mut device_set: Option<HashSet<u64>>,
- skip_lost_and_found: bool,
mut callback: F,
- entry_limit: usize,
catalog: Option<&mut dyn BackupCatalogWriter>,
+ options: PxarCreateOptions,
) -> Result<(), Error>
where
T: pxar::encoder::SeqWrite,
@@ -164,6 +177,7 @@ where
)
.map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
+ let mut device_set = options.device_set.clone();
if let Some(ref mut set) = device_set {
set.insert(stat.st_dev);
}
@@ -171,7 +185,9 @@ where
let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
let mut encoder = Encoder::new(writer, &metadata)?;
- if skip_lost_and_found {
+ let mut patterns = options.patterns.clone();
+
+ if options.skip_lost_and_found {
patterns.push(MatchEntry::parse_pattern(
"lost+found",
PatternFlag::PATH_NAME,
@@ -188,7 +204,7 @@ where
catalog,
path: PathBuf::new(),
entry_counter: 0,
- entry_limit,
+ entry_limit: options.entries_max,
current_st_dev: stat.st_dev,
device_set,
hardlinks: HashMap::new(),
diff --git a/src/pxar/mod.rs b/src/pxar/mod.rs
index 82998cf8..c10bb6fb 100644
--- a/src/pxar/mod.rs
+++ b/src/pxar/mod.rs
@@ -58,7 +58,7 @@ pub(crate) mod tools;
mod flags;
pub use flags::Flags;
-pub use create::create_archive;
+pub use create::{create_archive, PxarCreateOptions};
pub use extract::{extract_archive, ErrorHandler};
/// The format requires to build sorted directory lookup tables in
diff --git a/tests/catar.rs b/tests/catar.rs
index a6acf9af..2d9dea71 100644
--- a/tests/catar.rs
+++ b/tests/catar.rs
@@ -25,16 +25,18 @@ fn run_test(dir_name: &str) -> Result<(), Error> {
dir_name, nix::fcntl::OFlag::O_NOFOLLOW,
nix::sys::stat::Mode::empty())?;
+ let options = PxarCreateOptions {
+ entries_max: ENCODER_MAX_ENTRIES,
+ ..PxarCreateOptions::default()
+ };
+
create_archive(
dir,
writer,
- Vec::new(),
Flags::DEFAULT,
- None,
- false,
|_| Ok(()),
- ENCODER_MAX_ENTRIES,
None,
+ options,
)?;
Command::new("cmp")
--
2.20.1
next prev parent reply other threads:[~2021-01-25 13:43 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-01-25 13:42 [pbs-devel] [PATCH proxmox-backup(-qemu) 00/17] clippy refactorings Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 01/15] report: type-alias function call tuple Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 02/15] broadcast_future: refactor broadcast/future binding Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 03/15] client: refactor catalog upload spawning Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 04/15] allow complex Futures in tower_service impl Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 05/15] async index reader: typedef ReadFuture Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 06/15] systemd/time: extract Time/DateSpec structs Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 07/15] client: factor out UploadOptions Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 08/15] pxar: typedef on_error as ErrorHandler Fabian Grünbichler
2021-01-25 13:42 ` Fabian Grünbichler [this message]
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 10/15] pxar: extract PxarExtractOptions Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 11/15] authid: make Tokenname(Ref) derive Eq Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 12/15] derive/impl and use Default for some structs Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 13/15] verify: factor out common parameters Fabian Grünbichler
2021-01-25 13:42 ` [pbs-devel] [PATCH proxmox-backup 14/15] clippy: allow api functions with many arguments Fabian Grünbichler
2021-01-25 13:43 ` [pbs-devel] [PATCH proxmox-backup 15/15] clippy: more misc fixes Fabian Grünbichler
2021-01-25 13:43 ` [pbs-devel] [PATCH proxmox-backup-qemu 1/2] use UploadOptions for uploading Blobs Fabian Grünbichler
2021-01-25 13:43 ` [pbs-devel] [PATCH proxmox-backup-qemu 2/2] use new HttpClientOptions constructors Fabian Grünbichler
2021-01-26 9:44 ` [pbs-devel] applied series: [PATCH proxmox-backup(-qemu) 00/17] clippy refactorings Wolfgang Bumiller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210125134302.3394328-10-f.gruenbichler@proxmox.com \
--to=f.gruenbichler@proxmox.com \
--cc=pbs-devel@lists.proxmox.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox