From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id EA25E1FF398 for ; Mon, 27 May 2024 16:34:47 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 05C7D5F0F; Mon, 27 May 2024 16:34:57 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Mon, 27 May 2024 16:33:19 +0200 Message-Id: <20240527143323.456002-66-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20240527143323.456002-1-c.ebner@proxmox.com> References: <20240527143323.456002-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL -0.095 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LOTSOFHASH 0.25 Emails with lots of hash-like gibberish SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH v7 proxmox-backup 65/69] client: pxar: add archive creation with reference test X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Add a basic regression test for archive creation with reference metadata archive and index. Signed-off-by: Christian Ebner --- changes since version 6: - adapt to PxarVariant pxar interface - adapt to PxarLookaheadCache pbs-client/src/pxar/create.rs | 243 ++++++++++++++++++ tests/pxar/backup-client-pxar-data.mpxar | Bin 0 -> 15070 bytes tests/pxar/backup-client-pxar-data.ppxar.didx | Bin 0 -> 8096 bytes tests/pxar/backup-client-pxar-expected.mpxar | Bin 0 -> 15086 bytes 4 files changed, 243 insertions(+) create mode 100644 tests/pxar/backup-client-pxar-data.mpxar create mode 100644 tests/pxar/backup-client-pxar-data.ppxar.didx create mode 100644 tests/pxar/backup-client-pxar-expected.mpxar diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs index ff7e86804..bcadf12bd 100644 --- a/pbs-client/src/pxar/create.rs +++ b/pbs-client/src/pxar/create.rs @@ -1718,3 +1718,246 @@ fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec { content } + +#[cfg(test)] +mod tests { + use std::ffi::OsString; + use std::fs::File; + use std::fs::OpenOptions; + use std::io::{self, BufReader, Seek, SeekFrom, Write}; + use std::pin::Pin; + use std::process::Command; + use std::sync::mpsc; + use std::task::{Context, Poll}; + + use pbs_datastore::dynamic_index::DynamicIndexReader; + use pxar::accessor::sync::FileReader; + use pxar::encoder::SeqWrite; + + use crate::pxar::extract::Extractor; + use crate::pxar::OverwriteFlags; + + use super::*; + + struct DummyWriter { + file: Option, + } + + impl DummyWriter { + fn new>(path: Option

) -> Result { + let file = if let Some(path) = path { + Some( + OpenOptions::new() + .read(true) + .write(true) + .truncate(true) + .create(true) + .open(path)?, + ) + } else { + None + }; + Ok(Self { file }) + } + } + + impl Write for DummyWriter { + fn write(&mut self, data: &[u8]) -> io::Result { + if let Some(file) = self.file.as_mut() { + file.write_all(data)?; + } + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + if let Some(file) = self.file.as_mut() { + file.flush()?; + } + Ok(()) + } + } + + impl SeqWrite for DummyWriter { + fn poll_seq_write( + mut self: Pin<&mut Self>, + _cx: &mut Context, + buf: &[u8], + ) -> Poll> { + Poll::Ready(self.as_mut().write(buf)) + } + + fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + Poll::Ready(self.as_mut().flush()) + } + } + + fn prepare>(dir_path: P) -> Result<(), Error> { + let dir = nix::dir::Dir::open(dir_path.as_ref(), OFlag::O_DIRECTORY, Mode::empty())?; + + let fs_magic = detect_fs_type(dir.as_raw_fd()).unwrap(); + let stat = nix::sys::stat::fstat(dir.as_raw_fd()).unwrap(); + let mut fs_feature_flags = Flags::from_magic(fs_magic); + let metadata = get_metadata( + dir.as_raw_fd(), + &stat, + fs_feature_flags, + fs_magic, + &mut fs_feature_flags, + false, + )?; + + let mut extractor = Extractor::new( + dir, + metadata.clone(), + true, + OverwriteFlags::empty(), + fs_feature_flags, + ); + + let dir_metadata = Metadata { + stat: pxar::Stat::default().mode(0o777u64).set_dir().gid(0).uid(0), + ..Default::default() + }; + + let file_metadata = Metadata { + stat: pxar::Stat::default() + .mode(0o777u64) + .set_regular_file() + .gid(0) + .uid(0), + ..Default::default() + }; + + extractor.enter_directory( + OsString::from(format!("testdir")), + dir_metadata.clone(), + true, + )?; + + let size = 1024 * 1024; + let mut cursor = BufReader::new(std::io::Cursor::new(vec![0u8; size])); + for i in 0..10 { + extractor.enter_directory( + OsString::from(format!("folder_{i}")), + dir_metadata.clone(), + true, + )?; + for j in 0..10 { + cursor.seek(SeekFrom::Start(0))?; + extractor.extract_file( + CString::new(format!("file_{j}").as_str())?.as_c_str(), + &file_metadata, + size as u64, + &mut cursor, + true, + )?; + } + extractor.leave_directory()?; + } + + extractor.leave_directory()?; + + Ok(()) + } + + #[test] + fn test_create_archive_with_reference() -> Result<(), Error> { + let mut testdir = PathBuf::from("./target/testout"); + testdir.push(std::module_path!()); + + let _ = std::fs::remove_dir_all(&testdir); + let _ = std::fs::create_dir_all(&testdir); + + prepare(testdir.as_path())?; + + let previous_payload_index = Some(DynamicIndexReader::new(File::open( + "../tests/pxar/backup-client-pxar-data.ppxar.didx", + )?)?); + let metadata_archive = File::open("../tests/pxar/backup-client-pxar-data.mpxar").unwrap(); + let metadata_size = metadata_archive.metadata()?.len(); + let reader: MetadataArchiveReader = Arc::new(FileReader::new(metadata_archive)); + + let rt = tokio::runtime::Runtime::new().unwrap(); + let (suggested_boundaries, _rx) = mpsc::channel(); + let (forced_boundaries, _rx) = mpsc::channel(); + + rt.block_on(async move { + testdir.push("testdir"); + let source_dir = + nix::dir::Dir::open(testdir.as_path(), OFlag::O_DIRECTORY, Mode::empty()).unwrap(); + + let fs_magic = detect_fs_type(source_dir.as_raw_fd()).unwrap(); + let stat = nix::sys::stat::fstat(source_dir.as_raw_fd()).unwrap(); + let mut fs_feature_flags = Flags::from_magic(fs_magic); + + let metadata = get_metadata( + source_dir.as_raw_fd(), + &stat, + fs_feature_flags, + fs_magic, + &mut fs_feature_flags, + false, + )?; + + let writer = DummyWriter::new(Some("./target/backup-client-pxar-run.mpxar")).unwrap(); + let payload_writer = DummyWriter::new::(None).unwrap(); + + let mut encoder = Encoder::new( + pxar::PxarVariant::Split(writer, payload_writer), + &metadata, + Some(&[]), + ) + .await?; + + let mut archiver = Archiver { + feature_flags: Flags::from_magic(fs_magic), + fs_feature_flags: Flags::from_magic(fs_magic), + fs_magic, + callback: Box::new(|_| Ok(())), + patterns: Vec::new(), + catalog: None, + path: PathBuf::new(), + entry_counter: 0, + entry_limit: 1024, + current_st_dev: stat.st_dev, + device_set: None, + hardlinks: HashMap::new(), + file_copy_buffer: vec::undefined(4 * 1024 * 1024), + skip_e2big_xattr: false, + forced_boundaries: Some(forced_boundaries), + previous_payload_index, + suggested_boundaries: Some(suggested_boundaries), + cache: PxarLookaheadCache::new(), + reuse_stats: ReuseStats::default(), + }; + + let accessor = Accessor::new(pxar::PxarVariant::Unified(reader), metadata_size) + .await + .unwrap(); + let root = accessor.open_root().await.ok(); + archiver + .archive_dir_contents(&mut encoder, root, source_dir, true) + .await + .unwrap(); + + archiver + .flush_cached_reusing_if_below_threshold(&mut encoder, false) + .await + .unwrap(); + + encoder.finish().await.unwrap(); + encoder.close().await.unwrap(); + + let status = Command::new("diff") + .args([ + "../tests/pxar/backup-client-pxar-expected.mpxar", + "./target/backup-client-pxar-run.mpxar", + ]) + .status() + .expect("failed to execute diff"); + assert!(status.success()); + + Ok::<(), Error>(()) + }) + } +} diff --git a/tests/pxar/backup-client-pxar-data.mpxar b/tests/pxar/backup-client-pxar-data.mpxar new file mode 100644 index 0000000000000000000000000000000000000000..00f3dc295fb38062c23e6cf7cac9ae110beb0a65 GIT binary patch literal 15070 zcmeI3ZD<@t7{_Pd4&n>F7EIfqb#1^FO6}HK%_)tWO2s0r+iLp7VpnN`+SqK3@uiTk z0g)mo3n~RgSX7jP#Z`-;wUEWA!B4JW5kF|x5580@T|bDmXzQ8GN@tzklRN*x`)~`# z+|A9+Z+4!U-!r*zm%i41e0X5q&>}W-sk}V(=Du$q+4;h;F8=yl4}ZdoA2i1PeiW~F z7gkDF&G*_D^Edhj2X^*7yu)JuwZnyZhYt+&$+{a8M{=R@jqX44;jVQr_n5qS`Ja!? zJj=%~;8y>8^bO)nmIG_xu7%+&Cf=v??$*F?HnW6jmEx|0;T&euxV12x%N!baJq+hD zm&V-y!}-jkaa}N6zpO3D*w{v9smfybSqt4rptjHd`|MLmXZd*YdB}HcLEjPq_HYs}F67(1L&2w#Y%n&v?uz=3 zSjazEo-U<0$>&W#aU0 zDplb0RRf39x22dg4ySKhu>@R8-*xF*Vx%6v7kKeM>Dy6kA+B?*Z&z_>oMf`bW;a>I zL^!RI9-0s|F6I zZ%Z|498TZ1YSB2Hz8%%3aX5Xuszc*&`u0?p#^LnstDb;s>ANm{OZIGY=sQq-A+B?* z?@$eB98TYn8qzqNzGF3_agwFbV75rqn8xAsovI0q!|6LyQyPcUH`6j2htqdiWBlvb z8kruaZ&RxR&pTMO^j(*}C7Y-@^lfRT5Z5`@x2;(;4ySKNvuPYo->&A+IGnyc&82aY zmDgal@HLOd;q)D7K8?faJJbRihtqeYg)|PQ?^ufjTua||>07d@n?v7;77KBmV|}Mu zLgR4y&a{-q;q=Y)jK<;gUDg@@$9att98TY+UIm_af|D*4$wF^1TUfeD<8b=6b&JN~ z^zG<2jl=2N)g1xX(sy0@mMpX8(6^_%LR_VL68GJ=uX{8Or|&@bX&g@9p&rmUoW3JH zq;WWX$9hELaQaU4n8r!=RfE|g)e{Dw|)8i&)jZCEr8r*FrwX&g@9uHn!)oW4E7rExfY`-Vs3B-^;bY!Mhf zjl=0XGy(zF(sy0@mIR_X^c@+Y5Z5_AeaA*b<8b;;jF`sZ^qm?Bjl=0XGg2Cd(>E(+ zG!Ccla*375OpnvIS*il5g9T3CR>`Ds5^FS=E$osd;9B~Y>$^BF%I$l%bRK>#eY7&O zHYWHE=v;8~)Sh?xU(H|VW$)?(9aB$LPP~7)*uLYlq5Vhi+jHX|?PC2`OI{f&Z9MeB z>6K!Ahu7bI_2$0s#@C4T7d>;$s;8fP>(9z^v3}Xrc literal 0 HcmV?d00001 diff --git a/tests/pxar/backup-client-pxar-data.ppxar.didx b/tests/pxar/backup-client-pxar-data.ppxar.didx new file mode 100644 index 0000000000000000000000000000000000000000..a646218b5d504196443b17d62f3b22d171f011b8 GIT binary patch literal 8096 zcmeIw&x=k`9LMqR`E|?Ga2Ha_;uI^9yBJNz!Z9f+>6R!pO*b*jh^~^|JnlRqSshV| z%`G8TSEgiYa;C7OyUpIN$Ya@$KDb%dI01H%~o(*K{tQ zJ~q6+^=PT==^y&xJ9`F3sC!@cUYwa2-S=W{^1`mZr(YJX_P=dkztmp8Zd>P0&yH!m zYQlvAp+G1Q3WNfoKqwFjgaV;JC=d#S0-?bFT|iU3_TbPp*KU10`+DQ}SnEW!^~!_6 zmE|WN7T?dES$;Kh@A!s<^qNZtPc0p`(~IYOPkMW9;>Of`YkpHA&QS%qnjGXj)sSn*LylMjxtI}Kh5y=%W?c!m zglWhbmOw6L267ooA(yiZas|sFXITNcl3B=Atc09n736B>Am>>PxrTYj5pN(DbK=OZ zH1A4ee_TV(@C0%xH;~JC3b~wTkSll&Im-*kmE1zE;w9u9uOL@*2RYAc$Ti$Ujzj~w zSdc(=rA1dF`x6>+MkJ6+g@IfqQpn{ZgIpnU$XQW9t`rt>l_(+SL@(FU1`;o$o`auoKXqnQe_~QsT6X#${<&$9CB6_kSmpiT%}6LIaNWf zRt|Dr)sSnHha5!>U2*iP2{P$LIT8J_45t^7Nokw=%!_Ax+~4i?J=zyLa6C89 zJ@O3UiwDo@!`Q)L-SbmQh9m#&Zl18d#vMIli#0ud-r#bZF%Wv55GTG z=D+abM~$(6erm4+b4!J*XM3IV`5y+h4{qsybhE|&Yln054j&rqmvuKLj^sk)8{PB% zM_X6zEf;z7ykx98^L+dQZu!4Q-z3iBn7X*@U^tuQ^Q$wv6)>E`EdE&Q;I4<^TxQd_ zl`x#g92$264CgbK#@z_R1U7R;UX3YxGRT_u72~*lgs8Q)#{0j9b5a> z?2DIhA8zNZ*S-7XwomW5WYZDeF0cRj_D?3wgOk5@a0TY|UrzpUcHvKl7p$vkKXB&O z-6z*CeQTp$?Kp2=x@-K{%EhZsJW5pW}LsB zTQv3Jww=9syS(h4|IOK+j&S6Mn*RGP>m9!Lm-|jV&&QKLhg^R(`j!Z=%tywH3;8zh zQ1GcF8jMY^yIOt6Ead-K$2gMFH;GGFMB{M!PFYOjaQeFVG!Cb4TdBcCE(inZb;t}BbE7C;Kl!>Z&$H}b(Ka7 zoW4E9p>dLjH8#D6RU4dq#iemLeFut1<8b;86`#i8^c^Vyjl=0XRzezw(|4joG!Ccl zREcREPT!f52)MSs8`8H#5#{L_N$OKv_RZ8(SXU_yr*BiuXdF)8MV0YaS#@$8$=Zxf zZ*6L$g{7J_4ySKht<3NIRcCfMeLJc}<8bNxXEY{8a|D~={qt40oT@dL;991L~H0fHbP-tXE^&#jEKhJ^qm?pjl=0XGZGqy z(>E)mG!Cb4vyjm^oW6?%Rv<$!PTy9+q;WWX+l9*fsKi2IjV7aoQ?LYFTi?H~rZu z)_=9&wV}e=gCCw=D%N*-1HIR*@Be;$g;;;lbJs3=_UU*2DlHc47oFa}W{!4S$F7B9 o{h^z+M~)BcqpN0%^>=T6<;?i3wfjde7VGn`GkwA5n}40@Z-J;Sk^lez literal 0 HcmV?d00001 -- 2.39.2 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel