From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 723E51FF396 for ; Thu, 23 May 2024 12:05:13 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 784641CE4F; Thu, 23 May 2024 12:05:31 +0200 (CEST) Message-ID: <40542da1-4552-477e-8880-02f2d1eff904@proxmox.com> Date: Thu, 23 May 2024 12:04:56 +0200 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird Beta To: Proxmox Backup Server development discussion , Christian Ebner References: <20240514103421.289431-1-c.ebner@proxmox.com> <20240514103421.289431-62-c.ebner@proxmox.com> Content-Language: en-US From: Dominik Csapak In-Reply-To: <20240514103421.289431-62-c.ebner@proxmox.com> X-SPAM-LEVEL: Spam detection results: 0 AWL -0.109 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment KAM_LOTSOFHASH 0.25 Emails with lots of hash-like gibberish SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [create.rs] Subject: Re: [pbs-devel] [PATCH v6 proxmox-backup 61/65] client: pxar: add archive creation with reference test X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Transfer-Encoding: 7bit Content-Type: text/plain; charset="us-ascii"; Format="flowed" Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" interestingly this test runs through but if do ./target/debug/pxar list ./tests/pxar/backup-client-pxar-expected.mpxar i get the following error: "/" "/" "/" "/folder_0" "/folder_0/file_0" Error: unexpected entry type: UNKNOWN header (ef00315f656c6966) is that expected ? On 5/14/24 12:34, Christian Ebner wrote: > Add a basic regression test for archive creation with reference > metadata archive and index. > > Signed-off-by: Christian Ebner > --- > pbs-client/src/pxar/create.rs | 242 ++++++++++++++++++ > tests/pxar/backup-client-pxar-data.mpxar | Bin 0 -> 15070 bytes > tests/pxar/backup-client-pxar-data.ppxar.didx | Bin 0 -> 8096 bytes > tests/pxar/backup-client-pxar-expected.mpxar | Bin 0 -> 15086 bytes > 4 files changed, 242 insertions(+) > create mode 100644 tests/pxar/backup-client-pxar-data.mpxar > create mode 100644 tests/pxar/backup-client-pxar-data.ppxar.didx > create mode 100644 tests/pxar/backup-client-pxar-expected.mpxar > > diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs > index 287e47655..9349dd8bb 100644 > --- a/pbs-client/src/pxar/create.rs > +++ b/pbs-client/src/pxar/create.rs > @@ -1811,3 +1811,245 @@ fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec { > > content > } > + > +#[cfg(test)] > +mod tests { > + use std::ffi::OsString; > + use std::fs::File; > + use std::fs::OpenOptions; > + use std::io::{self, BufReader, Seek, SeekFrom, Write}; > + use std::pin::Pin; > + use std::process::Command; > + use std::sync::mpsc; > + use std::task::{Context, Poll}; > + > + use pbs_datastore::dynamic_index::DynamicIndexReader; > + use pxar::accessor::sync::FileReader; > + use pxar::encoder::SeqWrite; > + > + use crate::pxar::extract::Extractor; > + use crate::pxar::OverwriteFlags; > + > + use super::*; > + > + struct DummyWriter { > + file: Option, > + } > + > + impl DummyWriter { > + fn new>(path: Option

) -> Result { > + let file = if let Some(path) = path { > + Some( > + OpenOptions::new() > + .read(true) > + .write(true) > + .truncate(true) > + .create(true) > + .open(path)?, > + ) > + } else { > + None > + }; > + Ok(Self { file }) > + } > + } > + > + impl Write for DummyWriter { > + fn write(&mut self, data: &[u8]) -> io::Result { > + if let Some(file) = self.file.as_mut() { > + file.write_all(data)?; > + } > + Ok(data.len()) > + } > + > + fn flush(&mut self) -> io::Result<()> { > + if let Some(file) = self.file.as_mut() { > + file.flush()?; > + } > + Ok(()) > + } > + } > + > + impl SeqWrite for DummyWriter { > + fn poll_seq_write( > + mut self: Pin<&mut Self>, > + _cx: &mut Context, > + buf: &[u8], > + ) -> Poll> { > + Poll::Ready(self.as_mut().write(buf)) > + } > + > + fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { > + Poll::Ready(self.as_mut().flush()) > + } > + } > + > + fn prepare>(dir_path: P) -> Result<(), Error> { > + let dir = nix::dir::Dir::open(dir_path.as_ref(), OFlag::O_DIRECTORY, Mode::empty())?; > + > + let fs_magic = detect_fs_type(dir.as_raw_fd()).unwrap(); > + let stat = nix::sys::stat::fstat(dir.as_raw_fd()).unwrap(); > + let mut fs_feature_flags = Flags::from_magic(fs_magic); > + let metadata = get_metadata( > + dir.as_raw_fd(), > + &stat, > + fs_feature_flags, > + fs_magic, > + &mut fs_feature_flags, > + false, > + )?; > + > + let mut extractor = Extractor::new( > + dir, > + metadata.clone(), > + true, > + OverwriteFlags::empty(), > + fs_feature_flags, > + ); > + > + let dir_metadata = Metadata { > + stat: pxar::Stat::default().mode(0o777u64).set_dir().gid(0).uid(0), > + ..Default::default() > + }; > + > + let file_metadata = Metadata { > + stat: pxar::Stat::default() > + .mode(0o777u64) > + .set_regular_file() > + .gid(0) > + .uid(0), > + ..Default::default() > + }; > + > + extractor.enter_directory( > + OsString::from(format!("testdir")), > + dir_metadata.clone(), > + true, > + )?; > + > + let size = 1024 * 1024; > + let mut cursor = BufReader::new(std::io::Cursor::new(vec![0u8; size])); > + for i in 0..10 { > + extractor.enter_directory( > + OsString::from(format!("folder_{i}")), > + dir_metadata.clone(), > + true, > + )?; > + for j in 0..10 { > + cursor.seek(SeekFrom::Start(0))?; > + extractor.extract_file( > + CString::new(format!("file_{j}").as_str())?.as_c_str(), > + &file_metadata, > + size as u64, > + &mut cursor, > + true, > + )?; > + } > + extractor.leave_directory()?; > + } > + > + extractor.leave_directory()?; > + > + Ok(()) > + } > + > + #[test] > + fn test_create_archive_with_reference() -> Result<(), Error> { > + let mut testdir = PathBuf::from("./target/testout"); > + testdir.push(std::module_path!()); > + > + let _ = std::fs::remove_dir_all(&testdir); > + let _ = std::fs::create_dir_all(&testdir); > + > + prepare(testdir.as_path())?; > + > + let previous_payload_index = Some(DynamicIndexReader::new(File::open( > + "../tests/pxar/backup-client-pxar-data.ppxar.didx", > + )?)?); > + let metadata_archive = File::open("../tests/pxar/backup-client-pxar-data.mpxar").unwrap(); > + let metadata_size = metadata_archive.metadata()?.len(); > + let reader: MetadataArchiveReader = Arc::new(FileReader::new(metadata_archive)); > + > + let rt = tokio::runtime::Runtime::new().unwrap(); > + let (suggested_boundaries, _rx) = mpsc::channel(); > + let (forced_boundaries, _rx) = mpsc::channel(); > + > + rt.block_on(async move { > + testdir.push("testdir"); > + let source_dir = > + nix::dir::Dir::open(testdir.as_path(), OFlag::O_DIRECTORY, Mode::empty()).unwrap(); > + > + let fs_magic = detect_fs_type(source_dir.as_raw_fd()).unwrap(); > + let stat = nix::sys::stat::fstat(source_dir.as_raw_fd()).unwrap(); > + let mut fs_feature_flags = Flags::from_magic(fs_magic); > + > + let metadata = get_metadata( > + source_dir.as_raw_fd(), > + &stat, > + fs_feature_flags, > + fs_magic, > + &mut fs_feature_flags, > + false, > + )?; > + > + let mut writer = > + DummyWriter::new(Some("./target/backup-client-pxar-run.mpxar")).unwrap(); > + let mut payload_writer = Some(DummyWriter::new::(None).unwrap()); > + > + let mut encoder = > + Encoder::new(&mut writer, &metadata, payload_writer.as_mut(), Some(&[])).await?; > + > + let mut archiver = Archiver { > + feature_flags: Flags::from_magic(fs_magic), > + fs_feature_flags: Flags::from_magic(fs_magic), > + fs_magic, > + callback: Box::new(|_| Ok(())), > + patterns: Vec::new(), > + catalog: None, > + path: PathBuf::new(), > + entry_counter: 0, > + entry_limit: 1024, > + current_st_dev: stat.st_dev, > + device_set: None, > + hardlinks: HashMap::new(), > + file_copy_buffer: vec::undefined(4 * 1024 * 1024), > + skip_e2big_xattr: false, > + forced_boundaries: Some(forced_boundaries), > + previous_payload_index, > + suggested_boundaries: Some(suggested_boundaries), > + cached_entries: Vec::new(), > + cached_range: Range::default(), > + cached_last_chunk: None, > + cached_hardlinks: HashSet::new(), > + caching_enabled: false, > + reuse_stats: ReuseStats::default(), > + }; > + > + let accessor = Accessor::new(reader, metadata_size, None).await.unwrap(); > + let root = accessor.open_root().await.ok(); > + archiver > + .archive_dir_contents(&mut encoder, root, source_dir, true) > + .await > + .unwrap(); > + > + archiver > + .flush_cached_reusing_if_below_threshold(&mut encoder, false) > + .await > + .unwrap(); > + > + encoder.finish().await.unwrap(); > + encoder.close().await.unwrap(); > + > + let status = Command::new("diff") > + .args([ > + "../tests/pxar/backup-client-pxar-expected.mpxar", > + "./target/backup-client-pxar-run.mpxar", > + ]) > + .status() > + .expect("failed to execute diff"); > + assert!(status.success()); > + > + Ok::<(), Error>(()) > + }) > + } > +} > diff --git a/tests/pxar/backup-client-pxar-data.mpxar b/tests/pxar/backup-client-pxar-data.mpxar > new file mode 100644 > index 0000000000000000000000000000000000000000..00f3dc295fb38062c23e6cf7cac9ae110beb0a65 > GIT binary patch > literal 15070 > zcmeI3ZD<@t7{_Pd4&n>F7EIfqb#1^FO6}HK%_)tWO2s0r+iLp7VpnN`+SqK3@uiTk > z0g)mo3n~RgSX7jP#Z`-;wUEWA!B4JW5kF|x5580@T|bDmXzQ8GN@tzklRN*x`)~`# > z+|A9+Z+4!U-!r*zm%i41e0X5q&>}W-sk}V(=Du$q+4;h;F8=yl4}ZdoA2i1PeiW~F > z7gkDF&G*_D^Edhj2X^*7yu)JuwZnyZhYt+&$+{a8M{=R@jqX44;jVQr_n5qS`Ja!? > zJj=%~;8y>8^bO)nmIG_xu7%+&Cf=v??$*F?HnW6jmEx|0;T&euxV12x%N!baJq+hD > zm&V-y!}-jkaa}N6z zPwf8mu1mHb zeP!$T7i*WVKk;~>pO3D*w{v9smfybSqt4rptjHd`|MLm z7k_iMt1TS)zR-W~W!=Mf|9sD>XZd*YdB}HcLEjPq_HYs}F67(1L&2w#Y%n&v?uz=3 > zSjazEo-U<0$>&W#aU0 > zDplb0RRf39x22dg4ySKhu>@R8-*xF*Vx%6v7kKeM>Dy6kA+B?*Z&z_>oMf`bW;a>I > z zz9ow2nVpfOKE<8BGbI(`D#hVW-%QPD98TY5mGQr_Y8L^!RI9-0s|F6I > zZ%Z|498TZ1YSB2Hz8%%3aX5Xuszc*&`u0?p#^LnstDb;s>ANm{OZIGY=sQq-A+B?* > z?@$eB98TYn8qzqNzGF3_agwFbV75rqn8xAsovI0q!|6LyQyPcUH`6j2htqdiWBlvb > z8kruaZ&RxR&pTMO^j(*}C7Y-@^lfRT5Z5`@x2;(;4ySKNvuPYo->&A+IGnyc&82aY > zmDgal@HLOd;q)D7K8?faJJbRihtqeYg)|PQ?^ufjTua||>07d@n?v7;77KBmV|}Mu > zLgR4y&a{-q;q=Y)jK<;gUDg@@$9att98TY+UIm_af|D*4$wF^1TUfeD<8b=6b&JN~ > z^zG<2jl=2N)g1xX(sy0@mMpX8(6^_%LR_VL68GJ=uX{8Or|&@bX&g@9p&rmUoW3JH > zq;WWX$9hELaQaU4n8r!=RfE|g)e{ z^lchdAa#Pn>Dw|)8i&)jZCEr8r*FrwX&g@9uHn!)oW4E7rExfY`-Vs3B-^;bY!Mhf > zjl=0XGy(zF(sy0@mIR_X^c@+Y5Z5_AeaA*b<8b;;jF`sZ^qm?Bjl=0XGg2Cd(>E(+ > zG!Ccla*375OpnvIS*il5g9T3CR>`Ds5^FS=E$osd;9B~Y>$^BF%I$l%bRK>#eY7&O > zHYWHE=v;8~)Sh?xU(H|VW$)?(9aB$LPP~7)*uLYlq5Vhi+jHX|?PC2`OI{f&Z9MeB > z>6K!Ahu7bI_2$0s#@C4T7d>;$s;8fP>(9z^v3}X ee6jw{9WR|3pSSVg=*41v{&S{}`TgcUXZj0DmM>rc > > literal 0 > HcmV?d00001 > > diff --git a/tests/pxar/backup-client-pxar-data.ppxar.didx b/tests/pxar/backup-client-pxar-data.ppxar.didx > new file mode 100644 > index 0000000000000000000000000000000000000000..a646218b5d504196443b17d62f3b22d171f011b8 > GIT binary patch > literal 8096 > zcmeIw&x=k`9LMqR`E|?Ga2Ha_;uI^9yBJNz!Z9f+>6R!pO*b*jh^~^|JnlRqSshV| > z%`G8TSEgiYa;C7OyUpIN$Ya@$KDb%dI01H%~o(*K{tQ > zJ~q6+^=PT==^y&xJ9`F3sC!@cUYwa2-S=W{^1`mZr(YJX_P=dkztmp8Zd>P0&yH!m > zYQlvAp+G1Q3WNfoKqwFjgaV;JC=d#S0-?bFT|iU3_TbPp*KU10`+DQ}SnEW!^~!_6 > zmE|WN7T?dES$;Kh@A!s<^qNZtPc0p`(~IYOPkMW9;>Of`Ykp zkW0xxE~6B3Ic1P5D2JS-0&*o;$W>HA&QS%qnjGXj)sSn*LylMjxtI}Kh5y=%W?c!m > zglWhbmOw6L267ooA(yiZas|sFXITNcl3B=Atc09n736B>Am>>PxrTYj5pN(DbK=OZ > zH1A4ee_TV(@C0%xH;~JC3b~wTkSll&Im-*kmE1zE;w9u9uOL@*2RYAc$Ti$Ujzj~w > zSdc(=rA1dF`x6>+MkJ6+g@IfqQpn{ZgIpnU$XQW9t`rt>l_(+SL zawHqb#gZhlD=oVc*`L&qGcti(Dh=c^nL;j?8RQC?L(a+qa;3D8t7Hi|Co9O+(m~G4 > z8gh;FkR#PVE>@(FU1`;o$o`auoKXqnQe_~QsT6X#${<&$9CB6_kSmpiT%}6LIaNWf > zRt|Dr)sSnHha5!> Tczf@GpWkK|cE3#e4vqc=((xEu > > literal 0 > HcmV?d00001 > > diff --git a/tests/pxar/backup-client-pxar-expected.mpxar b/tests/pxar/backup-client-pxar-expected.mpxar > new file mode 100644 > index 0000000000000000000000000000000000000000..ae4a18c89749f3d7ec82623e84509df19943d03e > GIT binary patch > literal 15086 > zcmeI3Z-^9S9LJyew{ZQzQRvjGZ1W%mF~`ihExcw8BMEJ+&NoR;;T@Hij$M~!+%X3c > z5)=a!LLm(mg^)C*cv!*>U2*iP2{P$LIT8J_45t^7Nokw=%!_Ax+~4i?J=zyLa6C89 > zJ@O3UiwDo@!`Q)L-SbmQh9m#&Zl18d#vMIli#0ud-r#bZF%Wv55GTG > z=D+abM~$(6erm4+b4!J*XM3IV`5y+h4{qsybhE|&Yln054j&rqmvuKLj^sk)8{PB% > zM_X6zEf;z7ykx98^L+dQZu!4Q-z3iBn7X*@U^tuQ^Q$wv6)>E`EdE&Q;I4<^TxQd_ > zl`x#g92$264CgbK#@z_R1U7R;UX3YxGRT_u72~*lgs8Q)#{0j9b5a> > z?2DIhA8zNZ*S-7XwomW5WYZDeF0cRj_D?3wgOk5@a0TY|UrzpUcHvKl7p$vkKXB&O > z-6z*CeQTp$?Kp2=x@-K{%EhZsJW5pW}LsB > zTQv3Jww=9syS(h4|IOK+j&S6Mn*RGP>m9!Lm-|jV&&QKLhg^R(`j!Z=%tywH3;8zh > zQ1GcF8jMY^yIOt6Ead-K$2gMFH;GGFMB{M!PFYOjaQe zT~ru%K%Je5)3>FVG!Cb4TdBcCE(inZb;t}BbE7C;Kl!>Z&$H}b(Ka7 > zoW4E9p>dLjH8#D6RU4dq#iemLeFut1<8b;86`#i8^c^Vyjl=0XRzezw(|4joG!Ccl > zREcREPT!f52)MSs8`8H#5#{L_N$OKv_RZ8(SXU_yr*BiuXdF)8MV0YaS#@$8$=Zxf > zZ*6L$g{7J_4ySKht<3NIRcCfMeLJc}<8b zhV(5tv#p`;Q1yj%ond`PYCz*~`i|9*#^Lmxs1c2mY=tJHMXJU$4yW%-O=uiW-%Lws > z98TY+meDwzzKa^;Z^zaNy*PbanknGg`ff zhts#GIW!KZZ(nn1oMh)U87%_Mqj5NWhni30aQcq4fX3nU9cv+t!|6NGA_3RdcSHJ? > z?CRFgcdEt0y3TO+ooNY;!|9vpDUHMF+tf1}htqdaXZ(ZnIvE^J- zWqwp*fz!96TQm-*Z&$Zz98TY!?g+THz8liFWSg~yzJ1*l)^&#U9q1m7!|6NJeHw?; > zccce24yW%}4{01u--#a4IGnyyJ*IJzbJb+D$n=E9;q=Xnl*ZxoZ5o+?YwNoqeM{E8 > zHS}FHm_g<^xHnHM!=!OIecMK5epCjB)3;+-G!Cb4*RW|EPT!v4&^VmFeZ!@3IDH3( > zN8=>NxXEY{8a|D~={qt40oT@dL;991L~H0fHbP-tXE^&#jEKhJ^qm?pjl=0XGZGqy > z(>E)mG!Cb4vyjm^oW6?%Rv<$!PTy9+q;WWX+l9*fsKi2IjV7aoQ?LYFTi zj$IqN55JH;UaBtE1U~`Yb8ea1@!r7e`F&pYE#KEQ^-Sr+2Um#gyFMG*bL4>?H~rZu > z)_=9&wV}e=gCCw=D%N*-1HIR*@Be;$g;;;lbJs3=_UU*2DlHc47oFa}W{!4S$F7B9 > o{h^z+M~)BcqpN0%^>=T6<;?i3wfjde7VGn`GkwA5n}40@Z-J;Sk^lez > > literal 0 > HcmV?d00001 > _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel