Re: [pbs-devel] [PATCH v6 proxmox-backup 61/65] client: pxar: add archive creation with reference test

[Dominik Csapak <d.csapak@proxmox.com>]

interestingly this test runs through

but if do

./target/debug/pxar list ./tests/pxar/backup-client-pxar-expected.mpxar

i get the following error:

"/"
"/"
"/"
"/folder_0"
"/folder_0/file_0"
Error: unexpected entry type: UNKNOWN header (ef00315f656c6966)

is that expected ?

On 5/14/24 12:34, Christian Ebner wrote:
> Add a basic regression test for archive creation with reference
> metadata archive and index.
> 
> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
> ---
>   pbs-client/src/pxar/create.rs                 | 242 ++++++++++++++++++
>   tests/pxar/backup-client-pxar-data.mpxar      | Bin 0 -> 15070 bytes
>   tests/pxar/backup-client-pxar-data.ppxar.didx | Bin 0 -> 8096 bytes
>   tests/pxar/backup-client-pxar-expected.mpxar  | Bin 0 -> 15086 bytes
>   4 files changed, 242 insertions(+)
>   create mode 100644 tests/pxar/backup-client-pxar-data.mpxar
>   create mode 100644 tests/pxar/backup-client-pxar-data.ppxar.didx
>   create mode 100644 tests/pxar/backup-client-pxar-expected.mpxar
> 
> diff --git a/pbs-client/src/pxar/create.rs b/pbs-client/src/pxar/create.rs
> index 287e47655..9349dd8bb 100644
> --- a/pbs-client/src/pxar/create.rs
> +++ b/pbs-client/src/pxar/create.rs
> @@ -1811,3 +1811,245 @@ fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
>   
>       content
>   }
> +
> +#[cfg(test)]
> +mod tests {
> +    use std::ffi::OsString;
> +    use std::fs::File;
> +    use std::fs::OpenOptions;
> +    use std::io::{self, BufReader, Seek, SeekFrom, Write};
> +    use std::pin::Pin;
> +    use std::process::Command;
> +    use std::sync::mpsc;
> +    use std::task::{Context, Poll};
> +
> +    use pbs_datastore::dynamic_index::DynamicIndexReader;
> +    use pxar::accessor::sync::FileReader;
> +    use pxar::encoder::SeqWrite;
> +
> +    use crate::pxar::extract::Extractor;
> +    use crate::pxar::OverwriteFlags;
> +
> +    use super::*;
> +
> +    struct DummyWriter {
> +        file: Option<File>,
> +    }
> +
> +    impl DummyWriter {
> +        fn new<P: AsRef<Path>>(path: Option<P>) -> Result<Self, Error> {
> +            let file = if let Some(path) = path {
> +                Some(
> +                    OpenOptions::new()
> +                        .read(true)
> +                        .write(true)
> +                        .truncate(true)
> +                        .create(true)
> +                        .open(path)?,
> +                )
> +            } else {
> +                None
> +            };
> +            Ok(Self { file })
> +        }
> +    }
> +
> +    impl Write for DummyWriter {
> +        fn write(&mut self, data: &[u8]) -> io::Result<usize> {
> +            if let Some(file) = self.file.as_mut() {
> +                file.write_all(data)?;
> +            }
> +            Ok(data.len())
> +        }
> +
> +        fn flush(&mut self) -> io::Result<()> {
> +            if let Some(file) = self.file.as_mut() {
> +                file.flush()?;
> +            }
> +            Ok(())
> +        }
> +    }
> +
> +    impl SeqWrite for DummyWriter {
> +        fn poll_seq_write(
> +            mut self: Pin<&mut Self>,
> +            _cx: &mut Context,
> +            buf: &[u8],
> +        ) -> Poll<io::Result<usize>> {
> +            Poll::Ready(self.as_mut().write(buf))
> +        }
> +
> +        fn poll_flush(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Result<(), io::Error>> {
> +            Poll::Ready(self.as_mut().flush())
> +        }
> +    }
> +
> +    fn prepare<P: AsRef<Path>>(dir_path: P) -> Result<(), Error> {
> +        let dir = nix::dir::Dir::open(dir_path.as_ref(), OFlag::O_DIRECTORY, Mode::empty())?;
> +
> +        let fs_magic = detect_fs_type(dir.as_raw_fd()).unwrap();
> +        let stat = nix::sys::stat::fstat(dir.as_raw_fd()).unwrap();
> +        let mut fs_feature_flags = Flags::from_magic(fs_magic);
> +        let metadata = get_metadata(
> +            dir.as_raw_fd(),
> +            &stat,
> +            fs_feature_flags,
> +            fs_magic,
> +            &mut fs_feature_flags,
> +            false,
> +        )?;
> +
> +        let mut extractor = Extractor::new(
> +            dir,
> +            metadata.clone(),
> +            true,
> +            OverwriteFlags::empty(),
> +            fs_feature_flags,
> +        );
> +
> +        let dir_metadata = Metadata {
> +            stat: pxar::Stat::default().mode(0o777u64).set_dir().gid(0).uid(0),
> +            ..Default::default()
> +        };
> +
> +        let file_metadata = Metadata {
> +            stat: pxar::Stat::default()
> +                .mode(0o777u64)
> +                .set_regular_file()
> +                .gid(0)
> +                .uid(0),
> +            ..Default::default()
> +        };
> +
> +        extractor.enter_directory(
> +            OsString::from(format!("testdir")),
> +            dir_metadata.clone(),
> +            true,
> +        )?;
> +
> +        let size = 1024 * 1024;
> +        let mut cursor = BufReader::new(std::io::Cursor::new(vec![0u8; size]));
> +        for i in 0..10 {
> +            extractor.enter_directory(
> +                OsString::from(format!("folder_{i}")),
> +                dir_metadata.clone(),
> +                true,
> +            )?;
> +            for j in 0..10 {
> +                cursor.seek(SeekFrom::Start(0))?;
> +                extractor.extract_file(
> +                    CString::new(format!("file_{j}").as_str())?.as_c_str(),
> +                    &file_metadata,
> +                    size as u64,
> +                    &mut cursor,
> +                    true,
> +                )?;
> +            }
> +            extractor.leave_directory()?;
> +        }
> +
> +        extractor.leave_directory()?;
> +
> +        Ok(())
> +    }
> +
> +    #[test]
> +    fn test_create_archive_with_reference() -> Result<(), Error> {
> +        let mut testdir = PathBuf::from("./target/testout");
> +        testdir.push(std::module_path!());
> +
> +        let _ = std::fs::remove_dir_all(&testdir);
> +        let _ = std::fs::create_dir_all(&testdir);
> +
> +        prepare(testdir.as_path())?;
> +
> +        let previous_payload_index = Some(DynamicIndexReader::new(File::open(
> +            "../tests/pxar/backup-client-pxar-data.ppxar.didx",
> +        )?)?);
> +        let metadata_archive = File::open("../tests/pxar/backup-client-pxar-data.mpxar").unwrap();
> +        let metadata_size = metadata_archive.metadata()?.len();
> +        let reader: MetadataArchiveReader = Arc::new(FileReader::new(metadata_archive));
> +
> +        let rt = tokio::runtime::Runtime::new().unwrap();
> +        let (suggested_boundaries, _rx) = mpsc::channel();
> +        let (forced_boundaries, _rx) = mpsc::channel();
> +
> +        rt.block_on(async move {
> +            testdir.push("testdir");
> +            let source_dir =
> +                nix::dir::Dir::open(testdir.as_path(), OFlag::O_DIRECTORY, Mode::empty()).unwrap();
> +
> +            let fs_magic = detect_fs_type(source_dir.as_raw_fd()).unwrap();
> +            let stat = nix::sys::stat::fstat(source_dir.as_raw_fd()).unwrap();
> +            let mut fs_feature_flags = Flags::from_magic(fs_magic);
> +
> +            let metadata = get_metadata(
> +                source_dir.as_raw_fd(),
> +                &stat,
> +                fs_feature_flags,
> +                fs_magic,
> +                &mut fs_feature_flags,
> +                false,
> +            )?;
> +
> +            let mut writer =
> +                DummyWriter::new(Some("./target/backup-client-pxar-run.mpxar")).unwrap();
> +            let mut payload_writer = Some(DummyWriter::new::<PathBuf>(None).unwrap());
> +
> +            let mut encoder =
> +                Encoder::new(&mut writer, &metadata, payload_writer.as_mut(), Some(&[])).await?;
> +
> +            let mut archiver = Archiver {
> +                feature_flags: Flags::from_magic(fs_magic),
> +                fs_feature_flags: Flags::from_magic(fs_magic),
> +                fs_magic,
> +                callback: Box::new(|_| Ok(())),
> +                patterns: Vec::new(),
> +                catalog: None,
> +                path: PathBuf::new(),
> +                entry_counter: 0,
> +                entry_limit: 1024,
> +                current_st_dev: stat.st_dev,
> +                device_set: None,
> +                hardlinks: HashMap::new(),
> +                file_copy_buffer: vec::undefined(4 * 1024 * 1024),
> +                skip_e2big_xattr: false,
> +                forced_boundaries: Some(forced_boundaries),
> +                previous_payload_index,
> +                suggested_boundaries: Some(suggested_boundaries),
> +                cached_entries: Vec::new(),
> +                cached_range: Range::default(),
> +                cached_last_chunk: None,
> +                cached_hardlinks: HashSet::new(),
> +                caching_enabled: false,
> +                reuse_stats: ReuseStats::default(),
> +            };
> +
> +            let accessor = Accessor::new(reader, metadata_size, None).await.unwrap();
> +            let root = accessor.open_root().await.ok();
> +            archiver
> +                .archive_dir_contents(&mut encoder, root, source_dir, true)
> +                .await
> +                .unwrap();
> +
> +            archiver
> +                .flush_cached_reusing_if_below_threshold(&mut encoder, false)
> +                .await
> +                .unwrap();
> +
> +            encoder.finish().await.unwrap();
> +            encoder.close().await.unwrap();
> +
> +            let status = Command::new("diff")
> +                .args([
> +                    "../tests/pxar/backup-client-pxar-expected.mpxar",
> +                    "./target/backup-client-pxar-run.mpxar",
> +                ])
> +                .status()
> +                .expect("failed to execute diff");
> +            assert!(status.success());
> +
> +            Ok::<(), Error>(())
> +        })
> +    }
> +}
> diff --git a/tests/pxar/backup-client-pxar-data.mpxar b/tests/pxar/backup-client-pxar-data.mpxar
> new file mode 100644
> index 0000000000000000000000000000000000000000..00f3dc295fb38062c23e6cf7cac9ae110beb0a65
> GIT binary patch
> literal 15070
> zcmeI3ZD<@t7{_Pd4&n>F7EIfqb#1^FO6}HK%_)tWO2s0r+iLp7VpnN`+SqK3@uiTk
> z0g)mo3n~RgSX7jP#Z`-;wUEWA!B4JW5kF|x5580@T|bDmXzQ8GN@tzklRN*x`)~`#
> z+|A9+Z+4!U-!r*zm%i41e0X5q&>}W-sk}V(=Du$q+4;h;F8=yl4}ZdoA2i1PeiW~F
> z7gkDF&G*_D^Edhj2X^*7yu)JuwZnyZhYt+&$+{a8M{=R@jqX44;jVQr_n5qS`Ja!?
> zJj=%~;8y>8^bO)nmIG_xu7%+&Cf=v??$*F?HnW6jmEx|0;T&euxV12x%N!baJq+hD
> zm&V-y!}-jkaa}N6z<e54f#E_H2)HYTj;(+Fj+3hvDKpg*+uC;ZZa5IH;Qkxr`*hRW
> zPwf8mu1mHb<?ZtNpKkkPvV3Urmo~1zy#C9{-_I`n@$iyOh4%etZrOKo^U622=`*~%
> zeP!$T7i*WVKk;~>pO3D*w{v9smfybSqt4rptjHd`|MLm<Vqu)Wo?ABc{O-rP2Mg^x
> z7k_iMt1TS)zR-W~W!=Mf|9sD>XZd*YdB}HcLEjPq_HYs}F67(1L&2w#Y%n&v?uz=3
> zSjazEo-U<0$><xz#Vn$6IDIE9rg1oZr!1jyIDKa<rExfYGbN*OIDMBD#vM>&W#aU0
> zDplb0RRf39x22dg4ySKhu>@R8-*xF*Vx%6v7kKeM>Dy6kA+B?*Z&z_>oMf`bW;a>I
> z<m4$Xjl=2NS3DYr(|4fwG!CclPzh)pPT!Fd(m0&HV<n<-IDIEdOyh9+PL)K!we($=
> zz9ow2nVpfOKE<8BGbI(`D#hVW-%QPD98TY5mGQr_Y8<H~u^F3PY>L^!RI9-0s|F6I
> zZ%Z|498TZ1YSB2Hz8%%3aX5Xuszc*&`u0?p#^LnstDb;s>ANm{OZIGY=sQq-A+B?*
> z?@$eB98TYn8qzqNzGF3_agwFbV75rqn8xAsovI0q!|6LyQyPcUH`6j2htqdiWBlvb
> z8kruaZ&RxR&pTMO^j(*}C7Y-@^lfRT5Z5`@x2;(;4ySKNvuPYo->&A+IGnyc&82aY
> zmDgal@HLOd;q)D7K8?faJJbRihtqeYg)|PQ?^ufjTua||>07d@n?v7;77KBmV|}Mu
> zLgR4y&a{-q;q=Y)jK<;gUDg@@$9att98TY+UIm_af|D*4$wF^1TUfeD<8b=6b&JN~
> z^zG<2jl=2N)g1xX(sy0@mMpX8(6^_%LR_VL68GJ=uX{8Or|&@bX&g@9p&rmUoW3JH
> zq;WWX$9hELaQaU4n8r!=RfE|g)e{<r(|4w)G!Cb4W@G}crSH1*Es1+`=(}t%gFI5<
> z^lchdAa#Pn>Dw|)8i&)jZCEr8r*FrwX&g@9uHn!)oW4E7rExfY`-Vs3B-^;bY!Mhf
> zjl=0XGy(zF(sy0@mIR_X^c@+Y5Z5_AeaA*b<8b;;jF`sZ^qm?Bjl=0XGg2Cd(>E(+
> zG!Ccla*375OpnvIS*il5g9T3CR>`Ds5^FS=E$osd;9B~Y>$^BF%I$l%bRK>#eY7&O
> zHYWHE=v;8~)Sh?xU(H|VW$)?(9aB$LPP~7)*uLYlq5Vhi+jHX|?PC2`OI{f&Z9MeB
> z>6K!Ahu7bI_2$0s#@C4T7d>;$s;8fP>(9z^v3}X<gBuoTx3=wFD%QVullIW@VSRMn
> ee6jw{9WR|3pSSVg=*41v{&S{}`TgcUXZj0DmM>rc
> 
> literal 0
> HcmV?d00001
> 
> diff --git a/tests/pxar/backup-client-pxar-data.ppxar.didx b/tests/pxar/backup-client-pxar-data.ppxar.didx
> new file mode 100644
> index 0000000000000000000000000000000000000000..a646218b5d504196443b17d62f3b22d171f011b8
> GIT binary patch
> literal 8096
> zcmeIw&x=k`9LMqR`E|?Ga2Ha_;uI^9yBJNz!Z9f+>6R!pO*b*jh^~^|JnlRqSshV|
> z%`G8TSEgiYa;C7OyU<J|9b=l(l;<{OBpc7nKj5>pIN$Ya@$KDb%dI01H%~o(*K{tQ
> zJ~q6+^=PT==^y&xJ9`F3sC!@cUYwa2-S=W{^1`mZr(YJX_P=dkztmp8Zd>P0&yH!m
> zYQlvAp+G1Q3WNfoKqwFjgaV;JC=d#S0-?bFT|iU3_TbPp*KU10`+DQ}SnEW!^~!_6
> zmE|WN7T?dES$;Kh@A!s<^qNZtPc0p`(~IYOPkMW9;>Of`Ykp<tpHIL1?CtT1yY~$x
> zkW0xxE~6B3Ic1P5D2JS-0&*o;$W>HA&QS%qnjGXj)sSn*LylMjxtI}Kh5y=%W?c!m
> zglWhbmOw6L267ooA(yiZas|sFXITNcl3B=Atc09n736B>Am>>PxrTYj5pN(DbK=OZ
> zH1A4ee_TV(@C0%xH;~JC3b~wTkSll&Im-*kmE1zE;w9u9uOL@*2RYAc$Ti$Ujzj~w
> zSdc(=rA1dF`x6>+MkJ6+g@IfqQpn{ZgIpnU$XQW9t`rt>l_(+SL<PB8ILLWXL#`1X
> zawHqb#gZhlD=oVc*`L&qGcti(Dh=c^nL;j?8RQC?L(a+qa;3D8t7Hi|Co9O+(m~G4
> z8gh;FkR#PVE>@(FU1`;o$o`auoKXqnQe_~QsT6X#${<&$9CB6_kSmpiT%}6LIaNWf
> zRt|Dr)sSnHha5!><l=}TWLG-sN@RbLhMb8K$YqgPb4Mq~o{fC&+#LA+d)TeK+0;Jx
> Tczf@GpWkK|cE3#e4vqc=((xEu
> 
> literal 0
> HcmV?d00001
> 
> diff --git a/tests/pxar/backup-client-pxar-expected.mpxar b/tests/pxar/backup-client-pxar-expected.mpxar
> new file mode 100644
> index 0000000000000000000000000000000000000000..ae4a18c89749f3d7ec82623e84509df19943d03e
> GIT binary patch
> literal 15086
> zcmeI3Z-^9S9LJyew{ZQzQRvjGZ1W%mF~`ihExcw8BMEJ+&NoR;;T@Hij$M~!+%X3c
> z5)=a!LLm(mg^)C*cv!*>U2*iP2{P$LIT8J_45t^7Nokw=%!_Ax+~4i?J=zyLa6C89
> zJ@<T`XP)Qz{C>O3UiwDo@!`Q)L-SbmQh9m#&Zl18d#vMIli#0ud-r#bZF%Wv55GTG
> z=D+abM~$(6erm4+b4!J*XM3IV`5y+h4{qsybhE|&Yln054j&rqmvuKLj^sk)8{PB%
> zM_X6zEf;z7ykx98^L+dQZu!4Q-z3iBn7X*@U^tuQ^Q$wv6)>E`EdE&Q;I4<^TxQd_
> zl`x#g92$264CgbK#@z_R1<a#yJuqCzd>U7R;UX3YxGRT_u72~*lgs8Q)#{0j9b5a>
> z?2DIhA8zNZ*S-7XwomW5WYZDeF0cRj_D?3wgOk5@a0TY|UrzpUcHvKl7p$vkKXB&O
> z-6z*CeQTp$?Kp2=x@-K{%EhZsJW<on$5-9oJ+f)T?_cwA<n2e6WDh_1`2>5pW}LsB
> zTQv3Jww=9syS(h4|IOK+j&S6Mn*RGP>m9!Lm-|jV&&QKLhg^R(`j!Z=%tywH3;8zh
> zQ1GcF8jMY^yIOt6Ead-K$2gMFH;GGFMB{M!PFYOjaQe<zLgR4yW=cxqaQZftjK<;g
> zT~ru%K%Je5)3>FVG!Cb4TdB<N{8eXmIDI>cCE(inZb;t}BbE7C;Kl!>Z&$H}b(Ka7
> zoW4E9p>dLjH8#D6RU4dq#iemLeFut1<8b;86`#i8^c^Vyjl=0XRzezw(|4joG!Ccl
> zREcREPT!f52)MSs8`8H#5#{L_N$OKv_RZ8(SXU_yr*BiuXdF)8MV0YaS#@$8$=Zxf
> zZ*6L$g{7J_4ySKht<3NIRcCfMeLJc}<8b<RRh!1)^zEq*jl=2NS6v#1(|4eH0<Nv^
> zhV(5tv#p`;Q1yj%ond`PYCz*~`i|9*#^Lmxs1c2mY=tJHMXJU$4yW%-O=uiW-%Lws
> z98TY+meDwzzKa^;Z^zaNy*PbanknGg`ff<yl0(!Q`nI*oMxrt}T=wl~7LCK{+tq9u
> zhts#GIW!KZZ(nn1oMh)U87%_Mqj5NWhni30aQcq4fX3nU9cv+t!|6NGA_3RdcSHJ?
> z?CRFgcdEt0y3TO+ooNY;!|9vpDUHMF+tf1}htqdaXZ(ZnIvE^J-<EFDILStDGFsSr
> zWqwp*fz!96TQm-*Z&$Zz98TY!?g+THz8liFWSg~yzJ1*l)^&#U9q1m7!|6NJeHw?;
> zccce24yW%}4{01u--#a4IGnyyJ*IJzbJb+D$n=E9;q=Xnl*ZxoZ5o+?YwNoqeM{E8
> zHS}FHm_g<^xHnHM!=!OIecMK5epCjB)3;+-G!Cb4*RW|EPT!v4&^VmFeZ!@3IDH3(
> zN8=>NxXEY{8a|D~={qt40oT@dL;991L~H0fHbP-tXE^&#jEKhJ^qm?pjl=0XGZGqy
> z(>E)mG!Cb4vyjm^oW6?%Rv<$!PTy9+q;WWX+l9*fsKi2IjV7aoQ?LYFTi<eh*FG2J
> zj$IqN55JH;UaBtE1U~`Yb8ea1@!r7e`F&pYE#KEQ^-Sr+2Um#gyFMG*bL4>?H~rZu
> z)_=9&wV}e=gCCw=D%N*-1HIR*@Be;$g;;;lbJs3=_UU*2DlHc47oFa}W{!4S$F7B9
> o{h^z+M~)BcqpN0%^>=T6<;?i3wfjde7VGn`GkwA5n}40@Z-J;Sk^lez
> 
> literal 0
> HcmV?d00001
> 



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel