From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id B6B73941D0 for ; Wed, 21 Sep 2022 10:13:22 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id AD83324CC1 for ; Wed, 21 Sep 2022 10:12:52 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 21 Sep 2022 10:12:51 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 0738B442A9 for ; Wed, 21 Sep 2022 10:12:51 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Wed, 21 Sep 2022 10:12:39 +0200 Message-Id: <20220921081242.1139249-2-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220921081242.1139249-1-f.gruenbichler@proxmox.com> References: <20220921081242.1139249-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.149 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH proxmox-offline-mirror 1/4] pool: add diff and list helpers X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 21 Sep 2022 08:13:22 -0000 one for diffing two relative paths within a pool (e.g., for comparing snapshots), one for diffing two pools (e.g., for diffing mirror and mirror on medium), and one for listing paths. Signed-off-by: Fabian Grünbichler --- we could extend Diff with a list of error paths and make most of the errors here non-fatal, not sure whether that is nicer? src/pool.rs | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/types.rs | 16 ++++- 2 files changed, 181 insertions(+), 2 deletions(-) diff --git a/src/pool.rs b/src/pool.rs index b6047b8..5dba775 100644 --- a/src/pool.rs +++ b/src/pool.rs @@ -1,7 +1,7 @@ use std::{ cmp::max, collections::{hash_map::Entry, HashMap}, - fs::{hard_link, remove_dir, File}, + fs::{hard_link, remove_dir, File, Metadata}, ops::Deref, os::linux::fs::MetadataExt, path::{Path, PathBuf}, @@ -15,6 +15,8 @@ use proxmox_sys::fs::{create_path, file_get_contents, replace_file, CreateOption use proxmox_time::epoch_i64; use walkdir::WalkDir; +use crate::types::Diff; + #[derive(Debug)] /// Pool consisting of two (possibly overlapping) directory trees: /// - pool_dir contains checksum files added by `add_file` @@ -542,6 +544,169 @@ impl PoolLockGuard<'_> { std::fs::rename(&abs_from, &abs_to) .map_err(|err| format_err!("Failed to rename {abs_from:?} to {abs_to:?} - {err}")) } + + /// Calculate diff between two pool dirs + pub(crate) fn diff_dirs(&self, path: &Path, other_path: &Path) -> Result { + let mut diff = Diff::default(); + + let handle_entry = |entry: Result, + base: &Path, + other_base: &Path, + changed: Option<&mut Vec<(PathBuf, u64)>>, + missing: &mut Vec<(PathBuf, u64)>| + -> Result<(), Error> { + let path = entry?.into_path(); + + let meta = path.metadata()?; + if !meta.is_file() { + return Ok(()); + }; + + let relative = path.strip_prefix(base)?; + let mut absolute = other_base.to_path_buf(); + absolute.push(relative); + if absolute.exists() { + if let Some(changed) = changed { + let other_meta = absolute.metadata()?; + if other_meta.st_ino() != meta.st_ino() { + changed.push(( + relative.to_path_buf(), + meta.st_size().abs_diff(other_meta.st_size()), + )); + } + } + } else { + missing.push((relative.to_path_buf(), meta.st_size())); + } + + Ok(()) + }; + + let path = self.get_path(path)?; + let other_path = self.get_path(other_path)?; + + WalkDir::new(&path).into_iter().try_for_each(|entry| { + handle_entry( + entry, + &path, + &other_path, + Some(&mut diff.changed.paths), + &mut diff.removed.paths, + ) + })?; + WalkDir::new(&other_path) + .into_iter() + .try_for_each(|entry| { + handle_entry(entry, &other_path, &path, None, &mut diff.added.paths) + })?; + + Ok(diff) + } + + /// Calculate diff between two pools + pub(crate) fn diff_pools(&self, other: &Pool) -> Result { + let mut diff = Diff::default(); + + let handle_entry = |entry: Result, + pool: &Pool, + pool_csums: &HashMap, + other_pool: &Pool, + other_csums: &HashMap, + changed: Option<&mut Vec<(PathBuf, u64)>>, + missing: &mut Vec<(PathBuf, u64)>| + -> Result<(), Error> { + let path = entry?.into_path(); + + let meta = path.metadata()?; + if !meta.is_file() { + return Ok(()); + }; + + let base = &pool.link_dir; + + let relative = path.strip_prefix(base)?; + let absolute = other_pool.get_path(relative)?; + if absolute.exists() { + if let Some(changed) = changed { + let csum = match pool_csums.get(&meta.st_ino()) { + Some(csum) => csum, + None => { + eprintln!("{path:?} path not registered with pool."); + changed.push((relative.to_path_buf(), 0)); // TODO add warning/error field? + return Ok(()); + } + }; + let other_meta = absolute.metadata()?; + let other_csum = match other_csums.get(&other_meta.st_ino()) { + Some(csum) => csum, + None => { + eprintln!("{absolute:?} path not registered with pool."); + changed.push((relative.to_path_buf(), 0)); // TODO add warning/error field? + return Ok(()); + } + }; + if csum != other_csum { + changed.push(( + relative.to_path_buf(), + meta.st_size().abs_diff(other_meta.st_size()), + )); + } + } + } else { + missing.push((relative.to_path_buf(), meta.st_size())); + } + + Ok(()) + }; + + let other = other.lock()?; + let (csums, _) = self.get_inode_csum_map()?; + let (other_csums, _) = other.get_inode_csum_map()?; + + WalkDir::new(&self.link_dir) + .into_iter() + .try_for_each(|entry| { + handle_entry( + entry, + self, + &csums, + &other, + &other_csums, + Some(&mut diff.changed.paths), + &mut diff.removed.paths, + ) + })?; + WalkDir::new(&other.link_dir) + .into_iter() + .try_for_each(|entry| { + handle_entry( + entry, + &other, + &other_csums, + self, + &csums, + None, + &mut diff.added.paths, + ) + })?; + + Ok(diff) + } + + pub(crate) fn list_files(&self) -> Result, Error> { + let mut file_list = Vec::new(); + WalkDir::new(&self.link_dir) + .into_iter() + .try_for_each(|entry| -> Result<(), Error> { + let path = entry?.into_path(); + let meta = path.metadata()?; + let relative = path.strip_prefix(&self.link_dir)?; + + file_list.push((relative.to_path_buf(), meta)); + Ok(()) + })?; + Ok(file_list) + } } fn link_file_do(source: &Path, target: &Path) -> Result { diff --git a/src/types.rs b/src/types.rs index 7a1348a..3098a8d 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,4 +1,4 @@ -use std::{fmt::Display, str::FromStr}; +use std::{fmt::Display, path::PathBuf, str::FromStr}; use anyhow::Error; use proxmox_schema::{api, const_regex, ApiStringFormat, Schema, StringSchema, Updater}; @@ -140,3 +140,17 @@ impl FromStr for ProductType { } } } + +/// Entries of Diff +#[derive(Default)] +pub struct DiffMember { + pub paths: Vec<(PathBuf, u64)>, +} + +/// Differences between two pools or pool directories +#[derive(Default)] +pub struct Diff { + pub added: DiffMember, + pub changed: DiffMember, + pub removed: DiffMember, +} -- 2.30.2