From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 8DC77941FA for ; Wed, 21 Sep 2022 10:13:28 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 8485D24D3A for ; Wed, 21 Sep 2022 10:12:58 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS for ; Wed, 21 Sep 2022 10:12:57 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 607A14416F for ; Wed, 21 Sep 2022 10:12:57 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pve-devel@lists.proxmox.com Date: Wed, 21 Sep 2022 10:12:41 +0200 Message-Id: <20220921081242.1139249-4-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20220921081242.1139249-1-f.gruenbichler@proxmox.com> References: <20220921081242.1139249-1-f.gruenbichler@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.148 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pve-devel] [PATCH proxmox-offline-mirror 3/4] medium: add diff command X-BeenThere: pve-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox VE development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 21 Sep 2022 08:13:28 -0000 Signed-off-by: Fabian Grünbichler --- src/bin/proxmox_offline_mirror_cmds/medium.rs | 107 +++++++++++++++++- src/medium.rs | 103 ++++++++++++++++- 2 files changed, 207 insertions(+), 3 deletions(-) diff --git a/src/bin/proxmox_offline_mirror_cmds/medium.rs b/src/bin/proxmox_offline_mirror_cmds/medium.rs index b76e4e6..574f748 100644 --- a/src/bin/proxmox_offline_mirror_cmds/medium.rs +++ b/src/bin/proxmox_offline_mirror_cmds/medium.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::{Path, PathBuf}; use anyhow::Error; use serde_json::Value; @@ -220,6 +220,108 @@ async fn sync( Ok(Value::Null) } +#[api( + input: { + properties: { + config: { + type: String, + optional: true, + description: "Path to mirroring config file.", + }, + id: { + schema: MEDIA_ID_SCHEMA, + }, + verbose: { + type: bool, + optional: true, + default: false, + description: "Verbose output (print paths in addition to summary)." + }, + } + }, + )] +/// Diff a medium +async fn diff( + config: Option, + id: String, + verbose: bool, + _param: Value, +) -> Result { + let config = config.unwrap_or_else(get_config_path); + + let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?; + let config: MediaConfig = section_config.lookup("medium", &id)?; + let mut mirrors = Vec::with_capacity(config.mirrors.len()); + for mirror in &config.mirrors { + let mirror: MirrorConfig = section_config.lookup("mirror", mirror)?; + mirrors.push(mirror); + } + + let mut diffs = medium::diff(&config, mirrors)?; + let mut mirrors: Vec = diffs.keys().cloned().collect(); + mirrors.sort_unstable(); + + let sort_paths = + |(path, _): &(PathBuf, u64), (other_path, _): &(PathBuf, u64)| path.cmp(other_path); + + let mut first = true; + for mirror in mirrors { + if first { + first = false; + } else { + println!(); + } + + println!("Mirror '{mirror}'"); + if let Some(Some(mut diff)) = diffs.remove(&mirror) { + let mut total_size = 0; + println!("\t{} file(s) only on medium:", diff.added.paths.len()); + if verbose { + diff.added.paths.sort_unstable_by(sort_paths); + diff.changed.paths.sort_unstable_by(sort_paths); + diff.removed.paths.sort_unstable_by(sort_paths); + } + for (path, size) in diff.added.paths { + if verbose { + println!("\t\t{path:?}: +{size}b"); + } + total_size += size; + } + println!("\tTotal size: +{total_size}b"); + + total_size = 0; + println!( + "\n\t{} file(s) missing on medium:", + diff.removed.paths.len() + ); + for (path, size) in diff.removed.paths { + if verbose { + println!("\t\t{path:?}: -{size}b"); + } + total_size += size; + } + println!("\tTotal size: -{total_size}b"); + + total_size = 0; + println!( + "\n\t{} file(s) diff between source and medium:", + diff.changed.paths.len() + ); + for (path, size) in diff.changed.paths { + if verbose { + println!("\t\t{path:?}: +-{size}b"); + } + } + println!("\tSum of size differences: +-{total_size}b"); + } else { + // TODO + println!("\tNot yet synced or no longer available on source side."); + } + } + + Ok(Value::Null) +} + pub fn medium_commands() -> CommandLineInterface { let cmd_def = CliCommandMap::new() .insert( @@ -230,7 +332,8 @@ pub fn medium_commands() -> CommandLineInterface { "status", CliCommand::new(&API_METHOD_STATUS).arg_param(&["id"]), ) - .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"])); + .insert("sync", CliCommand::new(&API_METHOD_SYNC).arg_param(&["id"])) + .insert("diff", CliCommand::new(&API_METHOD_DIFF).arg_param(&["id"])); cmd_def.into() } diff --git a/src/medium.rs b/src/medium.rs index bfd35b4..4093fc1 100644 --- a/src/medium.rs +++ b/src/medium.rs @@ -1,5 +1,7 @@ use std::{ collections::{HashMap, HashSet}, + fs::Metadata, + os::linux::fs::MetadataExt, path::{Path, PathBuf}, }; @@ -16,7 +18,7 @@ use crate::{ generate_repo_file_line, mirror::pool, pool::Pool, - types::{Snapshot, SNAPSHOT_REGEX}, + types::{Diff, Snapshot, SNAPSHOT_REGEX}, }; #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -439,3 +441,102 @@ pub fn sync( Ok(()) } + +/// Sync medium's content according to config. +pub fn diff( + medium: &crate::config::MediaConfig, + mirrors: Vec, +) -> Result>, Error> { + let medium_base = Path::new(&medium.mountpoint); + if !medium_base.exists() { + bail!("Medium mountpoint doesn't exist."); + } + + let _lock = lock(medium_base)?; + + let state = + load_state(medium_base)?.ok_or_else(|| format_err!("Medium not yet initializes."))?; + + let mirror_state = get_mirror_state(medium, &state); + + let pools: HashMap = + state + .mirrors + .iter() + .fold(HashMap::new(), |mut map, (id, info)| { + map.insert(id.clone(), info.pool.clone()); + map + }); + + let mut diffs = HashMap::new(); + + let convert_file_list_to_diff = |files: Vec<(PathBuf, Metadata)>, added: bool| -> Diff { + files + .into_iter() + .fold(Diff::default(), |mut diff, (file, meta)| { + if !meta.is_file() { + return diff; + } + + let size = meta.st_size(); + if added { + diff.added.paths.push((file, size)); + } else { + diff.removed.paths.push((file, size)); + } + diff + }) + }; + + let get_target_pool = + |mirror_id: &str, mirror: Option<&MirrorConfig>| -> Result, Error> { + let mut mirror_base = medium_base.to_path_buf(); + mirror_base.push(Path::new(mirror_id)); + + let mut mirror_pool = medium_base.to_path_buf(); + let pool_dir = match pools.get(mirror_id) { + Some(pool_dir) => pool_dir.to_owned(), + None => { + if let Some(mirror) = mirror { + mirror_pool_dir(mirror) + } else { + return Ok(None); + } + } + }; + mirror_pool.push(pool_dir); + + Ok(Some(Pool::open(&mirror_base, &mirror_pool)?)) + }; + + for mirror in mirrors.into_iter() { + let source_pool: Pool = pool(&mirror)?; + + if !mirror_state.synced.contains(&mirror.id) { + let files = source_pool.lock()?.list_files()?; + diffs.insert(mirror.id, Some(convert_file_list_to_diff(files, false))); + continue; + } + + let target_pool = get_target_pool(mirror.id.as_str(), Some(&mirror))? + .ok_or_else(|| format_err!("Failed to open target pool."))?; + diffs.insert( + mirror.id, + Some(source_pool.lock()?.diff_pools(&target_pool)?), + ); + } + + for dropped in mirror_state.target_only { + match get_target_pool(&dropped, None)? { + Some(pool) => { + let files = pool.lock()?.list_files()?; + diffs.insert(dropped, Some(convert_file_list_to_diff(files, false))); + } + None => { + diffs.insert(dropped, None); + } + } + } + + Ok(diffs) +} -- 2.30.2