* [pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: add --dry-run parameter
2022-09-16 10:35 [pve-devel] [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions Fabian Grünbichler
@ 2022-09-16 10:35 ` Fabian Grünbichler
2022-09-16 10:35 ` [pve-devel] [PATCH proxmox-offline-mirror 2/3] cli: extract subscription key helper Fabian Grünbichler
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Fabian Grünbichler @ 2022-09-16 10:35 UTC (permalink / raw)
To: pve-devel
in dry-run mode, creating a snapshot will download (but not persist) the
Release files and any indices referenced within, but not download the
package files themselves. instead, any URLs that would still need to be
fetched are printed, and the statistics about to-be-fetched files and
bytes is updated accordingly.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/bin/proxmox_offline_mirror_cmds/mirror.rs | 20 +++-
src/mirror.rs | 108 ++++++++++++++----
2 files changed, 102 insertions(+), 26 deletions(-)
diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 8059c0a..0798f86 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -28,11 +28,22 @@ use super::get_config_path;
id: {
schema: MIRROR_ID_SCHEMA,
},
+ "dry-run": {
+ type: bool,
+ optional: true,
+ default: false,
+ description: "Only fetch indices and print summary of missing package files, don't store anything.",
+ }
},
},
)]
/// Create a new repository snapshot, fetching required/missing files from original repository.
-async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> Result<(), Error> {
+async fn create_snapshot(
+ config: Option<String>,
+ id: String,
+ dry_run: bool,
+ _param: Value,
+) -> Result<(), Error> {
let config = config.unwrap_or_else(get_config_path);
let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
@@ -61,7 +72,12 @@ async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> R
None
};
- proxmox_offline_mirror::mirror::create_snapshot(config, &Snapshot::now(), subscription)?;
+ proxmox_offline_mirror::mirror::create_snapshot(
+ config,
+ &Snapshot::now(),
+ subscription,
+ dry_run,
+ )?;
Ok(())
}
diff --git a/src/mirror.rs b/src/mirror.rs
index a1fc1a0..5126393 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -143,6 +143,7 @@ fn fetch_release(
config: &ParsedMirrorConfig,
prefix: &Path,
detached: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let (name, fetched, sig) = if detached {
println!("Fetching Release/Release.gpg files");
@@ -185,6 +186,13 @@ fn fetch_release(
..Default::default()
};
+ if dry_run {
+ return Ok(FetchResult {
+ data: verified,
+ fetched: fetched.fetched,
+ });
+ }
+
let locked = &config.pool.lock()?;
if !locked.contains(&csums) {
@@ -234,6 +242,7 @@ fn fetch_index_file(
reference: &FileReference,
uncompressed: Option<&FileReference>,
by_hash: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let url = get_dist_url(&config.repository, &reference.path);
let path = get_dist_path(&config.repository, prefix, &reference.path);
@@ -248,6 +257,9 @@ fn fetch_index_file(
.pool
.get_contents(&uncompressed.checksums, config.verify)?;
+ if dry_run {
+ return Ok(FetchResult { data, fetched: 0 });
+ }
// Ensure they're linked at current path
config.pool.lock()?.link_file(&reference.checksums, &path)?;
config
@@ -285,6 +297,7 @@ fn fetch_index_file(
reference.size,
&reference.checksums,
true,
+ dry_run,
)),
})
.ok_or_else(|| format_err!("Failed to retrieve {}", reference.path))??;
@@ -310,6 +323,14 @@ fn fetch_index_file(
&buf[..]
}
};
+ let res = FetchResult {
+ data: decompressed.to_owned(),
+ fetched: res.fetched,
+ };
+
+ if dry_run {
+ return Ok(res);
+ }
let locked = &config.pool.lock()?;
if let Some(uncompressed) = uncompressed {
@@ -322,10 +343,7 @@ fn fetch_index_file(
locked.link_file(&uncompressed.checksums, &uncompressed_path)?;
}
- Ok(FetchResult {
- data: decompressed.to_owned(),
- fetched: res.fetched,
- })
+ Ok(res)
}
/// Helper to fetch arbitrary files like binary packages.
@@ -340,6 +358,7 @@ fn fetch_plain_file(
max_size: usize,
checksums: &CheckSums,
need_data: bool,
+ dry_run: bool,
) -> Result<FetchResult, Error> {
let locked = &config.pool.lock()?;
let res = if locked.contains(checksums) {
@@ -355,6 +374,11 @@ fn fetch_plain_file(
fetched: 0,
}
}
+ } else if dry_run && !need_data {
+ FetchResult {
+ data: vec![],
+ fetched: 0,
+ }
} else {
let fetched = fetch_repo_file(
&config.client,
@@ -367,8 +391,10 @@ fn fetch_plain_file(
fetched
};
- // Ensure it's linked at current path
- locked.link_file(checksums, file)?;
+ if !dry_run {
+ // Ensure it's linked at current path
+ locked.link_file(checksums, file)?;
+ }
Ok(res)
}
@@ -433,6 +459,7 @@ pub fn create_snapshot(
config: MirrorConfig,
snapshot: &Snapshot,
subscription: Option<SubscriptionKey>,
+ dry_run: bool,
) -> Result<(), Error> {
let auth = if let Some(product) = &config.use_subscription {
match subscription {
@@ -477,11 +504,11 @@ pub fn create_snapshot(
};
// we want both on-disk for compat reasons
- let res = fetch_release(&config, prefix, true)?;
+ let res = fetch_release(&config, prefix, true, dry_run)?;
total_progress.update(&res);
let _release = parse_release(res, "Release")?;
- let res = fetch_release(&config, prefix, false)?;
+ let res = fetch_release(&config, prefix, false, dry_run)?;
total_progress.update(&res);
let release = parse_release(res, "InRelease")?;
@@ -591,6 +618,7 @@ pub fn create_snapshot(
reference,
uncompressed_ref,
release.aquire_by_hash,
+ dry_run,
) {
Ok(res) => res,
Err(err) if !reference.file_type.is_package_index() => {
@@ -632,6 +660,7 @@ pub fn create_snapshot(
}
println!("\nFetching packages..");
+ let mut dry_run_progress = Progress::new();
for (basename, references) in packages_indices {
let total_files = references.files.len();
if total_files == 0 {
@@ -643,30 +672,61 @@ pub fn create_snapshot(
let mut fetch_progress = Progress::new();
for package in references.files {
- let mut full_path = PathBuf::from(prefix);
- full_path.push(&package.file);
- let res = fetch_plain_file(
- &config,
- &get_repo_url(&config.repository, &package.file),
- &full_path,
- package.size,
- &package.checksums,
- false,
- )?;
- fetch_progress.update(&res);
+ let url = get_repo_url(&config.repository, &package.file);
+
+ if dry_run {
+ if config.pool.contains(&package.checksums) {
+ fetch_progress.update(&FetchResult {
+ data: vec![],
+ fetched: 0,
+ });
+ } else {
+ println!("\t(dry-run) GET missing '{url}' ({}b)", package.size);
+ fetch_progress.update(&FetchResult {
+ data: vec![],
+ fetched: package.size,
+ });
+ }
+ } else {
+ let mut full_path = PathBuf::from(prefix);
+ full_path.push(&package.file);
+
+ let res = fetch_plain_file(
+ &config,
+ &url,
+ &full_path,
+ package.size,
+ &package.checksums,
+ false,
+ dry_run,
+ )?;
+ fetch_progress.update(&res);
+ }
+
if fetch_progress.file_count() % (max(total_files / 100, 1)) == 0 {
println!("\tProgress: {fetch_progress}");
}
}
println!("\tProgress: {fetch_progress}");
- total_progress += fetch_progress;
+ if dry_run {
+ dry_run_progress += fetch_progress;
+ } else {
+ total_progress += fetch_progress;
+ }
}
- println!("\nStats: {total_progress}");
+ if dry_run {
+ println!("\nDry-run Stats (indices, downloaded but not persisted):\n{total_progress}");
+ println!("\nDry-run stats (packages, new == missing):\n{dry_run_progress}");
+ } else {
+ println!("\nStats: {total_progress}");
+ }
- println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
- let locked = config.pool.lock()?;
- locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+ if !dry_run {
+ println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
+ let locked = config.pool.lock()?;
+ locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+ }
Ok(())
}
--
2.30.2
^ permalink raw reply [flat|nested] 5+ messages in thread* [pve-devel] [PATCH proxmox-offline-mirror 3/3] cli: add `mirror snapshot create-all` command
2022-09-16 10:35 [pve-devel] [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions Fabian Grünbichler
2022-09-16 10:35 ` [pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: add --dry-run parameter Fabian Grünbichler
2022-09-16 10:35 ` [pve-devel] [PATCH proxmox-offline-mirror 2/3] cli: extract subscription key helper Fabian Grünbichler
@ 2022-09-16 10:35 ` Fabian Grünbichler
2022-09-16 12:31 ` [pve-devel] applied-series: [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions Thomas Lamprecht
3 siblings, 0 replies; 5+ messages in thread
From: Fabian Grünbichler @ 2022-09-16 10:35 UTC (permalink / raw)
To: pve-devel
that creates a new snapshot for each configured mirror, collecting the
results and printing a summary at the end. this should be suitable for
usage in a cron job or timer-triggered unit, with no output on stderr
for 100% OK execution runs.
Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
src/bin/proxmox_offline_mirror_cmds/mirror.rs | 78 ++++++++++++++++++-
1 file changed, 77 insertions(+), 1 deletion(-)
diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 4503971..3094146 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -1,8 +1,9 @@
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
use proxmox_section_config::SectionConfigData;
use proxmox_subscription::SubscriptionStatus;
use serde_json::Value;
+use std::collections::HashMap;
use proxmox_router::cli::{
format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
@@ -89,6 +90,80 @@ async fn create_snapshot(
Ok(())
}
+#[api(
+ input: {
+ properties: {
+ config: {
+ type: String,
+ optional: true,
+ description: "Path to mirroring config file.",
+ },
+ "dry-run": {
+ type: bool,
+ optional: true,
+ default: false,
+ description: "Only fetch indices and print summary of missing package files, don't store anything.",
+ }
+ },
+ },
+ )]
+/// Create a new repository snapshot for each configured mirror, fetching required/missing files
+/// from original repository.
+async fn create_snapshots(
+ config: Option<String>,
+ dry_run: bool,
+ _param: Value,
+) -> Result<(), Error> {
+ let config = config.unwrap_or_else(get_config_path);
+
+ let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
+ let mirrors: Vec<MirrorConfig> = section_config.convert_to_typed_array("mirror")?;
+
+ let mut results = HashMap::new();
+
+ for mirror in mirrors {
+ let mirror_id = mirror.id.clone();
+ println!("\nCREATING SNAPSHOT FOR '{mirror_id}'..");
+ let subscription = match get_subscription_key(§ion_config, &mirror) {
+ Ok(opt_key) => opt_key,
+ Err(err) => {
+ eprintln!("Skipping mirror '{mirror_id}' - {err})");
+ results.insert(mirror_id, Err(err));
+ continue;
+ }
+ };
+ let res = proxmox_offline_mirror::mirror::create_snapshot(
+ mirror,
+ &Snapshot::now(),
+ subscription,
+ dry_run,
+ );
+ if let Err(err) = &res {
+ eprintln!("Failed to create snapshot for '{mirror_id}' - {err}");
+ }
+
+ results.insert(mirror_id, res);
+ }
+
+ println!("\nSUMMARY:");
+ for (mirror_id, _res) in results.iter().filter(|(_, res)| res.is_ok()) {
+ println!("{mirror_id}: OK"); // TODO update once we have a proper return value
+ }
+
+ let mut fail = false;
+
+ for (mirror_id, res) in results.into_iter().filter(|(_, res)| res.is_err()) {
+ fail = true;
+ eprintln!("{mirror_id}: ERR - {}", res.unwrap_err());
+ }
+
+ if fail {
+ bail!("Failed to create snapshots for all configured mirrors.");
+ }
+
+ Ok(())
+}
+
#[api(
input: {
properties: {
@@ -204,6 +279,7 @@ pub fn mirror_commands() -> CommandLineInterface {
"create",
CliCommand::new(&API_METHOD_CREATE_SNAPSHOT).arg_param(&["id"]),
)
+ .insert("create-all", CliCommand::new(&API_METHOD_CREATE_SNAPSHOTS))
.insert(
"list",
CliCommand::new(&API_METHOD_LIST_SNAPSHOTS).arg_param(&["id"]),
--
2.30.2
^ permalink raw reply [flat|nested] 5+ messages in thread