[pve-devel] [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions

[pve-devel] [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions

[Fabian Grünbichler]

this series adds a new `--dry-run` parameter to `mirror snapshot
create`, and a new `mirror snapshot create-all` command.

it's based on top of my other series:

misc improvements - 20220915130918.727902-1-f.gruenbichler@proxmox.com

Fabian Grünbichler (3):
  mirror: add --dry-run parameter
  cli: extract subscription key helper
  cli: add `mirror snapshot create-all` command

 src/bin/proxmox_offline_mirror_cmds/mirror.rs | 149 +++++++++++++++---
 src/mirror.rs                                 | 108 ++++++++++---
 2 files changed, 208 insertions(+), 49 deletions(-)

-- 
2.30.2

[pve-devel] [PATCH proxmox-offline-mirror 1/3] mirror: add --dry-run parameter

[Fabian Grünbichler]

in dry-run mode, creating a snapshot will download (but not persist) the
Release files and any indices referenced within, but not download the
package files themselves. instead, any URLs that would still need to be
fetched are printed, and the statistics about to-be-fetched files and
bytes is updated accordingly.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_offline_mirror_cmds/mirror.rs |  20 +++-
 src/mirror.rs                                 | 108 ++++++++++++++----
 2 files changed, 102 insertions(+), 26 deletions(-)

diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 8059c0a..0798f86 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -28,11 +28,22 @@ use super::get_config_path;
             id: {
                 schema: MIRROR_ID_SCHEMA,
             },
+            "dry-run": {
+                type: bool,
+                optional: true,
+                default: false,
+                description: "Only fetch indices and print summary of missing package files, don't store anything.",
+            }
         },
     },
  )]
 /// Create a new repository snapshot, fetching required/missing files from original repository.
-async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> Result<(), Error> {
+async fn create_snapshot(
+    config: Option<String>,
+    id: String,
+    dry_run: bool,
+    _param: Value,
+) -> Result<(), Error> {
     let config = config.unwrap_or_else(get_config_path);
 
     let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
@@ -61,7 +72,12 @@ async fn create_snapshot(config: Option<String>, id: String, _param: Value) -> R
         None
     };
 
-    proxmox_offline_mirror::mirror::create_snapshot(config, &Snapshot::now(), subscription)?;
+    proxmox_offline_mirror::mirror::create_snapshot(
+        config,
+        &Snapshot::now(),
+        subscription,
+        dry_run,
+    )?;
 
     Ok(())
 }
diff --git a/src/mirror.rs b/src/mirror.rs
index a1fc1a0..5126393 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -143,6 +143,7 @@ fn fetch_release(
     config: &ParsedMirrorConfig,
     prefix: &Path,
     detached: bool,
+    dry_run: bool,
 ) -> Result<FetchResult, Error> {
     let (name, fetched, sig) = if detached {
         println!("Fetching Release/Release.gpg files");
@@ -185,6 +186,13 @@ fn fetch_release(
         ..Default::default()
     };
 
+    if dry_run {
+        return Ok(FetchResult {
+            data: verified,
+            fetched: fetched.fetched,
+        });
+    }
+
     let locked = &config.pool.lock()?;
 
     if !locked.contains(&csums) {
@@ -234,6 +242,7 @@ fn fetch_index_file(
     reference: &FileReference,
     uncompressed: Option<&FileReference>,
     by_hash: bool,
+    dry_run: bool,
 ) -> Result<FetchResult, Error> {
     let url = get_dist_url(&config.repository, &reference.path);
     let path = get_dist_path(&config.repository, prefix, &reference.path);
@@ -248,6 +257,9 @@ fn fetch_index_file(
                 .pool
                 .get_contents(&uncompressed.checksums, config.verify)?;
 
+            if dry_run {
+                return Ok(FetchResult { data, fetched: 0 });
+            }
             // Ensure they're linked at current path
             config.pool.lock()?.link_file(&reference.checksums, &path)?;
             config
@@ -285,6 +297,7 @@ fn fetch_index_file(
                 reference.size,
                 &reference.checksums,
                 true,
+                dry_run,
             )),
         })
         .ok_or_else(|| format_err!("Failed to retrieve {}", reference.path))??;
@@ -310,6 +323,14 @@ fn fetch_index_file(
             &buf[..]
         }
     };
+    let res = FetchResult {
+        data: decompressed.to_owned(),
+        fetched: res.fetched,
+    };
+
+    if dry_run {
+        return Ok(res);
+    }
 
     let locked = &config.pool.lock()?;
     if let Some(uncompressed) = uncompressed {
@@ -322,10 +343,7 @@ fn fetch_index_file(
         locked.link_file(&uncompressed.checksums, &uncompressed_path)?;
     }
 
-    Ok(FetchResult {
-        data: decompressed.to_owned(),
-        fetched: res.fetched,
-    })
+    Ok(res)
 }
 
 /// Helper to fetch arbitrary files like binary packages.
@@ -340,6 +358,7 @@ fn fetch_plain_file(
     max_size: usize,
     checksums: &CheckSums,
     need_data: bool,
+    dry_run: bool,
 ) -> Result<FetchResult, Error> {
     let locked = &config.pool.lock()?;
     let res = if locked.contains(checksums) {
@@ -355,6 +374,11 @@ fn fetch_plain_file(
                 fetched: 0,
             }
         }
+    } else if dry_run && !need_data {
+        FetchResult {
+            data: vec![],
+            fetched: 0,
+        }
     } else {
         let fetched = fetch_repo_file(
             &config.client,
@@ -367,8 +391,10 @@ fn fetch_plain_file(
         fetched
     };
 
-    // Ensure it's linked at current path
-    locked.link_file(checksums, file)?;
+    if !dry_run {
+        // Ensure it's linked at current path
+        locked.link_file(checksums, file)?;
+    }
 
     Ok(res)
 }
@@ -433,6 +459,7 @@ pub fn create_snapshot(
     config: MirrorConfig,
     snapshot: &Snapshot,
     subscription: Option<SubscriptionKey>,
+    dry_run: bool,
 ) -> Result<(), Error> {
     let auth = if let Some(product) = &config.use_subscription {
         match subscription {
@@ -477,11 +504,11 @@ pub fn create_snapshot(
     };
 
     // we want both on-disk for compat reasons
-    let res = fetch_release(&config, prefix, true)?;
+    let res = fetch_release(&config, prefix, true, dry_run)?;
     total_progress.update(&res);
     let _release = parse_release(res, "Release")?;
 
-    let res = fetch_release(&config, prefix, false)?;
+    let res = fetch_release(&config, prefix, false, dry_run)?;
     total_progress.update(&res);
     let release = parse_release(res, "InRelease")?;
 
@@ -591,6 +618,7 @@ pub fn create_snapshot(
                     reference,
                     uncompressed_ref,
                     release.aquire_by_hash,
+                    dry_run,
                 ) {
                     Ok(res) => res,
                     Err(err) if !reference.file_type.is_package_index() => {
@@ -632,6 +660,7 @@ pub fn create_snapshot(
     }
 
     println!("\nFetching packages..");
+    let mut dry_run_progress = Progress::new();
     for (basename, references) in packages_indices {
         let total_files = references.files.len();
         if total_files == 0 {
@@ -643,30 +672,61 @@ pub fn create_snapshot(
 
         let mut fetch_progress = Progress::new();
         for package in references.files {
-            let mut full_path = PathBuf::from(prefix);
-            full_path.push(&package.file);
-            let res = fetch_plain_file(
-                &config,
-                &get_repo_url(&config.repository, &package.file),
-                &full_path,
-                package.size,
-                &package.checksums,
-                false,
-            )?;
-            fetch_progress.update(&res);
+            let url = get_repo_url(&config.repository, &package.file);
+
+            if dry_run {
+                if config.pool.contains(&package.checksums) {
+                    fetch_progress.update(&FetchResult {
+                        data: vec![],
+                        fetched: 0,
+                    });
+                } else {
+                    println!("\t(dry-run) GET missing '{url}' ({}b)", package.size);
+                    fetch_progress.update(&FetchResult {
+                        data: vec![],
+                        fetched: package.size,
+                    });
+                }
+            } else {
+                let mut full_path = PathBuf::from(prefix);
+                full_path.push(&package.file);
+
+                let res = fetch_plain_file(
+                    &config,
+                    &url,
+                    &full_path,
+                    package.size,
+                    &package.checksums,
+                    false,
+                    dry_run,
+                )?;
+                fetch_progress.update(&res);
+            }
+
             if fetch_progress.file_count() % (max(total_files / 100, 1)) == 0 {
                 println!("\tProgress: {fetch_progress}");
             }
         }
         println!("\tProgress: {fetch_progress}");
-        total_progress += fetch_progress;
+        if dry_run {
+            dry_run_progress += fetch_progress;
+        } else {
+            total_progress += fetch_progress;
+        }
     }
 
-    println!("\nStats: {total_progress}");
+    if dry_run {
+        println!("\nDry-run Stats (indices, downloaded but not persisted):\n{total_progress}");
+        println!("\nDry-run stats (packages, new == missing):\n{dry_run_progress}");
+    } else {
+        println!("\nStats: {total_progress}");
+    }
 
-    println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
-    let locked = config.pool.lock()?;
-    locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+    if !dry_run {
+        println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
+        let locked = config.pool.lock()?;
+        locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
+    }
 
     Ok(())
 }
-- 
2.30.2

[pve-devel] [PATCH proxmox-offline-mirror 2/3] cli: extract subscription key helper

[Fabian Grünbichler]

for re-use in the (not-yet-created) create-all command.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_offline_mirror_cmds/mirror.rs | 51 +++++++++++--------
 1 file changed, 29 insertions(+), 22 deletions(-)

diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 0798f86..4503971 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -1,5 +1,6 @@
 use anyhow::{format_err, Error};
 
+use proxmox_section_config::SectionConfigData;
 use proxmox_subscription::SubscriptionStatus;
 use serde_json::Value;
 
@@ -17,6 +18,33 @@ use proxmox_offline_mirror::{
 
 use super::get_config_path;
 
+fn get_subscription_key(
+    config: &SectionConfigData,
+    mirror: &MirrorConfig,
+) -> Result<Option<SubscriptionKey>, Error> {
+    if let Some(product) = &mirror.use_subscription {
+        let subscriptions: Vec<SubscriptionKey> = config.convert_to_typed_array("subscription")?;
+        let key = subscriptions
+            .iter()
+            .find(|key| {
+                if let Ok(Some(info)) = key.info() {
+                    info.status == SubscriptionStatus::Active && key.product() == *product
+                } else {
+                    false
+                }
+            })
+            .ok_or_else(|| {
+                format_err!(
+                    "Need matching active subscription key for product {product}, but none found."
+                )
+            })?
+            .clone();
+        Ok(Some(key))
+    } else {
+        Ok(None)
+    }
+}
+
 #[api(
     input: {
         properties: {
@@ -49,28 +77,7 @@ async fn create_snapshot(
     let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
     let config: MirrorConfig = section_config.lookup("mirror", &id)?;
 
-    let subscription = if let Some(product) = &config.use_subscription {
-        let subscriptions: Vec<SubscriptionKey> =
-            section_config.convert_to_typed_array("subscription")?;
-        let key = subscriptions
-            .iter()
-            .find(|key| {
-                if let Ok(Some(info)) = key.info() {
-                    info.status == SubscriptionStatus::Active && key.product() == *product
-                } else {
-                    false
-                }
-            })
-            .ok_or_else(|| {
-                format_err!(
-                    "Need matching active subscription key for product {product}, but none found."
-                )
-            })?
-            .clone();
-        Some(key)
-    } else {
-        None
-    };
+    let subscription = get_subscription_key(&section_config, &config)?;
 
     proxmox_offline_mirror::mirror::create_snapshot(
         config,
-- 
2.30.2

[pve-devel] [PATCH proxmox-offline-mirror 3/3] cli: add `mirror snapshot create-all` command

[Fabian Grünbichler]

that creates a new snapshot for each configured mirror, collecting the
results and printing a summary at the end. this should be suitable for
usage in a cron job or timer-triggered unit, with no output on stderr
for 100% OK execution runs.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_offline_mirror_cmds/mirror.rs | 78 ++++++++++++++++++-
 1 file changed, 77 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox_offline_mirror_cmds/mirror.rs b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
index 4503971..3094146 100644
--- a/src/bin/proxmox_offline_mirror_cmds/mirror.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/mirror.rs
@@ -1,8 +1,9 @@
-use anyhow::{format_err, Error};
+use anyhow::{bail, format_err, Error};
 
 use proxmox_section_config::SectionConfigData;
 use proxmox_subscription::SubscriptionStatus;
 use serde_json::Value;
+use std::collections::HashMap;
 
 use proxmox_router::cli::{
     format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface,
@@ -89,6 +90,80 @@ async fn create_snapshot(
     Ok(())
 }
 
+#[api(
+    input: {
+        properties: {
+            config: {
+                type: String,
+                optional: true,
+                description: "Path to mirroring config file.",
+            },
+            "dry-run": {
+                type: bool,
+                optional: true,
+                default: false,
+                description: "Only fetch indices and print summary of missing package files, don't store anything.",
+            }
+        },
+    },
+ )]
+/// Create a new repository snapshot for each configured mirror, fetching required/missing files
+/// from original repository.
+async fn create_snapshots(
+    config: Option<String>,
+    dry_run: bool,
+    _param: Value,
+) -> Result<(), Error> {
+    let config = config.unwrap_or_else(get_config_path);
+
+    let (section_config, _digest) = proxmox_offline_mirror::config::config(&config)?;
+    let mirrors: Vec<MirrorConfig> = section_config.convert_to_typed_array("mirror")?;
+
+    let mut results = HashMap::new();
+
+    for mirror in mirrors {
+        let mirror_id = mirror.id.clone();
+        println!("\nCREATING SNAPSHOT FOR '{mirror_id}'..");
+        let subscription = match get_subscription_key(&section_config, &mirror) {
+            Ok(opt_key) => opt_key,
+            Err(err) => {
+                eprintln!("Skipping mirror '{mirror_id}' - {err})");
+                results.insert(mirror_id, Err(err));
+                continue;
+            }
+        };
+        let res = proxmox_offline_mirror::mirror::create_snapshot(
+            mirror,
+            &Snapshot::now(),
+            subscription,
+            dry_run,
+        );
+        if let Err(err) = &res {
+            eprintln!("Failed to create snapshot for '{mirror_id}' - {err}");
+        }
+
+        results.insert(mirror_id, res);
+    }
+
+    println!("\nSUMMARY:");
+    for (mirror_id, _res) in results.iter().filter(|(_, res)| res.is_ok()) {
+        println!("{mirror_id}: OK"); // TODO update once we have a proper return value
+    }
+
+    let mut fail = false;
+
+    for (mirror_id, res) in results.into_iter().filter(|(_, res)| res.is_err()) {
+        fail = true;
+        eprintln!("{mirror_id}: ERR - {}", res.unwrap_err());
+    }
+
+    if fail {
+        bail!("Failed to create snapshots for all configured mirrors.");
+    }
+
+    Ok(())
+}
+
 #[api(
     input: {
         properties: {
@@ -204,6 +279,7 @@ pub fn mirror_commands() -> CommandLineInterface {
             "create",
             CliCommand::new(&API_METHOD_CREATE_SNAPSHOT).arg_param(&["id"]),
         )
+        .insert("create-all", CliCommand::new(&API_METHOD_CREATE_SNAPSHOTS))
         .insert(
             "list",
             CliCommand::new(&API_METHOD_LIST_SNAPSHOTS).arg_param(&["id"]),
-- 
2.30.2

[pve-devel] applied-series: [PATCH proxmox-offline-mirror 0/3] --dry-run / create-all additions

[Thomas Lamprecht]

Am 16/09/2022 um 12:35 schrieb Fabian Grünbichler:
> this series adds a new `--dry-run` parameter to `mirror snapshot
> create`, and a new `mirror snapshot create-all` command.
> 
> it's based on top of my other series:
> 
> misc improvements - 20220915130918.727902-1-f.gruenbichler@proxmox.com
> 
> Fabian Grünbichler (3):
>   mirror: add --dry-run parameter
>   cli: extract subscription key helper
>   cli: add `mirror snapshot create-all` command
> 
>  src/bin/proxmox_offline_mirror_cmds/mirror.rs | 149 +++++++++++++++---
>  src/mirror.rs                                 | 108 ++++++++++---
>  2 files changed, 208 insertions(+), 49 deletions(-)
> 

applied, thanks!