[pve-devel] [PATCH proxmox-offline-mirror 1/2] fix #4259: mirror: add ignore-errors option

[pve-devel] [PATCH proxmox-offline-mirror 1/2] fix #4259: mirror: add ignore-errors option

[Fabian Grünbichler]

to make fetching errors from broken repositories non-fatal.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
based on top of "extend/add commands" series from
20220921081242.1139249-1-f.gruenbichler@proxmox.com

 src/bin/proxmox-offline-mirror.rs             |  2 ++
 src/bin/proxmox_offline_mirror_cmds/config.rs |  3 +++
 src/config.rs                                 |  8 ++++++++
 src/mirror.rs                                 | 19 ++++++++++++++++---
 4 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/bin/proxmox-offline-mirror.rs b/src/bin/proxmox-offline-mirror.rs
index 0a6e77e..222b561 100644
--- a/src/bin/proxmox-offline-mirror.rs
+++ b/src/bin/proxmox-offline-mirror.rs
@@ -386,6 +386,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
                 sync,
                 base_dir: base_dir.clone(),
                 use_subscription: None,
+                ignore_errors: false,
             });
         }
     }
@@ -399,6 +400,7 @@ fn action_add_mirror(config: &SectionConfigData) -> Result<Vec<MirrorConfig>, Er
         sync,
         base_dir,
         use_subscription,
+        ignore_errors: false,
     };
 
     configs.push(main_config);
diff --git a/src/bin/proxmox_offline_mirror_cmds/config.rs b/src/bin/proxmox_offline_mirror_cmds/config.rs
index b48a708..5ebf6d5 100644
--- a/src/bin/proxmox_offline_mirror_cmds/config.rs
+++ b/src/bin/proxmox_offline_mirror_cmds/config.rs
@@ -262,6 +262,9 @@ pub fn update_mirror(
     if let Some(verify) = update.verify {
         data.verify = verify
     }
+    if let Some(ignore_errors) = update.ignore_errors {
+        data.ignore_errors = ignore_errors
+    }
 
     config.set_data(&id, "mirror", &data)?;
     proxmox_offline_mirror::config::save_config(&config_file, &config)?;
diff --git a/src/config.rs b/src/config.rs
index 6c2f3e8..cb9a22b 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -41,6 +41,11 @@ use crate::types::{
         sync: {
             type: bool,
         },
+        "ignore-errors": {
+            type: bool,
+            optional: true,
+            default: false,
+        },
     }
 )]
 #[derive(Clone, Debug, Serialize, Deserialize, Updater)]
@@ -65,6 +70,9 @@ pub struct MirrorConfig {
     /// Use subscription key to access (required for Proxmox Enterprise repositories).
     #[serde(skip_serializing_if = "Option::is_none")]
     pub use_subscription: Option<ProductType>,
+    /// Whether to downgrade download errors to warnings
+    #[serde(default)]
+    pub ignore_errors: bool,
 }
 
 #[api(
diff --git a/src/mirror.rs b/src/mirror.rs
index 5bf9219..e655847 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -46,6 +46,7 @@ struct ParsedMirrorConfig {
     pub sync: bool,
     pub auth: Option<String>,
     pub client: Client,
+    pub ignore_errors: bool,
 }
 
 impl TryInto<ParsedMirrorConfig> for MirrorConfig {
@@ -74,6 +75,7 @@ impl TryInto<ParsedMirrorConfig> for MirrorConfig {
             sync: self.sync,
             auth: None,
             client,
+            ignore_errors: self.ignore_errors,
         })
     }
 }
@@ -691,7 +693,7 @@ pub fn create_snapshot(
                 let mut full_path = PathBuf::from(prefix);
                 full_path.push(&package.file);
 
-                let res = fetch_plain_file(
+                match fetch_plain_file(
                     &config,
                     &url,
                     &full_path,
@@ -699,8 +701,19 @@ pub fn create_snapshot(
                     &package.checksums,
                     false,
                     dry_run,
-                )?;
-                fetch_progress.update(&res);
+                ) {
+                    Ok(res) => fetch_progress.update(&res),
+                    Err(err) if config.ignore_errors => {
+                        let msg = format!(
+                            "{}: failed to fetch package '{}' - {}",
+                            basename, package.file, err,
+                        );
+                        eprintln!("{msg}");
+                    }
+                    res => {
+                        res?;
+                    }
+                }
             }
 
             if fetch_progress.file_count() % (max(total_files / 100, 1)) == 0 {
-- 
2.30.2

[pve-devel] [PATCH proxmox-offline-mirror 2/2] mirror: collect and summarize warnings

[Fabian Grünbichler]

the output can get quite long and warnings can easily be missed
otherwise.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/mirror.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/mirror.rs b/src/mirror.rs
index e655847..f8afd2b 100644
--- a/src/mirror.rs
+++ b/src/mirror.rs
@@ -494,6 +494,7 @@ pub fn create_snapshot(
     let prefix = Path::new(&prefix);
 
     let mut total_progress = Progress::new();
+    let mut warnings = Vec::new();
 
     let parse_release = |res: FetchResult, name: &str| -> Result<ReleaseFile, Error> {
         println!("Parsing {name}..");
@@ -624,10 +625,12 @@ pub fn create_snapshot(
                 ) {
                     Ok(res) => res,
                     Err(err) if !reference.file_type.is_package_index() => {
-                        eprintln!(
+                        let msg = format!(
                             "Failed to fetch '{:?}' type reference '{}', skipping - {err}",
                             reference.file_type, reference.path
                         );
+                        eprintln!("{msg}");
+                        warnings.push(msg);
                         failed_references.push(reference);
                         continue;
                     }
@@ -709,6 +712,7 @@ pub fn create_snapshot(
                             basename, package.file, err,
                         );
                         eprintln!("{msg}");
+                        warnings.push(msg);
                     }
                     res => {
                         res?;
@@ -735,8 +739,15 @@ pub fn create_snapshot(
         println!("\nStats: {total_progress}");
     }
 
+    if !warnings.is_empty() {
+        eprintln!("Warnings:");
+        for msg in warnings {
+            eprintln!("- {msg}");
+        }
+    }
+
     if !dry_run {
-        println!("Rotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
+        println!("\nRotating temp. snapshot in-place: {prefix:?} -> \"{snapshot}\"");
         let locked = config.pool.lock()?;
         locked.rename(prefix, Path::new(&format!("{snapshot}")))?;
     }
-- 
2.30.2

[pve-devel] applied: [PATCH proxmox-offline-mirror 1/2] fix #4259: mirror: add ignore-errors option

[Wolfgang Bumiller]

applied both patches and cleaned up 2 error handlers (one from this
patch and one older one)

On Fri, Sep 23, 2022 at 12:33:51PM +0200, Fabian Grünbichler wrote:
> to make fetching errors from broken repositories non-fatal.
> 
> Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
> ---
> based on top of "extend/add commands" series from
> 20220921081242.1139249-1-f.gruenbichler@proxmox.com
> 
>  src/bin/proxmox-offline-mirror.rs             |  2 ++
>  src/bin/proxmox_offline_mirror_cmds/config.rs |  3 +++
>  src/config.rs                                 |  8 ++++++++
>  src/mirror.rs                                 | 19 ++++++++++++++++---
>  4 files changed, 29 insertions(+), 3 deletions(-)