[pbs-devel] [PATCH proxmox-backup 1/3] debug: recover: allow ignoring missing/corrupt chunks

[pbs-devel] [PATCH proxmox-backup 1/3] debug: recover: allow ignoring missing/corrupt chunks

[Fabian Grünbichler]

replacing them with chunks of zero bytes.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_backup_debug/recover.rs | 98 +++++++++++++++++++++----
 1 file changed, 84 insertions(+), 14 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
index b118a71c..9c4aed3d 100644
--- a/src/bin/proxmox_backup_debug/recover.rs
+++ b/src/bin/proxmox_backup_debug/recover.rs
@@ -38,7 +38,19 @@ use pbs_tools::crypt_config::CryptConfig;
                 type: Boolean,
                 optional: true,
                 default: false,
-            }
+            },
+            "ignore-missing-chunks": {
+                description: "If a chunk is missing, warn and write 0-bytes instead to attempt partial recovery.",
+                type: Boolean,
+                optional: true,
+                default: false,
+            },
+            "ignore-corrupt-chunks": {
+                description: "If a chunk is corrupt, warn and write 0-bytes instead to attempt partial recovery.",
+                type: Boolean,
+                optional: true,
+                default: false,
+            },
         }
     }
 )]
@@ -49,6 +61,8 @@ fn recover_index(
     chunks: String,
     keyfile: Option<String>,
     skip_crc: bool,
+    ignore_missing_chunks: bool,
+    ignore_corrupt_chunks: bool,
     _param: Value,
 ) -> Result<(), Error> {
     let file_path = Path::new(&file);
@@ -89,22 +103,78 @@ fn recover_index(
         let digest_str = hex::encode(chunk_digest);
         let digest_prefix = &digest_str[0..4];
         let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
-        let mut chunk_file = std::fs::File::open(&chunk_path)
-            .map_err(|e| format_err!("could not open chunk file - {}", e))?;
 
-        data.clear();
-        chunk_file.read_to_end(&mut data)?;
-        let chunk_blob = DataBlob::from_raw(data.clone())?;
+        let create_zero_chunk = |msg: String| -> Result<(DataBlob, Option<&[u8; 32]>), Error> {
+            let info = index
+                .chunk_info(pos)
+                .ok_or_else(|| format_err!("Couldn't read chunk info from index at {pos}"))?;
+            let size = info.size();
 
-        if !skip_crc {
-            chunk_blob.verify_crc()?;
-        }
+            eprintln!("WARN: chunk {:?} {}", chunk_path, msg);
+            eprintln!("WARN: replacing output file {:?} with '\\0'", info.range,);
+
+            Ok((
+                DataBlob::encode(&vec![0; size as usize], crypt_conf_opt.as_ref(), true)?,
+                None,
+            ))
+        };
+
+        let (chunk_blob, chunk_digest) = match std::fs::File::open(&chunk_path) {
+            Ok(mut chunk_file) => {
+                data.clear();
+                chunk_file.read_to_end(&mut data)?;
+
+                // first chance for corrupt chunk - handling magic fails
+                DataBlob::from_raw(data.clone())
+                    .map(|blob| (blob, Some(chunk_digest)))
+                    .or_else(|err| {
+                        if ignore_corrupt_chunks {
+                            create_zero_chunk(format!("is corrupt - {err}"))
+                        } else {
+                            bail!("{err}");
+                        }
+                    })?
+            }
+            Err(err) => {
+                if ignore_missing_chunks && err.kind() == std::io::ErrorKind::NotFound {
+                    create_zero_chunk(format!("is missing"))?
+                } else {
+                    bail!("could not open chunk file - {}", err);
+                }
+            }
+        };
+
+        // second chance - we need CRC to detect truncated chunks!
+        let crc_res = if skip_crc {
+            Ok(())
+        } else {
+            chunk_blob.verify_crc()
+        };
+
+        let (chunk_blob, chunk_digest) = if let Err(crc_err) = crc_res {
+            if ignore_corrupt_chunks {
+                create_zero_chunk(format!("is corrupt - {crc_err}"))?
+            } else {
+                bail!("Error at chunk {:?} - {crc_err}", chunk_path);
+            }
+        } else {
+            (chunk_blob, chunk_digest)
+        };
+
+        // third chance - decoding might fail (digest, compression, encryption)
+        let decoded = chunk_blob
+            .decode(crypt_conf_opt.as_ref(), chunk_digest)
+            .or_else(|err| {
+                if ignore_corrupt_chunks {
+                    create_zero_chunk(format!("fails to decode - {err}"))?
+                        .0
+                        .decode(crypt_conf_opt.as_ref(), None)
+                } else {
+                    bail!("Failed to decode chunk {:?} = {}", chunk_path, err);
+                }
+            })?;
 
-        output_file.write_all(
-            chunk_blob
-                .decode(crypt_conf_opt.as_ref(), Some(chunk_digest))?
-                .as_slice(),
-        )?;
+        output_file.write_all(decoded.as_slice())?;
     }
 
     Ok(())
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 2/3] debug: move outfile_or_stdout to module for reuse

[Fabian Grünbichler]

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_backup_debug/inspect.rs | 18 +++---------------
 src/bin/proxmox_backup_debug/mod.rs     | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs
index 37bc6e05..e50c50cc 100644
--- a/src/bin/proxmox_backup_debug/inspect.rs
+++ b/src/bin/proxmox_backup_debug/inspect.rs
@@ -1,7 +1,6 @@
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{stdout, Read, Seek, SeekFrom, Write};
-use std::panic::{RefUnwindSafe, UnwindSafe};
+use std::io::{Read, Seek, SeekFrom, Write};
 use std::path::Path;
 
 use anyhow::{bail, format_err, Error};
@@ -27,18 +26,6 @@ use pbs_datastore::index::IndexFile;
 use pbs_datastore::DataBlob;
 use pbs_tools::crypt_config::CryptConfig;
 
-// Returns either a new file, if a path is given, or stdout, if no path is given.
-fn outfile_or_stdout<P: AsRef<Path>>(
-    path: Option<P>,
-) -> std::io::Result<Box<dyn Write + Send + Sync + Unpin + RefUnwindSafe + UnwindSafe>> {
-    if let Some(path) = path {
-        let f = File::create(path)?;
-        Ok(Box::new(f) as Box<_>)
-    } else {
-        Ok(Box::new(stdout()) as Box<_>)
-    }
-}
-
 /// Decodes a blob and writes its content either to stdout or into a file
 fn decode_blob(
     mut output_path: Option<&Path>,
@@ -61,7 +48,8 @@ fn decode_blob(
         _ => output_path,
     };
 
-    outfile_or_stdout(output_path)?.write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?;
+    crate::outfile_or_stdout(output_path)?
+        .write_all(blob.decode(crypt_conf_opt, digest)?.as_slice())?;
     Ok(())
 }
 
diff --git a/src/bin/proxmox_backup_debug/mod.rs b/src/bin/proxmox_backup_debug/mod.rs
index f092c585..31bc68c3 100644
--- a/src/bin/proxmox_backup_debug/mod.rs
+++ b/src/bin/proxmox_backup_debug/mod.rs
@@ -1,3 +1,22 @@
+use std::{
+    fs::File,
+    io::{stdout, Write},
+    panic::{RefUnwindSafe, UnwindSafe},
+    path::Path,
+};
+
 pub mod api;
 pub mod inspect;
 pub mod recover;
+
+// Returns either a new file, if a path is given, or stdout, if no path is given.
+pub(crate) fn outfile_or_stdout<P: AsRef<Path>>(
+    path: Option<P>,
+) -> std::io::Result<Box<dyn Write + Send + Sync + Unpin + RefUnwindSafe + UnwindSafe>> {
+    if let Some(path) = path {
+        let f = File::create(path)?;
+        Ok(Box::new(f) as Box<_>)
+    } else {
+        Ok(Box::new(stdout()) as Box<_>)
+    }
+}
-- 
2.30.2

[pbs-devel] [PATCH proxmox-backup 3/3] debug: recover: allow overriding output-path

[Fabian Grünbichler]

including to STDOUT.

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
---
 src/bin/proxmox_backup_debug/recover.rs | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
index 9c4aed3d..23366b86 100644
--- a/src/bin/proxmox_backup_debug/recover.rs
+++ b/src/bin/proxmox_backup_debug/recover.rs
@@ -51,6 +51,11 @@ use pbs_tools::crypt_config::CryptConfig;
                 optional: true,
                 default: false,
             },
+            "output-path": {
+                type: String,
+                description: "Output file path, defaults to `file` without extension, '-' means STDOUT.",
+                optional: true,
+            },
         }
     }
 )]
@@ -63,6 +68,7 @@ fn recover_index(
     skip_crc: bool,
     ignore_missing_chunks: bool,
     ignore_corrupt_chunks: bool,
+    output_path: Option<String>,
     _param: Value,
 ) -> Result<(), Error> {
     let file_path = Path::new(&file);
@@ -92,9 +98,16 @@ fn recover_index(
         None
     };
 
-    let output_filename = file_path.file_stem().unwrap().to_str().unwrap();
-    let output_path = Path::new(output_filename);
-    let mut output_file = File::create(output_path)
+    let output_path = output_path.unwrap_or_else(|| {
+        let filename = file_path.file_stem().unwrap().to_str().unwrap();
+        filename.to_string()
+    });
+
+    let output_path = match output_path.as_str() {
+        "-" => None,
+        path => Some(path),
+    };
+    let mut output_file = crate::outfile_or_stdout(output_path)
         .map_err(|e| format_err!("could not create output file - {}", e))?;
 
     let mut data = Vec::with_capacity(4 * 1024 * 1024);
-- 
2.30.2

Re: [pbs-devel] [PATCH proxmox-backup 1/3] debug: recover: allow ignoring missing/corrupt chunks

[Hannes Laimer]

Tested with both dynamic and fixed indices, everything worked as described.

Tested-by: Hannes Laimer <h.laimer@proxmox.com>

Am 23.05.22 um 16:11 schrieb Fabian Grünbichler:
> replacing them with chunks of zero bytes.
> 
> Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
> ---
>   src/bin/proxmox_backup_debug/recover.rs | 98 +++++++++++++++++++++----
>   1 file changed, 84 insertions(+), 14 deletions(-)
> 
> diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs
> index b118a71c..9c4aed3d 100644
> --- a/src/bin/proxmox_backup_debug/recover.rs
> +++ b/src/bin/proxmox_backup_debug/recover.rs
> @@ -38,7 +38,19 @@ use pbs_tools::crypt_config::CryptConfig;
>                   type: Boolean,
>                   optional: true,
>                   default: false,
> -            }
> +            },
> +            "ignore-missing-chunks": {
> +                description: "If a chunk is missing, warn and write 0-bytes instead to attempt partial recovery.",
> +                type: Boolean,
> +                optional: true,
> +                default: false,
> +            },
> +            "ignore-corrupt-chunks": {
> +                description: "If a chunk is corrupt, warn and write 0-bytes instead to attempt partial recovery.",
> +                type: Boolean,
> +                optional: true,
> +                default: false,
> +            },
>           }
>       }
>   )]
> @@ -49,6 +61,8 @@ fn recover_index(
>       chunks: String,
>       keyfile: Option<String>,
>       skip_crc: bool,
> +    ignore_missing_chunks: bool,
> +    ignore_corrupt_chunks: bool,
>       _param: Value,
>   ) -> Result<(), Error> {
>       let file_path = Path::new(&file);
> @@ -89,22 +103,78 @@ fn recover_index(
>           let digest_str = hex::encode(chunk_digest);
>           let digest_prefix = &digest_str[0..4];
>           let chunk_path = chunks_path.join(digest_prefix).join(digest_str);
> -        let mut chunk_file = std::fs::File::open(&chunk_path)
> -            .map_err(|e| format_err!("could not open chunk file - {}", e))?;
>   
> -        data.clear();
> -        chunk_file.read_to_end(&mut data)?;
> -        let chunk_blob = DataBlob::from_raw(data.clone())?;
> +        let create_zero_chunk = |msg: String| -> Result<(DataBlob, Option<&[u8; 32]>), Error> {
> +            let info = index
> +                .chunk_info(pos)
> +                .ok_or_else(|| format_err!("Couldn't read chunk info from index at {pos}"))?;
> +            let size = info.size();
>   
> -        if !skip_crc {
> -            chunk_blob.verify_crc()?;
> -        }
> +            eprintln!("WARN: chunk {:?} {}", chunk_path, msg);
> +            eprintln!("WARN: replacing output file {:?} with '\\0'", info.range,);
> +
> +            Ok((
> +                DataBlob::encode(&vec![0; size as usize], crypt_conf_opt.as_ref(), true)?,
> +                None,
> +            ))
> +        };
> +
> +        let (chunk_blob, chunk_digest) = match std::fs::File::open(&chunk_path) {
> +            Ok(mut chunk_file) => {
> +                data.clear();
> +                chunk_file.read_to_end(&mut data)?;
> +
> +                // first chance for corrupt chunk - handling magic fails
> +                DataBlob::from_raw(data.clone())
> +                    .map(|blob| (blob, Some(chunk_digest)))
> +                    .or_else(|err| {
> +                        if ignore_corrupt_chunks {
> +                            create_zero_chunk(format!("is corrupt - {err}"))
> +                        } else {
> +                            bail!("{err}");
> +                        }
> +                    })?
> +            }
> +            Err(err) => {
> +                if ignore_missing_chunks && err.kind() == std::io::ErrorKind::NotFound {
> +                    create_zero_chunk(format!("is missing"))?
> +                } else {
> +                    bail!("could not open chunk file - {}", err);
> +                }
> +            }
> +        };
> +
> +        // second chance - we need CRC to detect truncated chunks!
> +        let crc_res = if skip_crc {
> +            Ok(())
> +        } else {
> +            chunk_blob.verify_crc()
> +        };
> +
> +        let (chunk_blob, chunk_digest) = if let Err(crc_err) = crc_res {
> +            if ignore_corrupt_chunks {
> +                create_zero_chunk(format!("is corrupt - {crc_err}"))?
> +            } else {
> +                bail!("Error at chunk {:?} - {crc_err}", chunk_path);
> +            }
> +        } else {
> +            (chunk_blob, chunk_digest)
> +        };
> +
> +        // third chance - decoding might fail (digest, compression, encryption)
> +        let decoded = chunk_blob
> +            .decode(crypt_conf_opt.as_ref(), chunk_digest)
> +            .or_else(|err| {
> +                if ignore_corrupt_chunks {
> +                    create_zero_chunk(format!("fails to decode - {err}"))?
> +                        .0
> +                        .decode(crypt_conf_opt.as_ref(), None)
> +                } else {
> +                    bail!("Failed to decode chunk {:?} = {}", chunk_path, err);
> +                }
> +            })?;
>   
> -        output_file.write_all(
> -            chunk_blob
> -                .decode(crypt_conf_opt.as_ref(), Some(chunk_digest))?
> -                .as_slice(),
> -        )?;
> +        output_file.write_all(decoded.as_slice())?;
>       }
>   
>       Ok(())

[pbs-devel] applied-series: [PATCH proxmox-backup 1/3] debug: recover: allow ignoring missing/corrupt chunks

[Thomas Lamprecht]

On 23/05/2022 16:11, Fabian Grünbichler wrote:
> replacing them with chunks of zero bytes.
> 
> Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>
> ---
>  src/bin/proxmox_backup_debug/recover.rs | 98 +++++++++++++++++++++----
>  1 file changed, 84 insertions(+), 14 deletions(-)
> 
>

applied series with Hannes T-b tag, thanks!