From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 47DBD7239A for ; Mon, 23 May 2022 16:12:13 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 307AEC55A for ; Mon, 23 May 2022 16:11:43 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 2ECA4C551 for ; Mon, 23 May 2022 16:11:41 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id A5ACD43801 for ; Mon, 23 May 2022 16:11:40 +0200 (CEST) From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= To: pbs-devel@lists.proxmox.com Date: Mon, 23 May 2022 16:11:33 +0200 Message-Id: <20220523141135.921321-1-f.gruenbichler@proxmox.com> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.172 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record T_SCC_BODY_TEXT_LINE -0.01 - Subject: [pbs-devel] [PATCH proxmox-backup 1/3] debug: recover: allow ignoring missing/corrupt chunks X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 23 May 2022 14:12:13 -0000 replacing them with chunks of zero bytes. Signed-off-by: Fabian Grünbichler --- src/bin/proxmox_backup_debug/recover.rs | 98 +++++++++++++++++++++---- 1 file changed, 84 insertions(+), 14 deletions(-) diff --git a/src/bin/proxmox_backup_debug/recover.rs b/src/bin/proxmox_backup_debug/recover.rs index b118a71c..9c4aed3d 100644 --- a/src/bin/proxmox_backup_debug/recover.rs +++ b/src/bin/proxmox_backup_debug/recover.rs @@ -38,7 +38,19 @@ use pbs_tools::crypt_config::CryptConfig; type: Boolean, optional: true, default: false, - } + }, + "ignore-missing-chunks": { + description: "If a chunk is missing, warn and write 0-bytes instead to attempt partial recovery.", + type: Boolean, + optional: true, + default: false, + }, + "ignore-corrupt-chunks": { + description: "If a chunk is corrupt, warn and write 0-bytes instead to attempt partial recovery.", + type: Boolean, + optional: true, + default: false, + }, } } )] @@ -49,6 +61,8 @@ fn recover_index( chunks: String, keyfile: Option, skip_crc: bool, + ignore_missing_chunks: bool, + ignore_corrupt_chunks: bool, _param: Value, ) -> Result<(), Error> { let file_path = Path::new(&file); @@ -89,22 +103,78 @@ fn recover_index( let digest_str = hex::encode(chunk_digest); let digest_prefix = &digest_str[0..4]; let chunk_path = chunks_path.join(digest_prefix).join(digest_str); - let mut chunk_file = std::fs::File::open(&chunk_path) - .map_err(|e| format_err!("could not open chunk file - {}", e))?; - data.clear(); - chunk_file.read_to_end(&mut data)?; - let chunk_blob = DataBlob::from_raw(data.clone())?; + let create_zero_chunk = |msg: String| -> Result<(DataBlob, Option<&[u8; 32]>), Error> { + let info = index + .chunk_info(pos) + .ok_or_else(|| format_err!("Couldn't read chunk info from index at {pos}"))?; + let size = info.size(); - if !skip_crc { - chunk_blob.verify_crc()?; - } + eprintln!("WARN: chunk {:?} {}", chunk_path, msg); + eprintln!("WARN: replacing output file {:?} with '\\0'", info.range,); + + Ok(( + DataBlob::encode(&vec![0; size as usize], crypt_conf_opt.as_ref(), true)?, + None, + )) + }; + + let (chunk_blob, chunk_digest) = match std::fs::File::open(&chunk_path) { + Ok(mut chunk_file) => { + data.clear(); + chunk_file.read_to_end(&mut data)?; + + // first chance for corrupt chunk - handling magic fails + DataBlob::from_raw(data.clone()) + .map(|blob| (blob, Some(chunk_digest))) + .or_else(|err| { + if ignore_corrupt_chunks { + create_zero_chunk(format!("is corrupt - {err}")) + } else { + bail!("{err}"); + } + })? + } + Err(err) => { + if ignore_missing_chunks && err.kind() == std::io::ErrorKind::NotFound { + create_zero_chunk(format!("is missing"))? + } else { + bail!("could not open chunk file - {}", err); + } + } + }; + + // second chance - we need CRC to detect truncated chunks! + let crc_res = if skip_crc { + Ok(()) + } else { + chunk_blob.verify_crc() + }; + + let (chunk_blob, chunk_digest) = if let Err(crc_err) = crc_res { + if ignore_corrupt_chunks { + create_zero_chunk(format!("is corrupt - {crc_err}"))? + } else { + bail!("Error at chunk {:?} - {crc_err}", chunk_path); + } + } else { + (chunk_blob, chunk_digest) + }; + + // third chance - decoding might fail (digest, compression, encryption) + let decoded = chunk_blob + .decode(crypt_conf_opt.as_ref(), chunk_digest) + .or_else(|err| { + if ignore_corrupt_chunks { + create_zero_chunk(format!("fails to decode - {err}"))? + .0 + .decode(crypt_conf_opt.as_ref(), None) + } else { + bail!("Failed to decode chunk {:?} = {}", chunk_path, err); + } + })?; - output_file.write_all( - chunk_blob - .decode(crypt_conf_opt.as_ref(), Some(chunk_digest))? - .as_slice(), - )?; + output_file.write_all(decoded.as_slice())?; } Ok(()) -- 2.30.2