From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 1C35D7866F for ; Fri, 30 Apr 2021 11:44:36 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 0D005266DE for ; Fri, 30 Apr 2021 11:44:06 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 46D25266D3 for ; Fri, 30 Apr 2021 11:44:05 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 19A724298A for ; Fri, 30 Apr 2021 11:44:05 +0200 (CEST) Date: Fri, 30 Apr 2021 11:44:03 +0200 From: Wolfgang Bumiller To: Hannes Laimer Cc: pbs-devel@lists.proxmox.com Message-ID: <20210430094403.mmmlwfov2erjqhyu@wobu-vie.proxmox.com> References: <20210429110016.1467670-1-h.laimer@proxmox.com> <20210429110016.1467670-4-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20210429110016.1467670-4-h.laimer@proxmox.com> User-Agent: NeoMutt/20180716 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.021 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [inspect.rs] Subject: Re: [pbs-devel] [PATCH v5 proxmox-backup 3/4] add file inspection to pb-debug X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 30 Apr 2021 09:44:36 -0000 On Thu, Apr 29, 2021 at 01:00:15PM +0200, Hannes Laimer wrote: > Adds possibility to inspect .blob, .fidx and .didx files. For index > files a list of the chunks referenced will be printed in addition to > some other inforation. .blob files can be decoded into file or directly > into stdout. Options: > - file: path to the file > - [opt] decode: path to a file or stdout(-), if specidied, the file will be > decoded into the specified location [only for blob files, no effect > with index files] > - [opt] keyfile: path to a keyfile, needed if decode is specified and the > data was encrypted > > Signed-off-by: Hannes Laimer > --- > v5: > - combine path for fixed and dynamic index fliles to avoid duplicate > code > > v4: > - only the types of file that are passed by the user are check with the > magic number, when looking for index files just the filename ending > is checked -> don't have to open the file for that > - not sure if a function for the magic nr reading, seek reset makes > sense(?), it's just two lines > > src/bin/proxmox_backup_debug/inspect.rs | 119 ++++++++++++++++++++++-- > 1 file changed, 112 insertions(+), 7 deletions(-) > > diff --git a/src/bin/proxmox_backup_debug/inspect.rs b/src/bin/proxmox_backup_debug/inspect.rs > index dd6ee287..2d95448e 100644 > --- a/src/bin/proxmox_backup_debug/inspect.rs > +++ b/src/bin/proxmox_backup_debug/inspect.rs > @@ -1,19 +1,26 @@ > +use std::collections::HashSet; > +use std::fs::File; > +use std::io::{Read, Seek, SeekFrom}; > use std::path::Path; > > use anyhow::{format_err, Error}; > -use proxmox::api::cli::{ > - format_and_print_result, get_output_format, CliCommand, CliCommandMap, CommandLineInterface, > -}; > -use proxmox::api::{api, cli::*}; > use serde_json::{json, Value}; > use walkdir::WalkDir; > > +use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA}; > +use proxmox::api::{ > + api, > + cli::{ > + format_and_print_result, get_output_format, CliCommand, CliCommandMap, > + CommandLineInterface, OUTPUT_FORMAT, > + }, > +}; > use proxmox_backup::backup::{ > load_and_decrypt_key, CryptConfig, DataBlob, DynamicIndexReader, FixedIndexReader, IndexFile, > + COMPRESSED_BLOB_MAGIC_1_0, DYNAMIC_SIZED_CHUNK_INDEX_1_0, ENCRYPTED_BLOB_MAGIC_1_0, > + ENCR_COMPR_BLOB_MAGIC_1_0, FIXED_SIZED_CHUNK_INDEX_1_0, UNCOMPRESSED_BLOB_MAGIC_1_0, > }; > > -use crate::{get_encryption_key_password, KEYFILE_SCHEMA, PATH_SCHEMA}; > - > use proxmox_backup::tools::outfile_or_stdout; > > /// Decodes a blob and writes its content either to stdout or into a file > @@ -37,6 +44,102 @@ fn decode_blob( > Ok(()) > } > > +#[api( > + input: { > + properties: { > + file: { > + schema: PATH_SCHEMA, > + }, > + "decode": { > + schema: PATH_SCHEMA, > + optional: true, > + }, > + "keyfile": { > + schema: KEYFILE_SCHEMA, > + optional: true, > + }, > + "output-format": { > + schema: OUTPUT_FORMAT, > + optional: true, > + }, > + } > + } > +)] > +/// Inspect a file > +fn inspect_file( > + file: String, > + decode: Option, > + keyfile: Option, > + param: Value, > +) -> Result<(), Error> { > + let output_format = get_output_format(¶m); > + > + let mut file = File::open(Path::new(&file))?; > + let mut magic = [0; 8]; > + file.read_exact(&mut magic)?; > + file.seek(SeekFrom::Start(0))?; > + let val = match magic { > + UNCOMPRESSED_BLOB_MAGIC_1_0 > + | COMPRESSED_BLOB_MAGIC_1_0 > + | ENCRYPTED_BLOB_MAGIC_1_0 > + | ENCR_COMPR_BLOB_MAGIC_1_0 => { > + let data_blob = DataBlob::load_from_reader(&mut file)?; > + let key_file_path = keyfile.as_ref().map(Path::new); > + > + let (decode_output_path, to_stdout) = ( Same reasoning about `to_stdout` here as in the other patch. I think it can just be dropped. > + decode.as_ref().map(Path::new), > + decode.clone().map_or(false, |p| p.eq("-")), > + ); > + > + if decode_output_path.is_some() || to_stdout { > + decode_blob(decode_output_path, key_file_path, None, &data_blob)?; > + } > + > + let crypt_mode = data_blob.crypt_mode()?; > + Ok(json!({ Please remove the `Ok` wrapping here. > + "encryption": crypt_mode, > + "raw_size": data_blob.raw_size(), > + })) > + } > + FIXED_SIZED_CHUNK_INDEX_1_0 | DYNAMIC_SIZED_CHUNK_INDEX_1_0 => { > + let index: Box = match magic { > + FIXED_SIZED_CHUNK_INDEX_1_0 => { > + Ok(Box::new(FixedIndexReader::new(file)?) as Box) > + } > + DYNAMIC_SIZED_CHUNK_INDEX_1_0 => { > + Ok(Box::new(DynamicIndexReader::new(file)?) as Box) > + } > + _ => Err(format_err!("This is technically not possible")), > + }?; > + > + let mut ctime_str = index.index_ctime().to_string(); > + if let Ok(s) = proxmox::tools::time::strftime_local("%c", index.index_ctime()) { > + ctime_str = s; > + } > + > + let mut chunk_digests = HashSet::new(); > + > + for pos in 0..index.index_count() { > + let digest = index.index_digest(pos).unwrap(); > + chunk_digests.insert(proxmox::tools::digest_to_hex(digest)); > + } > + > + Ok(json!({ and here > + "size": index.index_size(), > + "ctime": ctime_str, > + "chunk-digests": chunk_digests > + })) > + } > + _ => Err(format_err!( and use `bail!` instead of `Err(format_err!(` followed by a `?` below. > + "Only .blob, .fidx and .didx files may be inspected" > + )), > + }?; ^ And just drop the `?` here. > + > + format_and_print_result(&val, &output_format); > + > + Ok(()) > +} > + > #[api( > input: { > properties: { > @@ -156,7 +259,9 @@ fn inspect_chunk( > } > > pub fn inspect_commands() -> CommandLineInterface { > - let cmd_def = CliCommandMap::new().insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK)); > + let cmd_def = CliCommandMap::new() > + .insert("file", CliCommand::new(&API_METHOD_INSPECT_FILE)) Here too, I think we should add: .arg_param(["file"]) > + .insert("chunk", CliCommand::new(&API_METHOD_INSPECT_CHUNK)); > > cmd_def.into() > } > -- > 2.20.1