From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 308CF605BA for ; Thu, 15 Oct 2020 06:25:23 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 11505E891 for ; Thu, 15 Oct 2020 06:25:23 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 5622EE881 for ; Thu, 15 Oct 2020 06:25:21 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 2692545D81 for ; Thu, 15 Oct 2020 06:25:21 +0200 (CEST) Date: Thu, 15 Oct 2020 06:25:04 +0200 (CEST) From: Dietmar Maurer To: Proxmox Backup Server development discussion , Stefan Reiter Message-ID: <291450438.94.1602735905280@webmail.proxmox.com> In-Reply-To: <20201014150416.5612-1-s.reiter@proxmox.com> References: <20201014150416.5612-1-s.reiter@proxmox.com> MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 7bit X-Priority: 3 Importance: Normal X-Mailer: Open-Xchange Mailer v7.10.4-Rev11 X-Originating-Client: open-xchange-appsuite X-SPAM-LEVEL: Spam detection results: 0 AWL 0.069 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com, datastore.rs] Subject: Re: [pbs-devel] [RFC proxmox-backup] api: datastore: determine size and blob type if not in manifest X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Oct 2020 04:25:23 -0000 This is a critical path, and I do not really want an additional delay here. We need a better solution for this. > On 10/14/2020 5:04 PM Stefan Reiter wrote: > > > Try to read the first 8 byte (DataBlobHeader magic) from any .blob file > not mentioned in the manifest. This allows classifying the > client.log.blob file as encrypted if it is, and showing the correct > symbol in the GUI for it. > > While already open, also determine the file size by seeking to End(0). > > Signed-off-by: Stefan Reiter > --- > > I noticed that the GUI would show the "download" icon enabled for > 'client.log.blob' files, even if they were encrypted. The button wouldn't work > of course, since the server can't decode encrypted blobs, so nothing would > happen except an error logged to the console. > > The problem is that currently we don't know if a blob file not listed in the > manifest is encrypted - but we can find out, since it's encoded into the header. > Not sure if this is worth the open() and read() for every unknown blob file, but > it would solve the problem... > > > src/api2/admin/datastore.rs | 25 +++++++++++++++++++++++-- > src/backup/data_blob.rs | 12 ++++++++++++ > 2 files changed, 35 insertions(+), 2 deletions(-) > > diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs > index 11223e6a..2a9ac469 100644 > --- a/src/api2/admin/datastore.rs > +++ b/src/api2/admin/datastore.rs > @@ -2,6 +2,7 @@ use std::collections::{HashSet, HashMap}; > use std::ffi::OsStr; > use std::os::unix::ffi::OsStrExt; > use std::sync::{Arc, Mutex}; > +use std::io::{Read, Seek, SeekFrom}; > > use anyhow::{bail, format_err, Error}; > use futures::*; > @@ -91,10 +92,30 @@ fn get_all_snapshot_files( > > for file in &info.files { > if file_set.contains(file) { continue; } > + > + // Try to determine size and crypt mode for unknown blobs > + let mut crypt_mode = None; > + let mut size = None; > + if file.ends_with(".blob") { > + let mut path = store.snapshot_path(&info.backup_dir); > + path.push(file); > + if let Ok(mut file) = std::fs::File::open(path) { > + let mut buffer = [0u8; 8]; > + if let Ok(n) = file.read(&mut buffer[..]) { > + if n == buffer.len() { > + crypt_mode = DataBlob::is_blob_magic(&buffer); > + } > + } > + if let Ok(pos) = file.seek(SeekFrom::End(0)) { > + size = Some(pos); > + } > + } > + } > + > files.push(BackupContent { > filename: file.to_string(), > - size: None, > - crypt_mode: None, > + size, > + crypt_mode, > }); > } > > diff --git a/src/backup/data_blob.rs b/src/backup/data_blob.rs > index 284dc243..626a5fe4 100644 > --- a/src/backup/data_blob.rs > +++ b/src/backup/data_blob.rs > @@ -321,6 +321,18 @@ impl DataBlob { > > Ok(()) > } > + > + /// Determine if the given value is a valid blob magic number. > + /// Returns CryptMode::Encrypt or CryptMode::None depending on type. > + pub fn is_blob_magic(magic: &[u8; 8]) -> Option { > + if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 || magic == &COMPRESSED_BLOB_MAGIC_1_0 { > + Some(CryptMode::None) > + } else if magic == &ENCRYPTED_BLOB_MAGIC_1_0 || magic == &ENCR_COMPR_BLOB_MAGIC_1_0 { > + Some(CryptMode::Encrypt) > + } else { > + None > + } > + } > } > > /// Builder for chunk DataBlobs > -- > 2.20.1 > > > > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel