From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 865056074D for ; Thu, 15 Oct 2020 09:08:07 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 75A23F56C for ; Thu, 15 Oct 2020 09:07:37 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 9AA5FF562 for ; Thu, 15 Oct 2020 09:07:36 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 5F9D045D7D for ; Thu, 15 Oct 2020 09:07:36 +0200 (CEST) Date: Thu, 15 Oct 2020 09:07:29 +0200 From: Fabian =?iso-8859-1?q?Gr=FCnbichler?= To: Proxmox Backup Server development discussion References: <20201014150416.5612-1-s.reiter@proxmox.com> In-Reply-To: <20201014150416.5612-1-s.reiter@proxmox.com> MIME-Version: 1.0 User-Agent: astroid/0.15.0 (https://github.com/astroidmail/astroid) Message-Id: <1602745607.jr8ncdt2wm.astroid@nora.none> Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable X-SPAM-LEVEL: Spam detection results: 0 AWL 0.033 Adjusted score from AWL reputation of From: address KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [proxmox.com, datastore.rs] Subject: Re: [pbs-devel] [RFC proxmox-backup] api: datastore: determine size and blob type if not in manifest X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Oct 2020 07:08:07 -0000 On October 14, 2020 5:04 pm, Stefan Reiter wrote: > Try to read the first 8 byte (DataBlobHeader magic) from any .blob file > not mentioned in the manifest. This allows classifying the > client.log.blob file as encrypted if it is, and showing the correct > symbol in the GUI for it. >=20 > While already open, also determine the file size by seeking to End(0). >=20 > Signed-off-by: Stefan Reiter > --- >=20 > I noticed that the GUI would show the "download" icon enabled for > 'client.log.blob' files, even if they were encrypted. The button wouldn't= work > of course, since the server can't decode encrypted blobs, so nothing woul= d > happen except an error logged to the console. >=20 > The problem is that currently we don't know if a blob file not listed in = the > manifest is encrypted - but we can find out, since it's encoded into the = header. > Not sure if this is worth the open() and read() for every unknown blob fi= le, but > it would solve the problem... couldn't we just store that information (once) when uploading the client lo= g? >=20 >=20 > src/api2/admin/datastore.rs | 25 +++++++++++++++++++++++-- > src/backup/data_blob.rs | 12 ++++++++++++ > 2 files changed, 35 insertions(+), 2 deletions(-) >=20 > diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs > index 11223e6a..2a9ac469 100644 > --- a/src/api2/admin/datastore.rs > +++ b/src/api2/admin/datastore.rs > @@ -2,6 +2,7 @@ use std::collections::{HashSet, HashMap}; > use std::ffi::OsStr; > use std::os::unix::ffi::OsStrExt; > use std::sync::{Arc, Mutex}; > +use std::io::{Read, Seek, SeekFrom}; > =20 > use anyhow::{bail, format_err, Error}; > use futures::*; > @@ -91,10 +92,30 @@ fn get_all_snapshot_files( > =20 > for file in &info.files { > if file_set.contains(file) { continue; } > + > + // Try to determine size and crypt mode for unknown blobs > + let mut crypt_mode =3D None; > + let mut size =3D None; > + if file.ends_with(".blob") { > + let mut path =3D store.snapshot_path(&info.backup_dir); > + path.push(file); > + if let Ok(mut file) =3D std::fs::File::open(path) { > + let mut buffer =3D [0u8; 8]; > + if let Ok(n) =3D file.read(&mut buffer[..]) { > + if n =3D=3D buffer.len() { > + crypt_mode =3D DataBlob::is_blob_magic(&buffer); > + } > + } > + if let Ok(pos) =3D file.seek(SeekFrom::End(0)) { > + size =3D Some(pos); > + } > + } > + } > + > files.push(BackupContent { > filename: file.to_string(), > - size: None, > - crypt_mode: None, > + size, > + crypt_mode, > }); > } > =20 > diff --git a/src/backup/data_blob.rs b/src/backup/data_blob.rs > index 284dc243..626a5fe4 100644 > --- a/src/backup/data_blob.rs > +++ b/src/backup/data_blob.rs > @@ -321,6 +321,18 @@ impl DataBlob { > =20 > Ok(()) > } > + > + /// Determine if the given value is a valid blob magic number. > + /// Returns CryptMode::Encrypt or CryptMode::None depending on type. > + pub fn is_blob_magic(magic: &[u8; 8]) -> Option { > + if magic =3D=3D &UNCOMPRESSED_BLOB_MAGIC_1_0 || magic =3D=3D &CO= MPRESSED_BLOB_MAGIC_1_0 { > + Some(CryptMode::None) > + } else if magic =3D=3D &ENCRYPTED_BLOB_MAGIC_1_0 || magic =3D=3D= &ENCR_COMPR_BLOB_MAGIC_1_0 { > + Some(CryptMode::Encrypt) > + } else { > + None > + } > + } > } > =20 > /// Builder for chunk DataBlobs > --=20 > 2.20.1 >=20 >=20 >=20 > _______________________________________________ > pbs-devel mailing list > pbs-devel@lists.proxmox.com > https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel >=20 >=20 >=20 =