From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id DCB981FF13A for ; Wed, 01 Apr 2026 09:55:44 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 32EDB10CEC; Wed, 1 Apr 2026 09:56:10 +0200 (CEST) From: Christian Ebner To: pbs-devel@lists.proxmox.com Subject: [PATCH proxmox-backup 20/20] sync: pull: decrypt snapshots with matching encryption key fingerprint Date: Wed, 1 Apr 2026 09:55:21 +0200 Message-ID: <20260401075521.176354-21-c.ebner@proxmox.com> X-Mailer: git-send-email 2.47.3 In-Reply-To: <20260401075521.176354-1-c.ebner@proxmox.com> References: <20260401075521.176354-1-c.ebner@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Bm-Milter-Handled: 55990f41-d878-4baa-be0a-ee34c49e34d2 X-Bm-Transport-Timestamp: 1775030091692 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.064 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 446RK2CE2APWSFOTAG3MOMUD6T2MPA3G X-Message-ID-Hash: 446RK2CE2APWSFOTAG3MOMUD6T2MPA3G X-MailFrom: c.ebner@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: Decrypt any backup snapshot during pull which was encrypted with a matching encryption key. Matching of keys is performed by comparing the fingerprint of the key as stored in the source manifest and the key configured for the pull sync jobs. If matching, pass along the key's crypto config to the index and chunk readers and write the local files unencrypted instead of simply downloading them. A new manifest file is written instead of the original one and files registered accordingly. Signed-off-by: Christian Ebner --- src/server/pull.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/src/server/pull.rs b/src/server/pull.rs index 05152d0dd..22b058056 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -3,6 +3,7 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::io::{BufReader, Read, Seek, Write}; +use std::os::fd::AsRawFd; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::SystemTime; @@ -10,11 +11,14 @@ use std::time::SystemTime; use anyhow::{bail, format_err, Context, Error}; use pbs_tools::crypt_config::CryptConfig; use proxmox_human_byte::HumanByte; +use serde_json::Value; +use tokio::fs::OpenOptions; +use tokio::io::AsyncWriteExt; use tracing::{info, warn}; use pbs_api_types::{ print_store_and_ns, ArchiveType, Authid, BackupArchiveName, BackupDir, BackupGroup, - BackupNamespace, GroupFilter, Operation, RateLimitConfig, Remote, SnapshotListItem, + BackupNamespace, CryptMode, GroupFilter, Operation, RateLimitConfig, Remote, SnapshotListItem, VerifyState, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, MAX_NAMESPACE_DEPTH, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP, }; @@ -397,6 +401,7 @@ async fn pull_single_archive<'a>( encountered_chunks: Arc>, crypt_config: Option>, backend: &DatastoreBackend, + new_manifest: Option>>, ) -> Result { let archive_name = &archive_info.filename; let mut path = snapshot.full_path(); @@ -446,6 +451,17 @@ async fn pull_single_archive<'a>( // Overwrite current tmp file so it will be persisted instead std::fs::rename(&path, &tmp_path)?; + + if let Some(new_manifest) = new_manifest { + let name = archive_name.as_str().try_into()?; + // size is indetical to original, encrypted index + new_manifest.lock().unwrap().add_file( + &name, + size, + csum, + CryptMode::None, + )?; + } } sync_stats.add(stats); @@ -484,6 +500,17 @@ async fn pull_single_archive<'a>( // Overwrite current tmp file so it will be persisted instead std::fs::rename(&path, &tmp_path)?; + + if let Some(new_manifest) = new_manifest { + let name = archive_name.as_str().try_into()?; + // size is indetical to original, encrypted index + new_manifest.lock().unwrap().add_file( + &name, + size, + csum, + CryptMode::None, + )?; + } } sync_stats.add(stats); @@ -522,6 +549,14 @@ async fn pull_single_archive<'a>( decrypted_tmpfile.rewind()?; let (csum, size) = sha256(&mut decrypted_tmpfile)?; + if let Some(new_manifest) = new_manifest { + let mut new_manifest = new_manifest.lock().unwrap(); + let name = archive_name.as_str().try_into()?; + new_manifest.add_file(&name, size, csum, CryptMode::None)?; + } + + nix::unistd::fsync(decrypted_tmpfile.as_raw_fd())?; + std::fs::rename(&decrypted_tmp_path, &tmp_path)?; Ok::<(), Error>(()) @@ -607,9 +642,11 @@ async fn pull_snapshot<'a>( let _ = std::fs::remove_file(&tmp_manifest_name); return Ok(sync_stats); // nothing changed } + // redownload also in case of encrypted, even if key would match as cannot + // fully verify otherwise due to file checksum mismatches. } - let manifest_data = tmp_manifest_blob.raw_data().to_vec(); + let mut manifest_data = tmp_manifest_blob.raw_data().to_vec(); let manifest = BackupManifest::try_from(tmp_manifest_blob)?; if ignore_not_verified_or_encrypted( @@ -629,6 +666,16 @@ async fn pull_snapshot<'a>( } let mut crypt_config = None; + let mut new_manifest = None; + if let Ok(Some(source_fingerprint)) = manifest.fingerprint() { + if let Some(config) = ¶ms.crypt_config { + if config.fingerprint() == *source_fingerprint.bytes() { + crypt_config = Some(Arc::clone(config)); + new_manifest = Some(Arc::new(Mutex::new(BackupManifest::new(snapshot.into())))); + info!("Found matching key fingerprint {source_fingerprint}, decrypt on pull"); + } + } + } let backend = ¶ms.target.backend; for item in manifest.files() { @@ -678,11 +725,38 @@ async fn pull_snapshot<'a>( encountered_chunks.clone(), crypt_config.clone(), backend, + new_manifest.clone(), ) .await?; sync_stats.add(stats); } + if let Some(new_manifest) = new_manifest { + let mut new_manifest = Arc::try_unwrap(new_manifest) + .map_err(|_arc| { + format_err!("failed to take ownership of still referenced new manifest") + })? + .into_inner() + .unwrap(); + + // copy over notes ecc, but drop encryption key fingerprint + new_manifest.unprotected = manifest.unprotected.clone(); + new_manifest.unprotected["key-fingerprint"] = Value::Null; + + let manifest_string = new_manifest.to_string(None)?; + let manifest_blob = DataBlob::encode(manifest_string.as_bytes(), None, true)?; + // update contents to be uploaded to backend + manifest_data = manifest_blob.raw_data().to_vec(); + + let mut tmp_manifest_file = OpenOptions::new() + .write(true) + .truncate(true) // clear pre-existing manifest content + .open(&tmp_manifest_name) + .await?; + tmp_manifest_file.write_all(&manifest_data).await?; + tmp_manifest_file.flush().await?; + } + if let Err(err) = std::fs::rename(&tmp_manifest_name, &manifest_name) { bail!("Atomic rename file {:?} failed - {}", manifest_name, err); } -- 2.47.3