From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [IPv6:2a01:7e0:0:424::9]) by lore.proxmox.com (Postfix) with ESMTPS id 90D4A1FF165 for ; Wed, 15 Jan 2025 15:16:46 +0100 (CET) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 1A69BDEE2; Wed, 15 Jan 2025 15:16:46 +0100 (CET) From: Christian Ebner To: pbs-devel@lists.proxmox.com Date: Wed, 15 Jan 2025 15:15:45 +0100 Message-Id: <20250115141547.376149-4-c.ebner@proxmox.com> X-Mailer: git-send-email 2.39.5 In-Reply-To: <20250115141547.376149-1-c.ebner@proxmox.com> References: <20250115141547.376149-1-c.ebner@proxmox.com> MIME-Version: 1.0 X-SPAM-LEVEL: Spam detection results: 0 AWL 0.030 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup 3/5] fix #6072: server: sync encrypted or verified snapshots only X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: Proxmox Backup Server development discussion Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Errors-To: pbs-devel-bounces@lists.proxmox.com Sender: "pbs-devel" Skip over snapshots which have not been verified or encrypted if the sync jobs has set the flags accordingly. A snapshot is considered as encrypted, if all the archives in the manifest have `CryptMode::Encrypt`. A snapshot is considered as verified, when the manifest's verify state is set to `VerifyState::Ok`. This allows to only synchronize a subset of the snapshots, which are known to be fine (verified) or which are known to be encrypted. The latter is of most interest for sync jobs in push direction to untrusted or less trusted remotes, where it might be desired to not expose unencrypted contents. Link to the bugtracker issue: https://bugzilla.proxmox.com/show_bug.cgi?id=6072 Signed-off-by: Christian Ebner --- src/server/pull.rs | 65 +++++++++++++++++++++++++++++++++++++++++++--- src/server/push.rs | 38 ++++++++++++++++++++++++--- 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/src/server/pull.rs b/src/server/pull.rs index 616d45eb9..c223e1b93 100644 --- a/src/server/pull.rs +++ b/src/server/pull.rs @@ -12,7 +12,7 @@ use tracing::info; use pbs_api_types::{ print_store_and_ns, ArchiveType, Authid, BackupArchiveName, BackupDir, BackupGroup, - BackupNamespace, GroupFilter, Operation, RateLimitConfig, Remote, VerifyState, + BackupNamespace, CryptMode, GroupFilter, Operation, RateLimitConfig, Remote, VerifyState, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, MAX_NAMESPACE_DEPTH, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP, }; @@ -344,6 +344,7 @@ async fn pull_single_archive<'a>( /// -- if not, pull it from the remote /// - Download log if not already existing async fn pull_snapshot<'a>( + params: &PullParameters, reader: Arc, snapshot: &'a pbs_datastore::BackupDir, downloaded_chunks: Arc>>, @@ -402,6 +403,55 @@ async fn pull_snapshot<'a>( let manifest = BackupManifest::try_from(tmp_manifest_blob)?; + if params.verified_only { + let mut snapshot_verified = false; + if let Ok(Some(verify_state)) = manifest.verify_state() { + if let VerifyState::Ok = verify_state.state { + snapshot_verified = true; + } + } + + if !snapshot_verified { + info!( + "Snapshot {} not verified but verified-only set, snapshot skipped", + snapshot.dir(), + ); + if is_new { + let path = snapshot.full_path(); + // safe to remove as locked by caller + std::fs::remove_dir_all(&path).map_err(|err| { + format_err!("removing temporary backup snapshot {path:?} failed - {err}") + })?; + } + return Ok(sync_stats); + } + } + + if params.encrypted_only { + let mut snapshot_encrypted = true; + // Consider only encrypted if all files in the manifest are marked as encrypted + for file in manifest.files() { + if file.chunk_crypt_mode() != CryptMode::Encrypt { + snapshot_encrypted = false; + } + } + + if !snapshot_encrypted { + info!( + "Snapshot {} not encrypted but encrypted-only set, snapshot skipped", + snapshot.dir(), + ); + if is_new { + let path = snapshot.full_path(); + // safe to remove as locked by caller + std::fs::remove_dir_all(&path).map_err(|err| { + format_err!("removing temporary backup snapshot {path:?} failed - {err}") + })?; + } + return Ok(sync_stats); + } + } + for item in manifest.files() { let mut path = snapshot.full_path(); path.push(&item.filename); @@ -466,6 +516,7 @@ async fn pull_snapshot<'a>( /// The `reader` is configured to read from the source backup directory, while the /// `snapshot` is pointing to the local datastore and target namespace. async fn pull_snapshot_from<'a>( + params: &PullParameters, reader: Arc, snapshot: &'a pbs_datastore::BackupDir, downloaded_chunks: Arc>>, @@ -475,7 +526,7 @@ async fn pull_snapshot_from<'a>( .datastore() .create_locked_backup_dir(snapshot.backup_ns(), snapshot.as_ref())?; - let result = pull_snapshot(reader, snapshot, downloaded_chunks, corrupt, is_new).await; + let result = pull_snapshot(params, reader, snapshot, downloaded_chunks, corrupt, is_new).await; if is_new { // Cleanup directory on error if snapshot was not present before @@ -621,8 +672,14 @@ async fn pull_group( .source .reader(source_namespace, &from_snapshot) .await?; - let result = - pull_snapshot_from(reader, &to_snapshot, downloaded_chunks.clone(), corrupt).await; + let result = pull_snapshot_from( + params, + reader, + &to_snapshot, + downloaded_chunks.clone(), + corrupt, + ) + .await; progress.done_snapshots = pos as u64 + 1; info!("percentage done: {progress}"); diff --git a/src/server/push.rs b/src/server/push.rs index 8ac6fbf37..ed9726dc9 100644 --- a/src/server/push.rs +++ b/src/server/push.rs @@ -11,10 +11,11 @@ use tracing::{info, warn}; use pbs_api_types::{ print_store_and_ns, ApiVersion, ApiVersionInfo, ArchiveType, Authid, BackupArchiveName, - BackupDir, BackupGroup, BackupGroupDeleteStats, BackupNamespace, GroupFilter, GroupListItem, - NamespaceListItem, Operation, RateLimitConfig, Remote, SnapshotListItem, CLIENT_LOG_BLOB_NAME, - MANIFEST_BLOB_NAME, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_READ, PRIV_REMOTE_DATASTORE_BACKUP, - PRIV_REMOTE_DATASTORE_MODIFY, PRIV_REMOTE_DATASTORE_PRUNE, + BackupDir, BackupGroup, BackupGroupDeleteStats, BackupNamespace, CryptMode, GroupFilter, + GroupListItem, NamespaceListItem, Operation, RateLimitConfig, Remote, SnapshotListItem, + VerifyState, CLIENT_LOG_BLOB_NAME, MANIFEST_BLOB_NAME, PRIV_DATASTORE_BACKUP, + PRIV_DATASTORE_READ, PRIV_REMOTE_DATASTORE_BACKUP, PRIV_REMOTE_DATASTORE_MODIFY, + PRIV_REMOTE_DATASTORE_PRUNE, }; use pbs_client::{BackupRepository, BackupWriter, HttpClient, MergedChunkInfo, UploadOptions}; use pbs_config::CachedUserInfo; @@ -810,6 +811,35 @@ pub(crate) async fn push_snapshot( } }; + if params.verified_only { + let mut snapshot_verified = false; + if let Ok(Some(verify_state)) = source_manifest.verify_state() { + if let VerifyState::Ok = verify_state.state { + snapshot_verified = true; + } + } + + if !snapshot_verified { + info!("Snapshot {snapshot} not verified but verified-only set, snapshot skipped"); + return Ok(stats); + } + } + + if params.encrypted_only { + let mut snapshot_encrypted = true; + // Consider only encrypted if all files in the manifest are marked as encrypted + for file in source_manifest.files() { + if file.chunk_crypt_mode() != CryptMode::Encrypt { + snapshot_encrypted = false; + } + } + + if !snapshot_encrypted { + info!("Snapshot {snapshot} not encrypted but encrypted-only set, snapshot skipped"); + return Ok(stats); + } + } + // Writer instance locks the snapshot on the remote side let backup_writer = BackupWriter::start( ¶ms.target.client, -- 2.39.5 _______________________________________________ pbs-devel mailing list pbs-devel@lists.proxmox.com https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel