From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [45.144.208.40]) by lore.proxmox.com (Postfix) with ESMTPS id EE75A1FF0B7 for ; Fri, 13 Nov 2026 10:06:13 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 3EEA92178D; Fri, 13 Nov 2026 10:06:06 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=stuttgart.de; s=s1022; t=1794560285; bh=Vm2vIcE7jNhmvxy+INw/jorTlfMzDt498uxME/5rnQg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=REaiuwQXHctoRRzX5qhNK35UnA7Db1a1lGACa9SnXy3CfMyAjgKf0IXOBca211Uqo OFHDvKf+qIq/WVJCinvzR5kwZUdXK0gBOQC86nihUcZP/Uiym42YEIMB8nuEWHp2Rr gi9WasL0JhpzyCXpWD4n7xZIHzO/2GguADkgnSClwglkccP4i/4SGAOCO3o9tHZOyk +Djj2lw/yl80T3EXISY5g25pSXAqT5Jf1sUsqfoce+yz2++dQZpq/YjF5zQSD/vP0v 3ugy6EMQOGTcssRqMdILN8EkNrvO1ITbYJONMkcog4sszlpPEfxtbtkLQ+QUgGFvOL UxKLn1D1f97hA== From: Jakob-Tobias Winter To: pbs-devel@lists.proxmox.com Subject: [RFC PATCH proxmox-backup 03/12] datastore: add attributes to backup snapshots Date: Fri, 13 Nov 2026 09:56:30 +0100 Message-ID: <20261113085637.57360-6-tobias@stuttgart.de> X-Mailer: git-send-email 2.53.0 In-Reply-To: <20261113085637.57360-3-tobias@stuttgart.de> References: <20261113085637.57360-3-tobias@stuttgart.de> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.023 Adjusted score from AWL reputation of From: address DKIM_SIGNED 0.1 Message has a DKIM or DK signature, not necessarily valid DKIM_VALID -0.1 Message has at least one valid DKIM or DK signature DKIM_VALID_AU -0.1 Message has a valid DKIM or DK signature from author's domain DKIM_VALID_EF -0.1 Message has a valid DKIM or DK signature from envelope-from domain DMARC_PASS -0.1 DMARC pass policy RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_PASS -0.001 SPF: HELO matches SPF record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: IOFVRE7WYLUPXSRLUHIZLT234YPORTDA X-Message-ID-Hash: IOFVRE7WYLUPXSRLUHIZLT234YPORTDA X-MailFrom: prvs=d19b210678=tobias@stuttgart.de X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header CC: Jakob-Tobias Winter X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: A backup server knows little about where a backup comes from. Things like the tags and the resource pool of a Proxmox VE guest are only known to the client, but they are what users want to select backups by, for example in tape backup jobs. Allow storing such information as attributes of a snapshot. They are kept in the unprotected part of the manifest, as suggested in [0]. That way they can also be set for encrypted backups, and they stay with the snapshot when it is synced or written to tape. Each attribute is a key with a list of values. A guest with several tags gets one list element for each tag, instead of one string that has to be split again. A backup group has the attributes of its newest snapshot which has attributes. Snapshots without attributes are skipped on purpose: a client which does not set them, or fails to set them once, must not make a group lose its attributes. The unprotected part of the manifest can be uploaded by the client, so values with an invalid format are dropped when reading it. [0] https://bugzilla.proxmox.com/show_bug.cgi?id=4186#c32 Signed-off-by: Jakob-Tobias Winter --- The patches for proxmox-backup need the pbs-api-types of patches 1 and 2, so the dependency has to be bumped once they are released. pbs-datastore/src/backup_info.rs | 18 ++++++- pbs-datastore/src/datastore.rs | 15 +++++- pbs-datastore/src/manifest.rs | 87 +++++++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 3 deletions(-) diff --git a/pbs-datastore/src/backup_info.rs b/pbs-datastore/src/backup_info.rs index cce2b699f..ea9dc9a2a 100644 --- a/pbs-datastore/src/backup_info.rs +++ b/pbs-datastore/src/backup_info.rs @@ -17,7 +17,7 @@ use proxmox_systemd::escape_unit; use pbs_api_types::{ ArchiveType, Authid, BACKUP_DATE_REGEX, BackupArchiveName, BackupGroupDeleteStats, - BackupNamespace, BackupType, GroupFilter, MANIFEST_BLOB_NAME, VerifyState, + BackupNamespace, BackupType, GroupFilter, MANIFEST_BLOB_NAME, SnapshotAttributes, VerifyState, }; use pbs_config::{BackupLockGuard, open_backup_lockfile}; @@ -148,6 +148,22 @@ impl BackupGroup { .max_by_key(|item| item.backup_dir.backup_time())) } + /// Returns the attributes of the backup group + /// + /// These are the attributes of the newest finished snapshot which has some stored. Snapshots + /// without stored attributes are skipped, so a group keeps its attributes if a client did not + /// provide them for newer snapshots. + pub fn attributes(&self) -> Result { + let mut backups = self.list_backups()?; + BackupInfo::sort_list(&mut backups, false); + + Ok(backups + .iter() + .filter(|info| info.is_finished()) + .find_map(|info| info.backup_dir.load_manifest().ok()?.0.attributes()) + .unwrap_or_default()) + } + pub fn last_successful_backup(&self) -> Result, Error> { let mut last = None; diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs index e6766a3b9..5b094a095 100644 --- a/pbs-datastore/src/datastore.rs +++ b/pbs-datastore/src/datastore.rs @@ -34,7 +34,7 @@ use pbs_api_types::{ ArchiveType, Authid, BackupGroupDeleteStats, BackupNamespace, BackupType, ChunkOrder, DEFAULT_SNAPSHOT_TIME_SKEW_SECS, DataStoreConfig, DatastoreBackendConfig, DatastoreBackendType, GarbageCollectionCacheStats, GarbageCollectionStatus, MAX_NAMESPACE_DEPTH, MaintenanceMode, - MaintenanceType, Operation, S3Statistics, UPID, + MaintenanceType, Operation, S3Statistics, SnapshotAttributes, UPID, }; use pbs_config::s3::S3_CFG_TYPE_ID; use pbs_config::{BackupLockGuard, ConfigVersionCache}; @@ -3458,6 +3458,19 @@ impl DataStore { Ok(()) } + /// Set the attributes for the given snapshot, replacing the existing ones. + pub fn set_attributes( + self: &Arc, + attributes: SnapshotAttributes, + snapshot: BackupDir, + ) -> Result<(), Error> { + let backend = self.backend().context("failed to get backend")?; + snapshot + .update_manifest(&backend, |manifest| manifest.set_attributes(&attributes)) + .map_err(|err| format_err!("unable to update manifest blob - {err}"))?; + Ok(()) + } + /// Renames a corrupt chunk, returning the new path if the chunk was renamed successfully. /// Returns with `Ok(None)` if the chunk source was not found. pub fn rename_corrupt_chunk(&self, digest: &[u8; 32]) -> Result, Error> { diff --git a/pbs-datastore/src/manifest.rs b/pbs-datastore/src/manifest.rs index 16f9668f2..61982d64a 100644 --- a/pbs-datastore/src/manifest.rs +++ b/pbs-datastore/src/manifest.rs @@ -9,7 +9,7 @@ use proxmox_sys::fs::{CreateOptions, replace_file}; use pbs_api_types::{ BackupArchiveName, BackupType, CLIENT_LOG_BLOB_NAME, CryptMode, Fingerprint, - MANIFEST_BLOB_NAME, SnapshotVerifyState, + MANIFEST_BLOB_NAME, SNAPSHOT_ATTRIBUTE_REGEX, SnapshotAttributes, SnapshotVerifyState, }; use pbs_tools::crypt_config::CryptConfig; @@ -273,6 +273,27 @@ impl BackupManifest { Ok(Some(serde_json::from_value::(verify)?)) } + /// Get the attributes of the snapshot + /// + /// Returns `None` if no attributes were stored for the snapshot. The unprotected part of the + /// manifest can be provided by the client, so values with an invalid format are dropped and + /// attributes which are not a list of values for each key count as not stored. + pub fn attributes(&self) -> Option { + let mut attributes = + SnapshotAttributes::deserialize(&self.unprotected["attributes"]).ok()?; + + for (key, values) in attributes.iter_mut() { + values.retain(|value| SNAPSHOT_ATTRIBUTE_REGEX.is_match(&format!("{key}={value}"))); + } + + Some(attributes) + } + + /// Set the attributes of the snapshot, replacing the existing ones + pub fn set_attributes(&mut self, attributes: &SnapshotAttributes) { + self.unprotected["attributes"] = json!(attributes); + } + /// Set the sync-source-signature used to detect snapshot changes across encrypted/decrypted /// sync flows. /// @@ -597,3 +618,67 @@ fn test_manifest_eq_ignoring_verify_state() -> Result<(), Error> { Ok(()) } + +#[test] +fn test_manifest_attributes() -> Result<(), Error> { + let crypt_config = CryptConfig::new([7; 32])?; + let mut manifest = BackupManifest::new("vm/100/2020-06-26T13:56:05Z".parse()?); + manifest.add_file( + &"drive-scsi0.img.fidx".try_into()?, + 200, + [1; 32], + CryptMode::Encrypt, + )?; + + // nothing stored yet + assert!(manifest.attributes().is_none()); + + let signature = manifest.signature(&crypt_config)?; + + let mut attributes = SnapshotAttributes::new(); + for (key, value) in [ + ("pve-tag", "web"), + ("pve-tag", "prod"), + ("pve-pool", "customers/a"), + ] { + attributes + .entry(key.to_string()) + .or_default() + .insert(value.to_string()); + } + manifest.set_attributes(&attributes); + + // not covered by the signature, so attributes can be set for encrypted backups + assert_eq!(manifest.signature(&crypt_config)?, signature); + + // each value is a list element of its own, in a stable order + assert_eq!( + manifest.unprotected["attributes"], + json!({ "pve-pool": ["customers/a"], "pve-tag": ["prod", "web"] }), + ); + + let data = manifest.to_string(Some(&crypt_config))?; + let mut manifest = BackupManifest::from_data(data.as_bytes(), Some(&crypt_config), false)?; + assert_eq!(manifest.attributes(), Some(attributes)); + + // stored, but empty + manifest.set_attributes(&SnapshotAttributes::new()); + assert_eq!(manifest.attributes(), Some(SnapshotAttributes::new())); + + // data which does not come from the server + manifest.unprotected["attributes"] = json!("pve-tag=prod"); + assert!(manifest.attributes().is_none()); + + manifest.unprotected["attributes"] = json!({ "pve-tag": "prod;web" }); + assert!(manifest.attributes().is_none()); + + manifest.unprotected["attributes"] = json!({ + "pve-tag": ["prod", "with space", "web;db"], + "Invalid Key": ["prod"], + }); + let attributes = manifest.attributes().unwrap(); + assert_eq!(attributes["pve-tag"].iter().collect::>(), ["prod"]); + assert!(attributes["Invalid Key"].is_empty()); + + Ok(()) +} base-commit: a7eb4e8a1477df960570fa72e9e7e2b8ac58bdab -- 2.53.0