From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from gate001.proxmox.com (gate001.proxmox.com [IPv6:2a0f:8001:1:32::40]) by lore.proxmox.com (Postfix) with ESMTPS id DE91D1FF0C0 for ; Wed, 18 Nov 2026 13:55:16 +0100 (CET) Received: from gate001.proxmox.com (localhost.localdomain [127.0.0.1]) by gate001.proxmox.com (Proxmox) with ESMTP id 7F2FD21689; Wed, 18 Nov 2026 13:55:16 +0100 (CET) Message-ID: <96bdde16-b17f-4b14-90cf-6830fdec5641@proxmox.com> Date: Wed, 18 Nov 2026 13:55:10 +0100 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird From: Samuel Rufinatscha Subject: Re: [RFC PATCH proxmox-backup 03/12] datastore: add attributes to backup snapshots To: Jakob-Tobias Winter , pbs-devel@lists.proxmox.com References: <20261113085637.57360-3-tobias@stuttgart.de> <20261113085637.57360-6-tobias@stuttgart.de> Content-Language: en-US In-Reply-To: <20261113085637.57360-6-tobias@stuttgart.de> Content-Type: text/plain; charset=UTF-8; format=flowed Content-Transfer-Encoding: 7bit X-SPAM-LEVEL: Spam detection results: 0 AWL 0.167 Adjusted score from AWL reputation of From: address DMARC_MISSING 0.1 Missing DMARC policy KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment (newer systems) RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Message-ID-Hash: 6I2TL4XTNYJFTMSUWEK6VR2YDHYYS6ZA X-Message-ID-Hash: 6I2TL4XTNYJFTMSUWEK6VR2YDHYYS6ZA X-MailFrom: s.rufinatscha@proxmox.com X-Mailman-Rule-Misses: dmarc-mitigation; no-senders; approved; loop; banned-address; emergency; member-moderation; nonmember-moderation; administrivia; implicit-dest; max-recipients; max-size; news-moderation; no-subject; digests; suspicious-header X-Mailman-Version: 3.3.10 Precedence: list List-Id: Proxmox Backup Server development discussion List-Help: List-Owner: List-Post: List-Subscribe: List-Unsubscribe: On 11/13/26 10:06 AM, Jakob-Tobias Winter wrote: > A backup server knows little about where a backup comes from. Things > like the tags and the resource pool of a Proxmox VE guest are only > known to the client, but they are what users want to select backups > by, for example in tape backup jobs. > > Allow storing such information as attributes of a snapshot. They are > kept in the unprotected part of the manifest, as suggested in [0]. > That way they can also be set for encrypted backups, and they stay > with the snapshot when it is synced or written to tape. > > Each attribute is a key with a list of values. A guest with several > tags gets one list element for each tag, instead of one string that > has to be split again. > > A backup group has the attributes of its newest snapshot which has > attributes. Snapshots without attributes are skipped on purpose: a > client which does not set them, or fails to set them once, must not > make a group lose its attributes. > > The unprotected part of the manifest can be uploaded by the client, > so values with an invalid format are dropped when reading it. > > [0] https://bugzilla.proxmox.com/show_bug.cgi?id=4186#c32 > > Signed-off-by: Jakob-Tobias Winter > --- > The patches for proxmox-backup need the pbs-api-types of patches 1 and > 2, so the dependency has to be bumped once they are released. > > pbs-datastore/src/backup_info.rs | 18 ++++++- > pbs-datastore/src/datastore.rs | 15 +++++- > pbs-datastore/src/manifest.rs | 87 +++++++++++++++++++++++++++++++- > 3 files changed, 117 insertions(+), 3 deletions(-) > > diff --git a/pbs-datastore/src/backup_info.rs b/pbs-datastore/src/backup_info.rs > index cce2b699f..ea9dc9a2a 100644 > --- a/pbs-datastore/src/backup_info.rs > +++ b/pbs-datastore/src/backup_info.rs > @@ -17,7 +17,7 @@ use proxmox_systemd::escape_unit; > > use pbs_api_types::{ > ArchiveType, Authid, BACKUP_DATE_REGEX, BackupArchiveName, BackupGroupDeleteStats, > - BackupNamespace, BackupType, GroupFilter, MANIFEST_BLOB_NAME, VerifyState, > + BackupNamespace, BackupType, GroupFilter, MANIFEST_BLOB_NAME, SnapshotAttributes, VerifyState, > }; > use pbs_config::{BackupLockGuard, open_backup_lockfile}; > > @@ -148,6 +148,22 @@ impl BackupGroup { > .max_by_key(|item| item.backup_dir.backup_time())) > } > > + /// Returns the attributes of the backup group > + /// > + /// These are the attributes of the newest finished snapshot which has some stored. Snapshots > + /// without stored attributes are skipped, so a group keeps its attributes if a client did not > + /// provide them for newer snapshots. > + pub fn attributes(&self) -> Result { > + let mut backups = self.list_backups()?; > + BackupInfo::sort_list(&mut backups, false); > + > + Ok(backups > + .iter() > + .filter(|info| info.is_finished()) > + .find_map(|info| info.backup_dir.load_manifest().ok()?.0.attributes()) this ok() would silently skip an unreadable/corrupt manifest and fallback to an older snapshot. i think it would make sense to propagate it instead > + .unwrap_or_default()) > + } > + > pub fn last_successful_backup(&self) -> Result, Error> { > let mut last = None; > > diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs > index e6766a3b9..5b094a095 100644 > --- a/pbs-datastore/src/datastore.rs > +++ b/pbs-datastore/src/datastore.rs > @@ -34,7 +34,7 @@ use pbs_api_types::{ > ArchiveType, Authid, BackupGroupDeleteStats, BackupNamespace, BackupType, ChunkOrder, > DEFAULT_SNAPSHOT_TIME_SKEW_SECS, DataStoreConfig, DatastoreBackendConfig, DatastoreBackendType, > GarbageCollectionCacheStats, GarbageCollectionStatus, MAX_NAMESPACE_DEPTH, MaintenanceMode, > - MaintenanceType, Operation, S3Statistics, UPID, > + MaintenanceType, Operation, S3Statistics, SnapshotAttributes, UPID, > }; > use pbs_config::s3::S3_CFG_TYPE_ID; > use pbs_config::{BackupLockGuard, ConfigVersionCache}; > @@ -3458,6 +3458,19 @@ impl DataStore { > Ok(()) > } > > + /// Set the attributes for the given snapshot, replacing the existing ones. > + pub fn set_attributes( > + self: &Arc, > + attributes: SnapshotAttributes, > + snapshot: BackupDir, > + ) -> Result<(), Error> { > + let backend = self.backend().context("failed to get backend")?; > + snapshot > + .update_manifest(&backend, |manifest| manifest.set_attributes(&attributes)) > + .map_err(|err| format_err!("unable to update manifest blob - {err}"))?; > + Ok(()) > + } > + > /// Renames a corrupt chunk, returning the new path if the chunk was renamed successfully. > /// Returns with `Ok(None)` if the chunk source was not found. > pub fn rename_corrupt_chunk(&self, digest: &[u8; 32]) -> Result, Error> { > diff --git a/pbs-datastore/src/manifest.rs b/pbs-datastore/src/manifest.rs > index 16f9668f2..61982d64a 100644 > --- a/pbs-datastore/src/manifest.rs > +++ b/pbs-datastore/src/manifest.rs > @@ -9,7 +9,7 @@ use proxmox_sys::fs::{CreateOptions, replace_file}; > > use pbs_api_types::{ > BackupArchiveName, BackupType, CLIENT_LOG_BLOB_NAME, CryptMode, Fingerprint, > - MANIFEST_BLOB_NAME, SnapshotVerifyState, > + MANIFEST_BLOB_NAME, SNAPSHOT_ATTRIBUTE_REGEX, SnapshotAttributes, SnapshotVerifyState, > }; > use pbs_tools::crypt_config::CryptConfig; > > @@ -273,6 +273,27 @@ impl BackupManifest { > Ok(Some(serde_json::from_value::(verify)?)) > } > > + /// Get the attributes of the snapshot > + /// > + /// Returns `None` if no attributes were stored for the snapshot. The unprotected part of the > + /// manifest can be provided by the client, so values with an invalid format are dropped and > + /// attributes which are not a list of values for each key count as not stored. > + pub fn attributes(&self) -> Option { > + let mut attributes = > + SnapshotAttributes::deserialize(&self.unprotected["attributes"]).ok()?; > + > + for (key, values) in attributes.iter_mut() { > + values.retain(|value| SNAPSHOT_ATTRIBUTE_REGEX.is_match(&format!("{key}={value}"))); > + } > + > + Some(attributes) > + } > + > + /// Set the attributes of the snapshot, replacing the existing ones > + pub fn set_attributes(&mut self, attributes: &SnapshotAttributes) { > + self.unprotected["attributes"] = json!(attributes); > + } > + > /// Set the sync-source-signature used to detect snapshot changes across encrypted/decrypted > /// sync flows. > /// > @@ -597,3 +618,67 @@ fn test_manifest_eq_ignoring_verify_state() -> Result<(), Error> { > > Ok(()) > } > + > +#[test] > +fn test_manifest_attributes() -> Result<(), Error> { > + let crypt_config = CryptConfig::new([7; 32])?; > + let mut manifest = BackupManifest::new("vm/100/2020-06-26T13:56:05Z".parse()?); > + manifest.add_file( > + &"drive-scsi0.img.fidx".try_into()?, > + 200, > + [1; 32], > + CryptMode::Encrypt, > + )?; > + > + // nothing stored yet > + assert!(manifest.attributes().is_none()); > + > + let signature = manifest.signature(&crypt_config)?; > + > + let mut attributes = SnapshotAttributes::new(); > + for (key, value) in [ > + ("pve-tag", "web"), > + ("pve-tag", "prod"), > + ("pve-pool", "customers/a"), > + ] { > + attributes > + .entry(key.to_string()) > + .or_default() > + .insert(value.to_string()); > + } > + manifest.set_attributes(&attributes); > + > + // not covered by the signature, so attributes can be set for encrypted backups > + assert_eq!(manifest.signature(&crypt_config)?, signature); > + > + // each value is a list element of its own, in a stable order > + assert_eq!( > + manifest.unprotected["attributes"], > + json!({ "pve-pool": ["customers/a"], "pve-tag": ["prod", "web"] }), > + ); > + > + let data = manifest.to_string(Some(&crypt_config))?; > + let mut manifest = BackupManifest::from_data(data.as_bytes(), Some(&crypt_config), false)?; > + assert_eq!(manifest.attributes(), Some(attributes)); > + > + // stored, but empty > + manifest.set_attributes(&SnapshotAttributes::new()); > + assert_eq!(manifest.attributes(), Some(SnapshotAttributes::new())); > + > + // data which does not come from the server > + manifest.unprotected["attributes"] = json!("pve-tag=prod"); > + assert!(manifest.attributes().is_none()); > + > + manifest.unprotected["attributes"] = json!({ "pve-tag": "prod;web" }); > + assert!(manifest.attributes().is_none()); > + > + manifest.unprotected["attributes"] = json!({ > + "pve-tag": ["prod", "with space", "web;db"], > + "Invalid Key": ["prod"], > + }); > + let attributes = manifest.attributes().unwrap(); > + assert_eq!(attributes["pve-tag"].iter().collect::>(), ["prod"]); > + assert!(attributes["Invalid Key"].is_empty()); > + > + Ok(()) > +} > > base-commit: a7eb4e8a1477df960570fa72e9e7e2b8ac58bdab