Re: [pbs-devel] [PATCH v5 proxmox-backup 30/31] api: datastore/namespace: return backup groups delete stats on remove

[Christian Ebner <c.ebner@proxmox.com>]

On 10/25/24 12:10, Fabian Grünbichler wrote:
> On October 18, 2024 10:42 am, Christian Ebner wrote:
>> Add and expose the backup group delete statistics by adding the
>> return type to the corresponding REST API endpoints.
>>
>> Further, add a `ignore-protected` flag to the api endpoints, allowing
>> to return without error when set.
> 
> logical changes look good to me, `ignore-protected` is just slightly
> confusing as a name - just reading that, I would think this removes the
> namespace/group even if something protected is found, not that it would
> make that "error" non-fatal.
> 
> can't really think of a catchy alternative though, but wanted to mention
> it in case somebody else can ;)

Renamed this to `error-on-protected` and switched the boolean logic. 
That should hopefully make it less susceptible to misinterpretation.

> 
> we should probably switch the UI over to present the results now that we
> can?

Not sure I fully understood what you mean here, the remove vanished 
stats are already exposed in the summary of the sync job.

> 
>>
>> Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
>> ---
>> changes since version 4:
>> - no changes
>>
>> changes since version 3:
>> - make new endpoint failure behaviour opt-in
>>
>>   pbs-datastore/src/datastore.rs | 20 ++++++++++++++------
>>   src/api2/admin/datastore.rs    | 29 +++++++++++++++++++++--------
>>   src/api2/admin/namespace.rs    | 31 ++++++++++++++++++++++---------
>>   src/server/pull.rs             |  6 ++++--
>>   4 files changed, 61 insertions(+), 25 deletions(-)
>>
>> diff --git a/pbs-datastore/src/datastore.rs b/pbs-datastore/src/datastore.rs
>> index c8701d2dd..68c7f2934 100644
>> --- a/pbs-datastore/src/datastore.rs
>> +++ b/pbs-datastore/src/datastore.rs
>> @@ -489,16 +489,22 @@ impl DataStore {
>>       ///
>>       /// Does *not* descends into child-namespaces and doesn't remoes the namespace itself either.
>>       ///
>> -    /// Returns true if all the groups were removed, and false if some were protected.
>> -    pub fn remove_namespace_groups(self: &Arc<Self>, ns: &BackupNamespace) -> Result<bool, Error> {
>> +    /// Returns a tuple with the first item being true if all the groups were removed, and false if some were protected.
>> +    /// The second item returns the remove statistics.
>> +    pub fn remove_namespace_groups(
>> +        self: &Arc<Self>,
>> +        ns: &BackupNamespace,
>> +    ) -> Result<(bool, BackupGroupDeleteStats), Error> {
>>           // FIXME: locking? The single groups/snapshots are already protected, so may not be
>>           // necessary (depends on what we all allow to do with namespaces)
>>           log::info!("removing all groups in namespace {}:/{ns}", self.name());
>>   
>>           let mut removed_all_groups = true;
>> +        let mut stats = BackupGroupDeleteStats::default();
>>   
>>           for group in self.iter_backup_groups(ns.to_owned())? {
>>               let delete_stats = group?.destroy()?;
>> +            stats.add(&delete_stats);
>>               removed_all_groups = removed_all_groups && delete_stats.all_removed();
>>           }
>>   
>> @@ -515,7 +521,7 @@ impl DataStore {
>>               }
>>           }
>>   
>> -        Ok(removed_all_groups)
>> +        Ok((removed_all_groups, stats))
>>       }
>>   
>>       /// Remove a complete backup namespace optionally including all it's, and child namespaces',
>> @@ -527,13 +533,15 @@ impl DataStore {
>>           self: &Arc<Self>,
>>           ns: &BackupNamespace,
>>           delete_groups: bool,
>> -    ) -> Result<bool, Error> {
>> +    ) -> Result<(bool, BackupGroupDeleteStats), Error> {
>>           let store = self.name();
>>           let mut removed_all_requested = true;
>> +        let mut stats = BackupGroupDeleteStats::default();
>>           if delete_groups {
>>               log::info!("removing whole namespace recursively below {store}:/{ns}",);
>>               for ns in self.recursive_iter_backup_ns(ns.to_owned())? {
>> -                let removed_ns_groups = self.remove_namespace_groups(&ns?)?;
>> +                let (removed_ns_groups, delete_stats) = self.remove_namespace_groups(&ns?)?;
>> +                stats.add(&delete_stats);
>>                   removed_all_requested = removed_all_requested && removed_ns_groups;
>>               }
>>           } else {
>> @@ -574,7 +582,7 @@ impl DataStore {
>>               }
>>           }
>>   
>> -        Ok(removed_all_requested)
>> +        Ok((removed_all_requested, stats))
>>       }
>>   
>>       /// Remove a complete backup group including all snapshots.
>> diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
>> index 7660dd7f6..81f81b2b3 100644
>> --- a/src/api2/admin/datastore.rs
>> +++ b/src/api2/admin/datastore.rs
>> @@ -34,10 +34,10 @@ use pxar::accessor::aio::Accessor;
>>   use pxar::EntryKind;
>>   
>>   use pbs_api_types::{
>> -    print_ns_and_snapshot, print_store_and_ns, Authid, BackupContent, BackupNamespace, BackupType,
>> -    Counts, CryptMode, DataStoreConfig, DataStoreListItem, DataStoreStatus,
>> -    GarbageCollectionJobStatus, GroupListItem, JobScheduleStatus, KeepOptions, Operation,
>> -    PruneJobOptions, SnapshotListItem, SnapshotVerifyState, BACKUP_ARCHIVE_NAME_SCHEMA,
>> +    print_ns_and_snapshot, print_store_and_ns, Authid, BackupContent, BackupGroupDeleteStats,
>> +    BackupNamespace, BackupType, Counts, CryptMode, DataStoreConfig, DataStoreListItem,
>> +    DataStoreStatus, GarbageCollectionJobStatus, GroupListItem, JobScheduleStatus, KeepOptions,
>> +    Operation, PruneJobOptions, SnapshotListItem, SnapshotVerifyState, BACKUP_ARCHIVE_NAME_SCHEMA,
>>       BACKUP_ID_SCHEMA, BACKUP_NAMESPACE_SCHEMA, BACKUP_TIME_SCHEMA, BACKUP_TYPE_SCHEMA,
>>       DATASTORE_SCHEMA, IGNORE_VERIFIED_BACKUPS_SCHEMA, MAX_NAMESPACE_DEPTH, NS_MAX_DEPTH_SCHEMA,
>>       PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_MODIFY, PRIV_DATASTORE_PRUNE,
>> @@ -267,8 +267,17 @@ pub fn list_groups(
>>                   type: pbs_api_types::BackupGroup,
>>                   flatten: true,
>>               },
>> +            "ignore-protected": {
>> +                type: bool,
>> +                optional: true,
>> +                default: false,
>> +                description: "No error when group cannot be deleted because of protected snapshots",
>> +            }
>>           },
>>       },
>> +    returns: {
>> +        type: BackupGroupDeleteStats,
>> +    },
>>       access: {
>>           permission: &Permission::Anybody,
>>           description: "Requires on /datastore/{store}[/{namespace}] either DATASTORE_MODIFY for any \
>> @@ -279,9 +288,10 @@ pub fn list_groups(
>>   pub async fn delete_group(
>>       store: String,
>>       ns: Option<BackupNamespace>,
>> +    ignore_protected: bool,
>>       group: pbs_api_types::BackupGroup,
>>       rpcenv: &mut dyn RpcEnvironment,
>> -) -> Result<Value, Error> {
>> +) -> Result<BackupGroupDeleteStats, Error> {
>>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>>   
>>       tokio::task::spawn_blocking(move || {
>> @@ -299,10 +309,13 @@ pub async fn delete_group(
>>   
>>           let delete_stats = datastore.remove_backup_group(&ns, &group)?;
>>           if !delete_stats.all_removed() {
>> -            bail!("group only partially deleted due to protected snapshots");
>> +            if ignore_protected {
>> +                warn!("group only partially deleted due to protected snapshots");
>> +            } else {
>> +                bail!("group only partially deleted due to protected snapshots");
>> +            }
>>           }
>> -
>> -        Ok(Value::Null)
>> +        Ok(delete_stats)
>>       })
>>       .await?
>>   }
>> diff --git a/src/api2/admin/namespace.rs b/src/api2/admin/namespace.rs
>> index 889dc1a3d..8f99eb401 100644
>> --- a/src/api2/admin/namespace.rs
>> +++ b/src/api2/admin/namespace.rs
>> @@ -1,13 +1,12 @@
>>   use anyhow::{bail, Error};
>> -use serde_json::Value;
>>   
>>   use pbs_config::CachedUserInfo;
>>   use proxmox_router::{http_bail, ApiMethod, Permission, Router, RpcEnvironment};
>>   use proxmox_schema::*;
>>   
>>   use pbs_api_types::{
>> -    Authid, BackupNamespace, NamespaceListItem, Operation, DATASTORE_SCHEMA, NS_MAX_DEPTH_SCHEMA,
>> -    PROXMOX_SAFE_ID_FORMAT,
>> +    Authid, BackupGroupDeleteStats, BackupNamespace, NamespaceListItem, Operation,
>> +    DATASTORE_SCHEMA, NS_MAX_DEPTH_SCHEMA, PROXMOX_SAFE_ID_FORMAT,
>>   };
>>   
>>   use pbs_datastore::DataStore;
>> @@ -138,6 +137,12 @@ pub fn list_namespaces(
>>                   optional: true,
>>                   default: false,
>>               },
>> +            "ignore-protected": {
>> +                type: bool,
>> +                optional: true,
>> +                default: false,
>> +                description: "No error when namespace cannot be deleted because of protected snapshots",
>> +            }
>>           },
>>       },
>>       access: {
>> @@ -149,24 +154,32 @@ pub fn delete_namespace(
>>       store: String,
>>       ns: BackupNamespace,
>>       delete_groups: bool,
>> +    ignore_protected: bool,
>>       _info: &ApiMethod,
>>       rpcenv: &mut dyn RpcEnvironment,
>> -) -> Result<Value, Error> {
>> +) -> Result<BackupGroupDeleteStats, Error> {
>>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>>   
>>       check_ns_modification_privs(&store, &ns, &auth_id)?;
>>   
>>       let datastore = DataStore::lookup_datastore(&store, Some(Operation::Write))?;
>>   
>> -    if !datastore.remove_namespace_recursive(&ns, delete_groups)? {
>> -        if delete_groups {
>> -            bail!("group only partially deleted due to protected snapshots");
>> +    let (removed_all, stats) = datastore.remove_namespace_recursive(&ns, delete_groups)?;
>> +    if !removed_all {
>> +        let err_msg = if delete_groups {
>> +            "group only partially deleted due to protected snapshots"
>> +        } else {
>> +            "only partially deleted due to existing groups but `delete-groups` not true"
>> +        };
>> +
>> +        if ignore_protected {
>> +            log::warn!("{err_msg}");
>>           } else {
>> -            bail!("only partially deleted due to existing groups but `delete-groups` not true ");
>> +            bail!(err_msg);
>>           }
>>       }
>>   
>> -    Ok(Value::Null)
>> +    Ok(stats)
>>   }
>>   
>>   pub const ROUTER: Router = Router::new()
>> diff --git a/src/server/pull.rs b/src/server/pull.rs
>> index 3117f7d2c..d7f5c42ea 100644
>> --- a/src/server/pull.rs
>> +++ b/src/server/pull.rs
>> @@ -645,10 +645,12 @@ fn check_and_remove_ns(params: &PullParameters, local_ns: &BackupNamespace) -> R
>>       check_ns_modification_privs(params.target.store.name(), local_ns, &params.owner)
>>           .map_err(|err| format_err!("Removing {local_ns} not allowed - {err}"))?;
>>   
>> -    params
>> +    let (removed_all, _delete_stats) = params
>>           .target
>>           .store
>> -        .remove_namespace_recursive(local_ns, true)
>> +        .remove_namespace_recursive(local_ns, true)?;
>> +
>> +    Ok(removed_all)
>>   }
>>   
>>   fn check_and_remove_vanished_ns(
>> -- 
>> 2.39.5
>>
>>
>>
>> _______________________________________________
>> pbs-devel mailing list
>> pbs-devel@lists.proxmox.com
>> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
>>
>>
>>
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel
> 
> 



_______________________________________________
pbs-devel mailing list
pbs-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel