Re: [pbs-devel] [PATCH proxmox-backup 4/5] jobs/api2: add checks for maintenance

[Dominik Csapak <d.csapak@proxmox.com>]

On 9/28/21 12:05, Hannes Laimer wrote:
> ---
>   src/api2/admin/datastore.rs | 48 +++++++++++++++++++++++++++++++++----
>   src/api2/pull.rs            |  5 +++-
>   src/server/gc_job.rs        |  5 ++--
>   src/server/prune_job.rs     |  4 +++-
>   src/server/verify_job.rs    |  5 ++--
>   5 files changed, 56 insertions(+), 11 deletions(-)
> 
> diff --git a/src/api2/admin/datastore.rs b/src/api2/admin/datastore.rs
> index 7e9a0ee0..f5ed6603 100644
> --- a/src/api2/admin/datastore.rs
> +++ b/src/api2/admin/datastore.rs
> @@ -29,7 +29,7 @@ use pxar::EntryKind;
>   use pbs_api_types::{ Authid, BackupContent, Counts, CryptMode,
>       DataStoreListItem, GarbageCollectionStatus, GroupListItem,
>       SnapshotListItem, SnapshotVerifyState, PruneOptions,
> -    DataStoreStatus, RRDMode, RRDTimeFrameResolution,
> +    DataStoreStatus, RRDMode, RRDTimeFrameResolution, MaintenanceType,
>       BACKUP_ARCHIVE_NAME_SCHEMA, BACKUP_ID_SCHEMA, BACKUP_TIME_SCHEMA,
>       BACKUP_TYPE_SCHEMA, DATASTORE_SCHEMA,
>       IGNORE_VERIFIED_BACKUPS_SCHEMA, UPID_SCHEMA,
> @@ -83,6 +83,7 @@ fn check_priv_or_backup_owner(
>       auth_id: &Authid,
>       required_privs: u64,
>   ) -> Result<(), Error> {
> +    store.check_maintenance(MaintenanceType::Offline)?;
>       let user_info = CachedUserInfo::new()?;
>       let privs = user_info.lookup_privs(&auth_id, &["datastore", store.name()]);
>   
> @@ -97,7 +98,7 @@ fn read_backup_index(
>       store: &DataStore,
>       backup_dir: &BackupDir,
>   ) -> Result<(BackupManifest, Vec<BackupContent>), Error> {
> -
> +    store.check_maintenance(MaintenanceType::Offline)?;
>       let (manifest, index_size) = store.load_manifest(backup_dir)?;
>   
>       let mut result = Vec::new();
> @@ -125,7 +126,7 @@ fn get_all_snapshot_files(
>       store: &DataStore,
>       info: &BackupInfo,
>   ) -> Result<(BackupManifest, Vec<BackupContent>), Error> {
> -
> +    store.check_maintenance(MaintenanceType::Offline)?;
>       let (manifest, mut files) = read_backup_index(&store, &info.backup_dir)?;
>   
>       let file_set = files.iter().fold(HashSet::new(), |mut acc, item| {
> @@ -172,6 +173,9 @@ pub fn list_groups(
>       let user_privs = user_info.lookup_privs(&auth_id, &["datastore", &store]);
>   
>       let datastore = DataStore::lookup_datastore(&store)?;
> +
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let list_all = (user_privs & PRIV_DATASTORE_AUDIT) != 0;
>   
>       let backup_groups = BackupInfo::list_backup_groups(&datastore.base_path())?;
> @@ -267,9 +271,11 @@ pub fn delete_group(
>   ) -> Result<Value, Error> {
>   
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
> +    let datastore = DataStore::lookup_datastore(&store)?;
> +
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
>   
>       let group = BackupGroup::new(backup_type, backup_id);
> -    let datastore = DataStore::lookup_datastore(&store)?;
>   
>       check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
>   
> @@ -316,6 +322,8 @@ pub fn list_snapshot_files(
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
>   
>       check_priv_or_backup_owner(&datastore, snapshot.group(), &auth_id, PRIV_DATASTORE_AUDIT | PRIV_DATASTORE_READ)?;
> @@ -362,9 +370,12 @@ pub fn delete_snapshot(
>   ) -> Result<Value, Error> {
>   
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
> +
> +    let datastore = DataStore::lookup_datastore(&store)?;
> +
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
>   
>       let snapshot = BackupDir::new(backup_type, backup_id, backup_time)?;
> -    let datastore = DataStore::lookup_datastore(&store)?;
>   
>       check_priv_or_backup_owner(&datastore, snapshot.group(), &auth_id, PRIV_DATASTORE_MODIFY)?;
>   
> @@ -415,6 +426,8 @@ pub fn list_snapshots (
>   
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let base_path = datastore.base_path();
>   
>       let groups = match (backup_type, backup_id) {
> @@ -686,6 +699,9 @@ pub fn verify(
>       rpcenv: &mut dyn RpcEnvironment,
>   ) -> Result<Value, Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
> +
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let ignore_verified = ignore_verified.unwrap_or(true);
>   
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
> @@ -832,6 +848,8 @@ pub fn prune(
>   
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       check_priv_or_backup_owner(&datastore, &group, &auth_id, PRIV_DATASTORE_MODIFY)?;
>   
>       let worker_id = format!("{}:{}/{}", store, &backup_type, &backup_id);
> @@ -1035,6 +1053,8 @@ pub fn garbage_collection_status(
>   
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +

i was confused here at first, why garbage collect why garbage collect
needs only read access, then noticed it's the status. but the status
is not really a 'read' operation on the datastore? its more like
the config or a task log since it only shows the status of the last gc

>       let status = datastore.last_gc_status();
>   
>       Ok(status)
> @@ -1111,6 +1131,8 @@ pub fn download_file(
>           let store = required_string_param(&param, "store")?;
>           let datastore = DataStore::lookup_datastore(store)?;
>   
> +        datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>           let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>   
>           let file_name = required_string_param(&param, "file-name")?.to_owned();
> @@ -1181,6 +1203,8 @@ pub fn download_file_decoded(
>           let store = required_string_param(&param, "store")?;
>           let datastore = DataStore::lookup_datastore(store)?;
>   
> +        datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>           let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>   
>           let file_name = required_string_param(&param, "file-name")?.to_owned();
> @@ -1372,6 +1396,8 @@ pub fn catalog(
>   ) -> Result<Vec<ArchiveEntry>, Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>   
>       let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
> @@ -1442,6 +1468,8 @@ pub fn pxar_file_download(
>           let store = required_string_param(&param, "store")?;
>           let datastore = DataStore::lookup_datastore(&store)?;
>   
> +        datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>           let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>   
>           let filepath = required_string_param(&param, "filepath")?.to_owned();
> @@ -1597,6 +1625,8 @@ pub fn get_group_notes(
>   ) -> Result<String, Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>       let backup_group = BackupGroup::new(backup_type, backup_id);
>   
> @@ -1639,6 +1669,8 @@ pub fn set_group_notes(
>   ) -> Result<(), Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>       let backup_group = BackupGroup::new(backup_type, backup_id);
>   
> @@ -1681,6 +1713,8 @@ pub fn get_notes(
>   ) -> Result<String, Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::Offline)?;
> +
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>       let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
>   
> @@ -1732,6 +1766,8 @@ pub fn set_notes(
>   ) -> Result<(), Error> {
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
>       let backup_dir = BackupDir::new(backup_type, backup_id, backup_time)?;
>   
> @@ -1777,6 +1813,8 @@ pub fn set_backup_owner(
>   
>       let datastore = DataStore::lookup_datastore(&store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       let backup_group = BackupGroup::new(backup_type, backup_id);
>   
>       let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
> diff --git a/src/api2/pull.rs b/src/api2/pull.rs
> index ea8faab8..b1669288 100644
> --- a/src/api2/pull.rs
> +++ b/src/api2/pull.rs
> @@ -9,7 +9,7 @@ use proxmox::api::{ApiMethod, Router, RpcEnvironment, Permission};
>   
>   use pbs_client::{HttpClient, BackupRepository};
>   use pbs_api_types::{
> -    Remote, Authid, SyncJobConfig,
> +    Remote, Authid, SyncJobConfig, MaintenanceType,
>       DATASTORE_SCHEMA, REMOTE_ID_SCHEMA, REMOVE_VANISHED_BACKUPS_SCHEMA,
>       PRIV_DATASTORE_BACKUP, PRIV_DATASTORE_PRUNE, PRIV_REMOTE_READ,
>   };
> @@ -71,6 +71,9 @@ pub fn do_sync_job(
>       to_stdout: bool,
>   ) -> Result<String, Error> {
>   
> +    let datastore = DataStore::lookup_datastore(&sync_job.store)?;
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       let job_id = format!("{}:{}:{}:{}",
>                            sync_job.remote,
>                            sync_job.remote_store,
> diff --git a/src/server/gc_job.rs b/src/server/gc_job.rs
> index 794fe146..be04b635 100644
> --- a/src/server/gc_job.rs
> +++ b/src/server/gc_job.rs
> @@ -1,7 +1,7 @@
>   use std::sync::Arc;
>   use anyhow::Error;
>   
> -use pbs_api_types::Authid;
> +use pbs_api_types::{Authid, MaintenanceType};
>   use pbs_tools::task_log;
>   use pbs_datastore::DataStore;
>   use proxmox_rest_server::WorkerTask;
> @@ -16,7 +16,8 @@ pub fn do_garbage_collection_job(
>       schedule: Option<String>,
>       to_stdout: bool,
>   ) -> Result<String, Error> {
> -
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       let store = datastore.name().to_string();
>   
>       let (email, notify) = crate::server::lookup_datastore_notify_settings(&store);
> diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs
> index fc6443e9..84131634 100644
> --- a/src/server/prune_job.rs
> +++ b/src/server/prune_job.rs
> @@ -5,7 +5,7 @@ use anyhow::Error;
>   use pbs_datastore::backup_info::BackupInfo;
>   use pbs_datastore::prune::compute_prune_info;
>   use pbs_datastore::DataStore;
> -use pbs_api_types::{Authid, PRIV_DATASTORE_MODIFY, PruneOptions};
> +use pbs_api_types::{Authid, MaintenanceType, PRIV_DATASTORE_MODIFY, PruneOptions};
>   use pbs_config::CachedUserInfo;
>   use pbs_tools::{task_log, task_warn};
>   use proxmox_rest_server::WorkerTask;
> @@ -20,6 +20,8 @@ pub fn prune_datastore(
>       datastore: Arc<DataStore>,
>       dry_run: bool,
>   ) -> Result<(), Error> {
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +
>       task_log!(worker, "Starting datastore prune on store \"{}\"", store);
>   
>       if dry_run {
> diff --git a/src/server/verify_job.rs b/src/server/verify_job.rs
> index 6aba97c9..3b2b1472 100644
> --- a/src/server/verify_job.rs
> +++ b/src/server/verify_job.rs
> @@ -1,7 +1,7 @@
>   use anyhow::{format_err, Error};
>   
>   use pbs_tools::task_log;
> -use pbs_api_types::{Authid, VerificationJobConfig};
> +use pbs_api_types::{Authid, VerificationJobConfig, MaintenanceType};
>   use proxmox_rest_server::WorkerTask;
>   use pbs_datastore::DataStore;
>   
> @@ -21,9 +21,10 @@ pub fn do_verification_job(
>       schedule: Option<String>,
>       to_stdout: bool,
>   ) -> Result<String, Error> {
> -
>       let datastore = DataStore::lookup_datastore(&verification_job.store)?;
>   
> +    datastore.check_maintenance(MaintenanceType::ReadOnly)?;
> +

why does a verification write access? because of the verify status in 
the manifest?
imho that reason would be good to have in a comment

>       let outdated_after = verification_job.outdated_after;
>       let ignore_verified_snapshots = verification_job.ignore_verified.unwrap_or(true);
>   
>