[pbs-devel] [PATCH v2 proxmox-backup 0/3] datastore: Create a prune job upon datastore creation

[pbs-devel] [PATCH v2 proxmox-backup 0/3] datastore: Create a prune job upon datastore creation

[Stefan Lendl]

prune-schedule was availible as an option in API and GUI but had no effect
because datastore prunes are handles by prune-jobs.
This creates a default prune job if prune-schedule is set when creating the
datastore.

Differences to v1:
  split into individual patches to separatly move logic to do_create_prune_job

Stefan Lendl (3):
  prune job: prune job creation in separate function
  pass worker context to do_create_prune_job
  fix #4374: create a prune job upon datastore creation

 src/api2/config/datastore.rs | 49 +++++++++++++++++++++++++++++-------
 src/api2/config/prune.rs     | 43 ++++++++++++++++++++-----------
 2 files changed, 68 insertions(+), 24 deletions(-)

-- 
2.42.0

[pbs-devel] [PATCH v2 proxmox-backup 1/3] prune job: prune job creation in separate function

[Stefan Lendl]

move logic for prune job creation to do_create_prune_job

Signed-off-by: Stefan Lendl <s.lendl@proxmox.com>
---
 src/api2/config/prune.rs | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/api2/config/prune.rs b/src/api2/config/prune.rs
index 6f391722..1e8b5526 100644
--- a/src/api2/config/prune.rs
+++ b/src/api2/config/prune.rs
@@ -56,6 +56,24 @@ pub fn list_prune_jobs(
     Ok(list)
 }
 
+pub fn do_create_prune_job(config: PruneJobConfig) -> Result<(), Error> {
+    let _lock = prune::lock_config()?;
+
+    let (mut section_config, _digest) = prune::config()?;
+
+    if section_config.sections.get(&config.id).is_some() {
+        param_bail!("id", "job '{}' already exists.", config.id);
+    }
+
+    section_config.set_data(&config.id, "prune", &config)?;
+
+    prune::save_config(&section_config)?;
+
+    crate::server::jobstate::create_state_file("prunejob", &config.id)?;
+
+    Ok(())
+}
+
 #[api(
     protected: true,
     input: {
@@ -81,21 +99,7 @@ pub fn create_prune_job(
 
     user_info.check_privs(&auth_id, &config.acl_path(), PRIV_DATASTORE_MODIFY, true)?;
 
-    let _lock = prune::lock_config()?;
-
-    let (mut section_config, _digest) = prune::config()?;
-
-    if section_config.sections.get(&config.id).is_some() {
-        param_bail!("id", "job '{}' already exists.", config.id);
-    }
-
-    section_config.set_data(&config.id, "prune", &config)?;
-
-    prune::save_config(&section_config)?;
-
-    crate::server::jobstate::create_state_file("prunejob", &config.id)?;
-
-    Ok(())
+    do_create_prune_job(config)
 }
 
 #[api(
-- 
2.42.0

[pbs-devel] [PATCH v2 proxmox-backup 2/3] pass worker context to do_create_prune_job

[Stefan Lendl]

pass the WorkerTaksContext to do_create_prune_job because we want
logging when calling within a worker context.

Signed-off-by: Stefan Lendl <s.lendl@proxmox.com>
---
 src/api2/config/prune.rs | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/api2/config/prune.rs b/src/api2/config/prune.rs
index 1e8b5526..4f7ce39c 100644
--- a/src/api2/config/prune.rs
+++ b/src/api2/config/prune.rs
@@ -1,5 +1,7 @@
 use anyhow::Error;
 use hex::FromHex;
+use proxmox_sys::task_log;
+use proxmox_sys::WorkerTaskContext;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
@@ -56,7 +58,10 @@ pub fn list_prune_jobs(
     Ok(list)
 }
 
-pub fn do_create_prune_job(config: PruneJobConfig) -> Result<(), Error> {
+pub fn do_create_prune_job(
+    config: PruneJobConfig,
+    worker: Option<&dyn WorkerTaskContext>,
+) -> Result<(), Error> {
     let _lock = prune::lock_config()?;
 
     let (mut section_config, _digest) = prune::config()?;
@@ -71,6 +76,10 @@ pub fn do_create_prune_job(config: PruneJobConfig) -> Result<(), Error> {
 
     crate::server::jobstate::create_state_file("prunejob", &config.id)?;
 
+    if let Some(worker) = worker {
+        task_log!(worker, "Prune job created: {}", config.id);
+    }
+
     Ok(())
 }
 
@@ -99,7 +108,7 @@ pub fn create_prune_job(
 
     user_info.check_privs(&auth_id, &config.acl_path(), PRIV_DATASTORE_MODIFY, true)?;
 
-    do_create_prune_job(config)
+    do_create_prune_job(config, None)
 }
 
 #[api(
-- 
2.42.0

[pbs-devel] [PATCH v2 proxmox-backup 3/3] fix #4374: create a prune job upon datastore creation

[Stefan Lendl]

creates a default prune job if prune-schedule is set when creating the
datastore.

Auto generates a name for a prune-job with a truncated uuid to avoid
collisions.

Prune settings were stored in the datastore config but have no effect.
Prune settings are not stored there anymore

Signed-off-by: Stefan Lendl <s.lendl@proxmox.com>
---
 src/api2/config/datastore.rs | 49 +++++++++++++++++++++++++++++-------
 1 file changed, 40 insertions(+), 9 deletions(-)

diff --git a/src/api2/config/datastore.rs b/src/api2/config/datastore.rs
index 5e013c39..746a5c01 100644
--- a/src/api2/config/datastore.rs
+++ b/src/api2/config/datastore.rs
@@ -9,11 +9,12 @@ use proxmox_router::{http_bail, Permission, Router, RpcEnvironment, RpcEnvironme
 use proxmox_schema::{api, param_bail, ApiType};
 use proxmox_section_config::SectionConfigData;
 use proxmox_sys::{task_warn, WorkerTaskContext};
+use proxmox_uuid::Uuid;
 
 use pbs_api_types::{
-    Authid, DataStoreConfig, DataStoreConfigUpdater, DatastoreNotify, DatastoreTuning,
-    DATASTORE_SCHEMA, PRIV_DATASTORE_ALLOCATE, PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY,
-    PROXMOX_CONFIG_DIGEST_SCHEMA, UPID_SCHEMA,
+    Authid, DataStoreConfig, DataStoreConfigUpdater, DatastoreNotify, DatastoreTuning, KeepOptions,
+    PruneJobConfig, PruneJobOptions, DATASTORE_SCHEMA, PRIV_DATASTORE_ALLOCATE,
+    PRIV_DATASTORE_AUDIT, PRIV_DATASTORE_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA, UPID_SCHEMA,
 };
 use pbs_config::BackupLockGuard;
 use pbs_datastore::chunk_store::ChunkStore;
@@ -21,7 +22,7 @@ use pbs_datastore::chunk_store::ChunkStore;
 use crate::api2::admin::{
     prune::list_prune_jobs, sync::list_sync_jobs, verify::list_verification_jobs,
 };
-use crate::api2::config::prune::delete_prune_job;
+use crate::api2::config::prune::{delete_prune_job, do_create_prune_job};
 use crate::api2::config::sync::delete_sync_job;
 use crate::api2::config::tape_backup_job::{delete_tape_backup_job, list_tape_backup_jobs};
 use crate::api2::config::verify::delete_verification_job;
@@ -91,10 +92,7 @@ pub(crate) fn do_create_datastore(
 
     pbs_config::datastore::save_config(&config)?;
 
-    jobstate::create_state_file("prune", &datastore.name)?;
-    jobstate::create_state_file("garbage_collection", &datastore.name)?;
-
-    Ok(())
+    jobstate::create_state_file("garbage_collection", &datastore.name)
 }
 
 #[api(
@@ -127,12 +125,45 @@ pub fn create_datastore(
     let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
     let to_stdout = rpcenv.env_type() == RpcEnvironmentType::CLI;
 
+    let prune_job_config = config.prune_schedule.as_ref().and_then(|schedule| {
+        let mut id = format!("default-{}-{}", config.name, Uuid::generate());
+        id.truncate(32);
+
+        Some(PruneJobConfig {
+            id,
+            store: config.name.clone(),
+            comment: None,
+            disable: false,
+            schedule: schedule.clone(),
+            options: PruneJobOptions {
+                keep: config.keep.clone(),
+                max_depth: None,
+                ns: None,
+            },
+        })
+    });
+
+    // clearing prune settings in the datastore config, as they are now handled by prune jobs
+    let config = DataStoreConfig {
+        prune_schedule: None,
+        keep: KeepOptions::default(),
+        ..config
+    };
+
     WorkerTask::new_thread(
         "create-datastore",
         Some(config.name.to_string()),
         auth_id.to_string(),
         to_stdout,
-        move |worker| do_create_datastore(lock, section_config, config, Some(&worker)),
+        move |worker| {
+            do_create_datastore(lock, section_config, config, Some(&worker))?;
+
+            if let Some(prune_job_config) = prune_job_config {
+                do_create_prune_job(prune_job_config, Some(&worker))
+            } else {
+                Ok(())
+            }
+        },
     )
 }
 
-- 
2.42.0

Re: [pbs-devel] [PATCH v2 proxmox-backup 0/3] datastore: Create a prune job upon datastore creation

[Gabriel Goller]

LGTM!
Just one small clippy warning:

warning: using `Option.and_then(|x| Some(y))`, which is more succinctly 
expressed as `map(|x| y)`
    --> src/api2/config/datastore.rs:128:28

Otherwise you can consider:
Tested-by: Gabriel Goller <g.goller@proxmox.com>

Re: [pbs-devel] [PATCH v2 proxmox-backup 0/3] datastore: Create a prune job upon datastore creation

[Stefan Lendl]

Gabriel Goller <g.goller@proxmox.com> writes:

> LGTM!
> Just one small clippy warning:
>
> warning: using `Option.and_then(|x| Some(y))`, which is more succinctly 
> expressed as `map(|x| y)`
>     --> src/api2/config/datastore.rs:128:28
>
> Otherwise you can consider:
> Tested-by: Gabriel Goller <g.goller@proxmox.com>

sent v3 incorporating the change