[pbs-devel] [PATCH proxmox-backup 0/2] proxy: scheduling cleanup

[pbs-devel] [PATCH proxmox-backup 0/2] proxy: scheduling cleanup

[Hannes Laimer]

Logic for gc and logrotate was not moved, could (should?) also be moved later.

 * moved logic for prune out of proxy
 * extract commonly used logic for scheduling into new function


Hannes Laimer (2):
  proxy: move prune logic into new file
  proxy: extract commonly used logic for scheduling into new function

 src/bin/proxmox-backup-proxy.rs | 245 ++++++++------------------------
 src/server.rs                   |   3 +
 src/server/prune_job.rs         |  91 ++++++++++++
 3 files changed, 151 insertions(+), 188 deletions(-)
 create mode 100644 src/server/prune_job.rs

-- 
2.20.1

[pbs-devel] [PATCH proxmox-backup 1/2] proxy: move prune logic into new file

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 67 +++---------------------
 src/server.rs                   |  3 ++
 src/server/prune_job.rs         | 91 +++++++++++++++++++++++++++++++++
 3 files changed, 100 insertions(+), 61 deletions(-)
 create mode 100644 src/server/prune_job.rs

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index ce290171..21c5e9fb 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -49,6 +49,7 @@ use proxmox_backup::tools::{
 
 use proxmox_backup::api2::pull::do_sync_job;
 use proxmox_backup::server::do_verification_job;
+use proxmox_backup::server::do_prune_job;
 
 fn main() -> Result<(), Error> {
     proxmox_backup::tools::setup_safe_path_env();
@@ -358,8 +359,6 @@ async fn schedule_datastore_prune() {
     use proxmox_backup::{
         backup::{
             PruneOptions,
-            BackupGroup,
-            compute_prune_info,
         },
         config::datastore::{
             self,
@@ -376,13 +375,6 @@ async fn schedule_datastore_prune() {
     };
 
     for (store, (_, store_config)) in config.sections {
-        let datastore = match DataStore::lookup_datastore(&store) {
-            Ok(datastore) => datastore,
-            Err(err) => {
-                eprintln!("lookup_datastore '{}' failed - {}", store, err);
-                continue;
-            }
-        };
 
         let store_config: DataStoreConfig = match serde_json::from_value(store_config) {
             Ok(c) => c,
@@ -441,64 +433,17 @@ async fn schedule_datastore_prune() {
 
         if next > now  { continue; }
 
-        let mut job = match Job::new(worker_type, &store) {
+        let job = match Job::new(worker_type, &store) {
             Ok(job) => job,
             Err(_) => continue, // could not get lock
         };
 
-        let store2 = store.clone();
-
-        if let Err(err) = WorkerTask::new_thread(
-            worker_type,
-            Some(store.clone()),
-            Userid::backup_userid().clone(),
-            false,
-            move |worker| {
-
-                job.start(&worker.upid().to_string())?;
-
-                let result = try_block!({
-
-                    worker.log(format!("Starting datastore prune on store \"{}\"", store));
-                    worker.log(format!("task triggered by schedule '{}'", event_str));
-                    worker.log(format!("retention options: {}", prune_options.cli_options_string()));
-
-                    let base_path = datastore.base_path();
-
-                    let groups = BackupGroup::list_groups(&base_path)?;
-                    for group in groups {
-                        let list = group.list_backups(&base_path)?;
-                        let mut prune_info = compute_prune_info(list, &prune_options)?;
-                        prune_info.reverse(); // delete older snapshots first
-
-                        worker.log(format!("Starting prune on store \"{}\" group \"{}/{}\"",
-                                store, group.backup_type(), group.backup_id()));
-
-                        for (info, keep) in prune_info {
-                            worker.log(format!(
-                                    "{} {}/{}/{}",
-                                    if keep { "keep" } else { "remove" },
-                                    group.backup_type(), group.backup_id(),
-                                    info.backup_dir.backup_time_string()));
-                            if !keep {
-                                datastore.remove_backup_dir(&info.backup_dir, true)?;
-                            }
-                        }
-                    }
-                    Ok(())
-                });
-
-                let status = worker.create_state(&result);
-
-                if let Err(err) = job.finish(status) {
-                    eprintln!("could not finish job state for {}: {}", worker_type, err);
-                }
+        let userid = Userid::backup_userid().clone();
 
-                result
-            }
-        ) {
-            eprintln!("unable to start datastore prune on store {} - {}", store2, err);
+        if let Err(err) = do_prune_job(job, prune_options, store.clone(), &userid, Some(event_str)) {
+            eprintln!("unable to start datastore prune job {} - {}", &store, err);
         }
+
     }
 }
 
diff --git a/src/server.rs b/src/server.rs
index dbaec645..86719c42 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -34,3 +34,6 @@ pub mod jobstate;
 
 mod verify_job;
 pub use verify_job::*;
+
+mod prune_job;
+pub use prune_job::*;
diff --git a/src/server/prune_job.rs b/src/server/prune_job.rs
new file mode 100644
index 00000000..86127733
--- /dev/null
+++ b/src/server/prune_job.rs
@@ -0,0 +1,91 @@
+use anyhow::Error;
+
+use proxmox::try_block;
+
+use crate::{
+    api2::types::*,
+    backup::{compute_prune_info, BackupGroup, DataStore, PruneOptions},
+    server::jobstate::Job,
+    server::WorkerTask,
+    task_log,
+};
+
+pub fn do_prune_job(
+    mut job: Job,
+    prune_options: PruneOptions,
+    store: String,
+    userid: &Userid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+    let datastore = DataStore::lookup_datastore(&store)?;
+
+    let worker_type = job.jobtype().to_string();
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        userid.clone(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            let result = try_block!({
+                task_log!(worker, "Starting datastore prune on store \"{}\"", store);
+
+                if let Some(event_str) = schedule {
+                    task_log!(worker, "task triggered by schedule '{}'", event_str);
+                }
+
+                task_log!(
+                    worker,
+                    "retention options: {}",
+                    prune_options.cli_options_string()
+                );
+
+                let base_path = datastore.base_path();
+
+                let groups = BackupGroup::list_groups(&base_path)?;
+                for group in groups {
+                    let list = group.list_backups(&base_path)?;
+                    let mut prune_info = compute_prune_info(list, &prune_options)?;
+                    prune_info.reverse(); // delete older snapshots first
+
+                    task_log!(
+                        worker,
+                        "Starting prune on store \"{}\" group \"{}/{}\"",
+                        store,
+                        group.backup_type(),
+                        group.backup_id()
+                    );
+
+                    for (info, keep) in prune_info {
+                        task_log!(
+                            worker,
+                            "{} {}/{}/{}",
+                            if keep { "keep" } else { "remove" },
+                            group.backup_type(),
+                            group.backup_id(),
+                            info.backup_dir.backup_time_string()
+                        );
+                        if !keep {
+                            datastore.remove_backup_dir(&info.backup_dir, true)?;
+                        }
+                    }
+                }
+                Ok(())
+            });
+
+            let status = worker.create_state(&result);
+
+            if let Err(err) = job.finish(status) {
+                eprintln!(
+                    "could not finish job state for {}: {}",
+                    job.jobtype().to_string(),
+                    err
+                );
+            }
+
+            result
+        },
+    )?;
+    Ok(upid_str)
+}
-- 
2.20.1

[pbs-devel] [PATCH proxmox-backup 2/2] proxy: extract commonly used logic for scheduling into new function

[Hannes Laimer]

Signed-off-by: Hannes Laimer <h.laimer@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 188 ++++++++++----------------------
 1 file changed, 56 insertions(+), 132 deletions(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 21c5e9fb..9222b075 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -402,48 +402,19 @@ async fn schedule_datastore_prune() {
             continue;
         }
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "prune";
+        if check_schedule(worker_type.to_string(), event_str.clone(), store.clone()) {
+            let job = match Job::new(worker_type, &store) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &store) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, store, err);
-                continue;
-            }
-        };
+            let userid = Userid::backup_userid().clone();
 
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
+            if let Err(err) = do_prune_job(job, prune_options, store.clone(), &userid, Some(event_str)) {
+                eprintln!("unable to start datastore prune job {} - {}", &store, err);
             }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now  { continue; }
-
-        let job = match Job::new(worker_type, &store) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let userid = Userid::backup_userid().clone();
-
-        if let Err(err) = do_prune_job(job, prune_options, store.clone(), &userid, Some(event_str)) {
-            eprintln!("unable to start datastore prune job {} - {}", &store, err);
         }
-
     }
 }
 
@@ -476,46 +447,18 @@ async fn schedule_datastore_sync_jobs() {
             None => continue,
         };
 
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
-
         let worker_type = "syncjob";
+        if check_schedule(worker_type.to_string(), event_str.clone(), job_id.clone()) {
+            let job = match Job::new(worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
 
-        let last = match jobstate::last_run_time(worker_type, &job_id) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
-                continue;
-            }
-        };
+            let userid = Userid::backup_userid().clone();
 
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
+            if let Err(err) = do_sync_job(job, job_config, &userid, Some(event_str)) {
+                eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
             }
-        };
-
-        let now = proxmox::tools::time::epoch_i64();
-
-        if next > now  { continue; }
-
-        let job = match Job::new(worker_type, &job_id) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-
-        let userid = Userid::backup_userid().clone();
-
-        if let Err(err) = do_sync_job(job, job_config, &userid, Some(event_str)) {
-            eprintln!("unable to start datastore sync job {} - {}", &job_id, err);
         }
     }
 }
@@ -546,38 +489,17 @@ async fn schedule_datastore_verify_jobs() {
             Some(ref event_str) => event_str.clone(),
             None => continue,
         };
-        let event = match parse_calendar_event(&event_str) {
-            Ok(event) => event,
-            Err(err) => {
-                eprintln!("unable to parse schedule '{}' - {}", event_str, err);
-                continue;
-            }
-        };
+
         let worker_type = "verificationjob";
-        let last = match jobstate::last_run_time(worker_type, &job_id) {
-            Ok(time) => time,
-            Err(err) => {
-                eprintln!("could not get last run time of {} {}: {}", worker_type, job_id, err);
-                continue;
-            }
-        };
-        let next = match compute_next_event(&event, last, false) {
-            Ok(Some(next)) => next,
-            Ok(None) => continue,
-            Err(err) => {
-                eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
-                continue;
+        if check_schedule(worker_type.to_string(), event_str.clone(), job_id.clone()) {
+            let job = match Job::new(&worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
+            let userid = Userid::backup_userid().clone();
+            if let Err(err) = do_verification_job(job, job_config, &userid, Some(event_str)) {
+                eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
             }
-        };
-        let now = proxmox::tools::time::epoch_i64();
-        if next > now { continue; }
-        let job = match Job::new(worker_type, &job_id) {
-            Ok(job) => job,
-            Err(_) => continue, // could not get lock
-        };
-        let userid = Userid::backup_userid().clone();
-        if let Err(err) = do_verification_job(job, job_config, &userid, Some(event_str)) {
-            eprintln!("unable to start datastore verification job {} - {}", &job_id, err);
         }
     }
 }
@@ -587,38 +509,10 @@ async fn schedule_task_log_rotate() {
     let worker_type = "logrotate";
     let job_id = "task_archive";
 
-    let last = match jobstate::last_run_time(worker_type, job_id) {
-        Ok(time) => time,
-        Err(err) => {
-            eprintln!("could not get last run time of task log archive rotation: {}", err);
-            return;
-        }
-    };
-
     // schedule daily at 00:00 like normal logrotate
     let schedule = "00:00";
 
-    let event = match parse_calendar_event(schedule) {
-        Ok(event) => event,
-        Err(err) => {
-            // should not happen?
-            eprintln!("unable to parse schedule '{}' - {}", schedule, err);
-            return;
-        }
-    };
-
-    let next = match compute_next_event(&event, last, false) {
-        Ok(Some(next)) => next,
-        Ok(None) => return,
-        Err(err) => {
-            eprintln!("compute_next_event for '{}' failed - {}", schedule, err);
-            return;
-        }
-    };
-
-    let now = proxmox::tools::time::epoch_i64();
-
-    if next > now {
+    if !check_schedule(worker_type.to_string(), schedule.to_string(), job_id.to_string()) {
         // if we never ran the rotation, schedule instantly
         match jobstate::JobState::load(worker_type, job_id) {
             Ok(state) => match state {
@@ -783,6 +677,36 @@ async fn generate_host_stats(save: bool) {
     });
 }
 
+fn check_schedule(worker_type: String, event_str: String, id: String) -> bool {
+    let event = match parse_calendar_event(&event_str) {
+        Ok(event) => event,
+        Err(err) => {
+            eprintln!("unable to parse schedule '{}' - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let last = match jobstate::last_run_time(&worker_type, &id) {
+        Ok(time) => time,
+        Err(err) => {
+            eprintln!("could not get last run time of {} {}: {}", worker_type, id, err);
+            return false;
+        }
+    };
+
+    let next = match compute_next_event(&event, last, false) {
+        Ok(Some(next)) => next,
+        Ok(None) => return false,
+        Err(err) => {
+            eprintln!("compute_next_event for '{}' failed - {}", event_str, err);
+            return false;
+        }
+    };
+
+    let now = proxmox::tools::time::epoch_i64();
+    next <= now
+}
+
 fn gather_disk_stats(disk_manager: Arc<DiskManage>, path: &Path, rrd_prefix: &str, save: bool) {
 
     match proxmox_backup::tools::disks::disk_usage(path) {
-- 
2.20.1

Re: [pbs-devel] [PATCH proxmox-backup 0/2] proxy: scheduling cleanup

[Dietmar Maurer]

Those patches are based on outdated code - please can you rebase them?

> On 10/29/2020 11:37 AM Hannes Laimer <h.laimer@proxmox.com> wrote:
> 
>  
> Logic for gc and logrotate was not moved, could (should?) also be moved later.
> 
>  * moved logic for prune out of proxy
>  * extract commonly used logic for scheduling into new function
> 
> 
> Hannes Laimer (2):
>   proxy: move prune logic into new file
>   proxy: extract commonly used logic for scheduling into new function
> 
>  src/bin/proxmox-backup-proxy.rs | 245 ++++++++------------------------
>  src/server.rs                   |   3 +
>  src/server/prune_job.rs         |  91 ++++++++++++
>  3 files changed, 151 insertions(+), 188 deletions(-)
>  create mode 100644 src/server/prune_job.rs
> 
> -- 
> 2.20.1
> 
> 
> 
> _______________________________________________
> pbs-devel mailing list
> pbs-devel@lists.proxmox.com
> https://lists.proxmox.com/cgi-bin/mailman/listinfo/pbs-devel