[pdm-devel] [PATCH datacenter-manager 2/2] server: add task log, auth log and access log rotation

[Lukas Wagner <l.wagner@proxmox.com>]

This adds a periodic task that runs once a day at midnight. This task
will rotate the task log archive, the auth log and also the access log
file.

Copied from PBS and adapted as needed. We don't have a place yet to
configure the 'max_days' parameter for rotating the task log archive, so
this one is always set to `None` at the moment.

Signed-off-by: Lukas Wagner <l.wagner@proxmox.com>
---
 server/src/bin/proxmox-datacenter-api/main.rs |   2 +-
 .../proxmox-datacenter-api/tasks/logrotate.rs | 195 ++++++++++++++++++
 .../bin/proxmox-datacenter-api/tasks/mod.rs   |   2 +
 .../bin/proxmox-datacenter-privileged-api.rs  |   5 +-
 4 files changed, 201 insertions(+), 3 deletions(-)
 create mode 100644 server/src/bin/proxmox-datacenter-api/tasks/logrotate.rs

diff --git a/server/src/bin/proxmox-datacenter-api/main.rs b/server/src/bin/proxmox-datacenter-api/main.rs
index 0e80c829..7459e68f 100644
--- a/server/src/bin/proxmox-datacenter-api/main.rs
+++ b/server/src/bin/proxmox-datacenter-api/main.rs
@@ -426,9 +426,9 @@ async fn run_task_scheduler() {
 
 async fn schedule_tasks() -> Result<(), Error> {
     // TODO: move out to own module, refactor PBS stuff for reuse & then add:
-    // - task log rotation
     // - stats (rrd) collection
     // - ...?
+    tasks::logrotate::schedule_task_log_rotate().await;
 
     Ok(())
 }
diff --git a/server/src/bin/proxmox-datacenter-api/tasks/logrotate.rs b/server/src/bin/proxmox-datacenter-api/tasks/logrotate.rs
new file mode 100644
index 00000000..6afb069b
--- /dev/null
+++ b/server/src/bin/proxmox-datacenter-api/tasks/logrotate.rs
@@ -0,0 +1,195 @@
+use anyhow::{format_err, Error};
+
+use proxmox_lang::try_block;
+use proxmox_rest_server::WorkerTask;
+use proxmox_sys::logrotate::LogRotate;
+use proxmox_time::CalendarEvent;
+
+use pdm_api_types::Authid;
+use server::jobstate::{self, Job, JobState};
+
+/// Rotate task logs, auth logs and access logs.
+///
+/// This task runs every day at midnight, except when it has never run before, then it runs
+/// immediately.
+pub async fn schedule_task_log_rotate() {
+    let worker_type = "logrotate";
+    let job_id = "access-log_and_task-archive";
+
+    // schedule daily at 00:00 like normal logrotate
+    let schedule = "00:00";
+
+    if !check_schedule(worker_type, schedule, job_id) {
+        // if we never ran the rotation, schedule instantly
+        match JobState::load(worker_type, job_id) {
+            Ok(JobState::Created { .. }) => {}
+            _ => return,
+        }
+    }
+
+    let mut job = match Job::new(worker_type, job_id) {
+        Ok(job) => job,
+        Err(_) => return, // could not get lock
+    };
+
+    if let Err(err) = WorkerTask::new_thread(
+        worker_type,
+        None,
+        Authid::root_auth_id().to_string(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+            proxmox_log::info!("starting task log rotation");
+
+            let result = try_block!({
+                let max_size = 512 * 1024 - 1; // an entry has ~ 100b, so > 5000 entries/file
+                let max_files = 20; // times twenty files gives > 100000 task entries
+
+                // TODO: Make this configurable
+                let max_days = None;
+
+                let options = proxmox_product_config::default_create_options();
+
+                let has_rotated = proxmox_rest_server::rotate_task_log_archive(
+                    max_size,
+                    true,
+                    Some(max_files),
+                    max_days,
+                    Some(options),
+                )?;
+
+                if has_rotated {
+                    log::info!("task log archive was rotated");
+                } else {
+                    log::info!("task log archive was not rotated");
+                }
+
+                let max_size = 32 * 1024 * 1024 - 1;
+                let max_files = 14;
+
+                let mut logrotate = LogRotate::new(
+                    pdm_buildcfg::API_ACCESS_LOG_FN,
+                    true,
+                    Some(max_files),
+                    Some(options),
+                )?;
+
+                if logrotate.rotate(max_size)? {
+                    println!("rotated access log, telling daemons to re-open log file");
+                    proxmox_async::runtime::block_on(command_reopen_access_logfiles())?;
+                    log::info!("API access log was rotated");
+                } else {
+                    log::info!("API access log was not rotated");
+                }
+
+                let mut logrotate = LogRotate::new(
+                    pdm_buildcfg::API_AUTH_LOG_FN,
+                    true,
+                    Some(max_files),
+                    Some(options),
+                )?;
+
+                if logrotate.rotate(max_size)? {
+                    println!("rotated auth log, telling daemons to re-open log file");
+                    proxmox_async::runtime::block_on(command_reopen_auth_logfiles())?;
+                    log::info!("API authentication log was rotated");
+                } else {
+                    log::info!("API authentication log was not rotated");
+                }
+
+                if has_rotated {
+                    log::info!("cleaning up old task logs");
+                    if let Err(err) = proxmox_rest_server::cleanup_old_tasks(true) {
+                        log::warn!("could not completely cleanup old tasks: {err}");
+                    }
+                }
+
+                Ok(())
+            });
+
+            let status = worker.create_state(&result);
+
+            if let Err(err) = job.finish(status) {
+                eprintln!("could not finish job state for {worker_type}: {err}");
+            }
+
+            result
+        },
+    ) {
+        eprintln!("unable to start task log rotation: {err}");
+    }
+}
+
+async fn command_reopen_access_logfiles() -> Result<(), Error> {
+    // only care about the most recent daemon instance for each, proxy & api, as other older ones
+    // should not respond to new requests anyway, but only finish their current one and then exit.
+    let sock = proxmox_daemon::command_socket::this_path();
+    let f1 =
+        proxmox_daemon::command_socket::send_raw(sock, "{\"command\":\"api-access-log-reopen\"}\n");
+
+    let pid = proxmox_rest_server::read_pid(pdm_buildcfg::PDM_API_PID_FN)?;
+    let sock = proxmox_daemon::command_socket::path_from_pid(pid);
+    let f2 =
+        proxmox_daemon::command_socket::send_raw(sock, "{\"command\":\"api-access-log-reopen\"}\n");
+
+    match futures::join!(f1, f2) {
+        (Err(e1), Err(e2)) => Err(format_err!(
+            "reopen commands failed, proxy: {e1}; api: {e2}"
+        )),
+        (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {e1}")),
+        (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {e2}")),
+        _ => Ok(()),
+    }
+}
+
+async fn command_reopen_auth_logfiles() -> Result<(), Error> {
+    // only care about the most recent daemon instance for each, proxy & api, as other older ones
+    // should not respond to new requests anyway, but only finish their current one and then exit.
+    let sock = proxmox_daemon::command_socket::this_path();
+    let f1 =
+        proxmox_daemon::command_socket::send_raw(sock, "{\"command\":\"api-auth-log-reopen\"}\n");
+
+    let pid = proxmox_rest_server::read_pid(pdm_buildcfg::PDM_API_PID_FN)?;
+    let sock = proxmox_daemon::command_socket::path_from_pid(pid);
+    let f2 =
+        proxmox_daemon::command_socket::send_raw(sock, "{\"command\":\"api-auth-log-reopen\"}\n");
+
+    match futures::join!(f1, f2) {
+        (Err(e1), Err(e2)) => Err(format_err!(
+            "reopen commands failed, proxy: {e1}; api: {e2}"
+        )),
+        (Err(e1), Ok(_)) => Err(format_err!("reopen commands failed, proxy: {e1}")),
+        (Ok(_), Err(e2)) => Err(format_err!("reopen commands failed, api: {e2}")),
+        _ => Ok(()),
+    }
+}
+
+fn check_schedule(worker_type: &str, event_str: &str, id: &str) -> bool {
+    let event: CalendarEvent = match event_str.parse() {
+        Ok(event) => event,
+        Err(err) => {
+            eprintln!("unable to parse schedule '{event_str}' - {err}");
+            return false;
+        }
+    };
+
+    let last = match jobstate::last_run_time(worker_type, id) {
+        Ok(time) => time,
+        Err(err) => {
+            eprintln!("could not get last run time of {worker_type} {id}: {err}");
+            return false;
+        }
+    };
+
+    let next = match event.compute_next_event(last) {
+        Ok(Some(next)) => next,
+        Ok(None) => return false,
+        Err(err) => {
+            eprintln!("compute_next_event for '{event_str}' failed - {err}");
+            return false;
+        }
+    };
+
+    let now = proxmox_time::epoch_i64();
+    next <= now
+}
diff --git a/server/src/bin/proxmox-datacenter-api/tasks/mod.rs b/server/src/bin/proxmox-datacenter-api/tasks/mod.rs
index f4d1d3a1..b6c3a70c 100644
--- a/server/src/bin/proxmox-datacenter-api/tasks/mod.rs
+++ b/server/src/bin/proxmox-datacenter-api/tasks/mod.rs
@@ -1,3 +1,5 @@
+pub mod logrotate;
+
 pub mod remote_node_mapping;
 pub mod remote_tasks;
 pub mod remote_updates;
diff --git a/server/src/bin/proxmox-datacenter-privileged-api.rs b/server/src/bin/proxmox-datacenter-privileged-api.rs
index 14ff4bf4..9b0a037b 100644
--- a/server/src/bin/proxmox-datacenter-privileged-api.rs
+++ b/server/src/bin/proxmox-datacenter-privileged-api.rs
@@ -30,10 +30,10 @@ fn main() -> Result<(), Error> {
         .tasklog_pbs()
         .init()?;
 
-    create_directories()?;
-
     proxmox_product_config::init(pdm_config::api_user()?, pdm_config::priv_user()?);
 
+    create_directories()?;
+
     let mut args = std::env::args();
     args.next();
     for arg in args {
@@ -63,6 +63,7 @@ fn create_directories() -> Result<(), Error> {
     let api_user = pdm_config::api_user()?;
 
     pdm_config::setup::create_configdir()?;
+    server::jobstate::create_jobstate_dir()?;
 
     pdm_config::setup::mkdir_perms(
         pdm_buildcfg::PDM_RUN_DIR,
-- 
2.47.3



_______________________________________________
pdm-devel mailing list
pdm-devel@lists.proxmox.com
https://lists.proxmox.com/cgi-bin/mailman/listinfo/pdm-devel