[PATCH 4/7] client: job: add job management CLI subcommands

[Thomas Lamprecht <t.lamprecht@proxmox.com>]

Add `job create`, `update`, `remove`, `enable`, `disable`, `list`
and `show`.

When a change leaves a job that should run but no scheduler is running,
the commands print how to start one, as nothing runs without it. A
running scheduler picks up changes within a minute.

Signed-off-by: Thomas Lamprecht <t.lamprecht@proxmox.com>
---
 proxmox-backup-client/src/job/mod.rs       | 358 ++++++++++++++++++++-
 proxmox-backup-client/src/job/scheduler.rs |  15 +
 2 files changed, 366 insertions(+), 7 deletions(-)

diff --git a/proxmox-backup-client/src/job/mod.rs b/proxmox-backup-client/src/job/mod.rs
index ffde90ad1..858dc2250 100644
--- a/proxmox-backup-client/src/job/mod.rs
+++ b/proxmox-backup-client/src/job/mod.rs
@@ -1,20 +1,364 @@
 //! Scheduled backup jobs: the `job` subcommands, the [`scheduler`] daemon and the [`runner`].
 
+use anyhow::{Error, bail, format_err};
+use serde_json::Value;
+
 use proxmox_router::cli::*;
+use proxmox_schema::{ApiType, ReturnType, api};
 
-use pbs_client::client_job::complete_job_id;
+use pbs_api_types::JOB_ID_SCHEMA;
+use pbs_client::client_job::{
+    self, ClientBackupJobConfig, ClientBackupJobConfigUpdater, ClientBackupJobStatus,
+    DeletableProperty, complete_job_id,
+};
 
 mod runner;
 mod scheduler;
 
+/// Refuse a schedule without any event left. It is valid syntax, but the job would never run.
+fn check_schedule_ahead(schedule: &str) -> Result<(), Error> {
+    let event: proxmox_time::CalendarEvent = schedule.parse()?;
+    if event
+        .compute_next_event(proxmox_time::epoch_i64())?
+        .is_none()
+    {
+        bail!("schedule '{schedule}' has no future event, the job would never run");
+    }
+    Ok(())
+}
+
+/// Render the backup target for the overview: a repository URL as configured, or the server and
+/// datastore of the atom form. Deliberately lossy, `job show` has the exact values.
+fn render_target(_value: &Value, record: &Value) -> Result<String, Error> {
+    if let Some(repository) = record.get("repository").and_then(Value::as_str) {
+        return Ok(repository.to_string());
+    }
+    let server = record
+        .get("server")
+        .and_then(Value::as_str)
+        .unwrap_or("localhost");
+    let store = record
+        .get("datastore")
+        .and_then(Value::as_str)
+        .unwrap_or("");
+    Ok(format!("{server}:{store}"))
+}
+
+/// Render when a job runs next, or why it does not.
+fn render_next_run(value: &Value, record: &Value) -> Result<String, Error> {
+    if record["disable"].as_bool().unwrap_or(false) {
+        return Ok("disabled".to_string());
+    }
+    if value.is_null() {
+        // an enabled job without a next run has a schedule that ran out
+        return Ok("never".to_string());
+    }
+    pbs_tools::format::render_epoch(value, record)
+}
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        },
+    },
+    returns: {
+        description: "List of configured backup jobs.",
+        type: Array,
+        items: { type: ClientBackupJobStatus },
+    },
+)]
+/// List the backup jobs, with their last and next run.
+fn list_jobs(param: Value) -> Result<Value, Error> {
+    let output_format = get_output_format(&param);
+    let (config, _digest) = client_job::config()?;
+
+    let mut jobs = Vec::with_capacity(config.sections.len());
+    for id in client_job::job_ids(&config) {
+        let job = client_job::lookup_job(&config, id)?;
+        jobs.push(ClientBackupJobStatus::with_state(job));
+    }
+
+    let options = default_table_format_options()
+        .column(ColumnConfig::new("id"))
+        .column(ColumnConfig::new("schedule"))
+        .column(
+            ColumnConfig::new("repository")
+                .header("target")
+                .renderer(render_target),
+        )
+        .column(ColumnConfig::new("last-run").renderer(pbs_tools::format::render_epoch))
+        .column(ColumnConfig::new("last-result"))
+        .column(ColumnConfig::new("next-run").renderer(render_next_run));
+
+    let mut data = serde_json::to_value(jobs)?;
+    format_and_print_result_full(
+        &mut data,
+        &API_METHOD_LIST_JOBS.returns,
+        &output_format,
+        &options,
+    );
+
+    Ok(Value::Null)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        },
+    },
+)]
+/// Show a backup job, with its last and next run.
+fn show_job(id: String, param: Value) -> Result<Value, Error> {
+    let output_format = get_output_format(&param);
+    let (config, _digest) = client_job::config()?;
+    let job = client_job::lookup_job(&config, &id)?;
+    let mut data = serde_json::to_value(ClientBackupJobStatus::with_state(job))?;
+
+    let return_type = ReturnType::new(false, &ClientBackupJobStatus::API_SCHEMA);
+    format_and_print_result_full(&mut data, &return_type, &output_format, &show_options());
+
+    Ok(Value::Null)
+}
+
+fn status_schema() -> &'static dyn proxmox_schema::ObjectSchemaType {
+    ClientBackupJobStatus::API_SCHEMA
+        .any_object()
+        .expect("job status schema is an object schema")
+}
+
+/// Columns for `job show`: every status property, with the job's identity first and its runs last.
+fn show_options() -> TableFormatOptions {
+    const LEADING: [&str; 2] = ["id", "schedule"];
+    const TRAILING: [&str; 4] = ["last-run", "last-result", "message", "next-run"];
+
+    let mut names: Vec<&str> = LEADING.to_vec();
+    names.extend(
+        status_schema()
+            .properties()
+            .map(|(name, _optional, _schema)| *name)
+            .filter(|name| !LEADING.contains(name) && !TRAILING.contains(name)),
+    );
+    names.extend(TRAILING);
+
+    let mut options = default_table_format_options();
+    for name in names {
+        let column = ColumnConfig::new(name);
+        options = match name {
+            "last-run" | "next-run" => {
+                options.column(column.renderer(pbs_tools::format::render_epoch))
+            }
+            "backupspec" | "exclude" | "include-dev" => {
+                options.column(column.renderer(render_string_list))
+            }
+            _ => options.column(column),
+        };
+    }
+    options
+}
+
+/// Render a list of strings one per line, rather than as a JSON array.
+fn render_string_list(value: &Value, _record: &Value) -> Result<String, Error> {
+    let Some(items) = value.as_array() else {
+        return Ok(String::new());
+    };
+    Ok(items
+        .iter()
+        .map(|item| item.as_str().unwrap_or_default())
+        .collect::<Vec<&str>>()
+        .join("\n"))
+}
+
+#[api(
+    input: {
+        properties: {
+            config: {
+                type: ClientBackupJobConfig,
+                flatten: true,
+            },
+        },
+    },
+)]
+/// Create a backup job.
+fn create_job(mut config: ClientBackupJobConfig) -> Result<(), Error> {
+    config.normalize();
+    config.validate()?;
+    check_schedule_ahead(&config.schedule)?;
+
+    let _lock = client_job::lock_config()?;
+    let (mut section_config, _digest) = client_job::config()?;
+
+    if section_config.sections.contains_key(&config.id) {
+        bail!("job '{}' already exists", config.id);
+    }
+
+    // state first, as a job without state is due at once
+    client_job::create_state(&config.id)?;
+
+    section_config.set_data(&config.id, client_job::JOB_SECTION_TYPE, &config)?;
+    client_job::save_config(&section_config)?;
+
+    if !config.disable.unwrap_or(false) {
+        scheduler::hint_if_not_running();
+    }
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            update: {
+                type: ClientBackupJobConfigUpdater,
+                flatten: true,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                },
+            },
+        },
+    },
+)]
+/// Update a backup job. Options that are not given keep their value.
+fn update_job(
+    id: String,
+    update: ClientBackupJobConfigUpdater,
+    delete: Option<Vec<DeletableProperty>>,
+) -> Result<(), Error> {
+    let _lock = client_job::lock_config()?;
+    let (mut section_config, _digest) = client_job::config()?;
+
+    let mut job = client_job::lookup_job(&section_config, &id)?;
+    // only a new schedule, so that a one-off job that already ran stays editable
+    let new_schedule = update.schedule.is_some();
+    job.apply_update(update, &delete.unwrap_or_default());
+    job.validate()?;
+    if new_schedule {
+        check_schedule_ahead(&job.schedule)?;
+    }
+
+    section_config.set_data(&id, client_job::JOB_SECTION_TYPE, &job)?;
+    client_job::save_config(&section_config)?;
+
+    if !job.disable.unwrap_or(false) {
+        scheduler::hint_if_not_running();
+    }
+
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            id: { schema: JOB_ID_SCHEMA },
+        },
+    },
+)]
+/// Enable a disabled backup job.
+fn enable_job(id: String) -> Result<(), Error> {
+    set_disable(&id, None)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: { schema: JOB_ID_SCHEMA },
+        },
+    },
+)]
+/// Disable a backup job. The scheduler no longer starts it, but 'job run' still does.
+fn disable_job(id: String) -> Result<(), Error> {
+    set_disable(&id, Some(true))
+}
+
+/// Enabling drops the property instead of storing `false`, so that an enabled job looks the same
+/// however it got there.
+fn set_disable(id: &str, value: Option<bool>) -> Result<(), Error> {
+    let _lock = client_job::lock_config()?;
+    let (mut section_config, _digest) = client_job::config()?;
+    let mut job = client_job::lookup_job(&section_config, id)?;
+    job.disable = value;
+    section_config.set_data(id, client_job::JOB_SECTION_TYPE, &job)?;
+    client_job::save_config(&section_config)?;
+    if value.is_none() {
+        scheduler::hint_if_not_running();
+    }
+    Ok(())
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+)]
+/// Remove a backup job.
+fn remove_job(id: String) -> Result<(), Error> {
+    let _lock = client_job::lock_config()?;
+    let (mut section_config, _digest) = client_job::config()?;
+    client_job::lookup_job(&section_config, &id)?;
+    if client_job::is_running(&id) {
+        bail!("cannot remove job '{id}' while it is running");
+    }
+
+    // the held job lock makes it safe to drop the job's files
+    let job_lock =
+        client_job::ClientJob::new(&id).map_err(|err| format_err!("cannot remove job: {err}"))?;
+
+    section_config.sections.remove(&id);
+    client_job::save_config(&section_config)?;
+    // only now, as a job without state would be due at once
+    client_job::remove_state(&job_lock)?;
+
+    Ok(())
+}
+
 pub fn cli() -> CliCommandMap {
+    let with_id = |info| {
+        CliCommand::new(info)
+            .arg_param(&["id"])
+            .completion_cb("id", complete_job_id)
+    };
+    let with_files = |command: CliCommand| {
+        command
+            .completion_cb("keyfile", complete_file_name)
+            .completion_cb("master-pubkey-file", complete_file_name)
+            .completion_cb("password-file", complete_file_name)
+            .completion_cb("encryption-password-file", complete_file_name)
+            .completion_cb("pre-run-hook", complete_file_name)
+    };
+    // no ID completion for `create`, which needs a new one
+    let create = with_files(CliCommand::new(&API_METHOD_CREATE_JOB).arg_param(&["id"]));
+
     CliCommandMap::new()
-        .insert(
-            "run",
-            CliCommand::new(&runner::API_METHOD_RUN_JOB)
-                .arg_param(&["id"])
-                .completion_cb("id", complete_job_id),
-        )
+        .insert("create", create)
+        .insert("update", with_files(with_id(&API_METHOD_UPDATE_JOB)))
+        .insert("remove", with_id(&API_METHOD_REMOVE_JOB))
+        .insert("enable", with_id(&API_METHOD_ENABLE_JOB))
+        .insert("disable", with_id(&API_METHOD_DISABLE_JOB))
+        .insert("list", CliCommand::new(&API_METHOD_LIST_JOBS))
+        .insert("show", with_id(&API_METHOD_SHOW_JOB))
+        .insert("run", with_id(&runner::API_METHOD_RUN_JOB))
         .insert(
             "scheduler",
             CliCommand::new(&scheduler::API_METHOD_RUN_SCHEDULER),
diff --git a/proxmox-backup-client/src/job/scheduler.rs b/proxmox-backup-client/src/job/scheduler.rs
index e8eaa7d0f..1c539bf52 100644
--- a/proxmox-backup-client/src/job/scheduler.rs
+++ b/proxmox-backup-client/src/job/scheduler.rs
@@ -325,3 +325,18 @@ fn acquire_scheduler_pidfile(path: &Path) -> Result<Flock<std::fs::File>, Error>
     write!(locked, "{}", std::process::id())?;
     Ok(locked)
 }
+
+/// Name of the systemd user service shipped for the scheduler.
+const SCHEDULER_UNIT: &str = "proxmox-backup-client-scheduler";
+
+/// Tell the user how to start a scheduler if none is running, as no job runs without one.
+pub(super) fn hint_if_not_running() {
+    if scheduler_pid_path().is_ok_and(|path| client_job::is_locked(&path)) {
+        return;
+    }
+    log::info!(
+        "note: no backup job scheduler is running, so jobs will not run. Enable it with\n  \
+        systemctl --user enable --now {SCHEDULER_UNIT}\n\
+        or run 'proxmox-backup-client job scheduler' in the foreground."
+    );
+}
-- 
2.47.3