[pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs

[pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs

[Christian Ebner]

This series of patches implements the logic to run regular sanity
check job, with the intention to check various states of e.g.
datastores, verification states of backups, possibly missing jobs
for datastores, uncovered namespace, load metrics exceeding some
threshold values, ecc. and allow to send a user notification on
success or error state, in an automated fashion.

Currently, only a datastore usage check is implemented as prove of
concept.

The intenition of this patch series is to get some feedback on the
implementation approach and find possible shortcommings in design
decisions.

Christian Ebner (8):
  api-types: jobs: add sanity checks job types
  config: implement sanity check job configuration
  api: config: sanity check jobs api endpoints
  server: add sanity check job email notifications
  server: implement sanity check job
  api: admin: add sanity check job api endpoints
  manager: add sanity check jobs management cli commands
  proxy: add sanity check task to scheduler

 pbs-api-types/src/jobs.rs                     | 106 +++++++
 pbs-config/src/lib.rs                         |   1 +
 pbs-config/src/sanity_check.rs                |  57 ++++
 src/api2/admin/mod.rs                         |   2 +
 src/api2/admin/sanity_check.rs                | 111 +++++++
 src/api2/config/mod.rs                        |   2 +
 src/api2/config/sanity_check.rs               | 296 ++++++++++++++++++
 src/bin/proxmox-backup-manager.rs             |   3 +-
 src/bin/proxmox-backup-proxy.rs               |  41 ++-
 src/bin/proxmox_backup_manager/mod.rs         |   2 +
 .../proxmox_backup_manager/sanity_check.rs    | 126 ++++++++
 src/server/email_notifications.rs             |  78 +++++
 src/server/mod.rs                             |   3 +
 src/server/sanity_check_job.rs                | 131 ++++++++
 14 files changed, 957 insertions(+), 2 deletions(-)
 create mode 100644 pbs-config/src/sanity_check.rs
 create mode 100644 src/api2/admin/sanity_check.rs
 create mode 100644 src/api2/config/sanity_check.rs
 create mode 100644 src/bin/proxmox_backup_manager/sanity_check.rs
 create mode 100644 src/server/sanity_check_job.rs

-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 1/8] api-types: jobs: add sanity checks job types

[Christian Ebner]

Defines the required types for managing sanity check jobs via the API.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-api-types/src/jobs.rs | 106 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs
index 1f5b3cf1..4f6b2cc7 100644
--- a/pbs-api-types/src/jobs.rs
+++ b/pbs-api-types/src/jobs.rs
@@ -57,12 +57,28 @@ pub const VERIFICATION_SCHEDULE_SCHEMA: Schema =
         .type_text("<calendar-event>")
         .schema();
 
+pub const SANITY_CHECK_SCHEDULE_SCHEMA: Schema =
+    StringSchema::new("Run sanity check job at specified schedule.")
+        .format(&ApiStringFormat::VerifyFn(
+            proxmox_time::verify_calendar_event,
+        ))
+        .type_text("<calendar-event>")
+        .schema();
+
 pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
     "Delete vanished backups. This remove the local copy if the remote backup was deleted.",
 )
 .default(false)
 .schema();
 
+pub const DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT: u8 = 90;
+pub const DATASTORE_USAGE_FULL_THRESHOLD_SCHEMA: Schema =
+    IntegerSchema::new("Datastore usage threshold level in percent for the datastore being full.")
+        .minimum(1)
+        .maximum(100)
+        .default(DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT as isize)
+        .schema();
+
 #[api(
     properties: {
         "next-run": {
@@ -753,3 +769,93 @@ pub struct PruneJobStatus {
     #[serde(flatten)]
     pub status: JobScheduleStatus,
 }
+
+#[api(
+    properties: {
+        "datastore-usage-full-threshold": {
+            schema: DATASTORE_USAGE_FULL_THRESHOLD_SCHEMA,
+            optional: true,
+        },
+        "notify-user": {
+            optional: true,
+            type: Userid,
+        },
+    }
+)]
+#[derive(Serialize, Deserialize, Default, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Sanity check options
+pub struct SanityCheckJobOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub datastore_usage_full_threshold: Option<u8>,
+    /// Send job email notification to this user
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub notify_user: Option<Userid>,
+}
+
+#[api(
+    properties: {
+        id: {
+            schema: JOB_ID_SCHEMA,
+        },
+        disable: {
+            type: Boolean,
+            optional: true,
+            default: false,
+        },
+        schedule: {
+            schema: SANITY_CHECK_SCHEDULE_SCHEMA,
+        },
+        comment: {
+            optional: true,
+            schema: SINGLE_LINE_COMMENT_SCHEMA,
+        },
+        options: {
+            type: SanityCheckJobOptions,
+        },
+    },
+)]
+#[derive(Deserialize, Serialize, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Sanity check configuration.
+pub struct SanityCheckJobConfig {
+    /// Unique ID to address this job
+    #[updater(skip)]
+    pub id: String,
+
+    /// Flag to disable job
+    #[serde(default, skip_serializing_if = "is_false")]
+    #[updater(serde(skip_serializing_if = "Option::is_none"))]
+    pub disable: bool,
+
+    /// When to schedule this job in calendar event notation
+    pub schedule: String,
+
+    /// Additional comment for this job
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub comment: Option<String>,
+
+    /// Configuration options for this job
+    #[serde(flatten)]
+    pub options: SanityCheckJobOptions,
+}
+
+#[api(
+    properties: {
+        config: {
+            type: SanityCheckJobConfig,
+        },
+        status: {
+            type: JobScheduleStatus,
+        },
+    },
+)]
+#[derive(Serialize, Deserialize, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Status of sanity check job
+pub struct SanityCheckJobStatus {
+    #[serde(flatten)]
+    pub config: SanityCheckJobConfig,
+    #[serde(flatten)]
+    pub status: JobScheduleStatus,
+}
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 2/8] config: implement sanity check job configuration

[Christian Ebner]

Implements the required methods and defines the location for storing
sanity check jobs configurations.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 pbs-config/src/lib.rs          |  1 +
 pbs-config/src/sanity_check.rs | 57 ++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 pbs-config/src/sanity_check.rs

diff --git a/pbs-config/src/lib.rs b/pbs-config/src/lib.rs
index 009c4d3c..dad9225e 100644
--- a/pbs-config/src/lib.rs
+++ b/pbs-config/src/lib.rs
@@ -9,6 +9,7 @@ pub mod metrics;
 pub mod network;
 pub mod prune;
 pub mod remote;
+pub mod sanity_check;
 pub mod sync;
 pub mod tape_job;
 pub mod token_shadow;
diff --git a/pbs-config/src/sanity_check.rs b/pbs-config/src/sanity_check.rs
new file mode 100644
index 00000000..e875733b
--- /dev/null
+++ b/pbs-config/src/sanity_check.rs
@@ -0,0 +1,57 @@
+use std::collections::HashMap;
+
+use anyhow::Error;
+use lazy_static::lazy_static;
+
+use proxmox_schema::*;
+use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
+
+use pbs_api_types::{SanityCheckJobConfig, JOB_ID_SCHEMA};
+
+use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
+
+lazy_static! {
+    pub static ref CONFIG: SectionConfig = init();
+}
+
+fn init() -> SectionConfig {
+    const OBJ_SCHEMA: &AllOfSchema = SanityCheckJobConfig::API_SCHEMA.unwrap_all_of_schema();
+
+    let plugin =
+        SectionConfigPlugin::new("sanity-check".to_string(), Some(String::from("id")), OBJ_SCHEMA);
+    let mut config = SectionConfig::new(&JOB_ID_SCHEMA);
+    config.register_plugin(plugin);
+
+    config
+}
+
+pub const SANITY_CHECK_CFG_FILENAME: &str = "/etc/proxmox-backup/sanity-check.cfg";
+pub const SANITY_CHECK_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.sanity-check.lck";
+
+/// Get exclusive lock
+pub fn lock_config() -> Result<BackupLockGuard, Error> {
+    open_backup_lockfile(SANITY_CHECK_CFG_LOCKFILE, None, true)
+}
+
+pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
+    let content = proxmox_sys::fs::file_read_optional_string(SANITY_CHECK_CFG_FILENAME)?;
+    let content = content.unwrap_or_default();
+
+    let digest = openssl::sha::sha256(content.as_bytes());
+    let data = CONFIG.parse(SANITY_CHECK_CFG_FILENAME, &content)?;
+
+    Ok((data, digest))
+}
+
+pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
+    let raw = CONFIG.write(SANITY_CHECK_CFG_FILENAME, config)?;
+    replace_backup_config(SANITY_CHECK_CFG_FILENAME, raw.as_bytes())
+}
+
+// shell completion helper
+pub fn complete_sanity_check_job_id(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
+    match config() {
+        Ok((data, _digest)) => data.sections.keys().map(|id| id.to_string()).collect(),
+        Err(_) => Vec::new(),
+    }
+}
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 3/8] api: config: sanity check jobs api endpoints

[Christian Ebner]

Adds the api endpoints to create, read/list, update and delete sanity
check configurations.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/api2/config/mod.rs          |   2 +
 src/api2/config/sanity_check.rs | 296 ++++++++++++++++++++++++++++++++
 2 files changed, 298 insertions(+)
 create mode 100644 src/api2/config/sanity_check.rs

diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 6cfeaea1..8409ad80 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -13,6 +13,7 @@ pub mod media_pool;
 pub mod metrics;
 pub mod prune;
 pub mod remote;
+pub mod sanity_check;
 pub mod sync;
 pub mod tape_backup_job;
 pub mod tape_encryption_keys;
@@ -30,6 +31,7 @@ const SUBDIRS: SubdirMap = &sorted!([
     ("metrics", &metrics::ROUTER),
     ("prune", &prune::ROUTER),
     ("remote", &remote::ROUTER),
+    ("sanity-check", &sync::ROUTER),
     ("sync", &sync::ROUTER),
     ("tape-backup-job", &tape_backup_job::ROUTER),
     ("tape-encryption-keys", &tape_encryption_keys::ROUTER),
diff --git a/src/api2/config/sanity_check.rs b/src/api2/config/sanity_check.rs
new file mode 100644
index 00000000..c708b952
--- /dev/null
+++ b/src/api2/config/sanity_check.rs
@@ -0,0 +1,296 @@
+use anyhow::Error;
+use hex::FromHex;
+use proxmox_sys::task_log;
+use proxmox_sys::WorkerTaskContext;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
+use proxmox_schema::{api, param_bail};
+
+use pbs_api_types::{
+    Authid, SanityCheckJobConfig, SanityCheckJobConfigUpdater, JOB_ID_SCHEMA, PRIV_SYS_AUDIT,
+    PRIV_SYS_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA,
+};
+use pbs_config::{sanity_check, CachedUserInfo};
+
+#[api(
+    input: {
+        properties: {},
+    },
+    returns: {
+        description: "List configured sanity checks schedules.",
+        type: Array,
+        items: { type: SanityCheckJobConfig },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Audit.",
+    },
+)]
+/// List all scheduled sanity check jobs.
+pub fn list_sanity_check_jobs(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<SanityCheckJobConfig>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT | PRIV_SYS_MODIFY, true)?;
+
+    let (config, digest) = sanity_check::config()?;
+    let list = config.convert_to_typed_array("sanity-check")?;
+
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(list)
+}
+
+pub fn do_create_sanity_check_job(
+    config: SanityCheckJobConfig,
+    worker: Option<&dyn WorkerTaskContext>,
+) -> Result<(), Error> {
+    let _lock = sanity_check::lock_config()?;
+
+    let (mut section_config, _digest) = sanity_check::config()?;
+
+    if section_config.sections.get(&config.id).is_some() {
+        param_bail!("id", "job '{}' already exists.", config.id);
+    }
+
+    section_config.set_data(&config.id, "sanity-check", &config)?;
+
+    sanity_check::save_config(&section_config)?;
+
+    crate::server::jobstate::create_state_file("sanitycheckjob", &config.id)?;
+
+    if let Some(worker) = worker {
+        task_log!(worker, "Sanity check job created: {}", config.id);
+    }
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            config: {
+                type: SanityCheckJobConfig,
+                flatten: true,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Modify.",
+    },
+)]
+/// Create a new sanity check job.
+pub fn create_sanity_check_job(
+    config: SanityCheckJobConfig,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+    do_create_sanity_check_job(config, None)
+}
+
+#[api(
+   input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+        },
+    },
+    returns: { type: SanityCheckJobConfig },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Audit.",
+    },
+)]
+/// Read a sanity check job configuration.
+pub fn read_sanity_check_job(
+    id: String,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<SanityCheckJobConfig, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT, true)?;
+
+    let (config, digest) = sanity_check::config()?;
+    let sanity_check_job: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(sanity_check_job)
+}
+
+#[api]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+    /// Delete the comment.
+    Comment,
+    /// Unset the disable flag.
+    Disable,
+    /// Unset the notify-user.
+    NotifyUser,
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            update: {
+                type: SanityCheckJobConfigUpdater,
+                flatten: true,
+            },
+            delete: {
+                description: "List of properties to delete.",
+                type: Array,
+                optional: true,
+                items: {
+                    type: DeletableProperty,
+                }
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Modify.",
+    },
+)]
+/// Update sanity check job config.
+#[allow(clippy::too_many_arguments)]
+pub fn update_sanity_check_job(
+    id: String,
+    update: SanityCheckJobConfigUpdater,
+    delete: Option<Vec<DeletableProperty>>,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+    let _lock = sanity_check::lock_config()?;
+
+    // pass/compare digest
+    let (mut config, expected_digest) = sanity_check::config()?;
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    let mut data: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+    if let Some(delete) = delete {
+        for delete_prop in delete {
+            match delete_prop {
+                DeletableProperty::Comment => {
+                    data.comment = None;
+                }
+                DeletableProperty::Disable => {
+                    data.disable = false;
+                }
+                DeletableProperty::NotifyUser => {
+                    data.options.notify_user = None;
+                }
+            }
+        }
+    }
+
+    let mut schedule_changed = false;
+    if let Some(schedule) = update.schedule {
+        schedule_changed = data.schedule != schedule;
+        data.schedule = schedule;
+    }
+
+    if let Some(value) = update.comment {
+        data.comment = Some(value);
+    }
+    if let Some(value) = update.disable {
+        data.disable = value;
+    }
+    if let Some(value) = update.options.notify_user {
+        data.options.notify_user = Some(value);
+    }
+
+    config.set_data(&id, "sanity-check", &data)?;
+
+    sanity_check::save_config(&config)?;
+
+    if schedule_changed {
+        crate::server::jobstate::update_job_last_run_time("sanitycheckjob", &id)?;
+    }
+
+    Ok(())
+}
+
+#[api(
+    protected: true,
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            digest: {
+                optional: true,
+                schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+            },
+        },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Modify.",
+    },
+)]
+/// Remove a sanity check job configuration
+pub fn delete_sanity_check_job(
+    id: String,
+    digest: Option<String>,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+    let _lock = sanity_check::lock_config()?;
+
+    let (mut config, expected_digest) = sanity_check::config()?;
+    if let Some(ref digest) = digest {
+        let digest = <[u8; 32]>::from_hex(digest)?;
+        crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+    }
+
+    if config.sections.remove(&id).is_none() {
+        http_bail!(NOT_FOUND, "job '{}' does not exist.", id);
+    }
+
+    sanity_check::save_config(&config)?;
+
+    crate::server::jobstate::remove_state_file("sanitycheckjob", &id)?;
+
+    Ok(())
+}
+
+const ITEM_ROUTER: Router = Router::new()
+    .get(&API_METHOD_READ_SANITY_CHECK_JOB)
+    .put(&API_METHOD_UPDATE_SANITY_CHECK_JOB)
+    .delete(&API_METHOD_DELETE_SANITY_CHECK_JOB);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_SANITY_CHECK_JOBS)
+    .post(&API_METHOD_CREATE_SANITY_CHECK_JOB)
+    .match_all("id", &ITEM_ROUTER);
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 4/8] server: add sanity check job email notifications

[Christian Ebner]

Defines the email templates and method to send success/failure
notification emails for the sanity check jobs.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/server/email_notifications.rs | 78 +++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/src/server/email_notifications.rs b/src/server/email_notifications.rs
index 43b55656..7672f30a 100644
--- a/src/server/email_notifications.rs
+++ b/src/server/email_notifications.rs
@@ -241,6 +241,36 @@ Please visit the web interface for further details:
 
 "###;
 
+const SANITY_CHECK_OK_TEMPLATE: &str = r###"
+
+Job ID:    {{jobname}}
+
+Sanity check successful.
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
+"###;
+
+const SANITY_CHECK_ERR_TEMPLATE: &str = r###"
+
+Job ID:    {{jobname}}
+
+Sanity checks failed:
+
+{{#each errors}}
+  {{this~}}
+{{/each}}
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
+"###;
+
 lazy_static::lazy_static! {
 
     static ref HANDLEBARS: Handlebars<'static> = {
@@ -272,6 +302,9 @@ lazy_static::lazy_static! {
 
             hb.register_template_string("certificate_renewal_err_template", ACME_CERTIFICATE_ERR_RENEWAL)?;
 
+            hb.register_template_string("sanity_check_ok_template", SANITY_CHECK_OK_TEMPLATE)?;
+            hb.register_template_string("sanity_check_err_template", SANITY_CHECK_ERR_TEMPLATE)?;
+
             Ok(())
         });
 
@@ -460,6 +493,48 @@ pub fn send_prune_status(
     Ok(())
 }
 
+pub fn send_sanity_check_status(
+    email: &str,
+    notify: Option<Notify>,
+    jobname: &str,
+    result: &Result<Vec<String>, Error>,
+) -> Result<(), Error> {
+    match notify {
+        None => { /* send notifications by default */ }
+        Some(notify) => {
+            if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+                return Ok(());
+            }
+        }
+    }
+
+    let (fqdn, port) = get_server_url();
+    let mut data = json!({
+        "jobname": jobname,
+        "fqdn": fqdn,
+        "port": port,
+    });
+
+    let (subject, text) = match result {
+        Ok(errors) if errors.is_empty() => (
+            format!("Sanity check successful"),
+            HANDLEBARS.render("sanity_check_ok_template", &data)?,
+        ),
+        Ok(errors) => {
+            data["errors"] = json!(errors);
+            (
+                format!("Sanity check failed"),
+                HANDLEBARS.render("sanity_check_err_template", &data)?,
+            )
+        }
+        Err(_) => return Ok(()),
+    };
+
+    send_job_status_mail(&email, &subject, &text)?;
+
+    Ok(())
+}
+
 pub fn send_sync_status(
     email: &str,
     notify: DatastoreNotify,
@@ -760,4 +835,7 @@ fn test_template_register() {
     assert!(HANDLEBARS.has_template("package_update_template"));
 
     assert!(HANDLEBARS.has_template("certificate_renewal_err_template"));
+
+    assert!(HANDLEBARS.has_template("sanity_check_ok_template"));
+    assert!(HANDLEBARS.has_template("sanity_check_err_template"));
 }
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 5/8] server: implement sanity check job

[Christian Ebner]

Adds the sanity check job execution logic and implements a check for
the datastore usage levels exceeding the config values threshold
level.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/server/mod.rs              |   3 +
 src/server/sanity_check_job.rs | 131 +++++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 src/server/sanity_check_job.rs

diff --git a/src/server/mod.rs b/src/server/mod.rs
index 4e3b68ac..b3fdc281 100644
--- a/src/server/mod.rs
+++ b/src/server/mod.rs
@@ -25,6 +25,9 @@ pub use gc_job::*;
 mod realm_sync_job;
 pub use realm_sync_job::*;
 
+mod sanity_check_job;
+pub use sanity_check_job::*;
+
 mod email_notifications;
 pub use email_notifications::*;
 
diff --git a/src/server/sanity_check_job.rs b/src/server/sanity_check_job.rs
new file mode 100644
index 00000000..a68b4bfd
--- /dev/null
+++ b/src/server/sanity_check_job.rs
@@ -0,0 +1,131 @@
+use std::sync::Arc;
+
+use anyhow::{format_err, Error};
+
+use proxmox_human_byte::HumanByte;
+use proxmox_sys::{task_error, task_log};
+
+use pbs_api_types::{
+    Authid, Operation, SanityCheckJobOptions, Userid, DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT,
+};
+use pbs_datastore::DataStore;
+use proxmox_rest_server::WorkerTask;
+
+use crate::server::{jobstate::Job, lookup_user_email};
+
+pub fn check_datastore_usage_full_threshold(
+    worker: Arc<WorkerTask>,
+    sanity_check_options: SanityCheckJobOptions,
+) -> Result<Vec<String>, Error> {
+    let (config, _digest) = pbs_config::datastore::config()?;
+    let threshold = sanity_check_options
+        .datastore_usage_full_threshold
+        .unwrap_or(DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT);
+    let mut errors = Vec::new();
+
+    task_log!(
+        worker,
+        "Checking datastore usage levels with {threshold}% threshold ..."
+    );
+    for (store, (_, _)) in &config.sections {
+        let datastore = match DataStore::lookup_datastore(store, Some(Operation::Read)) {
+            Ok(datastore) => datastore,
+            Err(err) => {
+                let msg = format!("failed to lookup datastore - {err}");
+                task_error!(worker, "{msg}");
+                errors.push(msg);
+                continue;
+            }
+        };
+
+        let status = match proxmox_sys::fs::fs_info(&datastore.base_path()) {
+            Ok(status) => status,
+            Err(err) => {
+                let msg = format!("failed to get datastore status - {err}");
+                task_error!(worker, "{msg}");
+                errors.push(msg);
+                continue;
+            }
+        };
+
+        let used = (status.used as f64 / status.total as f64 * 100f64).trunc() as u8;
+        if used >= threshold {
+            let msg = format!(
+                "Datastore '{store}' exceeded usage threshold!\n  used {} of {} ({used}%)",
+                HumanByte::from(status.used),
+                HumanByte::from(status.total),
+            );
+            task_error!(worker, "{msg}");
+            errors.push(msg);
+        } else {
+            task_log!(
+                worker,
+                "Datastore '{store}' below usage threshold, used {} of {} ({used}%)",
+                HumanByte::from(status.used),
+                HumanByte::from(status.total),
+            );
+        }
+    }
+
+    Ok(errors)
+}
+
+pub fn do_sanity_check_job(
+    mut job: Job,
+    sanity_check_options: SanityCheckJobOptions,
+    auth_id: &Authid,
+    schedule: Option<String>,
+) -> Result<String, Error> {
+    let worker_type = job.jobtype().to_string();
+    let auth_id = auth_id.clone();
+
+    let notify_user = sanity_check_options
+        .notify_user
+        .as_ref()
+        .unwrap_or_else(|| Userid::root_userid());
+    let email = lookup_user_email(notify_user);
+
+    let upid_str = WorkerTask::new_thread(
+        &worker_type,
+        Some(job.jobname().to_string()),
+        auth_id.to_string(),
+        false,
+        move |worker| {
+            job.start(&worker.upid().to_string())?;
+
+            task_log!(worker, "sanity check job '{}'", job.jobname());
+
+            if let Some(event_str) = schedule {
+                task_log!(worker, "task triggered by schedule '{event_str}'");
+            }
+
+            let result = check_datastore_usage_full_threshold(worker.clone(), sanity_check_options);
+            let job_result = match result {
+                Ok(ref errors) if errors.is_empty() => Ok(()),
+                Ok(_) => Err(format_err!(
+                    "sanity check failed - please check the log for details"
+                )),
+                Err(_) => Err(format_err!("sanity check failed - job aborted")),
+            };
+
+            let status = worker.create_state(&job_result);
+
+            if let Err(err) = job.finish(status) {
+                eprintln!("could not finish job state for {}: {err}", job.jobtype());
+            }
+
+            if let Some(email) = email {
+                task_log!(worker, "sending notification email to '{email}'");
+                if let Err(err) =
+                    crate::server::send_sanity_check_status(&email, None, job.jobname(), &result)
+                {
+                    log::error!("send sanity check notification failed: {err}");
+                }
+            }
+
+            job_result
+        },
+    )?;
+
+    Ok(upid_str)
+}
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 6/8] api: admin: add sanity check job api endpoints

[Christian Ebner]

Implements the api endpoints required to perform administration tasks
for sanity check jobs.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/api2/admin/mod.rs          |   2 +
 src/api2/admin/sanity_check.rs | 111 +++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 src/api2/admin/sanity_check.rs

diff --git a/src/api2/admin/mod.rs b/src/api2/admin/mod.rs
index 168dc038..8577fc56 100644
--- a/src/api2/admin/mod.rs
+++ b/src/api2/admin/mod.rs
@@ -8,6 +8,7 @@ pub mod datastore;
 pub mod metrics;
 pub mod namespace;
 pub mod prune;
+pub mod sanity_check;
 pub mod sync;
 pub mod traffic_control;
 pub mod verify;
@@ -17,6 +18,7 @@ const SUBDIRS: SubdirMap = &sorted!([
     ("datastore", &datastore::ROUTER),
     ("metrics", &metrics::ROUTER),
     ("prune", &prune::ROUTER),
+    ("sanity-check", &sanity_check::ROUTER),
     ("sync", &sync::ROUTER),
     ("traffic-control", &traffic_control::ROUTER),
     ("verify", &verify::ROUTER),
diff --git a/src/api2/admin/sanity_check.rs b/src/api2/admin/sanity_check.rs
new file mode 100644
index 00000000..990ce6f2
--- /dev/null
+++ b/src/api2/admin/sanity_check.rs
@@ -0,0 +1,111 @@
+//! Sanity Check Job Management
+
+use anyhow::{format_err, Error};
+use serde_json::Value;
+
+use proxmox_router::{
+    list_subdirs_api_method, ApiMethod, Permission, Router, RpcEnvironment, SubdirMap,
+};
+use proxmox_schema::api;
+use proxmox_sortable_macro::sortable;
+
+use pbs_api_types::{
+    Authid, SanityCheckJobConfig, SanityCheckJobStatus, JOB_ID_SCHEMA, PRIV_SYS_AUDIT,
+    PRIV_SYS_MODIFY,
+};
+use pbs_config::{sanity_check, CachedUserInfo};
+
+use crate::server::{
+    do_sanity_check_job,
+    jobstate::{compute_schedule_status, Job, JobState},
+};
+
+#[api(
+    returns: {
+        description: "List configured jobs and their status",
+        type: Array,
+        items: { type: SanityCheckJobStatus },
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Audit or Sys.Modify.",
+    },
+)]
+/// List all sanity check jobs
+pub fn list_sanity_check_jobs(
+    _param: Value,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<SanityCheckJobStatus>, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT | PRIV_SYS_MODIFY, true)?;
+
+    let (config, digest) = sanity_check::config()?;
+    let sanity_check_job_config: Vec<SanityCheckJobConfig> =
+        config.convert_to_typed_array("sanity-check")?;
+
+    let mut list = Vec::new();
+    for job in sanity_check_job_config {
+        let last_state = JobState::load("sanitycheckjob", &job.id)
+            .map_err(|err| format_err!("could not open statefile for {}: {err}", &job.id))?;
+
+        let mut status = compute_schedule_status(&last_state, Some(&job.schedule))?;
+        if job.disable {
+            status.next_run = None;
+        }
+
+        list.push(SanityCheckJobStatus {
+            config: job,
+            status,
+        });
+    }
+
+    rpcenv["digest"] = hex::encode(digest).into();
+
+    Ok(list)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            }
+        }
+    },
+    access: {
+        permission: &Permission::Anybody,
+        description: "Requires Sys.Modify.",
+    },
+)]
+/// Runs a sanity check job manually.
+pub fn run_sanity_check_job(
+    id: String,
+    _info: &ApiMethod,
+    rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+    let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+    let user_info = CachedUserInfo::new()?;
+    user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+    let (config, _digest) = sanity_check::config()?;
+    let sanity_check_job: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+    let job = Job::new("sanitycheckjob", &id)?;
+
+    let upid_str = do_sanity_check_job(job, sanity_check_job.options, &auth_id, None)?;
+
+    Ok(upid_str)
+}
+
+#[sortable]
+const SANITY_CHECK_INFO_SUBDIRS: SubdirMap =
+    &[("run", &Router::new().post(&API_METHOD_RUN_SANITY_CHECK_JOB))];
+
+const SANITY_CHECK_INFO_ROUTER: Router = Router::new()
+    .get(&list_subdirs_api_method!(SANITY_CHECK_INFO_SUBDIRS))
+    .subdirs(SANITY_CHECK_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+    .get(&API_METHOD_LIST_SANITY_CHECK_JOBS)
+    .match_all("id", &SANITY_CHECK_INFO_ROUTER);
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 7/8] manager: add sanity check jobs management cli commands

[Christian Ebner]

Expose the sanity check management commands to the CLI, allowing to
create, show/list, update, remove and run jobs.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/bin/proxmox-backup-manager.rs             |   3 +-
 src/bin/proxmox_backup_manager/mod.rs         |   2 +
 .../proxmox_backup_manager/sanity_check.rs    | 126 ++++++++++++++++++
 3 files changed, 130 insertions(+), 1 deletion(-)
 create mode 100644 src/bin/proxmox_backup_manager/sanity_check.rs

diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 115207f3..e17b18c7 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -447,6 +447,7 @@ async fn run() -> Result<(), Error> {
         .insert("acme", acme_mgmt_cli())
         .insert("cert", cert_mgmt_cli())
         .insert("subscription", subscription_commands())
+        .insert("sanity-check-job", sanity_check_job_commands())
         .insert("sync-job", sync_job_commands())
         .insert("verify-job", verify_job_commands())
         .insert("prune-job", prune_job_commands())
@@ -510,7 +511,7 @@ fn main() -> Result<(), Error> {
     proxmox_async::runtime::main(run())
 }
 
-/// Run the job of a given type (one of "prune", "sync", "verify"),
+/// Run the job of a given type (one of "prune", "sync", "verify", "sanity-check"),
 /// specified by the 'id' parameter.
 async fn run_job(job_type: &str, param: Value) -> Result<Value, Error> {
     let output_format = get_output_format(&param);
diff --git a/src/bin/proxmox_backup_manager/mod.rs b/src/bin/proxmox_backup_manager/mod.rs
index 8a1c140c..4d728636 100644
--- a/src/bin/proxmox_backup_manager/mod.rs
+++ b/src/bin/proxmox_backup_manager/mod.rs
@@ -16,6 +16,8 @@ mod prune;
 pub use prune::*;
 mod remote;
 pub use remote::*;
+mod sanity_check;
+pub use sanity_check::*;
 mod sync;
 pub use sync::*;
 mod verify;
diff --git a/src/bin/proxmox_backup_manager/sanity_check.rs b/src/bin/proxmox_backup_manager/sanity_check.rs
new file mode 100644
index 00000000..ff875b65
--- /dev/null
+++ b/src/bin/proxmox_backup_manager/sanity_check.rs
@@ -0,0 +1,126 @@
+
+use anyhow::Error;
+use serde_json::Value;
+
+use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
+use proxmox_schema::api;
+
+use pbs_api_types::JOB_ID_SCHEMA;
+
+use proxmox_backup::api2;
+
+#[api(
+    input: {
+        properties: {
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// List all sanity check jobs
+fn list_sanity_check_jobs(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let output_format = get_output_format(&param);
+
+    let info = &api2::config::sanity_check::API_METHOD_LIST_SANITY_CHECK_JOBS;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options()
+        .column(ColumnConfig::new("id"))
+        .column(ColumnConfig::new("schedule"))
+        .column(ColumnConfig::new("datastore-usage-full-threshold"));
+
+    format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Show sanity check job configuration
+fn show_sanity_check_job(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+    let output_format = get_output_format(&param);
+
+    let info = &api2::config::sanity_check::API_METHOD_READ_SANITY_CHECK_JOB;
+    let mut data = match info.handler {
+        ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+        _ => unreachable!(),
+    };
+
+    let options = default_table_format_options();
+    format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+    Ok(Value::Null)
+}
+
+#[api(
+    input: {
+        properties: {
+            id: {
+                schema: JOB_ID_SCHEMA,
+            },
+            "output-format": {
+                schema: OUTPUT_FORMAT,
+                optional: true,
+            },
+        }
+    }
+)]
+/// Run the specified sanity check job
+async fn run_sanity_check_job(param: Value) -> Result<Value, Error> {
+    crate::run_job("sanity-check", param).await
+}
+
+pub fn sanity_check_job_commands() -> CommandLineInterface {
+    let cmd_def = CliCommandMap::new()
+        .insert("list", CliCommand::new(&API_METHOD_LIST_SANITY_CHECK_JOBS))
+        .insert(
+            "show",
+            CliCommand::new(&API_METHOD_SHOW_SANITY_CHECK_JOB)
+                .arg_param(&["id"])
+                .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+        )
+        .insert(
+            "create",
+            CliCommand::new(&api2::config::sanity_check::API_METHOD_CREATE_SANITY_CHECK_JOB)
+                .arg_param(&["id"])
+                .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id)
+                .completion_cb("schedule", pbs_config::datastore::complete_calendar_event),
+        )
+        .insert(
+            "update",
+            CliCommand::new(&api2::config::sanity_check::API_METHOD_UPDATE_SANITY_CHECK_JOB)
+                .arg_param(&["id"])
+                .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id)
+                .completion_cb("schedule", pbs_config::datastore::complete_calendar_event),
+        )
+        .insert(
+            "run",
+            CliCommand::new(&API_METHOD_RUN_SANITY_CHECK_JOB)
+                .arg_param(&["id"])
+                .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+        )
+        .insert(
+            "remove",
+            CliCommand::new(&api2::config::sanity_check::API_METHOD_DELETE_SANITY_CHECK_JOB)
+                .arg_param(&["id"])
+                .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+        );
+
+    cmd_def.into()
+}
-- 
2.39.2

[pbs-devel] [RFC proxmox-backup 8/8] proxy: add sanity check task to scheduler

[Christian Ebner]

Execute configured sanity check tasks based on their configured time
schedule.

Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
 src/bin/proxmox-backup-proxy.rs | 41 ++++++++++++++++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 9c49026b..8efa0655 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -44,7 +44,7 @@ use proxmox_time::CalendarEvent;
 
 use pbs_api_types::{
     Authid, DataStoreConfig, Operation, PruneJobConfig, SyncJobConfig, TapeBackupJobConfig,
-    VerificationJobConfig,
+    VerificationJobConfig, SanityCheckJobConfig,
 };
 
 use proxmox_rest_server::daemon;
@@ -60,6 +60,7 @@ use proxmox_backup::api2::pull::do_sync_job;
 use proxmox_backup::api2::tape::backup::do_tape_backup_job;
 use proxmox_backup::server::do_prune_job;
 use proxmox_backup::server::do_verification_job;
+use proxmox_backup::server::do_sanity_check_job;
 
 fn main() -> Result<(), Error> {
     pbs_tools::setup_libc_malloc_opts();
@@ -454,6 +455,7 @@ async fn schedule_tasks() -> Result<(), Error> {
     schedule_datastore_verify_jobs().await;
     schedule_tape_backup_jobs().await;
     schedule_task_log_rotate().await;
+    schedule_task_sanity_check_jobs().await;
 
     Ok(())
 }
@@ -825,6 +827,43 @@ async fn schedule_task_log_rotate() {
     }
 }
 
+async fn schedule_task_sanity_check_jobs() {
+    let config = match pbs_config::sanity_check::config() {
+        Err(err) => {
+            eprintln!("unable to read sanity check job config - {err}");
+            return;
+        }
+        Ok((config, _digest)) => config,
+    };
+    for (job_id, (_, job_config)) in config.sections {
+        let job_config: SanityCheckJobConfig = match serde_json::from_value(job_config) {
+            Ok(c) => c,
+            Err(err) => {
+                eprintln!("sanity check job config from_value failed - {err}");
+                continue;
+            }
+        };
+
+        let worker_type = "sanitycheckjob";
+        let auth_id = Authid::root_auth_id().clone();
+        if check_schedule(worker_type, &job_config.schedule, &job_id) {
+            let job = match Job::new(worker_type, &job_id) {
+                Ok(job) => job,
+                Err(_) => continue, // could not get lock
+            };
+            if let Err(err) = do_sanity_check_job(
+                job,
+                job_config.options,
+                &auth_id,
+                Some(job_config.schedule),
+            ) {
+                eprintln!("unable to start sanity check job {job_id} - {err}");
+            }
+        };
+    }
+}
+
+
 async fn command_reopen_access_logfiles() -> Result<(), Error> {
     // only care about the most recent daemon instance for each, proxy & api, as other older ones
     // should not respond to new requests anyway, but only finish their current one and then exit.
-- 
2.39.2