* [pbs-devel] [RFC proxmox-backup 1/8] api-types: jobs: add sanity checks job types
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 2/8] config: implement sanity check job configuration Christian Ebner
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Defines the required types for managing sanity check jobs via the API.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-api-types/src/jobs.rs | 106 ++++++++++++++++++++++++++++++++++++++
1 file changed, 106 insertions(+)
diff --git a/pbs-api-types/src/jobs.rs b/pbs-api-types/src/jobs.rs
index 1f5b3cf1..4f6b2cc7 100644
--- a/pbs-api-types/src/jobs.rs
+++ b/pbs-api-types/src/jobs.rs
@@ -57,12 +57,28 @@ pub const VERIFICATION_SCHEDULE_SCHEMA: Schema =
.type_text("<calendar-event>")
.schema();
+pub const SANITY_CHECK_SCHEDULE_SCHEMA: Schema =
+ StringSchema::new("Run sanity check job at specified schedule.")
+ .format(&ApiStringFormat::VerifyFn(
+ proxmox_time::verify_calendar_event,
+ ))
+ .type_text("<calendar-event>")
+ .schema();
+
pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema = BooleanSchema::new(
"Delete vanished backups. This remove the local copy if the remote backup was deleted.",
)
.default(false)
.schema();
+pub const DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT: u8 = 90;
+pub const DATASTORE_USAGE_FULL_THRESHOLD_SCHEMA: Schema =
+ IntegerSchema::new("Datastore usage threshold level in percent for the datastore being full.")
+ .minimum(1)
+ .maximum(100)
+ .default(DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT as isize)
+ .schema();
+
#[api(
properties: {
"next-run": {
@@ -753,3 +769,93 @@ pub struct PruneJobStatus {
#[serde(flatten)]
pub status: JobScheduleStatus,
}
+
+#[api(
+ properties: {
+ "datastore-usage-full-threshold": {
+ schema: DATASTORE_USAGE_FULL_THRESHOLD_SCHEMA,
+ optional: true,
+ },
+ "notify-user": {
+ optional: true,
+ type: Userid,
+ },
+ }
+)]
+#[derive(Serialize, Deserialize, Default, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Sanity check options
+pub struct SanityCheckJobOptions {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub datastore_usage_full_threshold: Option<u8>,
+ /// Send job email notification to this user
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub notify_user: Option<Userid>,
+}
+
+#[api(
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ disable: {
+ type: Boolean,
+ optional: true,
+ default: false,
+ },
+ schedule: {
+ schema: SANITY_CHECK_SCHEDULE_SCHEMA,
+ },
+ comment: {
+ optional: true,
+ schema: SINGLE_LINE_COMMENT_SCHEMA,
+ },
+ options: {
+ type: SanityCheckJobOptions,
+ },
+ },
+)]
+#[derive(Deserialize, Serialize, Updater, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Sanity check configuration.
+pub struct SanityCheckJobConfig {
+ /// Unique ID to address this job
+ #[updater(skip)]
+ pub id: String,
+
+ /// Flag to disable job
+ #[serde(default, skip_serializing_if = "is_false")]
+ #[updater(serde(skip_serializing_if = "Option::is_none"))]
+ pub disable: bool,
+
+ /// When to schedule this job in calendar event notation
+ pub schedule: String,
+
+ /// Additional comment for this job
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub comment: Option<String>,
+
+ /// Configuration options for this job
+ #[serde(flatten)]
+ pub options: SanityCheckJobOptions,
+}
+
+#[api(
+ properties: {
+ config: {
+ type: SanityCheckJobConfig,
+ },
+ status: {
+ type: JobScheduleStatus,
+ },
+ },
+)]
+#[derive(Serialize, Deserialize, Clone, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+/// Status of sanity check job
+pub struct SanityCheckJobStatus {
+ #[serde(flatten)]
+ pub config: SanityCheckJobConfig,
+ #[serde(flatten)]
+ pub status: JobScheduleStatus,
+}
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 2/8] config: implement sanity check job configuration
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 1/8] api-types: jobs: add sanity checks job types Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 3/8] api: config: sanity check jobs api endpoints Christian Ebner
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Implements the required methods and defines the location for storing
sanity check jobs configurations.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
pbs-config/src/lib.rs | 1 +
pbs-config/src/sanity_check.rs | 57 ++++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+)
create mode 100644 pbs-config/src/sanity_check.rs
diff --git a/pbs-config/src/lib.rs b/pbs-config/src/lib.rs
index 009c4d3c..dad9225e 100644
--- a/pbs-config/src/lib.rs
+++ b/pbs-config/src/lib.rs
@@ -9,6 +9,7 @@ pub mod metrics;
pub mod network;
pub mod prune;
pub mod remote;
+pub mod sanity_check;
pub mod sync;
pub mod tape_job;
pub mod token_shadow;
diff --git a/pbs-config/src/sanity_check.rs b/pbs-config/src/sanity_check.rs
new file mode 100644
index 00000000..e875733b
--- /dev/null
+++ b/pbs-config/src/sanity_check.rs
@@ -0,0 +1,57 @@
+use std::collections::HashMap;
+
+use anyhow::Error;
+use lazy_static::lazy_static;
+
+use proxmox_schema::*;
+use proxmox_section_config::{SectionConfig, SectionConfigData, SectionConfigPlugin};
+
+use pbs_api_types::{SanityCheckJobConfig, JOB_ID_SCHEMA};
+
+use crate::{open_backup_lockfile, replace_backup_config, BackupLockGuard};
+
+lazy_static! {
+ pub static ref CONFIG: SectionConfig = init();
+}
+
+fn init() -> SectionConfig {
+ const OBJ_SCHEMA: &AllOfSchema = SanityCheckJobConfig::API_SCHEMA.unwrap_all_of_schema();
+
+ let plugin =
+ SectionConfigPlugin::new("sanity-check".to_string(), Some(String::from("id")), OBJ_SCHEMA);
+ let mut config = SectionConfig::new(&JOB_ID_SCHEMA);
+ config.register_plugin(plugin);
+
+ config
+}
+
+pub const SANITY_CHECK_CFG_FILENAME: &str = "/etc/proxmox-backup/sanity-check.cfg";
+pub const SANITY_CHECK_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.sanity-check.lck";
+
+/// Get exclusive lock
+pub fn lock_config() -> Result<BackupLockGuard, Error> {
+ open_backup_lockfile(SANITY_CHECK_CFG_LOCKFILE, None, true)
+}
+
+pub fn config() -> Result<(SectionConfigData, [u8; 32]), Error> {
+ let content = proxmox_sys::fs::file_read_optional_string(SANITY_CHECK_CFG_FILENAME)?;
+ let content = content.unwrap_or_default();
+
+ let digest = openssl::sha::sha256(content.as_bytes());
+ let data = CONFIG.parse(SANITY_CHECK_CFG_FILENAME, &content)?;
+
+ Ok((data, digest))
+}
+
+pub fn save_config(config: &SectionConfigData) -> Result<(), Error> {
+ let raw = CONFIG.write(SANITY_CHECK_CFG_FILENAME, config)?;
+ replace_backup_config(SANITY_CHECK_CFG_FILENAME, raw.as_bytes())
+}
+
+// shell completion helper
+pub fn complete_sanity_check_job_id(_arg: &str, _param: &HashMap<String, String>) -> Vec<String> {
+ match config() {
+ Ok((data, _digest)) => data.sections.keys().map(|id| id.to_string()).collect(),
+ Err(_) => Vec::new(),
+ }
+}
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 3/8] api: config: sanity check jobs api endpoints
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 1/8] api-types: jobs: add sanity checks job types Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 2/8] config: implement sanity check job configuration Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 4/8] server: add sanity check job email notifications Christian Ebner
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Adds the api endpoints to create, read/list, update and delete sanity
check configurations.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/api2/config/mod.rs | 2 +
src/api2/config/sanity_check.rs | 296 ++++++++++++++++++++++++++++++++
2 files changed, 298 insertions(+)
create mode 100644 src/api2/config/sanity_check.rs
diff --git a/src/api2/config/mod.rs b/src/api2/config/mod.rs
index 6cfeaea1..8409ad80 100644
--- a/src/api2/config/mod.rs
+++ b/src/api2/config/mod.rs
@@ -13,6 +13,7 @@ pub mod media_pool;
pub mod metrics;
pub mod prune;
pub mod remote;
+pub mod sanity_check;
pub mod sync;
pub mod tape_backup_job;
pub mod tape_encryption_keys;
@@ -30,6 +31,7 @@ const SUBDIRS: SubdirMap = &sorted!([
("metrics", &metrics::ROUTER),
("prune", &prune::ROUTER),
("remote", &remote::ROUTER),
+ ("sanity-check", &sync::ROUTER),
("sync", &sync::ROUTER),
("tape-backup-job", &tape_backup_job::ROUTER),
("tape-encryption-keys", &tape_encryption_keys::ROUTER),
diff --git a/src/api2/config/sanity_check.rs b/src/api2/config/sanity_check.rs
new file mode 100644
index 00000000..c708b952
--- /dev/null
+++ b/src/api2/config/sanity_check.rs
@@ -0,0 +1,296 @@
+use anyhow::Error;
+use hex::FromHex;
+use proxmox_sys::task_log;
+use proxmox_sys::WorkerTaskContext;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use proxmox_router::{http_bail, Permission, Router, RpcEnvironment};
+use proxmox_schema::{api, param_bail};
+
+use pbs_api_types::{
+ Authid, SanityCheckJobConfig, SanityCheckJobConfigUpdater, JOB_ID_SCHEMA, PRIV_SYS_AUDIT,
+ PRIV_SYS_MODIFY, PROXMOX_CONFIG_DIGEST_SCHEMA,
+};
+use pbs_config::{sanity_check, CachedUserInfo};
+
+#[api(
+ input: {
+ properties: {},
+ },
+ returns: {
+ description: "List configured sanity checks schedules.",
+ type: Array,
+ items: { type: SanityCheckJobConfig },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Audit.",
+ },
+)]
+/// List all scheduled sanity check jobs.
+pub fn list_sanity_check_jobs(
+ _param: Value,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<SanityCheckJobConfig>, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT | PRIV_SYS_MODIFY, true)?;
+
+ let (config, digest) = sanity_check::config()?;
+ let list = config.convert_to_typed_array("sanity-check")?;
+
+ rpcenv["digest"] = hex::encode(digest).into();
+
+ Ok(list)
+}
+
+pub fn do_create_sanity_check_job(
+ config: SanityCheckJobConfig,
+ worker: Option<&dyn WorkerTaskContext>,
+) -> Result<(), Error> {
+ let _lock = sanity_check::lock_config()?;
+
+ let (mut section_config, _digest) = sanity_check::config()?;
+
+ if section_config.sections.get(&config.id).is_some() {
+ param_bail!("id", "job '{}' already exists.", config.id);
+ }
+
+ section_config.set_data(&config.id, "sanity-check", &config)?;
+
+ sanity_check::save_config(§ion_config)?;
+
+ crate::server::jobstate::create_state_file("sanitycheckjob", &config.id)?;
+
+ if let Some(worker) = worker {
+ task_log!(worker, "Sanity check job created: {}", config.id);
+ }
+
+ Ok(())
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ config: {
+ type: SanityCheckJobConfig,
+ flatten: true,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Modify.",
+ },
+)]
+/// Create a new sanity check job.
+pub fn create_sanity_check_job(
+ config: SanityCheckJobConfig,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+ do_create_sanity_check_job(config, None)
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ },
+ },
+ returns: { type: SanityCheckJobConfig },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Audit.",
+ },
+)]
+/// Read a sanity check job configuration.
+pub fn read_sanity_check_job(
+ id: String,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<SanityCheckJobConfig, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT, true)?;
+
+ let (config, digest) = sanity_check::config()?;
+ let sanity_check_job: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+ rpcenv["digest"] = hex::encode(digest).into();
+
+ Ok(sanity_check_job)
+}
+
+#[api]
+#[derive(Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Deletable property name
+pub enum DeletableProperty {
+ /// Delete the comment.
+ Comment,
+ /// Unset the disable flag.
+ Disable,
+ /// Unset the notify-user.
+ NotifyUser,
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ update: {
+ type: SanityCheckJobConfigUpdater,
+ flatten: true,
+ },
+ delete: {
+ description: "List of properties to delete.",
+ type: Array,
+ optional: true,
+ items: {
+ type: DeletableProperty,
+ }
+ },
+ digest: {
+ optional: true,
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Modify.",
+ },
+)]
+/// Update sanity check job config.
+#[allow(clippy::too_many_arguments)]
+pub fn update_sanity_check_job(
+ id: String,
+ update: SanityCheckJobConfigUpdater,
+ delete: Option<Vec<DeletableProperty>>,
+ digest: Option<String>,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+ let _lock = sanity_check::lock_config()?;
+
+ // pass/compare digest
+ let (mut config, expected_digest) = sanity_check::config()?;
+ if let Some(ref digest) = digest {
+ let digest = <[u8; 32]>::from_hex(digest)?;
+ crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+ }
+
+ let mut data: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+ if let Some(delete) = delete {
+ for delete_prop in delete {
+ match delete_prop {
+ DeletableProperty::Comment => {
+ data.comment = None;
+ }
+ DeletableProperty::Disable => {
+ data.disable = false;
+ }
+ DeletableProperty::NotifyUser => {
+ data.options.notify_user = None;
+ }
+ }
+ }
+ }
+
+ let mut schedule_changed = false;
+ if let Some(schedule) = update.schedule {
+ schedule_changed = data.schedule != schedule;
+ data.schedule = schedule;
+ }
+
+ if let Some(value) = update.comment {
+ data.comment = Some(value);
+ }
+ if let Some(value) = update.disable {
+ data.disable = value;
+ }
+ if let Some(value) = update.options.notify_user {
+ data.options.notify_user = Some(value);
+ }
+
+ config.set_data(&id, "sanity-check", &data)?;
+
+ sanity_check::save_config(&config)?;
+
+ if schedule_changed {
+ crate::server::jobstate::update_job_last_run_time("sanitycheckjob", &id)?;
+ }
+
+ Ok(())
+}
+
+#[api(
+ protected: true,
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ digest: {
+ optional: true,
+ schema: PROXMOX_CONFIG_DIGEST_SCHEMA,
+ },
+ },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Modify.",
+ },
+)]
+/// Remove a sanity check job configuration
+pub fn delete_sanity_check_job(
+ id: String,
+ digest: Option<String>,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<(), Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+ let _lock = sanity_check::lock_config()?;
+
+ let (mut config, expected_digest) = sanity_check::config()?;
+ if let Some(ref digest) = digest {
+ let digest = <[u8; 32]>::from_hex(digest)?;
+ crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?;
+ }
+
+ if config.sections.remove(&id).is_none() {
+ http_bail!(NOT_FOUND, "job '{}' does not exist.", id);
+ }
+
+ sanity_check::save_config(&config)?;
+
+ crate::server::jobstate::remove_state_file("sanitycheckjob", &id)?;
+
+ Ok(())
+}
+
+const ITEM_ROUTER: Router = Router::new()
+ .get(&API_METHOD_READ_SANITY_CHECK_JOB)
+ .put(&API_METHOD_UPDATE_SANITY_CHECK_JOB)
+ .delete(&API_METHOD_DELETE_SANITY_CHECK_JOB);
+
+pub const ROUTER: Router = Router::new()
+ .get(&API_METHOD_LIST_SANITY_CHECK_JOBS)
+ .post(&API_METHOD_CREATE_SANITY_CHECK_JOB)
+ .match_all("id", &ITEM_ROUTER);
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 4/8] server: add sanity check job email notifications
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
` (2 preceding siblings ...)
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 3/8] api: config: sanity check jobs api endpoints Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 5/8] server: implement sanity check job Christian Ebner
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Defines the email templates and method to send success/failure
notification emails for the sanity check jobs.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/server/email_notifications.rs | 78 +++++++++++++++++++++++++++++++
1 file changed, 78 insertions(+)
diff --git a/src/server/email_notifications.rs b/src/server/email_notifications.rs
index 43b55656..7672f30a 100644
--- a/src/server/email_notifications.rs
+++ b/src/server/email_notifications.rs
@@ -241,6 +241,36 @@ Please visit the web interface for further details:
"###;
+const SANITY_CHECK_OK_TEMPLATE: &str = r###"
+
+Job ID: {{jobname}}
+
+Sanity check successful.
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
+"###;
+
+const SANITY_CHECK_ERR_TEMPLATE: &str = r###"
+
+Job ID: {{jobname}}
+
+Sanity checks failed:
+
+{{#each errors}}
+ {{this~}}
+{{/each}}
+
+
+Please visit the web interface for further details:
+
+<https://{{fqdn}}:{{port}}/#pbsServerAdministration:tasks>
+
+"###;
+
lazy_static::lazy_static! {
static ref HANDLEBARS: Handlebars<'static> = {
@@ -272,6 +302,9 @@ lazy_static::lazy_static! {
hb.register_template_string("certificate_renewal_err_template", ACME_CERTIFICATE_ERR_RENEWAL)?;
+ hb.register_template_string("sanity_check_ok_template", SANITY_CHECK_OK_TEMPLATE)?;
+ hb.register_template_string("sanity_check_err_template", SANITY_CHECK_ERR_TEMPLATE)?;
+
Ok(())
});
@@ -460,6 +493,48 @@ pub fn send_prune_status(
Ok(())
}
+pub fn send_sanity_check_status(
+ email: &str,
+ notify: Option<Notify>,
+ jobname: &str,
+ result: &Result<Vec<String>, Error>,
+) -> Result<(), Error> {
+ match notify {
+ None => { /* send notifications by default */ }
+ Some(notify) => {
+ if notify == Notify::Never || (result.is_ok() && notify == Notify::Error) {
+ return Ok(());
+ }
+ }
+ }
+
+ let (fqdn, port) = get_server_url();
+ let mut data = json!({
+ "jobname": jobname,
+ "fqdn": fqdn,
+ "port": port,
+ });
+
+ let (subject, text) = match result {
+ Ok(errors) if errors.is_empty() => (
+ format!("Sanity check successful"),
+ HANDLEBARS.render("sanity_check_ok_template", &data)?,
+ ),
+ Ok(errors) => {
+ data["errors"] = json!(errors);
+ (
+ format!("Sanity check failed"),
+ HANDLEBARS.render("sanity_check_err_template", &data)?,
+ )
+ }
+ Err(_) => return Ok(()),
+ };
+
+ send_job_status_mail(&email, &subject, &text)?;
+
+ Ok(())
+}
+
pub fn send_sync_status(
email: &str,
notify: DatastoreNotify,
@@ -760,4 +835,7 @@ fn test_template_register() {
assert!(HANDLEBARS.has_template("package_update_template"));
assert!(HANDLEBARS.has_template("certificate_renewal_err_template"));
+
+ assert!(HANDLEBARS.has_template("sanity_check_ok_template"));
+ assert!(HANDLEBARS.has_template("sanity_check_err_template"));
}
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 5/8] server: implement sanity check job
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
` (3 preceding siblings ...)
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 4/8] server: add sanity check job email notifications Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 6/8] api: admin: add sanity check job api endpoints Christian Ebner
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Adds the sanity check job execution logic and implements a check for
the datastore usage levels exceeding the config values threshold
level.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/server/mod.rs | 3 +
src/server/sanity_check_job.rs | 131 +++++++++++++++++++++++++++++++++
2 files changed, 134 insertions(+)
create mode 100644 src/server/sanity_check_job.rs
diff --git a/src/server/mod.rs b/src/server/mod.rs
index 4e3b68ac..b3fdc281 100644
--- a/src/server/mod.rs
+++ b/src/server/mod.rs
@@ -25,6 +25,9 @@ pub use gc_job::*;
mod realm_sync_job;
pub use realm_sync_job::*;
+mod sanity_check_job;
+pub use sanity_check_job::*;
+
mod email_notifications;
pub use email_notifications::*;
diff --git a/src/server/sanity_check_job.rs b/src/server/sanity_check_job.rs
new file mode 100644
index 00000000..a68b4bfd
--- /dev/null
+++ b/src/server/sanity_check_job.rs
@@ -0,0 +1,131 @@
+use std::sync::Arc;
+
+use anyhow::{format_err, Error};
+
+use proxmox_human_byte::HumanByte;
+use proxmox_sys::{task_error, task_log};
+
+use pbs_api_types::{
+ Authid, Operation, SanityCheckJobOptions, Userid, DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT,
+};
+use pbs_datastore::DataStore;
+use proxmox_rest_server::WorkerTask;
+
+use crate::server::{jobstate::Job, lookup_user_email};
+
+pub fn check_datastore_usage_full_threshold(
+ worker: Arc<WorkerTask>,
+ sanity_check_options: SanityCheckJobOptions,
+) -> Result<Vec<String>, Error> {
+ let (config, _digest) = pbs_config::datastore::config()?;
+ let threshold = sanity_check_options
+ .datastore_usage_full_threshold
+ .unwrap_or(DATASTORE_USAGE_FULL_THRESHOLD_DEFAULT);
+ let mut errors = Vec::new();
+
+ task_log!(
+ worker,
+ "Checking datastore usage levels with {threshold}% threshold ..."
+ );
+ for (store, (_, _)) in &config.sections {
+ let datastore = match DataStore::lookup_datastore(store, Some(Operation::Read)) {
+ Ok(datastore) => datastore,
+ Err(err) => {
+ let msg = format!("failed to lookup datastore - {err}");
+ task_error!(worker, "{msg}");
+ errors.push(msg);
+ continue;
+ }
+ };
+
+ let status = match proxmox_sys::fs::fs_info(&datastore.base_path()) {
+ Ok(status) => status,
+ Err(err) => {
+ let msg = format!("failed to get datastore status - {err}");
+ task_error!(worker, "{msg}");
+ errors.push(msg);
+ continue;
+ }
+ };
+
+ let used = (status.used as f64 / status.total as f64 * 100f64).trunc() as u8;
+ if used >= threshold {
+ let msg = format!(
+ "Datastore '{store}' exceeded usage threshold!\n used {} of {} ({used}%)",
+ HumanByte::from(status.used),
+ HumanByte::from(status.total),
+ );
+ task_error!(worker, "{msg}");
+ errors.push(msg);
+ } else {
+ task_log!(
+ worker,
+ "Datastore '{store}' below usage threshold, used {} of {} ({used}%)",
+ HumanByte::from(status.used),
+ HumanByte::from(status.total),
+ );
+ }
+ }
+
+ Ok(errors)
+}
+
+pub fn do_sanity_check_job(
+ mut job: Job,
+ sanity_check_options: SanityCheckJobOptions,
+ auth_id: &Authid,
+ schedule: Option<String>,
+) -> Result<String, Error> {
+ let worker_type = job.jobtype().to_string();
+ let auth_id = auth_id.clone();
+
+ let notify_user = sanity_check_options
+ .notify_user
+ .as_ref()
+ .unwrap_or_else(|| Userid::root_userid());
+ let email = lookup_user_email(notify_user);
+
+ let upid_str = WorkerTask::new_thread(
+ &worker_type,
+ Some(job.jobname().to_string()),
+ auth_id.to_string(),
+ false,
+ move |worker| {
+ job.start(&worker.upid().to_string())?;
+
+ task_log!(worker, "sanity check job '{}'", job.jobname());
+
+ if let Some(event_str) = schedule {
+ task_log!(worker, "task triggered by schedule '{event_str}'");
+ }
+
+ let result = check_datastore_usage_full_threshold(worker.clone(), sanity_check_options);
+ let job_result = match result {
+ Ok(ref errors) if errors.is_empty() => Ok(()),
+ Ok(_) => Err(format_err!(
+ "sanity check failed - please check the log for details"
+ )),
+ Err(_) => Err(format_err!("sanity check failed - job aborted")),
+ };
+
+ let status = worker.create_state(&job_result);
+
+ if let Err(err) = job.finish(status) {
+ eprintln!("could not finish job state for {}: {err}", job.jobtype());
+ }
+
+ if let Some(email) = email {
+ task_log!(worker, "sending notification email to '{email}'");
+ if let Err(err) =
+ crate::server::send_sanity_check_status(&email, None, job.jobname(), &result)
+ {
+ log::error!("send sanity check notification failed: {err}");
+ }
+ }
+
+ job_result
+ },
+ )?;
+
+ Ok(upid_str)
+}
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 6/8] api: admin: add sanity check job api endpoints
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
` (4 preceding siblings ...)
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 5/8] server: implement sanity check job Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 7/8] manager: add sanity check jobs management cli commands Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 8/8] proxy: add sanity check task to scheduler Christian Ebner
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Implements the api endpoints required to perform administration tasks
for sanity check jobs.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/api2/admin/mod.rs | 2 +
src/api2/admin/sanity_check.rs | 111 +++++++++++++++++++++++++++++++++
2 files changed, 113 insertions(+)
create mode 100644 src/api2/admin/sanity_check.rs
diff --git a/src/api2/admin/mod.rs b/src/api2/admin/mod.rs
index 168dc038..8577fc56 100644
--- a/src/api2/admin/mod.rs
+++ b/src/api2/admin/mod.rs
@@ -8,6 +8,7 @@ pub mod datastore;
pub mod metrics;
pub mod namespace;
pub mod prune;
+pub mod sanity_check;
pub mod sync;
pub mod traffic_control;
pub mod verify;
@@ -17,6 +18,7 @@ const SUBDIRS: SubdirMap = &sorted!([
("datastore", &datastore::ROUTER),
("metrics", &metrics::ROUTER),
("prune", &prune::ROUTER),
+ ("sanity-check", &sanity_check::ROUTER),
("sync", &sync::ROUTER),
("traffic-control", &traffic_control::ROUTER),
("verify", &verify::ROUTER),
diff --git a/src/api2/admin/sanity_check.rs b/src/api2/admin/sanity_check.rs
new file mode 100644
index 00000000..990ce6f2
--- /dev/null
+++ b/src/api2/admin/sanity_check.rs
@@ -0,0 +1,111 @@
+//! Sanity Check Job Management
+
+use anyhow::{format_err, Error};
+use serde_json::Value;
+
+use proxmox_router::{
+ list_subdirs_api_method, ApiMethod, Permission, Router, RpcEnvironment, SubdirMap,
+};
+use proxmox_schema::api;
+use proxmox_sortable_macro::sortable;
+
+use pbs_api_types::{
+ Authid, SanityCheckJobConfig, SanityCheckJobStatus, JOB_ID_SCHEMA, PRIV_SYS_AUDIT,
+ PRIV_SYS_MODIFY,
+};
+use pbs_config::{sanity_check, CachedUserInfo};
+
+use crate::server::{
+ do_sanity_check_job,
+ jobstate::{compute_schedule_status, Job, JobState},
+};
+
+#[api(
+ returns: {
+ description: "List configured jobs and their status",
+ type: Array,
+ items: { type: SanityCheckJobStatus },
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Audit or Sys.Modify.",
+ },
+)]
+/// List all sanity check jobs
+pub fn list_sanity_check_jobs(
+ _param: Value,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<Vec<SanityCheckJobStatus>, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_AUDIT | PRIV_SYS_MODIFY, true)?;
+
+ let (config, digest) = sanity_check::config()?;
+ let sanity_check_job_config: Vec<SanityCheckJobConfig> =
+ config.convert_to_typed_array("sanity-check")?;
+
+ let mut list = Vec::new();
+ for job in sanity_check_job_config {
+ let last_state = JobState::load("sanitycheckjob", &job.id)
+ .map_err(|err| format_err!("could not open statefile for {}: {err}", &job.id))?;
+
+ let mut status = compute_schedule_status(&last_state, Some(&job.schedule))?;
+ if job.disable {
+ status.next_run = None;
+ }
+
+ list.push(SanityCheckJobStatus {
+ config: job,
+ status,
+ });
+ }
+
+ rpcenv["digest"] = hex::encode(digest).into();
+
+ Ok(list)
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ }
+ }
+ },
+ access: {
+ permission: &Permission::Anybody,
+ description: "Requires Sys.Modify.",
+ },
+)]
+/// Runs a sanity check job manually.
+pub fn run_sanity_check_job(
+ id: String,
+ _info: &ApiMethod,
+ rpcenv: &mut dyn RpcEnvironment,
+) -> Result<String, Error> {
+ let auth_id: Authid = rpcenv.get_auth_id().unwrap().parse()?;
+ let user_info = CachedUserInfo::new()?;
+ user_info.check_privs(&auth_id, &["/"], PRIV_SYS_MODIFY, true)?;
+
+ let (config, _digest) = sanity_check::config()?;
+ let sanity_check_job: SanityCheckJobConfig = config.lookup("sanity-check", &id)?;
+
+ let job = Job::new("sanitycheckjob", &id)?;
+
+ let upid_str = do_sanity_check_job(job, sanity_check_job.options, &auth_id, None)?;
+
+ Ok(upid_str)
+}
+
+#[sortable]
+const SANITY_CHECK_INFO_SUBDIRS: SubdirMap =
+ &[("run", &Router::new().post(&API_METHOD_RUN_SANITY_CHECK_JOB))];
+
+const SANITY_CHECK_INFO_ROUTER: Router = Router::new()
+ .get(&list_subdirs_api_method!(SANITY_CHECK_INFO_SUBDIRS))
+ .subdirs(SANITY_CHECK_INFO_SUBDIRS);
+
+pub const ROUTER: Router = Router::new()
+ .get(&API_METHOD_LIST_SANITY_CHECK_JOBS)
+ .match_all("id", &SANITY_CHECK_INFO_ROUTER);
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 7/8] manager: add sanity check jobs management cli commands
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
` (5 preceding siblings ...)
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 6/8] api: admin: add sanity check job api endpoints Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 8/8] proxy: add sanity check task to scheduler Christian Ebner
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Expose the sanity check management commands to the CLI, allowing to
create, show/list, update, remove and run jobs.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox-backup-manager.rs | 3 +-
src/bin/proxmox_backup_manager/mod.rs | 2 +
.../proxmox_backup_manager/sanity_check.rs | 126 ++++++++++++++++++
3 files changed, 130 insertions(+), 1 deletion(-)
create mode 100644 src/bin/proxmox_backup_manager/sanity_check.rs
diff --git a/src/bin/proxmox-backup-manager.rs b/src/bin/proxmox-backup-manager.rs
index 115207f3..e17b18c7 100644
--- a/src/bin/proxmox-backup-manager.rs
+++ b/src/bin/proxmox-backup-manager.rs
@@ -447,6 +447,7 @@ async fn run() -> Result<(), Error> {
.insert("acme", acme_mgmt_cli())
.insert("cert", cert_mgmt_cli())
.insert("subscription", subscription_commands())
+ .insert("sanity-check-job", sanity_check_job_commands())
.insert("sync-job", sync_job_commands())
.insert("verify-job", verify_job_commands())
.insert("prune-job", prune_job_commands())
@@ -510,7 +511,7 @@ fn main() -> Result<(), Error> {
proxmox_async::runtime::main(run())
}
-/// Run the job of a given type (one of "prune", "sync", "verify"),
+/// Run the job of a given type (one of "prune", "sync", "verify", "sanity-check"),
/// specified by the 'id' parameter.
async fn run_job(job_type: &str, param: Value) -> Result<Value, Error> {
let output_format = get_output_format(¶m);
diff --git a/src/bin/proxmox_backup_manager/mod.rs b/src/bin/proxmox_backup_manager/mod.rs
index 8a1c140c..4d728636 100644
--- a/src/bin/proxmox_backup_manager/mod.rs
+++ b/src/bin/proxmox_backup_manager/mod.rs
@@ -16,6 +16,8 @@ mod prune;
pub use prune::*;
mod remote;
pub use remote::*;
+mod sanity_check;
+pub use sanity_check::*;
mod sync;
pub use sync::*;
mod verify;
diff --git a/src/bin/proxmox_backup_manager/sanity_check.rs b/src/bin/proxmox_backup_manager/sanity_check.rs
new file mode 100644
index 00000000..ff875b65
--- /dev/null
+++ b/src/bin/proxmox_backup_manager/sanity_check.rs
@@ -0,0 +1,126 @@
+
+use anyhow::Error;
+use serde_json::Value;
+
+use proxmox_router::{cli::*, ApiHandler, RpcEnvironment};
+use proxmox_schema::api;
+
+use pbs_api_types::JOB_ID_SCHEMA;
+
+use proxmox_backup::api2;
+
+#[api(
+ input: {
+ properties: {
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// List all sanity check jobs
+fn list_sanity_check_jobs(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+ let output_format = get_output_format(¶m);
+
+ let info = &api2::config::sanity_check::API_METHOD_LIST_SANITY_CHECK_JOBS;
+ let mut data = match info.handler {
+ ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+ _ => unreachable!(),
+ };
+
+ let options = default_table_format_options()
+ .column(ColumnConfig::new("id"))
+ .column(ColumnConfig::new("schedule"))
+ .column(ColumnConfig::new("datastore-usage-full-threshold"));
+
+ format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+ Ok(Value::Null)
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// Show sanity check job configuration
+fn show_sanity_check_job(param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<Value, Error> {
+ let output_format = get_output_format(¶m);
+
+ let info = &api2::config::sanity_check::API_METHOD_READ_SANITY_CHECK_JOB;
+ let mut data = match info.handler {
+ ApiHandler::Sync(handler) => (handler)(param, info, rpcenv)?,
+ _ => unreachable!(),
+ };
+
+ let options = default_table_format_options();
+ format_and_print_result_full(&mut data, &info.returns, &output_format, &options);
+
+ Ok(Value::Null)
+}
+
+#[api(
+ input: {
+ properties: {
+ id: {
+ schema: JOB_ID_SCHEMA,
+ },
+ "output-format": {
+ schema: OUTPUT_FORMAT,
+ optional: true,
+ },
+ }
+ }
+)]
+/// Run the specified sanity check job
+async fn run_sanity_check_job(param: Value) -> Result<Value, Error> {
+ crate::run_job("sanity-check", param).await
+}
+
+pub fn sanity_check_job_commands() -> CommandLineInterface {
+ let cmd_def = CliCommandMap::new()
+ .insert("list", CliCommand::new(&API_METHOD_LIST_SANITY_CHECK_JOBS))
+ .insert(
+ "show",
+ CliCommand::new(&API_METHOD_SHOW_SANITY_CHECK_JOB)
+ .arg_param(&["id"])
+ .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+ )
+ .insert(
+ "create",
+ CliCommand::new(&api2::config::sanity_check::API_METHOD_CREATE_SANITY_CHECK_JOB)
+ .arg_param(&["id"])
+ .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id)
+ .completion_cb("schedule", pbs_config::datastore::complete_calendar_event),
+ )
+ .insert(
+ "update",
+ CliCommand::new(&api2::config::sanity_check::API_METHOD_UPDATE_SANITY_CHECK_JOB)
+ .arg_param(&["id"])
+ .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id)
+ .completion_cb("schedule", pbs_config::datastore::complete_calendar_event),
+ )
+ .insert(
+ "run",
+ CliCommand::new(&API_METHOD_RUN_SANITY_CHECK_JOB)
+ .arg_param(&["id"])
+ .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+ )
+ .insert(
+ "remove",
+ CliCommand::new(&api2::config::sanity_check::API_METHOD_DELETE_SANITY_CHECK_JOB)
+ .arg_param(&["id"])
+ .completion_cb("id", pbs_config::sanity_check::complete_sanity_check_job_id),
+ );
+
+ cmd_def.into()
+}
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread* [pbs-devel] [RFC proxmox-backup 8/8] proxy: add sanity check task to scheduler
2023-12-13 15:38 [pbs-devel] [RFC proxmox-backup 0/8] implement sanity check jobs Christian Ebner
` (6 preceding siblings ...)
2023-12-13 15:38 ` [pbs-devel] [RFC proxmox-backup 7/8] manager: add sanity check jobs management cli commands Christian Ebner
@ 2023-12-13 15:38 ` Christian Ebner
7 siblings, 0 replies; 9+ messages in thread
From: Christian Ebner @ 2023-12-13 15:38 UTC (permalink / raw)
To: pbs-devel
Execute configured sanity check tasks based on their configured time
schedule.
Signed-off-by: Christian Ebner <c.ebner@proxmox.com>
---
src/bin/proxmox-backup-proxy.rs | 41 ++++++++++++++++++++++++++++++++-
1 file changed, 40 insertions(+), 1 deletion(-)
diff --git a/src/bin/proxmox-backup-proxy.rs b/src/bin/proxmox-backup-proxy.rs
index 9c49026b..8efa0655 100644
--- a/src/bin/proxmox-backup-proxy.rs
+++ b/src/bin/proxmox-backup-proxy.rs
@@ -44,7 +44,7 @@ use proxmox_time::CalendarEvent;
use pbs_api_types::{
Authid, DataStoreConfig, Operation, PruneJobConfig, SyncJobConfig, TapeBackupJobConfig,
- VerificationJobConfig,
+ VerificationJobConfig, SanityCheckJobConfig,
};
use proxmox_rest_server::daemon;
@@ -60,6 +60,7 @@ use proxmox_backup::api2::pull::do_sync_job;
use proxmox_backup::api2::tape::backup::do_tape_backup_job;
use proxmox_backup::server::do_prune_job;
use proxmox_backup::server::do_verification_job;
+use proxmox_backup::server::do_sanity_check_job;
fn main() -> Result<(), Error> {
pbs_tools::setup_libc_malloc_opts();
@@ -454,6 +455,7 @@ async fn schedule_tasks() -> Result<(), Error> {
schedule_datastore_verify_jobs().await;
schedule_tape_backup_jobs().await;
schedule_task_log_rotate().await;
+ schedule_task_sanity_check_jobs().await;
Ok(())
}
@@ -825,6 +827,43 @@ async fn schedule_task_log_rotate() {
}
}
+async fn schedule_task_sanity_check_jobs() {
+ let config = match pbs_config::sanity_check::config() {
+ Err(err) => {
+ eprintln!("unable to read sanity check job config - {err}");
+ return;
+ }
+ Ok((config, _digest)) => config,
+ };
+ for (job_id, (_, job_config)) in config.sections {
+ let job_config: SanityCheckJobConfig = match serde_json::from_value(job_config) {
+ Ok(c) => c,
+ Err(err) => {
+ eprintln!("sanity check job config from_value failed - {err}");
+ continue;
+ }
+ };
+
+ let worker_type = "sanitycheckjob";
+ let auth_id = Authid::root_auth_id().clone();
+ if check_schedule(worker_type, &job_config.schedule, &job_id) {
+ let job = match Job::new(worker_type, &job_id) {
+ Ok(job) => job,
+ Err(_) => continue, // could not get lock
+ };
+ if let Err(err) = do_sanity_check_job(
+ job,
+ job_config.options,
+ &auth_id,
+ Some(job_config.schedule),
+ ) {
+ eprintln!("unable to start sanity check job {job_id} - {err}");
+ }
+ };
+ }
+}
+
+
async fn command_reopen_access_logfiles() -> Result<(), Error> {
// only care about the most recent daemon instance for each, proxy & api, as other older ones
// should not respond to new requests anyway, but only finish their current one and then exit.
--
2.39.2
^ permalink raw reply [flat|nested] 9+ messages in thread