From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id 3DBC861242 for ; Tue, 20 Oct 2020 11:10:45 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 17790CD0C for ; Tue, 20 Oct 2020 11:10:45 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [212.186.127.180]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id CA8CEC45A for ; Tue, 20 Oct 2020 11:10:39 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 89A3C45E31 for ; Tue, 20 Oct 2020 11:10:39 +0200 (CEST) From: Hannes Laimer To: pbs-devel@lists.proxmox.com Date: Tue, 20 Oct 2020 11:10:04 +0200 Message-Id: <20201020091012.82723-3-h.laimer@proxmox.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20201020091012.82723-1-h.laimer@proxmox.com> References: <20201020091012.82723-1-h.laimer@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment RCVD_IN_DNSWL_MED -2.3 Sender listed at https://www.dnswl.org/, medium trust SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record URIBL_BLOCKED 0.001 ADMINISTRATOR NOTICE: The query to URIBL was blocked. See http://wiki.apache.org/spamassassin/DnsBlocklists#dnsbl-block for more information. [verify.rs, config.rs, mod.rs, data.store] Subject: [pbs-devel] [PATCH v4 proxmox-backup 02/10] api2: add verification job config endpoint X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 20 Oct 2020 09:10:45 -0000 Signed-off-by: Hannes Laimer --- src/api2/config.rs | 2 + src/api2/config/verify.rs | 274 ++++++++++++++++++++++++++++++++++++++ src/api2/types/mod.rs | 10 ++ src/config.rs | 1 + src/config/verify.rs | 189 ++++++++++++++++++++++++++ 5 files changed, 476 insertions(+) create mode 100644 src/api2/config/verify.rs create mode 100644 src/config/verify.rs diff --git a/src/api2/config.rs b/src/api2/config.rs index be7397c8..7a5129c7 100644 --- a/src/api2/config.rs +++ b/src/api2/config.rs @@ -4,11 +4,13 @@ use proxmox::list_subdirs_api_method; pub mod datastore; pub mod remote; pub mod sync; +pub mod verify; const SUBDIRS: SubdirMap = &[ ("datastore", &datastore::ROUTER), ("remote", &remote::ROUTER), ("sync", &sync::ROUTER), + ("verify", &verify::ROUTER) ]; pub const ROUTER: Router = Router::new() diff --git a/src/api2/config/verify.rs b/src/api2/config/verify.rs new file mode 100644 index 00000000..efc33a5c --- /dev/null +++ b/src/api2/config/verify.rs @@ -0,0 +1,274 @@ +use anyhow::{bail, Error}; +use serde_json::Value; +use ::serde::{Deserialize, Serialize}; + +use proxmox::api::{api, Router, RpcEnvironment}; +use proxmox::tools::fs::open_file_locked; + +use crate::api2::types::*; +use crate::config::verify::{self, VerificationJobConfig}; + +#[api( + input: { + properties: {}, + }, + returns: { + description: "List configured jobs.", + type: Array, + items: { type: verify::VerificationJobConfig }, + }, +)] +/// List all verification jobs +pub fn list_verification_jobs( + _param: Value, + mut rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + + let (config, digest) = verify::config()?; + + let list = config.convert_to_typed_array("verification")?; + + rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into(); + + Ok(list) +} + + +#[api( + protected: true, + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + store: { + schema: DATASTORE_SCHEMA, + }, + "ignore-verified": { + optional: true, + schema: IGNORE_VERIFIED_BACKUPS_SCHEMA, + }, + "outdated-after": { + optional: true, + schema: VERIFICATION_OUTDATED_AFTER_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + schedule: { + optional: true, + schema: VERIFICATION_SCHEDULE_SCHEMA, + }, + } + } +)] +/// Create a new verification job. +pub fn create_verification_job(param: Value) -> Result<(), Error> { + + let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; + + let verification_job: verify::VerificationJobConfig = serde_json::from_value(param.clone())?; + + let (mut config, _digest) = verify::config()?; + + if let Some(_) = config.sections.get(&verification_job.id) { + bail!("job '{}' already exists.", verification_job.id); + } + + config.set_data(&verification_job.id, "verification", &verification_job)?; + + verify::save_config(&config)?; + + crate::config::jobstate::create_state_file("verificationjob", &verification_job.id)?; + + Ok(()) +} + +#[api( + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + }, + }, + returns: { + description: "The verification job configuration.", + type: verify::VerificationJobConfig, + }, +)] +/// Read a verification job configuration. +pub fn read_verification_job( + id: String, + mut rpcenv: &mut dyn RpcEnvironment, +) -> Result { + let (config, digest) = verify::config()?; + + let verification_job = config.lookup("verification", &id)?; + rpcenv["digest"] = proxmox::tools::digest_to_hex(&digest).into(); + + Ok(verification_job) +} + +#[api()] +#[derive(Serialize, Deserialize)] +#[serde(rename_all="kebab-case")] +/// Deletable property name +pub enum DeletableProperty { + /// Delete the ignore verified property. + IgnoreVerified, + /// Delete the comment property. + Comment, + /// Delete the job schedule. + Schedule, + /// Delete outdated after property. + OutdatedAfter +} + +#[api( + protected: true, + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + store: { + optional: true, + schema: DATASTORE_SCHEMA, + }, + "ignore-verified": { + optional: true, + schema: IGNORE_VERIFIED_BACKUPS_SCHEMA, + }, + "outdated-after": { + optional: true, + schema: VERIFICATION_OUTDATED_AFTER_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + schedule: { + optional: true, + schema: VERIFICATION_SCHEDULE_SCHEMA, + }, + delete: { + description: "List of properties to delete.", + type: Array, + optional: true, + items: { + type: DeletableProperty, + } + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, +)] +/// Update verification job config. +pub fn update_verification_job( + id: String, + store: Option, + ignore_verified: Option, + outdated_after: Option, + comment: Option, + schedule: Option, + delete: Option>, + digest: Option, +) -> Result<(), Error> { + + let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; + + // pass/compare digest + let (mut config, expected_digest) = verify::config()?; + + if let Some(ref digest) = digest { + let digest = proxmox::tools::hex_to_digest(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + let mut data: verify::VerificationJobConfig = config.lookup("verification", &id)?; + + if let Some(delete) = delete { + for delete_prop in delete { + match delete_prop { + DeletableProperty::IgnoreVerified => { data.ignore_verified = None; }, + DeletableProperty::OutdatedAfter => { data.outdated_after = None; }, + DeletableProperty::Comment => { data.comment = None; }, + DeletableProperty::Schedule => { data.schedule = None; }, + } + } + } + + if let Some(comment) = comment { + let comment = comment.trim().to_string(); + if comment.is_empty() { + data.comment = None; + } else { + data.comment = Some(comment); + } + } + + if let Some(store) = store { data.store = store; } + + if ignore_verified.is_some() { data.ignore_verified = ignore_verified; } + if outdated_after.is_some() { data.outdated_after = outdated_after; } + if schedule.is_some() { data.schedule = schedule; } + + config.set_data(&id, "verification", &data)?; + + verify::save_config(&config)?; + + Ok(()) +} + +#[api( + protected: true, + input: { + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + digest: { + optional: true, + schema: PROXMOX_CONFIG_DIGEST_SCHEMA, + }, + }, + }, +)] +/// Remove a verification job configuration +pub fn delete_verification_job(id: String, digest: Option) -> Result<(), Error> { + + let _lock = open_file_locked(verify::VERIFICATION_CFG_LOCKFILE, std::time::Duration::new(10, 0), true)?; + + let (mut config, expected_digest) = verify::config()?; + + if let Some(ref digest) = digest { + let digest = proxmox::tools::hex_to_digest(digest)?; + crate::tools::detect_modified_configuration_file(&digest, &expected_digest)?; + } + + match config.sections.get(&id) { + Some(_) => { config.sections.remove(&id); }, + None => bail!("job '{}' does not exist.", id), + } + + verify::save_config(&config)?; + + crate::config::jobstate::remove_state_file("verificationjob", &id)?; + + Ok(()) +} + +const ITEM_ROUTER: Router = Router::new() + .get(&API_METHOD_READ_VERIFICATION_JOB) + .put(&API_METHOD_UPDATE_VERIFICATION_JOB) + .delete(&API_METHOD_DELETE_VERIFICATION_JOB); + +pub const ROUTER: Router = Router::new() + .get(&API_METHOD_LIST_VERIFICATION_JOBS) + .post(&API_METHOD_CREATE_VERIFICATION_JOB) + .match_all("id", &ITEM_ROUTER); \ No newline at end of file diff --git a/src/api2/types/mod.rs b/src/api2/types/mod.rs index 5a30bb89..f97db557 100644 --- a/src/api2/types/mod.rs +++ b/src/api2/types/mod.rs @@ -324,6 +324,16 @@ pub const REMOVE_VANISHED_BACKUPS_SCHEMA: Schema = BooleanSchema::new( .default(true) .schema(); +pub const IGNORE_VERIFIED_BACKUPS_SCHEMA: Schema = BooleanSchema::new( + "Do not verify backups that are already verified if their verification is not outdated.") + .default(true) + .schema(); + +pub const VERIFICATION_OUTDATED_AFTER_SCHEMA: Schema = IntegerSchema::new( + "Days after that a verification becomes outdated") + .minimum(1) + .schema(); + pub const SINGLE_LINE_COMMENT_SCHEMA: Schema = StringSchema::new("Comment (single line).") .format(&SINGLE_LINE_COMMENT_FORMAT) .schema(); diff --git a/src/config.rs b/src/config.rs index c2ac6da1..ab7fc81a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,6 +23,7 @@ pub mod network; pub mod remote; pub mod sync; pub mod user; +pub mod verify; /// Check configuration directory permissions /// diff --git a/src/config/verify.rs b/src/config/verify.rs new file mode 100644 index 00000000..af3d20a7 --- /dev/null +++ b/src/config/verify.rs @@ -0,0 +1,189 @@ +use anyhow::{Error}; +use lazy_static::lazy_static; +use std::collections::HashMap; +use serde::{Serialize, Deserialize}; + +use proxmox::api::{ + api, + schema::*, + section_config::{ + SectionConfig, + SectionConfigData, + SectionConfigPlugin, + } +}; + +use proxmox::tools::{fs::replace_file, fs::CreateOptions}; + +use crate::api2::types::*; + +lazy_static! { + static ref CONFIG: SectionConfig = init(); +} + + +#[api( + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + store: { + schema: DATASTORE_SCHEMA, + }, + "ignore-verified": { + optional: true, + schema: IGNORE_VERIFIED_BACKUPS_SCHEMA, + }, + "outdated-after": { + optional: true, + schema: VERIFICATION_OUTDATED_AFTER_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + schedule: { + optional: true, + schema: VERIFICATION_SCHEDULE_SCHEMA, + }, + } +)] +#[serde(rename_all="kebab-case")] +#[derive(Serialize,Deserialize)] +/// Verification Job +pub struct VerificationJobConfig { + pub id: String, + pub store: String, + #[serde(skip_serializing_if="Option::is_none")] + pub ignore_verified: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub outdated_after: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub comment: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub schedule: Option, +} + + +#[api( + properties: { + id: { + schema: JOB_ID_SCHEMA, + }, + store: { + schema: DATASTORE_SCHEMA, + }, + "ignore-verified": { + optional: true, + schema: IGNORE_VERIFIED_BACKUPS_SCHEMA, + }, + "outdated-after": { + optional: true, + schema: VERIFICATION_OUTDATED_AFTER_SCHEMA, + }, + comment: { + optional: true, + schema: SINGLE_LINE_COMMENT_SCHEMA, + }, + schedule: { + optional: true, + schema: VERIFICATION_SCHEDULE_SCHEMA, + }, + "next-run": { + description: "Estimated time of the next run (UNIX epoch).", + optional: true, + type: Integer, + }, + "last-run-state": { + description: "Result of the last run.", + optional: true, + type: String, + }, + "last-run-upid": { + description: "Task UPID of the last run.", + optional: true, + type: String, + }, + "last-run-endtime": { + description: "Endtime of the last run.", + optional: true, + type: Integer, + }, + } +)] +#[serde(rename_all="kebab-case")] +#[derive(Serialize,Deserialize)] +/// Status of Verification Job +pub struct VerificationJobStatus { + pub id: String, + pub store: String, + #[serde(skip_serializing_if="Option::is_none")] + pub ignore_verified: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub outdated_after: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub comment: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub schedule: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub next_run: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub last_run_state: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub last_run_upid: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub last_run_endtime: Option, +} + + +fn init() -> SectionConfig { + let obj_schema = match VerificationJobConfig::API_SCHEMA { + Schema::Object(ref obj_schema) => obj_schema, + _ => unreachable!(), + }; + + let plugin = SectionConfigPlugin::new("verification".to_string(), Some(String::from("id")), obj_schema); + let mut config = SectionConfig::new(&JOB_ID_SCHEMA); + config.register_plugin(plugin); + + config +} + +pub const VERIFICATION_CFG_FILENAME: &str = "/etc/proxmox-backup/verification.cfg"; +pub const VERIFICATION_CFG_LOCKFILE: &str = "/etc/proxmox-backup/.verification.lck"; + +pub fn config() -> Result<(SectionConfigData, [u8;32]), Error> { + + let content = proxmox::tools::fs::file_read_optional_string(VERIFICATION_CFG_FILENAME)?; + let content = content.unwrap_or_else(String::new); + + let digest = openssl::sha::sha256(content.as_bytes()); + let data = CONFIG.parse(VERIFICATION_CFG_FILENAME, &content)?; + Ok((data, digest)) +} + +pub fn save_config(config: &SectionConfigData) -> Result<(), Error> { + let raw = CONFIG.write(VERIFICATION_CFG_FILENAME, &config)?; + + let backup_user = crate::backup::backup_user()?; + let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640); + // set the correct owner/group/permissions while saving file + // owner(rw) = root, group(r)= backup + + let options = CreateOptions::new() + .perm(mode) + .owner(nix::unistd::ROOT) + .group(backup_user.gid); + + replace_file(VERIFICATION_CFG_FILENAME, raw.as_bytes(), options)?; + + Ok(()) +} + +// shell completion helper +pub fn complete_verification_job_id(_arg: &str, _param: &HashMap) -> Vec { + match config() { + Ok((data, _digest)) => data.sections.iter().map(|(id, _)| id.to_string()).collect(), + Err(_) => return vec![], + } +} \ No newline at end of file -- 2.20.1